commit cf145bcb3e0d7ec85671b459ecaf8432fc5dd93e Author: Filip Leonarski (Gitea) Date: Tue Oct 6 20:38:39 2026 +0000 Deploy site diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 000000000..756921077 --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. +config: edc143847c74a69dfff92f02b40aeb50 +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/ACKNOWLEDGEMENT.doctree b/.doctrees/ACKNOWLEDGEMENT.doctree new file mode 100644 index 000000000..16b8d0e28 Binary files /dev/null and b/.doctrees/ACKNOWLEDGEMENT.doctree differ diff --git a/.doctrees/BATTERY_REPORT.doctree b/.doctrees/BATTERY_REPORT.doctree new file mode 100644 index 000000000..bad6ef43c Binary files /dev/null and b/.doctrees/BATTERY_REPORT.doctree differ diff --git a/.doctrees/CBOR.doctree b/.doctrees/CBOR.doctree new file mode 100644 index 000000000..02fc9500a Binary files /dev/null and b/.doctrees/CBOR.doctree differ diff --git a/.doctrees/CHANGELOG.doctree b/.doctrees/CHANGELOG.doctree new file mode 100644 index 000000000..ae8aed302 Binary files /dev/null and b/.doctrees/CHANGELOG.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS.doctree b/.doctrees/CPU_DATA_ANALYSIS.doctree new file mode 100644 index 000000000..8932c426d Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS_DECISIONS.doctree b/.doctrees/CPU_DATA_ANALYSIS_DECISIONS.doctree new file mode 100644 index 000000000..6caa30935 Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS_DECISIONS.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS_IMAGE.doctree b/.doctrees/CPU_DATA_ANALYSIS_IMAGE.doctree new file mode 100644 index 000000000..66f034541 Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS_IMAGE.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS_INDEXING.doctree b/.doctrees/CPU_DATA_ANALYSIS_INDEXING.doctree new file mode 100644 index 000000000..28d14b631 Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS_INDEXING.doctree differ diff --git a/.doctrees/CPU_DATA_ANALYSIS_INTEGRATION.doctree b/.doctrees/CPU_DATA_ANALYSIS_INTEGRATION.doctree new file mode 100644 index 000000000..c8ee62fa4 Binary files /dev/null and b/.doctrees/CPU_DATA_ANALYSIS_INTEGRATION.doctree differ diff --git a/.doctrees/DEPLOYMENT.doctree b/.doctrees/DEPLOYMENT.doctree new file mode 100644 index 000000000..559868278 Binary files /dev/null and b/.doctrees/DEPLOYMENT.doctree differ diff --git a/.doctrees/DETECTORS.doctree b/.doctrees/DETECTORS.doctree new file mode 100644 index 000000000..9ea184887 Binary files /dev/null and b/.doctrees/DETECTORS.doctree differ diff --git a/.doctrees/DETECTOR_GEOMETRY.doctree b/.doctrees/DETECTOR_GEOMETRY.doctree new file mode 100644 index 000000000..2a7c5f39a Binary files /dev/null and b/.doctrees/DETECTOR_GEOMETRY.doctree differ diff --git a/.doctrees/EXTERNAL_TEST_DATA.doctree b/.doctrees/EXTERNAL_TEST_DATA.doctree new file mode 100644 index 000000000..30d930084 Binary files /dev/null and b/.doctrees/EXTERNAL_TEST_DATA.doctree differ diff --git a/.doctrees/FPGA.doctree b/.doctrees/FPGA.doctree new file mode 100644 index 000000000..a34581467 Binary files /dev/null and b/.doctrees/FPGA.doctree differ diff --git a/.doctrees/FPGA_DATA_ANALYSIS.doctree b/.doctrees/FPGA_DATA_ANALYSIS.doctree new file mode 100644 index 000000000..1109c050c Binary files /dev/null and b/.doctrees/FPGA_DATA_ANALYSIS.doctree differ diff --git a/.doctrees/FPGA_DESIGN.doctree b/.doctrees/FPGA_DESIGN.doctree new file mode 100644 index 000000000..ed08df361 Binary files /dev/null and b/.doctrees/FPGA_DESIGN.doctree differ diff --git a/.doctrees/FPGA_LICENSE.doctree b/.doctrees/FPGA_LICENSE.doctree new file mode 100644 index 000000000..9e76d8d42 Binary files /dev/null and b/.doctrees/FPGA_LICENSE.doctree differ diff --git a/.doctrees/FPGA_NETWORK.doctree b/.doctrees/FPGA_NETWORK.doctree new file mode 100644 index 000000000..d2de1a390 Binary files /dev/null and b/.doctrees/FPGA_NETWORK.doctree differ diff --git a/.doctrees/FPGA_PCIE_DRIVER.doctree b/.doctrees/FPGA_PCIE_DRIVER.doctree new file mode 100644 index 000000000..198b6da65 Binary files /dev/null and b/.doctrees/FPGA_PCIE_DRIVER.doctree differ diff --git a/.doctrees/FPGA_SETTINGS.doctree b/.doctrees/FPGA_SETTINGS.doctree new file mode 100644 index 000000000..a3ee3969b Binary files /dev/null and b/.doctrees/FPGA_SETTINGS.doctree differ diff --git a/.doctrees/HARDWARE.doctree b/.doctrees/HARDWARE.doctree new file mode 100644 index 000000000..e49e27976 Binary files /dev/null and b/.doctrees/HARDWARE.doctree differ diff --git a/.doctrees/HDF5.doctree b/.doctrees/HDF5.doctree new file mode 100644 index 000000000..484f69933 Binary files /dev/null and b/.doctrees/HDF5.doctree differ diff --git a/.doctrees/IMAGE_STREAM.doctree b/.doctrees/IMAGE_STREAM.doctree new file mode 100644 index 000000000..4b7574fb6 Binary files /dev/null and b/.doctrees/IMAGE_STREAM.doctree differ diff --git a/.doctrees/JFJOCH_BROKER.doctree b/.doctrees/JFJOCH_BROKER.doctree new file mode 100644 index 000000000..cd33a4d36 Binary files /dev/null and b/.doctrees/JFJOCH_BROKER.doctree differ diff --git a/.doctrees/JFJOCH_VIEWER.doctree b/.doctrees/JFJOCH_VIEWER.doctree new file mode 100644 index 000000000..6f0abab1f Binary files /dev/null and b/.doctrees/JFJOCH_VIEWER.doctree differ diff --git a/.doctrees/JFJOCH_WRITER.doctree b/.doctrees/JFJOCH_WRITER.doctree new file mode 100644 index 000000000..a75939c03 Binary files /dev/null and b/.doctrees/JFJOCH_WRITER.doctree differ diff --git a/.doctrees/LICENSE.doctree b/.doctrees/LICENSE.doctree new file mode 100644 index 000000000..728fbb6b4 Binary files /dev/null and b/.doctrees/LICENSE.doctree differ diff --git a/.doctrees/NAMING.doctree b/.doctrees/NAMING.doctree new file mode 100644 index 000000000..7badc0968 Binary files /dev/null and b/.doctrees/NAMING.doctree differ diff --git a/.doctrees/OPENAPI.doctree b/.doctrees/OPENAPI.doctree new file mode 100644 index 000000000..d8b3ecf86 Binary files /dev/null and b/.doctrees/OPENAPI.doctree differ diff --git a/.doctrees/OPENAPI_SPECS.doctree b/.doctrees/OPENAPI_SPECS.doctree new file mode 100644 index 000000000..2c353cb88 Binary files /dev/null and b/.doctrees/OPENAPI_SPECS.doctree differ diff --git a/.doctrees/PIXEL_MASK.doctree b/.doctrees/PIXEL_MASK.doctree new file mode 100644 index 000000000..38a730776 Binary files /dev/null and b/.doctrees/PIXEL_MASK.doctree differ diff --git a/.doctrees/PYTHON_CLIENT.doctree b/.doctrees/PYTHON_CLIENT.doctree new file mode 100644 index 000000000..291b525dc Binary files /dev/null and b/.doctrees/PYTHON_CLIENT.doctree differ diff --git a/.doctrees/RELEASE_CONTENTS.doctree b/.doctrees/RELEASE_CONTENTS.doctree new file mode 100644 index 000000000..8b6731d8a Binary files /dev/null and b/.doctrees/RELEASE_CONTENTS.doctree differ diff --git a/.doctrees/REPOSITORIES.doctree b/.doctrees/REPOSITORIES.doctree new file mode 100644 index 000000000..6e83f4dc7 Binary files /dev/null and b/.doctrees/REPOSITORIES.doctree differ diff --git a/.doctrees/RUGNUX.doctree b/.doctrees/RUGNUX.doctree new file mode 100644 index 000000000..9b15d3080 Binary files /dev/null and b/.doctrees/RUGNUX.doctree differ diff --git a/.doctrees/RUGNUX_ADVANCED.doctree b/.doctrees/RUGNUX_ADVANCED.doctree new file mode 100644 index 000000000..361b2de98 Binary files /dev/null and b/.doctrees/RUGNUX_ADVANCED.doctree differ diff --git a/.doctrees/RUGNUX_CALIBRATION.doctree b/.doctrees/RUGNUX_CALIBRATION.doctree new file mode 100644 index 000000000..bc378c66c Binary files /dev/null and b/.doctrees/RUGNUX_CALIBRATION.doctree differ diff --git a/.doctrees/RUGNUX_FORMATS.doctree b/.doctrees/RUGNUX_FORMATS.doctree new file mode 100644 index 000000000..077dae2cb Binary files /dev/null and b/.doctrees/RUGNUX_FORMATS.doctree differ diff --git a/.doctrees/RUGNUX_INSTALL.doctree b/.doctrees/RUGNUX_INSTALL.doctree new file mode 100644 index 000000000..e5c1921e1 Binary files /dev/null and b/.doctrees/RUGNUX_INSTALL.doctree differ diff --git a/.doctrees/RUGNUX_INTEGRATION.doctree b/.doctrees/RUGNUX_INTEGRATION.doctree new file mode 100644 index 000000000..005a1fa7b Binary files /dev/null and b/.doctrees/RUGNUX_INTEGRATION.doctree differ diff --git a/.doctrees/RUGNUX_OVERVIEW.doctree b/.doctrees/RUGNUX_OVERVIEW.doctree new file mode 100644 index 000000000..ea1b334d6 Binary files /dev/null and b/.doctrees/RUGNUX_OVERVIEW.doctree differ diff --git a/.doctrees/RUGNUX_REPORT.doctree b/.doctrees/RUGNUX_REPORT.doctree new file mode 100644 index 000000000..2c8ef334b Binary files /dev/null and b/.doctrees/RUGNUX_REPORT.doctree differ diff --git a/.doctrees/RUGNUX_TUTORIAL.doctree b/.doctrees/RUGNUX_TUTORIAL.doctree new file mode 100644 index 000000000..f4c47d1fe Binary files /dev/null and b/.doctrees/RUGNUX_TUTORIAL.doctree differ diff --git a/.doctrees/SECURITY.doctree b/.doctrees/SECURITY.doctree new file mode 100644 index 000000000..7a6514128 Binary files /dev/null and b/.doctrees/SECURITY.doctree differ diff --git a/.doctrees/SOFTWARE.doctree b/.doctrees/SOFTWARE.doctree new file mode 100644 index 000000000..74752681b Binary files /dev/null and b/.doctrees/SOFTWARE.doctree differ diff --git a/.doctrees/SOFTWARE_INTEGRATION.doctree b/.doctrees/SOFTWARE_INTEGRATION.doctree new file mode 100644 index 000000000..eaca894d9 Binary files /dev/null and b/.doctrees/SOFTWARE_INTEGRATION.doctree differ diff --git a/.doctrees/TESTS.doctree b/.doctrees/TESTS.doctree new file mode 100644 index 000000000..ccdd7a40a Binary files /dev/null and b/.doctrees/TESTS.doctree differ diff --git a/.doctrees/THIRD_PARTY_NOTICES.doctree b/.doctrees/THIRD_PARTY_NOTICES.doctree new file mode 100644 index 000000000..ae014638f Binary files /dev/null and b/.doctrees/THIRD_PARTY_NOTICES.doctree differ diff --git a/.doctrees/TOOLS.doctree b/.doctrees/TOOLS.doctree new file mode 100644 index 000000000..5af4c0328 Binary files /dev/null and b/.doctrees/TOOLS.doctree differ diff --git a/.doctrees/VERSIONING.doctree b/.doctrees/VERSIONING.doctree new file mode 100644 index 000000000..e68bd9491 Binary files /dev/null and b/.doctrees/VERSIONING.doctree differ diff --git a/.doctrees/WEB_FRONTEND.doctree b/.doctrees/WEB_FRONTEND.doctree new file mode 100644 index 000000000..fa9db7b49 Binary files /dev/null and b/.doctrees/WEB_FRONTEND.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 000000000..209670b61 Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 000000000..ae23874ab Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/python_client/README.doctree b/.doctrees/python_client/README.doctree new file mode 100644 index 000000000..e4c0ef172 Binary files /dev/null and b/.doctrees/python_client/README.doctree differ diff --git a/.doctrees/python_client/docs/AzimIntSettings.doctree b/.doctrees/python_client/docs/AzimIntSettings.doctree new file mode 100644 index 000000000..023070deb Binary files /dev/null and b/.doctrees/python_client/docs/AzimIntSettings.doctree differ diff --git a/.doctrees/python_client/docs/BraggIntegrationSettings.doctree b/.doctrees/python_client/docs/BraggIntegrationSettings.doctree new file mode 100644 index 000000000..c3ef6f83b Binary files /dev/null and b/.doctrees/python_client/docs/BraggIntegrationSettings.doctree differ diff --git a/.doctrees/python_client/docs/BrokerStatus.doctree b/.doctrees/python_client/docs/BrokerStatus.doctree new file mode 100644 index 000000000..132fe9db0 Binary files /dev/null and b/.doctrees/python_client/docs/BrokerStatus.doctree differ diff --git a/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree new file mode 100644 index 000000000..2f30d3f52 Binary files /dev/null and b/.doctrees/python_client/docs/CalibrationStatisticsInner.doctree differ diff --git a/.doctrees/python_client/docs/DarkMaskSettings.doctree b/.doctrees/python_client/docs/DarkMaskSettings.doctree new file mode 100644 index 000000000..b7ed23aa0 Binary files /dev/null and b/.doctrees/python_client/docs/DarkMaskSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettings.doctree b/.doctrees/python_client/docs/DatasetSettings.doctree new file mode 100644 index 000000000..477a9c4fa Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettings.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree new file mode 100644 index 000000000..ef560a438 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsSmargon.doctree differ diff --git a/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree new file mode 100644 index 000000000..1b34cc7b7 Binary files /dev/null and b/.doctrees/python_client/docs/DatasetSettingsXrayFluorescenceSpectrum.doctree differ diff --git a/.doctrees/python_client/docs/DefaultApi.doctree b/.doctrees/python_client/docs/DefaultApi.doctree new file mode 100644 index 000000000..a6c93be7b Binary files /dev/null and b/.doctrees/python_client/docs/DefaultApi.doctree differ diff --git a/.doctrees/python_client/docs/Detector.doctree b/.doctrees/python_client/docs/Detector.doctree new file mode 100644 index 000000000..006089ef6 Binary files /dev/null and b/.doctrees/python_client/docs/Detector.doctree differ diff --git a/.doctrees/python_client/docs/DetectorList.doctree b/.doctrees/python_client/docs/DetectorList.doctree new file mode 100644 index 000000000..f951b0d76 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorList.doctree differ diff --git a/.doctrees/python_client/docs/DetectorListElement.doctree b/.doctrees/python_client/docs/DetectorListElement.doctree new file mode 100644 index 000000000..dda31eb6e Binary files /dev/null and b/.doctrees/python_client/docs/DetectorListElement.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModule.doctree b/.doctrees/python_client/docs/DetectorModule.doctree new file mode 100644 index 000000000..bfdcac794 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModule.doctree differ diff --git a/.doctrees/python_client/docs/DetectorModuleDirection.doctree b/.doctrees/python_client/docs/DetectorModuleDirection.doctree new file mode 100644 index 000000000..b510009fd Binary files /dev/null and b/.doctrees/python_client/docs/DetectorModuleDirection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorPowerState.doctree b/.doctrees/python_client/docs/DetectorPowerState.doctree new file mode 100644 index 000000000..115ce3324 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorPowerState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSelection.doctree b/.doctrees/python_client/docs/DetectorSelection.doctree new file mode 100644 index 000000000..765b84ce8 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSelection.doctree differ diff --git a/.doctrees/python_client/docs/DetectorSettings.doctree b/.doctrees/python_client/docs/DetectorSettings.doctree new file mode 100644 index 000000000..2557ede2c Binary files /dev/null and b/.doctrees/python_client/docs/DetectorSettings.doctree differ diff --git a/.doctrees/python_client/docs/DetectorState.doctree b/.doctrees/python_client/docs/DetectorState.doctree new file mode 100644 index 000000000..84750d048 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorState.doctree differ diff --git a/.doctrees/python_client/docs/DetectorStatus.doctree b/.doctrees/python_client/docs/DetectorStatus.doctree new file mode 100644 index 000000000..316cb6cbf Binary files /dev/null and b/.doctrees/python_client/docs/DetectorStatus.doctree differ diff --git a/.doctrees/python_client/docs/DetectorTiming.doctree b/.doctrees/python_client/docs/DetectorTiming.doctree new file mode 100644 index 000000000..22dc0c03b Binary files /dev/null and b/.doctrees/python_client/docs/DetectorTiming.doctree differ diff --git a/.doctrees/python_client/docs/DetectorType.doctree b/.doctrees/python_client/docs/DetectorType.doctree new file mode 100644 index 000000000..f4e0a6a95 Binary files /dev/null and b/.doctrees/python_client/docs/DetectorType.doctree differ diff --git a/.doctrees/python_client/docs/ErrorMessage.doctree b/.doctrees/python_client/docs/ErrorMessage.doctree new file mode 100644 index 000000000..41404f389 Binary files /dev/null and b/.doctrees/python_client/docs/ErrorMessage.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterFormat.doctree b/.doctrees/python_client/docs/FileWriterFormat.doctree new file mode 100644 index 000000000..42d7d43c9 Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterFormat.doctree differ diff --git a/.doctrees/python_client/docs/FileWriterSettings.doctree b/.doctrees/python_client/docs/FileWriterSettings.doctree new file mode 100644 index 000000000..7a71193d2 Binary files /dev/null and b/.doctrees/python_client/docs/FileWriterSettings.doctree differ diff --git a/.doctrees/python_client/docs/FpgaStatusInner.doctree b/.doctrees/python_client/docs/FpgaStatusInner.doctree new file mode 100644 index 000000000..ed1965ab2 Binary files /dev/null and b/.doctrees/python_client/docs/FpgaStatusInner.doctree differ diff --git a/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree new file mode 100644 index 000000000..3430aa012 Binary files /dev/null and b/.doctrees/python_client/docs/GeomRefinementAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/GridScan.doctree b/.doctrees/python_client/docs/GridScan.doctree new file mode 100644 index 000000000..c7e653816 Binary files /dev/null and b/.doctrees/python_client/docs/GridScan.doctree differ diff --git a/.doctrees/python_client/docs/ImageBufferStatus.doctree b/.doctrees/python_client/docs/ImageBufferStatus.doctree new file mode 100644 index 000000000..04efdce5f Binary files /dev/null and b/.doctrees/python_client/docs/ImageBufferStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImageFormatSettings.doctree b/.doctrees/python_client/docs/ImageFormatSettings.doctree new file mode 100644 index 000000000..d795ce4ba Binary files /dev/null and b/.doctrees/python_client/docs/ImageFormatSettings.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherStatus.doctree b/.doctrees/python_client/docs/ImagePusherStatus.doctree new file mode 100644 index 000000000..2be4badd9 Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherStatus.doctree differ diff --git a/.doctrees/python_client/docs/ImagePusherType.doctree b/.doctrees/python_client/docs/ImagePusherType.doctree new file mode 100644 index 000000000..b9e53dc3c Binary files /dev/null and b/.doctrees/python_client/docs/ImagePusherType.doctree differ diff --git a/.doctrees/python_client/docs/IndexingAlgorithm.doctree b/.doctrees/python_client/docs/IndexingAlgorithm.doctree new file mode 100644 index 000000000..e150689f1 Binary files /dev/null and b/.doctrees/python_client/docs/IndexingAlgorithm.doctree differ diff --git a/.doctrees/python_client/docs/IndexingSettings.doctree b/.doctrees/python_client/docs/IndexingSettings.doctree new file mode 100644 index 000000000..e6fe1cca7 Binary files /dev/null and b/.doctrees/python_client/docs/IndexingSettings.doctree differ diff --git a/.doctrees/python_client/docs/InstrumentMetadata.doctree b/.doctrees/python_client/docs/InstrumentMetadata.doctree new file mode 100644 index 000000000..769d4b73d Binary files /dev/null and b/.doctrees/python_client/docs/InstrumentMetadata.doctree differ diff --git a/.doctrees/python_client/docs/IntegrationModel.doctree b/.doctrees/python_client/docs/IntegrationModel.doctree new file mode 100644 index 000000000..5c3b85f83 Binary files /dev/null and b/.doctrees/python_client/docs/IntegrationModel.doctree differ diff --git a/.doctrees/python_client/docs/JfjochSettings.doctree b/.doctrees/python_client/docs/JfjochSettings.doctree new file mode 100644 index 000000000..0859b393f Binary files /dev/null and b/.doctrees/python_client/docs/JfjochSettings.doctree differ diff --git a/.doctrees/python_client/docs/JfjochStatistics.doctree b/.doctrees/python_client/docs/JfjochStatistics.doctree new file mode 100644 index 000000000..96d2ecff2 Binary files /dev/null and b/.doctrees/python_client/docs/JfjochStatistics.doctree differ diff --git a/.doctrees/python_client/docs/MeasurementStatistics.doctree b/.doctrees/python_client/docs/MeasurementStatistics.doctree new file mode 100644 index 000000000..a5e569773 Binary files /dev/null and b/.doctrees/python_client/docs/MeasurementStatistics.doctree differ diff --git a/.doctrees/python_client/docs/PcieDevicesInner.doctree b/.doctrees/python_client/docs/PcieDevicesInner.doctree new file mode 100644 index 000000000..909a5bfd2 Binary files /dev/null and b/.doctrees/python_client/docs/PcieDevicesInner.doctree differ diff --git a/.doctrees/python_client/docs/PixelMaskStatistics.doctree b/.doctrees/python_client/docs/PixelMaskStatistics.doctree new file mode 100644 index 000000000..d3e9f09da Binary files /dev/null and b/.doctrees/python_client/docs/PixelMaskStatistics.doctree differ diff --git a/.doctrees/python_client/docs/Plot.doctree b/.doctrees/python_client/docs/Plot.doctree new file mode 100644 index 000000000..b9bb9578a Binary files /dev/null and b/.doctrees/python_client/docs/Plot.doctree differ diff --git a/.doctrees/python_client/docs/PlotUnitX.doctree b/.doctrees/python_client/docs/PlotUnitX.doctree new file mode 100644 index 000000000..5f0ace53b Binary files /dev/null and b/.doctrees/python_client/docs/PlotUnitX.doctree differ diff --git a/.doctrees/python_client/docs/Plots.doctree b/.doctrees/python_client/docs/Plots.doctree new file mode 100644 index 000000000..299ab240f Binary files /dev/null and b/.doctrees/python_client/docs/Plots.doctree differ diff --git a/.doctrees/python_client/docs/PowderCalibrationFitSigma.doctree b/.doctrees/python_client/docs/PowderCalibrationFitSigma.doctree new file mode 100644 index 000000000..3a283111b Binary files /dev/null and b/.doctrees/python_client/docs/PowderCalibrationFitSigma.doctree differ diff --git a/.doctrees/python_client/docs/PowderCalibrationOutput.doctree b/.doctrees/python_client/docs/PowderCalibrationOutput.doctree new file mode 100644 index 000000000..2dc2f66bb Binary files /dev/null and b/.doctrees/python_client/docs/PowderCalibrationOutput.doctree differ diff --git a/.doctrees/python_client/docs/PowderCalibrationQuality.doctree b/.doctrees/python_client/docs/PowderCalibrationQuality.doctree new file mode 100644 index 000000000..927c07f1a Binary files /dev/null and b/.doctrees/python_client/docs/PowderCalibrationQuality.doctree differ diff --git a/.doctrees/python_client/docs/PowderCalibrationSpotCheck.doctree b/.doctrees/python_client/docs/PowderCalibrationSpotCheck.doctree new file mode 100644 index 000000000..747636404 Binary files /dev/null and b/.doctrees/python_client/docs/PowderCalibrationSpotCheck.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimList.doctree b/.doctrees/python_client/docs/RoiAzimList.doctree new file mode 100644 index 000000000..f9a57907f Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimList.doctree differ diff --git a/.doctrees/python_client/docs/RoiAzimuthal.doctree b/.doctrees/python_client/docs/RoiAzimuthal.doctree new file mode 100644 index 000000000..9f8e90848 Binary files /dev/null and b/.doctrees/python_client/docs/RoiAzimuthal.doctree differ diff --git a/.doctrees/python_client/docs/RoiBox.doctree b/.doctrees/python_client/docs/RoiBox.doctree new file mode 100644 index 000000000..929be63d0 Binary files /dev/null and b/.doctrees/python_client/docs/RoiBox.doctree differ diff --git a/.doctrees/python_client/docs/RoiBoxList.doctree b/.doctrees/python_client/docs/RoiBoxList.doctree new file mode 100644 index 000000000..65cdc1ef3 Binary files /dev/null and b/.doctrees/python_client/docs/RoiBoxList.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircle.doctree b/.doctrees/python_client/docs/RoiCircle.doctree new file mode 100644 index 000000000..3fd5dacf9 Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircle.doctree differ diff --git a/.doctrees/python_client/docs/RoiCircleList.doctree b/.doctrees/python_client/docs/RoiCircleList.doctree new file mode 100644 index 000000000..ea9ddcafc Binary files /dev/null and b/.doctrees/python_client/docs/RoiCircleList.doctree differ diff --git a/.doctrees/python_client/docs/RoiDefinitions.doctree b/.doctrees/python_client/docs/RoiDefinitions.doctree new file mode 100644 index 000000000..e282ae2cf Binary files /dev/null and b/.doctrees/python_client/docs/RoiDefinitions.doctree differ diff --git a/.doctrees/python_client/docs/RotationAxis.doctree b/.doctrees/python_client/docs/RotationAxis.doctree new file mode 100644 index 000000000..91189243b Binary files /dev/null and b/.doctrees/python_client/docs/RotationAxis.doctree differ diff --git a/.doctrees/python_client/docs/ScanResult.doctree b/.doctrees/python_client/docs/ScanResult.doctree new file mode 100644 index 000000000..c6b205335 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResult.doctree differ diff --git a/.doctrees/python_client/docs/ScanResultImagesInner.doctree b/.doctrees/python_client/docs/ScanResultImagesInner.doctree new file mode 100644 index 000000000..48359b4a7 Binary files /dev/null and b/.doctrees/python_client/docs/ScanResultImagesInner.doctree differ diff --git a/.doctrees/python_client/docs/SpotFindingSettings.doctree b/.doctrees/python_client/docs/SpotFindingSettings.doctree new file mode 100644 index 000000000..d8e56788f Binary files /dev/null and b/.doctrees/python_client/docs/SpotFindingSettings.doctree differ diff --git a/.doctrees/python_client/docs/StandardDetectorGeometry.doctree b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree new file mode 100644 index 000000000..04e06feaf Binary files /dev/null and b/.doctrees/python_client/docs/StandardDetectorGeometry.doctree differ diff --git a/.doctrees/python_client/docs/TcpSettings.doctree b/.doctrees/python_client/docs/TcpSettings.doctree new file mode 100644 index 000000000..355545e87 Binary files /dev/null and b/.doctrees/python_client/docs/TcpSettings.doctree differ diff --git a/.doctrees/python_client/docs/UnitCell.doctree b/.doctrees/python_client/docs/UnitCell.doctree new file mode 100644 index 000000000..536658f22 Binary files /dev/null and b/.doctrees/python_client/docs/UnitCell.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree new file mode 100644 index 000000000..6af290dcc Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqMetadataSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree new file mode 100644 index 000000000..cbd200c29 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqPreviewSettings.doctree differ diff --git a/.doctrees/python_client/docs/ZeromqSettings.doctree b/.doctrees/python_client/docs/ZeromqSettings.doctree new file mode 100644 index 000000000..633af6f57 Binary files /dev/null and b/.doctrees/python_client/docs/ZeromqSettings.doctree differ diff --git a/ACKNOWLEDGEMENT.html b/ACKNOWLEDGEMENT.html new file mode 100644 index 000000000..8a1c95650 --- /dev/null +++ b/ACKNOWLEDGEMENT.html @@ -0,0 +1 @@ + Acknowledgements — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Acknowledgements

Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, “Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors” (2023), J. Synchrotron Rad., 30, 227-234 doi:10.1107/S1600577522010268.

Funding and support

The project is supported by:

  • Innosuisse via Innovation Project “NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications” (101.535.1 IP-ENG; Apr 2023 - Sep 2025).

  • ETH Domain via Open Research Data Contribute project (Jan - Dec 2023).

  • AMD University Program with donation of licenses of Ethernet IP cores and Vivado software.

Crystallographic methods adopted from other packages

The analysis pipeline reimplements methods first published, and in most cases first implemented, by other crystallographic software. The code below is Jungfraujoch’s own; the methods are theirs, and are acknowledged here. Where a package’s source was consulted this is said explicitly. Most of these packages are neither linked nor vendored; the three that are - GEMMI, traccc and fast-feedback-indexer - also carry a licence obligation, recorded in THIRD_PARTY_NOTICES.md.

Spot finding

CrystFEL — spot finding, the three-ring integration region, the serial/stills processing model, and the per-frame indexing acceptance test (indexing_peak_check() in peaks.c). T. A. White, R. A. Kirian, A. V. Martin, A. Aquila, K. Nass, A. Barty and H. N. Chapman, “CrystFEL: a software suite for snapshot serial crystallography” (2012), J. Appl. Cryst. 45, 335-341 doi:10.1107/S0021889812002312. The self-calibrating spot finder’s per-resolution-ring background statistics, with the Bragg peaks excluded by iterated clipping, follow Cheetah’s peakfinder8: A. Barty, R. A. Kirian, F. R. N. C. Maia, M. Hantke, C. H. Yoon, T. A. White and H. N. Chapman, “Cheetah: software for high-throughput reduction and analysis of serial femtosecond X-ray diffraction data” (2014), J. Appl. Cryst. 47, 1118-1131 doi:10.1107/S1600576714007626.

Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, A. Salzburger, S. N. Swatman and B. Yeo, “traccc: GPU track reconstruction library for HEP experiments” (2025), arXiv:2505.22822; traccc. The CPU spot extractor adapts its SparseCCL source, and the CUDA spot extractor follows the design of its GPU counterpart - a backward-neighbour graph over a sorted hit list, resolved by a parallel union-find. traccc is MPL-2.0; see THIRD_PARTY_NOTICES.md. The SparseCCL algorithm itself is A. Hennequin, B. Couturier, V. V. Gligorov and L. Lacassagne, “SparseCCL: Connected Components Labeling and Analysis for sparse images” (2019), DASIP 2019, 65-70 doi:10.1109/DASIP48288.2019.9049184.

Indexing

MOSFLM — the Rossmann FFT autoindexing algorithm and post-refinement practice, including which parameters are safe to refine per image and which must be refined over a wedge. The autoindexing algorithm itself — projecting the reciprocal-space points onto many directions and Fourier-transforming the 1D projection histograms — is I. Steller, R. Bolotovsky and M. G. Rossmann, “An algorithm for automatic indexing of oscillation images using Fourier analysis” (1997), J. Appl. Cryst. 30, 1036-1040 doi:10.1107/S0021889897008777; MOSFLM is the implementation whose practice is followed. A. G. W. Leslie and H. R. Powell, “Processing diffraction data with MOSFLM” (2007), in Evolving Methods for Macromolecular Crystallography, NATO Science Series II, vol. 245, 41-51 doi:10.1007/978-1-4020-6316-9_4; T. G. G. Battye, L. Kontogiannis, O. Johnson, H. R. Powell and A. G. W. Leslie, “iMOSFLM: a new graphical interface for diffraction-image processing with MOSFLM” (2011), Acta Cryst. D67, 271-281 doi:10.1107/S0907444910048675; H. R. Powell, T. G. G. Battye, L. Kontogiannis, O. Johnson and A. G. W. Leslie, “Integrating macromolecular X-ray diffraction data with the graphical user interface iMosflm” (2017), Nat. Protoc. 12, 1310-1325 doi:10.1038/nprot.2017.037.

fast-feedback-indexer — the known-cell indexer for serial stills (-X ffbidx) is PSI’s fast-feedback-indexer library, linked at build time (BSD-3-Clause; see THIRD_PARTY_NOTICES.md), which implements the TORO algorithm: P. Gasparotto, L. Barba, H.-C. Stadler, G. Assmann, H. Mendonça, A. W. Ashton, M. Janousch, F. Leonarski and B. Béjar, “TORO Indexer: a PyTorch-based indexing algorithm for kilohertz serial crystallography” (2024), J. Appl. Cryst. 57, 931-944 doi:10.1107/S1600576724003182.

Cell reduction and lattice symmetry

GEMMI — symmetry operations, unit-cell and structure-factor machinery, and MTZ / XDS_ASCII I/O. Vendored in gemmi_gph/, so it also carries a licence obligation. M. Wojdyr, “GEMMI: A library for structural biology” (2022), J. Open Source Softw. 7, 4200 doi:10.21105/joss.04200.

Křivý & Gruber’s Niggli reduction, and the lattice-character table — the reduction that puts every candidate cell in a comparable form is I. Křivý and B. Gruber, “A unified algorithm for determining the reduced (Niggli) cell” (1976), Acta Cryst. A32, 297-298 doi:10.1107/S0567739476000636, used through GEMMI’s implementation; the table of lattice characters that maps a reduced cell to Bravais lattices and centrings follows International Tables for Crystallography Vol. A, Table 9.2.5.1.

Grosse-Kunstleve, Sauter & Adams’s numerically stable cell reduction - the magnitude-scaled tolerance that decides the sign of a structurally-zero scalar product, and with it the Niggli type a reduced cell is presented in. R. W. Grosse-Kunstleve, N. K. Sauter and P. D. Adams, “Numerically stable algorithms for the computation of reduced unit cells” (2004), Acta Cryst. A60, 1-6 doi:10.1107/S010876730302186X.

Le Page’s metric-symmetry search - the obliquity of each of the 81 candidate two-folds of a reduced cell, which is what tells a run that its lattice metric hosts more rotational symmetry than the group its intensities supported, and the derivation of the conventional axes from that rotation group, which is what the run then offers to the space-group search as a second lattice candidate. The two-fold search is used through GEMMI’s implementation of it. Y. Le Page, “The derivation of the axes of the conventional unit cell from the dimensions of the Buerger-reduced cell” (1982), J. Appl. Cryst. 15, 255-259 doi:10.1107/S0021889882011959.

Integration and rotation geometry

XDS — rotation geometry and notation, the reciprocal Lorentz and partiality treatment, the maximum-likelihood mosaicity estimate, the MINPK criterion for rejecting a reflection whose predicted profile is not cleanly its own, the intensity-based test for a centred lattice, the recognition of shaded detector regions by comparing a pixel’s background against the background at its own resolution (DEFPIX), and the scaling correction surfaces indexed by image number and detector region. W. Kabsch, “XDS” (2010), Acta Cryst. D66, 125-132 doi:10.1107/S0907444909047337; W. Kabsch, “Integration, scaling, space-group assignment and post-refinement” (2010), Acta Cryst. D66, 133-144 doi:10.1107/S0907444909047374.

Profile fitting with reweighted, de-biased variances is the Kabsch/Otwinowski iteration, from the second XDS paper above and from Z. Otwinowski and W. Minor, “Processing of X-ray diffraction data collected in oscillation mode” (1997), Methods Enzymol. 276, 307-326 doi:10.1016/S0076-6879(97)76066-X.

The two-dimensional integration architecture — integrating each image in the detector plane and only afterwards assembling a reflection’s partials into a full across images, as against three-dimensional profile fitting through the image stack — is the architecture of DENZO/SCALEPACK and MOSFLM, and it is the one Rugnux’s rotation pipeline follows (per-image profile-fitted integration, then partials combined into fulls; §9 and §10.6 of the data-analysis reference). The Otwinowski & Minor citation above and the MOSFLM citations below carry the credit for the paradigm as well as for the specifics taken from each.

Space group, twinning and pseudo-symmetry

POINTLESS (CCP4) — the space-group search. Stage A scores each candidate rotation operator by the correlation of I(h) with I(Rh) on resolution-normalised intensities (E²), as POINTLESS does — both arms of a symmetry pair sit at the same |s|, so on raw intensities the resolution fall-off is variance shared between them and lifts a false operator’s correlation as much as a true one’s; the screw-axis test scores a predicted-absent class against the rest of its own axial row rather than against a global mean or a fixed cut, and lets confidence fall away with the number of axial reflections instead of refusing below a count; the glide-plane test is that same test applied to a zone, scoring the extinguished class against the rest of its own plane, as POINTLESS scores zonal absences. P. Evans, “Scaling and assessment of data quality” (2006), Acta Cryst. D62, 72-82 doi:10.1107/S0907444905036693; P. R. Evans, “An introduction to data reduction: space-group determination, scaling and intensity statistics” (2011), Acta Cryst. D67, 282-292 doi:10.1107/S090744491003982X; P. R. Evans and G. N. Murshudov, “How good are my data and what is the resolution?” (2013), Acta Cryst. D69, 1204-1214 doi:10.1107/S0907444913000061; J. Agirre, M. Atanasova, H. Bagdonas et al., “The CCP4 suite: integrative software for macromolecular crystallography” (2023), Acta Cryst. D79, 449-461 doi:10.1107/S2059798323003595.

Baur and Kassner — the convention for which of two absence-equivalent glide groups the space-group search writes (P2/c over Pc, C2/c over Cc): the centrosymmetric one, because a missed inversion centre is the common error among published small-molecule space-group assignments. W. H. Baur and D. Kassner, “The perils of Cc: comparing the frequencies of falsely assigned space groups with their general population” (1992), Acta Cryst. B48, 356-369 doi:10.1107/S0108768191014726.

The centre-of-symmetry statistics are Wilson’s and Howells, Phillips & Rogers’s: the intensity distributions of acentric and centric structures and the cumulative N(z) test built on them, read here on the general reflections of the Laue class beside <|E^2-1|> and Padilla and Yeates’s L test (taken at its centric value, 2/pi). A. J. C. Wilson, “The probability distribution of X-ray intensities” (1949), Acta Cryst. 2, 318-321 doi:10.1107/S0365110X49000813; E. R. Howells, D. C. Phillips and D. Rogers, “The probability distribution of X-ray intensities. II. Experimental investigation and the X-ray detection of centres of symmetry” (1950), Acta Cryst. 3, 210-214 doi:10.1107/S0365110X50000513.

The twinning L test is Padilla and Yeates’s: pairing each acentric reflection with a symmetry-independent neighbour and reading the first and second moments of L = (I1−I2)/(I1+I2) against their untwinned and perfect-twin values. J. E. Padilla and T. O. Yeates, “A statistic for local intensity differences: robustness to anisotropy and pseudo-centering and utility for detecting twinning” (2003), Acta Cryst. D59, 1124-1130 doi:10.1107/S0907444903007947. Their title claims robustness to pseudo-centering, and this program’s partner steps deliver it: a step of 2 along an axis preserves the class of a half-integer pseudo-translation, which is what a pseudo-centering is. That robustness does not extend to a pseudo-translation which is not half-integer, and the partner steps are restricted when one is detected - see the translational-pseudo-symmetry note below.

Translational pseudo-symmetry is detected from the native Patterson computed from the merged intensities, and the interpretation of an off-origin peak as a pseudo-translation between copies of the contents of the asymmetric unit - together with the modulation it puts on the intensities, which is the second half of the test here - is Read, Adams and McCoy’s. Their fitted peak-height table is not used: the peak is scored against a per-dataset within-shell permutation null instead, because the noise floor of the statistic depends strongly on how many reflections a dataset has. The same modulation is what the axial systematic-absence test scores against, so that a reflection class a pseudo-translation merely suppresses is not read as extinct and does not buy a screw axis; the estimate of its depth there is our own, measured per axial row from the merged intensities rather than from the Patterson vector. R. J. Read, P. D. Adams and A. J. McCoy, “Intensity statistics in the presence of translational noncrystallographic symmetry” (2013), Acta Cryst. D69, 176-183 doi:10.1107/S0907444912045374.

Scaling, merging and data quality

DIALS — the resolution cutoff from the CC1/2 fall-off, per-observation outlier rejection at merge, the scaling error model, and the treatment of a reflection whose background is contaminated. Its published behaviour, and in places its source, settled several choices here. G. Winter, D. G. Waterman, J. M. Parkhurst et al., “DIALS: implementation and evaluation of a new integration package” (2018), Acta Cryst. D74, 85-97 doi:10.1107/S2059798317017235; D. G. Waterman, G. Winter, R. J. Gildea et al., “Diffraction-geometry refinement in the DIALS framework” (2016), Acta Cryst. D72, 558-575 doi:10.1107/S2059798316002187; J. Beilsten-Edmands, G. Winter, R. Gildea et al., “Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling” (2020), Acta Cryst. D76, 385-399 doi:10.1107/S2059798320003198; J. M. Parkhurst, G. Winter, D. G. Waterman et al., “Robust background modelling in DIALS” (2016), J. Appl. Cryst. 49, 1912-1921 doi:10.1107/S1600576716013595.

Wilson outlier test — judging an observation that has no symmetry mates against the acentric and centric intensity distributions of its resolution shell, with the symmetry enhancement factor, is Wilson’s statistics; rejecting only observations that are also significant, and keeping a reflection whose observations are all large, follows AIMLESS’s EMAX test. A. J. C. Wilson, “The probability distribution of X-ray intensities” (1949), Acta Cryst. 2, 318-321 doi:10.1107/S0365110X49000813; P. Evans, “Scaling and assessment of data quality” (2006), Acta Cryst. D62, 72-82 doi:10.1107/S0907444905036693.

Amplitudes from intensities — the posterior-mean amplitude of each merged intensity under the acentric and centric Wilson priors is French and Wilson’s; giving no amplitude to an intensity more than 3.7 sigma below zero, and leaving such intensities out of the prior, follows CCP4’s ctruncate (C. Ballard and N. Stein). So does scaling each reflection’s Wilson prior by the anisotropy tensor along its direction, which ctruncate has done by default since its version 1.7 (“use anisotropy in prior for truncate procedure”); rugnux uses its own tensor for it (see Diffraction anisotropy below). S. French and K. Wilson, “On the treatment of negative intensity observations” (1978), Acta Cryst. A34, 517-525 doi:10.1107/S0567739478001114; ctruncate is cited through the CCP4 suite: M. D. Winn, C. C. Ballard, K. D. Cowtan et al., “Overview of the CCP4 suite and current developments” (2011), Acta Cryst. D67, 235-242 doi:10.1107/S0907444910045749.

Absorption as spherical harmonics — describing an empirical absorption correction as a series of real spherical harmonics of the beam directions in the crystal frame is Blessing’s; its use as a restrained scaling surface of the diffracted-beam direction follows SCALA and AIMLESS. R. H. Blessing, “An empirical correction for absorption anisotropy” (1995), Acta Cryst. A51, 33-38 doi:10.1107/S0108767394005726; P. Evans, “Scaling and assessment of data quality” (2006), Acta Cryst. D62, 72-82 doi:10.1107/S0907444905036693.

Diffraction anisotropy — the description of the overall fall-off by a single anisotropic displacement tensor, its symmetry constraints, and the fact that only its deviatoric part is determined (the isotropic part being degenerate with the overall scale) are Sheriff and Hendrickson’s. The estimator fits that tensor to the observed intensity distribution, taking sigma(I) into account, in the sense of Popov and Bourenkov. The directional diffraction limits - <I/sigma(I)> in a cone about each principal direction, and the reporting of the anisotropic deltaB as the range of the principal components - follow AIMLESS. Rugnux reports these; it corrects no intensity and removes no reflection on a directional criterion. S. Sheriff and W. A. Hendrickson, “Description of overall anisotropy in diffraction from macromolecular crystals” (1987), Acta Cryst. A43, 118-121 doi:10.1107/S010876738709977X; A. N. Popov and G. P. Bourenkov, “Choice of data-collection parameters based on statistic modelling” (2003), Acta Cryst. D59, 1145-1153 doi:10.1107/S0907444903008163; P. R. Evans and G. N. Murshudov, “How good are my data and what is the resolution?” (2013), Acta Cryst. D69, 1204-1214 doi:10.1107/S0907444913000061.

Data-quality statistics follow the established conventions rather than any one program: R_meas and R_pim, CC1/2 and CC*, the per-shell CC(model, data) between F^2_calc and F^2_obs, and the reporting of I/sigma(I). K. Diederichs and P. A. Karplus, “Improved R-factors for diffraction data analysis in macromolecular crystallography” (1997), Nat. Struct. Biol. 4, 269-275 doi:10.1038/nsb0497-269; P. A. Karplus and K. Diederichs, “Linking crystallographic model and data quality” (2012), Science 336, 1030-1033 doi:10.1126/science.1218231; K. Diederichs and P. A. Karplus, “Better models by discarding data?” (2013), Acta Cryst. D69, 1215-1222 doi:10.1107/S0907444913001121.

The Whittaker smoother — the per-frame scale of the rotation fulls is a penalised least-squares curve (a second-difference penalty, the smoothness chosen by cross-validation) in the form P. H. C. Eilers gave Whittaker’s graduation: P. H. C. Eilers, “A perfect smoother” (2003), Anal. Chem. 75, 3631-3636 doi:10.1021/ac034173t; E. T. Whittaker, “On a new method of graduation” (1923), Proc. Edinburgh Math. Soc. 41, 63-75 doi:10.1017/S0013091500077853.

Fisher’s z-transformation — the cross-validation of the scaling correction surfaces averages the change of the half-set CC1/2 over resolution shells on atanh(CC), so that the shells near CC = 1, where a multiplicative error shows, are not outweighed by the noise of the shells without signal. R. A. Fisher, “Frequency distribution of the values of the correlation coefficient in samples from an indefinitely large population” (1915), Biometrika 10, 507-521 doi:10.2307/2331838.

The frame disposition - which stretches of a rotation sweep are kept, carried at reduced weight or dropped from the merge - decides every exclusion on delta-CC1/2, the change in the overall CC1/2 when a group of images is left out, measured in the sigma-tau form so that no random half-dataset split is involved. The statistic, the Fisher transformation used to compare it across CC1/2 values, its standard error going as the inverse square root of the reflection count, and the rejection discipline (never remove a group whose delta-CC1/2 is positive or near zero; remove a little, re-form the reference and repeat) are all taken from its authors, whose XDSCC12 is the reference implementation. G. Assmann, W. Brehm and K. Diederichs, “Identification of rogue datasets in serial crystallography” (2016), J. Appl. Cryst. 49, 1021-1028 doi:10.1107/S1600576716005471; G. M. Assmann, M. Wang and K. Diederichs, “Making a difference in multi-data-set crystallography: simple and deterministic data-scaling/selection methods” (2020), Acta Cryst. D76, 636-652 doi:10.1107/S2059798320006348. That the same statistic belongs at scaling, applied to groups of images rather than to whole datasets, follows DIALS (dials.scale, delta-CC1/2 image-group filtering): J. Beilsten-Edmands, G. Winter, R. Gildea et al., “Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling” (2020), Acta Cryst. D76, 385-399 doi:10.1107/S2059798320003198.

Uncertainty conventions follow the IUCr Commission on Crystallographic Nomenclature: D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., “Statistical descriptors in crystallography: Report of the IUCr Subcommittee on Statistical Descriptors” (1989), Acta Cryst. A45, 63-75 doi:10.1107/S0108767388009596; D. Schwarzenbach, S. C. Abrahams, H. D. Flack, E. Prince and A. J. C. Wilson, “Statistical descriptors in crystallography. II. Report of a Working Group on Expression of Uncertainty in Measurement” (1995), Acta Cryst. A51, 565-569 doi:10.1107/S0108767395002340.

Physical corrections and calibration

Sensor absorption at oblique incidence, and the flight path — the angle-dependent quantum efficiency of a flat sensor, the radial parallax variance that comes from the same integral, and the attenuation of a reflection in the air between the sample and its pixel are all the Beer-Lambert law taken along a ray that crosses t/cos(alpha) of sensor, or D/cos(alpha) of air, and converts at a random depth. The attenuation coefficients, for silicon, CdTe, dry air and helium alike, are the NIST tabulation: J. H. Hubbell and S. M. Seltzer, “Tables of X-Ray Mass Attenuation Coefficients and Mass Energy-Absorption Coefficients from 1 keV to 20 MeV for Elements Z = 1 to 92 and 48 Additional Substances of Dosimetric Interest” (1995, data updated 2004), NIST Standard Reference Database 126 doi:10.18434/T4D01F.

Polarization correction — the azimuthal polarization factor applied to the azimuthally integrated profile, to the integrated Bragg intensities and to the ring background the beam-stop shadow test compares a pixel against is the one derived for a partially polarized synchrotron source by R. Kahn, R. Fourme, A. Gadet, J. Janin, C. Dumas and D. Andre, “Macromolecular crystallography with synchrotron radiation: photographic data collection and polarization correction” (1982), J. Appl. Cryst. 15, 330-337 doi:10.1107/S0021889882012060.

Hexagonal-ice ring positions — the eleven ring \(d\) spacings from 3.895 to 1.522 Å that the ice-ring score, the ice-ring flagging and the ice calibrant are all built on are taken from the measurements of Moreau and co-workers, not enumerated from a cell. D. W. Moreau, H. Atakisi and R. E. Thorne, “Ice in biomolecular cryocrystallography” (2021), Acta Cryst. D77, 540-554 doi:10.1107/S2059798321001170.

That list ends at 1.522 Å by its own scope, so the eight bands below it are calculated here rather than taken from anyone: ice Ih structure factors on the oxygen sublattice, kept where they reach 3% of the strongest line, which reproduces the eleven measured positions exactly. The lattice constants are Röttger and co-workers’. K. Röttger, A. Endriss, J. Ihringer, S. Doyle and W. F. Kuhs, “Lattice constants and thermal expansion of H2O and D2O ice Ih between 10 and 265 K” (1994), Acta Cryst. B50, 644-648 doi:10.1107/S0108768194004933.

Model-based analysis and maps

Bulk-solvent correction and overall scaling — the model’s structure factors are put on the observed scale with an overall factor, an anisotropic B and a flat bulk-solvent term, the flat-mask model of A. Fokine and A. Urzhumtsev, “Flat bulk-solvent model: obtaining optimal parameters” (2002), Acta Cryst. D58, 1387-1392 doi:10.1107/S0907444902010284, which is also the source of the starting values and of the range those two parameters are physically meaningful over. The procedure that fits them — a grid search over that range for the solvent pair, with the overall scale and the anisotropic B refitted at every grid point — follows P. V. Afonine, R. W. Grosse-Kunstleve and P. D. Adams, “A robust bulk-solvent correction and anisotropic scaling procedure” (2005), Acta Cryst. D61, 850-855 doi:10.1107/S0907444905007894. The fit is unweighted, as in both that procedure and REFMAC5: G. N. Murshudov, P. Skubak, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long and A. A. Vagin, “REFMAC5 for the refinement of macromolecular crystal structures” (2011), Acta Cryst. D67, 355-367 doi:10.1107/S0907444911001314.

sigma_A map coefficients — the maps written by --model are weighted by a maximum-likelihood sigma_A estimated per resolution shell, giving 2mFo-DFc and mFo-DFc rather than 2Fo-Fc and Fo-Fc. What is taken is the formalism itself: the Rice and Woolfson likelihoods of |Fo| given |Fc| and sigma_A, the figure of merit m and the scale D that follow from it, and the result that the bias-corrected coefficient is 2mFo-DFc for an acentric reflection and mFo for a centric one. R. J. Read, “Improved Fourier coefficients for maps using phases from partial structures with errors” (1986), Acta Cryst. A42, 140-149 doi:10.1107/S0108767386099622.

ANODE — reading the anomalous difference map at the atoms of a supplied model and reporting the strongest sites by name, instead of searching the map for blobs. The map itself is the textbook anomalous difference Fourier; what is taken from ANODE is that reading: A. Thorn and G. M. Sheldrick, “ANODE: anomalous and heavy-atom density calculation” (2011), J. Appl. Cryst. 44, 1285-1287 doi:10.1107/S0021889811041768.

Uniform random rotations — the null a supplied model is scored against re-orients that model at random about its own centroid, and the rotations are drawn uniformly from SO(3) through a uniform random unit quaternion. K. Shoemake, “Uniform Random Rotations”, in Graphics Gems III, ed. D. Kirk, Academic Press (1992), 124-132 (no DOI).

Variable projection — the rigid-body placement of a supplied model re-fits the overall scale at every step, and its Jacobian folds that re-fit in by projecting the scale’s own derivatives out of the placement’s, in Kaufman’s simplified form of Golub and Pereyra’s derivative of the reduced problem. G. H. Golub and V. Pereyra, “The Differentiation of Pseudo-Inverses and Nonlinear Least Squares Problems Whose Variables Separate” (1973), SIAM J. Numer. Anal. 10, 413-432 doi:10.1137/0710036. L. Kaufman, “A variable projection method for solving separable nonlinear least squares problems” (1975), BIT 15, 49-57 doi:10.1007/BF01932995.

Software and computing methods

Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading the result, follows Jon Wright (ESRF): “Experiences with GPU decompression for bitshuffle + LZ4 data”, HDF5 User Group meeting (2021), and bslz4decoders. The CUDA kernels in Jungfraujoch are its own, but the approach is his.

Removing the small islands of solvent from the bulk-solvent mask on the GPU labels the connected components with a parallel union-find, following D. P. Playne and K. Hawick, “A New Algorithm for Parallel Connected-Component Labelling on GPUs” (2018), IEEE Trans. Parallel Distrib. Syst. 29, 1217-1230 doi:10.1109/TPDS.2018.2799216.

This software uses the Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license. J. D. Hunter, “Matplotlib: A 2D graphics environment” (2007), Comput. Sci. Eng. 9, 90-95 doi:10.1109/MCSE.2007.55.

File formats read from a published specification

CBF / imgCIF - the native miniCBF reader implements the x-CBF_BYTE_OFFSET compression scheme and reads the imgCIF _axis table (the laboratory directions of the image’s fast and slow pixel directions, of the goniometer axes and of a 2theta arm) from the specification alone; no CBFlib or other CBF code is used, so there is no licence obligation, only this credit. H. J. Bernstein and A. P. Hammersley, “Specification of the Crystallographic Binary File (CBF/imgCIF)” (2006), International Tables for Crystallography Vol. G, 37-43 doi:10.1107/97809553602060000729; A. P. Hammersley, H. J. Bernstein and J. D. Westbrook, “Image dictionary (imgCIF)” (2006), International Tables for Crystallography Vol. G, 444-458 doi:10.1107/97809553602060000746.

d*TREK SMV - the SMV reader reads the d*TREK header vocabulary written by Rigaku’s CrystalClear (Saturn and R-AXIS detectors: detector and spatial-distortion vectors, detector circles, 2theta arm, encoded overflows) from the headers themselves, interpreting the detector vectors as dxtbx does for these detectors; no d*TREK or dxtbx code is used. J. W. Pflugrath, “The finer things in X-ray diffraction data collection” (1999), Acta Cryst. D55, 1718-1725 doi:10.1107/S090744499900935X; dxtbx: J. M. Parkhurst, A. S. Brewster, L. Fuentes-Montero, D. G. Waterman et al., “dxtbx: the diffraction experiment toolbox” (2014), J. Appl. Cryst. 47, 1459-1465 doi:10.1107/S1600576714011996.

Public diffraction data used for testing

In addition to in-house datasets collected at SLS 2.0, Jungfraujoch is tested against public diffraction data collected on other people’s beamlines, on detectors and in file formats we do not produce ourselves - most of it at other facilities, a few sets at the Swiss Light Source but not by this system. That data was collected and published by other people. Every dataset used, the DOI to cite for it, and the deposition it belongs to are listed in EXTERNAL_TEST_DATA; we thank the depositors, and the repositories that make the data findable and citable.

IRRMC, the Integrated Resource for Reproducibility in Macromolecular Crystallography (Minor lab, University of Virginia), is the source of most of them. IRRMC releases its data under CC0 and asks that the DOI of the dataset be cited. M. Grabowski, K. M. Langner, M. Cymborowski, P. J. Porebski, P. Sroka, H. Zheng, D. R. Cooper, M. D. Zimmerman, M.-A. Elsliger, S. K. Burley and W. Minor, “A public database of macromolecular diffraction experiments” (2016), Acta Cryst. D72, 1181-1193 doi:10.1107/S2059798316014716; M. Grabowski, M. Cymborowski, P. J. Porebski, T. Osinski, I. G. Shabalin, D. R. Cooper and W. Minor, “The Integrated Resource for Reproducibility in Macromolecular Crystallography: Experiences of the first four years” (2019), Struct. Dyn. 6, 064301 doi:10.1063/1.5128672.

SBGrid Data Bank, the structural biology community’s data publication service (SBGrid Consortium, Harvard Medical School). P. A. Meyer, S. Socias, J. Key, E. Ransey, E. C. Tjon, A. Buschiazzo et al., “Data publication with the structural biology data grid supports live analysis” (2016), Nat. Commun. 7, 10882 doi:10.1038/ncomms10882.

Zenodo, CERN’s open repository, hosts datasets deposited there directly by the groups that collected them. European Organization for Nuclear Research and OpenAIRE, “Zenodo” (2013), CERN doi:10.25495/7GXK-RD71. Some of those deposits are described in IUCrData Raw Data Letters; the letters are cited on the EXTERNAL_TEST_DATA page, beside the datasets they describe.

MXRDR, the Macromolecular Xtallography Raw Data Repository (ICM, University of Warsaw), releases its data under CC0 and asks that the DOI of the dataset be cited.

XRDa, the Xtal Raw Data Archive (Protein Data Bank Japan), which publishes raw diffraction images - X-ray, electron and neutron - and mints a DOI for each; it asks that the DOI of the dataset be cited. It has no canonical citation paper.

The ESRF data portal, through which the European Synchrotron publishes raw data under its data policy (CC BY 4.0, with the dataset DOI to be cited). R. Dimper, A. Götz, A. De Maria, V. A. Solé, M. Chaillet and B. Lebayle, “ESRF Data Policy, Storage, and Services” (2019), Synchrotron Rad. News 32, 7-12 doi:10.1080/08940886.2019.1608119.

The Keele University research data repository and UQ eSpace (The University of Queensland), which host the raw images of datasets deposited there by the groups that collected them; the dataset DOIs are cited on the EXTERNAL_TEST_DATA page.

The beamline, resolution, space group and unit cell quoted for each dataset are the values deposited with the corresponding PDB entry, read from the RCSB PDB data API. H. M. Berman, J. Westbrook, Z. Feng, G. Gilliland, T. N. Bhat, H. Weissig, I. N. Shindyalov and P. E. Bourne, “The Protein Data Bank” (2000), Nucleic Acids Res. 28, 235-242 doi:10.1093/nar/28.1.235.

Generative AI usage declaration

Large language models were used extensively in developing this code. Jungfraujoch development was supported with JetBrains AI (mostly GPT models) to refactor and verify particular code fragments. Rugnux was developed with the assistance of Claude Code (mostly the Opus model). This documentation was written with the assistance of Claude Opus and Fable models.

\ No newline at end of file diff --git a/BATTERY_REPORT.html b/BATTERY_REPORT.html new file mode 100644 index 000000000..a4a6db3f4 --- /dev/null +++ b/BATTERY_REPORT.html @@ -0,0 +1 @@ + Rugnux battery - 20261005-2102_3770f4_rc174-final-refmac — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Rugnux battery - 20261005-2102_3770f4_rc174-final-refmac

  • rugnux 1.0.0-rc.174, sha256 fee1c90710c43704, build flags: Release CXX_FLAGS=‘-march=x86-64-v3’ CUDA=ON

  • runner 3770f42c9, host mpc2898.psi.ch, 2026-10-05T21:02:33 -> 2026-10-06T06:58:55

  • hardware: AMD Ryzen 9 5950X 16-Core Processor, 3.40 GHz nominal (acpi_cppc), 5.08 GHz max boost, 16C/32T, 1 socket(s), 125 GiB RAM, GPU: NVIDIA GeForce RTX 5080 (16303 MiB)

  • arms: open, inhouse; 249 sets (full arm)

  • timing: NOT a reference - the GPU was shared

  • command options: (none beyond output selection)

  • command, open arm: rugnux --model <deposited coordinates> <input> (without –model where there is no deposited model: small molecules, unpublished sets)

  • command, inhouse arm: rugnux --report-resolution <XDS d_min>,<XDS d_max> [--model <reference model>] <input>: the second statistics table at XDS’s range is report-only; where XDS kept Friedel mates apart the scorer reads the run’s own per-hand table, which needs no flag. –model where the set names a PDB entry of its crystal form (the standard proteins)

Commentary

This is the validation battery of release 1.0.0-rc.174 (the binary built from 3770f42c9 with the CI flags), run once per set with the REFMAC model check, against the rc.173 battery as baseline. The private arm ran too; as always, only its aggregate is stated here: 23 pass, 2 fail, 2 not scored, with no verdict changed from rc.173.

  • Verdicts. Open arm 188 pass / 11 fail / 3 not scored over 202 sets, in-house 46 / 1 over 47. Every failure was known before this run. Seven are point groups called too high on twinned or pseudo-symmetric crystals (2wnq, 4bwl, 5ebi, 6oww, 6p8j, 8c3e, 3r6o), two are screw axes (5cc8, 9hnc), two are halved lattices (6z9g, 9min), and KDP is scored against I-42d while Rugnux reports the group with the same absences, I4_1md. On the 171 open sets both runs have, 164 of 168 scored pass against 166 of 169 at rc.173 (5ebi and 9hnc lost, 7mzt now not scored); the in-house sets both runs have all pass. The 31 open sets added since rc.173 (twinned crystals, home-source and other facilities) carry most of the failures.

  • R-free. The deposited model refined by REFMAC against Rugnux’s data comes within 0.05 of the same refinement against the depositor’s data on 174 of 182 structures (rc.173: 154 of 159). On the 160 structures both runs scored, the median change is -0.001, 33 better and 15 worse by more than 0.005. Two losses are worth a look: 9qw8 (0.264 to 0.338, with a coarser resolution cut, 1.71 against 1.59 A) and 9i80 (0.234 to 0.279 at the same resolution, a twinned P4_1 crystal).

  • Resolution. Rugnux’s own cut is finer than the deposited one on 173 of 184 structures.

  • Processing time. The median over all 249 sets is 28 s (open arm 33 s, rc.173 44 s); on the sets both runs have, the median time ratio is 0.88 on the open arm and 0.78 on the in-house arm. The battery times include reading the images from disk, so they understate the gain in computation. The in-house arm alone (standard proteins, small molecules and the two no-crystal controls; the proteins now also run the model check) has a median of 13.7 s against 17.4 s at rc.173; its slowest sets are KDP (93 s), L-cystine at 20 keV (57 s) and the two controls (44-48 s), which search for a lattice that is not there. The header line “the GPU was shared” only records that the run used --gpulock; no set saw another process on the GPU.

  • Small molecules. SHELXL R1 against the published structures is at or below XDS on the organic sets (aspirin 0.036, citric acid 0.037, HEPES 0.031) and well below it on KDP (0.048 against 0.112); YAG (0.102) and L-cystine (0.14-0.16) remain the weak cases.

Processing time of every open and in-house set

The same, over the first 60 s

Processing time of the in-house sets, over the first 60 s

R-free of the deposited model refined against the depositor's data and against Rugnux's data

High-resolution limit: deposition against Rugnux

Summary

arm

sets

pass

of which alt

fail

unscored

not run

pass rate

median res. gain

median ISa

median R_meas

median time s

total time min

open

202

188

5

11

3

0

188/199 (94%)

+12.1%

13.6

13.8%

33

176.2

inhouse

47

46

0

1

0

0

46/47 (98%)

+6.3%

21.8

9.8%

14

14.0

Plot in report.html: verdicts per arm

Pass rate is over the scored sets (pass + fail). of which alt counts the passes that are already in pass: rows where the answer matched an accepted alternative reference rather than the deposition, listed under Accepted alternatives below. In the bar chart they are drawn as their own segment. Every number in this table is rugnux’s own run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave out the no-crystal controls. R_meas and CC1/2 are pooled over each run’s own resolution range, so they describe a run and do not rank two; the like-for-like comparison with XDS is the reference-range table below. Each set runs once, so every time includes reading its images from disk (unless they were still in the page cache from before the run).

By population

arm

population

sets

pass rate

median res. gain

open

SMV

2

2/2 (100%)

+11.2%

open

cbf

97

90/95 (95%)

+12.2%

open

cdte

3

3/3 (100%)

+5.1%

open

cubic

13

13/13 (100%)

+14.7%

open

diffuse

4

4/4 (100%)

+9.9%

open

h5

57

56/57 (98%)

+12.6%

open

hexagonal

16

15/15 (100%)

+13.9%

open

home-source

11

8/11 (73%)

+13.2%

open

long-axis

2

2/2 (100%)

+16.2%

open

long-wavelength

1

1/1 (100%)

-

open

low-resolution

1

1/1 (100%)

-8.8%

open

marCCD

15

14/15 (93%)

+12.4%

open

marccd

8

8/8 (100%)

+11.5%

open

monoclinic

49

43/49 (88%)

+12.1%

open

nxs

2

1/1 (100%)

+8.5%

open

orthorhombic

48

45/47 (96%)

+12.0%

open

pseudo-merohedral

4

2/4 (50%)

+10.3%

open

pseudo-symmetry

7

5/7 (71%)

+9.9%

open

small-molecule

6

5/5 (100%)

-12.9%

open

smv

21

17/21 (81%)

+10.9%

open

tetragonal

24

23/24 (96%)

+10.3%

open

tncs

5

3/5 (60%)

+12.6%

open

triclinic

13

13/13 (100%)

+9.5%

open

trigonal

19

18/19 (95%)

+13.7%

open

twin

14

10/14 (71%)

+10.6%

open

two-wavelength

2

2/2 (100%)

+10.8%

inhouse

control

2

2/2 (100%)

-

inhouse

cubic

1

1/1 (100%)

+2.2%

inhouse

cytochrome-c

3

3/3 (100%)

+9.2%

inhouse

h5

47

46/47 (98%)

+6.3%

inhouse

hexagonal

2

2/2 (100%)

-

inhouse

insulin

9

9/9 (100%)

+5.2%

inhouse

iodine

2

2/2 (100%)

+16.9%

inhouse

long-wavelength

7

7/7 (100%)

+1.1%

inhouse

lysozyme

12

12/12 (100%)

+6.7%

inhouse

monoclinic

3

3/3 (100%)

+2.6%

inhouse

myoglobin

6

6/6 (100%)

-2.8%

inhouse

orthorhombic

1

1/1 (100%)

+2.6%

inhouse

pink-beam

3

3/3 (100%)

+16.1%

inhouse

small-molecule

8

7/8 (88%)

+2.6%

inhouse

tetragonal

1

0/1 (0%)

+10.3%

inhouse

thaumatin

7

7/7 (100%)

+2.7%

inhouse

twin

4

4/4 (100%)

-22.9%

Distributions (sets that merged, controls left out)

arm

metric

n

min

q1

median

q3

max

open

ISa

202

4.1

9.5

13.6

19.8

45.5

open

R_meas

202

2.5%

8.8%

13.8%

22.2%

366.1%

open

CC1/2

202

0.917

0.994

0.998

0.999

1.000

open

res. gain %

198

-68.3

+8.1

+12.1

+16.8

+34.3

open

low-res shell R_meas

202

1.7%

4.1%

5.5%

8.2%

33.8%

open

R_model, shell-scaled

196

0.096

0.180

0.209

0.247

0.580

open

R_free (placement, within-run only)

196

0.121

0.188

0.218

0.255

0.575

open

radial misfit

196

0.03

0.09

0.12

0.16

1.10

open

R_free ratio

196

0.813

0.961

1.030

1.118

3.380

open

anomalous map at the scatterers, sigma

196

-3.42

0.70

1.51

3.51

29.13

open

REFMAC R_free ratio

192

0.807

0.981

0.998

1.027

1.393

open

dCC vs depositor, all

193

-0.317

-0.018

-0.004

+0.003

+0.194

open

dCC vs depositor, outer 2 shells

193

-0.664

-0.032

-0.000

+0.030

+0.318

open

CC(Fc^2) past the deposited limit

178

-0.013

0.229

0.297

0.367

0.926

open

time s

202

5

18

33

71

401

inhouse

ISa

44

1.9

7.7

21.8

32.9

53.4

inhouse

R_meas

45

2.6%

6.3%

9.8%

25.2%

125.6%

inhouse

CC1/2

45

0.702

0.998

0.999

1.000

1.000

inhouse

res. gain %

43

-42.2

+1.8

+6.3

+11.1

+21.5

inhouse

ref-range ISa / XDS

43

0.209

0.838

1.104

1.264

6.410

inhouse

ref-range R_meas / XDS

43

0.207

0.838

0.942

1.054

2.123

inhouse

low-res shell R_meas

45

2.2%

3.7%

4.8%

9.3%

41.8%

inhouse

ref-range low-res R_meas / XDS

43

0.273

0.871

1.021

1.149

2.681

inhouse

ref-range CC1/2 noise / XDS

43

0.03

0.33

0.54

0.80

8.58

inhouse

R_model, shell-scaled

37

0.183

0.227

0.253

0.287

0.520

inhouse

R_free (placement, within-run only)

37

0.175

0.234

0.287

0.325

0.572

inhouse

radial misfit

37

0.02

0.07

0.10

0.18

0.27

inhouse

R_free ratio

37

0.940

1.226

1.531

1.724

2.624

inhouse

anomalous map at the scatterers, sigma

37

0.35

1.45

3.11

6.19

11.50

inhouse

time s

45

5

9

13

18

93

Per set, against the reference

Plot in report.html: open: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 = finer than the reference

Plot in report.html: inhouse: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 = finer than the reference

Plot in report.html: inhouse: ISa of the reference-range table (the error model refitted on it) / XDS’s ISa; above 1 = rugnux’s error model is the better one. One point per set, sorted.

Plot in report.html: inhouse: R_meas over the reference range / XDS’s R_meas; below 1 = rugnux’s merge is the more consistent one. One point per set, sorted.

Plot in report.html: inhouse: R_meas of the lowest-resolution shell of the reference-range table / of XDS’s table; below 1 = rugnux’s strong reflections agree better. One point per set, sorted.

Plot in report.html: inhouse: half-set noise over the reference range read off CC1/2 (1 / CC1/2 - 1) / XDS’s; 2 = as noisy as XDS’s merge would be with half its observations (not scored). One point per set, sorted.

Plot in report.html: R_free of the deposited model against our merge, as rugnux reports it with –model (a rigid-body placement, scored on rugnux’s own free set: a trend number, not a refinement) / the R_free the depositor published, one point per set, sorted; below 1 = lower than published

Plot in report.html: REFMAC check (–model-check): R_free of the deposited model against our merge / against the deposited structure factors, both on the depositor’s free set with the same protocol, one point per set, sorted; below 1 = our data fit the model better

Against the depositor’s data

Per resolution shell, the rank correlation of our merged intensities with |Fc|^2 minus that of the depositor’s own data, on the reflections both carry (d < 4 A, eight shells of equal count), |Fc| from the deposited model as it is (no bulk solvent, no refinement; depdata_check.py). The model was refined against the depositor’s data, so zero is already a good result: across a corpus the median sits near -0.01. ‘Past the limit’ is CC(I, |Fc|^2) of our reflections in the outermost of up to three shells beyond the deposited data’s limit, which the model never saw: clearly above zero there is signal. Reported, never scored.

set

dep. data

our d_min

dep. d_min

common refl.

dCC all

dCC outer 2

CC past limit

to d A

REFMAC ratio

ISa

tags

3r6o

F

1.53

1.95

16829

-0.314

-0.664

0.076

1.54

1.393

4.5

smv, tetragonal, home-source

2wnq

F

1.65

1.80

101895

-0.148

-0.626

0.039

1.65

1.066

10.3

smv, monoclinic, twin, pseudo-merohedral

4bwl

F

1.68

2.00

72820

-0.317

-0.520

0.117

1.67

1.030

14.3

smv, monoclinic, twin, pseudo-merohedral

2xfw

F

1.55

1.65

147572

-0.127

-0.515

0.258

1.55

1.158

14.1

smv, monoclinic, twin, pseudo-merohedral

3p85

F

1.62

1.90

24299

-0.162

-0.404

0.261

1.62

0.879

8.1

smv, hexagonal, home-source

2wnz

F

1.85

1.85

96083

-0.106

-0.396

-

-

1.113

14.8

smv, monoclinic, pseudo-symmetry

9qw2

F

1.76

1.92

43869

-0.172

-0.360

0.234

1.75

1.185

7.8

cbf, monoclinic, pseudo-symmetry

2wnn

F

1.44

1.65

123469

-0.055

-0.351

0.204

1.44

0.807

14.1

smv, monoclinic, twin, pseudo-merohedral

3mc4

F

1.79

1.95

24530

-0.161

-0.333

0.280

1.79

1.365

12.5

smv, trigonal, home-source

9qw8

I

1.71

1.80

38358

-0.192

-0.326

0.192

1.70

1.272

9.2

h5, triclinic

9lxl

I

2.06

2.19

18039

-0.277

-0.270

0.068

2.05

1.058

6.2

h5, tetragonal

7bgu

I

2.30

2.43

14571

-0.094

-0.246

0.009

2.30

1.140

10.0

marCCD, triclinic

9hnc

F

1.63

1.88

404705

-0.062

-0.237

0.376

1.63

0.923

13.6

cbf, monoclinic

6yqf

I

3.02

3.33

933

-0.134

-0.233

0.015

3.01

1.001

4.2

cbf, orthorhombic

6r72

F

4.39

3.95

18989

-0.092

-0.232

-

-

1.038

17.9

h5, monoclinic

9rcs

I

3.27

3.01

2052

-0.208

-0.171

-

-

0.928

7.3

h5, monoclinic, cdte

6zqy

F

1.70

1.85

47653

-0.029

-0.144

0.242

1.70

1.071

12.7

smv

5ky6

F

1.54

1.94

98442

-0.074

-0.142

0.159

1.54

1.076

6.1

marccd

8qq7

I

3.16

3.62

5507

-0.104

-0.107

-0.013

3.15

1.065

6.4

cbf, hexagonal

8k1g

F

1.63

2.09

34025

-0.077

-0.107

0.281

1.62

0.974

11.7

cbf, tetragonal

3meb

F

1.53

1.90

35703

-0.026

-0.107

0.330

1.54

0.993

15.6

smv, monoclinic, home-source

7arr

I

0.92

1.10

47848

-0.036

-0.103

0.293

0.92

1.016

18.9

cbf, triclinic

6zqr

F

1.76

1.93

37169

-0.051

-0.098

0.297

1.76

1.064

9.1

smv

6u7g

I

1.88

2.35

87246

-0.055

-0.097

0.243

1.88

0.980

13.3

h5, monoclinic

5epe

F

1.77

1.90

22792

-0.015

-0.093

0.322

1.77

1.039

9.9

marCCD, cubic

9pbb

I

1.78

2.16

11031

-0.048

-0.087

0.160

1.78

0.980

17.0

cbf, monoclinic

9hs7

F

1.70

1.70

9595

-0.131

-0.083

-0.008

1.69

0.958

13.3

cbf, hexagonal

8c3e

I

1.79

2.10

4079

-0.038

-0.076

0.162

1.80

1.031

5.8

cbf, trigonal, home-source, twin

6jgh

I

0.87

0.94

139810

-0.014

-0.070

0.333

0.87

1.069

6.6

marccd

6qaj

I

2.70

2.63

4163

-0.132

-0.069

-

-

0.951

13.2

cbf

5f6m

I

1.09

1.10

68757

-0.006

-0.059

0.364

1.09

1.032

23.0

cbf, orthorhombic, diffuse

7tcd

I

1.65

1.70

27140

-0.043

-0.057

0.083

1.65

0.968

18.4

h5, monoclinic

7os3

I

1.96

2.18

32881

-0.004

-0.054

0.355

1.96

0.985

24.4

cbf, orthorhombic

5t39

I

1.01

1.10

92595

-0.010

-0.053

0.400

1.01

1.003

16.2

marccd

6cdl

I

1.13

1.24

56384

-0.008

-0.051

0.353

1.13

1.039

11.6

marccd

8dyz

I

1.14

1.27

24752

-0.007

-0.050

0.625

1.14

0.958

37.8

cbf, tetragonal, diffuse

6oww

F

2.72

3.84

3373

+0.031

-0.049

0.053

2.71

0.938

11.8

cbf, monoclinic, pseudo-symmetry

9upt

F

2.03

2.37

24477

-0.036

-0.046

0.324

2.03

1.056

6.5

SMV, hexagonal

6z8o

F

2.21

2.20

22416

-0.136

-0.045

-

-

1.043

13.9

h5, monoclinic

9b22

I

1.14

1.30

96597

-0.014

-0.044

0.422

1.14

1.007

16.4

h5, monoclinic

7brr

I

1.24

1.40

102801

-0.023

-0.041

0.274

1.25

0.981

16.6

h5, monoclinic

5vml

F

1.92

1.70

14226

-0.015

-0.038

-

-

1.053

10.9

smv, tetragonal, home-source

9p7q

I

1.76

2.21

11981

-0.031

-0.037

0.282

1.76

1.017

10.8

cbf, monoclinic

7raa

I

2.58

2.69

7523

-0.065

-0.036

0.046

2.58

0.990

11.0

cbf

8a1a

F

1.93

2.05

101602

-0.009

-0.036

0.355

1.94

0.991

18.8

h5, hexagonal

5reo

F

1.65

1.88

18392

-0.026

-0.036

0.338

1.65

0.991

23.7

cbf, monoclinic

5m17

I

0.98

1.03

185105

-0.006

-0.034

0.311

0.98

1.024

16.2

cbf, tetragonal

8y74

F

1.68

1.90

58040

-0.010

-0.032

0.288

1.69

0.962

8.6

h5, monoclinic

6s1u

I

1.75

1.90

17747

-0.003

-0.031

0.344

1.75

0.975

13.8

marccd

9qvv

I

2.49

2.72

16358

-0.014

-0.030

0.141

2.50

0.947

37.3

nxs, orthorhombic, pseudo-symmetry

7atg

I

0.60

0.60

54556

-0.004

-0.026

-

-

-

22.9

cbf, orthorhombic

5lzl

F

2.86

3.47

21840

-0.014

-0.026

0.231

2.87

1.027

15.2

cbf, trigonal

9zlo

I

1.56

2.00

22408

-0.011

-0.024

0.242

1.56

1.011

24.1

h5, orthorhombic

9fhc

F

1.92

2.20

76515

-0.002

-0.024

0.353

1.92

1.042

12.3

marCCD, cubic

6iu8

I

2.32

2.70

15286

+0.004

-0.024

0.175

2.32

0.984

8.1

cbf, trigonal

8r5r

F

2.80

3.08

12960

-0.032

-0.024

0.211

2.80

1.003

19.3

h5, orthorhombic

6oel

I

2.85

3.10

13233

-0.013

-0.024

0.272

2.85

0.965

9.9

SMV, cubic

9yl4

I

3.61

3.70

1130

-0.010

-0.023

0.175

3.61

0.987

9.5

cbf, orthorhombic

9gdj

I

1.40

1.47

157444

-0.007

-0.023

0.299

1.40

1.021

12.9

h5, tetragonal, cdte

8agq

I

0.97

1.09

98576

-0.013

-0.022

0.298

0.97

1.153

14.1

cbf, monoclinic

9ih9

F

1.40

1.70

78425

-0.005

-0.021

0.381

1.40

1.007

12.7

h5, monoclinic

9q66

F

2.03

2.01

87912

-0.017

-0.021

-

-

1.017

13.1

h5, monoclinic

7mzt

I

3.12

4.05

8489

-0.074

-0.020

0.083

3.11

0.991

4.1

cbf, orthorhombic

9zm0

I

1.80

2.10

13858

-0.014

-0.020

0.115

1.80

0.984

9.0

h5, monoclinic

7kcn

I

1.39

1.46

43289

-0.011

-0.020

0.569

1.39

1.055

11.7

cbf, tetragonal

6ze4

I

1.30

1.60

136778

-0.054

-0.020

0.323

1.29

1.040

9.3

cbf, orthorhombic

9e2t

I

2.29

2.28

32088

-0.110

-0.019

-

-

0.980

6.8

cbf, triclinic

9gjx

I

2.06

2.40

52052

-0.001

-0.019

0.268

2.06

1.033

40.1

h5, monoclinic

7q6j

I

1.98

2.20

41720

+0.005

-0.019

0.215

1.98

0.996

11.0

cbf, orthorhombic, pseudo-symmetry

6fvz

F

1.49

1.80

105993

-0.019

-0.018

0.386

1.50

0.998

20.5

cbf, orthorhombic

6ttn

I

1.08

1.12

126153

-0.001

-0.018

0.223

1.08

1.020

14.5

cbf, orthorhombic

6iu5

I

2.12

2.25

30820

-0.010

-0.018

0.231

2.12

1.008

8.0

cbf, trigonal, twin

7n2s

I

2.56

2.37

10827

-0.009

-0.018

-

-

0.968

9.6

cbf, monoclinic

6fid

F

1.98

2.20

10527

-0.005

-0.018

0.819

1.97

0.977

13.6

cbf, orthorhombic

6p8p

I

1.46

1.64

64791

-0.015

-0.017

0.289

1.45

1.036

15.5

cbf, tetragonal

6iu6

I

2.36

2.90

10758

+0.003

-0.016

0.252

2.36

1.063

7.2

cbf, trigonal, twin

6w75

F

1.69

1.95

99979

-0.010

-0.016

0.320

1.69

0.991

15.8

marccd

9h0q

I

2.10

2.55

45699

-0.005

-0.016

0.343

2.11

0.980

18.6

h5

6h5t

F

1.48

1.69

28184

-0.007

-0.015

0.356

1.48

1.057

9.3

marCCD, tetragonal

9crw

I

2.28

2.49

53873

-0.021

-0.014

0.130

2.28

1.010

15.8

h5, monoclinic

6rlr

I

1.92

2.00

19974

+0.002

-0.014

0.189

1.92

1.022

16.4

h5, triclinic, twin

5uth

F

1.72

1.95

27663

-0.015

-0.012

0.436

1.73

1.009

9.5

smv, trigonal, home-source

6w4h

F

1.62

1.80

70331

-0.003

-0.012

0.419

1.62

0.996

18.5

marCCD, trigonal

9jq9

I

1.65

1.90

13865

+0.001

-0.011

0.509

1.65

1.005

19.5

cbf, orthorhombic, home-source

6rym

I

1.45

1.46

17018

-0.005

-0.009

-

-

1.022

22.9

smv, tetragonal

6hv2

F

1.41

1.71

19192

-0.035

-0.008

0.102

1.41

0.829

13.7

h5, hexagonal

8dz7

I

1.19

1.34

13603

-0.002

-0.008

0.809

1.19

1.009

35.1

cbf, orthorhombic, diffuse

9chw

I

1.60

2.16

20358

-0.005

-0.008

0.648

1.60

1.006

21.3

marCCD, hexagonal

7l84

I

1.70

1.71

11375

-0.001

-0.008

-

-

0.976

11.4

cbf, tetragonal

5mln

F

1.26

1.59

60800

-0.012

-0.007

0.409

1.26

1.017

21.5

cbf

5ojv

I

1.82

2.06

63705

-0.030

-0.005

0.267

1.82

0.998

13.2

cbf, orthorhombic, pseudo-symmetry

6pxb

I

1.39

1.75

50825

+0.008

-0.005

0.097

1.40

0.991

12.1

cbf, trigonal

7ris

I

1.51

1.72

22161

-0.001

-0.004

0.221

1.51

0.987

31.0

h5, trigonal

6o2h

I

1.10

1.21

11612

-0.002

-0.003

0.926

1.10

1.157

23.9

cbf, triclinic, diffuse

8xtf

I

1.84

2.13

27230

+0.000

-0.003

0.420

1.83

0.956

7.6

h5, trigonal

7l6j

F

1.51

1.78

37485

-0.002

-0.001

0.414

1.51

1.013

10.5

marCCD, cubic

7bgt

I

1.78

1.93

33462

-0.003

-0.000

0.308

1.79

0.999

17.4

marCCD, triclinic

9sl0

I

1.36

1.60

67358

+0.002

+0.001

0.243

1.37

0.996

20.2

h5, orthorhombic

6cee

I

1.38

1.55

14202

-0.001

+0.001

0.678

1.38

0.993

23.9

smv, orthorhombic, home-source

3ky7

F

1.92

2.35

11401

+0.002

+0.001

0.182

1.92

0.992

12.2

marCCD, cubic

6i3j

I

2.32

2.59

34974

-0.006

+0.002

0.366

2.32

1.154

6.8

marCCD, orthorhombic

8tyy

I

1.28

1.68

44790

-0.001

+0.002

0.476

1.28

0.989

16.3

cbf, cubic

9i80

I

1.59

1.95

68449

-0.011

+0.002

0.351

1.59

1.024

6.8

h5, tetragonal, twin

3inp

I

1.70

2.05

26251

-0.005

+0.003

0.276

1.70

0.984

13.2

marCCD, cubic

8sqt

I

1.88

2.20

9296

-0.011

+0.004

0.320

1.88

1.044

29.9

h5, cubic

6wzo

I

1.04

1.42

91585

-0.010

+0.004

0.355

1.04

0.965

16.8

cbf, triclinic

6vww

F

1.99

2.20

58979

-0.003

+0.004

0.345

1.98

1.025

8.0

cbf, hexagonal, twin

6zr0

F

1.66

1.94

40650

-0.006

+0.005

0.345

1.65

1.014

24.5

cbf

9mh4

I

2.78

3.05

9490

+0.006

+0.005

0.206

2.79

1.017

13.8

h5, cubic

8dqb

I

2.05

2.50

19165

-0.002

+0.006

0.382

2.05

1.005

22.1

h5, cubic

6nen

I

1.77

2.15

10259

+0.003

+0.007

0.363

1.77

1.008

6.3

smv

6gvk

I

1.42

1.55

32916

+0.002

+0.007

0.227

1.42

0.989

19.8

cbf, monoclinic

8v4j

I

1.10

1.30

41379

+0.001

+0.008

0.580

1.10

0.983

22.1

smv, tetragonal

9zmu

I

1.72

1.98

21684

-0.002

+0.009

0.189

1.71

0.997

11.6

h5, hexagonal

6v2r

I

1.38

1.57

9330

-0.000

+0.009

0.458

1.38

0.996

24.4

smv, tetragonal, home-source

6cs9

I

1.72

1.85

5090

-0.000

+0.010

0.328

1.72

1.018

13.1

smv, monoclinic

9vyb

I

1.67

2.12

5191

-0.007

+0.012

0.352

1.68

0.995

22.1

h5, orthorhombic

7yzx

F

1.88

1.90

83105

-0.031

+0.013

0.319

1.88

1.025

10.4

cbf, hexagonal

8v2t

I

1.17

1.40

33174

+0.001

+0.014

0.369

1.16

0.975

11.9

cbf, tetragonal

9gqg

I

1.81

2.00

15522

-0.004

+0.014

0.241

1.81

1.036

12.8

h5, trigonal

9ig7

F

2.02

2.60

26626

-0.007

+0.015

0.258

2.02

1.030

11.1

cbf, orthorhombic

8egn

F

1.63

1.95

38629

-0.011

+0.015

0.276

1.62

0.997

22.9

cbf, orthorhombic

6f3p

F

1.13

1.35

235840

-0.001

+0.015

0.386

1.13

0.999

9.4

marccd

8iya

F

1.91

2.43

16262

-0.002

+0.016

0.239

1.92

0.998

5.8

h5, monoclinic

6g1f

I

1.93

2.25

131738

+0.001

+0.017

0.237

1.93

1.018

21.0

cbf

6fwc

F

1.41

1.70

126460

-0.001

+0.018

0.394

1.41

0.980

27.3

cbf, orthorhombic

9jzo

F

1.15

1.40

47773

+0.002

+0.018

0.769

1.15

1.045

8.1

cbf, triclinic

8sqq

I

1.90

2.25

8697

+0.006

+0.018

0.331

1.91

0.937

17.6

h5, cubic

5jk4

I

1.02

1.01

133760

-0.001

+0.018

-

-

0.992

14.8

smv, monoclinic

9bn8

I

1.21

1.35

119114

-0.001

+0.019

0.488

1.21

0.975

19.8

h5, tetragonal

7dkp

F

1.18

1.45

131351

+0.004

+0.020

0.679

1.18

0.984

26.1

h5, monoclinic

8qaw

F

1.30

1.55

256134

+0.001

+0.022

0.356

1.30

0.957

11.0

cbf, trigonal, long-axis

9fcf

F

1.75

2.36

9898

-0.018

+0.022

0.120

1.75

0.976

7.0

cbf

11if

I

1.36

1.51

27273

-0.005

+0.022

0.286

1.36

0.982

25.7

h5, tetragonal

6ukf

I

0.96

1.00

143260

+0.004

+0.023

0.327

0.96

0.957

9.8

cbf, monoclinic

5ebi

F

0.85

1.09

39168

+0.010

+0.024

0.192

0.85

0.996

10.9

marCCD, monoclinic, twin

36gk

F

2.09

2.28

84276

-0.011

+0.024

0.313

2.09

1.014

11.8

h5, orthorhombic

8u0i

I

1.38

1.54

16737

+0.005

+0.025

0.346

1.38

1.008

17.0

cbf, tetragonal

7t5t

I

1.24

1.35

101432

-0.002

+0.025

0.251

1.24

1.017

16.5

cbf, tetragonal

9c18

I

1.71

1.90

25042

+0.010

+0.025

0.261

1.72

0.957

10.8

h5, triclinic

6h2p_1p89A

F

1.76

1.88

85465

+0.004

+0.026

0.525

1.76

1.043

22.8

cbf, orthorhombic, long-wavelength, two-wavelength

6jgj

F

0.65

0.77

250738

-0.049

+0.026

0.387

0.65

1.127

15.1

cbf, orthorhombic, cdte

6moj

I

2.43

2.43

13971

-0.050

+0.029

-

-

0.977

8.3

cbf, tetragonal

9o0h

I

2.02

2.24

15448

+0.030

+0.029

0.266

2.03

0.987

5.8

cbf, orthorhombic

8qj5

I

1.29

1.63

101763

+0.003

+0.029

0.214

1.29

1.058

8.4

cbf, monoclinic

8s38

F

1.63

1.89

121216

-0.003

+0.030

0.356

1.63

0.999

22.3

cbf, orthorhombic

8sqo

I

1.32

1.55

33951

+0.002

+0.033

0.369

1.32

0.987

14.2

h5, cubic

9z72

I

2.00

2.38

28228

+0.009

+0.033

0.272

2.00

0.994

11.8

cbf, trigonal, long-axis

5src

I

0.97

1.05

138681

+0.015

+0.035

0.247

0.97

0.988

19.6

cbf, tetragonal

6pxc

F

1.41

1.60

15636

+0.004

+0.035

0.300

1.41

1.019

10.2

cbf, orthorhombic

5cc8

F

1.53

2.00

28144

+0.038

+0.036

0.447

1.53

0.950

11.2

smv, orthorhombic, home-source, tncs

6iu9

I

2.74

3.00

9191

+0.011

+0.036

0.152

2.73

0.983

5.4

cbf, trigonal, twin

8pqd

F

1.30

1.50

102729

+0.004

+0.036

0.241

1.30

0.995

15.0

h5, orthorhombic

9s02

F

1.42

1.65

187371

-0.006

+0.036

0.235

1.42

0.969

26.0

h5, orthorhombic

8xtg

I

1.54

2.00

177630

-0.004

+0.037

0.285

1.54

1.001

7.3

cbf, trigonal

5nw5

I

7.07

6.50

9834

-0.050

+0.038

-

-

1.053

7.7

cbf, orthorhombic, low-resolution

9yzk

I

3.87

4.50

13466

+0.003

+0.038

0.017

3.88

0.995

9.3

cbf, monoclinic

9ea5

I

1.64

2.00

51543

+0.008

+0.040

0.315

1.64

0.986

26.7

cbf, monoclinic

6h2p_native

F

1.32

1.48

188243

-0.007

+0.041

0.295

1.32

0.959

20.0

cbf, orthorhombic, two-wavelength

7pq7

F

1.37

1.55

51889

-0.001

+0.041

0.152

1.38

1.007

14.0

cbf, monoclinic

9khr

I

1.38

2.00

11649

+0.008

+0.042

0.277

1.38

0.984

9.5

marCCD, orthorhombic

8xte

I

1.64

1.99

195244

+0.005

+0.043

0.307

1.63

0.999

12.4

cbf, trigonal

8tha

I

1.33

1.68

8018

+0.001

+0.044

0.316

1.33

0.980

27.7

cbf, hexagonal

6pb3

I

1.84

2.05

15267

-0.002

+0.045

0.230

1.84

1.002

25.4

cbf, hexagonal

8owm

F

1.48

1.70

293908

+0.004

+0.046

0.388

1.48

0.981

22.2

cbf, triclinic

7ph1

F

1.08

1.18

120936

-0.007

+0.050

0.379

1.08

0.991

16.0

cbf, orthorhombic

8sa8

I

1.10

1.30

409824

+0.008

+0.051

0.484

1.10

0.977

22.0

h5, monoclinic

9w3y

I

1.19

1.50

61343

+0.008

+0.051

0.454

1.19

0.995

20.2

h5, orthorhombic

7n0i

I

1.68

1.80

117513

+0.018

+0.051

0.154

1.68

1.041

10.9

cbf, orthorhombic, tncs

8rud

I

1.57

2.10

77721

+0.003

+0.052

0.297

1.58

0.927

11.9

cbf, monoclinic

6hwj

I

1.71

1.98

52132

+0.007

+0.052

0.370

1.71

0.967

39.0

cbf, monoclinic

8t7r

I

3.23

3.84

13698

+0.042

+0.054

0.284

3.23

0.998

7.8

cbf, monoclinic

7orr

I

1.62

1.79

16877

+0.002

+0.056

0.357

1.62

1.020

26.5

h5, cubic

8oic

I

2.34

2.51

78747

-0.018

+0.056

0.249

2.34

0.956

20.0

h5, triclinic

8v4o

I

2.10

2.70

59576

+0.012

+0.058

0.262

2.10

0.997

15.5

h5, hexagonal

9q41

I

1.68

1.95

42195

+0.001

+0.059

0.396

1.67

0.972

7.9

h5, orthorhombic

9z44

I

6.73

7.20

1893

+0.013

+0.059

0.089

6.81

1.031

8.5

cbf, monoclinic

7qij

I

3.59

4.10

135378

+0.027

+0.061

0.082

3.60

0.996

9.4

cbf, orthorhombic

9fcg

F

1.38

1.54

37974

+0.002

+0.062

0.454

1.38

1.004

9.7

cbf, tetragonal

7qis

F

1.75

1.83

93162

+0.016

+0.064

0.276

1.75

1.024

17.4

cbf, hexagonal

6jgi

I

0.75

0.85

187106

+0.003

+0.069

0.408

0.75

0.980

8.5

marCCD, orthorhombic

9rp9

I

1.90

2.10

19429

+0.024

+0.073

0.358

1.91

1.033

32.7

h5, monoclinic

7ou1

F

1.40

1.65

176468

+0.008

+0.085

0.392

1.40

0.966

9.0

marccd

7rji

I

1.48

1.71

16700

+0.004

+0.087

0.327

1.48

0.977

8.4

cbf, trigonal

5jvn

I

2.24

2.90

21045

+0.030

+0.092

0.235

2.24

1.031

15.6

cbf, hexagonal

7k1l

F

1.91

2.25

54383

+0.012

+0.093

0.336

1.91

1.016

9.9

cbf, hexagonal

8ys9

I

1.31

1.46

75646

+0.007

+0.096

0.379

1.31

0.982

15.4

h5, orthorhombic

9t6s

I

1.75

2.00

25144

+0.017

+0.102

0.237

1.75

0.948

29.9

h5, orthorhombic, tncs

6p8j

I

1.28

1.47

191664

+0.004

+0.120

0.387

1.28

1.062

5.3

cbf, monoclinic, pseudo-symmetry

6toc

I

1.64

1.85

6080

+0.015

+0.132

0.362

1.64

0.917

24.8

cbf, tetragonal, twin

9i0a

F

1.81

2.22

60317

+0.018

+0.164

0.200

1.80

0.984

14.1

h5, orthorhombic

8xbp

F

1.66

2.00

24156

+0.013

+0.245

0.121

1.65

1.063

18.6

h5, monoclinic

5j23

F

2.17

2.30

56769

+0.194

+0.318

0.387

2.17

1.012

11.6

marCCD, trigonal, twin

No comparison: 9min (the merge does not match the model in any setting); 9rci (the merge does not match the model in any setting); 6z9g (the merge does not match the model in any setting)

Like for like with XDS: the reference-range table

The same merge, binned a second time over XDS’s range (–report-resolution; report-only, nothing was processed differently for it), against XDS’s CORRECT.LP totals. Where rugnux’s own cut is coarser than the reference (‘coverage’ in the last column), the shells past it are not merged at all: the completeness there is coverage of XDS’s range, not a quality loss, and the other rugnux numbers are over the shells it reached. XDS’s totals are over its own merged range, which is the reference range except where the reference d_min was derived (see below).

set

arm

range A

own d_min

compl % (rugnux / XDS)

mult

I/sigma

R_meas

low-res R_meas (to d A)

CC1/2

CC1/2 noise ratio

ISa

reading

aspirin_x10sa_20keV

inhouse

50.000 0.680

0.66

85.2 / 84.0

6.0 / 3.1

43.5

3.0% / 3.3%

2.2% / 3.3% (2.04 / 2.01)

0.9997 / 0.999

0.20

36.4 / 27.4

like for like

aspirin_x10sa_25keV

inhouse

50.000 0.550

0.53

87.6 / 86.6

6.0 / 3.1

35.5

3.1% / 3.4%

2.2% / 3.2% (1.65 / 1.63)

0.9996 / 0.999

0.27

36.4 / 29.2

like for like

citricacid_x10sa_20keV

inhouse

50.000 0.680

0.67

83.2 / 82.0

5.8 / 3.1

45.4

3.6% / 4.1%

3.8% / 5.0% (2.04 / 2.02)

0.9986 / 0.997

0.40

24.0 / 20.4

like for like

cytc_x06da_1

inhouse

50.000 1.875

1.70

100.0 / 99.9

10.5 / 7.7

14.8

8.4% / 8.4%

3.7% / 3.3% (5.59 / 5.58)

0.9996 / 0.999

0.27

17.8 / 24.5

like for like

cytc_x06da_2

inhouse

50.000 1.690

1.57

100.0 / 99.9

10.2 / 7.8

12.9

9.1% / 9.7%

3.5% / 3.2% (5.05 / 5.03)

0.9996 / 0.999

0.27

21.5 / 27.0

like for like

cytc_x10sa

inhouse

50.000 2.039

1.95

99.9 / 99.8

10.6 / 10.7

9.5

18.0% / 23.2%

3.9% / 3.6% (6.08 / 6.05)

0.9990 / 0.999

0.67

26.6 / 31.8

like for like

hepes_x10sa_20keV

inhouse

50.000 0.680

0.66

93.0 / 91.8

10.4 / 5.7

82.7

2.6% / 3.1%

3.5% / 3.7% (2.04 / 2.00)

0.9994 / 0.999

0.40

39.7 / 28.5

like for like

insu_H_x06da_notwin

inhouse

50.000 1.544

1.42

98.7 / 97.4

4.8 / 3.5

15.5

5.9% / 6.8%

4.3% / 4.5% (4.61 / 4.61)

0.9988 / 0.998

0.48

20.6 / 17.7

like for like

insu_H_x06da_twin

inhouse

50.000 1.455

1.38

96.8 / 95.0

4.5 / 3.6

8.0

12.5% / 11.8%

10.3% / 10.6% (4.35 / 4.34)

0.9869 / 0.988

1.05

6.2 / 6.8

like for like

insu_I_x06da_13keV

inhouse

50.000 1.635

1.47

100.0 / 100.0

20.2 / 17.2

9.4

26.8% / 27.9%

8.1% / 9.3% (4.88 / 4.85)

0.9982 / 0.998

0.72

26.2 / 18.8

like for like

insu_I_x06da_5keV

inhouse

50.000 2.450

2.43

96.0 / 91.7

15.4 / 11.2

33.9

6.2% / 7.3%

4.4% / 4.8% (7.28 / 7.21)

0.9995 / 0.999

0.33

28.1 / 17.5

like for like

insu_I_x06da_5keV_2

inhouse

50.000 2.450

2.42

96.3 / 91.8

15.9 / 12.7

27.1

7.9% / 7.6%

5.7% / 4.8% (7.28 / 7.21)

0.9991 / 0.999

0.60

25.3 / 20.0

like for like

insu_I_x06da_6keV

inhouse

50.000 2.040

2.03

95.0 / 92.4

15.5 / 12.4

34.7

5.8% / 6.5%

4.8% / 4.7% (6.08 / 6.03)

0.9995 / 0.999

0.33

20.6 / 17.9

like for like

insu_I_x06da_low_isa

inhouse

50.000 1.300

1.44

74.3 / 90.8

16.6 / 13.3

7.7

22.6% / 22.0%

16.1% / 16.7% (3.89 / 3.92)

0.9949 / 0.998

2.04

5.6 / 4.2

coverage: own cut 1.44 A is coarser than the reference, 1 shell(s) not merged

insu_I_x06da_ref

inhouse

50.000 1.621

1.40

100.0 / 100.0

16.1 / 13.8

17.3

25.8% / 34.9%

6.2% / 14.3% (4.84 / 4.82)

0.9994 / 0.998

0.24

33.4 / 25.1

like for like

insu_I_x06da_weak

inhouse

999.000 1.080

1.64

28.9 / 92.4

40.1 / 29.0

15.8

16.7% / 40.1%

6.2% / 5.4% (3.24 / 3.22)

0.9997 / 0.999

0.20

15.0 / 18.9

coverage: own cut 1.64 A is coarser than the reference, 5 shell(s) not merged

kdp_x10sa_20keV

inhouse

50.000 0.740

0.66

99.7 / 99.2

10.4 / 6.1

48.0

5.0% / 24.1%

4.7% / 17.2% (2.22 / 2.06)

0.9991 / 0.969

0.03

26.3 / 4.1

like for like

lysoI_micromax_mono

inhouse

50.000 1.650

1.36

100.0 / 100.0

6.5 / 6.8

22.2

5.8% / 6.8%

3.1% / 2.8% (4.93 / 4.99)

0.9994 / 0.999

0.40

39.5 / 31.4

like for like

lysoI_micromax_pink

inhouse

50.000 1.650

1.38

100.0 / 100.0

6.5 / 6.8

20.1

6.6% / 7.9%

3.2% / 2.8% (4.93 / 4.99)

0.9993 / 0.999

0.47

36.4 / 29.2

like for like

lyso_micromax_mono

inhouse

50.000 1.500

1.18

100.0 / 100.0

6.0 / 6.2

24.4

4.7% / 4.7%

2.5% / 2.3% (4.48 / 4.54)

0.9996 / 1.000

0.80

40.9 / 39.9

like for like

lyso_micromax_pink

inhouse

50.000 1.450

1.20

99.9 / 99.9

5.8 / 6.0

21.4

4.8% / 5.1%

2.4% / 2.2% (4.34 / 4.39)

0.9996 / 1.000

0.80

41.2 / 37.4

like for like

lyso_x06da_5keV

inhouse

50.000 2.450

2.43

87.7 / 86.4

11.1 / 8.8

34.0

5.7% / 6.4%

5.4% / 4.7% (7.28 / 7.22)

0.9991 / 0.998

0.36

28.2 / 19.4

like for like

lyso_x06da_atten_wedge

inhouse

50.000 1.264

1.19

100.0 / 99.9

12.8 / 11.0

8.8

35.8% / 23.9%

6.9% / 6.9% (3.78 / 3.76)

0.9981 / 0.998

0.76

13.3 / 16.6

like for like

lyso_x06da_half_image

inhouse

50.000 1.650

1.57

100.0 / 99.6

5.5 / 5.3

3.8

110.9% / 59.5%

33.0% / 25.8% (4.93 / 4.91)

0.7920 / 0.958

5.92

7.0 / 6.6

like for like

lyso_x06da_ice

inhouse

50.000 1.431

1.34

100.0 / 100.0

10.5 / 11.9

9.4

16.6% / 21.9%

4.6% / 5.5% (4.28 / 4.26)

0.9986 / 0.998

0.56

23.3 / 23.3

like for like

lyso_x06da_ref

inhouse

50.000 1.200

0.99

100.0 / 100.0

13.8 / 13.8

36.0

4.3% / 4.5%

2.7% / 2.9% (3.59 / 3.58)

0.9998 / 1.000

0.40

30.3 / 28.3

like for like

lyso_x10sa_90deg_1

inhouse

50.000 1.966

1.83

100.0 / 99.3

3.5 / 3.5

6.9

15.2% / 18.2%

4.5% / 4.9% (5.86 / 5.83)

0.9956 / 0.995

0.80

20.7 / 20.8

like for like

lyso_x10sa_90deg_2

inhouse

50.000 1.973

1.85

100.0 / 99.3

3.5 / 3.5

6.8

15.1% / 18.0%

4.6% / 5.1% (5.88 / 5.86)

0.9956 / 0.994

0.68

19.5 / 18.1

like for like

lyso_x10sa_strong

inhouse

999.000 1.180

1.36

66.0 / 65.8

21.5 / 11.5

9.1

23.8% / 11.2%

11.4% / 7.7% (3.54 / 3.52)

0.9966 / 0.998

1.36

5.5 / 8.6

coverage: own cut 1.36 A is coarser than the reference, 2 shell(s) not merged

myob_x06da

inhouse

50.000 1.422

1.23

99.8 / 99.4

3.5 / 3.5

7.5

12.5% / 23.2%

4.2% / 10.5% (4.25 / 4.23)

0.9963 / 0.987

0.27

26.0 / 7.6

like for like

myob_x06da_powder_1

inhouse

50.000 1.495

1.73

64.8 / 74.2

5.3 / 2.9

1.5

76.4% / 49.3%

27.7% / 12.5% (4.47 / 4.44)

0.7653 / 0.966

8.58

1.9 / 5.5

coverage: own cut 1.73 A is coarser than the reference, 2 shell(s) not merged

myob_x06da_powder_2

inhouse

999.000 0.990

1.41

35.0 / 58.0

5.5 / 4.7

0.9

125.6% / 110.6%

45.9% / 70.6% (2.97 / 2.98)

0.9076 / 0.655

0.19

2.5 / 2.2

coverage: own cut 1.41 A is coarser than the reference, 4 shell(s) not merged

myob_x06da_sparse

inhouse

50.000 2.000

1.77

100.0 / 73.7

5.5 / 5.2

3.1

38.6% / 32.6%

17.2% / 14.6% (5.96 / 5.92)

0.9004 / 0.972

3.77

3.2 / 5.5

like for like

myob_x06da_split

inhouse

50.000 1.506

1.96

45.9 / 77.7

6.0 / 4.4

2.0

71.2% / 57.7%

24.4% / 9.1% (4.50 / 4.48)

0.9199 / 0.984

5.19

2.6 / 12.4

coverage: own cut 1.96 A is coarser than the reference, 3 shell(s) not merged

myob_x10sa

inhouse

50.000 1.742

1.57

98.1 / 97.7

3.2 / 3.3

5.7

16.0% / 24.7%

7.6% / 18.6% (5.20 / 5.16)

0.9900 / 0.965

0.27

10.3 / 5.2

like for like

thau_bl1a_3p8keV

inhouse

100.000 3.100

3.02

93.5 / 92.2

9.1 / 7.9

29.7

6.5% / 5.7%

6.3% / 3.9% (9.26 / 9.23)

0.9966 / 0.997

0.97

23.6 / 30.8

like for like

thau_bl1a_4p6keV

inhouse

100.000 2.530

2.47

93.2 / 92.0

9.0 / 7.9

27.2

6.4% / 6.2%

5.2% / 3.9% (7.57 / 7.56)

0.9979 / 0.998

0.84

38.2 / 35.6

like for like

thau_bl1a_6p5keV

inhouse

100.000 1.780

1.73

93.2 / 91.8

8.9 / 8.0

19.6

7.3% / 7.2%

4.6% / 4.1% (5.33 / 5.33)

0.9988 / 0.999

0.80

43.4 / 34.5

like for like

thau_micromax_pink

inhouse

50.000 1.400

1.24

99.0 / 98.5

9.5 / 9.4

15.3

8.7% / 8.0%

4.8% / 4.0% (4.19 / 4.17)

0.9991 / 0.999

0.60

16.0 / 21.2

like for like

thau_x10sa_0p1deg

inhouse

50.000 2.197

1.91

99.5 / 99.0

11.4 / 11.4

12.4

13.6% / 23.0%

6.9% / 10.4% (6.54 / 6.51)

0.9981 / 0.997

0.54

10.8 / 9.2

like for like

thau_x10sa_16keV

inhouse

50.000 1.300

1.22

94.3 / 94.3

18.5 / 18.4

24.9

7.0% / 7.2%

2.8% / 2.7% (3.89 / 3.89)

0.9998 / 1.000

0.40

53.5 / 44.5

like for like

thau_x10sa_injection

inhouse

50.000 1.280

1.26

86.3 / 86.2

10.1 / 10.0

37.3

3.9% / 3.7%

2.6% / 2.2% (3.83 / 3.83)

0.9998 / 1.000

0.40

33.1 / 36.2

like for like

yag_x10sa_20keV

inhouse

50.000 0.680

0.67

99.6 / 99.0

36.5 / 28.4

21.8

39.4% / 43.2%

41.8% / 43.6% (2.04 / 1.99)

0.9298 / 0.947

1.34

3.0 / 3.2

like for like

Data quality is not scored; this table is for a human to judge. CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the half-set noise; the noise ratio is rugnux’s over XDS’s, with XDS’s CC1/2 taken at the bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS’s merge with half its observations. The low-resolution R_meas is the lowest shell of each table (the shells are XDS’s, so both cover the same reflections); reported like everything here.

Small molecules: SHELXL refinement of the published structure

The set’s structure from the Crystallography Open Database (COD id) refined against our p.hkl by SHELXL with one fixed recipe (shelx_check.py: data reindexed into the COD setting, non-H anisotropic, H fixed at the COD positions, EXTI refined, three rounds with the weights SHELXL suggests, MERG 2), the way a chemical crystallographer judges data. WGHT a near its 0.2 ceiling and a large EXTI mean the strong reflections or the sigmas are off; K top is / of SHELXL’s strongest analysis-of-variance bin (below 1: strong reflections read low). Fixed-model R1(F) scores the data against |Fc| of the COD model as published, one scale, nothing refined. R(int) means something only for unmerged data. Reported, never scored.

set

arm

COD

own d_min

R1 >4sig (n)

wR2

GooF

EXTI

WGHT a / b

peak / hole e/A3

R(int) / R(sigma)

K top

fixed-model R1(F)

status

cytidine

open

2001311

0.58

0.0613 (3657)

0.1762

1.061

0.039

0.047 / 0.10

0.27 / -0.24

0.0865 / 0.0883

1.009

0.1084

ok

dnba

open

4510615

0.81

0.0266 (1171)

0.0706

1.072

0.002

0.030 / 1.87

0.21 / -0.21

0.0202 / 0.0225

1.008

0.0795

ok

lalanine

open

2311261

0.65

0.0311 (1070)

0.0939

1.150

0.000

0.055 / 0.07

0.25 / -0.25

0.0204 / 0.0258

1.018

0.0439

ok

metformin

open

2108029

0.51

0.0322 (4453)

0.0960

1.101

0.000

0.041 / 0.40

0.59 / -0.38

0.0284 / 0.0206

1.012

0.0373

ok

nidppe

open

2012031

0.51

0.0414 (9289)

0.1022

1.036

0.004

0.037 / 0.55

0.58 / -0.65

0.0726 / 0.0525

0.993

0.1541

ok

aspirin_x10sa_20keV

inhouse

7050897

0.66

0.0363 (2288)

0.1095

1.100

0.009

0.058 / 0.29

0.42 / -0.39

0.0274 / 0.0171

1.005

0.0819

ok

aspirin_x10sa_25keV

inhouse

7050897

0.53

0.0358 (4379)

0.1165

1.077

0.012

0.069 / 0.11

0.54 / -0.45

0.0288 / 0.0180

1.005

0.0908

ok

citricacid_x10sa_20keV

inhouse

5000063

0.67

0.0374 (2092)

0.1026

1.092

0.149

0.052 / 0.36

0.43 / -0.26

0.0329 / 0.0233

1.009

0.2755

ok

hepes_x10sa_20keV

inhouse

2224210

0.66

0.0310 (3358)

0.0895

1.061

0.053

0.052 / 0.80

0.44 / -0.54

0.0246 / 0.0118

1.015

0.0404

ok

kdp_x10sa_20keV

inhouse

9009999

0.66

0.0475 (305)

0.1208

1.363

0.028

0.000 / 6.44

0.82 / -0.76

0.0494 / 0.0238

0.990

0.2658

ok

lcystine_x10sa_20keV

inhouse

1513328

0.67

0.1575 (1216)

0.3812

1.457

0.000

0.200 / 0.00

2.21 / -1.17

0.3066 / 0.1109

1.396

0.2102

ok

lcystine_x10sa_25keV

inhouse

1513328

0.53

0.1397 (2305)

0.3903

1.465

0.000

0.200 / 0.00

2.20 / -2.01

0.2667 / 0.0919

1.256

0.2363

ok

yag_x10sa_20keV

inhouse

2003066

0.67

0.1021 (247)

0.2472

1.142

0.584

0.181 / 18.31

2.69 / -6.79

0.4973 / 0.1053

0.789

0.1740

ok

XDS merged past its own signal (CC1/2 of its finest shell not significant), so the reference d_min is where XDS’s CC1/2 falls through 0.30, and that is also the d_min of the reference-range table: cytc_x10sa (XDS range 2.04 A, reference 2.27 A); insu_I_x06da_weak (XDS range 1.08 A, reference 1.81 A); lyso_x10sa_strong (XDS range 1.18 A, reference 1.24 A). XDS’s pooled R_meas, CC1/2 and completeness are over its whole range.

Failures

set

arm

cause

reason

3r6o

open

sym_over

I 41 2 2 vs reference I 41

5cc8

open

sym_screw

P 21 21 21 vs reference P 21 21 2

5ebi

open

sym_over

C 2 2 21 vs reference P 1 21 1

9hnc

open

sym_screw

P 1 21 1 vs reference P 1 2 1

9min

open

lattice_halved

primitive volume ratio 0.49

6z9g

open

lattice_halved

primitive volume ratio 0.50

8c3e

open

sym_over

P 6 2 2 vs reference P 31 2 1

4bwl

open

sym_over

C 2 2 21 vs reference P 1 21 1

2wnq

open

sym_over

C 2 2 21 vs reference P 1 21 1

6oww

open

sym_over

P 41 21 2 vs reference P 1 21 1

6p8j

open

sym_over

P 21 21 2 vs reference P 1 21 1

kdp_x10sa_20keV

inhouse

sym_other

I 41 m d vs reference I -4 2 d

Not scored: 7k1l (P 6 vs reference P 63: the screw along c is undeterminable from these data (offered P 61 | P 65 | P 62 | P 64 | P 63)); 7mzt (P 21 21 21 vs reference P 21 21 2: the screw along c is undeterminable from these data (offered P 21 21 2)); cuhf2 (no reference to score against)

Accepted alternatives

5 of the passes above are rows that accept more than one reference: the deposition and our reduction disagree, the disagreement is real, and no test available to us settles it, so either answer passes as long as the program picks one of them. These are open questions, not errors attributed to the deposition; the manifest’s ref keeps the deposited values verbatim in every one of them.

  • 6pxb (open): we report P 31 1 2, the reference is P 32; accepted as P 32 1 2. The deposition merged in point group 3 (its 55502 unique reflections to 1.747 A are what Laue class -3 holds, twice what -3 1 m would) and refined six chains in P 32. The intensities read point group 312 instead: the three added two-folds correlate at 0.98-0.99, at or above the three-folds nobody disputes (0.98), against 0.37-0.40 for the 321 and 622 operators, POINTLESS on our P1 merge picks P -3 1 m (likelihood 1.000), and the reflections centric in 312 but not in 3 are distributed as centric (+435 nats), which a twin law cannot produce. Against that, the deposited chains pair under the added two-fold at 0.3-0.7 A, more than coordinate error at 1.75 A, and the model tells the two indexings apart (R 0.22 against 0.25), so the two-fold may be a near-exact non-crystallographic one; ZANUDA settles on P 32 2 1, whose operators these data do not support. Neither answer is established, so both are accepted; P 31 1 2, which the data cannot separate from P 32 1 2, is accepted as its hand

  • 6toc (open): we report P 42 2 2, the reference is P 42; accepted as P 42 2 2. Our reduction and an independent POINTLESS run on our own P1 merge both read point group 422, and the deposited asymmetric unit’s two chains are related by the very two-fold the higher group adds, to 0.16 A CA RMSD over 43 residues - coordinate error at 1.85 A. Merging in P 42 2 2 costs 0.0006 in R_meas for 1.75x the multiplicity and correlates better with the deposited model (0.9802 vs 0.9764). The refinement test is NOT unanimous: ZANUDA 1.097 refines P 42 2 2 to R-free 0.2561 against P 42’s 0.2636 at half the parameters and reports the deposited assignment incorrect, while an independent Refmac 5.8.0431 comparison on a symmetry-consistent free set puts P 42 ahead by 0.004-0.020 depending on cycle count - less than the spread between refinement protocols. Neither answer is established: a pseudo-symmetry too exact for any test we have remains a live explanation, and so does the deposited assignment. Either is accepted

  • 8xte (open): we report P 31 2 1, the reference is P 32; accepted as P 31 2 1. The evidence favours the higher group here. The twin-immune centric zone of the added two-folds - reflections the higher group makes centric, which are their own twin mates and so cannot be made to read centric by a merohedral twin law - gives <|E^2-1|> = 0.946 +/- 0.012 against a centric expectation of 0.968 and an acentric 0.736, at +707 nats. Re-refinement on a shared free set with the twin law removed from both sides favours P 3_2 2 1 (0.2177/0.2322) over P 3_2 (0.2460/0.2612), and the deposited entry’s published R values reproduce only with an undeclared twin law h,-h-k,-l at alpha = 0.50, which is itself a 321-symmetric target. No refinement R can close the question in principle, because a merohedral twin at exactly alpha = 0.5 and true 321 predict identical intensities; the case rests on the centric zone. Both answers are accepted, ours being the better supported

  • 8xtg (open): we report P 31 2 1, the reference is P 32; accepted as P 31 2 1. The evidence favours the DEPOSITION here, and this row must not be read as the 8xte one. Every correlation-based instrument we have - our own operator correlations, POINTLESS (0.85 on our P1 merge) - reads point group 321, but our own twin-immune centric-zone test, the only one that separates real symmetry from pseudo-symmetry, read <|E^2-1|> = 0.869 at -44.9 nats against the promotion when it assumed an untwinned zone; read at the fraction of the crystal’s other twin laws (rc174) it now favours the promotion, with an L-test twin fraction of 0.20-0.26. Whether this crystal is partially twinned or purely pseudo-symmetric is not established. Both answers are accepted, the deposition being the better supported

  • 9rci (open): we report P 1, the reference is P 1; accepted as cell 35.869 39.297 199.976. The deposited cell is the (0,1/2,1/2)-centred sublattice of the supercell we report, to 0.17%, and both descriptions of this lattice are defensible. The Patterson has an off-origin peak at 62.5% of the origin, so a real translational NCS relates the two halves of our cell: describing the crystal by the doubled cell with the near-translation left in the content, or by its sublattice with the near-translation absorbed into the lattice, is a choice, not a measurement. The alternative cell is the deposited one doubled along c with the centring removed (c’ = b + 2c), computed from the deposited cell alone - not from our output

Per set

Space group: on the open arm the data’s own determination. Where –model put the model’s enantiomorph on the label, the label is in the note and the group scored is the search’s.

R_free and R_work describe one run against its model and do not carry across runs: the model is scaled to the data by an overall factor and an anisotropic B, so whatever the amplitudes’ radial profile does that this shape cannot follow is reported as R, and a change in the profile alone moves R_free further than a real change in the data does. R_model (shell-scaled) is the same R with one free scale per resolution shell removing exactly that, over every reflection rather than the free 5%, and it is the column to read in a delta table. Radial misfit is how big that per-shell rescale had to be (the RMS of its logarithm): when it moves between two runs, R_free between them means little. Both are reported for a human to judge; nothing is scored on them.

Anom. sigma is the anomalous difference map (F+ - F- on the model’s phases) read at the model’s anomalous scatterers - every atom from phosphorus up: the S of Met and Cys, metals, Cl, I - and averaged, in units of the map’s r.m.s.: how much anomalous signal the merge carries, on the model’s yardstick. Reported, never scored.

set

arm

verdict

space group

ref

cell dev %

V ratio

d_min

ref d_min

gain

compl %

mult

R_meas

low-res R_meas

CC1/2

ISa

ref ISa

idx

R_model (shell-scaled)

radial misfit

CC model

R_free (within-run)

R_work

dep R_free

R_free ratio

anom. sigma

REFMAC R_free

REFMAC dep data

REFMAC ratio

dCC dep. outer

model fit

time s

note

11if

open

pass

P 41

P 43

0.12

0.997

1.36

1.51

+10.0%

88.5

11.1

4.8%

2.8%

1.000

25.7

-

-

0.192

0.134

0.966

0.188

0.197

0.210

0.898

2.920

0.229

0.233

0.982

0.022

ACCEPTED

12

P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model

2wnn

open

pass

P 1 21 1

P 1 21 1

0.11

1.003

1.44

1.65

+12.5%

73.7

3.6

7.2%

4.6%

0.997

14.1

-

-

0.279

0.174

0.913

0.286

0.284

0.249

1.148

1.000

0.253

0.313

0.807

-0.351

NOT_TESTED

76

2wnq

open

fail

C 2 2 21

P 1 21 1

68.55

0.997

1.65

1.80

+8.1%

97.1

7.1

10.8%

6.0%

0.997

10.3

-

-

0.536

0.275

0.397

0.545

0.549

0.253

2.153

0.020

0.278

0.261

1.066

-0.626

NOT_TESTED

63

C 2 2 21 vs reference P 1 21 1

2wnz

open

pass

P 1 21 1

P 1 21 1

0.13

1.001

1.85

1.85

-0.2%

99.7

3.7

12.2%

4.0%

0.996

14.8

-

-

0.207

0.220

0.936

0.232

0.237

0.225

1.031

0.940

0.255

0.229

1.113

-0.396

NOT_TESTED

61

2xfw

open

pass

P 1 21 1

P 1 21 1

0.11

1.000

1.55

1.65

+5.8%

99.7

3.7

14.8%

4.4%

0.996

14.1

-

-

0.200

0.157

0.951

0.220

0.218

0.182

1.209

1.110

0.215

0.186

1.158

-0.515

ACCEPTED

93

36gk

open

pass

I 2 2 2

I 2 2 2

0.09

0.999

2.09

2.28

+8.2%

99.7

13.8

15.7%

7.3%

0.998

11.8

-

-

0.203

0.109

0.950

0.203

0.212

0.223

0.913

2.800

0.255

0.252

1.014

0.024

NOT_TESTED

38

3inp

open

pass

F 41 3 2

F 41 3 2

0.04

1.001

1.70

2.05

+16.9%

99.7

18.8

10.8%

5.5%

0.999

13.2

-

-

0.239

0.070

0.946

0.239

0.240

0.177

1.351

2.980

0.240

0.244

0.984

0.003

NOT_TESTED

46

3ky7

open

pass

P 43 3 2

P 43 3 2

0.03

1.001

1.92

2.35

+18.3%

99.7

25.7

10.7%

5.9%

0.999

12.2

-

-

0.339

0.043

0.899

0.335

0.341

0.255

1.312

0.400

0.352

0.355

0.992

0.001

NOT_TESTED

58

3mc4

open

pass

R 3:H

H 3

0.06

0.998

1.79

1.95

+8.1%

79.9

3.3

10.0%

8.4%

0.994

12.5

-

-

0.252

0.061

0.900

0.256

0.254

0.156

1.642

1.400

0.261

0.191

1.365

-0.333

ACCEPTED

10

3meb

open

pass

P 1 21 1

P 1 21 1

0.43

0.992

1.53

1.90

+19.4%

31.7

2.2

13.9%

9.5%

0.989

15.6

-

-

0.201

1.105

0.954

0.203

0.205

0.214

0.950

-0.190

0.229

0.230

0.993

-0.107

NOT_TESTED

6

3p85

open

pass

P 63 2 2

P 63 2 2

0.50

1.007

1.62

1.90

+14.7%

86.8

10.8

12.1%

10.2%

0.998

8.1

-

-

0.194

0.117

0.953

0.201

0.201

0.216

0.930

7.760

0.203

0.232

0.879

-0.404

NOT_TESTED

10

3r6o

open

fail

I 41 2 2

I 41

0.43

0.990

1.53

1.95

+21.4%

43.1

7.2

19.1%

17.2%

0.985

4.5

-

-

0.302

0.725

0.807

0.307

0.304

0.185

1.657

1.580

0.310

0.223

1.393

-0.664

NOT_TESTED

5

I 41 2 2 vs reference I 41

4bwl

open

fail

C 2 2 21

P 1 21 1

72.04

1.004

1.68

2.00

+16.1%

99.4

7.0

14.1%

5.7%

0.998

14.3

-

-

0.520

0.130

0.383

0.524

0.524

0.235

2.230

0.300

0.234

0.227

1.030

-0.520

NOT_TESTED

59

C 2 2 21 vs reference P 1 21 1

5cc8

open

fail

P 21 21 21

P 21 21 2

0.06

1.000

1.53

1.75

+12.6%

48.5

3.9

8.0%

6.7%

0.997

11.2

-

-

0.171

0.165

0.959

0.184

0.178

0.194

0.950

4.010

0.203

0.214

0.950

0.036

ACCEPTED

11

P 21 21 21 vs reference P 21 21 2

5ebi

open

fail

C 2 2 21

P 1 21 1

40.37

1.001

0.85

1.09

+21.6%

99.4

6.7

12.3%

7.2%

0.998

10.9

-

-

0.580

0.135

0.340

0.571

0.584

0.169

3.380

-0.150

0.181

0.181

0.996

0.024

REJECTED

46

C 2 2 21 vs reference P 1 21 1

5epe

open

pass

F 2 3

F 2 3

0.00

1.000

1.77

1.90

+7.0%

99.7

15.1

16.5%

8.4%

0.998

9.9

-

-

0.173

0.109

0.951

0.181

0.182

0.167

1.086

11.950

0.208

0.200

1.039

-0.093

ACCEPTED

36

5f6m

open

pass

P 21 21 21

P 21 21 21

0.08

1.002

1.09

1.10

+1.0%

78.8

4.0

5.1%

3.7%

0.998

23.0

-

-

0.152

0.132

0.972

0.165

0.160

0.154

1.065

5.990

0.163

0.158

1.032

-0.059

NOT_TESTED

8

5j23

open

pass

R 3:H

H 3

0.15

0.998

2.17

2.30

+5.8%

99.7

5.2

15.4%

6.0%

0.996

11.6

-

-

0.226

0.145

0.939

0.234

0.237

0.169

1.386

0.860

0.188

0.185

1.012

0.318

NOT_TESTED

34

5jk4

open

pass

P 1 21 1

P 1 21 1

0.12

1.000

1.02

1.10

+7.5%

84.9

4.0

8.9%

4.1%

0.998

14.8

-

-

0.109

0.092

0.980

0.123

0.121

0.125

0.990

3.690

0.152

0.154

0.992

0.018

NOT_TESTED

25

5jvn

open

pass

P 6 2 2

P 6 2 2

0.03

0.999

2.24

2.90

+22.8%

98.7

16.8

16.3%

5.4%

0.999

15.6

-

-

0.228

0.137

0.884

0.249

0.249

0.235

1.059

1.360

0.250

0.242

1.031

0.092

NOT_TESTED

42

5ky6

open

pass

P 1 21 1

P 1 21 1

0.64

0.994

1.54

1.94

+20.5%

96.8

3.2

25.4%

10.6%

0.986

6.1

-

-

0.233

0.123

0.937

0.243

0.241

0.223

1.091

0.550

0.246

0.229

1.076

-0.142

NOT_TESTED

91

5lzl

open

pass

P 31 2 1

P 31 2 1

0.29

0.992

2.86

3.47

+17.6%

94.9

8.9

17.1%

4.8%

0.998

15.2

-

-

0.222

0.139

0.878

0.229

0.233

0.250

0.915

1.260

0.264

0.257

1.027

-0.026

NOT_TESTED

18

5m17

open

pass

I 4

I 4

0.08

0.998

0.98

1.03

+4.6%

92.6

6.0

5.5%

5.1%

0.996

16.2

-

-

0.132

0.220

0.961

0.158

0.159

0.130

1.217

7.090

0.175

0.171

1.024

-0.034

NOT_TESTED

97

5mln

open

pass

P 21 21 2

P 21 2 21

0.10

0.999

1.26

1.60

+21.3%

99.7

8.1

10.4%

3.5%

0.999

21.5

-

-

0.181

0.144

0.949

0.188

0.197

0.180

1.046

2.170

0.194

0.190

1.017

-0.007

ACCEPTED

22

5nw5

open

pass

P 21 21 21

P 21 21 21

0.19

0.997

7.07

6.50

-8.8%

99.7

6.3

33.3%

16.7%

0.927

7.7

-

-

0.395

0.254

0.806

0.448

0.419

0.277

1.618

0.090

0.342

0.325

1.053

0.038

NOT_TESTED

46

5ojv

open

pass

P 21 21 2

P 21 21 2

0.41

0.993

1.82

2.06

+11.7%

98.5

6.5

14.5%

4.6%

0.998

13.2

-

-

0.173

0.158

0.960

0.187

0.192

0.194

0.964

1.990

0.213

0.214

0.998

-0.005

NOT_TESTED

237

5reo

open

pass

C 1 2 1

C 1 2 1

0.35

1.009

1.65

1.88

+12.1%

99.4

3.1

22.2%

8.1%

0.990

23.7

-

-

0.202

0.068

0.960

0.199

0.206

0.227

0.879

0.750

0.231

0.233

0.991

-0.036

NOT_TESTED

9

5src

open

pass

P 41

P 43

0.09

0.999

0.97

1.05

+7.7%

99.0

6.2

5.7%

3.1%

0.999

19.6

-

-

0.161

0.190

0.962

0.183

0.180

0.175

1.042

6.050

0.208

0.211

0.988

0.035

ACCEPTED

38

P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model

5t39

open

pass

P 1 21 1

P 1 21 1

0.13

1.003

1.01

1.10

+8.5%

90.9

4.2

7.3%

4.4%

0.998

16.2

-

-

0.151

0.088

0.956

0.156

0.153

0.154

1.014

6.470

0.183

0.182

1.003

-0.053

NOT_TESTED

71

5uth

open

pass

P 31 2 1

P 31 2 1

0.21

0.995

1.72

1.95

+11.7%

82.8

8.4

12.4%

7.5%

0.997

9.5

-

-

0.194

0.165

0.952

0.217

0.210

0.202

1.074

2.800

0.222

0.220

1.009

-0.012

ACCEPTED

10

5vml

open

pass

P 42 21 2

P 42 21 2

0.02

1.000

1.92

1.70

-12.6%

81.0

7.4

10.2%

9.8%

0.996

10.9

-

-

0.156

0.050

0.964

0.165

0.158

0.171

0.966

3.400

0.180

0.171

1.053

-0.038

NOT_TESTED

6

6cdl

open

pass

P 21 21 2

P 21 21 2

1.22

1.023

1.13

1.25

+9.5%

71.7

6.4

7.8%

6.4%

0.997

11.6

-

-

0.147

0.233

0.949

0.159

0.159

0.181

0.881

4.600

0.195

0.187

1.039

-0.051

NOT_TESTED

49

6cee

open

pass

P 21 21 21

P 21 21 21

0.04

1.001

1.38

1.55

+10.9%

87.8

6.1

5.4%

3.6%

0.999

23.9

-

-

0.166

0.137

0.952

0.174

0.182

0.169

1.028

9.050

0.181

0.182

0.993

0.001

NOT_TESTED

7

6cs9

open

pass

P 1 21 1

P 1 21 1

0.06

0.999

1.72

1.85

+7.1%

91.3

3.8

9.9%

4.3%

0.998

13.1

-

-

0.211

0.143

0.902

0.205

0.226

0.227

0.903

0.770

0.257

0.252

1.018

0.010

NOT_TESTED

29

6f3p

open

pass

C 1 2 1

C 1 2 1

0.12

0.997

1.13

1.35

+16.1%

99.3

3.6

10.0%

5.4%

0.997

9.4

-

-

0.143

0.214

0.968

0.169

0.169

0.127

1.331

-3.420

0.173

0.174

0.999

0.015

NOT_TESTED

128

6fid

open

pass

P 21 21 21

P 21 21 21

0.41

1.008

1.98

2.20

+10.1%

72.9

12.4

9.4%

10.4%

0.998

13.6

-

-

0.193

0.054

0.943

0.196

0.198

0.220

0.892

11.260

0.237

0.242

0.977

-0.018

NOT_TESTED

28

6fvz

open

pass

C 2 2 2

C 2 2 2

0.43

0.989

1.49

1.80

+17.2%

99.5

6.7

24.2%

5.1%

0.996

20.5

-

-

0.198

0.094

0.954

0.202

0.206

0.199

1.017

1.330

0.207

0.207

0.998

-0.018

NOT_TESTED

38

6fwc

open

pass

C 2 2 2

C 2 2 2

0.04

0.999

1.41

1.70

+16.9%

94.7

4.0

16.4%

4.3%

0.995

27.3

-

-

0.188

0.098

0.959

0.196

0.198

0.189

1.037

1.440

0.197

0.201

0.980

0.018

NOT_TESTED

26

6g1f

open

pass

C 1 2 1

C 1 2 1

0.04

1.000

1.93

2.25

+14.1%

99.6

3.8

11.8%

3.9%

0.997

21.0

-

-

0.208

0.120

0.954

0.222

0.220

0.211

1.054

1.040

0.231

0.226

1.018

0.017

NOT_TESTED

116

6gvk

open

pass

C 1 2 1

C 1 2 1

0.09

0.999

1.42

1.55

+8.5%

99.6

6.7

5.2%

3.3%

0.999

19.8

-

-

0.200

0.179

0.950

0.215

0.212

0.210

1.021

1.880

0.225

0.227

0.989

0.007

NOT_TESTED

94

6h2p_1p89A

open

pass

C 2 2 21

C 2 2 21

0.03

1.000

1.76

-

-

88.1

10.2

8.5%

5.2%

0.999

22.8

-

-

0.143

0.076

0.971

0.149

0.147

0.170

0.873

9.200

0.170

0.163

1.043

0.026

NOT_TESTED

217

6h2p_native

open

pass

C 2 2 21

C 2 2 21

0.04

0.999

1.32

1.48

+10.8%

99.5

6.6

13.7%

3.9%

0.999

20.0

-

-

0.161

0.067

0.973

0.165

0.165

0.170

0.968

3.330

0.175

0.182

0.959

0.041

NOT_TESTED

113

6h5t

open

pass

I 4 2 2

I 4 2 2

0.40

0.991

1.48

1.69

+12.4%

99.4

7.4

14.6%

8.3%

0.996

9.3

-

-

0.196

0.192

0.941

0.222

0.218

0.191

1.164

9.000

0.227

0.214

1.057

-0.015

NOT_TESTED

35

6hv2

open

pass

P 61 2 2

P 61 2 2

0.04

1.001

1.41

1.71

+17.6%

99.8

29.2

19.8%

5.5%

1.000

13.7

-

-

0.219

0.279

0.947

0.239

0.242

0.263

0.910

7.830

0.271

0.326

0.829

-0.008

NOT_TESTED

32

6hwj

open

pass

P 1 21 1

P 1 21 1

0.10

1.001

1.71

1.98

+13.4%

97.5

3.5

8.6%

2.4%

0.999

39.0

-

-

0.187

0.129

0.960

0.192

0.198

0.214

0.894

1.600

0.215

0.223

0.967

0.052

NOT_TESTED

14

6i3j

open

pass

F 2 2 2

F 2 2 2

0.08

0.999

2.32

2.59

+10.5%

98.7

7.2

24.6%

16.3%

0.977

6.8

-

-

0.203

0.172

0.914

0.237

0.234

0.226

1.049

1.730

0.219

0.190

1.154

0.002

NOT_TESTED

75

6iu5

open

pass

P 31

P 31

0.02

1.000

2.12

2.25

+6.0%

99.1

4.8

18.2%

11.4%

0.993

8.0

-

-

0.212

0.079

0.932

0.216

0.222

0.250

0.863

7.630

0.261

0.258

1.008

-0.018

ACCEPTED

88

6iu6

open

pass

P 31

P 31

0.33

0.996

2.36

2.90

+18.6%

97.4

4.2

11.5%

7.9%

0.996

7.2

-

-

0.221

0.165

0.950

0.233

0.238

0.211

1.102

3.540

0.223

0.209

1.063

-0.016

ACCEPTED

91

6iu8

open

pass

P 31

P 31

0.02

1.000

2.32

2.70

+14.1%

99.7

10.2

14.1%

7.5%

0.998

8.1

-

-

0.270

0.173

0.910

0.282

0.282

0.215

1.315

1.230

0.211

0.215

0.984

-0.024

NOT_TESTED

13

6iu9

open

pass

P 31

P 31

0.22

1.005

2.74

3.00

+8.8%

99.7

4.7

14.4%

9.8%

0.993

5.4

-

-

0.310

0.096

0.811

0.327

0.312

0.265

1.234

1.430

0.250

0.254

0.983

0.036

ACCEPTED

86

6jgh

open

pass

P 21 21 21

P 21 21 21

0.39

1.008

0.87

0.94

+7.1%

99.3

7.2

23.4%

10.9%

0.990

6.6

-

-

0.136

0.170

0.964

0.156

0.157

0.129

1.206

2.070

0.174

0.163

1.069

-0.070

NOT_TESTED

104

6jgi

open

pass

P 21 21 21

P 21 21 21

0.20

0.997

0.75

0.85

+12.0%

98.5

7.0

14.3%

8.5%

0.995

8.5

-

-

0.112

0.144

0.975

0.128

0.127

0.112

1.145

6.270

0.145

0.148

0.980

0.069

NOT_TESTED

115

6jgj

open

pass

P 21 21 21

P 21 21 21

0.28

0.992

0.65

0.77

+15.7%

85.0

7.2

7.4%

4.6%

0.999

15.1

-

-

0.164

0.057

0.962

0.167

0.166

0.125

1.333

2.600

0.188

0.167

1.127

0.026

NOT_TESTED

49

6moj

open

pass

I 41 2 2

I 41 2 2

0.08

0.998

2.43

2.43

+0.1%

99.7

26.7

52.4%

8.9%

0.998

8.3

-

-

0.248

0.210

0.917

0.257

0.262

0.250

1.028

1.050

0.276

0.282

0.977

0.029

NOT_TESTED

125

6nen

open

pass

P 3 1 2

P 3 1 2

0.07

0.998

1.77

2.15

+17.7%

99.6

21.6

28.5%

10.2%

0.997

6.3

-

-

0.211

0.054

0.936

0.218

0.215

0.208

1.050

1.000

0.212

0.210

1.008

0.007

ACCEPTED

19

6o2h

open

pass

P 1

P 1

0.74

0.982

1.10

1.21

+9.5%

30.3

1.1

7.0%

8.3%

0.977

23.9

-

-

0.096

0.132

0.934

0.129

0.127

0.117

1.103

0.110

0.179

0.155

1.157

-0.003

ACCEPTED

13

6oel

open

pass

F 41 3 2

F 41 3 2

0.00

1.000

2.85

3.10

+8.1%

99.7

41.4

36.5%

8.2%

0.998

9.9

-

-

0.248

0.136

0.815

0.255

0.259

0.256

0.998

0.700

0.278

0.288

0.965

-0.024

NOT_TESTED

40

6oww

open

fail

P 41 21 2

P 1 21 1

0.19

0.998

2.72

3.84

+29.3%

99.7

42.0

203.3%

14.0%

0.994

11.8

-

-

0.363

0.076

0.857

0.376

0.368

0.340

1.105

2.030

0.321

0.342

0.938

-0.049

ACCEPTED

222

P 41 21 2 vs reference P 1 21 1

6p8j

open

fail

P 21 21 2

P 1 21 1

0.33

0.992

1.28

1.47

+13.1%

99.6

6.6

23.7%

12.0%

0.990

5.3

-

-

0.241

0.099

0.907

0.251

0.247

0.229

1.094

1.180

0.257

0.242

1.062

0.120

NOT_TESTED

143

P 21 21 2 vs reference P 1 21 1

6p8p

open

pass

P 4

P 4

0.14

1.003

1.46

1.64

+10.9%

99.7

6.7

13.2%

5.1%

0.998

15.5

-

-

0.189

0.143

0.950

0.204

0.203

0.198

1.030

1.890

0.210

0.203

1.036

-0.017

NOT_TESTED

20

6pb3

open

pass

P 6

P 6

0.17

0.995

1.84

2.05

+10.1%

93.6

10.3

6.5%

3.0%

1.000

25.4

-

-

0.214

0.108

0.956

0.229

0.219

0.252

0.908

10.600

0.270

0.269

1.002

0.045

ACCEPTED

32

6pxb

open

pass

P 31 1 2

P 32

0.23

0.994

1.39

1.75

+20.3%

99.7

10.0

8.8%

4.6%

0.999

12.1

-

-

0.240

0.258

0.959

0.279

0.261

0.263

1.059

0.620

0.291

0.294

0.991

-0.005

NOT_TESTED

16

P 31 1 2 vs reference P 32: accepted alternative P 32 1 2 - a knife-edge this battery does not decide

6pxc

open

pass

I 2 2 2

I 2 2 2

0.23

0.996

1.41

1.60

+11.9%

96.0

5.7

8.1%

5.6%

0.998

10.2

-

-

0.210

0.164

0.960

0.214

0.217

0.210

1.022

0.690

0.226

0.222

1.019

0.035

NOT_TESTED

32

6qaj

open

pass

C 2 2 21

C 2 2 21

0.41

0.993

2.70

2.90

+6.8%

99.7

9.2

24.4%

5.1%

0.997

13.2

-

-

0.321

0.230

0.858

0.332

0.331

0.291

1.139

4.460

0.311

0.327

0.951

-0.069

NOT_TESTED

65

6r72

open

pass

P 1 21 1

P 1 21 1

1.32

0.979

4.39

3.95

-11.2%

99.7

7.1

16.5%

3.5%

1.000

17.9

-

-

0.362

0.045

0.586

0.367

0.364

0.321

1.143

0.130

0.394

0.380

1.038

-0.232

NOT_TESTED

26

6rlr

open

pass

P 1

P 1

0.03

1.001

1.92

2.00

+3.9%

97.6

3.5

12.9%

4.1%

0.998

16.4

-

-

0.250

0.067

0.896

0.254

0.254

0.279

0.909

0.680

0.287

0.281

1.022

-0.014

NOT_TESTED

18

6rym

open

pass

P 41

P 43

0.06

0.998

1.45

1.46

+0.5%

79.7

3.9

4.2%

4.1%

0.998

22.9

-

-

0.170

0.246

0.940

0.190

0.183

0.184

1.033

14.400

0.198

0.194

1.022

-0.009

ACCEPTED

16

P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model

6s1u

open

pass

P 1 21 1

P 1 21 1

0.13

1.002

1.75

1.90

+7.8%

95.0

3.8

22.3%

5.6%

0.989

13.8

-

-

0.202

0.089

0.951

0.200

0.210

0.235

0.849

0.600

0.244

0.251

0.975

-0.031

NOT_TESTED

13

6toc

open

pass

P 42 2 2

P 42

0.30

0.991

1.64

1.85

+11.6%

99.7

24.3

9.4%

2.3%

1.000

24.8

-

-

0.276

0.136

0.973

0.270

0.281

0.265

1.018

0.540

0.245

0.267

0.917

0.132

ACCEPTED

11

P 42 2 2 vs reference P 42: accepted alternative P 42 2 2 - a knife-edge this battery does not decide

6ttn

open

pass

P 21 21 21

P 21 21 21

0.41

1.012

1.08

1.12

+3.2%

98.5

11.9

10.7%

4.5%

0.999

14.5

-

-

0.132

0.155

0.975

0.154

0.149

0.146

1.051

8.200

0.175

0.172

1.020

-0.018

NOT_TESTED

45

6u7g

open

pass

P 1 21 1

P 1 21 1

0.13

1.003

1.88

2.35

+20.1%

78.4

3.3

8.0%

4.1%

0.998

13.3

-

-

0.198

0.110

0.933

0.200

0.204

0.218

0.916

1.570

0.232

0.237

0.980

-0.097

NOT_TESTED

71

6ukf

open

pass

P 1 21 1

P 1 21 1

0.08

1.001

0.96

1.00

+4.3%

91.5

6.8

9.5%

5.8%

0.998

9.8

-

-

0.165

0.074

0.954

0.161

0.167

0.166

0.970

2.260

0.183

0.191

0.957

0.023

NOT_TESTED

48

6v2r

open

pass

P 41 21 2

P 41 21 2

0.02

0.999

1.38

1.60

+13.6%

92.6

11.3

5.2%

2.9%

1.000

24.4

-

-

0.205

0.207

0.945

0.209

0.230

0.235

0.889

12.960

0.262

0.263

0.996

0.009

NOT_TESTED

8

6vww

open

pass

P 63

P 63

0.20

1.005

1.99

2.20

+9.7%

99.6

5.6

16.4%

8.9%

0.993

8.0

-

-

0.240

0.118

0.925

0.243

0.249

0.178

1.370

0.670

0.256

0.249

1.025

0.004

NOT_TESTED

15

6w4h

open

pass

P 31 2 1

P 31 2 1

0.02

1.000

1.62

1.80

+9.8%

97.8

6.9

7.6%

3.9%

0.999

18.5

-

-

0.163

0.160

0.967

0.169

0.176

0.163

1.036

4.710

0.195

0.196

0.996

-0.012

ACCEPTED

71

6w75

open

pass

P 31 2 1

P 32 2 1

0.01

1.000

1.69

1.95

+13.5%

99.7

10.5

12.0%

4.7%

0.999

15.8

-

-

0.175

0.170

0.957

0.188

0.187

0.174

1.075

3.590

0.192

0.194

0.991

-0.016

ACCEPTED

94

P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model

6wzo

open

pass

P 1

P 1

0.04

1.000

1.04

1.42

+26.8%

67.0

3.8

6.2%

3.7%

0.999

16.8

-

-

0.174

0.120

0.959

0.175

0.175

0.173

1.013

2.680

0.178

0.184

0.965

0.004

NOT_TESTED

54

6yqf

open

pass

P 21 21 2

P 21 21 2

0.71

1.017

3.02

3.33

+9.3%

99.7

5.7

70.0%

11.0%

0.982

4.2

-

-

0.434

0.156

0.742

0.442

0.448

0.367

1.204

-0.200

0.420

0.420

1.001

-0.233

NOT_TESTED

23

6z8o

open

pass

P 1 21 1

P 1 21 1

0.63

1.016

2.21

2.20

-0.4%

90.7

3.6

17.4%

5.8%

0.995

13.9

-

-

0.276

0.069

0.905

0.281

0.280

0.275

1.023

1.240

0.298

0.286

1.043

-0.045

NOT_TESTED

64

6z9g

open

fail

P 1 21 1

P 1 21 1

22.53

0.501

1.59

1.76

+9.4%

81.5

4.3

11.4%

4.9%

0.997

14.8

-

-

0.530

0.184

0.142

0.542

0.536

0.240

2.260

0.050

-

-

-

-

NOT_TESTED

94

primitive volume ratio 0.50

6ze4

open

pass

P 21 21 21

P 21 21 21

0.59

1.009

1.30

1.60

+18.9%

96.2

8.0

20.2%

6.7%

0.993

9.3

-

-

0.213

0.079

0.966

0.218

0.217

0.202

1.077

1.790

0.193

0.185

1.040

-0.020

NOT_TESTED

75

6zqr

open

pass

P 4

P 4

0.40

1.010

1.76

1.93

+8.8%

99.7

8.2

19.3%

8.1%

0.996

9.1

-

-

0.197

0.168

0.949

0.204

0.213

0.191

1.066

1.350

0.214

0.201

1.064

-0.098

NOT_TESTED

17

6zqy

open

pass

P 4

P 4

0.12

0.997

1.70

1.85

+8.2%

98.8

7.9

23.3%

5.6%

0.994

12.7

-

-

0.213

0.158

0.948

0.222

0.227

0.196

1.131

1.360

0.220

0.205

1.071

-0.144

ACCEPTED

32

6zr0

open

pass

P 4

P 4

0.08

1.001

1.66

1.94

+14.7%

98.0

3.7

13.1%

4.1%

0.993

24.5

-

-

0.216

0.068

0.960

0.218

0.218

0.210

1.039

1.800

0.222

0.219

1.014

0.005

ACCEPTED

57

7arr

open

pass

P 1

P 1

0.28

0.993

0.92

1.10

+16.7%

70.3

3.5

3.7%

3.0%

0.999

18.9

-

-

0.154

0.197

0.964

0.166

0.165

0.161

1.033

0.000

0.203

0.200

1.016

-0.103

ACCEPTED

35

7atg

open

pass

P 21 21 21

P 21 21 21

0.08

1.002

0.60

0.60

+0.5%

86.8

3.8

5.2%

7.1%

0.995

22.9

-

-

0.127

0.201

0.442

0.176

0.175

0.095

1.857

6.470

-

-

-

-0.026

NOT_TESTED

28

7bgt

open

pass

P 1

P 1

0.32

0.991

1.78

1.93

+7.6%

97.9

2.2

13.7%

5.6%

0.993

17.4

-

-

0.195

0.118

0.955

0.207

0.206

0.212

0.977

0.230

0.225

0.226

0.999

-0.000

NOT_TESTED

12

7bgu

open

pass

P 1

P 1

0.17

0.996

2.30

2.43

+5.6%

94.7

2.9

12.4%

4.5%

0.937

10.0

-

-

0.286

0.108

0.801

0.304

0.304

0.236

1.288

0.260

0.268

0.235

1.140

-0.246

NOT_TESTED

41

7brr

open

pass

P 1 21 1

P 1 21 1

0.09

0.998

1.24

1.35

+7.9%

96.8

6.3

6.6%

3.8%

0.999

16.6

-

-

0.192

0.095

0.963

0.191

0.196

0.197

0.966

3.070

0.205

0.209

0.981

-0.041

NOT_TESTED

17

7dkp

open

pass

P 1 21 1

P 1 21 1

0.06

1.002

1.18

1.45

+18.3%

66.3

7.3

11.9%

4.6%

0.998

26.1

-

-

0.158

0.080

0.969

0.161

0.163

0.160

1.001

2.830

0.170

0.172

0.984

0.020

NOT_TESTED

29

7k1l

open

unscored

P 6

P 63

0.27

0.995

1.91

2.25

+15.1%

98.9

6.5

16.1%

8.3%

0.994

9.9

-

-

0.261

0.098

0.909

0.268

0.268

0.192

1.401

0.780

0.271

0.266

1.016

0.093

NOT_TESTED

20

P 6 vs reference P 63: the screw along c is undeterminable from these data (offered P 61 / P 65 / P 62 / P 64 / P 63)

7kcn

open

pass

P 41 2 2

P 41 2 2

0.07

1.000

1.39

1.46

+4.7%

92.8

20.7

7.8%

7.4%

0.999

11.7

-

-

0.172

0.154

0.961

0.189

0.190

0.182

1.041

8.440

0.209

0.198

1.055

-0.020

NOT_TESTED

33

7l6j

open

pass

I 41 3 2

I 41 3 2

0.00

1.000

1.51

1.78

+15.3%

99.7

29.0

20.9%

7.5%

0.999

10.5

-

-

0.172

0.179

0.963

0.187

0.188

0.154

1.211

29.130

0.181

0.178

1.013

-0.001

NOT_TESTED

107

7l84

open

pass

P 41 21 2

P 43 21 2

0.06

0.999

1.70

1.70

+0.0%

91.6

33.5

9.1%

13.0%

0.999

11.4

-

-

0.150

0.090

0.911

0.169

0.163

0.162

1.040

17.580

0.179

0.183

0.976

-0.008

ACCEPTED

17

P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model

7mzt

open

unscored

P 21 21 21

P 21 21 2

0.56

0.998

3.12

4.07

+23.4%

99.7

10.6

366.1%

33.8%

0.971

4.1

-

-

0.414

0.104

0.556

0.426

0.418

0.372

1.144

-0.040

0.394

0.398

0.991

-0.020

ACCEPTED

33

P 21 21 21 vs reference P 21 21 2: the screw along c is undeterminable from these data (offered P 21 21 2)

7n0i

open

pass

P 21 21 21

P 21 21 21

0.22

0.996

1.68

2.20

+23.8%

99.7

6.6

14.5%

5.7%

0.998

10.9

-

-

0.267

0.259

0.916

0.295

0.294

0.271

1.085

0.530

0.296

0.284

1.041

0.051

NOT_TESTED

27

7n2s

open

pass

P 1 21 1

P 1 21 1

0.07

1.000

2.56

2.37

-8.1%

97.6

3.4

56.7%

8.7%

0.917

9.6

-

-

0.299

0.111

0.822

0.312

0.308

0.311

1.002

0.340

0.315

0.325

0.968

-0.018

NOT_TESTED

12

7orr

open

pass

I 2 3

I 21 3

0.03

0.999

1.62

1.79

+9.3%

99.6

15.7

6.7%

4.0%

1.000

26.5

-

-

0.183

0.164

0.965

0.196

0.194

0.184

1.062

5.080

0.184

0.181

1.020

0.056

NOT_TESTED

20

I 2 3 vs reference I 21 3 (UNDECIDABLE from intensities)

7os3

open

pass

P 21 21 21

P 21 21 21

0.10

1.002

1.96

2.18

+10.1%

83.6

11.4

8.4%

3.8%

0.999

24.4

-

-

0.181

0.153

0.955

0.194

0.193

0.224

0.866

6.150

0.233

0.236

0.985

-0.054

NOT_TESTED

33

7ou1

open

pass

P 1 21 1

P 1 21 1

0.04

0.999

1.40

1.65

+15.0%

87.9

3.4

15.9%

7.7%

0.991

9.0

-

-

0.202

0.098

0.955

0.209

0.210

0.222

0.944

1.520

0.229

0.237

0.966

0.085

NOT_TESTED

22

7ph1

open

pass

I 2 2 2

I 2 2 2

0.12

0.998

1.08

1.18

+8.1%

99.7

7.1

11.3%

5.4%

0.998

16.0

-

-

0.155

0.158

0.968

0.174

0.172

0.166

1.051

3.210

0.207

0.209

0.991

0.050

NOT_TESTED

77

7pq7

open

pass

C 1 2 1

C 1 2 1

0.22

0.993

1.37

1.55

+11.4%

98.1

3.7

6.5%

4.3%

0.998

14.0

-

-

0.188

0.193

0.946

0.215

0.209

0.200

1.074

2.320

0.230

0.229

1.007

0.041

NOT_TESTED

14

7q6j

open

pass

P 21 21 21

P 21 21 21

0.25

0.998

1.98

2.20

+9.9%

99.7

7.3

14.6%

5.4%

0.996

11.0

-

-

0.224

0.137

0.945

0.236

0.240

0.236

1.000

1.620

0.258

0.260

0.996

-0.019

NOT_TESTED

126

7qij

open

pass

P 21 21 21

P 21 21 21

0.36

0.993

3.59

4.10

+12.4%

99.7

6.8

24.1%

5.6%

0.996

9.4

-

-

0.357

0.065

0.831

0.364

0.362

0.325

1.121

0.180

0.363

0.364

0.996

0.061

NOT_TESTED

92

7qis

open

pass

P 61

P 61

0.05

0.999

1.75

1.83

+4.3%

99.6

9.1

20.9%

6.2%

0.997

17.4

-

-

0.166

0.195

0.965

0.192

0.191

0.189

1.019

1.410

0.214

0.209

1.024

0.064

NOT_TESTED

31

7raa

open

pass

P 41 21 2

P 43 21 2

0.04

0.999

2.58

2.69

+4.1%

98.7

35.1

17.2%

5.5%

1.000

11.0

-

-

0.296

0.217

0.902

0.335

0.304

0.294

1.140

0.530

0.305

0.308

0.990

-0.036

ACCEPTED

182

P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model

7ris

open

pass

P 31 2 1

P 32 2 1

0.08

1.000

1.51

1.72

+12.0%

96.3

17.9

10.0%

3.3%

1.000

31.0

-

-

0.188

0.047

0.961

0.189

0.190

0.201

0.942

2.260

0.206

0.209

0.987

-0.004

ACCEPTED

24

P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model

7rji

open

pass

R 3 2:H

H 3 2

0.26

1.008

1.48

1.71

+13.3%

99.3

28.9

13.1%

8.2%

0.999

8.4

-

-

0.206

0.119

0.947

0.198

0.212

0.224

0.883

6.960

0.227

0.232

0.977

0.087

NOT_TESTED

27

7t5t

open

pass

P 42 21 2

P 42 21 2

0.04

0.999

1.24

1.35

+8.2%

98.4

12.6

6.5%

4.4%

0.999

16.5

-

-

0.163

0.246

0.960

0.192

0.188

0.168

1.145

11.230

0.193

0.189

1.017

0.025

NOT_TESTED

60

7tcd

open

pass

C 1 2 1

C 1 2 1

0.20

1.004

1.65

1.70

+2.8%

69.3

7.1

8.6%

4.3%

0.999

18.4

-

-

0.191

0.195

0.956

0.211

0.208

0.252

0.835

1.170

0.248

0.257

0.968

-0.057

NOT_TESTED

28

7yzx

open

pass

P 63 2 2

P 63 2 2

0.25

0.994

1.88

1.90

+1.3%

99.7

14.1

20.5%

6.2%

0.998

10.4

-

-

0.212

0.106

0.951

0.215

0.215

0.229

0.938

1.380

0.236

0.230

1.025

0.013

NOT_TESTED

49

8a1a

open

pass

P 61

P 65

0.42

0.987

1.93

2.05

+5.9%

99.7

41.8

52.0%

8.1%

0.997

18.8

-

-

0.177

0.029

0.962

0.179

0.178

0.185

0.968

1.820

0.178

0.179

0.991

-0.036

ACCEPTED

94

P 61 vs reference P 65 (hand only (needs anomalous)); labelled P 65 from the model

8agq

open

pass

C 1 2 1

C 1 2 1

0.30

0.991

0.97

1.09

+11.6%

95.6

6.0

9.2%

4.3%

0.999

14.1

-

-

0.181

0.212

0.945

0.204

0.203

0.149

1.373

6.460

0.201

0.174

1.153

-0.022

NOT_TESTED

26

8c3e

open

fail

P 6 2 2

P 31 2 1

0.24

0.994

1.79

2.10

+14.5%

98.0

7.0

29.4%

11.9%

0.980

5.8

-

-

0.359

0.290

0.758

0.400

0.388

0.249

1.607

0.960

0.381

0.370

1.031

-0.076

ACCEPTED

5

P 6 2 2 vs reference P 31 2 1

8dqb

open

pass

I 2 3

I 2 3

0.08

0.998

2.05

2.50

+18.1%

97.5

10.3

14.3%

4.4%

0.996

22.1

-

-

0.230

0.112

0.939

0.237

0.243

0.239

0.990

3.080

0.254

0.252

1.005

0.006

NOT_TESTED

14

8dyz

open

pass

P 41 21 2

P 43 21 2

0.08

0.999

1.14

1.27

+10.4%

64.2

3.8

3.1%

2.4%

0.999

37.8

-

-

0.120

0.046

0.973

0.121

0.122

0.133

0.907

4.220

0.171

0.178

0.958

-0.050

ACCEPTED

12

P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model

8dz7

open

pass

P 21 21 21

P 21 21 21

0.08

0.998

1.19

1.34

+11.2%

37.2

3.2

3.0%

2.7%

0.999

35.1

-

-

0.118

0.068

0.968

0.128

0.122

0.136

0.942

5.140

0.172

0.170

1.009

-0.008

NOT_TESTED

10

8egn

open

pass

P 21 21 21

P 21 21 21

0.07

1.001

1.63

1.95

+16.6%

91.2

5.4

6.3%

3.3%

0.999

22.9

-

-

0.197

0.154

0.953

0.215

0.205

0.219

0.979

2.530

0.230

0.231

0.997

0.015

NOT_TESTED

14

8iya

open

pass

C 1 2 1

C 1 2 1

0.16

0.996

1.91

2.43

+21.3%

97.9

5.5

21.6%

11.5%

0.989

5.8

-

-

0.247

0.033

0.946

0.234

0.249

0.247

0.948

0.400

0.254

0.254

0.998

0.016

NOT_TESTED

8

8k1g

open

pass

I 4 2 2

I 4 2 2

0.82

1.019

1.63

2.09

+22.1%

99.7

20.6

25.6%

6.8%

0.999

11.7

-

-

0.206

0.200

0.956

0.227

0.226

0.207

1.093

2.260

0.209

0.215

0.974

-0.107

NOT_TESTED

43

8oic

open

pass

P 1

P 1

0.02

1.000

2.34

2.80

+16.4%

98.2

3.6

24.4%

4.6%

0.988

20.0

-

-

0.238

0.112

0.933

0.248

0.246

0.251

0.988

0.410

0.249

0.260

0.956

0.056

ACCEPTED

42

8owm

open

pass

P 1

P 1

0.02

1.000

1.48

1.70

+13.2%

96.3

3.6

10.2%

3.4%

0.998

22.2

-

-

0.164

0.127

0.971

0.177

0.176

0.172

1.031

1.370

0.183

0.186

0.981

0.046

NOT_TESTED

67

8pqd

open

pass

P 21 21 21

P 21 21 21

0.03

1.000

1.30

1.50

+13.0%

94.5

13.5

9.0%

5.5%

0.999

15.0

-

-

0.184

0.216

0.961

0.204

0.200

0.194

1.051

3.570

0.209

0.210

0.995

0.036

NOT_TESTED

63

8qaw

open

pass

R 3:H

H 3

0.04

0.999

1.30

1.55

+16.3%

99.4

9.9

12.2%

7.6%

0.998

11.0

-

-

0.143

0.163

0.958

0.153

0.157

0.161

0.950

11.400

0.175

0.183

0.957

0.022

NOT_TESTED

401

8qj5

open

pass

P 1 21 1

P 1 21 1

0.45

0.987

1.29

1.63

+20.9%

98.1

6.1

17.7%

7.7%

0.996

8.4

-

-

0.192

0.121

0.941

0.206

0.203

0.190

1.083

1.500

0.204

0.193

1.058

0.029

NOT_TESTED

20

8qq7

open

pass

P 62 2 2

P 64 2 2

0.65

1.017

3.16

3.62

+12.7%

99.4

24.9

19.0%

10.5%

0.997

6.4

-

-

0.432

0.370

0.330

0.429

0.436

0.320

1.340

0.750

0.351

0.330

1.065

-0.107

ACCEPTED

13

P 62 2 2 vs reference P 64 2 2 (hand only (needs anomalous)); labelled P 64 2 2 from the model

8r5r

open

pass

P 21 21 21

P 21 21 21

0.04

1.001

2.80

3.08

+9.0%

99.7

10.1

22.7%

4.7%

0.998

19.3

-

-

0.263

0.144

0.884

0.284

0.271

0.252

1.127

0.430

0.253

0.252

1.003

-0.024

NOT_TESTED

30

8rud

open

pass

P 1 21 1

P 1 21 1

0.40

0.989

1.57

2.10

+25.1%

99.7

6.6

37.8%

6.3%

0.991

11.9

-

-

0.253

0.081

0.924

0.261

0.258

0.262

0.997

0.690

0.242

0.262

0.927

0.052

NOT_TESTED

236

8s38

open

pass

I 2 2 2

I 21 21 21

0.16

0.996

1.63

1.89

+13.8%

99.5

6.7

8.6%

3.5%

0.999

22.3

-

-

0.180

0.133

0.962

0.185

0.188

0.187

0.990

2.280

0.204

0.204

0.999

0.030

NOT_TESTED

125

I 2 2 2 vs reference I 21 21 21 (UNDECIDABLE from intensities)

8sa8

open

pass

I 1 2 1

I 1 2 1

0.02

1.000

1.10

1.30

+15.0%

85.7

7.2

11.7%

3.7%

0.999

22.0

-

-

0.155

0.144

0.972

0.169

0.169

0.152

1.109

3.790

0.181

0.185

0.977

0.051

ACCEPTED

94

8sqo

open

pass

P 4 3 2

P 4 3 2

0.13

0.996

1.32

1.55

+14.7%

99.7

70.4

23.4%

6.1%

1.000

14.2

-

-

0.183

0.180

0.957

0.206

0.199

0.177

1.159

7.020

0.190

0.193

0.987

0.033

NOT_TESTED

72

8sqq

open

pass

F 4 3 2

F 4 3 2

0.17

0.995

1.90

2.25

+15.5%

99.7

39.2

25.1%

5.4%

0.999

17.6

-

-

0.211

0.105

0.963

0.206

0.217

0.245

0.839

3.100

0.242

0.258

0.937

0.018

NOT_TESTED

63

8sqt

open

pass

F 4 3 2

F 4 3 2

0.11

0.997

1.88

2.20

+14.7%

99.7

21.6

20.1%

3.5%

0.999

29.9

-

-

0.223

0.109

0.967

0.226

0.232

0.262

0.860

2.080

0.269

0.257

1.044

0.004

NOT_TESTED

28

8t7r

open

pass

C 1 2 1

C 1 2 1

0.28

0.991

3.23

3.84

+15.8%

99.1

4.0

31.2%

8.3%

0.984

7.8

-

-

0.283

0.078

0.755

0.285

0.290

0.263

1.086

0.430

0.282

0.283

0.998

0.054

NOT_TESTED

63

8tha

open

pass

P 62

P 64

0.17

0.995

1.33

1.68

+21.1%

99.1

19.0

16.3%

3.2%

0.999

27.7

-

-

0.212

0.091

0.961

0.220

0.217

0.215

1.026

4.370

0.215

0.220

0.980

0.044

ACCEPTED

19

P 62 vs reference P 64 (hand only (needs anomalous)); labelled P 64 from the model

8tyy

open

pass

F 4 3 2

F 4 3 2

0.06

1.002

1.28

1.68

+23.6%

98.0

38.5

16.6%

5.2%

0.999

16.3

-

-

0.160

0.151

0.970

0.180

0.175

0.163

1.104

13.520

0.161

0.162

0.989

0.002

NOT_TESTED

184

8u0i

open

pass

P 41 21 2

P 43 21 2

0.06

1.001

1.38

1.54

+10.6%

98.1

11.0

8.1%

3.4%

0.999

17.0

-

-

0.179

0.054

0.964

0.183

0.182

0.210

0.871

3.370

0.212

0.210

1.008

0.025

ACCEPTED

18

P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model

8v2t

open

pass

P 42 21 2

P 42 21 2

0.24

1.005

1.17

1.40

+16.8%

90.9

9.7

8.8%

5.1%

0.999

11.9

-

-

0.166

0.213

0.969

0.181

0.180

0.163

1.110

10.860

0.161

0.165

0.975

0.014

NOT_TESTED

31

8v4j

open

pass

P 42 21 2

P 42 21 2

0.04

1.000

1.10

1.31

+16.0%

89.5

11.4

5.4%

3.1%

1.000

22.1

-

-

0.170

0.112

0.965

0.177

0.176

0.172

1.026

12.350

0.169

0.172

0.983

0.008

NOT_TESTED

110

8v4o

open

pass

P 61 2 2

P 61 2 2

0.06

1.001

2.10

2.70

+22.2%

99.7

20.4

31.9%

6.0%

0.998

15.5

-

-

0.236

0.157

0.929

0.253

0.254

0.240

1.056

0.960

0.247

0.247

0.997

0.058

NOT_TESTED

63

8xbp

open

pass

C 1 2 1

C 1 2 1

0.15

0.999

1.66

1.99

+16.7%

81.0

6.6

11.2%

4.1%

0.999

18.6

-

-

0.313

0.177

0.892

0.329

0.320

0.288

1.141

1.210

0.329

0.310

1.063

0.245

NOT_TESTED

21

8xte

open

pass

P 31 2 1

P 32

0.06

1.001

1.64

1.99

+17.8%

99.5

9.4

25.2%

7.7%

0.996

12.4

-

-

0.275

0.121

0.913

0.287

0.284

0.219

1.307

0.910

0.290

0.290

0.999

0.043

NOT_TESTED

34

P 31 2 1 vs reference P 32: accepted alternative P 31 2 1 - a knife-edge this battery does not decide

8xtf

open

pass

R 3 2:H

H 3 2

0.22

1.006

1.84

2.13

+13.7%

99.6

18.9

73.3%

16.7%

0.989

7.6

-

-

0.191

0.044

0.964

0.186

0.193

0.210

0.884

1.400

0.208

0.218

0.956

-0.003

NOT_TESTED

31

8xtg

open

pass

P 31 2 1

P 32

0.05

1.002

1.54

2.00

+22.8%

99.7

9.9

27.3%

10.7%

0.994

7.3

-

-

0.275

0.134

0.903

0.284

0.284

0.211

1.345

0.880

0.280

0.280

1.001

0.037

ACCEPTED

50

P 31 2 1 vs reference P 32: accepted alternative P 31 2 1 - a knife-edge this battery does not decide

8y74

open

pass

C 1 2 1

C 1 2 1

0.34

0.992

1.68

1.90

+11.4%

97.6

6.2

13.0%

7.8%

0.997

8.6

-

-

0.214

0.104

0.939

0.225

0.220

0.240

0.935

1.230

0.257

0.267

0.962

-0.032

NOT_TESTED

11

8ys9

open

pass

P 21 21 21

P 21 21 21

0.16

0.998

1.31

1.46

+10.5%

99.7

13.4

13.3%

4.6%

0.999

15.4

-

-

0.176

0.104

0.962

0.188

0.182

0.197

0.955

3.110

0.209

0.213

0.982

0.096

NOT_TESTED

29

9b22

open

pass

P 1 21 1

P 1 21 1

0.07

1.002

1.14

1.30

+12.1%

77.1

6.1

6.2%

3.5%

0.999

16.4

-

-

0.159

0.121

0.968

0.166

0.164

0.164

1.014

0.160

0.196

0.194

1.007

-0.044

NOT_TESTED

17

9bn8

open

pass

P 41

P 41

0.09

0.997

1.21

1.35

+10.6%

95.4

11.6

8.1%

3.8%

1.000

19.8

-

-

0.151

0.104

0.967

0.159

0.157

0.158

1.011

3.770

0.177

0.182

0.975

0.019

ACCEPTED

27

9c18

open

pass

P 1

P 1

0.61

0.985

1.71

1.90

+9.8%

97.4

3.6

27.3%

8.5%

0.986

10.8

-

-

0.221

0.067

0.947

0.221

0.224

0.229

0.965

0.710

0.224

0.234

0.957

0.025

ACCEPTED

11

9chw

open

pass

P 61

P 61

0.04

1.001

1.60

2.16

+25.9%

72.5

3.1

5.8%

3.3%

0.998

21.3

-

-

0.180

0.140

0.962

0.184

0.186

0.210

0.879

1.730

0.216

0.214

1.006

-0.008

NOT_TESTED

79

9crw

open

pass

P 1 21 1

P 1 21 1

0.23

0.994

2.28

2.49

+8.4%

97.2

7.0

8.7%

3.7%

0.999

15.8

-

-

0.247

0.132

0.955

0.262

0.260

0.278

0.943

0.690

0.296

0.293

1.010

-0.014

NOT_TESTED

16

9e2t

open

pass

P 1

P 1

0.06

1.001

2.29

2.28

-0.4%

91.6

5.7

29.1%

7.3%

0.992

6.8

-

-

0.230

0.047

0.937

0.232

0.233

0.242

0.958

0.420

0.254

0.259

0.980

-0.019

NOT_TESTED

67

9ea5

open

pass

P 1 21 1

P 1 21 1

0.85

0.998

1.64

2.00

+17.9%

90.9

6.7

17.9%

4.2%

0.997

26.7

-

-

0.196

0.111

0.954

0.202

0.204

0.207

0.976

1.200

0.212

0.215

0.986

0.040

ACCEPTED

94

9fcf

open

pass

P 4

P 4

0.04

1.001

1.75

2.36

+25.8%

99.6

13.1

34.5%

10.0%

0.994

7.0

-

-

0.292

0.064

0.923

0.295

0.295

0.257

1.147

0.850

0.269

0.276

0.976

0.022

ACCEPTED

146

9fcg

open

pass

P 4

P 4

0.07

0.999

1.38

1.54

+10.2%

89.9

11.3

14.1%

8.3%

0.997

9.7

-

-

0.180

0.111

0.956

0.189

0.191

0.198

0.952

2.420

0.202

0.201

1.004

0.062

ACCEPTED

48

9fhc

open

pass

I 2 3

I 2 3

0.25

0.993

1.92

2.20

+12.9%

95.2

18.4

17.0%

4.7%

0.998

12.3

-

-

0.239

0.072

0.917

0.244

0.244

0.238

1.024

1.200

0.250

0.240

1.042

-0.024

ACCEPTED

107

9gdj

open

pass

P 41 21 2

P 41 21 2

0.16

0.996

1.40

1.47

+5.1%

99.7

13.1

11.2%

6.1%

0.999

12.9

-

-

0.154

0.199

0.968

0.172

0.176

0.175

0.983

2.890

0.199

0.195

1.021

-0.023

NOT_TESTED

289

9gjx

open

pass

P 1 21 1

P 1 21 1

0.13

0.997

2.06

2.40

+14.1%

96.3

6.7

16.3%

3.1%

0.997

40.1

-

-

0.193

0.146

0.952

0.224

0.216

0.219

1.020

0.840

0.234

0.227

1.033

-0.019

NOT_TESTED

28

9gqg

open

pass

P 31 2 1

P 32 2 1

0.01

1.000

1.81

2.00

+9.6%

99.6

7.8

11.4%

6.0%

0.998

12.8

-

-

0.232

0.116

0.912

0.243

0.244

0.261

0.932

1.790

0.276

0.267

1.036

0.014

ACCEPTED

57

P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model

9h0q

open

pass

R 3 2:H

H 3 2

0.41

0.990

2.10

2.55

+17.6%

99.7

10.6

17.4%

4.5%

0.998

18.6

-

-

0.205

0.111

0.945

0.214

0.219

0.222

0.965

1.100

0.231

0.236

0.980

-0.016

NOT_TESTED

44

9hnc

open

fail

P 1 21 1

P 1 2 1

0.09

0.999

1.63

1.88

+13.0%

90.9

7.0

12.8%

5.3%

0.998

13.6

-

-

0.246

0.149

0.948

0.255

0.254

0.211

1.208

0.880

0.227

0.246

0.923

-0.237

NOT_TESTED

66

P 1 21 1 vs reference P 1 2 1

9hs7

open

pass

P 61

P 65

0.08

0.999

1.70

1.70

+0.2%

99.7

10.2

11.7%

5.0%

0.999

13.3

-

-

0.213

0.206

0.965

0.236

0.229

0.239

0.988

1.260

0.244

0.255

0.958

-0.083

ACCEPTED

26

P 61 vs reference P 65 (hand only (needs anomalous)); labelled P 65 from the model

9i0a

open

pass

P 21 21 2

P 21 21 2

0.43

1.008

1.81

2.22

+18.6%

98.9

12.7

16.9%

5.1%

0.999

14.1

-

-

0.224

0.116

0.955

0.233

0.235

0.237

0.984

0.950

0.255

0.259

0.984

0.164

NOT_TESTED

59

9i80

open

pass

P 41

P 41

0.07

1.002

1.59

1.95

+18.2%

99.7

13.0

25.1%

10.7%

0.992

6.8

-

-

0.219

0.117

0.905

0.221

0.225

0.203

1.092

1.890

0.212

0.207

1.024

0.002

NOT_TESTED

116

9ig7

open

pass

P 21 21 2

P 21 21 2

0.17

0.996

2.02

2.60

+22.3%

99.7

13.7

21.7%

7.3%

0.991

11.1

-

-

0.234

0.081

0.931

0.249

0.239

0.259

0.963

0.800

0.286

0.277

1.030

0.015

NOT_TESTED

82

9ih9

open

pass

C 1 2 1

C 1 2 1

0.21

0.997

1.40

1.70

+17.6%

96.0

2.4

11.2%

5.0%

0.996

12.7

-

-

0.199

0.075

0.952

0.207

0.204

0.203

1.020

1.020

0.204

0.202

1.007

-0.021

NOT_TESTED

20

9jq9

open

pass

P 21 21 21

P 21 21 21

0.12

0.997

1.65

1.90

+13.2%

84.9

10.2

6.9%

4.3%

0.999

19.5

-

-

0.212

0.151

0.936

0.242

0.223

0.237

1.022

4.820

0.241

0.239

1.005

-0.011

NOT_TESTED

6

9jzo

open

pass

P 1

P 1

0.22

0.996

1.15

1.40

+18.0%

61.1

3.3

5.8%

4.5%

0.997

8.1

-

-

0.179

0.047

0.960

0.176

0.180

0.195

0.906

4.390

0.203

0.194

1.045

0.018

NOT_TESTED

11

9khr

open

pass

P 21 21 21

P 21 21 21

0.12

0.997

1.38

2.00

+30.9%

96.9

6.5

18.8%

11.3%

0.994

9.5

-

-

0.244

0.116

0.942

0.259

0.254

0.235

1.104

2.020

0.245

0.249

0.984

0.042

NOT_TESTED

13

9lxl

open

pass

P 41 21 2

P 41 21 2

0.49

1.013

2.06

2.19

+6.1%

99.7

14.7

41.9%

10.2%

0.996

6.2

-

-

0.306

0.095

0.896

0.321

0.304

0.234

1.371

0.690

0.302

0.286

1.058

-0.270

NOT_TESTED

60

9mh4

open

pass

P 21 3

P 21 3

0.31

0.991

2.78

3.05

+8.8%

99.7

40.5

20.2%

4.9%

1.000

13.8

-

-

0.229

0.150

0.922

0.232

0.237

0.215

1.079

0.980

0.242

0.238

1.017

0.005

NOT_TESTED

24

9min

open

fail

P 21 21 2

P 21 21 21

36.97

0.494

1.86

2.05

+9.2%

99.6

23.4

26.5%

6.8%

0.999

10.5

-

-

0.566

0.207

0.221

0.575

0.577

0.267

2.150

-0.000

-

-

-

-

NOT_TESTED

73

primitive volume ratio 0.49

9o0h

open

pass

P 21 21 21

P 21 21 21

0.25

0.994

2.02

2.24

+9.7%

99.6

12.7

65.1%

12.8%

0.985

5.8

-

-

0.242

0.079

0.941

0.249

0.247

0.249

0.999

0.110

0.263

0.267

0.987

0.029

NOT_TESTED

59

9p7q

open

pass

C 1 2 1

C 1 2 1

0.14

1.003

1.76

2.21

+20.3%

88.0

2.6

23.8%

8.8%

0.985

10.8

-

-

0.269

0.036

0.950

0.282

0.270

0.259

1.090

0.870

0.267

0.262

1.017

-0.037

NOT_TESTED

12

9pbb

open

pass

C 1 2 1

C 1 2 1

0.15

0.997

1.78

2.17

+17.8%

73.7

2.3

19.1%

6.0%

0.993

17.0

-

-

0.233

0.035

0.953

0.231

0.234

0.223

1.037

0.670

0.233

0.238

0.980

-0.087

NOT_TESTED

16

9q41

open

pass

C 2 2 21

C 2 2 21

0.20

1.004

1.68

1.95

+14.1%

99.7

6.9

29.0%

12.9%

0.984

7.9

-

-

0.188

0.099

0.959

0.194

0.195

0.213

0.908

0.480

0.208

0.214

0.972

0.059

NOT_TESTED

23

9q66

open

pass

P 1 21 1

P 1 21 1

0.34

0.990

2.03

2.01

-0.9%

99.7

7.1

32.1%

8.2%

0.990

13.1

-

-

0.204

0.084

0.903

0.216

0.216

0.266

0.813

0.660

0.280

0.275

1.017

-0.021

NOT_TESTED

30

9qvv

open

pass

I 2 2 2

I 2 2 2

0.42

0.989

2.49

2.72

+8.5%

87.8

13.8

11.2%

2.3%

1.000

37.3

-

-

0.235

0.204

0.867

0.239

0.246

0.279

0.854

1.640

0.259

0.273

0.947

-0.030

NOT_TESTED

110

9qw2

open

pass

P 1 21 1

P 1 21 1

0.77

1.012

1.76

1.92

+8.5%

99.6

3.4

18.5%

6.6%

0.958

7.8

-

-

0.240

0.040

0.930

0.245

0.242

0.212

1.155

0.540

0.260

0.220

1.185

-0.360

NOT_TESTED

98

9qw8

open

pass

P 1

P 1

0.46

1.000

1.71

1.80

+5.2%

97.0

2.9

20.0%

7.6%

0.989

9.2

-

-

0.294

0.148

0.890

0.312

0.302

0.243

1.286

0.330

0.317

0.249

1.272

-0.326

NOT_TESTED

66

9rci

open

pass

P 1

P 1

97.58

1.990

1.76

1.66

-5.8%

85.2

2.6

29.6%

12.3%

0.951

7.0

-

-

0.561

0.084

0.284

0.555

0.564

0.283

1.959

-0.020

-

-

-

-

NOT_TESTED

22

P 1 vs reference P 1: accepted alternative cell 35.869 39.297 199.976 - a knife-edge this battery does not decide

9rcs

open

pass

P 1 21 1

P 1 21 1

0.93

1.022

3.27

3.01

-8.5%

99.7

7.0

23.4%

9.9%

0.994

7.3

-

-

0.318

0.124

0.502

0.341

0.325

0.317

1.078

0.380

0.336

0.363

0.928

-0.171

ACCEPTED

51

9rp9

open

pass

C 1 2 1

C 1 2 1

0.16

0.996

1.90

2.10

+9.3%

99.6

6.1

17.4%

4.8%

0.997

32.7

-

-

0.204

0.138

0.931

0.233

0.225

0.222

1.051

2.470

0.235

0.228

1.033

0.073

NOT_TESTED

25

9s02

open

pass

P 21 21 2

P 21 21 2

0.06

0.999

1.42

1.65

+13.8%

98.7

12.4

10.2%

3.3%

0.999

26.0

-

-

0.180

0.109

0.961

0.189

0.189

0.193

0.977

2.250

0.195

0.202

0.969

0.036

ACCEPTED

127

9sl0

open

pass

P 21 21 21

P 21 21 21

0.37

0.990

1.36

1.60

+14.9%

99.3

12.3

8.0%

3.7%

1.000

20.2

-

-

0.245

0.156

0.912

0.255

0.256

0.264

0.967

2.200

0.269

0.270

0.996

0.001

NOT_TESTED

45

9t6s

open

pass

P 21 21 21

P 21 21 21

0.06

1.000

1.75

2.00

+12.6%

95.7

5.1

11.2%

3.6%

0.999

29.9

-

-

0.215

0.044

0.970

0.232

0.216

0.241

0.960

2.510

0.239

0.252

0.948

0.102

NOT_TESTED

12

9upt

open

pass

P 6

P 6

0.20

0.994

2.03

2.37

+14.4%

99.7

5.7

24.4%

10.8%

0.989

6.5

-

-

0.218

0.088

0.942

0.218

0.225

0.215

1.014

1.310

0.228

0.216

1.056

-0.046

NOT_TESTED

77

9vyb

open

pass

P 21 21 21

P 21 21 21

0.40

0.989

1.67

2.12

+21.3%

86.9

10.0

8.2%

3.6%

0.999

22.1

-

-

0.246

0.082

0.934

0.253

0.249

0.269

0.938

0.420

0.262

0.264

0.995

0.012

NOT_TESTED

34

9w3y

open

pass

P 21 21 21

P 21 21 21

0.25

1.004

1.19

1.50

+20.4%

99.7

6.8

24.7%

5.2%

0.997

20.2

-

-

0.187

0.088

0.973

0.198

0.197

0.198

0.998

4.930

0.214

0.215

0.995

0.051

NOT_TESTED

11

9yl4

open

pass

P 21 21 21

P 21 21 21

0.08

1.001

3.61

3.70

+2.5%

99.7

13.4

31.8%

9.4%

0.996

9.5

-

-

0.293

0.066

0.942

0.303

0.297

0.283

1.069

5.750

0.284

0.288

0.987

-0.023

NOT_TESTED

90

9yzk

open

pass

I 1 2 1

I 1 2 1

0.20

0.996

3.87

5.10

+24.1%

98.5

3.3

30.9%

4.9%

0.997

9.3

-

-

0.355

0.294

0.867

0.406

0.412

0.304

1.335

0.180

0.312

0.314

0.995

0.038

ACCEPTED

12

9z44

open

pass

I 1 2 1

I 1 2 1

1.35

0.964

6.73

7.20

+6.5%

98.2

3.3

22.3%

8.9%

0.981

8.5

-

-

0.334

0.163

0.782

0.324

0.343

0.345

0.939

-0.030

0.361

0.350

1.031

0.059

ACCEPTED

27

9z72

open

pass

P 31 2 1

P 31 2 1

0.15

1.004

2.00

2.38

+16.1%

99.7

9.9

80.3%

12.6%

0.991

11.8

-

-

0.240

0.048

0.912

0.255

0.243

0.269

0.948

0.570

0.275

0.277

0.994

0.033

NOT_TESTED

44

9zlo

open

pass

P 21 21 21

P 21 21 21

0.39

0.995

1.56

2.00

+21.9%

99.7

12.7

12.3%

3.9%

0.999

24.1

-

-

0.224

0.065

0.942

0.231

0.228

0.219

1.053

1.750

0.225

0.222

1.011

-0.024

NOT_TESTED

25

9zm0

open

pass

P 1 21 1

P 1 21 1

0.16

0.997

1.80

2.10

+14.5%

99.7

6.5

25.0%

9.9%

0.996

9.0

-

-

0.255

0.098

0.955

0.275

0.259

0.298

0.923

4.590

0.297

0.302

0.984

-0.020

NOT_TESTED

9

9zmu

open

pass

P 61 2 2

P 65 2 2

0.25

1.007

1.72

1.98

+13.4%

99.7

28.0

30.1%

7.3%

0.999

11.6

-

-

0.278

0.153

0.893

0.284

0.289

0.294

0.966

0.890

0.303

0.304

0.997

0.009

ACCEPTED

31

P 61 2 2 vs reference P 65 2 2 (hand only (needs anomalous)); labelled P 65 2 2 from the model

cuhf2

open

unscored

P 4 2 2

-

-

-

0.51

-

-

82.1

13.9

2.8%

2.1%

1.000

45.5

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

32

no reference to score against

cytidine

open

pass

P 21 21 21

P 21 21 21

0.31

0.995

0.58

-

-

91.1

3.6

10.3%

9.7%

0.993

8.7

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

24

dnba

open

pass

C 1 2/c 1

C 1 2/c 1

0.06

0.999

0.81

0.48

-68.3%

75.8

2.8

2.5%

1.8%

1.000

36.8

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

6

lalanine

open

pass

P 21 21 21

P 21 21 21

0.07

0.998

0.65

-

-

77.3

2.6

2.6%

1.7%

1.000

42.4

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

9

metformin

open

pass

P 1 21/c 1

P 1 21/c 1

0.17

1.001

0.51

0.45

-12.9%

73.5

4.8

3.1%

2.8%

1.000

30.8

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

5

nidppe

open

pass

P 1 21/c 1

P 1 21/c 1

0.28

0.997

0.51

0.77

+34.3%

66.6

5.3

7.9%

4.0%

0.999

32.9

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

5

aspirin_x10sa_20keV

inhouse

pass

P 1 21/c 1

P 1 21/c 1

0.05

1.001

0.66

0.68

+2.6%

79.5

5.9

3.0%

2.2%

1.000

36.4

27.4

-

-

-

-

-

-

-

-

-

-

-

-

-

-

9

aspirin_x10sa_25keV

inhouse

pass

P 1 21/c 1

P 1 21/c 1

0.05

1.001

0.53

0.55

+3.6%

79.1

6.0

3.1%

2.2%

1.000

36.4

29.2

-

-

-

-

-

-

-

-

-

-

-

-

-

-

11

citricacid_x10sa_20keV

inhouse

pass

P 1 21/c 1

P 1 21/c 1

0.10

1.001

0.67

0.68

+1.8%

79.2

5.8

3.6%

3.8%

0.999

24.1

20.4

-

-

-

-

-

-

-

-

-

-

-

-

-

-

11

cytc_x06da_1

inhouse

pass

P 31 2 1

P 31 2 1

0.04

0.999

1.70

1.88

+9.2%

98.7

18.0

9.3%

3.9%

1.000

17.5

24.5

-

0.264

0.273

0.927

0.335

0.293

0.296

1.132

5.380

-

-

-

-

ACCEPTED

14

labelled P 32 2 1 from the model

cytc_x06da_2

inhouse

pass

P 31 2 1

P 31 2 1

0.11

1.002

1.57

1.69

+7.3%

99.5

17.4

9.8%

3.5%

1.000

21.2

27.0

-

0.253

0.221

0.924

0.306

0.279

0.286

1.069

6.320

-

-

-

-

ACCEPTED

15

labelled P 32 2 1 from the model

cytc_x10sa

inhouse

pass

P 31 2 1

P 31 2 1

0.13

1.003

1.95

2.27

+13.9%

99.6

20.1

20.0%

4.0%

0.999

26.2

31.8

-

0.262

0.263

0.938

0.357

0.287

0.340

1.050

2.690

-

-

-

-

ACCEPTED

23

labelled P 32 2 1 from the model

hepes_x10sa_20keV

inhouse

pass

P b c a

P b c a

0.07

0.999

0.66

0.68

+2.6%

86.8

10.3

2.6%

3.5%

1.000

39.6

28.5

-

-

-

-

-

-

-

-

-

-

-

-

-

-

12

insu_H_x06da_notwin

inhouse

pass

R 3:H

R 3

0.06

0.999

1.42

1.54

+8.4%

91.3

8.6

6.2%

4.2%

0.999

20.4

17.7

-

0.229

0.110

0.935

0.215

0.233

0.229

0.940

6.050

-

-

-

-

ACCEPTED

9

insu_H_x06da_twin

inhouse

pass

R 3:H

R 3

0.04

1.001

1.38

1.46

+5.2%

90.6

8.5

12.1%

10.3%

0.995

6.2

6.8

-

0.264

0.088

0.885

0.253

0.267

0.229

1.108

4.400

-

-

-

-

NOT_TESTED

7

insu_I_x06da_13keV

inhouse

pass

I 2 3

I 2 3

0.27

1.008

1.47

1.64

+9.9%

99.7

38.9

32.1%

8.4%

0.999

-

18.8

-

0.234

0.029

0.940

0.259

0.234

0.161

1.606

1.330

-

-

-

-

ACCEPTED

15

insu_I_x06da_5keV

inhouse

pass

I 2 3

I 2 3

0.03

0.999

2.43

2.45

+0.9%

95.4

28.1

6.6%

4.4%

1.000

28.1

17.5

-

0.221

0.105

0.923

0.245

0.228

0.161

1.523

7.950

-

-

-

-

ACCEPTED

6

insu_I_x06da_5keV_2

inhouse

pass

I 2 3

I 2 3

0.05

0.999

2.42

2.45

+1.1%

93.9

29.5

8.4%

5.8%

0.999

25.3

20.0

-

0.219

0.263

0.899

0.294

0.274

0.161

1.828

9.120

-

-

-

-

NOT_TESTED

5

insu_I_x06da_6keV

inhouse

pass

I 2 3

I 2 3

0.04

0.999

2.03

2.04

+0.7%

95.6

28.4

6.1%

4.8%

1.000

20.6

17.9

-

0.213

0.037

0.933

0.236

0.213

0.161

1.464

11.500

-

-

-

-

ACCEPTED

6

insu_I_x06da_low_isa

inhouse

pass

I 2 3

I 2 3

0.08

1.002

1.44

1.30

-10.4%

99.9

31.9

22.4%

16.1%

0.998

5.6

4.2

-

0.226

0.095

0.933

0.261

0.229

0.161

1.619

3.110

-

-

-

-

NOT_TESTED

11

insu_I_x06da_ref

inhouse

pass

I 2 3

I 2 3

0.13

1.004

1.40

1.62

+13.4%

99.7

24.8

30.6%

6.2%

0.999

32.2

25.1

-

0.226

0.077

0.937

0.245

0.228

0.161

1.522

3.330

-

-

-

-

NOT_TESTED

10

insu_I_x06da_weak

inhouse

pass

I 2 3

I 2 3

0.79

1.024

1.64

1.81

+9.2%

99.5

40.1

16.7%

4.9%

1.000

15.0

18.9

-

0.286

0.147

0.919

0.315

0.289

0.161

1.958

1.920

-

-

-

-

NOT_TESTED

16

kdp_x10sa_20keV

inhouse

fail

I 41 m d

I -4 2 d

0.07

1.002

0.66

0.74

+10.3%

90.9

16.7

5.2%

4.6%

0.999

25.1

4.1

-

-

-

-

-

-

-

-

-

-

-

-

-

-

93

I 41 m d vs reference I -4 2 d

lcystine_x10sa_20keV

inhouse

pass

P 61 2 2

P 61 2 2

0.49

0.991

0.67

-

-

92.9

19.5

30.6%

18.4%

1.000

5.7

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

57

lcystine_x10sa_25keV

inhouse

pass

P 61 2 2

P 61 2 2

0.52

0.990

0.53

-

-

92.1

20.9

26.7%

13.4%

1.000

5.5

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

23

lysoI_micromax_mono

inhouse

pass

P 41 21 2

P 43 21 2

0.06

1.001

1.36

1.65

+17.6%

98.9

9.6

7.0%

2.9%

1.000

37.9

31.4

-

0.270

0.080

0.906

0.290

0.274

0.177

1.638

3.080

-

-

-

-

ACCEPTED

21

P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lysoI_micromax_pink

inhouse

pass

P 41 21 2

P 43 21 2

0.09

1.002

1.38

1.65

+16.1%

99.3

9.8

8.0%

3.1%

1.000

35.1

29.2

-

0.275

0.084

0.900

0.297

0.279

0.177

1.677

2.470

-

-

-

-

ACCEPTED

13

P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_micromax_mono

inhouse

pass

P 41 21 2

P 43 21 2

0.16

1.003

1.18

1.50

+21.5%

79.1

8.3

5.0%

2.5%

1.000

39.9

39.9

-

0.234

0.151

0.932

0.249

0.243

0.177

1.407

4.530

-

-

-

-

ACCEPTED

20

P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_micromax_pink

inhouse

pass

P 41 21 2

P 43 21 2

0.16

1.004

1.20

1.45

+17.0%

83.6

8.3

5.1%

2.5%

1.000

40.6

37.4

-

0.236

0.180

0.930

0.260

0.253

0.177

1.468

4.650

-

-

-

-

ACCEPTED

14

P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_x06da_5keV

inhouse

pass

P 41 21 2

P 41 21 2

0.29

1.009

2.43

2.45

+0.7%

86.2

19.7

6.5%

5.4%

0.999

28.2

19.4

-

0.282

0.175

0.825

0.297

0.297

0.177

1.675

5.830

-

-

-

-

ACCEPTED

6

labelled P 43 21 2 from the model

lyso_x06da_atten_wedge

inhouse

pass

P 41 21 2

P 41 21 2

0.25

1.006

1.19

1.26

+6.0%

99.7

23.1

42.5%

7.2%

0.998

13.1

16.6

-

0.295

0.073

0.896

0.306

0.296

0.177

1.729

1.930

-

-

-

-

ACCEPTED

17

labelled P 43 21 2 from the model

lyso_x06da_half_image

inhouse

pass

P 41 21 2

P 43 21 2

0.71

1.021

1.57

1.65

+5.1%

99.7

9.8

118.0%

33.5%

0.969

6.8

6.6

-

0.278

0.061

0.889

0.289

0.282

0.177

1.629

0.830

-

-

-

-

ACCEPTED

26

P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_x06da_ice

inhouse

pass

P 41 21 2

P 4 2 2

0.09

1.002

1.34

1.43

+6.4%

99.7

19.4

18.8%

4.7%

0.999

22.5

23.3

-

0.287

0.103

0.902

0.313

0.292

0.177

1.765

2.660

-

-

-

-

ACCEPTED

13

P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_x06da_ref

inhouse

pass

P 41 21 2

P 4 2 2

0.04

1.000

0.99

1.20

+17.3%

85.7

22.2

4.8%

2.7%

1.000

29.9

28.3

-

0.272

0.089

0.902

0.287

0.275

0.177

1.622

9.660

-

-

-

-

ACCEPTED

15

P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS); labelled P 43 21 2 from the model

lyso_x10sa_90deg_1

inhouse

pass

P 41 21 2

P 41 21 2

0.12

0.998

1.83

1.97

+6.9%

99.7

6.4

17.5%

4.6%

0.998

20.7

20.8

-

0.380

0.099

0.817

0.369

0.386

0.177

2.083

0.350

-

-

-

-

ACCEPTED

8

labelled P 43 21 2 from the model

lyso_x10sa_90deg_2

inhouse

pass

P 41 21 2

P 41 21 2

0.16

0.997

1.85

1.97

+6.3%

99.7

6.4

17.0%

4.8%

0.998

19.0

18.1

-

0.378

0.104

0.819

0.372

0.385

0.177

2.100

0.560

-

-

-

-

ACCEPTED

8

labelled P 43 21 2 from the model

lyso_x10sa_strong

inhouse

pass

P 41 21 2

P 41 21 2

0.63

0.984

1.36

1.24

-9.5%

99.4

21.5

23.8%

10.3%

0.997

5.5

8.6

-

0.375

0.197

0.844

0.368

0.379

0.177

2.076

1.560

-

-

-

-

ACCEPTED

36

labelled P 43 21 2 from the model

myob_x06da

inhouse

pass

P 1 21 1

P 1 21 1

0.12

1.003

1.23

1.42

+13.4%

99.7

6.7

17.0%

4.1%

0.998

24.7

7.6

-

0.229

0.105

0.953

0.233

0.236

0.218

1.070

3.410

-

-

-

-

NOT_TESTED

8

myob_x06da_powder_1

inhouse

pass

P 1 21 1

P 1 21 1

0.02

1.000

1.73

1.50

-15.6%

99.2

5.3

76.4%

25.3%

0.765

1.9

5.5

-

0.520

0.234

0.280

0.572

0.524

0.218

2.624

0.380

-

-

-

-

ACCEPTED

12

myob_x06da_powder_2

inhouse

pass

P 1 21 1

P 1 21 1

0.34

0.995

1.41

0.99

-42.2%

99.4

5.5

125.6%

33.4%

0.908

2.5

2.2

-

0.447

0.181

0.092

0.490

0.464

0.218

2.247

1.090

-

-

-

-

NOT_TESTED

12

myob_x06da_sparse

inhouse

pass

P 1 21 1

P 1 21 1

0.44

0.993

1.77

2.00

+11.7%

99.7

5.5

45.7%

18.3%

0.902

3.0

5.5

-

0.398

0.133

0.770

0.365

0.402

0.218

1.674

0.890

-

-

-

-

NOT_TESTED

16

myob_x06da_split

inhouse

pass

P 1 21 1

P 1 21 1

0.24

0.998

1.96

1.51

-30.1%

99.7

6.0

71.2%

20.4%

0.920

2.6

12.4

-

0.387

0.120

0.219

0.375

0.391

0.218

1.718

0.480

-

-

-

-

NOT_TESTED

12

myob_x10sa

inhouse

pass

P 1 21 1

P 1 21 1

0.31

1.004

1.57

1.74

+10.0%

97.5

5.7

17.2%

7.7%

0.995

10.2

5.2

-

0.244

0.079

0.936

0.246

0.248

0.218

1.131

2.130

-

-

-

-

NOT_TESTED

14

nothing_1

inhouse

pass

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

48

no lattice reported, as expected

nothing_2

inhouse

pass

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

-

44

no lattice reported, as expected

thau_bl1a_3p8keV

inhouse

pass

P 41 21 2

P 4 2 2

0.02

1.000

3.02

3.10

+2.7%

88.7

15.5

7.3%

6.5%

0.998

23.6

30.8

-

0.185

0.031

0.935

0.188

0.186

0.151

1.241

8.910

-

-

-

-

NOT_TESTED

9

P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS)

thau_bl1a_4p6keV

inhouse

pass

P 41 21 2

P 4 2 2

0.08

0.998

2.47

2.53

+2.5%

88.4

15.8

6.8%

5.3%

0.999

38.2

35.6

-

0.183

0.027

0.943

0.175

0.185

0.151

1.154

8.530

-

-

-

-

NOT_TESTED

8

P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS)

thau_bl1a_6p5keV

inhouse

pass

P 41 21 2

P 4 2 2

0.04

0.999

1.73

1.78

+2.6%

88.5

16.1

7.5%

4.7%

0.999

42.8

34.5

-

0.191

0.052

0.951

0.183

0.194

0.151

1.212

6.690

-

-

-

-

NOT_TESTED

11

P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS)

thau_micromax_pink

inhouse

pass

P 41 21 2

P 41 21 2

0.13

0.997

1.24

1.40

+11.1%

95.1

13.8

8.8%

5.0%

0.999

15.9

21.2

-

0.190

0.202

0.962

0.213

0.215

0.151

1.407

4.020

-

-

-

-

NOT_TESTED

16

thau_x10sa_0p1deg

inhouse

pass

P 41 21 2

P 41 21 2

0.23

0.993

1.91

2.20

+13.0%

87.1

17.3

14.4%

7.3%

0.998

10.8

9.2

-

0.236

0.027

0.932

0.232

0.238

0.151

1.531

1.260

-

-

-

-

NOT_TESTED

22

thau_x10sa_16keV

inhouse

pass

P 41 21 2

P 41 21 2

0.02

1.000

1.22

1.30

+6.3%

82.4

17.6

7.1%

2.8%

1.000

53.4

44.5

-

0.228

0.119

0.942

0.231

0.235

0.151

1.525

2.340

-

-

-

-

NOT_TESTED

18

thau_x10sa_injection

inhouse

pass

P 41 21 2

P 41 21 2

0.02

1.000

1.26

1.28

+1.6%

82.5

19.0

3.9%

2.6%

1.000

33.1

36.2

-

0.227

0.024

0.939

0.223

0.229

0.151

1.474

9.340

-

-

-

-

NOT_TESTED

25

yag_x10sa_20keV

inhouse

pass

I a -3 d

I a -3 d

0.03

1.001

0.67

0.68

+2.2%

96.6

34.9

39.4%

41.8%

0.702

3.0

3.2

-

-

-

-

-

-

-

-

-

-

-

-

-

-

17

Delta vs baseline 20260929-2003_cca7bf_r12-rc173-refmac

Baseline rugnux 1.0.0-rc.173. Pass rates on the sets both runs have:

arm

common sets

baseline

this run

open

171

166/169 (98%)

164/168 (98%)

inhouse

39

39/39 (100%)

39/39 (100%)

Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with compare --rerun-changed before believing a single-set change):

set

arm

verdict

space group

d_min

ISa

R_meas

CC1/2

cell dev %

R_model (shell-scaled)

R_free

radial misfit

anom. sigma

ref-range R_meas

ref-range low-res R_meas

SHELXL R1

SHELXL wR2

SHELXL GooF

SHELXL EXTI

time s

beyond noise

insu_H_x06da_twin

inhouse

pass

R 3:H

1.38

6.3 -> 6.2

12.8% -> 12.1%

0.994 -> 0.995

0.04

- -> 0.264

- -> 0.253

- -> 0.09

- -> 4.40

13.2% -> 12.5%

11.5% -> 10.3%

-

-

-

-

12 -> 7

r_meas, refres_r_meas, refres_lowres_r_meas

insu_I_x06da_ref

inhouse

pass

I 2 3

1.40

31.1 -> 32.2

27.8% -> 30.6%

0.999

0.13

- -> 0.226

- -> 0.245

- -> 0.08

- -> 3.33

24.0% -> 25.8%

5.9% -> 6.2%

-

-

-

-

15 -> 10

r_meas, refres_r_meas, refres_lowres_r_meas

lysoI_micromax_pink

inhouse

pass

P 41 21 2

1.42 -> 1.38

33.5 -> 35.1

7.4% -> 8.0%

1.000

0.09

- -> 0.275

- -> 0.297

- -> 0.08

- -> 2.47

6.3% -> 6.6%

3.1% -> 3.2%

-

-

-

-

17 -> 13

d_min, isa, r_meas, refres_r_meas

lyso_micromax_mono

inhouse

pass

P 41 21 2

1.20 -> 1.18

39.0 -> 39.9

4.9% -> 5.0%

1.000

0.16

- -> 0.234

- -> 0.249

- -> 0.15

- -> 4.53

4.6% -> 4.7%

2.5%

-

-

-

-

24 -> 20

completeness

lyso_micromax_pink

inhouse

pass

P 41 21 2

1.26 -> 1.20

35.7 -> 40.6

5.2% -> 5.1%

1.000

0.16

- -> 0.236

- -> 0.260

- -> 0.18

- -> 4.65

4.9% -> 4.8%

2.5% -> 2.4%

-

-

-

-

17 -> 14

d_min, isa, completeness, refres_isa

lyso_x06da_5keV

inhouse

pass

P 41 21 2

2.43

25.6 -> 28.2

6.6% -> 6.5%

0.999

0.29

- -> 0.282

- -> 0.297

- -> 0.17

- -> 5.83

5.9% -> 5.7%

5.4%

-

-

-

-

8 -> 6

isa, refres_isa

lyso_x06da_atten_wedge

inhouse

pass

P 41 21 2

1.16 -> 1.19

13.0 -> 13.1

35.8% -> 42.5%

0.999 -> 0.998

0.25

- -> 0.295

- -> 0.306

- -> 0.07

- -> 1.93

30.9% -> 35.8%

7.1% -> 6.9%

-

-

-

-

19 -> 17

d_min, r_meas, completeness, refres_r_meas

lyso_x06da_half_image

inhouse

pass

P 41 21 2

1.56 -> 1.57

7.3 -> 6.8

101.0% -> 118.0%

0.972 -> 0.969

0.71

- -> 0.278

- -> 0.289

- -> 0.06

- -> 0.83

94.7% -> 110.9%

25.6% -> 33.0%

-

-

-

-

47 -> 26

isa, r_meas, refres_r_meas, refres_isa, refres_lowres_r_meas, time

lyso_x10sa_90deg_1

inhouse

pass

P 41 21 2

1.86 -> 1.83

18.8 -> 20.7

18.1% -> 17.5%

0.997 -> 0.998

0.12

- -> 0.380

- -> 0.369

- -> 0.10

- -> 0.35

16.3% -> 15.2%

4.7% -> 4.5%

-

-

-

-

10 -> 8

isa, refres_r_meas, refres_isa

lyso_x10sa_90deg_2

inhouse

pass

P 41 21 2

1.88 -> 1.85

17.7 -> 19.0

17.7% -> 17.0%

0.998

0.16

- -> 0.378

- -> 0.372

- -> 0.10

- -> 0.56

16.2% -> 15.1%

4.8% -> 4.6%

-

-

-

-

9 -> 8

isa, refres_r_meas, refres_isa

lyso_x10sa_strong

inhouse

pass

P 41 21 2

1.37 -> 1.36

4.9 -> 5.5

24.0% -> 23.8%

0.996 -> 0.997

0.63

- -> 0.375

- -> 0.368

- -> 0.20

- -> 1.56

24.0% -> 23.8%

12.9% -> 11.4%

-

-

-

-

40 -> 36

isa, refres_isa, refres_lowres_r_meas

myob_x06da_powder_1

inhouse

pass

P 1 2 1 -> P 1 21 1

1.38 -> 1.73

2.4 -> 1.9

98.7% -> 76.4%

0.929 -> 0.765

0.02

- -> 0.520

- -> 0.572

- -> 0.23

- -> 0.38

83.4% -> 76.4%

17.0% -> 27.7%

-

-

-

-

16 -> 12

space group, d_min, isa, r_meas, cc_half, refres_r_meas, refres_isa, refres_lowres_r_meas

myob_x06da_powder_2

inhouse

pass

P 1 2 1 -> P 1 21 1

1.41

2.8 -> 2.5

151.9% -> 125.6%

0.888 -> 0.908

0.34

- -> 0.447

- -> 0.490

- -> 0.18

- -> 1.09

151.9% -> 125.6%

58.8% -> 45.9%

-

-

-

-

18 -> 12

space group, isa, r_meas, cc_half, refres_r_meas, refres_isa, refres_lowres_r_meas

myob_x06da_sparse

inhouse

pass

P 1 2 1 -> P 1 21 1

1.77

3.0

49.1% -> 45.7%

0.893 -> 0.902

0.44

- -> 0.398

- -> 0.365

- -> 0.13

- -> 0.89

41.8% -> 38.6%

19.9% -> 17.2%

-

-

-

-

40 -> 16

space group, r_meas, cc_half, refres_r_meas, refres_lowres_r_meas, time

myob_x06da_split

inhouse

pass

P 1 21 1

1.84 -> 1.96

3.1 -> 2.6

69.9% -> 71.2%

0.525 -> 0.920

0.21 -> 0.24

- -> 0.387

- -> 0.375

- -> 0.12

- -> 0.48

69.9% -> 71.2%

20.6% -> 24.4%

-

-

-

-

19 -> 12

d_min, isa, cc_half, refres_isa, refres_lowres_r_meas

myob_x10sa

inhouse

pass

P 1 21 1

1.62 -> 1.57

9.1 -> 10.2

18.4% -> 17.2%

0.994 -> 0.995

0.31

- -> 0.244

- -> 0.246

- -> 0.08

- -> 2.13

17.7% -> 16.0%

7.9% -> 7.6%

-

-

-

-

18 -> 14

d_min, isa, r_meas, refres_r_meas, refres_isa

thau_bl1a_3p8keV

inhouse

pass

P 41 21 2

3.02

17.4 -> 23.6

8.1% -> 7.3%

0.997 -> 0.998

0.02

- -> 0.185

- -> 0.188

- -> 0.03

- -> 8.91

7.4% -> 6.5%

6.9% -> 6.3%

-

-

-

-

10 -> 9

isa, r_meas, refres_r_meas, refres_isa, refres_lowres_r_meas

thau_bl1a_4p6keV

inhouse

pass

P 41 21 2

2.47

33.8 -> 38.2

7.0% -> 6.8%

0.999

0.08

- -> 0.183

- -> 0.175

- -> 0.03

- -> 8.53

6.6% -> 6.4%

5.2%

-

-

-

-

11 -> 8

isa, refres_isa

thau_bl1a_6p5keV

inhouse

pass

P 41 21 2

1.73

- -> 42.8

7.2% -> 7.5%

0.999

0.04

- -> 0.191

- -> 0.183

- -> 0.05

- -> 6.69

7.1% -> 7.3%

4.6%

-

-

-

-

15 -> 11

isa, refres_isa

thau_micromax_pink

inhouse

pass

P 41 21 2

1.28 -> 1.24

15.4 -> 15.9

8.6% -> 8.8%

0.999

0.13

- -> 0.190

- -> 0.213

- -> 0.20

- -> 4.02

8.6% -> 8.7%

4.9% -> 4.8%

-

-

-

-

21 -> 16

d_min, completeness

thau_x10sa_0p1deg

inhouse

pass

P 41 21 2

2.04 -> 1.91

7.0 -> 10.8

18.7% -> 14.4%

0.999 -> 0.998

0.23

- -> 0.236

- -> 0.232

- -> 0.03

- -> 1.26

18.4% -> 13.6%

9.1% -> 6.9%

-

-

-

-

23 -> 22

d_min, isa, r_meas, completeness, refres_r_meas, refres_isa, refres_lowres_r_meas

11if

open

pass

P 41

1.38 -> 1.36

25.5 -> 25.7

4.6% -> 4.8%

1.000

0.12

0.191 -> 0.192

0.200 -> 0.188

0.19 -> 0.13

- -> 2.92

-

-

-

-

-

-

14 -> 12

completeness, radial_misfit, rfree

5ebi

open

pass -> fail

P 1 21 1 -> C 2 2 21

0.90 -> 0.85

14.4 -> 10.9

10.6% -> 12.3%

0.997 -> 0.998

0.05 -> 40.37

0.549 -> 0.580

0.535 -> 0.571

0.07 -> 0.14

- -> -0.15

-

-

-

-

-

-

52 -> 46

verdict pass->fail, space group, d_min, isa, r_meas, cell_dev_pct, rmodel_shell_scaled, radial_misfit, rfree

5epe

open

pass

F 2 3

1.78 -> 1.77

10.0 -> 9.9

14.3% -> 16.5%

0.998

0.00

0.166 -> 0.173

0.175 -> 0.181

0.08 -> 0.11

- -> 11.95

-

-

-

-

-

-

67 -> 36

r_meas, rmodel_shell_scaled, rfree, time

5j23

open

pass

R 3:H

2.17

11.5 -> 11.6

15.2% -> 15.4%

0.996

0.15

0.227 -> 0.226

0.234

0.13 -> 0.14

- -> 0.86

-

-

-

-

-

-

62 -> 34

time

5jvn

open

pass

P 6 2 2

2.24

15.6

16.3%

0.998 -> 0.999

0.03

0.234 -> 0.228

0.250 -> 0.249

0.14

- -> 1.36

-

-

-

-

-

-

40 -> 42

rmodel_shell_scaled

5ky6

open

pass

P 1 21 1

1.55 -> 1.54

7.2 -> 6.1

27.3% -> 25.4%

0.987 -> 0.986

0.64

0.245 -> 0.233

0.249 -> 0.243

0.11 -> 0.12

- -> 0.55

-

-

-

-

-

-

92 -> 91

isa, r_meas, rmodel_shell_scaled, rfree

5m17

open

pass

I 4

0.98

16.0 -> 16.2

6.0% -> 5.5%

0.994 -> 0.996

0.08

0.132

0.159 -> 0.158

0.23 -> 0.22

- -> 7.09

-

-

-

-

-

-

102 -> 97

r_meas

5mln

open

pass

P 21 21 2

1.26

21.4 -> 21.5

10.4%

0.999

0.10

0.184 -> 0.181

0.189 -> 0.188

0.14

- -> 2.17

-

-

-

-

-

-

45 -> 22

time

5nw5

open

pass

P 21 21 21

7.10 -> 7.07

8.8 -> 7.7

32.0% -> 33.3%

0.936 -> 0.927

0.19

0.401 -> 0.395

0.440 -> 0.448

0.36 -> 0.25

- -> 0.09

-

-

-

-

-

-

47 -> 46

isa, cc_half, rmodel_shell_scaled, radial_misfit, rfree

5t39

open

pass

P 1 21 1

1.01

16.0 -> 16.2

7.0% -> 7.3%

0.998

0.13

0.152 -> 0.151

0.165 -> 0.156

0.17 -> 0.09

- -> 6.47

-

-

-

-

-

-

75 -> 71

r_meas, radial_misfit, rfree

6cdl

open

pass

P 21 21 2

1.15 -> 1.13

11.2 -> 11.6

8.2% -> 7.8%

0.996 -> 0.997

1.22

0.150 -> 0.147

0.174 -> 0.159

0.28 -> 0.23

- -> 4.60

-

-

-

-

-

-

57 -> 49

r_meas, completeness, rfree

6f3p

open

pass

C 1 2 1

1.13

9.4

10.6% -> 10.0%

0.997

0.12

0.144 -> 0.143

0.167 -> 0.169

0.18 -> 0.21

- -> -3.42

-

-

-

-

-

-

130 -> 128

r_meas

6fwc

open

pass

C 2 2 2

1.41

27.2 -> 27.3

16.4%

0.995

0.04

0.189 -> 0.188

0.197 -> 0.196

0.10

- -> 1.44

-

-

-

-

-

-

48 -> 26

time

6h2p_1p89A

open

pass

C 2 2 21

1.76

22.5 -> 22.8

8.3% -> 8.5%

0.999

0.03

0.144 -> 0.143

0.154 -> 0.149

0.11 -> 0.08

- -> 9.20

-

-

-

-

-

-

172 -> 217

rfree, time

6h2p_native

open

pass

C 2 2 21

1.32

19.9 -> 20.0

13.7%

0.999

0.04

0.170 -> 0.161

0.174 -> 0.165

0.08 -> 0.07

- -> 3.33

-

-

-

-

-

-

110 -> 113

rmodel_shell_scaled, rfree

6h5t

open

pass

I 4 2 2

1.48

9.2 -> 9.3

15.2% -> 14.6%

0.996

0.40

0.203 -> 0.196

0.218 -> 0.222

0.13 -> 0.19

- -> 9.00

-

-

-

-

-

-

35

rmodel_shell_scaled, radial_misfit

6hv2

open

pass

P 61 2 2

1.42 -> 1.41

13.6 -> 13.7

18.1% -> 19.8%

1.000

0.04

0.235 -> 0.219

0.247 -> 0.239

0.46 -> 0.28

- -> 7.83

-

-

-

-

-

-

37 -> 32

r_meas, rmodel_shell_scaled, radial_misfit, rfree

6i3j

open

pass

F 2 2 2

2.32

7.0 -> 6.8

23.5% -> 24.6%

0.983 -> 0.977

0.08

0.202 -> 0.203

0.233 -> 0.237

0.16 -> 0.17

- -> 1.73

-

-

-

-

-

-

83 -> 75

cc_half

6iu6

open

pass

P 31

2.38 -> 2.36

7.0 -> 7.2

14.2% -> 11.5%

0.992 -> 0.996

0.33

0.221

0.234 -> 0.233

0.19 -> 0.17

- -> 3.54

-

-

-

-

-

-

87 -> 91

r_meas

6iu9

open

pass

P 31

2.73 -> 2.74

5.3 -> 5.4

18.0% -> 14.4%

0.989 -> 0.993

0.22

0.306 -> 0.310

0.316 -> 0.327

0.11 -> 0.10

- -> 1.43

-

-

-

-

-

-

129 -> 86

r_meas, rfree, time

6jgh

open

pass

P 21 21 21

0.87

6.7 -> 6.6

22.7% -> 23.4%

0.989 -> 0.990

0.39

0.148 -> 0.136

0.169 -> 0.156

0.19 -> 0.17

- -> 2.07

-

-

-

-

-

-

105 -> 104

rmodel_shell_scaled, rfree

6moj

open

pass

I 41 2 2

2.43

8.3

52.6% -> 52.4%

0.998

0.08

0.275 -> 0.248

0.285 -> 0.257

0.37 -> 0.21

- -> 1.05

-

-

-

-

-

-

124 -> 125

rmodel_shell_scaled, radial_misfit, rfree

6nen

open

pass

P 3 1 2

1.72 -> 1.77

6.3

26.9% -> 28.5%

0.997

0.07

0.210 -> 0.211

0.212 -> 0.218

0.08 -> 0.05

- -> 1.00

-

-

-

-

-

-

24 -> 19

d_min, r_meas, rfree

6o2h

open

pass

P 1

1.10

20.8 -> 23.9

7.2% -> 7.0%

0.978 -> 0.977

0.74

0.099 -> 0.096

0.131 -> 0.129

0.21 -> 0.13

- -> 0.11

-

-

-

-

-

-

16 -> 13

isa, radial_misfit

6oel

open

pass

F 41 3 2

2.85

9.9

36.6% -> 36.5%

0.998

0.00

0.248

0.255

0.14

- -> 0.70

-

-

-

-

-

-

57 -> 40

time

6p8p

open

pass

P 4

1.47 -> 1.46

15.3 -> 15.5

12.4% -> 13.2%

0.998

0.14

0.192 -> 0.189

0.209 -> 0.204

0.18 -> 0.14

- -> 1.89

-

-

-

-

-

-

25 -> 20

r_meas

6pxb

open

pass

P 31 1 2

1.39

12.0 -> 12.1

8.8%

0.999

0.23

0.245 -> 0.240

0.277 -> 0.279

0.31 -> 0.26

- -> 0.62

-

-

-

-

-

-

38 -> 16

radial_misfit, time

6pxc

open

pass

I 2 2 2

1.43 -> 1.41

10.1 -> 10.2

8.5% -> 8.1%

0.996 -> 0.998

0.23

0.209 -> 0.210

0.215 -> 0.214

0.17 -> 0.16

- -> 0.69

-

-

-

-

-

-

36 -> 32

completeness

6qaj

open

pass

C 2 2 21

2.70

13.1 -> 13.2

24.6% -> 24.4%

0.997

0.41

0.376 -> 0.321

0.390 -> 0.332

0.48 -> 0.23

- -> 4.46

-

-

-

-

-

-

67 -> 65

rmodel_shell_scaled, radial_misfit, rfree

6r72

open

pass

P 1 21 1

4.39

15.7 -> 17.9

14.8% -> 16.5%

0.999 -> 1.000

1.32

0.374 -> 0.362

0.383 -> 0.367

0.06 -> 0.04

- -> 0.13

-

-

-

-

-

-

29 -> 26

isa, r_meas, rmodel_shell_scaled, rfree

6rlr

open

pass

P 1

2.07 -> 1.92

13.2 -> 16.4

11.0% -> 12.9%

0.998

0.03

0.249 -> 0.250

0.258 -> 0.254

0.08 -> 0.07

- -> 0.68

-

-

-

-

-

-

18

d_min, isa, r_meas

6s1u

open

pass

P 1 21 1

1.75

13.7 -> 13.8

22.4% -> 22.3%

0.993 -> 0.989

0.13

0.204 -> 0.202

0.208 -> 0.200

0.09

- -> 0.60

-

-

-

-

-

-

36 -> 13

rfree, time

6toc

open

pass

P 42 2 2

1.64

24.3 -> 24.8

9.4%

1.000

0.30

0.276

0.269 -> 0.270

0.14

- -> 0.54

-

-

-

-

-

-

49 -> 11

time

6u7g

open

pass

P 1 21 1

1.92 -> 1.88

13.2 -> 13.3

8.3% -> 8.0%

0.998

0.13

0.201 -> 0.198

0.206 -> 0.200

0.14 -> 0.11

- -> 1.57

-

-

-

-

-

-

70 -> 71

d_min, completeness, rfree

6ukf

open

pass

P 1 21 1

0.95 -> 0.96

9.4 -> 9.8

10.7% -> 9.5%

0.997 -> 0.998

0.07 -> 0.08

0.166 -> 0.165

0.162 -> 0.161

0.09 -> 0.07

- -> 2.26

-

-

-

-

-

-

54 -> 48

r_meas

6vww

open

pass

P 63

2.00 -> 1.99

8.6 -> 8.0

15.9% -> 16.4%

0.993

0.20

0.239 -> 0.240

0.241 -> 0.243

0.10 -> 0.12

- -> 0.67

-

-

-

-

-

-

24 -> 15

isa

6wzo

open

pass

P 1

1.06 -> 1.04

16.9 -> 16.8

5.8% -> 6.2%

0.998 -> 0.999

0.04

0.179 -> 0.174

0.186 -> 0.175

0.24 -> 0.12

- -> 2.68

-

-

-

-

-

-

57 -> 54

d_min, r_meas, completeness, radial_misfit, rfree

6yqf

open

pass

P 21 21 2

3.05 -> 3.02

3.9 -> 4.2

50.3% -> 70.0%

0.988 -> 0.982

0.71

0.432 -> 0.434

0.470 -> 0.442

0.33 -> 0.16

- -> -0.20

-

-

-

-

-

-

24 -> 23

isa, r_meas, cc_half, radial_misfit, rfree

6z8o

open

pass

P 1 21 1

2.23 -> 2.21

13.8 -> 13.9

17.1% -> 17.4%

0.995

0.63

0.290 -> 0.276

0.295 -> 0.281

0.09 -> 0.07

- -> 1.24

-

-

-

-

-

-

62 -> 64

rmodel_shell_scaled, rfree

6ze4

open

pass

P 21 21 21

1.30

9.2 -> 9.3

20.4% -> 20.2%

0.992 -> 0.993

0.59

0.232 -> 0.213

0.237 -> 0.218

0.11 -> 0.08

- -> 1.79

-

-

-

-

-

-

73 -> 75

rmodel_shell_scaled, rfree

6zqr

open

pass

P 4

1.79 -> 1.76

8.7 -> 9.1

19.6% -> 19.3%

0.996

0.40

0.190 -> 0.197

0.198 -> 0.204

0.12 -> 0.17

- -> 1.35

-

-

-

-

-

-

45 -> 17

rmodel_shell_scaled, rfree, time

6zqy

open

pass

P 4

1.72 -> 1.70

13.8 -> 12.7

18.5% -> 23.3%

0.997 -> 0.994

0.12

0.212 -> 0.213

0.226 -> 0.222

0.20 -> 0.16

- -> 1.36

-

-

-

-

-

-

62 -> 32

isa, r_meas, time

6zr0

open

pass

P 4

1.73 -> 1.66

24.3 -> 24.5

11.1% -> 13.1%

0.997 -> 0.993

0.08

0.217 -> 0.216

0.224 -> 0.218

0.14 -> 0.07

- -> 1.80

-

-

-

-

-

-

61 -> 57

d_min, r_meas, completeness, radial_misfit, rfree

7arr

open

pass

P 1

0.92

17.3 -> 18.9

4.7% -> 3.7%

0.993 -> 0.999

0.28

0.155 -> 0.154

0.175 -> 0.166

0.27 -> 0.20

- -> 0.00

-

-

-

-

-

-

41 -> 35

isa, r_meas, cc_half, radial_misfit, rfree

7atg

open

pass

P 21 21 21

0.60

22.5 -> 22.9

5.1% -> 5.2%

0.995

0.08

0.135 -> 0.127

0.199 -> 0.176

0.23 -> 0.20

- -> 6.47

-

-

-

-

-

-

41 -> 28

rmodel_shell_scaled, rfree, time

7bgt

open

pass

P 1

1.77 -> 1.78

14.7 -> 17.4

15.0% -> 13.7%

0.989 -> 0.993

0.32

0.197 -> 0.195

0.207

0.12

- -> 0.23

-

-

-

-

-

-

32 -> 12

isa, r_meas, time

7brr

open

pass

P 1 21 1

1.27 -> 1.24

15.8 -> 16.6

6.5% -> 6.6%

0.999

0.09

0.191 -> 0.192

0.195 -> 0.191

0.15 -> 0.10

- -> 3.07

-

-

-

-

-

-

25 -> 17

d_min, completeness, radial_misfit

7mzt

open

fail -> unscored

P 21 21 21

3.25 -> 3.12

3.4 -> 4.1

269.6% -> 366.1%

0.968 -> 0.971

0.56

0.452 -> 0.414

0.466 -> 0.426

0.16 -> 0.10

- -> -0.04

-

-

-

-

-

-

37 -> 33

verdict fail->unscored, d_min, isa, r_meas, rmodel_shell_scaled, radial_misfit, rfree

7n0i

open

pass

P 21 21 21

1.65 -> 1.68

10.7 -> 10.9

14.0% -> 14.5%

0.998

0.22

0.290 -> 0.267

0.310 -> 0.295

0.36 -> 0.26

- -> 0.53

-

-

-

-

-

-

58 -> 27

rmodel_shell_scaled, radial_misfit, rfree, time

7n2s

open

pass

P 1 21 1

2.57 -> 2.56

9.6

52.8% -> 56.7%

0.903 -> 0.917

0.07

0.315 -> 0.299

0.334 -> 0.312

0.14 -> 0.11

- -> 0.34

-

-

-

-

-

-

42 -> 12

r_meas, cc_half, rmodel_shell_scaled, rfree, time

7orr

open

pass

I 2 3

1.65 -> 1.62

25.5 -> 26.5

6.4% -> 6.7%

1.000

0.03

0.181 -> 0.183

0.190 -> 0.196

0.14 -> 0.16

- -> 5.08

-

-

-

-

-

-

23 -> 20

r_meas, rfree

7ou1

open

pass

P 1 21 1

1.41 -> 1.40

9.5 -> 9.0

16.1% -> 15.9%

0.991

0.04

0.206 -> 0.202

0.211 -> 0.209

0.09 -> 0.10

- -> 1.52

-

-

-

-

-

-

44 -> 22

time

7ph1

open

pass

I 2 2 2

1.08

15.9 -> 16.0

11.4% -> 11.3%

0.998

0.12

0.160 -> 0.155

0.178 -> 0.174

0.17 -> 0.16

- -> 3.21

-

-

-

-

-

-

87 -> 77

rmodel_shell_scaled

7pq7

open

pass

C 1 2 1

1.38 -> 1.37

13.7 -> 14.0

7.1% -> 6.5%

0.998

0.22

0.189 -> 0.188

0.209 -> 0.215

0.18 -> 0.19

- -> 2.32

-

-

-

-

-

-

16 -> 14

r_meas, rfree

7qij

open

pass

P 21 21 21

3.59

9.4

24.5% -> 24.1%

0.996

0.36

0.374 -> 0.357

0.381 -> 0.364

0.07

- -> 0.18

-

-

-

-

-

-

94 -> 92

rmodel_shell_scaled, rfree

7ris

open

pass

P 31 2 1

1.51

30.2 -> 31.0

10.0%

1.000

0.08

0.196 -> 0.188

0.195 -> 0.189

0.03 -> 0.05

- -> 2.26

-

-

-

-

-

-

29 -> 24

rmodel_shell_scaled, rfree

7tcd

open

pass

C 1 2 1

1.72 -> 1.65

15.0 -> 18.4

10.0% -> 8.6%

0.999

0.20

0.205 -> 0.191

0.211

0.14 -> 0.20

- -> 1.17

-

-

-

-

-

-

32 -> 28

d_min, isa, r_meas, completeness, rmodel_shell_scaled, radial_misfit

8a1a

open

pass

P 61

1.95 -> 1.93

17.9 -> 18.8

41.5% -> 52.0%

0.998 -> 0.997

0.42

0.171 -> 0.177

0.177 -> 0.179

0.06 -> 0.03

- -> 1.82

-

-

-

-

-

-

100 -> 94

r_meas, rmodel_shell_scaled

8agq

open

pass

C 1 2 1

0.97

14.0 -> 14.1

9.8% -> 9.2%

0.998 -> 0.999

0.30

0.180 -> 0.181

0.204

0.21

- -> 6.46

-

-

-

-

-

-

29 -> 26

r_meas

8dqb

open

pass

I 2 3

2.06 -> 2.05

22.3 -> 22.1

13.5% -> 14.3%

0.996

0.08

0.230

0.240 -> 0.237

0.13 -> 0.11

- -> 3.08

-

-

-

-

-

-

26 -> 14

r_meas, time

8dz7

open

pass

P 21 21 21

1.19

33.7 -> 35.1

3.2% -> 3.0%

0.998 -> 0.999

0.08

0.119 -> 0.118

0.129 -> 0.128

0.07

- -> 5.14

-

-

-

-

-

-

15 -> 10

r_meas

8egn

open

pass

P 21 21 21

1.64 -> 1.63

22.7 -> 22.9

6.1% -> 6.3%

0.999

0.07

0.200 -> 0.197

0.214 -> 0.215

0.16 -> 0.15

- -> 2.53

-

-

-

-

-

-

15 -> 14

completeness

8k1g

open

pass

I 4 2 2

1.63

11.6 -> 11.7

25.7% -> 25.6%

0.999

0.82

0.219 -> 0.206

0.236 -> 0.227

0.23 -> 0.20

- -> 2.26

-

-

-

-

-

-

48 -> 43

rmodel_shell_scaled, rfree

8oic

open

pass

P 1

2.37 -> 2.34

19.6 -> 20.0

20.4% -> 24.4%

0.991 -> 0.988

0.02

0.234 -> 0.238

0.250 -> 0.248

0.15 -> 0.11

- -> 0.41

-

-

-

-

-

-

42

r_meas

8qj5

open

pass

P 1 21 1

1.35 -> 1.29

8.4

17.1% -> 17.7%

0.996

0.45

0.189 -> 0.192

0.203 -> 0.206

0.14 -> 0.12

- -> 1.50

-

-

-

-

-

-

59 -> 20

d_min, completeness, time

8qq7

open

pass

P 62 2 2

3.16

6.3 -> 6.4

18.8% -> 19.0%

0.993 -> 0.997

0.65

0.457 -> 0.432

0.464 -> 0.429

0.50 -> 0.37

- -> 0.75

-

-

-

-

-

-

17 -> 13

rmodel_shell_scaled, radial_misfit, rfree

8r5r

open

pass

P 21 21 21

2.89 -> 2.80

15.1 -> 19.3

27.8% -> 22.7%

0.997 -> 0.998

0.11 -> 0.04

0.274 -> 0.263

0.290 -> 0.284

0.14

- -> 0.43

-

-

-

-

-

-

35 -> 30

d_min, isa, r_meas, rmodel_shell_scaled, rfree

8rud

open

pass

P 1 21 1

1.70 -> 1.57

12.3 -> 11.9

29.5% -> 37.8%

0.992 -> 0.991

0.40

0.244 -> 0.253

0.255 -> 0.261

0.11 -> 0.08

- -> 0.69

-

-

-

-

-

-

284 -> 236

d_min, r_meas, rmodel_shell_scaled, rfree

8sa8

open

pass

I 1 2 1

1.11 -> 1.10

22.0

11.2% -> 11.7%

0.999

0.02

0.158 -> 0.155

0.175 -> 0.169

0.17 -> 0.14

- -> 3.79

-

-

-

-

-

-

99 -> 94

rfree

8sqo

open

pass

P 4 3 2

1.33 -> 1.32

14.2

22.1% -> 23.4%

1.000

0.13

0.183

0.211 -> 0.206

0.20 -> 0.18

- -> 7.02

-

-

-

-

-

-

72

r_meas, rfree

8sqt

open

pass

F 4 3 2

1.89 -> 1.88

29.6 -> 29.9

18.9% -> 20.1%

0.999

0.11

0.221 -> 0.223

0.224 -> 0.226

0.12 -> 0.11

- -> 2.08

-

-

-

-

-

-

30 -> 28

r_meas

8t7r

open

pass

C 1 2 1

3.22 -> 3.23

9.9 -> 7.8

35.3% -> 31.2%

0.984

0.28

0.288 -> 0.283

0.288 -> 0.285

0.06 -> 0.08

- -> 0.43

-

-

-

-

-

-

66 -> 63

isa, r_meas

8tha

open

pass

P 62

1.34 -> 1.33

25.9 -> 27.7

13.7% -> 16.3%

1.000 -> 0.999

0.17

0.208 -> 0.212

0.225 -> 0.220

0.16 -> 0.09

- -> 4.37

-

-

-

-

-

-

23 -> 19

isa, r_meas, radial_misfit

8u0i

open

pass

P 41 21 2

1.40 -> 1.38

16.1 -> 17.0

7.9% -> 8.1%

0.999

0.06

0.176 -> 0.179

0.180 -> 0.183

0.07 -> 0.05

- -> 3.37

-

-

-

-

-

-

24 -> 18

isa

8v4o

open

pass

P 61 2 2

2.10

15.5

31.9%

0.998

0.06

0.241 -> 0.236

0.253

0.16

- -> 0.96

-

-

-

-

-

-

92 -> 63

time

8xbp

open

pass

C 1 2 1

1.72 -> 1.66

16.2 -> 18.6

10.8% -> 11.2%

0.999

0.15

0.315 -> 0.313

0.325 -> 0.329

0.15 -> 0.18

- -> 1.21

-

-

-

-

-

-

22 -> 21

d_min, isa, completeness

8xtf

open

pass

R 3 2:H

1.84

7.5 -> 7.6

85.8% -> 73.3%

0.987 -> 0.989

0.22

0.191

0.185 -> 0.186

0.04

- -> 1.40

-

-

-

-

-

-

38 -> 31

r_meas

8y74

open

pass

C 1 2 1

1.71 -> 1.68

8.8 -> 8.6

12.3% -> 13.0%

0.997

0.34

0.212 -> 0.214

0.224 -> 0.225

0.12 -> 0.10

- -> 1.23

-

-

-

-

-

-

24 -> 11

r_meas, time

8ys9

open

pass

P 21 21 21

1.35 -> 1.31

15.4

11.0% -> 13.3%

0.999

0.16

0.173 -> 0.176

0.191 -> 0.188

0.14 -> 0.10

- -> 3.11

-

-

-

-

-

-

31 -> 29

d_min, r_meas

9b22

open

pass

P 1 21 1

1.18 -> 1.14

16.3 -> 16.4

6.0% -> 6.2%

0.999

0.07

0.161 -> 0.159

0.181 -> 0.166

0.20 -> 0.12

- -> 0.16

-

-

-

-

-

-

21 -> 17

d_min, completeness, radial_misfit, rfree

9bn8

open

pass

P 41

1.20 -> 1.21

19.8

7.5% -> 8.1%

0.999 -> 1.000

0.09

0.151

0.172 -> 0.159

0.17 -> 0.10

- -> 3.77

-

-

-

-

-

-

30 -> 27

r_meas, radial_misfit, rfree

9c18

open

pass

P 1

1.76 -> 1.71

10.2 -> 10.8

23.4% -> 27.3%

0.988 -> 0.986

0.61

0.222 -> 0.221

0.221

0.06 -> 0.07

- -> 0.71

-

-

-

-

-

-

13 -> 11

d_min, isa, r_meas

9crw

open

pass

P 1 21 1

2.29 -> 2.28

15.5 -> 15.8

8.7%

0.999

0.23

0.257 -> 0.247

0.271 -> 0.262

0.14 -> 0.13

- -> 0.69

-

-

-

-

-

-

19 -> 16

rmodel_shell_scaled, rfree

9e2t

open

pass

P 1

2.33 -> 2.29

6.9 -> 6.8

28.1% -> 29.1%

0.990 -> 0.992

0.06

0.248 -> 0.230

0.254 -> 0.232

0.11 -> 0.05

- -> 0.42

-

-

-

-

-

-

70 -> 67

rmodel_shell_scaled, radial_misfit, rfree

9fcf

open

pass

P 4

1.81 -> 1.75

7.8 -> 7.0

28.6% -> 34.5%

0.995 -> 0.994

0.04

0.285 -> 0.292

0.293 -> 0.295

0.06

- -> 0.85

-

-

-

-

-

-

142 -> 146

d_min, isa, r_meas, rmodel_shell_scaled

9fcg

open

pass

P 4

1.39 -> 1.38

9.3 -> 9.7

15.4% -> 14.1%

0.996 -> 0.997

0.10 -> 0.07

0.179 -> 0.180

0.184 -> 0.189

0.09 -> 0.11

- -> 2.42

-

-

-

-

-

-

138 -> 48

r_meas, time

9fhc

open

pass

I 2 3

1.97 -> 1.92

11.6 -> 12.3

22.4% -> 17.0%

0.995 -> 0.998

0.25

0.245 -> 0.239

0.250 -> 0.244

0.04 -> 0.07

- -> 1.20

-

-

-

-

-

-

104 -> 107

d_min, isa, r_meas, completeness, rmodel_shell_scaled, rfree

9gdj

open

pass

P 41 21 2

1.40

12.8 -> 12.9

10.9% -> 11.2%

0.999

0.16

0.153 -> 0.154

0.178 -> 0.172

0.24 -> 0.20

- -> 2.89

-

-

-

-

-

-

355 -> 289

rfree

9gjx

open

pass

P 1 21 1

2.15 -> 2.06

35.2 -> 40.1

12.5% -> 16.3%

0.999 -> 0.997

0.13

0.188 -> 0.193

0.225 -> 0.224

0.18 -> 0.15

- -> 0.84

-

-

-

-

-

-

33 -> 28

d_min, isa, r_meas, completeness

9h0q

open

pass

R 3 2:H

2.13 -> 2.10

18.7 -> 18.6

15.4% -> 17.4%

0.998

0.41

0.203 -> 0.205

0.233 -> 0.214

0.18 -> 0.11

- -> 1.10

-

-

-

-

-

-

72 -> 44

r_meas, radial_misfit, rfree, time

9hnc

open

pass -> fail

P 1 2 1 -> P 1 21 1

1.65 -> 1.63

1.4 -> 13.6

52.8% -> 12.8%

0.882 -> 0.998

0.13 -> 0.09

0.461 -> 0.246

0.465 -> 0.255

0.16 -> 0.15

- -> 0.88

-

-

-

-

-

-

80 -> 66

verdict pass->fail, space group, isa, r_meas, cc_half, completeness, rmodel_shell_scaled, rfree

9hs7

open

pass

P 61

1.68 -> 1.70

13.3

11.8% -> 11.7%

0.999

0.08

0.251 -> 0.213

0.279 -> 0.236

0.58 -> 0.21

- -> 1.26

-

-

-

-

-

-

27 -> 26

rmodel_shell_scaled, radial_misfit, rfree

9i0a

open

pass

P 21 21 2

1.81

13.8 -> 14.1

16.8% -> 16.9%

0.999

0.43

0.228 -> 0.224

0.239 -> 0.233

0.17 -> 0.12

- -> 0.95

-

-

-

-

-

-

59

radial_misfit, rfree

9i80

open

pass

P 41

1.61 -> 1.59

6.7 -> 6.8

23.2% -> 25.1%

0.991 -> 0.992

0.07

0.216 -> 0.219

0.221

0.15 -> 0.12

- -> 1.89

-

-

-

-

-

-

158 -> 116

r_meas, time

9ig7

open

pass

P 21 21 2

2.03 -> 2.02

11.0 -> 11.1

18.7% -> 21.7%

0.991

0.17

0.235 -> 0.234

0.255 -> 0.249

0.18 -> 0.08

- -> 0.80

-

-

-

-

-

-

85 -> 82

r_meas, radial_misfit, rfree

9jzo

open

pass

P 1

1.15

8.4 -> 8.1

9.2% -> 5.8%

0.994 -> 0.997

0.22

0.175 -> 0.179

0.175 -> 0.176

0.08 -> 0.05

- -> 4.39

-

-

-

-

-

-

17 -> 11

r_meas

9khr

open

pass

P 21 21 21

1.38

9.4 -> 9.5

18.9% -> 18.8%

0.994

0.12

0.248 -> 0.244

0.261 -> 0.259

0.11 -> 0.12

- -> 2.02

-

-

-

-

-

-

33 -> 13

time

9min

open

fail

P 21 21 2

1.87 -> 1.86

10.4 -> 10.5

26.6% -> 26.5%

0.998 -> 0.999

36.97

0.573 -> 0.566

0.578 -> 0.575

0.20 -> 0.21

- -> -0.00

-

-

-

-

-

-

77 -> 73

rmodel_shell_scaled

9o0h

open

pass

P 21 21 21

2.01 -> 2.02

6.1 -> 5.8

52.6% -> 65.1%

0.985

0.25

0.247 -> 0.242

0.255 -> 0.249

0.04 -> 0.08

- -> 0.11

-

-

-

-

-

-

64 -> 59

isa, r_meas, rmodel_shell_scaled, rfree

9p7q

open

pass

C 1 2 1

1.81 -> 1.76

10.6 -> 10.8

25.7% -> 23.8%

0.988 -> 0.985

0.14

0.270 -> 0.269

0.285 -> 0.282

0.03 -> 0.04

- -> 0.87

-

-

-

-

-

-

15 -> 12

d_min, r_meas, completeness

9pbb

open

pass

C 1 2 1

1.83 -> 1.78

14.5 -> 17.0

20.7% -> 19.1%

0.995 -> 0.993

0.15

0.236 -> 0.233

0.236 -> 0.231

0.04

- -> 0.67

-

-

-

-

-

-

19 -> 16

d_min, isa, r_meas, completeness

9q41

open

pass

C 2 2 21

1.69 -> 1.68

7.8 -> 7.9

27.3% -> 29.0%

0.984

0.20

0.187 -> 0.188

0.191 -> 0.194

0.09 -> 0.10

- -> 0.48

-

-

-

-

-

-

57 -> 23

r_meas, time

9q66

open

pass

P 1 21 1

2.04 -> 2.03

13.0 -> 13.1

31.7% -> 32.1%

0.990

0.34

0.204

0.216

0.09 -> 0.08

- -> 0.66

-

-

-

-

-

-

68 -> 30

time

9qw8

open

pass

P 1

1.59 -> 1.71

11.3 -> 9.2

14.6% -> 20.0%

0.994 -> 0.989

0.12 -> 0.46

0.237 -> 0.294

0.248 -> 0.312

0.10 -> 0.15

- -> 0.33

-

-

-

-

-

-

73 -> 66

d_min, isa, r_meas, cc_half, cell_dev_pct, rmodel_shell_scaled, radial_misfit, rfree

9rci

open

pass

P 1

1.82 -> 1.76

6.0 -> 7.0

29.4% -> 29.6%

0.942 -> 0.951

97.59 -> 97.58

0.576 -> 0.561

0.571 -> 0.555

0.09 -> 0.08

- -> -0.02

-

-

-

-

-

-

25 -> 22

d_min, isa, cc_half, rmodel_shell_scaled, rfree

9rcs

open

pass

P 1 21 1

3.37 -> 3.27

6.0 -> 7.3

24.3% -> 23.4%

0.992 -> 0.994

0.93

0.383 -> 0.318

0.442 -> 0.341

0.19 -> 0.12

- -> 0.38

-

-

-

-

-

-

55 -> 51

d_min, isa, rmodel_shell_scaled, radial_misfit, rfree

9rp9

open

pass

C 1 2 1

1.91 -> 1.90

28.8 -> 32.7

16.1% -> 17.4%

0.997

0.16

0.206 -> 0.204

0.224 -> 0.233

0.08 -> 0.14

- -> 2.47

-

-

-

-

-

-

23 -> 25

isa, r_meas, radial_misfit, rfree

9sl0

open

pass

P 21 21 21

1.38 -> 1.36

20.0 -> 20.2

7.6% -> 8.0%

1.000

0.37

0.245

0.256 -> 0.255

0.15 -> 0.16

- -> 2.20

-

-

-

-

-

-

50 -> 45

r_meas

9t6s

open

pass

P 21 21 21

1.76 -> 1.75

24.9 -> 29.9

10.5% -> 11.2%

0.999

0.06

0.220 -> 0.215

0.236 -> 0.232

0.03 -> 0.04

- -> 2.51

-

-

-

-

-

-

28 -> 12

isa, r_meas, time

9upt

open

pass

P 6

2.03

6.4 -> 6.5

25.9% -> 24.4%

0.988 -> 0.989

0.20

0.227 -> 0.218

0.229 -> 0.218

0.08 -> 0.09

- -> 1.31

-

-

-

-

-

-

76 -> 77

r_meas, rmodel_shell_scaled, rfree

9vyb

open

pass

P 21 21 21

1.71 -> 1.67

16.7 -> 22.1

8.0% -> 8.2%

1.000 -> 0.999

0.40

0.245 -> 0.246

0.258 -> 0.253

0.07 -> 0.08

- -> 0.42

-

-

-

-

-

-

41 -> 34

d_min, isa, completeness, rfree

9w3y

open

pass

P 21 21 21

1.20 -> 1.19

19.9 -> 20.2

23.4% -> 24.7%

0.997

0.25

0.186 -> 0.187

0.200 -> 0.198

0.11 -> 0.09

- -> 4.93

-

-

-

-

-

-

17 -> 11

r_meas

9yl4

open

pass

P 21 21 21

3.61

9.5

32.0% -> 31.8%

0.985 -> 0.996

0.08

0.295 -> 0.293

0.306 -> 0.303

0.07

- -> 5.75

-

-

-

-

-

-

85 -> 90

cc_half

9yzk

open

pass

I 1 2 1

3.86 -> 3.87

7.8 -> 9.3

29.7% -> 30.9%

0.996 -> 0.997

0.20

0.375 -> 0.355

0.394 -> 0.406

0.29

- -> 0.18

-

-

-

-

-

-

15 -> 12

isa, rmodel_shell_scaled, rfree

9z44

open

pass

I 1 2 1

6.86 -> 6.73

5.9 -> 8.5

26.6% -> 22.3%

0.953 -> 0.981

1.35

0.336 -> 0.334

0.325 -> 0.324

0.17 -> 0.16

- -> -0.03

-

-

-

-

-

-

32 -> 27

isa, r_meas, cc_half

9z72

open

pass

P 31 2 1

1.97 -> 2.00

11.8

76.6% -> 80.3%

0.991

0.15

0.241 -> 0.240

0.256 -> 0.255

0.06 -> 0.05

- -> 0.57

-

-

-

-

-

-

152 -> 44

time

9zlo

open

pass

P 21 21 21

1.58 -> 1.56

23.0 -> 24.1

12.0% -> 12.3%

0.999

0.39

0.235 -> 0.224

0.241 -> 0.231

0.05 -> 0.07

- -> 1.75

-

-

-

-

-

-

26 -> 25

rmodel_shell_scaled, rfree

9zm0

open

pass

P 1 21 1

1.81 -> 1.80

8.6 -> 9.0

23.7% -> 25.0%

0.996

0.16

0.262 -> 0.255

0.283 -> 0.275

0.12 -> 0.10

- -> 4.59

-

-

-

-

-

-

11 -> 9

r_meas, rmodel_shell_scaled, rfree

9zmu

open

pass

P 61 2 2

1.71 -> 1.72

10.8 -> 11.6

34.0% -> 30.1%

0.999

0.25

0.276 -> 0.278

0.285 -> 0.284

0.17 -> 0.15

- -> 0.89

-

-

-

-

-

-

39 -> 31

isa, r_meas

cuhf2

open

unscored

P 2 2 2 -> P 4 2 2

0.52 -> 0.51

3.8 -> 45.5

17.4% -> 2.8%

0.988 -> 1.000

-

-

-

-

-

-

-

-

-

-

-

56 -> 32

space group, isa, r_meas, cc_half, completeness, time

cytidine

open

pass

P 21 21 21

0.58

4.5 -> 8.7

20.6% -> 10.3%

0.980 -> 0.993

0.31

-

-

-

-

-

-

0.1018 -> 0.0613

0.3180 -> 0.1762

1.124 -> 1.061

0.096 -> 0.039

29 -> 24

isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL goof, SHELXL exti

dnba

open

pass

C 1 2/c 1

0.81

4.3 -> 36.8

16.7% -> 2.5%

0.979 -> 1.000

0.06

-

-

-

-

-

-

0.0607 -> 0.0266

0.1531 -> 0.0706

1.036 -> 1.072

0.004 -> 0.002

10 -> 6

isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL goof

lalanine

open

pass

P 21 21 21

0.65

7.5 -> 42.4

12.1% -> 2.6%

0.994 -> 1.000

0.07

-

-

-

-

-

-

0.0759 -> 0.0311

0.2201 -> 0.0939

1.147 -> 1.150

0.027 -> 0.000

14 -> 9

isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL exti

metformin

open

pass

P 1 21/c 1

0.51

13.6 -> 30.8

6.4% -> 3.1%

0.998 -> 1.000

0.17

-

-

-

-

-

-

0.0527 -> 0.0322

0.1569 -> 0.0960

1.073 -> 1.101

0.000

8 -> 5

isa, r_meas, SHELXL r1, SHELXL wr2

nidppe

open

pass

P 1 21/c 1

0.51

25.6 -> 32.9

8.2% -> 7.9%

0.998 -> 0.999

0.28

-

-

-

-

-

-

0.0441 -> 0.0414

0.1171 -> 0.1022

1.046 -> 1.036

0.006 -> 0.004

9 -> 5

isa, SHELXL wr2

aspirin_x10sa_20keV

inhouse

- -> pass

- -> P 1 21/c 1

- -> 0.66

- -> 36.4

- -> 3.0%

- -> 1.000

- -> 0.05

-

-

-

-

- -> 3.0%

- -> 2.2%

- -> 0.0363

- -> 0.1095

- -> 1.100

- -> 0.009

- -> 9

only in B

aspirin_x10sa_25keV

inhouse

- -> pass

- -> P 1 21/c 1

- -> 0.53

- -> 36.4

- -> 3.1%

- -> 1.000

- -> 0.05

-

-

-

-

- -> 3.1%

- -> 2.2%

- -> 0.0358

- -> 0.1165

- -> 1.077

- -> 0.012

- -> 11

only in B

citricacid_x10sa_20keV

inhouse

- -> pass

- -> P 1 21/c 1

- -> 0.67

- -> 24.1

- -> 3.6%

- -> 0.999

- -> 0.10

-

-

-

-

- -> 3.6%

- -> 3.8%

- -> 0.0374

- -> 0.1026

- -> 1.092

- -> 0.149

- -> 11

only in B

hepes_x10sa_20keV

inhouse

- -> pass

- -> P b c a

- -> 0.66

- -> 39.6

- -> 2.6%

- -> 1.000

- -> 0.07

-

-

-

-

- -> 2.6%

- -> 3.5%

- -> 0.0310

- -> 0.0895

- -> 1.061

- -> 0.053

- -> 12

only in B

kdp_x10sa_20keV

inhouse

- -> fail

- -> I 41 m d

- -> 0.66

- -> 25.1

- -> 5.2%

- -> 0.999

- -> 0.07

-

-

-

-

- -> 5.0%

- -> 4.7%

- -> 0.0475

- -> 0.1208

- -> 1.363

- -> 0.028

- -> 93

only in B

lcystine_x10sa_20keV

inhouse

- -> pass

- -> P 61 2 2

- -> 0.67

- -> 5.7

- -> 30.6%

- -> 1.000

- -> 0.49

-

-

-

-

-

-

- -> 0.1575

- -> 0.3812

- -> 1.457

- -> 0.000

- -> 57

only in B

lcystine_x10sa_25keV

inhouse

- -> pass

- -> P 61 2 2

- -> 0.53

- -> 5.5

- -> 26.7%

- -> 1.000

- -> 0.52

-

-

-

-

-

-

- -> 0.1397

- -> 0.3903

- -> 1.465

- -> 0.000

- -> 23

only in B

yag_x10sa_20keV

inhouse

- -> pass

- -> I a -3 d

- -> 0.67

- -> 3.0

- -> 39.4%

- -> 0.702

- -> 0.03

-

-

-

-

- -> 39.4%

- -> 41.8%

- -> 0.1021

- -> 0.2472

- -> 1.142

- -> 0.584

- -> 17

only in B

2wnn

open

- -> pass

- -> P 1 21 1

- -> 1.44

- -> 14.1

- -> 7.2%

- -> 0.997

- -> 0.11

- -> 0.279

- -> 0.286

- -> 0.17

- -> 1.00

-

-

-

-

-

-

- -> 76

only in B

2wnq

open

- -> fail

- -> C 2 2 21

- -> 1.65

- -> 10.3

- -> 10.8%

- -> 0.997

- -> 68.55

- -> 0.536

- -> 0.545

- -> 0.28

- -> 0.02

-

-

-

-

-

-

- -> 63

only in B

2wnz

open

- -> pass

- -> P 1 21 1

- -> 1.85

- -> 14.8

- -> 12.2%

- -> 0.996

- -> 0.13

- -> 0.207

- -> 0.232

- -> 0.22

- -> 0.94

-

-

-

-

-

-

- -> 61

only in B

2xfw

open

- -> pass

- -> P 1 21 1

- -> 1.55

- -> 14.1

- -> 14.8%

- -> 0.996

- -> 0.11

- -> 0.200

- -> 0.220

- -> 0.16

- -> 1.11

-

-

-

-

-

-

- -> 93

only in B

3mc4

open

- -> pass

- -> R 3:H

- -> 1.79

- -> 12.5

- -> 10.0%

- -> 0.994

- -> 0.06

- -> 0.252

- -> 0.256

- -> 0.06

- -> 1.40

-

-

-

-

-

-

- -> 10

only in B

3meb

open

- -> pass

- -> P 1 21 1

- -> 1.53

- -> 15.6

- -> 13.9%

- -> 0.989

- -> 0.43

- -> 0.201

- -> 0.203

- -> 1.10

- -> -0.19

-

-

-

-

-

-

- -> 6

only in B

3p85

open

- -> pass

- -> P 63 2 2

- -> 1.62

- -> 8.1

- -> 12.1%

- -> 0.998

- -> 0.50

- -> 0.194

- -> 0.201

- -> 0.12

- -> 7.76

-

-

-

-

-

-

- -> 10

only in B

3r6o

open

- -> fail

- -> I 41 2 2

- -> 1.53

- -> 4.5

- -> 19.1%

- -> 0.985

- -> 0.43

- -> 0.302

- -> 0.307

- -> 0.72

- -> 1.58

-

-

-

-

-

-

- -> 5

only in B

4bwl

open

- -> fail

- -> C 2 2 21

- -> 1.68

- -> 14.3

- -> 14.1%

- -> 0.998

- -> 72.04

- -> 0.520

- -> 0.524

- -> 0.13

- -> 0.30

-

-

-

-

-

-

- -> 59

only in B

5cc8

open

- -> fail

- -> P 21 21 21

- -> 1.53

- -> 11.2

- -> 8.0%

- -> 0.997

- -> 0.06

- -> 0.171

- -> 0.184

- -> 0.17

- -> 4.01

-

-

-

-

-

-

- -> 11

only in B

5jk4

open

- -> pass

- -> P 1 21 1

- -> 1.02

- -> 14.8

- -> 8.9%

- -> 0.998

- -> 0.12

- -> 0.109

- -> 0.123

- -> 0.09

- -> 3.69

-

-

-

-

-

-

- -> 25

only in B

5ojv

open

- -> pass

- -> P 21 21 2

- -> 1.82

- -> 13.2

- -> 14.5%

- -> 0.998

- -> 0.41

- -> 0.173

- -> 0.187

- -> 0.16

- -> 1.99

-

-

-

-

-

-

- -> 237

only in B

5uth

open

- -> pass

- -> P 31 2 1

- -> 1.72

- -> 9.5

- -> 12.4%

- -> 0.997

- -> 0.21

- -> 0.194

- -> 0.217

- -> 0.17

- -> 2.80

-

-

-

-

-

-

- -> 10

only in B

5vml

open

- -> pass

- -> P 42 21 2

- -> 1.92

- -> 10.9

- -> 10.2%

- -> 0.996

- -> 0.02

- -> 0.156

- -> 0.165

- -> 0.05

- -> 3.40

-

-

-

-

-

-

- -> 6

only in B

6cee

open

- -> pass

- -> P 21 21 21

- -> 1.38

- -> 23.9

- -> 5.4%

- -> 0.999

- -> 0.04

- -> 0.166

- -> 0.174

- -> 0.14

- -> 9.05

-

-

-

-

-

-

- -> 7

only in B

6cs9

open

- -> pass

- -> P 1 21 1

- -> 1.72

- -> 13.1

- -> 9.9%

- -> 0.998

- -> 0.06

- -> 0.211

- -> 0.205

- -> 0.14

- -> 0.77

-

-

-

-

-

-

- -> 29

only in B

6gvk

open

- -> pass

- -> C 1 2 1

- -> 1.42

- -> 19.8

- -> 5.2%

- -> 0.999

- -> 0.09

- -> 0.200

- -> 0.215

- -> 0.18

- -> 1.88

-

-

-

-

-

-

- -> 94

only in B

6oww

open

- -> fail

- -> P 41 21 2

- -> 2.72

- -> 11.8

- -> 203.3%

- -> 0.994

- -> 0.19

- -> 0.363

- -> 0.376

- -> 0.08

- -> 2.03

-

-

-

-

-

-

- -> 222

only in B

6p8j

open

- -> fail

- -> P 21 21 2

- -> 1.28

- -> 5.3

- -> 23.7%

- -> 0.990

- -> 0.33

- -> 0.241

- -> 0.251

- -> 0.10

- -> 1.18

-

-

-

-

-

-

- -> 143

only in B

6rym

open

- -> pass

- -> P 41

- -> 1.45

- -> 22.9

- -> 4.2%

- -> 0.998

- -> 0.06

- -> 0.170

- -> 0.190

- -> 0.25

- -> 14.40

-

-

-

-

-

-

- -> 16

only in B

6v2r

open

- -> pass

- -> P 41 21 2

- -> 1.38

- -> 24.4

- -> 5.2%

- -> 1.000

- -> 0.02

- -> 0.205

- -> 0.209

- -> 0.21

- -> 12.96

-

-

-

-

-

-

- -> 8

only in B

7bgu

open

- -> pass

- -> P 1

- -> 2.30

- -> 10.0

- -> 12.4%

- -> 0.937

- -> 0.17

- -> 0.286

- -> 0.304

- -> 0.11

- -> 0.26

-

-

-

-

-

-

- -> 41

only in B

7q6j

open

- -> pass

- -> P 21 21 21

- -> 1.98

- -> 11.0

- -> 14.6%

- -> 0.996

- -> 0.25

- -> 0.224

- -> 0.236

- -> 0.14

- -> 1.62

-

-

-

-

-

-

- -> 126

only in B

8c3e

open

- -> fail

- -> P 6 2 2

- -> 1.79

- -> 5.8

- -> 29.4%

- -> 0.980

- -> 0.24

- -> 0.359

- -> 0.400

- -> 0.29

- -> 0.96

-

-

-

-

-

-

- -> 5

only in B

8v2t

open

- -> pass

- -> P 42 21 2

- -> 1.17

- -> 11.9

- -> 8.8%

- -> 0.999

- -> 0.24

- -> 0.166

- -> 0.181

- -> 0.21

- -> 10.86

-

-

-

-

-

-

- -> 31

only in B

8v4j

open

- -> pass

- -> P 42 21 2

- -> 1.10

- -> 22.1

- -> 5.4%

- -> 1.000

- -> 0.04

- -> 0.170

- -> 0.177

- -> 0.11

- -> 12.35

-

-

-

-

-

-

- -> 110

only in B

9jq9

open

- -> pass

- -> P 21 21 21

- -> 1.65

- -> 19.5

- -> 6.9%

- -> 0.999

- -> 0.12

- -> 0.212

- -> 0.242

- -> 0.15

- -> 4.82

-

-

-

-

-

-

- -> 6

only in B

9lxl

open

- -> pass

- -> P 41 21 2

- -> 2.06

- -> 6.2

- -> 41.9%

- -> 0.996

- -> 0.49

- -> 0.306

- -> 0.321

- -> 0.10

- -> 0.69

-

-

-

-

-

-

- -> 60

only in B

9qvv

open

- -> pass

- -> I 2 2 2

- -> 2.49

- -> 37.3

- -> 11.2%

- -> 1.000

- -> 0.42

- -> 0.235

- -> 0.239

- -> 0.20

- -> 1.64

-

-

-

-

-

-

- -> 110

only in B

9qw2

open

- -> pass

- -> P 1 21 1

- -> 1.76

- -> 7.8

- -> 18.5%

- -> 0.958

- -> 0.77

- -> 0.240

- -> 0.245

- -> 0.04

- -> 0.54

-

-

-

-

-

-

- -> 98

only in B

9s02

open

- -> pass

- -> P 21 21 2

- -> 1.42

- -> 26.0

- -> 10.2%

- -> 0.999

- -> 0.06

- -> 0.180

- -> 0.189

- -> 0.11

- -> 2.25

-

-

-

-

-

-

- -> 127

only in B

\ No newline at end of file diff --git a/CBOR.html b/CBOR.html new file mode 100644 index 000000000..75ef3d12a --- /dev/null +++ b/CBOR.html @@ -0,0 +1 @@ + CBOR messages — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

CBOR messages

To communicate between the FPGA-equipped receiver system and the writers, Jungfraujoch uses binary CBOR encoding with the tinycbor library (Intel). The protocol is based on and compatible with DECTRIS Stream2. There are minor differences at the moment:

  • LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed

  • A few fields are currently absent

  • Extra fields are present beyond DECTRIS standard

  • There are calibration and metadata messages defined beyond DECTRIS specification

Start message

Field name

Type

Description

Present in DECTRIS format

type

String

value “start”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

detector_distance

float

Detector distance [m]

detector_translation

Array(float)

Detector translation vector [m]

X

beam_center_x

float

Beam center in X direction [pixels]

X

beam_center_y

float

Beam center in Y direction [pixels]

X

direct_beam_x

float (optional)

Where the undeflected beam lands on the detector, X [pixels]. Not the same point as beam_center_x, which is the PONI - the foot of the perpendicular from the sample - and separates from the beam position as soon as the detector is tilted. This is the number a program that asks for “the beam centre” (XDS ORGX, for one) wants

direct_beam_y

float (optional)

Where the undeflected beam lands on the detector, Y [pixels] (XDS ORGY)

countrate_correction_enabled

bool

Countrate correction enabled

X

countrate_correction_lookup_table

uint32 array (optional)

Maps a measured count c to its corrected value [c], as sent by a DECTRIS detector

X

flatfield_enabled

bool

Flatfield enabled

X

virtual_pixel_interpolation_enabled

bool (optional)

Virtual pixel interpolation enabled, as reported by a DECTRIS detector

X

number_of_images

uint64

Number of images in the series

X

image_size_x

uint64

Image width [pixels]

X

image_size_y

uint64

Image height [pixels]

X

mirror_y

bool

Whether the assembled image is mirrored in Y relative to the detector’s raw readout order. True is the MX convention - row 0 at the top of the detector seen from the sample - and is what absence of the key means

detector_orientation_mirror_y

bool

Whether the assembled image is mirrored in Y relative to the frame the PONI angles are stated in. A different setting from mirror_y above, which is about the module layout; this one changes no pixel. Absence means false

detector_orientation_quarter_turns

int

How many multiples of 90 degrees about the beam the assembled image is turned by, relative to the frame the PONI angles are stated in (0-3). Absence means 0

incident_energy

float

X-ray energy [eV]

X

incident_wavelength

float

X-ray wavelength [Angstrom]

X

incident_wavelength_spread

float (optional)

FWHM of the X-ray wavelength distribution [Angstrom] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic

beam_size_x

float (optional)

Horizontal size of the X-ray beam at the sample [m] (first element of NXmx incident_beam_size)

beam_size_y

float (optional)

Vertical size of the X-ray beam at the sample [m] (second element of NXmx incident_beam_size)

frame_time

float

Frame time, if multiple frames per trigger [s]

X

count_time

float

Exposure time [s]

X

saturation_value

int64

Maximum valid sample value

X

error_value

int64 (optional)

Value used in images to describe pixels that are in error state or missing

pixel_size_x

float

Pixel width [m]

X

pixel_size_y

float

Pixel height [m]

X

sensor_thickness

float

Sensor thickness [m]

X

sensor_material

string

Sensor material

X

arm_date

date

Approximate date of arming

X

pixel_mask_enabled

bool

Pixel mask applied on images

X

detector_description

string

Name of the detector

X

detector_serial_number

string

Detector serial number

X

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

fluorescence

object (optional)

X-ray fluorescence spectrum collected at start

- energy

Array(float)

Energy of measuring point [eV]

- data

Array(float)

Fluorescence scan result data [arbitrary units]; must be strictly the same length as energy

goniometer

Map

Definition of rotation axis (optional)

X

- AXIS

string

Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch

X

- - increment

float

Rotation axis increment (per image) in degree [deg]

X

- - start

float

Rotation axis start angle [deg]

X

- - axis

Array(float)

Vector for the rotation axis

- - helical_step

Array(float)

Translation for helical scan for 1 image [m]

- - screening_wedge

Array(float)

Wedge for screening [deg] (increment would correspond to difference between screening points)

grid_scan

object

Grid scan definition (optional). Send goniometer with it, increment 0, to state the angle the spindle stood at; without one the spindle is recorded at 0, meaning “nobody said”

- n_fast

uint64

Number of elements along fast axis

- n_slow

uint64

Number of elements along slow axis

- step_x_axis

float

Step along X axis, can be negative [m]

- step_y_axis

float

Step along Y axis, can be negative [m]

- snake_scan

bool

Snake scan (rows alternate direction)

- vertical_scan

bool

Vertical scan (enabled: fast direction = Y, disabled: fast direction = X)

jungfrau_conversion_enabled

bool (optional)

Applying JUNGFRAU pixel conversion (to photons or keV)

jungfrau_conversion_factor

float (optional)

Factor used for JUNGFRAU conversion [eV]

geometry_transformation_enabled

bool (optional)

Transformation from detector module geometry (512x1024) to full detector geometry

pixel_mask

Map(string -> Image)

Pixel mask - multiple in case of storage cells

X

channels

Array(string)

List of image channels

X

max_spot_count

uint64

Maximum number of spots identified in spot finding

max_extra_lattices

uint64

Maximum number of extra lattices

storage_cell_number

uint64 (optional)

Number of storage cells used by JUNGFRAU

storage_cell_delay

Rational

Delay of storage cells in JUNGFRAU

threshold_energy

Map(string -> float)

Per-channel threshold energy [eV] (map of channel name to value)

image_dtype

string

Pixel type of the image data: uint8, uint16, uint32 (DECTRIS), plus int8, int16, int32 as a Jungfraujoch extension. Sole wire encoding of both the bit depth and the sign, and must agree with the per-image typed-array tag

X

unit_cell

object (optional)

Unit cell of the system: a, b, c [angstrom] and alpha, beta, gamma [degree]

az_int_q_bin_count

uint64

Number of azimuthal integration bins in the radial direction

az_int_phi_bin_count

uint64

Number of azimuthal integration bins in the phi angle direction

az_int_bin_to_q

Array(float)

Q value for each azimuthal integration bin [angstrom^-1]

az_int_bin_to_two_theta

Array(float)

Two theta angle value for each azimuthal integration bin [deg]

az_int_bin_to_phi

Array(float)

Phi value for each azimuthal integration bin [deg]

az_int_map

Image

Mapping between pixel and bin number

summation

uint64

Factor of frame summation

user_data

string

JSON serialized to string that can contain the following fields (all fields are optional):

X

- file_prefix

string

File prefix

- images_per_file

uint64

Number of images written per file

- images_per_trigger

uint64

Number of images collected per trigger

- source_name

string

Facility name

- source_type

string

Type of X-ray source (use NXsource/type values, for example “Synchrotron X-ray Source” or “Free-Electron Laser”)

- instrument_name

string

Instrument name

- sample_name

string

Name of the sample

- user

any valid JSON

Value of header_appendix provided at collection start to Jungfraujoch

- attenuator_transmission

float

Attenuator transmission []

- total_flux

float

Total flux [ph/s]

- space_group_number

uint64

Space group number

- summation_mode

string

Summation mode (internal|fpga|cpu)

- overwrite

bool

Overwrite existing HDF5 files

- file_format

int

File writer format: 0 = only data files, 1 = NXmx legacy external links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF (retired; rejected), 5 = TIFF (retired; rejected), 6 = no file written

- roi

Array(object)

ROI configurations; each element is one of:

type “box”: xmin, xmax, ymin, ymax (numbers)

type “circle”: r, x, y (numbers)

type “azim”: qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector

- gain_file_names

Array(string)

Names of JUNGFRAU gain files used for the current detector

- write_master_file

bool

With multiple sockets, it selects which socket will provide master file

- write_images

bool

Write images in the HDF5 file (if false, will only write metadata)

- data_reduction_factor_serialmx

uint64

Data reduction factor for serial MX

- experiment_group

string

ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number

- jfjoch_release

string

Jungfraujoch release number

- socket_number

uint64

Number of ZeroMQ socket (on jfjoch_broker side) used for transmission

- bit_depth_readout

uint64

Bit depth of the stored image (see note below), copied to NXmx bit_depth_readout

- underload_value

int64

Lowest valid value; copied to NXmx underload_value. 0 for an unsigned image, INTx_MIN + 1 for a signed one

- writer_notification_zmq_addr

string

ZeroMQ address to inform jfjoch_broker about writers that finished operation

- xfel_pulse_id

uint64

Pulse IDs are recorded for images

- ring_current_mA

float

Ring current at the start of the measurement

- sample_temperature_K

float

Sample temperature [K]

- detect_ice_rings

bool

Ice ring detection feature is enabled

- indexing_algorithm

string

Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none

- geom_refinement_algorithm

string

Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center

- poni_rot1

float

Tilt of the detector rot1 according to PyFAI PONI convention [rad]

- poni_rot2

float

Tilt of the detector rot2 according to PyFAI PONI convention [rad]

- poni_rot3

float

Tilt of the detector rot3 according to PyFAI PONI convention [rad]

See DECTRIS documentation for definition of Image as MultiDimArray with optional compression.

Image message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “image”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

real_time

Rational

Exposure time

X

start_time

Rational

Exposure start time (highly approximate)

X

end_time

Rational

Exposure end time (highly approximate)

X

spots

Array(object)

Spots:

- x

float

observed position in x (pixels)

- y

float

observed position in y (pixels)

- I

float

intensity (photons)

- maxc

int64

max count (photons)

- ice_ring

bool

spot in resolution range for ice rings

- indexed

bool

indexed solution

- latt

int64

Lattice to which the peak belongs (negative number = not indexed)

- image

int64

image number the spot belongs to

- h

int64

Miller index (indexed spots only)

- k

int64

Miller index (indexed spots only)

- l

int64

Miller index (indexed spots only)

- dist_ewald

float

distance to Ewald sphere [Angstrom^-1] (indexed spots only)

reflections

Array(object)

Reflections:

- h

int64

Miller index

- k

int64

Miller index

- l

int64

Miller index

- x

float

predicted position in x (pixels)

- y

float

predicted position in y (pixels)

- obs_x

float

observed position in x (pixels)

- obs_y

float

observed position in y (pixels)

- d

float

resolution [Angstrom]

- I

float

integrated intensity (photons)

- bkg

float

mean background value (photons)

- var_bkg

float

non-signal (background) part of sigma^2, carried to the merge (photons^2)

- sigma

float

standard deviation, estimated from counting statistics (photons)

- image

int64

image number the reflection belongs to

- rp

float

Distance to Ewald sphere [Angstrom^-1]

- rlp

float

Reciprocal Lorentz-polarization factor: the multiplier taking the raw integrated count toward a quantity proportional to |F|^2. Lorentz x polarization only - a still has no Lorentz term, so there it is the polarization alone

- qe

float

Sensor efficiency at the reflection’s angle of incidence, QE(0)/QE(alpha); <= 1, and 1 where the sensor is opaque or unknown. Carried beside rlp, not inside it: the total correction is rlp * qe. Optional

- flight

float

Attenuation of the reflection in the flight path between the sample and its pixel, normalised to normal incidence; >= 1, and exactly 1 for a vacuum path. Carried beside rlp and qe: the total correction is rlp * qe * flight. Optional

- partiality

float

Partiality of the reflection

- phi

float

phi angle from XDS: difference from middle of current frame, not absolute [deg]

- zeta

float

Lorentz zeta factor (reciprocal-space geometry term)

- image_scale_corr

float

Per-image scale correction; I_true = image_scale_corr * I

spot_count

uint64

Spot count

spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

spot_count_low_res

uint64

Number of spots in low resolution (prior to filtering)

spot_count_indexed

uint64

Number of spots which fit indexing solution within a given tolerance

az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

NaN is used for empty bins and has to be taken care by the receiver

az_int_profile_std

Array(float)

Standard deviation for azimuthal integration. (NaN for less than 2 samples)

az_int_profile_count

Array(uint64)

Number of pixels contributing to azimuthal bin

indexing_result

bool

Indexing successful

indexing_lattice_count

int64

Number of indexing lattices found for this image

indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

indexing_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

profile_radius

float

Profile radius of the image - describes distance of observed reflections from the Ewald sphere [Angstrom^-1]

integrated_reflections

int64

Count of integrated reflections

mosaicity

float

Angular range of spots in image from a rotation scan [degree]

b_factor

float

Estimated B-factor (Angstrom^2)

compression_time

float

Time spent on compression/decompressing image [s]

preprocessing_time

float

Time spent on preparing the image for analysis [s]

azint_time

float

Time spent on azimuthal integration [s]

spot_finding_time

float

Time spent on spot finding [s]

indexing_time

float

Time spent on indexing [s]

refinement_time

float

Time spent on refinement of indexing solution and experimental geometry [s]

index_analysis_time

float

Time spent on analyzing indexing solution, calculating profile radius and mosaicity [s]

bragg_prediction_time

float

Time spent on predicting Bragg spots [s]

integration_time

float

Time spent on Bragg integration [s]

image_scale_time

float

Time spent on on-the-fly scaling [s]

processing_time

float

Total processing time [s]

xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

lattice_type

object

Bravais lattice classification of the indexing result (present only if available)

X

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

jf_info

uint64

Detector info field

receiver_aq_dev_delay

uint64

Receiver internal delay

receiver_free_send_buf

uint64

Receiver internal number of available buffer locations

receiver_buf_in_sending

uint64

Receiver internal number of buffer locations currently in sending/writing

receiver_buf_in_preparation

uint64

Receiver internal number of buffer locations currently in processing

storage_cell

uint64

Storage cell number

saturated_pixel_count

uint64

Saturated pixel count

pixel_sum

uint64

Sum of all pixels, excl. error and saturation

error_pixel_count

uint64

Error pixel count

strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

min_viable_pixel_value

int64

Minimal pixel value, excl. error and saturation

max_viable_pixel_value

int64

Maximal pixel value, excl. error and saturation

resolution_estimate

float

Resolution the merged data are predicted to reach, from this image’s spots alone [Angstrom]

X

data_collection_efficiency

float

Image collection efficiency []

packets_expected

uint64

Number of packets expected per image (in units of 2 kB)

packets_received

uint64

Number of packets received per image (in units of 2 kB)

bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons]

ice_ring_score

float

Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

spindle_blind_fraction

float

Fraction (0-1) of a rotation sweep’s blind cone this orientation makes unrecoverable, as a lone-2-fold worst-case bound; >= 0.5 should engage a recovery protocol, and ABSENT means the frame could not be assessed, which automation must treat the same way

spot_count_ice_control

float

Spots in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands’ own q width (control for spot_count_ice_rings)

beam_corr_x

float

Beam center correction X applied during processing [pixel]

X

beam_corr_y

float

Beam center correction Y applied during processing [pixel]

X

image_scale_factor

float

Scaling result: Image scale factor (g)

X

image_scale_mosaicity

float

Scaling result: Image scale mosaicity [deg]

X

image_scale_cc

float

Scaling result: Image scale CC

X

adu_histogram

Array(uint64)

ADU histogram

roi_integrals

object

Results of ROI calculation

X

- sum

int64

Sum of pixels in ROI area [photons]

- sum_square

int64

Sum of squares of pixels in ROI area [photons]

- pixels

uint64

Valid pixels in ROI area

- max_count

int64

Highest count in ROI area [photons]

- x_weighted_sum

int64

ROI pixel X position multiplied by photon count [photons * pixels]

- y_weighted_sum

int64

ROI pixel Y position multiplied by photon count [photons * pixels]

user_data

string

Optional user defined text information - this is image_appendix serialized to JSON format

X

data

Map(string -> Image)

Image

X

Metadata message

Field name

Type

Description

Present in DECTRIS format

Optional

type

String

value “metadata”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

images

Array(object)

Array of images (order and size of the array are not guaranteed)

X

- image_id

uint64

Number of image within the series; for MX lossy compression this is sequential excluding removed frames

X

- original_image_id

uint64

Number of image within the series; for MX lossy compression this includes removed frames in the count

- real_time

Rational

Exposure time

X

- start_time

Rational

Exposure start time (highly approximate)

X

- end_time

Rational

Exposure end time (highly approximate)

X

- spot_count

uint64

Spot count

- spot_count_ice_rings

uint64

Number of spots within identified rings (experimental)

- az_int_profile

Array(float)

Azimuthal integration results, use az_int_bin_to_q from start message for legend

- indexing_result

bool

Indexing successful

- indexing_lattice

Array(9 * float)

Indexing result real lattice; present only if indexed

X

- indexing_unit_cell

object

Indexing result unit cell: a, b, c [angstrom] and alpha, beta, gamma [degree]; present only if indexed

X

Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided

- xfel_pulse_id

uint64

Bunch ID (for pulsed source, e.g., SwissFEL)

X

- xfel_event_code

uint64

Event code (for pulsed source, e.g., SwissFEL)

X

- jf_info

uint64

Detector info field

- receiver_aq_dev_delay

uint64

Receiver internal delay

- receiver_free_send_buf

uint64

Receiver internal number of available send buffers

- storage_cell

uint64

Storage cell number

- saturated_pixel_count

uint64

Saturated pixel count

- error_pixel_count

uint64

Error pixel count

- strong_pixel_count

uint64

Strong pixel count (first stage of spot finding)

- data_collection_efficiency

float

Image collection efficiency []

- bkg_estimate

float

Mean value for pixels in resolution range from 3.0 to 5.0 A [photons] (with solid angle/polarization corrections, if applied)

X

- resolution_estimate

float

Predicted merged resolution, from spots alone

X

- adu_histogram

Array(uint64)

ADU histogram

X

- roi_integrals

object

Results of ROI calculation

X

- - sum

int64

Sum of pixels in ROI area [photons]

- - sum_square

int64

Sum of squares of pixels in ROI area [photons]

- - pixels

uint64

Valid pixels in ROI area

- - max_count

int64

Highest count in ROI area [photons]

End message

Field name

Type

Description

Present in DECTRIS format

type

String

value “end”

X

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

series_unique_id

string

Unique text ID of the series (run_name parameter)

X

series_id

uint64

Unique numeric ID of the series (run_number parameter)

X

end_date

string

Approximate end date

max_image_number

uint64

Number of image with the highest number; counted from 1 to distinguish zero images and one image

transformations

Array(object) (optional)

Sample transformation chain in mounting order, base first. Each element mirrors a NeXus NXtransformations axis: name, transformation_type (rotation/translation), units, vector, offset, depends_on (the axis this one is mounted on, empty for the base), and values - a single number for an axis that does not move, otherwise one per image. An ARRAY because the order matters and a CBOR map has none. Optional: when absent the writer builds the same chain from the start message. It is in the END message because a producer may want to report positions that were measured rather than commanded, which are only known once the run is over

images_collected

uint64

Number of images collected

images_sent_to_write

uint64

Number of images sent to writer; if writer queues were full, it is possible this is less than images collected

data_collection_efficiency

float

Overall network packets collected / network packets expected

az_int_result

Map(text->Array(float))

Azimuthal integration results, use az_int_bin_to_q from start message for legend

adu_histogram

Map(text->Array(uint64))

ADU values histogram

adu_histogram_bin_width

uint64

Width of bins in the above histogram [ADU]

max_receiver_delay

uint64

Internal performance of Jungfraujoch

bkg_estimate

float

Mean background estimate for the whole run

spindle_blind_fraction

float

Run mean of the per-image spindle_blind_fraction, over the frames that had one

spindle_lost_unique_fraction

float

Fraction (0-1) of unique reflections the mounting made unmeasurable, exact under the measured point group; offline (Rugnux) only

indexing_rate

float

Mean indexing rate for the whole run

unit_cell

object (optional)

Unit cell of the system, based on the actual experiment: a, b, c [angstrom] and alpha, beta, gamma [degree]

rotation_lattice_type

object

Bravais lattice classification of the total rotation solution over the run, if available; same schema as lattice_type

- centering

string

One-letter centering code: P, A, B, C, I, F, or R

- niggli_class

int64

Integer identifier for the Niggli-reduced Bravais class

- system

string

Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic

rotation_lattice

Array(9 * float)

Real-space lattice basis, flattened 3x3 in row-major order

rotation_extra_lattices

Array(Array(9*float))

Additional indexed lattices (orientation variants); present only if found

data_collection_efficiency_image

Array(float)

Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context

spot_count

Array(int32)

Per-image spot count

spot_count_ice_ring

Array(int32)

Per-image number of spots within identified ice-ring resolution ranges

key is singular here; the per-image message uses spot_count_ice_rings

spot_count_low_res

Array(int32)

Per-image number of low-resolution spots

spot_count_indexed

Array(int32)

Per-image number of spots fitting indexing solution

image_indexed

Array(uint8)

Per-image indexing result; 0 = not indexed, nonzero = indexed

v_bkg_estimate

Array(float)

Per-image background estimate

v_spindle_blind_fraction

Array(float)

Per-image spindle_blind_fraction; NaN where the frame had no value (which is “cannot say”, not zero)

ice_ring_score

Array(float)

Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice)

spot_count_ice_control

Array(float)

Per-image spot count in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands’ q width

ice_ring_score_mean

float

Mean ice-ring score for the whole run (1 = no ice)

profile_radius

Array(float)

Per-image profile radius [Angstrom^-1]

mosaicity

Array(float)

Per-image mosaicity [degree]

bFactor

Array(float)

Per-image estimated B-factor [Angstrom^2]

resolution_estimate

Array(float)

Per-image predicted merged resolution, from spots alone [Angstrom]

min_viable_pixel_value

Array(int64)

Per-image minimum valid pixel value, excluding error/saturated pixels

max_viable_pixel_value

Array(int64)

Per-image maximum valid pixel value, excluding error/saturated pixels

saturated_pixel_count

Array(int32)

Per-image saturated pixel count

error_pixel_count

Array(int32)

Per-image error pixel count

image_scale_factor

Array(float)

Per-image scale factor, if scaling/merging was performed

integrated_reflections

Array(int32)

Per-image count of integrated reflections

indexed_lattice_count

Array(int32)

Per-image count of indexed lattices

niggli_class

Array(uint8)

Per-image Niggli class identifier for indexed images; 0 if unavailable

pixel_sum

Array(int64)

Per-image sum of all valid pixels, excluding error/saturated pixels

image_scale_mosaicity

Array(float)

Scaling result: Image scale mosaicity [deg]

image_scale_cc

Array(float)

Scaling result: Image scale CC

End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted.

Calibration message

Field name

Type

Description

Present in DECTRIS format

type

String

value “calibration”

magic_number

uint64

Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver

data

Map(string -> Image)

Calibration map (only single pedestal array per message)

User data

Facilities often need to forward more metadata than Jungfraujoch models explicitly. For this reason two fields can be provided: header_appendix (sent with the start message) and image_appendix (sent with the image message). To increase flexibility, both appendices can contain any valid JSON message. These appendices are serialized into string and stored in CBOR messages as user_data.

Notably for start message, user_data can contain more information (non-DECTRIS compliant metadata). Therefore user_data is serialized by Jungfraujoch as CBOR object. There is member user which contains header_appendix defined in OpenAPI of Jungfraujoch.

Notes on images and compression

  • Images are encoded as DECTRIS MultiDimArray with typed array tags:

    • For RGB: shape [3, height, width], type: u8

    • For grayscale: shape [height, width], type according to bit depth and sign (e.g., uint16 LE)

  • Compression:

    • Uncompressed: raw CBOR byte string

    • Bitshuffle+LZ4: tag with [“bslz4”, elem_size, bytes]

    • Bitshuffle+Zstandard: tag with [“bszstd”, elem_size, bytes]

Notes on typed arrays

Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays.

Common tags used in this protocol include:

  • float32 little-endian arrays for Array(float)

  • uint8 arrays for compact boolean/integer flags such as image_indexed

  • int32 little-endian arrays for per-image counts

  • int64 little-endian arrays for large per-image integer values

  • uint64 little-endian arrays for histograms

\ No newline at end of file diff --git a/CHANGELOG.html b/CHANGELOG.html new file mode 100644 index 000000000..26789b3d8 --- /dev/null +++ b/CHANGELOG.html @@ -0,0 +1 @@ + Changelog — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Changelog

1.0.0

1.0.0-rc.174

  • Rugnux: Performance improvements on GPU and CPU (more of the pre-scan and of scaling on the GPU, faster CPU spot finding and crystal refinement), with unchanged results.

  • Rugnux: More robust processing - patches of persistently hot pixels are masked, an inconsistent merge triggers a retry at the measured beam centre, and builds targeting different CPU levels give the same results.

  • Rugnux: Improved scaling and merging - reflections with an overloaded pixel are dropped, as in XDS, sparse rotation sweeps are scaled more reliably, and French-Wilson amplitudes use an anisotropic Wilson prior.

  • Rugnux: Improved space-group determination - glide planes in groups without a centre of symmetry, screw axes from short or weak axial rows kept when a higher group is adopted, and more reliable decisions on twinned and pseudo-symmetric crystals.

  • Rugnux: Improved small-molecule processing - spots that grow wider than the integration disk and split spots are integrated over their measured footprint, sparse lattices are integrated on every frame, and the .hkl file holds unmerged scaled reflections (SHELX HKLF 4).

  • Rugnux: Reads Rigaku d*TREK SMV images (Saturn CCD), including detector 2theta and encoded pixel overflows; home-source (rotating-anode) datasets were added to the validation battery.

  • jfjoch_viewer: Fixed processing failing at the end with “Wrong JPEG library version” on Linux; the merge window shows the space group with proper subscripts and a checklist of crystal pathologies.

1.0.0-rc.173

  • jfjoch_broker: Optional per-dataset authentication - statistics, images and plots can require a bearer token, which jfjoch_viewer supports.

  • jfjoch_viewer: Dark mode and a theme-matched colour scheme, a magnifier panel, and simpler contrast and background controls.

  • Rugnux: Multiple performance improvements on GPU and CPU (CPU-only processing up to 40% faster, faster image decoding on ARM), with unchanged results.

  • Rugnux: --model rigid-body refinement runs on the GPU, and the model-validation check is faster and more reliable.

  • Rugnux: Improved scaling and merging - error model, outlier rejection, absorption correction and French-Wilson amplitudes now agree more closely with XDS and ctruncate.

  • Rugnux: Improved integration - radial background on powder and ice rings, crowded rotation data keep their reflections, and CPU-only builds integrate large unit cells as GPU builds do.

  • Rugnux: More robust detector geometry - measured beam centre, X-ray bandwidth and goniometer rate, and geometry refinement accepted only on significant evidence.

  • Rugnux: Merged files are written in the standard setting, or in the setting of a reference MTZ, structure-factor mmCIF or model, with its free-R flags.

  • Rugnux: Richer report - ice and powder rings, further lattices, superstructure candidates and mosaicity, with warnings worded as prompts to check.

  • Rugnux: Clear error messages when a data set needs more GPU or host memory than is available.

1.0.0-rc.172

  • Fixed jfjoch_broker cancelling every data collection with a CUDA “out of memory” error after long operation: GPU memory no longer leaks with each collection.

  • Rugnux scales a rotation sweep until the per-frame scales settle instead of for a fixed three rounds, and says so when they did not - merged intensities, and the space group, resolution cut and frame rejection read off them, change accordingly; --scaling-iterations is now the cap on that loop (default 100).

  • Rugnux places every frame of a marCCD, SMV or miniCBF series at the spindle angle its own header states, so a series with missing frames, or with angles written modulo 360, is no longer read at the wrong geometry or refused.

  • Every rotation run writes two diagnostic files beside its reflections: <prefix>_detector.jpg, the detector projection with the pixel mask and the detected beam-stop shadow drawn on it, and <prefix>_plot.txt, one row per image.

1.0.0-rc.171

  • Rugnux: basic support for CCD images (marCCD, SMV) and for gzipped miniCBF.

  • jfjoch_viewer: opens the CCD formats, and fixes to the dataset plots.

  • Documentation updates.

1.0.0-rc.170

  • Fixed a jfjoch_broker crash during indexing: sorting no longer misbehaves on non-finite values, and GPU FFT indexer kernel launches are now error-checked.

  • Rugnux needs about 40% less peak memory to scale, merge and post-refine rotation data, with identical results.

  • rugnux --model: the placed coordinate file carries the space group its own coordinates obey, and says so when that is not the group the reflection files beside it carry.

  • jfjoch_viewer: fixes in the dataset plots, inspector and layout; spot markers lose their black outline by default (a checkbox under “Image features” restores it) and the highest-pixel markers are white boxes around the pixel.

1.0.0-rc.169

  • Building Jungfraujoch no longer needs zlib or Eigen installed on the machine, and the dependencies the build fetches are pinned and updated to current releases.

  • Rugnux: improvements in indexing, lattice selection and geometry post-refinement, which index crystals that previously returned no lattice and keep the better of the two geometries a run measures.

  • Rugnux: improvements in beam-centre measurement, beam-stop detection and space-group determination.

  • Rugnux: the unit cell reported with a determined space group now obeys that group - a cell whose symmetry was confirmed from the intensities is re-refined under it, and a cell the group cannot describe is reported with a warning rather than as it stands.

  • Rugnux drops the stretches of a rotation sweep whose removal measurably improves the merged intensities and reports what became of every frame, and decides the resolution cut on the crystal’s own diffraction rather than on its ice rings.

  • The Rugnux results report is machine-readable - every line that is not KEY= value data starts with # - and states the build it was written by, its authorship and its terms of use (REPORT_VERSION= 8).

  • jfjoch_viewer: improvements in the file manager (CBF frames beside HDF5 datasets, a remembered root), the dataset plots, the inspector and the image statistics, plus a settable font size, a view of the Rugnux results report, usable performance over a remote display (ssh -X) and a reset of all settings to defaults; the reciprocal-space window is removed.

  • Broker fixes around DECTRIS collections and dark-mask calibration: re-initialising after a run that never started no longer freezes the broker, a cancelled calibration is abandoned instead of reported as done, and a collection whose start message never arrives ends by itself.

1.0.0-rc.168

  • Rugnux is substantially faster - a corpus of 145 rotation datasets processes in about two thirds of the time - with identical results.

  • A crystal whose lattice looks more symmetric than it is because the beam centre is off is no longer processed on the wrong cell.

  • Rugnux prints at startup, and writes at the foot of every results report, a short acknowledgement of the X-ray research community whose methods it implements and of the open-source projects it builds on; ACKNOWLEDGEMENT.md now ships in every package beside LICENSE and THIRD_PARTY_NOTICES.md.

1.0.0-rc.167

  • rugnux --model reports CC(model, data) - the correlation of the merged intensities with the placed, scaled model - by resolution shell, on the same shells as CC1/2, with the reflection count and a significance for each.

  • rugnux --model fits the model’s scale, anisotropic B and bulk-solvent parameters on the working reflections only, so the R-free it reports is measured against a model no free reflection helped scale.

  • The bulk-solvent parameters of rugnux --model are searched over their physically meaningful range instead of being fitted without bounds, so a model is never scaled with a solvent term that has silently switched itself off.

  • The rigid-body placement of rugnux --model uses the same bounded bulk solvent as the reported fit, so a model is no longer placed against a target carrying a solvent term with no physical meaning.

  • rugnux --model puts the model into the data’s own description of the lattice before placing it, so a model whose cell is written on other axes - I-centred where the run indexed C-centred, a different unique axis, a permuted orthorhombic cell - is placed rather than scored where it was read; MODEL_CHANGE_OF_BASIS= and MODEL_SETTING_AS_READ= report it when it happens.

  • The Rugnux results report opens with a summary - VERDICT= (OK, WARNINGS, UNUSABLE, FAILED), VERDICT_TEXT=, PATHOLOGY_FLAGS= with one closed-vocabulary code per condition that warned, and the WARNING: lines, which used to close the file - and the sections after it are renumbered 1-5 with no gaps.

  • rugnux --developer writes the full results report - the pipeline-internal keys and the long explanations the default report now leaves out - and --finalist-ledger adds the evidence for every space group the search considered, not only the one it adopted.

  • The results report warns when the merged data carry no usable signal and when too little of reciprocal space was measured inside the fitted resolution, and omits FITTED_RESOLUTION where the CC1/2 curve it is fitted on never falls off.

  • Rugnux detects translational pseudo-symmetry and reports it under the PSEUDO_TRANSLATION flag as TNCS_DETECTED= and the TNCS_* keys - a translation the merged data are exactly invariant under is reported as UNDECLARED_LATTICE_TRANSLATION= under LATTICE_TRANSLATION instead - and a detected pseudo-translation can no longer buy a false screw axis in the space-group search or hide a twin from the L-test (L_TEST_VS_TNCS=).

  • The space-group search determines glide planes from zonal systematic absences, so a non-Sohncke space group such as P 2_1/c or Pbca is named where the run previously stopped at its Sohncke subgroup; SOHNCKE_SPACE_GROUP= carries the best Sohncke group beside it on every run that searched, and a centre of symmetry is never claimed.

  • Where the cell metric carries more rotational symmetry than the Bravais class the indexer named, the extra rotations are put to the intensities and the space-group search is asked again on the metric’s own cell - adopted only where the intensities confirm the higher symmetry - so a lattice that is nearly but not exactly hexagonal, or whose reduction landed in a sub-cell, still reaches its true point group.

  • Systematic-absence calls rest on the evidence rather than on counts: a screw axis whose absent class the data show extinct is no longer refused because a handful of reflections in it read as present, and SPACE_GROUP_ALTERNATIVES= no longer drops a candidate that differs only on a zone the sweep never measured.

  • A reference correlation measured on too few reflections is refused instead of scored zero, so a run given a reference MTZ is no longer reindexed on an operator that mapped almost everything outside the reference’s coverage.

  • A frame counts as indexed from 6 spots on its lattice rather than 9, so a weakly diffracting crystal whose frames cannot carry 9 is no longer refused the lattice it fits; --min-indexed-spots overrides it.

  • -C accepts a known cell in any equivalent description - conventional or primitive, centred or not - instead of only the reduced primitive form, so a centred cell given the way it is published no longer makes the run report that it found no lattice.

  • Each reflection is corrected for the sensor’s quantum efficiency at the angle it meets the detector (attenuation lengths from the NIST tables, which also fixes the spot-width parallax term on CdTe) and for the attenuation of the flight path between the sample and its pixel; --flight-path air|helium|vacuum declares the medium - default air, since no file states it - and the report says what was assumed and what it was worth. The unmerged MTZ records the factors in new QE and FLIGHT columns beside LP, so raw counts are I / LP * QE * FLIGHT, and _process.h5 in new optional qe and flight datasets.

  • Rotation geometry post-refinement fits the crystal and the detector at once, against the observed spot positions and the observed rocking angles together, so the refined distance depends far less on how wrong the file’s distance was.

  • A coarsely sliced sweep integrates correctly: partials are joined into one rocking event by angle rather than by frame count, so two crossings of the Ewald sphere are no longer summed into one full, and at 0.5 degrees per image or coarser the per-frame geometry refinement accepts a spot whose miss the exposure’s own rotation accounts for.

  • rugnux --mode scale reports the detector tilt and direct beam of the geometry it re-scaled at, instead of zeros that read as a flat detector, and no longer warns that no image was indexed on a run whose lattice came from its input file.

  • Every rotation run that determined a space group and merged reports what the mounting cost: SPINDLE_LOST_UNIQUE_FRACTION= is the fraction (0-1) of unique reflections the mounting made unmeasurable under the measured point group, also written to the master as /entry/MX/spindleLostUniqueFraction and what the mounting warning fires on; SPINDLE_SYMMETRY_AXIS_ANGLE_DEG= / SPINDLE_SYMMETRY_AXIS_ORDER= describe the mounting in the --developer report.

  • Stills and grid scans carry a per-image spindle_blind_fraction - how much of a rotation sweep’s blind cone this orientation would make unrecoverable, 0.5 and above calling for a second orientation - through the CBOR stream, HDF5 (/entry/MX/spindleBlindFraction), the plot and scan-result APIs, and the viewer and frontend plots; an absent value means the frame could not be assessed and is not a 0.

  • jfjoch_viewer gains Help entries for the mouse shortcuts and the acknowledgements, and Inspector toggles that hide non-indexed spots and spots on ice rings.

  • The results report’s REPORT_VERSION is 7.

1.0.0-rc.166

  • rugnux --model treats the model as a hypothesis: it decides the enantiomorph and the indexing only where its R-work beats that of the same model in random orientations, and a model the data reject is still scored, placed and mapped, but leaves the reflection files byte for byte what a run with no model writes.

  • rugnux --model places the model against the data as a rigid body before scoring it, writes sigma_A-weighted 2mFo-DFc and mFo-DFc maps in place of the unweighted 2Fo-Fc and Fo-Fc, and writes the model as it was placed - <prefix>_model.cif, and <prefix>_model.pdb where the PDB format can express the cell - in the cell and space group of the reflection files beside it.

  • rugnux and jfjoch_viewer read PILATUS miniCBF sweeps natively, and open masters written at other facilities, including Eiger 1.x and third-party NXmx.

  • rugnux determines the lattice and the space group more reliably - the true cell where the first pass offers a whole-number multiple of it, so a pseudo-translated crystal keeps its full-length axis and a small molecule is indexed on its own cell rather than a protein-sized one, and the point group, the setting and the systematic absences - and -S refuses or re-seats a fixed space group whose symmetry axes the indexed cell does not carry.

  • rugnux measures the beam centre on every run and indexes with it when the file’s value indexes nothing, refines only the detector-tilt component the data determine - a beam-centre error is no longer reported as a tilt - and places a detector swung out on a 2theta arm where the file says it stands.

  • rugnux writes the unmerged MTZ by default, with a P1 merge beside it, a batch header for every image the observations span, and events kept to the same --min-captured-fraction as the merge, so a wrong space group can be re-merged in a scaling program without reprocessing.

  • rugnux writes reflection files in the conventions downstream programs read: FreeR_flag is 0 for the test set and 1 for the working set - it was the other way round - the merged and P1 MTZ carry the reserved HKL_base dataset so a CCP4 program reads the wavelength instead of falling back to 1.54187 A, and the merged mmCIF marks the free set as _refln.status = f.

  • The Rugnux results report carries the space groups the data cannot separate and the enantiomorph state, the model’s verdict and what it was allowed to decide, the detector geometry measured and what a single sweep cannot determine, the resolution the CC1/2 fit reached, which reciprocal axis each anisotropic diffraction limit belongs to, and twinning measured before and after the space group was decided; REPORT_VERSION is 6, and SPACE_GROUP_ENANTIOMORPH= DETERMINED_FROM_MODEL is now ASSUMED_FROM_MODEL.

  • rugnux --mode calibration writes <prefix>.json beside the .poni, holding the geometry as a jfjoch_broker dataset_settings body, and refuses a fit that is not a measurement - no .poni, a non-zero exit, converged recorded in the .json; --no-refine-tilt holds the detector tilt at the file’s value instead of zeroing it.

  • A snake grid scan with a negative slow step and an even number of rows no longer has its positions mirrored along the fast axis in the HDF5 master and the grid map, so the positions recorded for that configuration change; jfjoch_viewer draws grid scan cells in the proportion of the scan steps, labels the merge-statistics plot over the range the axis is drawn on, and builds its powder-calibration ring list from the loaded dataset’s space group as well as its cell, so a centred sample cell no longer scales the whole fit.

  • The HDF5 master records direct_beam_x/direct_beam_y - where the undeflected beam lands, sent on the CBOR start message too - the beam size at the sample as incident_beam_size from the new dataset_settings beam_size_x_um/beam_size_y_um, and /entry/MX/peakCountUnfiltered; dataset_settings accepts any smargon.chi_deg, which was restricted to 0-90 degrees.

  • Reported completeness counts the reflections the beam stop, a detector mask or the low-resolution limit kept out of the merge as missing: the denominator, and the resolution shells it is binned into, now span the run’s declared resolution range rather than the range of the reflections that survived, so the innermost shell boundary moves and its numbers are not comparable with those of an earlier release.

  • The Rugnux results report carries CC_ANOM beside SIGANO, overall and per shell: the anomalous difference measured from one half of the observations correlated against the same difference from the other half, which says whether there is an anomalous signal to phase on without depending on the error model. Reported on Friedel-merged runs too; it matches AIMLESS and phenix, and XDS’s similarly named Anomal Corr is a different quantity.

  • A quantity a Rugnux run did not measure is left out of the results report altogether instead of being written as nan - SIGANO= on a Friedel-merged run, which is the default, is the case a script meets first - and a merging-statistics shell prints - in its place.

  • FITTED_RESOLUTION= in the results report is the resolution fit of the run’s own merge rather than of the P1 cross-check, and the unmerged MTZ is written in the space-group setting the run adopted rather than in that setting’s reference one.

  • rugnux --mode calibration refuses to write a .poni for a detector whose stored image is mirrored or turned by a multiple of 90 degrees, the format having no field for it.

  • CC1/2 is computed on half-sets of equal size, so every reflection measured more than once contributes to it, as in XDS and phenix.merging_statistics, and a CPU-only build now reports the same value as a CUDA one; reported CC1/2 values move slightly, most where multiplicity is low.

  • The Rugnux manual is reorganised into task pages with a run overview and worked phenix / REFMAC5 / Phaser / SHELXC/D/E / POINTLESS-AIMLESS / careless examples, and the HDF5 and API documentation say how a grid scan records the angle its spindle stood at: the goniometer axis with a step of 0.

1.0.0-rc.165

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • rugnux --model adopts the model’s space group as a label where the data were merged in its enantiomorph, instead of reindexing the reflections - which swapped I(+) with I(-).

  • rugnux --model warns, naming the atom, when the anomalous density at the model’s atoms comes out inverted, which means the data and the model are in opposite hands.

  • rugnux --model writes an anomalous difference map (<prefix>_anom.ccp4) when the merge kept the Bijvoet split, and names the ten model atoms it peaks highest on as ANOMALOUS_SITE_01.._10.

  • MEAN_ATOM_DENSITY_SIGMA is read from the map by cubic rather than linear interpolation and comes out around a tenth higher; it is no longer comparable with the figure earlier versions printed.

  • rugnux --model reads an mmCIF coordinate file as well as a PDB one, gzipped or not, taking the format from the file’s content rather than its name.

  • A model rugnux --model cannot use is reported as a WARNING: line in the results report instead of only in the log.

  • The Rugnux results report has a 10. MODEL VALIDATION section when --model was given; REPORT_VERSION is 4, WARNINGS moves to section 11 and no existing key changed.

  • The Rugnux results report records how the run was invoked, what it cost and what it ran on: COMMAND_LINE=, WALL_TIME= and GPU_COUNT= / GPU=.

  • Rugnux says which GPUs it can see before it starts processing.

  • rugnux --export-unmerged also writes <prefix>_unmerged.mtz on a --no-merge run, and is ignored on a run with no output prefix instead of writing a file called _unmerged.mtz.

  • /start asks the writer whether the run can be written before the detector is armed, so a run whose master file already exists, or whose output directory cannot be created, is refused up front with the writer’s own message. This needs the TCP image stream or the built-in HDF5 writer; the ZeroMQ stream is unchanged.

  • A calibration that fails goes to Error carrying the reason instead of Inactive, so /wait_till_done and /wait_until_running report it; a cancelled calibration still goes to Inactive.

  • /wait_till_done answers 500 with the message when a collection ended in an error. A cancelled collection and a collection that only triggered a warning still answer 200.

  • A pending start failure is discarded by /cancel and /deactivate, as it already was by /start and /initialize.

  • /scan_result no longer reports the previous run’s images after a collection that failed to start, or after /deactivate.

  • The TCP image stream protocol version is 4. jfjoch_writer and jfjoch_broker have to be of the same release, as before.

1.0.0-rc.164

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Rugnux now tells you whether a crystal diffracts anisotropically and how far it reaches in each direction, without a second program: a new 9. DIFFRACTION ANISOTROPY section in <prefix>_report.txt and matching _reflns.pdbx_aniso_B_tensor_* / _reflns.jfjoch_aniso_* items in the merged mmCIF report the anisotropic deltaB, the diffraction limit along each principal direction, and a NOT DETECTED / DETECTED / CANNOT DETERMINE verdict measured against the data set’s own systematic error. It is a description only - no intensity is corrected, no reflection is removed, and the merged data do not depend on direction.

  • Rugnux can hand its integrated observations to another scaling program: --export-unmerged writes <prefix>_unmerged.mtz, an unmerged MTZ readable by aimless, pointless, careless and iotbx.merging_statistics, in --mode mx and --mode scale alike. Each rotation reflection’s partials are summed into one full; --export-unmerged-partials writes one row per image instead. Intensities carry the Lorentz-polarization factor and nothing else, since those programs scale the data themselves. Lattice-centring absences are not written; screw and glide absences are.

  • Rugnux integrates crystals with broad spots better - where it changes anything, per-shell mean I/sigma improves by up to 31% and R_meas by up to 24% - because on rotation data the integration signal radius is now taken from the crystal’s own measured spot width instead of a fixed 4 px. --adaptive-integration-radius=off restores the fixed radius and an explicit --integration-radius still overrides both. The widened radius applies to the final integration pass only, and a pattern too dense for it is re-integrated at 4 px with a note in the log.

  • Rugnux discards fewer stills reflections for want of a background ring, improving per-shell R_meas over most of the signal-bearing range: the stills background ring now runs to 14 px instead of 12. The gain reverses in shells below a mean I/sigma of about 4.

  • Rugnux determines the space group with thresholds that mean the same thing on a weak crystal as on a strong one: symmetry operators are scored on resolution-normalised intensities (E squared) instead of raw merged intensities, and a reflection counts as genuinely present on its counting significance instead of on the merged I/sigma, which saturates at the merge’s own ISa. The search resolution cut is no longer able to move the answer, and the twin-law H bound moves from 1.70 to 1.85, which stops one class of correct high-symmetry assignment being refused as twinning.

  • Rugnux says what the space-group search tested and what it could not: the twin-law disagreement H is printed for every operator together with the adopted point group’s H ratio and its bound; alternatives that are not on the reported lattice are named with how their cell differs; and a lattice centring the data could not test - the crystal having been integrated on the primitive sub-cell, so the reflections it extinguishes were never measured - is marked UNTESTED and warned about where it is adopted, as coming from the lattice metric rather than from the intensities.

  • Rugnux --mode scale re-merges a _process.h5 in the right symmetry without being told it: the file now records the space group on every run - a two-pass rotation run wrote none before, so re-merging defaulted to P1 - together with the change of basis under /entry/MX/reindexMatrix where the lattice was re-seated, and --mode scale also reports the Wilson B-factor estimate instead of WILSON_B= nan. A file written before this stops with a message naming the two cells and the override to use, instead of failing inside the merge. A third-party reader of a _process.h5 must apply reindexMatrix where it is present.

  • Rugnux installs on its own, as a package called rugnux - dnf install rugnux or apt install rugnux - instead of arriving inside jfjoch-viewer. It pulls in none of the acquisition stack, so a machine that only processes data no longer has to carry the broker, the detector libraries or Qt to get it. Installing it over a jfjoch-viewer from rc.163 or earlier, which still owns /usr/bin/rugnux, upgrades cleanly rather than failing on the duplicate file.

  • Rugnux is also a standalone download, built for arm64 as well as x86_64: rugnux-<version>-linux-{x86_64|aarch64}-cuda<major>.tgz and rugnux-<version>-win64-cuda<major>.zip on the release page, for machines that are not managed by a package manager. The aarch64 build targets GH200 and DGX Spark, and is untested on hardware.

  • Every portable Linux binary is now a single self-contained file: cuFFT is linked statically instead of being shipped beside the executable and found through an rpath, so rugnux and jfjoch_viewer need nothing but an NVIDIA driver, and only to use the GPU. The .rpm/.deb continue to take cuFFT from the distribution. The developer utilities jfjoch_extract_hkl and jfjoch_recompress are no longer packaged anywhere.

  • Jungfraujoch needs six fewer shared libraries on the machine - libopenblas and libmetis, and libgfortran, libquadmath, libgomp and libz behind them - because the Ceres LAPACK, METIS and SuiteSparse back-ends are no longer built. Nothing in the code ever selected them, and results are unchanged.

  • The PCIe driver DKMS package builds for the kernel it is being installed for instead of the running one, so a module built while a kernel update is being applied loads after the reboot.

  • The PCIe driver builds on RHEL 9.5 and later, and on their CentOS Stream, Rocky and AlmaLinux equivalents, where the vm_flags kernel interface was backported into the 5.14 kernel.

  • A data collection started with async_start that fails to start - a writer refusing to overwrite an existing file, for instance - is reported as an error by /wait_until_running and /wait_till_done instead of as a timeout and a successful collection respectively. The error message is the one the writer gave.

  • A calibration that is cancelled or that fails to collect its pedestals is no longer reported as a successful one. The broker goes to Inactive with an error message and has to be initialized again, instead of sitting in Idle looking ready to measure while holding partial pedestals - data collected in that state was silently mis-converted.

  • A failed /initialize is reported to /wait_until_running and /wait_till_done as soon as it happens, instead of when their timeout expires.

  • space_group_number accepts space groups up to 230 in the API schema, so cubic space groups can be recorded. The broker always accepted them; the generated clients rejected them before the request was sent.

  • The results report’s REPORT_VERSION is 3, two sections having been added. Existing key names and table columns are unchanged.

  • The merged statistics table has 9 resolution shells instead of 10, which is what XDS reports. The bins were already XDS’s - equal steps in 1/d^2 between the lowest- and the highest-resolution reflection the merge kept - so at the same resolution limits the two tables now have the same shell boundaries and can be read row for row. --resolution-shells sets a different count.

  • rugnux --model now settles the frame the merged reflections are written in, not only the frame the R-factors and the maps are computed in: the .mtz/.cif/.hkl come out in the model’s indexing, and where the data were merged in the model’s enantiomorph they take the model’s hand and space group - which on anomalous data puts I(+) and I(-) the right way round. The indexing choice is logged with the winning R-free and the runner-up, so a decision made within noise is visible.

  • rugnux --model can resolve the indexing ambiguity of a serial stills run, which a model could not do before: structure factors computed from the model become the per-image reference, the same role a reference MTZ plays. It needs the cell and space group up front (-C / -S). Without one or the other, a merohedral serial run still merges both hands together and says so.

  • The Rugnux documentation opens with a quick start - the default run, and runs with a reference MTZ, with a model, or with the space group and cell pinned - and explains the indexing ambiguity: what it costs on rotation and on serial data, and which of -z / --model resolves it in each case. The long reference pages now carry a table of contents.

1.0.0-rc.163

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Packaging: each package installs its license notices under a directory of its own - share/doc/jfjoch_broker, share/doc/jfjoch_writer, share/doc/jfjoch_viewer, share/doc/jfjoch_driver_dkms - instead of all of them into the shared share/doc/jfjoch, so jfjoch and jfjoch-writer can be upgraded one at a time instead of only together.

1.0.0-rc.162

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

Files written by Jungfraujoch now import correctly in DIALS, XDS and pyFAI. A tilted detector, a grid scan, a still recorded at a goniometer position, and saturated or unreadable pixels were each described in a way that a third-party program acted on wrongly. If you process Jungfraujoch data outside Jungfraujoch, prefer this release to any earlier one.

  • HDF5: the detector tilt (rot1/rot2/rot3) is exported correctly in the NXmx transformation chain; untilted geometries are unaffected.

  • HDF5: a still recorded at a goniometer position is no longer read back as a single image, and a grid scan records a stationary spindle so a program that requires a rotation axis can open it.

  • HDF5: the sample transformation chain is written in mounting order, with a Smargon head position told apart from the spindle, one entry per image, module_offset as a float unit vector, and offset_units on every offset.

  • HDF5: saturated, underloaded and unreadable pixels are described so a downstream program masks them - saturation_value, underload_value, error_value and bit_depth_readout are written correctly, and a data file missing next to a VDS master reads as the error marker rather than as zero counts.

  • HDF5: the rotation axis is read back under whatever name it carries, and mirror_y records whether the assembled image is mirrored in Y relative to the detector’s raw readout.

  • A grid scan and a goniometer axis can both be set; they are no longer alternatives.

  • images_per_file is chosen from the acquisition when it is not given: a rotation sweep of at most 20000 images goes into a single data file, a grid scan splits on whole fast-axis rows, and stills and serial keep 1000.

  • The writer refuses a stream whose start message declares a different pixel format than its images carry, and a DECTRIS detector sending signed images is no longer declared unsigned.

  • The image stream can carry the sample transformation chain (transformations, in the END message); a producer that does not send it gets the same chain built by the writer.

  • Rugnux: fixing the space group with -S no longer prevents the lattice from being found - a lattice indexed in a different setting is reindexed into that group’s own setting, and a run whose crystal does not have that group’s lattice stops and names the cell it indexed as, rather than reporting statistics that cannot describe it.

  • Rugnux: the per-image resolution estimate now predicts the resolution the merged data reach rather than the highest-resolution spot found, and is reported as SPOT_RESOLUTION_ESTIMATE.

  • Rugnux: two runs of the same command on the same images produce the same merged intensities; the azimuthal profile written alongside them is not yet reproducible in the same way.

  • Rugnux: the offline lattice refinement is bounded by iterations rather than by a wall clock, so a loaded machine can no longer refine to a different lattice; a live acquisition keeps its real-time bound.

  • Rugnux: the detector-frame modulation correction is fitted on a grid spanning the detector, so whether it is applied no longer depends on how far integration reached.

  • Rugnux: the geometry pre-pass no longer writes <prefix>_01.mtz, _01.cif, _01.hkl and _01_image.dat; the refined second pass writes those files under <prefix>, and that is the result to use.

  • Rugnux: _process.h5 describes the pixel format of the images it links to, and is written on a thread of its own.

  • Rugnux: the detector geometry is also logged in XDS’s convention (ORGX/ORGY, detector axis vectors, rotation axis), so it can be compared with an XDS refinement.

  • Rugnux: an image integrated in pyFAI through the .poni file written by --mode calibration comes out with the correct azimuth, and the file declares pyFAI’s orientation, which needs pyFAI 2024.01 or newer. Radial integration is unchanged.

  • Rugnux: a rotation run is substantially faster throughout - beam-stop detection, first-pass indexing, geometry refinement, integration, scaling and merging - and observations outside the scaling resolution range are dropped as they are ingested. The refined geometry, the space group chosen and the merged statistics are unchanged.

  • Faster spot finding and indexing, on the broker as well as in Rugnux; the spots found and the lattices indexed are unchanged.

  • A run reserves substantially less GPU memory: nothing is allocated for buffers that are never read, and a worker builds only the engines it uses.

  • Rugnux: with -N left at its default the per-image loop of --mode mx uses at most 16 workers per GPU, rather than one per hardware thread; an explicit -N is obeyed as given.

  • CUDA 12 builds now contain device code for Volta, so the RHEL 8 packages and the portable Linux .tgz run on a V100; the CUDA 13 artefacts (RHEL 9, Ubuntu, Windows) remain Turing and newer.

  • The build resolves a single Eigen for the whole project, and refuses to configure if Ceres picks up a different one; a build that mixed two Eigen versions was undefined behaviour and crashed at -O2.

  • Documentation: a security page, and the supported GPU generations and minimum NVIDIA driver version of every released artefact.

Breaking change to OpenAPI - regenerate the client (jfjoch-client 1.0.0-rc.162, frontend/src/client):

  • dataset_settings.images_per_file is no longer default: 1000 and no longer accepts 0; it is optional, and its minimum is 1. A client sending 0 (previously “one file for the whole run”) is now rejected - omit the field instead, which for a rotation sweep gives the same single file.

  • file_writer_format now defaults to NXmxVDS, matching the server’s own default and the layout recommended for DIALS, XDS and CrystFEL. A generated client that fills in schema defaults and does not set the format explicitly will write VDS masters where it previously wrote legacy ones; set NXmxLegacy explicitly to keep them.

1.0.0-rc.161

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Rugnux: significantly better quality of results, and faster. A large rework of integration, scaling, merging, geometry refinement and space-group determination, together with measurements the program previously made no attempt at - the direct beam before indexing, the beam stop, the goniometer rotation scale, and the stretches of a sweep the crystal did not deliver. A rotation dataset typically gains observations at better <I/sigma> and R_meas, and every mx and scale run writes a <prefix>_report.txt results report modelled on XDS’s CORRECT.LP. Many defaults moved with it: spot detection is self-calibrating, beam-stop detection and rotation geometry post-refinement are on, resolution limits default to as far as the detector reaches, and ice-ring handling engages only where the crystal is measured to have ice.

  • jfjoch_viewer: the beam-stop shadow, the detector calibration and the beam-centre measurement are reachable from “Analyze dataset”; the settings panel reports how the sample moved and how polarized the beam was; image rendering and interaction are faster.

  • Performance: bitshuffle+LZ4 images are decoded on the GPU rather than on the host, with the bitshuffle inverse fused into preprocessing so the decompressed frame is never held in device memory.

  • Broker, writer, packaging and build: image-slot lifetime and locking fixes, per-image datasets sized by the images actually written, the Debian/Ubuntu broker package renamed to jfjoch, and image_analysis compiling under MSVC again.

Breaking change to the Rugnux command line:

  • --azint-only and --scale are removed, replaced by --mode azint and --mode scale; the full pipeline is --mode mx and remains the default. A script passing the old flags now fails with the list of valid modes rather than silently running the wrong one.

  • -t/--stride is refused on rotation data: skipping frames cuts every reflection’s rocking curve, so the combined fulls and their partiality would be measured over frames the sweep never recorded. Select a contiguous range with -s/-e instead. --mode azint and --force-still still take a stride.

Breaking changes to OpenAPI - regenerate the client (jfjoch-client 1.0.0-rc.161, frontend/src/client) or read the affected fields as optional:

  • image_scale_b is removed from the plot_type enum, so a client requesting that plot now gets an error rather than a curve.

  • azim_int_settings.high_q_recipA, spot_finding_settings.high_resolution_limit and spot_finding_settings.low_resolution_limit are no longer required. All three mean “no limit at that end” when unset and are omitted from the response instead of carrying a placeholder value, which raises in a client generated from an rc.160-or-earlier spec. A value of 0 is still accepted and means the same thing.

Breaking changes to the stored formats - a consumer reading these fields must treat them as optional:

  • The per-image image-scale B factor is no longer computed, so /entry/MX/imageScaleBFactor is absent from newly written HDF5 files and the corresponding key is absent from the CBOR DataMessage and END blocks. Files written by rc.160 and earlier still contain it and still open; nothing in the pipeline reads it any more.

  • _reflns.jfjoch_diffrn_ISa now carries the whole-range 1/sqrt(a*b) that XDS’s ISa denotes, and the error-model a and b are reported in XDS’s convention; the strong-reflection asymptote moves to _reflns.jfjoch_diffrn_ISa_asymptotic. A file written by an earlier version carries the asymptote under the plain ISa name.

1.0.0-rc.160

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Rugnux: Rotation geometry post-refinement is now on by default (--rotation-no-postrefine to disable; also a viewer checkbox). A first pass integrates at the header geometry, then the shared detector distance + beam centre and the crystal cell/goniometer-axis are post-refined over all frames (cross-validated, committed only for a small < 1 % move, with the gauge-weak beam centre restrained toward the header); a second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical <prefix>_* output; the header-geometry pass is kept as <prefix>_01_*.

  • Rugnux: Optional per-batch relative-B correction for rotation (--relative-b[=deg], default 10°-of-rotation batches when bare, off otherwise) - a cross-validated, curvature-smoothed resolution×dose correction beyond the single global decay slope.

  • Rugnux: Always-on radiation-damage report for rotation - the per-image scale correlation-to-merge and mosaicity versus dose, plus the relative B-factor change over the run (first→last) as a scalar and a per-batch relative-B curve, printed to the log and written to the merged mmCIF. Report-only; it never alters the merge.

  • Rugnux: De-novo space-group search ranks candidate lattice centerings by net absences (systematically-absent minus violating), not the gross absent count, fixing an over-centering of a genuinely C-centred lattice to F.

  • Rugnux: Record the producing software (name and version) and the refined detector distance and beam centre in the merged mmCIF (and the software in the MTZ history).

  • Bragg integration: Carry the box-sum observed centroid through the profile-fit path, so the observed spot centroid is emitted in every integrator mode.

  • Rugnux: Report ISa as the counting-subtracted strong-reflection asymptote, not 1/b of the whole-range fit; it also sets the merged-sigma floor. CC1/2, R-meas and per-obs sigmas unchanged.

  • Frontend: Azimuthal-integration Q fields (Q spacing / Low Q / High Q) accept 5 decimals (was 3), matching the 1e-5 q_spacing minimum; number-field precision is now configurable.

  • Rugnux: Add a dataset-wide Wilson B-factor estimate to the merged output (mmCIF, stats table, log); the per-image viewer Wilson B emits NaN for implausible fits.

  • Rugnux: De-novo space-group search vetoes a merohedral-twin over-promotion whose systematic error-model b balloons past a calibrated bound (keeps R3 as R3, not R32).

  • Rugnux: De-novo space-group search decides lattice centering from the strength (mean I/sigma) of the systematically-absent class, not a per-reflection violation count.

  • Rugnux: Recover lattice centering on weak / low-energy data via a floor-independent test (rate of significant absent vs present reflections), fixing a missed I-centring at 5/13 keV.

  • Rugnux: Report anomalous signal-to-noise SigAno = <|I(+)-I(-)|>/ per shell and overall (mmCIF PDBx items + stats-table column); anomalous merges only.

  • Rugnux: De-novo space-group search recovers a genuine high-symmetry group on weak data with a broken sigma model by confirming on the systematic-b test alone (restores an F432 case).

  • Rugnux: Print the adopted space group and unit cell as a one-line summary at the end of the run (de-novo or user-fixed -S).

  • Rugnux: Score the radiation-damage decay cross-validation on a sigma-independent (R-meas-like) metric, so a spurious slope can’t pass by reshaping sigmas.

  • Rugnux: Fix de-novo rotation indexing committing a spurious axis-multiple supercell (collapsing to P1) via a cross-scheme smaller-cell tie-break on near-integer volume ratios.

  • Rugnux: Widen refined-cell angle bounds to [30, 150] deg (rotation candidate and per-frame stills refinement); check refined angles against the reference cell.

  • Rugnux: -S/--space-group now accepts a Hermann-Mauguin symbol (e.g. P43212) as well as a space-group number.

  • Rugnux: Warn when the chosen cell/space group carries an indexing (merohedral) ambiguity needing a reference to resolve.

  • Indexing: Requesting the FFTW (CPU) indexer on a GPU node now fails with a clear, actionable message (rotation always uses the GPU FFT indexer there).

  • Rugnux: Stills --refine-geometry[=N|off] - first-pass bundle-adjust of beam/distance/cell then re-index (default ON with a reference cell); accepts reference F/FP columns.

  • Rugnux: Per-image geometry refinement -r flex tries all three algorithms per image and keeps the best (old name multi kept as an alias).

  • Rugnux: Experimental stills partiality --still-partiality (Gaussian excitation-error) and --partiality-uncertainty <num> down-weighting the least-complete partials.

  • Rugnux: Default the stills Bragg-integration box to r=6 (integration radii 6, 8, 12).

  • Rugnux: Self-referenced stills scale in a single pass (fixing a weak-data collapse); a reference MTZ (-z) fixes SG/cell/ambiguity but never anchors the scale (stills and rotation).

  • Rugnux: Cap normalised intensity (E^2) on second-lattice overlaps in the de-novo space-group search, so strong overlaps don’t skew the symmetry decision.

  • Rugnux: Fix --scale on a self-contained _process.h5 (stored reflections and error model reload correctly).

  • Rugnux: Add --spot-low-resolution <num> (default 50 A) and --min-pix-per-spot <num> (default 2) to tune spot finding on weak serial data.

  • jfjoch_viewer: Expose stills processing settings in the settings dock, rename geometry-refinement multi to flex, and refit the initial image on resize.

  • jfjoch_writer: Remove the CBF and TIFF image writers - only NXmx HDF5 is written (all three layouts remain).

  • Reader: Treat a negative total_flux in a stored dataset as unknown/absent rather than a valid flux.

  • Packaging: Build the self-contained Linux viewer against a static libdbus with glib disabled; add parallel image-build and in-container viewer-verification scripts.

  • Rugnux: Write anomalous data as a standard CCP4 anomalous MTZ (one row per reflection: IMEAN, I(+)/I(-), F/F(+)/F(-)), readable by aimless/mtz2sca/ANODE.

  • Rugnux: Always write merged reflections as both <prefix>.mtz and <prefix>.cif; the --scaling-output selector and text .hkl output are removed.

  • Rugnux: Add a detector-plane modulation (flat-field) correction surface to rotation scaling (cross-validated, on by default; --no-scaling-corrections disables all), dropping R-meas.

  • Rugnux: Add optional stills detector-plane modulation (--stills-modulation, default off) - the same cross-validated surface for the on-the-fly stills path.

  • Bragg integration: Local background is now a symmetric trimmed mean of the ring (--background-trim <f>, default 0.10; monochromatic), improving <I/sigma> and resolution-edge CC1/2.

1.0.0-rc.159

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Rugnux: Add --model model.pdb - score the merged data against an atomic model and compute initial maps. It reports R-work/R-free (scaling the model to the observed amplitudes with an overall scale, an anisotropic B and a flat bulk solvent - the standard few-parameter model, so a batch of maps stays directly comparable) and writes 2Fo-Fc / Fo-Fc electron-density maps (CCP4) plus a map-coefficient MTZ. The structure itself is not refined; the model is only re-fractionalised into the data cell.

  • Rugnux: The merged reflection output now carries French-Wilson amplitudes (|F| and its sigma) next to the intensities - MTZ F/SIGF, mmCIF _refln.F_meas_au, and the text HKL - computed with the correct centric/acentric Wilson prior and epsilon multiplicity, so a downstream program (e.g. phenix.refine) can refine against amplitudes. The intensity columns are unchanged.

  • Rugnux: R-free test-set flags are now assigned deterministically and consistently across symmetry - a Bijvoet pair I(+)/I(-) is never split between the work and free sets, and the assignment is a reproducible per-hkl hash that depends only on the reflection index, so every dataset of one crystal form gets the same ~5% free set (what a multi-dataset campaign such as PanDDA needs). On small data the fraction is floored so the test set stays large enough for a stable R-free (~500 reflections, capped at 10%); it stays flat at 5% on ordinary data. When a reference MTZ carries a FreeR_flag column its test set is imported instead, letting a whole campaign inherit one shared free set.

  • Rugnux: A reference MTZ (--reference-mtz) can now fix the space group and cell for rotation data too (previously rejected), without being used to scale - the rotation merge stays self-consistent. When the crystal has an indexing (merohedral) ambiguity - a lattice symmetry higher than its Laue symmetry, e.g. P3/P4/P6/C2 - the reference also resolves it: each candidate reindexing (identity plus the twin-law cosets of the metric symmetry) is scored by its intensity correlation against the reference and the data are re-merged in the best-correlating one. This is a metric-preserving relabelling of hkl (the cell is unchanged) and a no-op for a holohedral crystal (which has no twin laws).

  • Rugnux: --model validation now aligns the data to the model before scoring - the observed reflections are reindexed into the model’s enantiomorph when the two differ only by hand (indistinguishable from merged intensities). A merohedral indexing ambiguity is resolved against the reference MTZ when one is given (so a whole campaign shares one indexing convention); only with a model and no reference does validation fall back to fitting each candidate reindexing and keeping the lowest R-free.

  • Rugnux: De-novo symmetry - recover a genuine high-symmetry group whose data are imperfectly scaled. Such a merge’s within-orbit chi² lands just past the self-consistency bound (each real symmetry step adds a little systematic scatter), right where a merohedral twin also lands, so the chi² ratio alone cannot separate them. The candidate is now rescued when the extra intensity-proportional systematic error it invokes stays small relative to the confirmed subgroup - a genuine symmetry step gains multiplicity without inflating the merge error model’s b, whereas a twin forces non-equivalent reflections together and b balloons. Fixes a cubic case (I23 instead of I222) with no change to any other crystal in the test battery, including the twins that must stay in their lower symmetry.

  • Docs: Document the French-Wilson amplitude estimation, R-free flagging, reference-based space-group/ambiguity resolution, and model-based validation/maps in CPU_DATA_ANALYSIS.md.

  • Frontend: The status-bar pill now shows a progress bar during detector calibration (previously only during measurement), and the calibration state and its button are labelled “Calibration”/“CALIBRATE” (the internal Pedestal state name is unchanged for back-compatibility).

1.0.0-rc.158

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (cos^3 of that angle) instead of cos^3(2*theta), so it is correct for a tilted detector and matches PyFAI solidAngleArray and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (XtalOptimizer) no longer silently ignores an imported PONI rot3 (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured rot3 through the full PONI rotation.

  • jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting \\); write processing outputs next to the input file, with a Browse button and independent _process.h5 / merged .mtz/.cif toggles; and show the determined space group in the merge-statistics window.

  • jfjoch_viewer: Connect to a broker over https (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow.

  • jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source.

  • Rugnux: Accept an absolute -o output prefix in offline processing.

  • Rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged).

  • Rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing.

  • Rugnux: De-novo symmetry robustness - don’t over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don’t flag twinning in holohedral Laue classes where no twin law can exist.

  • jfjoch_writer: Fold the refined beam centre into the NXmx detector translation vector too (not only the informational beam_center fields), so a reprocessed _process.h5 has a self-consistent refined geometry.

  • Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames.

  • Packaging: The self-contained Linux viewer .tgz now bundles cuFFT, so it runs without a system CUDA toolkit (.deb/.rpm are unchanged, distro-managed).

  • Docs: Documentation updated to match the current analysis code and CLI.

1.0.0-rc.157

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • Rugnux: Rebrand the offline data-processing subsystem as rugnux and consolidate all offline analysis into the single rugnux binary - jfjoch_process is now rugnux, the former jfjoch_azint is now rugnux --azint-only, and jfjoch_scale is now rugnux --scale (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (--no-merge disables them), replacing the previous opt-in -M, --scale-merge.

  • Rugnux: CLI fixes - default -N to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require --wavelength > 0, and correct the reproduced command line and --scale reference-cell handling.

  • Rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics.

  • Rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection’s rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below <I/sigma> = 1.

  • Rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below --min-captured-fraction (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned).

  • Rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved jfjoch prefix.

  • jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation.

  • jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a “reduce frame rate” warning.

  • jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint.

  • jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed).

  • jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types.

  • jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails.

  • jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak.

  • Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation.

  • CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*.

1.0.0-rc.156

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output.

  • jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed.

  • jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections.

  • jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw).

  • jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path.

  • jfjoch_process: Unify still and rotation processing on a single –force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental –reciprocal-profile option is removed.

  • jfjoch_process: Add EXPERIMENTAL ice-ring detection (–detect-ice-rings) that excludes ice reflections from scaling.

  • jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend.

  • jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata.

  • jfjoch_writer: Don’t write empty grid-scan position arrays when the dataset has no images.

  • Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don’t fit rather than aborting), and add the jfjoch_recompress tool.

  • jfjoch_viewer: Report “Multiple lattices detected” and grey out “Analyze dataset” on a live connection.

  • jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons.

  • CI: Build Windows (CUDA and non-CUDA) installers.

  • CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz.

1.0.0-rc.155

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D)

  • jfjoch_viewer: Some graphical improvements.

  • jfjoch_viewer: Simplify and unify data analysis settings.

  • jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker “dies” in the middle of data acquisition.

1.0.0-rc.154

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability)

1.0.0-rc.153

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

  • jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing

  • jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files

  • jfjoch_broker: Add ROI calculation in non-FPGA workflow

  • jfjoch_broker: Fixes to TCP image pusher

  • jfjoch_broker: Remove NUMA bindings

  • jfjoch_broker: Improvements to indexing

  • jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values

  • jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors

  • jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs

  • jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results

1.0.0-rc.152

  • jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5)

  • jfjoch_viewer: Adjust Q bounds for azimuthal integration

  • jfjoch_azint: Add tool to do quick azimuthal integration

1.0.0-rc.151

  • jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname

1.0.0-rc.150

  • jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin.

  • jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins.

1.0.0-rc.149

  • XDS plugin: Fix HDF5 mutex to run on multiple processors

1.0.0-rc.148

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Improve azimuthal integration (add <I^2> calculation)

  • jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated)

  • jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin

1.0.0-rc.147

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015.

  • jfjoch_viewer: Add reciprocal space viewer

  • jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset

  • jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress)

1.0.0-rc.146

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

  • jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension)

  • jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset.

  • jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling.

  • jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file.

  • jfjoch_scale: Dedicated tool for rescaling/merging existing data.

  • jfjoch_viewer: Fix bugs where pixel labels were displayed on a wrong pixel.

WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being.

1.0.0-rc.145

This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

  • Default HDF5 writing mode is with VDS, not soft-links - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.

  • XDS plugin: Improve performance of VDS reading.

  • jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.

  • jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

1.0.0-rc.144

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M)

1.0.0-rc.143

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment

1.0.0-rc.142

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8

  • Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent

1.0.0-rc.141

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times

  • frontend: Fix selection of FFTW in indexing settings

1.0.0-rc.140

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection

  • jfjoch_broker: Minor bug fixes for rare conditions

  • jfjoch_process: Significantly improve performance

1.0.0-rc.139

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on /start

  • jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on ‘/start’

  • jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite workflow preparation time

  • jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received

  • jfjoch_broker: Use nanoseconds internally for frame/image/readout time

  • jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished)

  • jfjoch_broker: Improve profiling of different data analysis steps

  • jfjoch_broker: Record integration reflection count

  • jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds)

  • jfjoch_broker: Fix bug where ‘/wait_till_done’ got deadlocked

  • jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets

Breaking changes: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time.

1.0.0-rc.138

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time

  • jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations

  • jfjoch_broker: Goniometer axis name is converted to lowercase

  • jfjoch_broker: Fix bug, where wrong HTTP error codes were returned

  • jfjoch_process: Improve sigma estimation during merging (K. Takaba)

  • jfjoch_process: Modify spot finding thresholds

1.0.0-rc.137

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Better track time for each operation in the processing stack

  • jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress)

  • jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress)

  • jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0)

1.0.0-rc.136

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress).

1.0.0-rc.135

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4)

  • jfjoch_viewer: Improve image render performance

1.0.0-rc.134

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long

  • jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata

  • XDS plugin: Add generation of Jungfraujoch plugin for XDS

  • CI: Add tests with XDS and DIALS (xia2.ssx)

1.0.0-rc.133

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132.

  • jfjoch_broker: Use httplib for HTTP server instead of Pistache

  • jfjoch_broker: Drop OpenSSL support

  • jfjoch_broker: Base work for multi-lattice support in the future

  • jfjoch_broker: Improve recording time of data analysis steps

  • jfjoch_writer: Save per-image information about data analysis timing

  • Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib)

1.0.0-rc.132

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • Documentation: Fix equation rendering

1.0.0-rc.131

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS)

  • jfjoch_viewer: Fix calibration (pedestal) images being open flipped

  • jfjoch_process: Add space group detection (EXPERIMENTAL)

1.0.0-rc.130

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Rotation indexer has two retries if it fails

  • jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot)

  • jfjoch_broker: Integration calculates background mask based on R2 radius

  • jfjoch_process: HDF5 files are not saved by default

1.0.0-rc.129

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files)

  • jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction

  • jfjoch_broker: Image pusher statistics are accessible via the REST interface

  • jfjoch_writer: Supports TCP image socket and for these auto-forking option

1.0.0-rc.128

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Handle properly reuse of image buffer locations

  • jfjoch_broker: Fix bug in counting idle slots

  • jfjoch_broker: Force obtuse angle for monoclinic cells

  • jfjoch_process: Change scaling refinement tolerance

1.0.0-rc.127

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Default EIGER readout time is 20 microseconds

  • jfjoch_broker: Multiple improvements regarding performance

  • jfjoch_broker: Image buffer allows to track frames in preparation and sending

  • jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer

  • jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets

  • jfjoch_writer: Fixes regarding properly closing files in long data collections

  • jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code

1.0.0-rc.126

This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg.

1.0.0-rc.125

This is an UNSTABLE release. This version adds scaling and merging. These are experimental at the moment, and should not be used for production analysis. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124.

  • jfjoch_broker: Improve logic on switching on/off spot finding

  • jfjoch_broker: Increase maximum spot count for FFBIDX to 65536

  • jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD)

  • jfjoch_process: Add scaling and merging functionality - program is experimental at the moment and should not be used for production analysis

  • jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection

  • jfjoch_writer: Save more information about each reflection

1.0.0-rc.124

This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123.

  • jfjoch_broker: Improve reflection position prediction and Bragg integration code.

  • jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation.

  • jfjoch_writer: Fix saving mosaicity properly in HDF5 file.

  • jfjoch_viewer: Introduce high-dynamic range mode for images

  • jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%)

  • jfjoch_viewer: Zoom-in numbers have better readability

1.0.0-rc.123

This is an UNSTABLE release.

  • jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility

  • jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M)

  • jfjoch_viewer: Add more color schemes for better image readability

  • HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable

  • HDF5: suppress warning if path (upstream group) doesn’t exist when checking if leaf exists

1.0.0-rc.122

This is an UNSTABLE release.

  • jfjoch_broker: Add thresholding to prefer shorter vectors after FFT

  • jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress)

  • jfjoch_broker: Update nlohmann::json to 3.12.0

  • jfjoch_viewer: Display file opening errors

  • jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available

1.0.0-rc.121

This is an UNSTABLE release.

  • jfjoch_broker: Report changes in the image buffer, so viewer doesn’t reload constantly

  • jfjoch_viewer: Improve performance of loading images

  • jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes

  • jfjoch_viewer: Auto-foreground calculated with histogram

  • jfjoch_viewer: Fix rare segmentation fault

1.0.0-rc.120

This is an UNSTABLE release.

  • jfjoch_broker: Improve performance of binary plot export

1.0.0-rc.119

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Add binary export of data analysis plots over OpenAPI

  • jfjoch_broker: Minor fixes to HTTP error handling

  • jfjoch_viewer: Prefer binary plots over JSON plots

  • jfjoch_viewer: Change foreground with F button + wheel

  • jfjoch_viewer: Change way how angles are displayed

  • jfjoch_viewer: Display resolution of the mouse cursor in top left corner

1.0.0-rc.118

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement.

  • jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations

1.0.0-rc.117

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add ROI results to the dataset info plots

  • jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment

1.0.0-rc.116

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_viewer: Add binning options in the context menu

1.0.0-rc.115

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Default spot finding settings can be configured via config JSON

  • jfjoch_viewer: FFT analysis of data in the dataset plot

1.0.0-rc.114

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Fix generating JPEG images with resolution estimation

1.0.0-rc.113

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Improve handling of rotation indexing

  • jfjoch_broker: More information saved in CBOR end message (WIP)

  • jfjoch_writer: Save rotation indexing lattice parameters and Niggli class

  • jfjoch_viewer: Remove (for now) primitive cell information

  • jfjoch_viewer: Use angle for dataset info plot for rotation scans

1.0.0-rc.112

This is an UNSTABLE release and not recommended for production use (please use rc.111 instead).

  • jfjoch_broker: Experimental rotation (3D) indexing

  • jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values

1.0.0-rc.111

This is an UNSTABLE release.

  • jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment)

  • jfjoch_viewer: Fix auto contrast not refreshing image

1.0.0-rc.110

This is an UNSTABLE release.

  • jfjoch_broker: Add auto-contrast option for preview images

  • Frontend: Add logo image

  • jfjoch_viewer: Add logo image

  • jfjoch_viewer: For image chart allow to set min value to zero

  • jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure

  • jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there)

  • Documentation: Add logo image

1.0.0-rc.109

This is an UNSTABLE release.

  • jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard

  • jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors

1.0.0-rc.108

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN)

  • jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default

  • jfjoch_viewer: Fit to window of image is now default when size of the grid changes

1.0.0-rc.107

This is an UNSTABLE release.

  • jfjoch_viewer: Minor polishing of new functionality

  • jfjoch_broker: Use NaN for empty azimuthal bins

1.0.0-rc.106

This is an UNSTABLE release.

  • jfjoch_viewer: Allow for multiple dataset info plots

  • jfjoch_viewer: Highlight current element in grid

1.0.0-rc.105

This is an UNSTABLE release.

  • jfjoch_viewer: Clean-up widgets slightly

  • jfjoch_viewer: Limit right panel to 600 pixels

  • jfjoch_viewer: Parse crystal symmetry type

  • jfjoch_viewer: Grid scan view takes color map and can be fit to zoom

1.0.0-rc.104

This is an UNSTABLE release.

  • jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier)

  • jfjoch_viewer: Display grid scan results in 2D (work in progress)

  • jfjoch_viewer: Improve auto-scaling on start of images (work in progress)

  • jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots

1.0.0-rc.103

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

  • jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates

1.0.0-rc.102

This is an UNSTABLE release.

  • jfjoch_viewer: Minor improvements to the viewer

1.0.0-rc.101

This is an UNSTABLE release.

  • jfjoch_viewer: Auto load is better handling change of states

  • jfjoch_viewer: Fix DBus registration

  • jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update

  • jfjoch_viewer: Calculate ROI in a more efficient way

1.0.0-rc.100

This is an UNSTABLE release.

  • jfjoch_viewer: Fix dbus registration

  • jfjoch_viewer: Remove background slider for diffraction image

  • jfjoch_viewer: Adjustments for 2D azimuthal image viewer

1.0.0-rc.99

This is an UNSTABLE release.

  • jfjoch_broker: Fix output during mask data collection

1.0.0-rc.98

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization

1.0.0-rc.97

This is an UNSTABLE release and not recommended for production use (please use rc.96 instead).

  • jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask

  • jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems)

  • jfjoch_viewer: Add option to handle user pixel mask (experimental)

  • jfjoch_viewer: More options for ROI

  • jfjoch_viewer: Add window to display calibration

1.0.0-rc.96

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • jfjoch_broker: Remove PNG preview, no dependency on libpng

  • jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds)

  • jfjoch_viewer: Show data collection time in dataset tooltip

  • jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate)

  • jfjoch_viewer: Auto foreground value

  • Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds

  • Remove OpenBLAS dependency

1.0.0-rc.95

This is an UNSTABLE release.

  • Fixes in CI pipeline

  • Add git-lfs to Rocky8 docker image

Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved.

1.0.0-rc.94

This is an UNSTABLE release.

  • FFTIndexer: Add limit on angles to avoid colinear vectors

  • Docker images: Add 3D Qt

  • Gitea: Fixes to the pipeline

1.0.0-rc.93

This is an UNSTABLE release.

  • CI: Fixes to Gitlab based pipeline

  • PCIe driver: Fix PCIe revision being hex number

1.0.0-rc.92

This is an UNSTABLE release.

  • jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample.

1.0.0-rc.91

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated

  • jfjoch_broker: Spot finding and integration predictions are ported to a GPU

  • jfjoch_broker: Estimate resolution

  • jfjoch_broker: Lattice search

  • jfjoch_broker: Many more improvements in image analysis

1.0.0-rc.90

This is an UNSTABLE release.

  • jfjoch_broker: for indexing min index spots for a viable cell can be changed via OpenAPI

  • jfjoch_viewer: Optional auto-reanalyze images

  • jfjoch_writer: Add option where no files at all are saved

  • Documentation: improvements

1.0.0-rc.89

This is an UNSTABLE release.

  • jfjoch_broker: Fix resolution estimation code

  • jfjoch_broker: Fix Wilson B-factor calculation code

  • jfjoch_viewer: Improve display of plots

  • jfjoch_viewer: Fix segmentation fault

  • jfjoch_viewer: Display missing metadata when using HTTP

  • jfjoch_viewer: Fix bug when opening the same file twice

1.0.0-rc.88

This is an UNSTABLE release.

  • jfjoch_viewer: Add resolution estimation to the image information

  • jfjoch_broker: Minor changes to resolution estimate routine

1.0.0-rc.87

This is an UNSTABLE release.

  • jfjoch_viewer: Display more image metadata (angle / exposure time)

  • jfjoch_viewer: Improve I/sigma and B-factor plots

  • jfjoch_broker: Estimate resolution based on visible spots

1.0.0-rc.86

This is an UNSTABLE release.

  • jfjoch_broker: Update logic when initializing detector to make it a bit more resilient

  • Gitea pipelines have nocuda option for all architectures

1.0.0-rc.85

This is an UNSTABLE release.

  • jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image

  • jfjoch_viewer: Handle spot count better in dataset info plots

  • jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled

1.0.0-rc.84

This is an UNSTABLE release.

  • jfjoch_broker: Write in log which detector is being initialized

  • Changes to automated build system

1.0.0-rc.83

This is an UNSTABLE release.

  • jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before)

  • CI: Fix script to generate python client

1.0.0-rc.82

This is an UNSTABLE release.

  • jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment)

  • Frontend: Minor fixes

  • Build scripts: Minor fixes to FFTW

1.0.0-rc.81

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing

  • jfjoch_broker: Add FFTW based indexer for CPU only indexing

  • jfjoch_broker: Enable saving X-ray fluorescence spectra

  • jfjoch_writer: Write total spot count (before filtering)

  • jfjoch_viewer: Add more information on source, sample, and button to show ice rings

  • jfjoch_viewer: Enable data processing inside the viewer

CI: Moving from Gitlab to Gitea at PSI

1.0.0-rc.80

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer

1.0.0-rc.79

This is an UNSTABLE release.

  • jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance

  • jfjoch_writer: Save nimages_per_trigger in detectorSpecific

1.0.0-rc.78

This is an UNSTABLE release.

  • jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore)

1.0.0-rc.77

This is an UNSTABLE release.

  • jfjoch_viewer: Display detector and dataset settings with tooltips

  • jfjoch_viewer: Clean excessive HDF5 warnings

  • jfjoch_viewer: Display unit cell

  • jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset

1.0.0-rc.76

This is an UNSTABLE release.

  • jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones

  • jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median)

  • jfjoch_writer: Save space group number (non-NXmx addition) in addition to name

  • jfjoch_viewer: Fix the bug on reading space_group as string

  • jfjoch_viewer: Add missing resolution labels on rings

  • jfjoch_viewer: Remove Q value from the status bar

1.0.0-rc.75

This is an UNSTABLE release.

  • jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being

1.0.0-rc.74

This is an UNSTABLE release.

  • jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing)

  • jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization

1.0.0-rc.73

This is an UNSTABLE release.

  • jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default

1.0.0-rc.72

This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Refactor of indexing and geometry refinement code

  • jfjoch_broker: Handle space group/centering in refinement code

  • jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images

  • jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion

  • jfjoch_writer: Experimental saving of reflections

  • jfjoch_writer: Save space group name as string

  • jfjoch_viewer: Add profile radius and B-factor

  • jfjoch_viewer: Show 4 digits for wavelength

  • jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings)

  • FPGA: Use UDP destination port to distinguish between detector modules and data streams

  • FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up)

  • FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene)

1.0.0-rc.71

This is an UNSTABLE release.

  • jfjoch_broker: Remove resolution estimation via machine learning

  • jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers

  • jfjoch_viewer: Fix error when HDF5 files with indexing results couldn’t be loaded on a machine without GPU

1.0.0-rc.70

This is an UNSTABLE release. This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. For production use we recommend waiting for a future bug-fix release.

  • jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement

  • jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards)

  • jfjoch_broker: Enable geometry refinement - first try (work in progress)

  • jfjoch_viewer: Fix deadlock when opening HTTP connections

  • jfjoch_viewer: Display rings as ellipses with detector tilt

  • jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image

  • jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3)

  • Add Google Ceres a non-linear least-square optimization library to Jungfraujoch

  • Add experimental detector calibration routines (for LaB6)

  • Improve documentation on the ZeroMQ writer notification socket and detector geometry

1.0.0-rc.69

This is an UNSTABLE release.

  • jfjoch_viewer: Metadata can be modified for an open dataset (no option to save)

  • jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments

  • jfjoch_viewer: Resolution rings not enabled by default

  • jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file

1.0.0-rc.68

This is an UNSTABLE release.

  • jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector

  • jfjoch_broker: Default detector settings can be configured for each detector separately

  • jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings

  • jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing

  • jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter

  • jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI

1.0.0-rc.67

This is an UNSTABLE release.

  • jfjoch_broker: Enable SSL

  • jfjoch_broker: Wilson B-factor only provided if fit is relatively OK (R^2 > 0.3); this will be refined much more in the future

1.0.0-rc.66

This is an UNSTABLE release.

  • jfjoch_broker: Indexers operate as a thread pool

  • jfjoch_viewer: Increase interval between loading images + fix too many verbose messages

1.0.0-rc.65

This is an UNSTABLE release.

  • jfjoch_broker: Print information regarding used image pushers

  • jfjoch_viewer: Allow syncing with Jungfraujoch server

  • OpenAPI: Clarify licensing terms in the file

1.0.0-rc.64

This is an UNSTABLE release.

  • jfjoch_broker: Fix issue in receiver light with very long preparation time for threads

  • jfjoch_broker: Add verbose option

  • jfjoch_broker: Don’t trigger pedestal if critical settings are not changed when loading detector settings

  • jfjoch_broker: Detector left in busy state when detector settings were improper

  • jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times

  • jfjoch_lite_perf_test: Add verbose option

1.0.0-rc.63

This is an UNSTABLE release.

  • jfjoch_broker: Save NX/NY for grid scan result

  • jfjoch_broker: Add processing time to CBOR output and plot

  • jfjoch_writer: Add processing time to data file

1.0.0-rc.62

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where low resolution spots were not counted properly

  • jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count

  • jfjoch_broker: Add more spot count information to CBOR

  • jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles

1.0.0-rc.61

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector

  • jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results

  • jfjoch_broker: High resolution limit for low res. spot counting can be adjusted

1.0.0-rc.60

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released

  • jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices

  • jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength

1.0.0-rc.59

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock.

  • jfjoch_writer: Verbose option for debugging.

1.0.0-rc.58

This is an UNSTABLE release.

  • jfjoch_viewer: Fix memory leak

  • jfjoch_writer: Add detector_number/serial_number to master file

1.0.0-rc.57

This is an UNSTABLE release.

  • jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible

  • jfjoch_viewer: “Movie” mode

1.0.0-rc.56

This is an UNSTABLE release.

  • jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation

1.0.0-rc.55

This is an UNSTABLE release.

  • jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation

1.0.0-rc.54

This is an UNSTABLE release.

  • jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not)

  • jfjoch_broker: Don’t remove shared memory area when deactivating detector

  • jfjoch_writer: Save writer release

  • jfjoch_viewer: Increase time for the messages in the status bar

1.0.0-rc.53

This is an UNSTABLE release.

  • PCIe driver: Imperfect solution for RHEL 9.5+ changes

  • jfjoch_writer: Fix to angle containers for AutoProc compatibility

  • jfjoch_fpga_test: Use consecutive number for devices, not interleaved

1.0.0-rc.52

This is an UNSTABLE release.

  • jfjoch_viewer: Use warmer colors to distinguish from AareGUI

  • jfjoch_viewer: Minor adjustments to DBus setting image number

  • jfjoch_broker: Fix in low resolution spot count plotting

1.0.0-rc.51

This is an UNSTABLE release.

  • jfjoch_broker: Send preview in PNG format

  • jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range

  • jfjoch_broker: Provide ML resolution estimation in scan result

  • jfjoch_broker: Allow removing beam center in web preview

1.0.0-rc.50

This is an UNSTABLE release.

  • The release fixes some of many bugs introduced in recent releases

  • jfjoch_viewer: display predictions for indexed cells

1.0.0-rc.49

This is an UNSTABLE release.

  • jfjoch_broker: Add sample temperature (K) and ring current (mA) to metadata

  • jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current

1.0.0-rc.48

This is an UNSTABLE release.

  • jfjoch_broker: fix the bug when a unit cell was not exported for a scan result.

1.0.0-rc.47

This is an UNSTABLE release.

  • jfjoch_viewer: fix dbus service path

  • jfjoch_writer: fix CBF/TIFF writing

1.0.0-rc.46

This is an UNSTABLE release.

  • jfjoch_viewer: remove dependency on image analysis

1.0.0-rc.45

This is an UNSTABLE release.

  • jfjoch_broker: Detector list returns pixel size (mm)

1.0.0-rc.44

This is an UNSTABLE release.

  • jfjoch_broker: more general definition of scan result export

Breaking changes:

  • It removes additions to OpenAPI from 1.0.0-rc.43

  • It makes changes to the “unit_cell” definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations.

1.0.0-rc.43

This is an UNSTABLE release.

  • jfjoch_broker: Export grid scan results into a single data structure

1.0.0-rc.42

This is an UNSTABLE release.

  • jfjoch_broker: Add pixel_sum to CBOR output.

  • jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine

  • jfjoch_writer: Save pixel_sum

1.0.0-rc.41

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction

  • jfjoch_broker: Export GPU count via OpenAPI

  • jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector

  • FPGA: Increase the number of integration bins to 2048

1.0.0-rc.40

This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment.

  • jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures

  • jfjoch_broker: Improve metadata for rotation data collection

  • jfjoch_broker: Better handling of plotting

  • jfjoch_broker: FFT based indexing

  • jfjoch_broker: Integration, first try, results not saved at the moment

  • jfjoch_broker: Internal improvements in image handling

  • jfjoch_writer: Multiple adjustments adapt to changes in this release for new features

  • jfjoch_writer: New state management model to improve clarity of error reporting

  • jfjoch_viewer: Remote control via DBus

  • Frontend: Multiple adjustments for new features

  • Frontend: Grid scan plots

WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken.

1.0.0-rc.39

  • FPGA: Bugfix for pixel masked for data analysis if summation was on

  • jfjoch_viewer: Fix segmentation fault when cursor was outside of image

1.0.0-rc.38

  • jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via REST

  • jfjoch_broker: Neural net model can use all 4 quadrants of the detector

  • jfjoch_broker: For EIGER image time can be provided through /start

  • jfjoch_viewer: Add image list option

  • jfjoch_viewer: Drawing circular ROIs with shift

  • jfjoch_viewer: Enable image summation

  • jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer

1.0.0-rc.37

  • jfjoch_broker: Make locking rules more flexible

  • jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors

  • jfjoch_viewer: Add status bar

1.0.0-rc.36

This is an UNSTABLE release. Wait for a new version to use in a production environment.

  • jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code)

  • jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation)

  • jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data

  • jfjoch_writer: Increase buffering to 1000 images in the receiver

  • jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5

1.0.0-rc.35

This is an UNSTABLE release, not properly tested. Wait for a new version before using it in production.

  • jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting.

  • jfjoch_writer: Save EIGER energy threshold

  • jfjoch_writer: Add /entry/sample/goniometer for compatibility with eiger2cbf program

1.0.0-rc.34

This is an UNSTABLE release - introducing new features, but not properly tested. Wait for a new version before using it in production.

  • jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings)

  • jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells

  • jfjoch_broker: Handle situation when detector doesn’t want to gracefully stop (to be tested)

  • jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots

  • jfjoch_viewer: Can extract data analysis results from “legacy” format

  • jfjoch_viewer: Display dataset name

  • FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels

  • FPGA: Add pixel sum to module statistics

  • FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI

  • FPGA: Spot finder is back to full dynamic range (24-bit)

  • FPGA: More debug features for internal FIFOs

Known issues:

  • ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts.

  • EIGER data acquisition has an issue that is currently debugged

1.0.0-rc.33

  • jfjoch_broker: Fix issue with EIGER settings being loaded improperly

1.0.0-rc.32

  • jfjoch_broker: Refactor code for azimuthal integration for further improvements

  • jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later)

  • jfjoch_writer: Fix too much verbose information

  • FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20)

1.0.0-rc.31

This is UNSTABLE release - introducing many features, but still needs more testing. Expecting soon to put bugfix release.

  • jfjoch_writer: Allow to enable overwriting existing files (not enabled by default)

  • jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default)

  • jfjoch_viewer: Image viewer, early test version

  • jfjoch_broker: Fixes to counting packets per dataset/image

  • jfjoch_broker: Image buffer is accessible for outside to check images

  • jfjoch_broker: error/saturated pixels and dedicated ROI “beam” can be tracked online

  • jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU

  • jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation

  • jfjoch_broker: Fix bug in EIGER initializing

  • jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot

  • PCIe driver: Add PCIe link speed and width

  • FPGA: Improve counting error/saturated/min/max pixels

  • FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas

  • FPGA: Require Vivado 2022.2

Warning: There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency.

1.0.0-rc.30

  • jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to “lose” images within the writer

1.0.0-rc.29

  • jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU

  • jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file

  • jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info)

  • jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly

Warning - two breaking changes to OpenAPI:

  • Handling of ROIs is through /config/roi path only for both circle and box ROIs, paths in /roi are no longer accessible

  • broker_status structure introduced in 1.0.0-rc.28 has member message and not error_message to allow handling info/warning messages as well

1.0.0-rc.28

  • jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI

  • jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not compliant with OpenAPI specs

  • jfjoch_test: remove header that failed when CUDA is absent during compilation

  • frontend: add soft trigger button in data collection tab

  • frontend: show error message when in error state

  • CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF)

1.0.0-rc.27

  • jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation

  • jfjoch_broker: update pistache web server version

  • jfjoch_writer: minor changes to republish logic

  • Improvements to documentation

1.0.0-rc.26

  • jfjoch_broker: implement ZeroMQ stream for image metadata information

  • jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent

  • jfjoch_broker: add crystal lattice plots

  • jfjoch_broker: remove empty bins from the plots

  • jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU “long” summation

  • jfjoch_broker: Fix bug when mean background estimation / indexing rate were affected by previous experiment

  • jfjoch_writer: fix missing “-w” parameter

  • jfjoch_writer: temporary files have “.tmp” suffix

  • jfjoch_writer: refactor logic for watermarks

  • jfjoch_writer: report on internal FIFO utilization

  • jfjoch_writer: clean-up naming for azimuthal integration and background estimate

  • jfjoch_writer: write final background estimate and indexing rate in the master file

  • tools/: remove unnecessary tools, make naming consistent

  • CBOR: Add indexing rate and background estimate to end message

  • CBOR: Clean-up documentation

1.0.0-rc.25

  • Updates to documentation

  • License set to GPLv3 / OHL-S

  • Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms

  • Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5

1.0.0-rc.24

New FPGA functionality:

  • EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit “real” depth is 23-bit + 1-bit signed)

  • Output possible to 8, 16 and 32-bit data

  • Threshold is applied before summation

  • Pixel mask can be applied on FPGA

  • Mark pixels with ADC content = 0 as bad pixels

  • FPGA stores semantic version information (access via /sys/class/misc/jfjoch…/version)

New software functionality:

  • Long summation (above 256 frames) done on CPU

  • Mechanism to save arbitrary data to HDF5 file

  • ZeroMQ preview has option to send start message

  • Rework pixel mask + add statistics displayed in web interface

Bug fixes:

  • Web frontend: Update preview image automatically during data acquisition

  • jfjoch_broker: Error handling if CUDA driver is not installed

  • jfjoch_broker: Correctly update progress during pedestal

  • jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF

  • jfjoch_action_test: enable HLS simulation

Documentation improvement and placement in a dedicated directory

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS.html b/CPU_DATA_ANALYSIS.html new file mode 100644 index 000000000..8bed90340 --- /dev/null +++ b/CPU_DATA_ANALYSIS.html @@ -0,0 +1 @@ + CPU-side crystallographic data analysis (Jungfraujoch) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

CPU-side crystallographic data analysis (Jungfraujoch)

This document describes the crystallographic algorithms implemented in Jungfraujoch for CPU- and GPU-side real‑time and near‑real‑time data analysis.

Scope. The pipeline covered here comprises:

  1. geometry mapping and corrections,

  2. azimuthal integration (powder/radial profiles),

  3. Bragg spot finding (strong pixels → connected components → spot descriptors),

  4. indexing (still and rotation modes),

  5. Bravais lattice / centering inference,

  6. geometry and lattice refinement,

  7. reflection prediction (still and rotation),

  8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free),

  9. scaling and merging,

  10. merge-level error modelling, outlier rejection and the resolution cutoff,

  11. space-group determination from the merged intensities (Laue group, screw axes, glide planes, centering, the centre of symmetry), the twinning check and the translational pseudo-symmetry check,

  12. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref),

  13. amplitude estimation (French–Wilson) and R-free test-set flagging,

  14. optional model-based validation: rigid-body placement of a supplied model, R-free against it, sigma_A-weighted 2mFo−DFc / mFo−DFc electron-density maps, and an anomalous difference map with the strongest anomalous sites named.

The reference is split into four parts, in pipeline order; the section numbers run continuously across them and are the ones the rest of the documentation cites.

References

The methods draw on, and in places reimplement, solutions from:

  • W. Kabsch, “XDS”, Acta Cryst. D66 (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts).

  • W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, Acta Cryst. D66 (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors).

  • T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts).

  • J. Kieffer & J. P. Wright, “PyFAI: a Python library for high performance azimuthal integration on GPU”, Powder Diffraction 28 (2013), S339-S350 (detector geometry definition, azimuthal integration)

  • I. Steller, R. Bolotovsky & M. G. Rossmann, “An algorithm for automatic indexing of oscillation images using Fourier analysis”, J. Appl. Cryst. 30 (1997), 1036-1040 (the projection/1D-FFT autoindexing algorithm of §5).

  • H. Powell, “The Rossmann Fourier autoindexing algorithm in MOSFLM”, Acta Cryst. D55 (1999), 1690-1695 (the MOSFLM implementation of it, whose practice is followed)

  • P. Gasparotto, L. Barba, H.-C. Stadler et al., “TORO Indexer: a PyTorch-based indexing algorithm for kilohertz serial crystallography”, J. Appl. Cryst. 57 (2024), 931-944 (the algorithm of the ffbidx fast-feedback indexer, §4).

  • I. Křivý & B. Gruber, “A unified algorithm for determining the reduced (Niggli) cell”, Acta Cryst. A32 (1976), 297-298, and International Tables for Crystallography Vol. A, Table 9.2.5.1 (the Niggli reduction and the lattice-character table of §5.3/§6).

  • J. E. Padilla & T. O. Yeates, “A statistic for local intensity differences: robustness to anisotropy and pseudo-centering and utility for detecting twinning”, Acta Cryst. D59 (2003), 1124-1130 (the L test, §13.2).

  • A. J. C. Wilson, “The probability distribution of X-ray intensities”, Acta Cryst. 2 (1949), 318-321, and E. R. Howells, D. C. Phillips & D. Rogers, “The probability distribution of X-ray intensities. II. Experimental investigation and the X-ray detection of centres of symmetry”, Acta Cryst. 3 (1950), 210-214 (the acentric and centric intensity distributions behind the centre-of-symmetry statistics of §13.1 and the Wilson outlier test of §13.3).

  • W. H. Baur & D. Kassner, “The perils of Cc: comparing the frequencies of falsely assigned space groups with their general population”, Acta Cryst. B48 (1992), 356-369 (writing the centrosymmetric group where the absences cannot decide the centre, §13.1).

  • R. J. Read, P. D. Adams & A. J. McCoy, “Intensity statistics in the presence of translational noncrystallographic symmetry”, Acta Cryst. D69 (2013), 176-183 (the native-Patterson detection of translational pseudo-symmetry, and the intensity modulation it produces, which the axial-zone screw-absence test scores against).

  • A. Barty, R. A. Kirian, F. R. N. C. Maia et al., “Cheetah: software for high-throughput reduction and analysis of serial femtosecond X-ray diffraction data”, J. Appl. Cryst. 47 (2014), 1118-1131 (peakfinder8: the per-resolution-ring background statistics of §3.2).

  • A. Hennequin, B. Couturier, V. V. Gligorov & L. Lacassagne, “SparseCCL: Connected Components Labeling and Analysis for sparse images”, DASIP 2019, 65-70 (the connected-component labelling of §3.4, used via ACTS/traccc).

  • S. French & K. Wilson, “On the treatment of negative intensity observations”, Acta Cryst. A34 (1978), 517-525 (Bayesian amplitude estimation from intensities), and CCP4’s ctruncate (C. Ballard & N. Stein), whose anisotropic Wilson prior §10.8 follows, cited through M. D. Winn et al., “Overview of the CCP4 suite and current developments”, Acta Cryst. D67 (2011), 235-242.

  • A. T. Brünger, “Free R value: a novel statistical quantity for assessing the accuracy of crystal structures”, Nature 355 (1992), 472-475 (R-free cross-validation).

  • P. H. C. Eilers, “A perfect smoother”, Anal. Chem. 75 (2003), 3631-3636, after E. T. Whittaker, “On a new method of graduation”, Proc. Edinburgh Math. Soc. 41 (1923), 63-75 (the penalised smoother of the fulls’ per-frame scale, §10.6).

  • R. A. Fisher, “Frequency distribution of the values of the correlation coefficient in samples from an indefinitely large population”, Biometrika 10 (1915), 507-521 (the z-transformation on which the correction surfaces’ held-out half-set CC1/2 is compared).

  • M. Wojdyr, “GEMMI: A library for structural biology”, J. Open Source Softw. 7 (2022), 4200 (model / structure-factor / map machinery used in §14).

  • J. P. Wright, “Experiences with GPU decompression for bitshuffle + LZ4 data”, HDF5 User Group meeting (2021), and github.com/jonwright/bslz4decoders (device-side decoding of bitshuffle+LZ4 images, §0).

  • A. Thorn & G. M. Sheldrick, “ANODE: anomalous and heavy-atom density calculation”, J. Appl. Cryst. 44 (2011), 1285-1287 (anomalous difference density read at the model’s sites).

  • R. Kahn, R. Fourme, A. Gadet, J. Janin, C. Dumas & D. Andre, “Macromolecular crystallography with synchrotron radiation: photographic data collection and polarization correction”, J. Appl. Cryst. 15 (1982), 330-337 (the azimuthal polarization factor of §2.2, applied to the azimuthal profile, the Bragg intensities and the ring background the beam-stop shadow test compares against).

  • R. J. Read, “Improved Fourier coefficients for maps using phases from partial structures with errors”, Acta Cryst. A42 (1986), 140-149 (the sigma_A formalism and the m, D weighting of the map coefficients of §14.4).

  • A. Fokine & A. Urzhumtsev, “Flat bulk-solvent model: obtaining optimal parameters”, Acta Cryst. D58 (2002), 1387-1392 (the flat bulk-solvent model, its optimal parameters and the range they are physically meaningful over, used when scaling a model to the data in §14).

  • P. V. Afonine, R. W. Grosse-Kunstleve & P. D. Adams, “A robust bulk-solvent correction and anisotropic scaling procedure”, Acta Cryst. D61 (2005), 850-855 (the grid search over that range that fits k_sol and b_sol, with the overall scale and anisotropic B refitted at each grid point).

  • K. Shoemake, “Uniform Random Rotations”, in Graphics Gems III, ed. D. Kirk, Academic Press (1992), 124-132 (the uniform random rotations the model-fit null of §14.5 is built from).

  • G. H. Golub & V. Pereyra, “The differentiation of pseudo-inverses and nonlinear least squares problems whose variables separate”, SIAM J. Numer. Anal. 10 (1973), 413-432, and L. Kaufman, “A variable projection method for solving separable nonlinear least squares problems”, BIT 15 (1975), 49-57 (the scale re-fit folded into the rigid-body Jacobian of §14.8).

  • Z. Otwinowski & W. Minor, “Processing of X-ray diffraction data collected in oscillation mode”, Methods Enzymol. 276 (1997), 307-326 (reweighted, de-biased profile-fit variances).

  • G. Winter et al., “DIALS: implementation and evaluation of a new integration package”, Acta Cryst. D74 (2018), 85-97, and J. Beilsten-Edmands et al., Acta Cryst. D76 (2020), 385-399 (CC1/2 resolution cutoff, merge outlier rejection, scaling error model).

  • R. H. Blessing, “An empirical correction for absorption anisotropy”, Acta Cryst. A51 (1995), 33-38 (absorption as spherical harmonics of the beam directions).

  • P. Evans, “Scaling and assessment of data quality”, Acta Cryst. D62 (2006), 72-82, and P. R. Evans, Acta Cryst. D67 (2011), 282-292 (POINTLESS: operator-by-operator point-group scoring, and the axial-zone screw-absence test).

  • A. G. W. Leslie & H. R. Powell, “Processing diffraction data with MOSFLM” (2007), NATO Science Series II 245, 41-51 (post-refinement practice: what is refined per image and what over a wedge).

  • D. W. Moreau, H. Atakisi & R. E. Thorne, “Ice in biomolecular cryocrystallography”, Acta Cryst. D77 (2021), 540-554 (measured hexagonal-ice ring positions, used by the ice-ring score, the ice flagging and the ice calibrant).

  • K. Röttger, A. Endriss, J. Ihringer, S. Doyle & W. F. Kuhs, “Lattice constants and thermal expansion of H2O and D2O ice Ih between 10 and 265 K”, Acta Cryst. B50 (1994), 644-648 (the ice Ih cell the ring positions below 1.522 Å are calculated from).

  • S. Sheriff & W. A. Hendrickson, “Description of overall anisotropy in diffraction from macromolecular crystals”, Acta Cryst. A43 (1987), 118-121 (the overall anisotropic B tensor and its symmetry constraints), and A. N. Popov & G. P. Bourenkov, “Choice of data-collection parameters based on statistic modelling”, Acta Cryst. D59 (2003), 1145-1153 (the sigma-aware estimation of the anisotropy of the observed intensity distribution, part of that paper’s statistic modelling).

  • P. R. Evans & G. N. Murshudov, “How good are my data and what is the resolution?”, Acta Cryst. D69 (2013), 1204-1214 (AIMLESS: the anisotropic deltaB as the range of the principal components, and diffraction limits from a cone about each principal direction).

  • G. Assmann, W. Brehm & K. Diederichs, “Identification of rogue datasets in serial crystallography”, J. Appl. Cryst. 49 (2016), 1021-1028, and G. M. Assmann, M. Wang & K. Diederichs, Acta Cryst. D76 (2020), 636-652 (XDSCC12: sigma-tau CC1/2, delta-CC1/2, the Fisher transformation and the rejection discipline the frame disposition follows).

  • K. Diederichs & P. A. Karplus, Nat. Struct. Biol. 4 (1997), 269-275, and P. A. Karplus & K. Diederichs, Science 336 (2012), 1030-1033 (R_meas / R_pim, CC1/2 and CC*).

  • IUCr Commission on Crystallographic Nomenclature, “Statistical descriptors in crystallography”, Acta Cryst. A45 (1989), 63-75, and Acta Cryst. A51 (1995), 565-569 (uncertainty conventions).

(list is not exhaustive; the full citations, with DOIs, are in ACKNOWLEDGEMENT.md)

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS_DECISIONS.html b/CPU_DATA_ANALYSIS_DECISIONS.html new file mode 100644 index 000000000..17d67d799 --- /dev/null +++ b/CPU_DATA_ANALYSIS_DECISIONS.html @@ -0,0 +1 @@ + Data analysis: space group and validation (§13–§14) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Data analysis: space group and validation (§13–§14)

Part of the CPU/GPU data-analysis reference; the section numbers are continuous across its four parts.

13. Space-group determination and merge-level decisions

13.1 Space-group determination

When no space group is supplied, a POINTLESS-like search scores Laue-group symmetry (CC of \(E^2(h)\) vs \(E^2(Rh)\) — the intensities normalised by the mean of their own resolution shell — plus merge self-consistency) and detects screw/centering absences from the \(P1\)-merged intensities. Three tests weigh a promotion to higher symmetry, all aimed at the merohedral twin, whose twin law forces non-equivalent reflections together and so mimics symmetry:

  1. Merge self-consistency (\(\chi^2\) under the candidate group, relative to the confirmed subgroup). On its own this is not sufficient: it is a ratio to an error model that moves with the amount of data — the parent’s systematic term grows as \(\sigma\) shrinks with \(1/\sqrt{N}\), while a twin’s is already saturated — so its verdict depends on how much data the search saw.

  2. Error-model \(b\) (the intensity-proportional systematic). A genuine symmetry step gains multiplicity without inflating \(b\); merging a twin law’s extra operator inflates it. This is a rescue and never a veto: a \(\chi^2\)-borderline promotion whose \(b\) stays close to the confirmed subgroup’s is confirmed on it, which is what recovers a genuine high-symmetry group on imperfectly-scaled data, but it cannot demote a promotion that passed. The veto it once carried was removed. It was guarded by “\(H\) could not be computed”, so it could only ever act where the one statistic that discriminates here is blind, and what it compared there was not a measurement but the error-model bisection’s own ceiling — which made a space group depend on the compiler flags, the same source refusing a correct \(P2_12_12_1\) in one build and not in the other. No bound repairs it: genuine promotions have since been measured up to 5.8 where the bound was calibrated at 1.62 against a twin at 2.1, so the two populations have swapped sides. The case it was meant to catch — a merohedral twin too sparse for \(H\) — is answered below it by the added-operator \(R\) gate, whose reference is the global best rather than the parent.

  3. Operator disagreement, a sigma-free statistic \(H=\mathrm{median}\,|I_1-I_2|/(I_1+I_2)\), formed as the ratio of the operators a promotion adds to the parent’s own, measured on the same reflections. Normalising against the parent divides out the systematic floor that symmetry mates carry on real data, which varies by crystal and by operator; a median is used because a twin perturbs every pair whereas a badly-measured minority perturbs only the tail. Where a candidate has several parents of the same order, it is judged against the worst of them, since a rival subgroup can itself contain the twin laws.

The correlation is on resolution-normalised intensity \(E^2 = I/\langle I\rangle(\text{shell})\), normalised over exactly the reflections the correlation pairs. Both members of a symmetry pair lie at the same \(|s|\), so on raw \(I\) the resolution fall-off is variance shared perfectly between the two arms and appears as a positive correlation for any pairing at all: a shell-matched random pairing — the exact null for a metrically-allowed false operator — scores a median 0.31 across the rotation battery, and on one crystal 0.53 — above the bound the correlation is tested against. That floor varies more from crystal to crystal (spread 0.46) than the whole true/false gap is wide (0.38), so an absolute bound on the raw statistic is a different test on every crystal; and it moves with the search resolution cut, which is what made that cut a symmetry-deciding parameter. Normalised, the floor has a median of 0.015, never exceeds 0.06, and barely moves with the cut.

Both the correlation stage and the absence tests need to know whether a reflection is genuinely present, and that question is asked of its counting significance, not of the merged \(I/\sigma\). A merged \(\sigma\) carries the error model’s intensity-proportional term, \(\sigma^2 = a\,\sigma_0^2 + (b\,I)^2\), so merged \(I/\sigma\) saturates — at \(\mathrm{ISa}\sqrt{n}\) for a reflection observed \(n\) times, and at \(\mathrm{ISa}\) exactly for one observed once. Above that knee it stops rising with the intensity: on the weakest search merge of the rotation battery the \(I/\sigma\) of every decile of \(E^2\) reads 1.58–1.65 against an \(\mathrm{ISa}\) of 1.70, so reflections an order of magnitude apart in real intensity report the same number. A single constant applied there demands anywhere between 2.2 and 11.9 in counting significance depending on the crystal.

The nominal cut \(T\) (default 3.0) is therefore converted once, using the ISa of the merge being searched. For a reflection observed once \(\sigma_\text{counting}^2 = \sigma^2 - (b\,I)^2\), so \(I/\sigma_\text{counting} \ge T\) is exactly

\[\frac{I}{\sigma} \;\ge\; \frac{T}{\sqrt{1 + (T/\mathrm{ISa})^2}}\]

The converted cut lies strictly below \(\mathrm{ISa}\) for every \(\mathrm{ISa}\), so it is always reachable by the reflection the ceiling binds hardest, and it is within 1 % of \(T\) on any merge with \(\mathrm{ISa} \ge 21\) — a healthy merge is left exactly where it was. Multiplicity is taken as 1 deliberately rather than estimated, for the same reason. Where the merge reports no ISa (or \(b = 0\)) the cut is used as it stands.

The candidates are enumerated in every setting the refined cell can host, not only in the settings the International Tables call the reference one. A setting is a statement about direction — \(P112_1\) puts its \(2_1\) on \(c\) where \(P12_11\) puts it on \(b\), and both are space group 4 — so a search restricted to reference settings can only ever put a screw or a centring on the axis the convention chose, whatever the data say. Two things bound the widened set. The cell’s own metric is asked once, when a point group’s rotation set is chosen — a set whose rotations it does not host (\(\alpha=\beta=90^\circ\) for a \(2\)-fold on \(c\)) is not offered — and every setting of a chosen set is then offered without asking again, because a cell refined free after integration sits a few tenths of a degree off \(90^\circ\) and asking per setting kept only the reference one; and a candidate predicting exactly the absences another candidate already predicts is dropped as the same hypothesis under a second name. A non-reference setting is additionally refused when its centring class holds no reflection in this merge, since there is then nothing to confirm it with. Because the candidates of a point group all share its rotations, this cannot raise the symmetry: it decides which axes carry the screws and the centring, never how many operators there are. The setting found this way is the one the decisions are made in. The files are then written in the ITA standard setting (as XDS and POINTLESS write it: \(P2_12_12\) with the pure axis on \(c\) and \(a<b\), monoclinic \(b\)-unique and \(C\)-centred with \(\beta\ge 90^\circ\), the other orthorhombic groups \(a\le b\le c\), triclinic reduced), or in the one the user supplied — a reference MTZ, a model that fits, the axis order of -C, a non-standard -S symbol, in that priority. That is an integer, determinant \(+1\) change of basis applied after every decision, to the merged and unmerged files and the _process.h5 alike, and reported as SETTING_OPERATOR; where the written group is not the reference setting (because the user asked for another) the report and the files name it by its full Hermann–Mauguin symbol. A group given with -S is placed on the axes its absences name before the merge: every change of basis that keeps its metric and centring is a candidate, each class of axial and zonal reflections some candidate predicts absent is read as the ratio of its mean \(I/\sigma\) to that of the reflections none predicts absent, and the candidate that calls absent the classes that read weak and present the ones that read strong wins.

Several space groups may be indistinguishable on the measured data. Where they are, the search scores them identically and all of them are named in the result rather than one being reported as the answer: some are enantiomorph pairs, which merged intensities cannot distinguish in principle, others differ only by a screw condition that the centering condition already implies, so the screw has no observable signature at all, and others differ only on a zone the sweep never measured. Equality is keyed on the measured absence evidence, not on a count of predicted absences — two settings of one point group routinely predict a different number of absences on an unmeasured zone, which argues nothing either way. Among the tied candidates, where at least one axial zone was judged, the representative is the one claiming a screw on the rows this merge holds no control class for — a stated prior, not a measurement: a crystal that has already shown one screw is the group carrying the rest an order of magnitude more often than not, and a sweep short of half a turn routinely records one axial row and not the next. Centring-driven axial conditions do not count towards the claim, so a centred group and its screw partner still tie. What remains tied is represented by the lowest space-group number, which is a convention and not a measurement.

Glide planes are detected the same way screws are, from the zone they extinguish: a glide removes half of a whole zone (\(h0l\) with \(h+l\) odd for an \(n\) glide perpendicular to \(\mathbf{b}\), and so on), and the extinguished class is scored against the rest of its own plane, as POINTLESS scores zonal absences. The statistic is per reflection rather than the zone’s sum — a plane holds hundreds to thousands of reflections where an axial row holds tens, so a summed statistic reaches enormous values on a class that is merely a few times weak, where per reflection the false and the genuine zones separate cleanly — and a zone that cannot be measured refuses its candidate rather than abstaining, because a glide is an extra claim on top of a group that already fits without it. Candidates whose glides share a plane and differ only in the translation (the \(c\) and the \(n\) of the same plane) are one zone, keyed by the rotation part of the improper operator, not two scored twice. A Sohncke group has no improper operator at all, so on chiral data the zone loop never runs and every Sohncke candidate scores exactly what it scored before. Both readings are reported on every searched run — SPACE_GROUP_NAME= carries the glide where one was found, and SOHNCKE_SPACE_GROUP= the best group without — because a crystal of chiral molecules cannot have a glide plane, and a reader who knows the sample is a protein must be able to read the Sohncke answer without processing the images again. Non-Sohncke candidates are enumerated by Laue class, so the groups without a centre of symmetry (Pc, Pna2_1, I-42d, Fdd2) are candidates as well as the centrosymmetric ones; a candidate is enumerated only where its absences differ from a Sohncke candidate’s, because an inversion centre or a mirror alone predicts exactly what the Sohncke subgroup predicts. A centre of symmetry is then settled by the absences where they can settle it: no group without a centre predicts the absences of P2_1/c, Pbca or Ia-3d, and no centrosymmetric group those of I-42d or Fdd2. Where two non-Sohncke groups predict the same absences (C2/c and Cc, Pnma and Pna2_1, I4_1md and I-42d) they are reported as alternatives these data cannot separate (SPACE_GROUP_CENTRE=NOT_DETERMINED), and the centrosymmetric one is written by default — a missed centre is the common error in small-molecule space-group assignment (Baur & Kassner 1992). That default gives way only when the intensity distribution of the general reflections reads acentric on <|L|>, <|E²-1|> and N(0.1), each calibrated against acentric and centric intensities simulated with the reflections’ own sigmas, every reading lies in the range a Wilson population can produce, the reflections centric in both groups read centric, and the lattice excludes twinning (the metric admits no rotation beyond the Laue class and no twin-domain lattice was found), because a twinned centrosymmetric crystal reads exactly like an untwinned acentric one. Between two groups without a centre, the lowest-numbered is written.

The Lorentz factor \(\zeta\) (§8.3) governs how well a reflection can be measured, so when the spindle lies in a plane of the lattice, an operator permuting the two in-plane axes samples a different mixture of measurement qualities than one that only flips signs. The search is therefore run a second time on a merge of only the well-measured observations (--search-min-zeta, rotation default 0.85), both answers are reported, and where they disagree the merge of all the observations decides. The filter discards 40–80 % of the observations, which can starve an operator correlation the full merge confirms and can equally leave an operator confirmed that the full merge refuses, so the decision — the point group as well as the absences, which live in the weak reflections the filter removes — rests on the arm with every observation behind it. A tie (same order, different symmetry) is reported with both candidates named, for trying in molecular replacement.

Both search merges are cut in resolution where the data stop carrying signal: the shell means of \(\langle I/\sigma\rangle\) are given their best non-increasing fit (pool-adjacent-violators, weighted by shell size) and the cut is where the fit drops under 1. Cutting at the first single shell to dip under 1 made the cut a coin toss on a merge whose \(\langle I/\sigma\rangle\) is flat near 1: one crystal was cut at 0.85 Å or 1.19 Å depending on the compiler flags, and the coarser cut left its glide zones too few absences to be judged. The fit moves only as much as its input does, and on a profile that already falls monotonically the cut is unchanged.

Both search merges also drop the frames whose fitted per-frame scale came out below a tenth of the run median — the stretches where the crystal was barely in the beam. The scale enters as \(1/G\), so such a frame’s intensities arrive amplified tenfold or more, with their \(\sigma\) amplified by the identical factor and the scale’s own error nowhere in it; the production merge survives that because a reflection in the determined symmetry is measured ten or twenty times and --reject-outliers removes the amplified observation, but the search merges in \(P1\), where a reflection has two or three observations and no majority exists to call any of them an outlier. On a crystal that repeatedly left the beam over a full turn this cost the \(422\) point group outright, its operators reading \(0.08\)–\(0.38\) on a merge that gives \(\mathrm{CC}_{1/2} = 96\,\%\) in that same point group once it is assumed. Like the \(\zeta\) filter, this is a filter of the search passes alone — the production merge keeps every frame.

Screw axes are scored per axial zone, not pooled over them. The conditions on \(h00\), \(0k0\) and \(00l\) are independent, so their log-likelihoods add, and a zone that was never measured is allowed to abstain rather than to argue: pooling let one unmeasured row veto a confirmed one, and it read three genuine screws as weaker than two whenever the third row was shallow. A screw is claimed when its absence evidence reaches 8 nats. The bound is set on the zones themselves, every single-axis candidate of a battery of rotation datasets labelled against the deposited group: a false zone with no violation reads at most +4 nats, a true one from +10, and the true zones below the former bound of 20 were all short rows — a \(2_1\) along a monoclinic axis of 25–30 Å reaches four to six odd reflections inside the search’s resolution range, which on weak data sit at a few per cent of their row rather than at zero. Two absences clear the bound if both read below about 2 % of their row.

A pseudo-translation is divided out before an axial absence is judged. A translational pseudo-symmetry (§13.2) splits the reflections into two classes by the parity of the index along the translation, one systematically strong and the other systematically weak. Where the translation is half-integer along an axis, those two classes are exactly the absent class and the control class of a screw on that same axis, so the absence test would divide one by the other and pay the suppression twice — reading a class that is present but suppressed as extinct, and buying a screw the crystal does not have. The modulation is therefore measured along each axis from the intensities themselves — per axis, not pooled, since one row can be strongly modulated while its neighbours are not — and divided out of both the absence evidence and the violation count before the screw is scored. Only order-two screws are treated this way; a three-fold with a one-third translation is left unanswered rather than answered no.

The violation count can be deferred to the absences, one zone at a time. A screw candidate is refused when too many of its predicted-absent reflections read as present — but on a strong axial row near the spindle those readings need not be structure factors at all: the same reflection can read far positive at one Ewald crossing and negative at the other, the directional smear tail of the strong reflections beside it. A zone whose per-reflection absence evidence clears the claim bar may therefore stand despite its count, since the likelihood has already priced those reflections in and still reads the class as extinct. The deferral is licensed zone by zone — the evidence is a group-level number while the count indicts particular zones, so an overwhelming genuine zone must not pay another zone’s debts — and it is not available on a zone corrected for a measured pseudo-translation, where the corrected count is the one instrument the modulation does not reach.

Centering is accepted when the systematically-absent class is weak relative to the present one by either of two floor-independent tests: its mean signed \(I/\sigma\) well below the present mean, or its rate of individually-significant reflections well below the present class’s own significant rate. The second test covers weak and low-energy data, where a positive intensity floor (background and profile leakage) lifts the absent class’s mean \(I/\sigma\) well above zero and, when the present class is itself weak, carries the plain mean ratio past its bound; a false centering fails both tests, its absent class being as strong as the present one. When several centerings pass they are ranked by a Beta-tail likelihood rather than by a count of net absences: a count lets a centering that extinguishes many more reflections out-rank one that violates far fewer, and the likelihood weighs the violations against the class each belongs to instead. The acceptance bounds above are unchanged by that ranking.

A confirmed promotion that is refused is decided by the twin-immune zone of its added operators, and only where that is undecided by merging under it. The zone (§13.2) is the one reading that does not depend on the per-frame scales or on a reference operator: read on the all-observation P1 merge where the two search arms disagree, and on the adopted group’s own merge at the remerge, an index-2 promotion whose calibrated zone evidence is acentric by 20 nats stays refused — on all observations, and the first added operator is recorded as the twin law — and one whose zone is centric by 20 nats is taken. Where the zone cannot decide (a higher index, no zone reflections in the shells with signal, within the margin), the merge is asked. Every gate that refuses a confirmed higher point group is a ratio — the added operators’ disagreement against the parent’s, or their \(R\) against the best-agreeing operator anywhere — and both references are properties of how the crystal was mounted rather than of its symmetry. A 2-fold within a degree of the spindle records its mates on the same detector pixel half a turn later, so it carries no geometry-dependent systematic at all, and by being that clean it makes every other operator look bad against it; the best-agreeing operator can also be one of the operators under test, which collapses the ratio on worse data and raises it on better. So where a promotion is confirmed and then refused, the merge is asked instead of the ratio. Both groups are scaled and merged over one pinned resolution range — the adopted group’s own cut — and the promotion is taken only if neither \(R_\mathrm{meas}\) nor ISa gets worse. \(R_\mathrm{meas}\) is multiplicity-corrected, so folding non-equivalent reflections together has to inflate it, and the error model is refitted per merge, so ISa says whether the extra multiplicity was bought with systematic disagreement. \(CC_{1/2}\) and \(\langle I/\sigma\rangle\) cannot arbitrate this: both rise on a false promotion too. Two details the comparison turns on: the refused point group’s representative is primitive and symmorphic, so the arm merged under it takes the adopted group’s centring — merged in the bare representative, a centred crystal is handed its centring-absent class as data, which inverts the decision — and the absence stage is asked of the adopted merge, so both arms carry the same absence classes and differ by the added rotations alone. That merge no longer holds the adopted group’s screw-absent axial reflections, so the higher group also keeps the screws the adopted group decided — among the higher candidates the merge cannot separate, the one that predicts every such axial absence is taken — rather than the lowest-numbered one, which claims none. The cost is two extra merges, and only on a run that records a refusal, which is a few in a hundred.

On a merge that reads twinned, the search asks the zone itself. A twin of an index-2 subgroup by exactly the operators a promotion adds defeats every agreement gate once its fraction is high — the added operators then agree like real ones — so where the P1 merge’s \(\langle|L|\rangle\) lies in the partial-twin band (0.375 to 0.44) and the candidate does not hold every rotation of its lattice (so a twin law outside it is possible), the candidate is refused when the twin-immune zone of one of its index-2 subgroups reads acentric by 20 nats. Below 0.375 something other than a twin compresses the intensities, the zones with them, and they are not read. A \(P3_1\) crystal twinned at \(\alpha \ge 0.4\) by its 321 law, whose 32 promotion every agreement gate passed, reads −230 nats there.

A point group’s own operators must agree with each other. A point group is the claim that all of its operators are symmetries of one crystal, and a mixture of real and false operators has a specific shape: the real ones form a subgroup — the true group’s intersection with the candidate — and everything outside it is false. The gate therefore reads the operators sorted by their \(R\) against the merge, finds every prefix that closes into a subgroup, and takes the largest factor by which every operator outside such a subgroup reads worse than every operator inside it; above 3.2 the candidate is refused. A genuine group whose operators merely spread over a continuum — anisotropic systematic error spreads a cubic crystal’s operators twofold — has no such gap. The earlier form, worst-agreeing over best-agreeing operator, divided by an extreme order statistic and refused a genuine \(432\) as soon as one of its operators happened to agree unusually well.

A group that holds every rotation of its lattice is refused when merging under it narrows the L-test like a twin. Every gate above compares the added operators with a reference, and a pseudo-symmetric structure or a twin defeats them all. For a candidate that holds every rotation its lattice admits no twin law exists, so intensities merged under it that read like a twin’s are not a twin. The Padilla–Yeates \(L\)-test is read twice on the same pairs of the \(P1\) merge — as measured, and with every intensity replaced by the mean of its orbit under the candidate — and the candidate is refused where the merged \(\langle|L| angle\) reads twin-like (\(0.375 \le \langle|L| angle < 0.44\)) and the averaging closed a third or more of the gap from the unmerged value to 0.375. Genuine lattice-holohedral groups close 0.01–0.25 of it. A merge reading below 0.375 is not read: nothing a twin or a false operator does reaches that, so something else compresses the intensities.

The lattice class is re-asked where the metric knows more. The Bravais class comes from Niggli reduction and a lattice-character lookup (§6), and that lookup can land short of the truth or beside it: near the Niggli type-I/type-II boundary it is decided by the last digits of the refined cell, and a lattice that is nearly but not exactly hexagonal matches the hexagonal character even when no point group that class can host carries the two-folds the data actually have. Two recoveries run, both settled by the intensities. Every rotation the cell metric can host beyond the named class — read off Le Page’s two-fold search on the lattice itself, which measures each rotation’s obliquity in a primitive basis and owes nothing to the character table — is put to the intensities as a single operator, scored exactly as the search scores its own operators, on the same reflection population and the same \(E^2\) normalisation. And where the metric group is larger than the adopted class’s holohedry, the merge is reindexed into the metric group’s conventional cell and the space-group search is run again there, with every gate live; the reindex is committed only where the search in the new setting confirms a strictly higher point group and the centring the new cell describes, so a pseudo-symmetric metric leaves the answer already in hand standing.

13.2 Twinning check, and translational pseudo-symmetry

A Padilla–Yeates \(L\)-test (\(\langle|L|\rangle\), \(\langle L^2\rangle\) — 0.500 and 0.333 untwinned, 0.375 and 0.200 for a perfect twin) and the second moment \(\langle I^2\rangle/\langle I\rangle^2\) (2.0 for untwinned acentric data, 1.5 for a perfect twin) are written to the merged mmCIF as a twinning diagnostic. Both are taken on intensities divided by their resolution-shell mean, over the shells whose \(\langle I/\sigma\rangle\) reaches 1, with Wilson outliers rejected; the \(L\)-test pairs are therefore compared on one scale even where two index steps span a steep fall-off, as in phenix.xtriage and ctruncate, and the selection is by shell, never by the individual reflection’s \(I/\sigma\), which would cut the weak tail and bias \(\langle|L|\rangle\) down. The twin fraction is quoted from the statistic that carries the verdict — the \(L\)-test unless the call rests on the second moment alone — and a second moment is not turned into a fraction under a detected pseudo-translation, which inflates it. A merohedral twin law exists only where the Laue class is a proper subgroup of the lattice holohedry, so in the high-symmetry holohedral classes (\(4/mmm\), \(6/mmm\), \(m\bar{3}m\), and \(\bar{3}m\) on a rhombohedral lattice) no twin is called. The \(L\)-test is still read there, for a different question: merging \(I(h)\) with \(I(Th)\) under a false operator \(T\) gives \((I(h)+I(Th))/2\) whatever the twin fraction, which has the perfect-twin distribution, so \(\langle|L|\rangle\) below 0.42 in a holohedral class is reported as an adopted operator averaging unequal intensities — the space group is too high, or a twin law was absorbed into the point group — a warning, never a change to the space group. A genuine operator leaves the untwinned 0.5. Reflections that overlap along a very long axis narrow the distribution the same way, which is one reason it stays a warning. The same numbers measured on the P1 merge the space-group search was given, before any point group was adopted, are reported beside them (_BEFORE_SEARCH), with that merge’s own pseudo-translation declared and the reflections its lattice centring extinguishes left out. The low-symmetry holohedral classes (\(\bar{1}\), \(2/m\), \(mmm\)) also admit no strictly merohedral law, but they stay eligible for the twin call on purpose: pseudo-merohedral twinning through an accidentally special metric cannot be ruled out from the symmetry alone, and those are the classes it happens in.

Beside them, on rotation data, the run reports twin-immune zone evidence for the operators the adopted point group adds over each of its index-2 subgroups, read on the P1 cross-check merge. Under a twin law \(T\), \(I_\mathrm{obs}(h)=(1-\alpha)I(h)+\alpha I(Th)\); a reflection whose twin mate is itself up to the subgroup and Friedel is untouched at every \(\alpha\), and those are exactly the reflections centric in the group but acentric in the subgroup. They read centric (\(\langle|E^2-1|\rangle = 0.968\)) if the added operators are real and acentric (0.736) if they are a twin law or a pseudo-symmetry — the one intensity statistic that still separates the two at \(\alpha = 0.5\), where every operator statistic reads “real”. Each zone is normalised against its own mean in resolution bins (and within the two phase classes of a detected pseudo-translation), read only in shells with \(\langle I/\sigma\rangle \ge 5\) because noise inflates every class towards centric, and reported with \(n\), a standard error and the centric-over-acentric Wilson log-likelihood ratio in nats (both densities convolved with each reflection’s measurement error), beside the acentric control. It is read absolutely, never as the difference to the control: a perfect twin’s control (0.541) plus noise inflation would otherwise look like true symmetry. An absolute reading is only as good as the normalisation, though, and the acentric control certifies it: an acentric population reads \(-0.130\) nats per reflection when the normalisation is right, and whatever the control reads above that is the normalisation’s — anisotropy, a pseudo-translation, a pseudo-centring, noise all inflate every class towards centric alike — so the zone, normalised the same way, carries the same per reflection and the calibrated evidence has it taken off (a twinned control reads below the expectation, so nothing is taken off a twin). The centric side is read twinned at the fraction the lattice’s other operations show by their own correlation — the strongest CC of a lattice rotation outside the group, relative to the mean CC of the group’s own operators, inverted through \(\rho = 2\alpha(1-\alpha)/((1-\alpha)^2+\alpha^2)\) — because a twin by another law reaches a genuine zone exactly as it reaches the control: a genuine 321 crystal twinned by a 622 law read its 2-folds at −206 nats against the untwinned centric density and +209 at that law’s fraction of 0.07. The operators’ own twin law cannot reach their zone, so the acentric side stays untwinned, and the control’s expectation is taken at the same fraction. Measured, a 6/m crystal with a 67 Ų anisotropy read its control at \(+0.10\) nats per reflection and the zone of a refused 622 the same, so the zone’s \(+124\) nats were the normalisation’s; calibrated it reads \(-115\), and genuine promotions keep \(+200\) and above. The calibrated zone is what a refused promotion is decided on (above). It is an in-house method: published practice (Yeates’ \(H\)-test, xtriage) excludes these reflections as uninformative about the twin fraction, and no published test was found that uses them positively; the one precedent here is a trigonal crystal whose twin-immune zone read centric and whose higher group was then confirmed by refinement.

Translational pseudo-symmetry — two copies of the contents of the asymmetric unit related by a pure translation that is not a lattice vector — is looked for on every merging run, because it is the classic predictor of a failed molecular replacement, and because it raises the second moment where twinning lowers it, so each can mask the other’s test. The detection is the native-Patterson route of Read, Adams & McCoy (see the references): the largest off-origin peak of a Patterson computed from the merged intensities, as a fraction of the origin peak, with the peak vector then refined against the intensity modulation it should produce — the ratio of the strongest to the weakest bin mean of \(\langle E^2\rangle\) over the phase \(\mathrm{frac}(\mathbf{h}\cdot\mathbf{u})\). Both halves are scored against a null computed for the crystal at hand rather than against a fixed bound — both against the same intensities permuted within resolution shells — the peak against that map, the modulation against the same measurement made on the shuffled intensities and started where the measurement starts — because the noise floor of the peak statistic spans an order of magnitude across data sets, so no fixed percentage means the same thing twice. The shuffle leaves the reflection count, the \(E^2\) distribution and the phase-bin populations untouched and removes only the correlation between a reflection’s intensity and \(\mathbf{h}\cdot\mathbf{u}\), which is the thing the modulation claims to see. Re-running the greedy search from random starting vectors over the same intensities does not work as a control: a greedy search started anywhere walks into a real modulation’s basin, so it measures the search and whatever modulation the crystal carries — measured over 110 merged datasets, single draws of that control reached 199× and 6162× on crystals whose own modulation reads 4.0× and 67×, and on 28 of the 110 it came out at or above the signal it is subtracted from. Requiring both halves is what keeps the false-positive rate down; either alone over-calls by about a factor of two. A translation the merged data are exactly invariant under is reported as an undeclared lattice translation instead — a translation the data are exactly invariant under is a lattice vector by definition, so the centring or the cell is wrong, not the packing — and the pseudo-symmetry search continues underneath it, so a real pseudo-translation sitting under an undeclared centring is still found. The finding itself is report-only (the TNCS_* keys of <prefix>_report.txt): it gates nothing and changes no reflection, no scale and no group — but the axial-absence test (§13.1) and the \(L\)-test here both correct themselves against the modulation it measures.

The \(L\)-test partners are chosen so a pseudo-translation cannot silence it. The test compares each reflection with a partner a fixed index step away, and a pseudo-translation biases \(\langle|L|\rangle\) upwards unless the partner shares its modulation class. The ordinary step of 2 preserves the class of a half-integer translation — the pseudo-centering the statistic’s authors call it robust to — but not of one of a third, which inflates \(\langle|L|\rangle\) past the bound that is read as evidence against twinning and silently loses the twin call on a crystal that has one. Where a pseudo-translation is detected, the partner steps are restricted to those that preserve its class; where no step does, the statistic is dropped from the twin verdict in both directions — it can no longer indicate a twin and can no longer be read as proof that there is none — and the second moment decides alone. L_TEST_VS_TNCS= in the report says which of the three happened.

13.3 Outlier rejection

Merging applies an optional per-observation median-based \(N\sigma\) cut (--reject-outliers, default 6σ for rot3d, off otherwise). On the rotation path the median is weighted by each full’s counting variance taken at the reflection’s expected intensity, as the merge weights are (§10.4): an observation’s own Poisson variance falls with its own intensity, so own-variance weights hand the median to whichever equivalents came out low — on a strongly absorbing crystal, where some frames read equivalents ten to a hundred times down, the median sat on the absorbed ones and the test removed correct measurements far above it. The band about the median keeps the counting part linear, \(\pm N\sigma_\text{counting}\), but takes the systematic part as a factor, \(e^{\pm N s}\) about the expected intensity: the error model’s systematic term is multiplicative — absorption, illuminated volume, a scale that is off — and as likely to be a factor \(1/f\) as \(f\), while a band symmetric in \(I\) reaches far below the median and only a little above it once that term is large. \(s\) is the spread of \(\ln(I/\text{median})\) measured on the merge’s own fulls (their weighted median of \(|\ln(I/\text{median})|\), weighted by \((\langle I\rangle/\sigma_\text{counting})^2\) so the fulls whose ratio counting noise does not blur carry it), with the error model’s \(b\) as the fallback where it cannot be measured; to first order in \(Ns\) the band is the linear one. The same \(N\sigma\) cut is fed back into the error model: after an initial \(a,b\) fit the parameters are re-fit once on the reflections that survive rejection (dropping any whose squared deviation exceeds \(N^2\,[a\,\sigma^2 + (b\,\langle I\rangle)^2]\)), so the calibrated errors describe the reflections that actually enter the merge rather than the pre-rejection pool.

The median needs three observations, so a reflection measured once or twice — typically one good observation and one artefact (a hot pixel, a zinger) after Friedel merging — is never tested by it, and an artefact that looks precise (thousands of counts, a small relative \(\sigma\)) can outweigh mates from weak frames and become the median itself. Every observation of the written merge is therefore also judged by Wilson statistics. Beyond 4 Å its \(E^2 = I/(\varepsilon\,\langle I/\varepsilon\rangle_\mathrm{shell})\) is compared with a bound set by a budget of \(\alpha = 0.01\) expected false rejections per dataset: with \(N\) observations tested, \(\ln(2N/\alpha)\) for acentrics and twice that for centrics, and the observation’s lower confidence limit \(I - z\sigma\) (with its own \(\sigma\), and \(z\) from the same budget) must exceed it, so noise in weak shells does not trigger it; a shell whose \(\langle I\rangle\) is not itself established at that significance is not judged. The bound is widened by the tail scale of the dataset’s own intensity distribution, measured on its well-measured observations (1 for a Wilson crystal; larger under a pseudo-translation or anisotropy). An improbable singleton is dropped; an improbable observation with company only when most of the reflection’s other observations are probable and it disagrees with their mean beyond the errors — several large observations confirm each other. An observation whose signal disk lost pixels to the mask or to saturation is never counted as such a witness, since its profile estimate may be low: a strong reflection is not replaced by its clipped mate. The count is OBSERVATIONS_REJECTED_WILSON= in the report (included in OBSERVATIONS_REJECTED=), and the developer report lists each observation with its image and detector position.

13.4 Automatic resolution cutoff

By default the reported/written high-resolution limit is trimmed where \(\mathrm{CC}_{1/2}\) falls off: a logistic is fitted to \(\mathrm{CC}_{1/2}(s)\), and the limit is set one reported-shell width past the point where the fit crosses 0.30 — deliberately “one shell too far”, so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. --scaling-high-resolution overrides the limit and --resolution-cutoff off disables it.

Ice sits out of this fit, and only this fit. A powder ring is reproducible: past the crystal’s own limit the ice is still there and still the same in both half-sets, so the two halves agree about the ring, and a Pearson \(\mathrm{CC}_{1/2}\) cannot tell that agreement from diffraction. Shells have been measured at \(\mathrm{CC}_{1/2}=0.765\) where \(\langle I/\sigma\rangle\) is \(-0.1\) and \(R_\mathrm{meas}\) is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, still written and still counted in the shell table (§10.10), and what changes is only that the resolution decision now reads the same curve that scaling, the error model and the space-group search already read.

A cut the shell table refutes is not quoted. Two things keep the number inside what the bins show. The fitted crossing has to lie inside the fitted bins: on a real fall-off it sits between the last bin at the target and the first bin below it, with the extension bins to spare, and a crossing past the last fitted bin is an extrapolation of a fall-off these data never showed — the crossing is then read off the bins themselves, between the two that straddle the target. And no single resolution is quoted at all where the curve is not a fall-off: where \(\mathrm{CC}_{1/2}\) never reaches the target in any shell, where the fit reads finer than the finest shell whose \(\mathrm{CC}_{1/2}\) still reaches it, or where \(\mathrm{CC}_{1/2}\) climbs back over the target after falling below it — a noise shell that climbs back is not the edge of the data. The report then says which of the three happened and points at the shell table instead.

13.5 Diffraction anisotropy

How fast the intensity falls off with resolution can depend on direction. Rugnux measures that and reports it. No intensity is corrected and no reflection is removed on a directional criterion; the one use of the tensor is the Wilson prior of the French–Wilson amplitudes (§10.8), so F/SIGF follow the fall-off along each reflection’s direction while the intensities do not depend on direction at all.

The tensor. A deviatoric anisotropic displacement tensor is fitted to the merged intensities as

\[\ln \langle I(\mathbf{s})\rangle = c(\text{shell}) - \tfrac{1}{2}\,\mathbf{s}^\mathsf{T} B\, \mathbf{s},\qquad \mathbf{s} = \text{reciprocal-space vector},\ |\mathbf{s}| = 1/d\]

(§10.6 and §14.2 write \(s\) for \(\sin\theta/\lambda = 1/2d\), so their \(s^2 = 1/4d^2\); the two conventions give the same exponent \(-(B/2)(1/d^2)\), and \(B\) is the same \(B\).)

with one free constant per resolution shell, so every isotropic feature — the Wilson curve, an ice ring, a noise floor, a scaling error — is absorbed exactly and only the \(\ell = 2\) angular part drives the tensor. For an isotropic \(B\) this reduces to the ordinary Wilson plot, so \(B\) here is the ordinary crystallographic (\(B = 8\pi^2 U\)) \(B\), directly comparable with phenix.xtriage’s B_cart, ctruncate’s anisotropic \(B\) eigenvalues and AIMLESS’s anisotropic \(\Delta B\). Only the deviatoric part is fitted: the isotropic part is degenerate with the overall scale. The tensor is constrained to the directions the Laue class allows — five free deviatoric parameters in triclinic, three in monoclinic, two in orthorhombic, one in tetragonal, trigonal and hexagonal, and none at all in cubic, where symmetry forces \(\Delta B\) to be exactly zero.

The fit is on intensities, with no positivity cut. Fitting amplitudes, or dropping non-positive intensities as an amplitude-based tool must, loses roughly 40% of the signal: in a direction that has died half the merged intensities are negative, so a positivity cut keeps only the positive noise excursions and flattens the fall-off exactly where the anisotropy is largest.

Two different quantities are reported, and they are not interchangeable. \(\Delta B\) (the range of the principal components) is a rate; the diffraction limit along each principal direction — where \(\langle I/\sigma(I)\rangle\) in a 20° cone about that direction falls through 2, read by interpolation in \(s^2\) over equal-count shells — is where the signal actually runs out. A crystal can have a large \(\Delta B\) and almost no spread in directional limit, or the reverse. Where \(\langle I/\sigma(I)\rangle\) never falls through 2 in a direction, the limit returned is the edge of the measured data rather than the crystal’s own; such a direction is marked — with a < in the report, a 1 in ANISOTROPY_D_MIN_CENSORED, and a note in the mmCIF — so the spread is not read as a measurement when it is a lower bound. Both constants are AIMLESS’s defaults (cone half-angle 20°, \(\langle I/\sigma\rangle = 2\)). The 2 is a per-direction diagnostic level only: the dataset-wide resolution cut (§13.4) is CC\(_{1/2}\)-based, and the two criteria are not interchangeable.

The resolution signature. A genuine Debye–Waller \(B\) makes the directional deficit a straight line through the origin in \(s^2\). The per-shell \(\ell = 2\) amplitude is therefore fitted against \(s^2\) and the curve is classified: linear (a real \(B\)), flat (a deficit that does not follow \(\exp(-\tfrac12 \mathbf{s}^\mathsf{T} B \mathbf{s})\) at all, so the fitted \(\Delta B\) describes the data with the wrong functional form and may be an under-estimate), or convex (a deficit that grows faster than \(s^2\), which a \(B\) cannot do). The verdict is re-derived at 8 and at 16 shells, and reported as undetermined if it moves.

The verdict, and what it is measured against. Whether an anisotropy is real is not decided against a counting-statistics error bar. Real data carry systematic error far larger than counting error, and gating on the latter reports anisotropy on datasets that have none. Instead the data set measures its own systematic error: in the tensor directions the Laue class forbids, the true tensor is exactly zero whatever the crystal is, so whatever is measured there is systematic. That measurement needs the unmerged observations — a merge has exact Laue symmetry by construction, and the forbidden directions are identically zero in it — so it is made on the scaled, rocking-curve-assembled observations. The counting part is subtracted, the counting error of the directions actually being tested is added back, and the result is the floor. What is tested against it is \(\Delta B_\text{linear}\) — the part of the fall-off that actually follows \(\exp(-\tfrac12\mathbf{s}^\mathsf{T}B\mathbf{s})\), clamped at zero — and not the headline \(\Delta B\); the report names which of the two it is quoting. The ratio is banded: below 2 not established, 2–3.5 marginal, above 3.5 established, above 5 strong. The bands are calibrated against known ground truth — merging cubic crystals in proper subgroups of their own Laue class, where the true anisotropy is exactly zero — which puts the false-positive rate at 24 % at 2.0, 10 % at 3.5 and 5 % at 5.0.

The report says NOT DETECTED, DETECTED, or CANNOT DETERMINE, and the third is a real answer rather than an evasion. It is returned when the Laue class is triclinic (no forbidden direction exists, so there is no internal measurement of the systematic error and no substitute for it), when the observed rotation range is under about 90° (a lab-fixed systematic then reaches several tensor directions instead of one), when the merged data are at the noise floor, when the scale model carried no dose term (an uncorrected dose ramp manufactures anisotropy that no significance test can see through), or when no unmerged observations were available. The smallest \(\Delta B\) that could have been established on the data set is reported with the verdict; it is set by the systematic error rather than by counting, so it does not improve with more reflections or a longer exposure.

A too-high space-group assignment is the one failure mode that is silent: real anisotropy is then pushed into the directions used to measure the systematic error, which inflates the floor and biases the answer towards reporting none. A caution says so, but only where that is actually indicated — a single free direction, no detection, and a forbidden-direction measurement far above its own counting noise — rather than on every tetragonal, trigonal and hexagonal data set.

Everything lands in <prefix>_report.txt section 4 (ANISOTROPY_* keys), in the printed statistics, and in the merged mmCIF: the eigen-decomposition of the tensor as the standard _reflns.pdbx_aniso_B_tensor_* items (relative to the weakest direction, since only the deviatoric part is determined), and the directional limits, the shape and the verdict under the _reflns.jfjoch_aniso_* local prefix.

13.6 Practical notes and limitations

  • Bragg integration is profile-fitted by default (per-shell Gaussian profile, Kabsch extraction; §9.3), with plain box summation available as a fallback (--integrator boxsum). The profiles are built per frame from that frame’s strong spots, which suits fast-feedback and serial/streaming use; a profile shared across many frames (as in full offline workflows) is not currently formed.

  • Space-group symmetry beyond centering absences is not enforced during prediction/integration unless the space group is supplied and used downstream.

  • Resolution masking is controllable, and so is every stage of ice-ring handling (§3.3, §10.10). None of it runs unless the crystal is measured to have ice, because the fixed bands are a fixed cost in unique reflections whether it does or not.

  • Rotation vs still modes differ substantially in prediction and scaling: partiality is angle-driven in rotation data, while stills are predicted within an excitation-error window and get their partiality from the default-on per-crystal tilt post-refinement (§10.2) — or unit partiality with --simple-stills.

  • Amplitudes and intensities. The merged output carries both intensities (mmCIF intensity_meas, MTZ IMEAN/SIGIMEAN) and French–Wilson amplitudes (mmCIF F_meas_au, MTZ F/SIGF; §10.8), so a downstream program can refine against either.

14. Model-based validation: R-free against a model and electron-density maps

Offline (rugnux --model model.pdb) the merged data can be scored against a supplied atomic model and electron-density maps computed — enough to confirm that a model fits the data and to inspect the density, not a substitute for refinement. The structure itself is not refined; the model is re-fractionalized into the data unit cell and then placed as one rigid body (§14.8), and the observed amplitudes are the French–Wilson \(|F|\) from §10.8, so the R-free and the maps use exactly the same amplitudes as the written reflection file. The model, structure-factor, bulk-solvent and FFT machinery is provided by GEMMI.

14.1 Model structure factors

The model electron density is sampled on a grid (IT92 X-ray form factors, with a Refmac-compatible Gaussian blur chosen for the grid spacing) and Fourier-transformed to structure factors \(F_\mathrm{calc}(hkl)\) up to the data resolution.

14.2 Bulk solvent and scaling

A flat bulk-solvent mask around the model is transformed to \(F_\mathrm{mask}\), and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale \(k\), an anisotropic \(B\), and the flat-solvent parameters \(k_\mathrm{sol}, B_\mathrm{sol}\):

\( F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2. \)

This is the standard, few-parameter scaling model used by refinement programs. No free-form per-resolution-shell rescale is applied: such a rescale is dataset-specific and reshapes each map’s radial amplitude profile differently, which would make maps from a multi-dataset campaign no longer directly comparable.

That decision has a cost, and it is paid by the R-factors rather than by the maps. \(k\exp(-\mathbf{s}^\mathsf{T} B \mathbf{s})\) can only bend one way with resolution, so whatever a dataset’s radial amplitude profile does that this shape cannot follow is reported as R — which makes R a reading of this dataset and not a quantity comparable with another reduction of the same crystal. Both uses are wanted, so both are served, separately: the maps and R_WORK / R_FREE keep the scale above, and a second reading, R_MODEL_SHELL_SCALED, applies one free scale per resolution shell to the same fit and is reported beside them, with MODEL_RADIAL_MISFIT saying how much that rescale had to do. Nothing is written from the second reading; it never reaches a map, a map coefficient or a decision (§14.3, and RUGNUX_REPORT).

The fit sees the working reflections only; the parameters it returns are then applied to every reflection, free ones included, so that R-free (§14.3) is computed against them. The parameters are few, but they are fitted by minimising the very sum R is made of, and a free reflection that helped choose them is no longer held out.

14.3 R-work and R-free

Crystallographic R-factors are reported over the work and free sets (the §10.7 flags):

\( R = \frac{\sum \big|\,|F_o| - |F_\mathrm{model}|\,\big|}{\sum |F_o|}, \)

with R-free the same sum restricted to the free set. Every parameter \(F_\mathrm{model}\) carries — the scaling of §14.2 and the rigid-body placement of §14.8 alike — is fitted on the working set alone, so the free reflections are held out of the fit as well as out of the sum. Two choices are still selected with the free set rather than fitted to it, and both are single discrete decisions rather than continuous parameters: the rigid-body step is committed only if it lowers R-free (§14.8), and the alternative indexing of §14.6 is the candidate with the lowest R-free. Both are the conventional use of a test set to accept or reject a step; both leave R-free very slightly optimistic where they fire.

14.4 Electron-density maps

Two maps are formed with the model phases \(\varphi_\mathrm{model}\), both \(\sigma_A\) weighted: a \(2mF_o-DF_c\) map and an \(mF_o-DF_c\) difference map, each inverse-Fourier-transformed to a real-space CCP4 map (<prefix>_2fofc.ccp4, <prefix>_fofc.ccp4). A map-coefficient MTZ (<prefix>_maps.mtz: FP, FC, PHIC, FWT/PHWT, DELFWT/PHDELWT, FOM, FREE) is written alongside so the maps can be reopened or rebuilt in Coot / PyMOL.

\(\sigma_A\) is estimated by maximum likelihood per resolution shell, on the free reflections only — on the working set the model has been fitted to the data, so \(\sigma_A\) would come out too high and the weighting would understate exactly the model error the map is meant to reveal. The shells are cut by equal reflection count, and it is the number of shells that is chosen from the size of the free set (about 50 free reflections to a shell, at most 20 shells), so no shell is thin by construction; a shell that still ends up with fewer than 10 free reflections takes the estimate made over the whole free set instead. Acentric and centric reflections enter with their own likelihoods (Rice and Woolfson respectively), and the epsilon factor is divided out before normalising both amplitudes to \(\langle|E|^2\rangle = 1\) within the shell. The figure of merit is then \(m = I_1(X)/I_0(X)\) with \(X = 2\sigma_A|E_o||E_c|/(1-\sigma_A^2)\) for an acentric reflection and \(m = \tanh(X/2)\) for a centric one, and \(D = \sigma_A\sqrt{\Sigma_o/\Sigma_c}\) carries \(F_c\) onto the observed amplitudes’ scale. The \(2F_o-F_c\) combination becomes \(2mF_o - DF_c\) for acentric reflections and \(mF_o\) for centric ones — a centric reflection’s phase is either exactly right or 180° wrong, never in between — while the difference coefficient is \(mF_o - DF_c\) throughout.

\(m\) and \(D\) are estimated on each dataset separately, so two datasets of one crystal form get slightly different weights and their maps are to that extent no longer scaled identically — the same property §14.2 deliberately protects by refusing a free-form per-shell rescale. The two are not the same thing: the weighting never rescales \(F_o\) and never touches the R-factors of §14.3, and the difference between two datasets’ \(\sigma_A\) curves is the difference in how well the model explains each of them, which is what a screening campaign is looking for. A PanDDA-style analysis consumes \(2mF_o-DF_c\) maps and normalises each dataset’s map against the ensemble before comparing them. The per-reflection FOM is written to the MTZ so the weighting can be read off and undone.

14.5 Does the model fit? The null it is scored against

A model supplied with --model is a hypothesis about the crystal, not an instruction. It is always scaled, always placed (§14.8) and always scored, and its R-factors and maps are always reported — but before it is allowed to change anything about the reflections that are written, the data have to accept it. There are only two such things (§14.6), and where the model claims neither the question is never put: see the end of this section.

No fixed threshold on R can decide that, and this was measured three ways. The classical acentric random-structure value is \(R = 0.586\) at unit scale, but the scale here is fitted to minimise the very sum the R is made of, which pulls it to about 0.550; observed nulls on real data land at 0.599–0.615; and the value moves with the model — its atom count and its B-factors — as much as with the data, so a cut calibrated on one pair misjudges the next. In one measured arm an unrelated protein reached a lower raw R-free than the correct model did on other data.

The only null that fits both the model and the data is therefore made out of them: the same model is re-oriented at random about its own centroid and run through the identical path — the same scaling, the same rigid-body placement, the same R — nine times, and the real fit is asked how far above the resulting distribution it sits (MODEL_FIT_SIGMA, accepted at 3σ). The replicates are placed as well as fitted, or the comparison would be between a placed model and unplaced nulls and the margin would be inflated by the placement rather than by the model. Measured on one rotation data set: the crystal’s own model +15.0σ, an unrelated protein +1.8σ, and the correct model rigidly rotated 90° −1.0σ.

A replicate is a sample of the null only while it stays away from the model’s own solution. A draw within the rigid body’s reach of an orientation equivalent to the model’s — under the space group’s rotations or a twin law of the lattice — is drawn again before it is placed, and a replicate the placement nevertheless carries to within that reach is drawn and placed again afterwards, each replicate from a generator of its own so the result does not depend on the order the concurrent replicates finish in.

R-work carries the decision, not R-free. Nothing is refined against the working set here — the scale has a handful of parameters (a scale, an anisotropic \(B\) and two solvent terms) and the placement six — so R-work carries no optimism, and it has an order of magnitude more reflections than R-free, and so that much more power to separate the two arms. R-free is still reported, and is still what the placement of §14.8 is committed on, since that is a fit.

What the verdict gates is exactly the two decisions of §14.6 that rewrite the data: the space-group label and the indexing. It does not gate the R-factors, the maps or the rigid-body placement, which are statements about the model and cannot corrupt a reflection. A rejected model therefore leaves the written files byte for byte what a run with no model would have produced, and the report says so beside the R it was rejected on: a negative result, not a failed run.

The null is only built where a decision is actually pending. A model can change exactly two things, and it does not always claim either: one already in the space group the data were merged in, on a crystal with no merohedral ambiguity, asserts no enantiomorph and prefers the data’s own indexing, so there is nothing to arbitrate and nothing for a null to gate. That case reports MODEL_FIT= NOT_TESTED and skips the replicates. It is not an edge case — it is the isomorphous fragment-screening run, the one that has to deliver a map seconds after the last image — and it is why the ordering matters: the indexing probe is a handful of scaling fits and runs first anyway, so whether the expensive part is worth paying for is known before it starts.

The count is nine, and it is the spread that sets it rather than the mean: the verdict is (mean − real)/sd of the sample, the relative error on an sd from \(n\) draws is \(1/\sqrt{2(n-1)}\), and a sample that happens to come out narrow is what turns a model that does not fit into one that appears to. Measured on the case nearest the gate — an unrelated protein at 1.83σ against a threshold of 3 — the chance of reading over the gate on a different seed is 31 % at \(n=3\), 17 % at \(n=5\) and 6 % at \(n=9\). Nine is affordable only because the replicates run at once: the null costs the slowest of them rather than their sum, and the slowest of nine is barely above the slowest of five.

The null costs one full fit and one rigid-body placement per replicate. The replicates share nothing — each is the same model under a different rotation, scored the same way — so each takes a copy of the model and of its structure factors and they run concurrently, on the run’s own thread budget (-N); the real model is not touched by any of them. Measured on an idle machine, five replicates cost 2.5 s together where running them one after another costs 10 s, on a --mode scale run that merges in 2 s. What remains is the cost of the slowest replicate: how many placement evaluations an orientation needs varies by nearly half between them, so the concurrency saturates around 4×, and more threads than replicates buy nothing. The orientations are drawn up front, in order, from the fixed seed, so replicate \(i\) gets the same orientation whatever order the threads run in — a σ that depended on the scheduling would not be a measurement. Where the null is skipped, --model costs what it did before the null existed (measured: indistinguishable, ~4 s against ~4 s, on a loaded machine where a single run scatters by a second). Where it runs, that is the price of the answer being a measurement rather than a threshold.

14.6 Aligning the data to the model: enantiomorph and indexing ambiguity

The model fixes a definite hand and indexing, but the merged data need not share them, so before comparison the observed reflections are brought into the model’s frame.

  • Enantiomorph / screw. When the data space group is the enantiomorph of the model’s (e.g. data \(P4_12_12\), model \(P4_32_12\); or \(P3_1/P3_2\)), the two are indistinguishable from merged intensities — \(|F_\mathrm{calc}|\) is invariant under the change of hand, so R-free cannot choose between them and probing would be meaningless. Where the model was accepted (§14.5), its group is therefore adopted as the label the reflections are written under, and the reflections themselves are left untouched. Where it was not, the label is left alone: adopting it is arithmetic on two group numbers, which a model that does not belong to this crystal can do exactly as readily as one that does, and the anomalous map of §14.7 vetoes it outright where it says the two are in opposite hands. The two groups of an enantiomorphic pair differ only in the translations of their operations: their rotations are identical, so they transform \(hkl\) identically, share a reciprocal ASU, and assign the Bijvoet hands identically. The label carries no handedness, and there is nothing about it to undo. Reindexing by the change-of-hand operator — which is the inversion — would instead swap \(I(+)\) with \(I(-)\), flipping every anomalous difference on the strength of a label the space-group search itself reports as undetermined; where the model is genuinely the wrong enantiomorph for the crystal, it would manufacture agreement rather than reveal the mismatch. What does carry the hand is the indexing the data already have, from the diffraction geometry, and the anomalous differences that come with it. §14.7 is what tests them against the model.

  • Indexing (merohedral) ambiguity. When the crystal has a merohedral ambiguity (§10.9), the observed intensities do differ between indexings, and the right one is chosen against the best available reference. If a reference MTZ was supplied, the data were already reindexed to agree with it (§10.9 — by the reference-intensity correlation, at the merge stage for rotation data or per image in stills scaling), and model validation keeps that authoritative choice. Only with a model and no reference does validation resolve the ambiguity itself, as a fallback: the scaled model is fit to each reindexing of the data (identity plus the twin-law cosets) and the one giving the lowest R-free is kept — but only where its lead over the runner-up beats the lead the same model in a random orientation takes, since a random model also picks a winner, and, as measured, by a comparable margin. Where it does not, the data keep the indexing they were merged in. The candidate operators are enumerated from the data’s space group, not the model’s: it is the observed intensities that are being relabelled, and asking the model’s group enumerates nothing at all wherever the two differ. This matters for a multi-dataset campaign — a single shared reference fixes one indexing convention for every dataset, whereas an independent per-dataset lowest-R-free choice could send borderline datasets to different conventions. A no-op either way for a holohedral crystal (no twin laws). The two decisions then reach the written output differently, because only one of them moves reflections. The ambiguity choice is applied to the merged reflections themselves — and to the integrated observations behind the unmerged export — which are written after this step, so the reflection file, the R-factors and the maps describe one indexing. The change of hand changes only the space group the files are written under (the model’s enantiomorph): no reflection moves, exactly as the first bullet says, so \(I(+)\) and \(I(-)\) stay as measured and §14.7’s hand check remains a genuine test rather than an agreement manufactured by reindexing. The ambiguity choice is reported with that margin and with the null it was judged against, since the margin alone is what cannot say whether the data decided or the two came out within noise of each other.

14.7 Anomalous difference map and the sites it names

Where the merge kept the Bijvoet split (§10.5) — which a rotation merge does by default, whether or not the mates were averaged — an anomalous difference map is computed as well, with coefficients

\( \big(|F(+)| - |F(-)|\big)\, e^{i(\varphi_\mathrm{model} - \pi/2)}, \)

over the acentric reflections that have both hands (a centric reflection has no anomalous difference, only noise). Turning the model phase back by 90° is what makes the anomalous scattering, which is 90° out of phase with the normal scattering, add up in the real part: the map’s peaks then sit on the anomalous scatterers. It is written as <prefix>_anom.ccp4. Its hand is the one §14.6 settled: with the mates the wrong way round every peak becomes a trough, so a map of clean peaks is itself a check that the frame is right.

Rather than searching the map for blobs and leaving a list of coordinates, the map is read at the model’s own atom centres (hydrogens excluded — they scatter no anomalous signal), and the ten highest, in units of the map’s r.m.s., are reported in the log and as ANOMALOUS_SITE_01…ANOMALOUS_SITE_10 in the results report. Each site is therefore named — the atom, residue and chain it belongs to — which is what says what carries the signal, not just where it is. The reading is cubic, not linear: the map is sampled every \(d_\mathrm{min}/3\), and a peak that sharp read by trilinear interpolation comes out up to a quarter low — unevenly enough to reorder the sites. This reading is the one ANODE reports. The same reading averaged over the model’s anomalous scatterers — every atom from phosphorus (\(Z = 15\)) up: the S of Met and Cys, metals, Cl, I — is reported as ANOMALOUS_SCATTERER_MEAN_SIGMA (with their count, ANOMALOUS_SCATTERERS): one number for how much anomalous signal the merge carries, which does not depend on which ten atoms happen to come out on top. Below phosphorus (C, N, O, Na, Mg) \(f''\) is a fraction of sulfur’s at any wavelength these data are taken at, and counting those atoms would only dilute the mean with noise.

Because the map is built on the model’s phases and the data’s own indexing, it is also the only test of whether the two agree about the hand (§14.6). A model and a dataset in opposite hands turn every anomalous peak into a trough, so a map whose deepest hole at an atom is both deeper than \(5\sigma\) and deeper than its highest peak says so, and the run reports it as a warning naming that atom. It is not repaired by reindexing: that would make the two agree by construction and destroy the evidence for which of the model and the data is in the wrong hand. Note that R-free cannot see this at all — a mirrored model gives R-free to four decimal places unchanged, and an exactly inverted anomalous map.

The list is always ten entries long, so it is their height that carries the information: on a sulfur-SAD dataset the sulfurs fill the top of the list and are followed by a clear drop to the couple of sigma that is the map’s noise, while a dataset with no anomalous signal has no such separation and lists ten unrelated atoms at noise level. A scatterer the model does not contain — a bound ion, a soaked heavy atom — is by construction invisible in the list, and is what the map file is for.

14.8 Rigid-body placement of the model

Re-fractionalizing a model into the data cell puts it in the right box but not necessarily in the right place: a non-isomorphous cell squeezes the box without moving the body inside it, and the body’s own position in the cell differs from crystal to crystal. Six parameters recover that — an angle-axis rotation about the model’s own centroid, then a translation. Rotating about the centroid rather than the cell origin is what keeps the rotation from moving the body bodily, so the two triplets are close to independent. Parameters are carried as six lengths in ångström (the rotation vector multiplied by the model’s r.m.s. radius), so a unit of each moves a typical atom by the same amount.

It is one rigid body. A fragment-screening model arrives already solved and isomorphous, and what is being recovered is the crystal’s movement, not the molecule’s; splitting it into domains, or giving a bound ligand six parameters of its own, would refine against evidence these data do not separately carry — and the ligand is what the difference map is there to show, not to model away.

The refinement walks a coarse-to-fine ladder, 6 Å → 4.5 Å → 3.5 Å, each zone starting from the previous one’s answer. It stops at 3.5 Å because that is where rigid-body refinement is conventionally run and because the movement being recovered is a few tenths of an ångström, a tenth of that resolution; a finer zone costs \((1/d)^3\) in grid points and reflections for a placement it cannot meaningfully sharpen. Each evaluation recomputes \(F_\mathrm{calc}\) and the solvent mask for the moved model and re-fits the scale of §14.2 — otherwise the target would measure the scale as much as the placement, and the body would translate to repair a scale error instead of moving to where the density is. The minimiser is Levenberg–Marquardt on the amplitude residuals. \(F_\mathrm{calc}\) is computed from one copy of the model, gridded without symmetrization and transformed once, and composed over the space group’s operators, \(F(\mathbf h) = n_\mathrm{cen}\sum_{(R,\mathbf t)} e^{2\pi i\,\mathbf h\cdot\mathbf t}\,F_1(\mathbf hR)\) — the transform of the symmetrized map, rearranged. That makes the derivative with respect to the translation exact and free (\(F_1(\mathbf k)\) only picks up the phase \(e^{2\pi i\,\mathbf s_\mathbf k\cdot\Delta}\)). The rotation’s three columns are forward differences, the step a fixed fraction of the zone’s resolution, each needing only one copy gridded and transformed. The Jacobian holds the scale and the bulk-solvent mask at the evaluation’s; the residuals keep both exact, and the scale re-fit is folded into the Jacobian by variable projection (Golub & Pereyra 1973, in Kaufman’s 1975 form), so a step is judged on the target the evaluations actually compute.

The refinement sees only the working reflections. The step is then committed only if it lowers R-free, computed at full resolution on the free set it never saw; otherwise the model is put back exactly where it was read and the maps are the ones it would have given. The whole addition costs about two seconds. Note that the origin is a gauge in some space groups — free in all three directions in \(P1\), and along the unique axis in a polar group — so those components of the translation are undetermined; nothing is done about that beyond the Levenberg–Marquardt damping and the R-free gate, which between them make an undetermined direction harmless rather than unstable.

Where a CUDA GPU is present, the refinement’s target is evaluated on it: the same density, the same composition of \(F_\mathrm{calc}\) from one copy of the model, the same bulk-solvent mask with its islands removed, the same scale fit and the same Jacobian, computed on the device, where a fit takes a fraction of a second rather than several seconds. The device is chosen once per validation - the real fit and every replicate of the null of §14.5 on the same one - and the CPU is used where there is no GPU, where the card has too little free memory for the fit’s buffers, or after a CUDA failure, in which case the whole validation is run again on the CPU. The two agree to rounding and not bit for bit: the device measures distances in single precision and transforms with cuFFT instead of FFTW, and GEMMI’s scale fit, whose stopping rule leaves it unconverged at about \(10^{-4}\) of \(|F|\), can respond to that rounding with a step of its own. A rigid-body step accepted or rejected on an R-free difference below about \(5\cdot10^{-4}\) can therefore go either way between the two; every kernel is deterministic, so either one gives the same answer on every run.

The rotation and translation actually taken are reported (RIGID_BODY_ROTATION_DEG, RIGID_BODY_SHIFT_A), together with the R-free before it (R_FREE_BEFORE_RIGID_BODY), so what the placement bought is visible.

The coordinates as placed are written as <prefix>_model.cif, and as <prefix>_model.pdb where the PDB format can hold the cell (the fragment-screening tools the file feeds — PanDDA, and the pair dimple produces — take a PDB beside the MTZ), so there is a coordinate file that describes the maps: the input’s chains, residues, ligands, waters, B-factors, occupancies and anisotropic \(U\)s, moved, in the same cell and space group as <prefix>.mtz. Taking the frame from the written reflections rather than from the input model matters — §14.6 may have relabelled them to the model’s enantiomorph, which is neither the data’s original group nor, necessarily, the model’s. It is written whenever the maps are, not only where the rigid-body step was committed: the model is re-fractionalized into the data cell and may be relabelled whatever the placement decided, so an unmoved model is still not the input file; and a model the null of §14.5 rejected is scored, placed and mapped like any other, which is precisely the case where the density is worth looking at.

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS_IMAGE.html b/CPU_DATA_ANALYSIS_IMAGE.html new file mode 100644 index 000000000..f45776078 --- /dev/null +++ b/CPU_DATA_ANALYSIS_IMAGE.html @@ -0,0 +1 @@ + Data analysis: from images to spots (§0–§3) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Data analysis: from images to spots (§0–§3)

Part of the CPU/GPU data-analysis reference; the section numbers are continuous across its four parts.

0. Getting the image onto the GPU: device-side bitshuffle+LZ4 decoding

Images arrive bitshuffle+LZ4 compressed (HDF5 filter 32008), and everything from §1 onwards runs on the GPU when one is present. Instead of decompressing on the host and uploading the image, the compressed chunk is uploaded — a few MB rather than tens of MB — and decoded on the device. The approach follows Jon Wright (ESRF); the kernels are Jungfraujoch’s own.

One kernel does the work: one CUDA block owns one bitshuffle block, from the compressed payload through to finished pixels.

  1. LZ4 into shared memory, one warp per bitshuffle block. Blocks are independent, so the parallelism is across them; within the warp every lane runs the same sequence parser over the same bytes, and the literal and match copies are split across the 32 lanes so the stores coalesce. An overlapping match is treated as a pattern of period offset sourced from bytes that already precede the write position, which keeps it parallel rather than a serial byte loop; offset == 1 (a run of one repeated byte, the common case in sparse detector data) and power-of-two offsets avoid the modulo altogether. Because the lanes cooperate on the copies, each one is followed by __syncwarp() — a later match can read bytes another lane wrote, and since Volta that ordering is not implicit.

  2. The bitshuffle inverse fused with preprocessing. The whole CUDA block then reads that shared buffer back: one thread owns one group of 8 elements across every byte-plane, so once it has transposed its 8 bytes out of each plane it holds 8 complete elements — and it applies the pixel mask, the error marker and the saturation cap and emits 8 finished int32 pixels directly. Nothing of the block reaches device memory but the pixels — neither the bitshuffled bytes nor the decompressed image is ever materialised. For 8-bit images there is a single plane and the assembly degenerates to a copy.

Decoding into shared memory is worth more than the bandwidth it saves: an LZ4 match reads back bytes written a few sequences earlier, so every copy step is a dependent round trip — tens of cycles in shared memory against hundreds in device memory. It is paid for in residency, because the buffer holds a whole bitshuffle block, and a block larger than 16 kB (larger than either writer this pipeline reads produces) falls back to a pair of kernels instead, the first writing the shuffled image to device memory and the second un-transposing and preprocessing out of it.

The block offsets inside the container can only be discovered by reading the block lengths in order, so that scan stays on the host.

Only BSHUF_LZ4 is decoded on the device. For the zstd variants (BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF), and for uncompressed or float images, BSLZ4DecoderGPU::Supports() returns false and the pipeline decompresses on the host and uploads as before.

The container arrives off the network or off disk and is not trusted. Everything checkable on the host — declared sizes, the block scan, a block size that is not a multiple of 8 elements, a block count the chunk could not hold, trailing bytes — is rejected before any work is queued; the kernel additionally flags a block that did not decode to exactly its declared length, which becomes an exception once the caller has synchronised. That last check matters because the decode buffers are reused frame to frame: a block that stopped early would leave the previous image’s most significant byte-plane in place, which reads not as a missing corner but as real pixels several powers of two too bright.

1. Geometry, reciprocal-space mapping, and basic quantities

1.1 Coordinate conventions

For a pixel coordinate \((x,y)\) (in pixels), Jungfraujoch converts to a laboratory direction vector via:

  1. shift by the beam-centre pixel \((x_\mathrm{beam}, y_\mathrm{beam})\) — the PONI, pyFAI’s point of normal incidence, which coincides with the direct-beam impact point only for an untilted detector (see Detector geometry),

  2. scale by pixel size \(p\) (mm),

  3. set detector distance \(D\) (mm),

  4. apply detector orientation rotation \(R_\mathrm{det}\) (PyFAI-like parameterization).

The unnormalized detector coordinate (mm) is: \( \mathbf{r}_\mathrm{det}(x,y) = \begin{pmatrix} (x-x_\mathrm{beam})p\\ (y-y_\mathrm{beam})p\\ D \end{pmatrix}. \)

The lab-frame vector is: \( \mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. \)

By this construction \((x_\mathrm{beam}, y_\mathrm{beam})\) maps to \((0,0,D)\) before the rotation — the point where the detector normal through the sample meets the detector — which is what makes it the PONI rather than the direct beam; the two differ by \(D\tan(\mathrm{tilt})\) on a tilted detector. The laboratory frame is fixed the same way everywhere in the system: \(+z\) along the beam propagation, \(+x\) along increasing pixel column (the fast axis) and \(+y\) along increasing pixel row (the slow axis) — a right-handed triple that coincides with XDS’s laboratory frame, which is what makes the geometry echo of Rugnux directly comparable. The absolute hand of an indexing — and with it the Bijvoet hands of §14.6–§14.7 — follows from this convention.

Let the incident wavevector magnitude be \(k = 1/\lambda\) in Å\(^{-1}\), and define: \( \mathbf{S}_0 = (0,0,k). \)

The reciprocal-space scattering vector associated with pixel \((x,y)\) is: \( \mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. \)

This \(\mathbf{s}\) is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement.

1.2 Two-theta, azimuth, resolution and \(q\)

The scattering angle \(2\theta\) is computed from \(\mathbf{r}_\mathrm{lab}\) via: \( 2\theta = \mathrm{atan2}\!\left(\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2},\; z_\mathrm{lab}\right), \)

evaluated as a two-argument arctangent, so the mapping stays correct where a strongly tilted detector’s far corner reaches past \(2\theta = 90°\).

Resolution (Å) at a pixel is: \( d = \frac{\lambda}{2\sin\theta}. \)

The magnitude \(q = 2\pi/d\) is used for radial binning and ice-ring handling.

1.3 Distance from the Ewald sphere

For a reciprocal lattice point \(\mathbf{p}\) (Å\(^{-1}\)), define: \( \Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. \) Jungfraujoch uses \(|\Delta_\mathrm{Ewald}|\) as an operational proxy for excitation error. This appears in:

  • still prediction (accept if \(|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}\)),

  • profile radius estimation (see §11.1),

  • still partiality option in scaling/merging (§10.2).

1.4 The beam centre

A run starts from the beam centre in the file, or from --beam-x/--beam-y where they are given. Header centres are often typed rather than measured, and on rotation data a wrong one is not repaired by anything downstream: the error is fixed in the laboratory frame, so accumulating a sweep smears every reciprocal-lattice point around a circle, and a displacement \(\delta p\) on the detector multiplies the FFT amplitude at an axis of length \(a\) by \(J_0(2\pi\,\delta p\,a/(D\lambda))\). Past the first zero the true axis is gone and its harmonic wins. Rugnux therefore measures the centre itself on every run and treats the measurement as a second hypothesis to be tested against the file’s, not as a replacement for it. The steps, in the order they run:

Step

Runs

Effect on the centre

Header check

every run, before a frame is read

none; reported

Background measurement (--beam-center-check)

every run with the beam-stop pre-scan

none at this point; reported

--estimate-beam-center

only when asked

replaces the file’s before indexing, where measured precisely enough

Second first pass at the measured centre

rotation

adopts the measured centre in the cases listed below

Both centres judged on the merge

rotation, two-pass

adopts the measured centre where its first pass merges better

--beam-center-search

rotation, after a first pass that indexes under half the validation frames

steps the centre a pixel at a time

Post-refinement (§7.5)

rotation, two-pass

refines the centre from the integrated reflections, within 15 px

On most runs the measured centre is reported, the second first pass finds the same lattice at both centres, and the run continues at the file’s centre until post-refinement moves it.

What the file says. Before any frame is read, a centre within one pixel of the geometric centre of the detector, or a whole number of pixels in both coordinates, is reported as a value written rather than measured, and a centre that lands on a masked pixel is a warning — a real beam does not sit on a dead pixel or in a module gap. These lines change nothing.

Measured on every run (--beam-center-check, on by default). The isotropy of the scattered background places the point the beam lands on; the leverage is the curvature of the solvent ring. The fit runs over the mean image the beam-stop pre-scan already builds (§1.5), with the opaque part of the shadow masked, so it costs no frames of its own. For the same reason --detect-beam-stop=off leaves nothing to measure it on, and the check then does nothing.

The fit starts with a whole-detector capture. The centrosymmetry score of the projection about a candidate centre \(\mathbf{c}\), \(\sum_\mathbf{x} I(\mathbf{x})\,I(2\mathbf{c}-\mathbf{x})\), is the self-convolution \((I*I)(2\mathbf{c})\), so one FFT pair scores every candidate centre on the detector at half-pixel spacing: the cost is \(O(N\log N)\) and independent of how far the header is from the truth. Three surfaces are scored — the 2D point inversion, which measures the same isotropic background the walk below fits, and a 1D line mirror per detector axis, which along the spindle is exact Friedel physics (a reflection’s mate half a turn later lands mirrored in the line through the beam across the spindle). The beam-stop shadow is blanked out of the scored image, a one-sided opaque obstruction being a centrosymmetry defect in its own right. What comes back is a shortlist and a margin per surface, not a centre: the surface can be locally flat over tens of pixels, and the peak-to-runner-up margin says so. A local walk is then seeded at the capture and refines it; where it declines there it is started again from the file’s centre, and only where neither start gives it something to fit does the capture stand alone. The transforms run on the GPU where one is present and on FFTW otherwise, with everything that decides anything shared between the two.

The run log then carries:

  • Beam centre capture: the strongest point-symmetry centre, the two line-mirror coordinates and the three margins (under about 3 % the surface is flat).

  • Beam centre check: the file puts the direct beam at (x,y), the scattered background puts it at (x,y) +- σ px - a difference of d px, against the t px this geometry asks the centre to be right to. The comparison is against the direct beam, not the PONI: beam_x_pxl is the point of normal incidence, and the two part by \(D\tan(\mathrm{rot})/p\) as soon as the detector is tilted, while the background is centrosymmetric about where the beam lands. \(t = 0.183\,D\lambda/(p\,a_\mathrm{max})\) is the displacement at which the \(J_0\) factor above has fallen to 0.70 for the longest axis of the cell given with -C (200 Å where none is given); it runs from well under a pixel to several. It is reported, not used as a gate: a centre change far smaller than \(t\) can still decide between a cell and its harmonic.

  • One of three follow-ups: the difference is under three times the fit’s σ (the file’s centre is as good as this measurement can tell); more than \(t\) of it lies across the spindle, where it can cost peaks or an axis; or it is a real difference, mostly along the spindle, where a wrong centre does not announce itself.

None of these lines moves the centre. What consumes the measurement is the second first pass below (rotation runs), --estimate-beam-center’s fall-through, and the bound on the post-refinement.

The second first pass (rotation). After the first pass — and after the rotation-axis sign rescue below — the first pass is indexed again at the measured centre, however small the difference. A trial centre gets its own azimuthal mapping, spot engines and spot cache: raw centroids do not move with the centre, but which spots are in the list does (the resolution limit, the ice-ring flag, the beam-stop mask and the strongest-\(N\) ranking are all radial), and the starting centre’s spots are parked and restored so a trial that adopts nothing leaves the run as it was.

The indexed frame count is not used to choose between two centres that both index: acceptance is a fractional-Miller test, so a cell twice as long must place every spot twice as accurately to score the same, and a halved axis can index more frames than the true cell. “Indexes” below means more than half of the validation frames; “same lattice” means the same Bravais class with primitive volumes within 2 %. The outcomes:

  1. The file’s centre indexes and the measured one does not. The file’s centre stands.

  2. Both index the same lattice. The file’s centre is kept, and the log says the difference does not decide this crystal’s cell. A centre off by about one reflection spacing still indexes the same cell, with part of the sweep given indices one out, so where the two centres are further apart than both \(t\) and three σ of the fit, the measured centre is kept as a hypothesis for the merge (next paragraph).

  3. Both index the same cell in a different Bravais class. Whether the centre could have decided the class is arithmetic: a shift of \(n\) pixels of size \(p\) moves an axis of length \(a\) by \(n\,p\,a/(D\lambda)\) relative, and the lattice search holds an axis equality to 3 %. Where the shift is worth that much on the pair of reduced-cell axes the equality compares, both centres are carried to the merge; otherwise the file’s centre and its class are kept.

  4. Both index, on different lattices. Both cells are reported in a warning. Where the primitive volumes differ by an integer factor of 2 to 4 — an axis harmonic, which a centre error along the spindle produces — the choice is made on the pooled validation spots: the measured centre is adopted where its lattice, larger or smaller, beats chance and puts a share of the spots on itself over its own wrong-spindle null that exceeds the file’s by more than the binomial noise of the two (z = 3.29). Otherwise the file’s centre is kept and the warning suggests --estimate-beam-center.

  5. The file’s centre does not index. Nothing defends it, so:

    • where the measured centre indexes, it is adopted;

    • where it does not, the rotation-axis sign is flipped at the measured centre, since each of the two errors can hide the other; where that indexes, both are adopted;

    • where it still does not, the first-pass depth ladder of §6 (how much of each frame the pass reads) is run at both centres, so the centre moves only when it is the centre that pays. The measured centre is adopted where its lean rung indexes and beats the file’s; where both index the same lattice once less of each frame is read, both centres are carried to the merge; where the file’s centre wins or ties, the depth was the problem and the file’s centre stays;

    • where nothing indexes at either centre, the pooled spots decide as in case 4; failing that, the file’s centre is put back, so a run that fails for some other reason fails at the geometry it was given.

Both centres judged on the merge (rotation, two-pass). Where case 3 or 5 carried both centres, the whole first pass is run again at the measured centre — with its own short-axis pass and post-refinement — and the two arms are compared on their search merges (\(P1\), whole range, before the correction surfaces). Where case 2 held a hypothesis, this is done only if the file’s arm merges inconsistently: \(\mathrm{CC}_{1/2}\) below 0.9 in the lowest-resolution shell of its search merge, where every reflection is strong and a consistent merge reads close to 1. The measured centre wins where:

  • both arms ended on the same lattice, and it merges more reflections at \(I/\sigma \ge 2\), or its lowest-shell \(\mathrm{CC}_{1/2}\) is higher by more than 0.05;

  • the arms ended on different lattices, and its search-merge \(\mathrm{CC}_{1/2}\) is higher by more than 0.05.

Otherwise, or where the measured arm does not complete, the run keeps the file’s centre. The log lines are Beam centre check: running the first pass again at the measured centre … followed by … the run adopts the measured centre and the lattice it finds or the measured centre does not win. Over the validation battery about one rotation run in ten runs this second arm, and about one in ten ends on the measured centre by one of the routes above.

After a first pass that failed. A rotation sweep leaves two further things the input often cannot settle, both decided on the same count the rest of the first pass uses — the right answer indexes and the wrong one does not.

  • The rotation-axis sign. A miniCBF header names the axis but gives no direction, and an NXmx vector is only meaningful together with the detector mounting. So after a poor first pass the opposite sign is tried and whichever indexes more validation frames is kept, the flipped one only where its lattice also beats chance on the pooled spots (§4.1), so that one frame against none on a sparse pattern cannot flip it. The decision holds for the rest of the run, and it flips the axis, not the angles, because prediction reads the axis too. It runs before the second first pass above and before the long-axis rescue, since with the sign wrong every candidate lattice is wrong.

  • The beam-centre search (--beam-center-search[=N|off], on by default, N = 12 px). Where the first pass, after the second first pass above, still indexes fewer than half the validation frames, the centre is stepped a pixel at a time out to N px along both detector axes, each rung with its own spots, and the first rung that indexes a majority and beats the starting count is adopted (Beam centre from indexing: …). Both directions are searched deliberately: an error across the spindle collapses the indexed fraction and announces itself, while an error along it leaves the transform’s peaks sharp, holds nearly every frame indexed and lets the lattice fit commit to an axis harmonic. A rung whose primitive volume is an integer or \(\sqrt{3}\) multiple of the starting cell’s is refused for that reason. The search is skipped where the background measurement places the centre further away than both N px and three σ, since no rung could reach it, and it stops after two rings where no rung has reached a third of the majority. The step is a flat pixel: derived from the \(J_0\) law it would have to use the cell the failed pass returned, which can be a small spurious sub-cell and asks for a step that jumps the lobe being looked for.

Post-refinement. On a two-pass rotation run, whatever centre the first pass ends on is refined together with the distance, cell, orientation and axis by the post-refinement of §7.5, and the second pass is integrated there. The background measurement does not move that fit; it only widens where the fit may go: the refined beam is committed only within 15 px of the nearer of the pass-1 centre and the measured one (the measured one counting where its σ is within \(\max(1\ \mathrm{px}, t)\)), so a header that is far out can still be corrected. Where the refined pass is judged worse, the run goes back to the pass-1 geometry.

Committed before indexing (--estimate-beam-center, off by default). Here the centre is measured before anything is indexed and used in place of the file’s. On a sweep that reaches half a turn it comes from the spot positions alone; two exact facts about a rotation sweep supply the two coordinates.

Friedel mates half a turn apart. The Laue condition fixes the component of \(\mathbf{q}\) along the beam, \(q_\parallel = -\lVert\mathbf{q}\rVert^2\lambda/2\). Rotating 180° about the spindle \(\mathbf{m}\) negates the two components perpendicular to \(\mathbf{m}\) and taking \(-h\) negates all three, so together they negate only the component along \(\mathbf{m}\) and leave \(q_\parallel\) untouched. With the spindle perpendicular to the beam, \(-h\) therefore diffracts at \(\varphi+180°\) exactly where \(h\) diffracts at \(\varphi\), and its spot sits at the mirror image of \(h\)’s along the spindle. This gives the beam coordinate along the spindle. Only the reciprocal lattice’s centrosymmetry is needed for the geometry; Friedel’s law \(|F(h)|=|F(-h)|\) is used separately, to tell a true pairing from an accidental one.

The second crossing. The same reflection meets the Ewald sphere twice, at two angles that are generally not 180° apart, differing only in the sign of the lab component perpendicular to both \(\mathbf{m}\) and the beam. This gives the remaining coordinate. The two crossings are separated by a sweep angle fixed by the reflection’s own position, which is what identifies genuine pairs.

Neither observable requires a cell or an orientation matrix: each candidate pairing votes for a beam coordinate, and the true value accumulates while wrong pairings scatter. A Friedel pair needs both \(\varphi\) and \(\varphi+180°\) recorded, so a sweep of \(S°\) yields only \(S-180\) degrees’ worth of pairs.

The mirror is exact in the laboratory frame, so it is sensitive to the spindle direction. A skew of the spindle about the beam spreads the vote rather than shifting it, and is fitted alongside the centre (--no-fit-spindle keeps the axis from the file); a tilt of the spindle towards the beam is measured and reported but not applied, being confounded with the detector tilt. Nothing inside the fit can tell that the vote settled on the wrong periodic maximum — every frame pair agrees with every other — so its σ is how far the answer moves when the search is started from a different position. The frames are read in pairs half a turn apart, twice as many as the beam-stop projection uses and away from both ends of the sweep, where shutter synchronisation can spoil an image; where the answer does not come out on them, up to 400 more are read.

Where the spot symmetry does not come out — in practice on sweeps shorter than about 220° — the centre is taken from the background measurement above. The estimate replaces the file’s centre only where its σ is within \(\max(1\ \mathrm{px}, t)\) and it moves the centre by more than three σ; otherwise, or where neither method measures anything, the file’s centre is kept. The log line is Beam centre from spot symmetry|background: (x,y) -> (x,y), moved d px, sigma σ px against a c px ceiling => COMMIT (or reject, with the reason). --estimate-beam-center is ignored where --beam-x/--beam-y are given, and on stills where --refine-geometry has already placed the centre from indexed spots.

Stills. There is no second first pass, no search and no post-refinement on stills; the background measurement is reported only. The centre moves through --estimate-beam-center (background estimator) or through the stills geometry refinement, --refine-geometry, which bundle-adjusts the beam centre, distance and cell over strongly indexed frames and is on by default where a reference cell is given.

What ends up in the output. The report’s BEAM_CENTRE (the PONI) and DIRECT_BEAM are the geometry the result was integrated at, and so is JFJOCH_DATASET_SETTINGS; POSTREFINE_BEAM_CENTRE gives the post-refinement’s move as before -> after. Which of the steps above changed the centre, and why, is in the run log only.

Options.

  • --beam-x/--beam-y set the starting centre and switch --estimate-beam-center off. The background check still runs and can still adopt the measured centre in the cases above; add --beam-center-check=off to hold a typed centre until post-refinement.

  • --beam-center-check=off skips the background measurement and the second first pass, and removes the measured centre from the post-refinement bound.

  • --beam-center-search=off or =N turns off or resizes the search after a failed pass.

  • --estimate-beam-center measures and commits the centre before indexing; --no-fit-spindle keeps the spindle direction from the file in that estimate.

  • --detect-beam-stop=off removes the projection the background measurement is read from.

  • --rotation-no-postrefine removes the post-refinement, and with it the merge-judged arms.

1.5 Finding the beam stop

The beam stop and its holder arm shadow part of the detector. A reflection behind them is attenuated but otherwise ordinary — it integrates low, with a plausible \(\sigma\), and no outlier test catches it — so the shadow is found and masked instead. --detect-beam-stop[=N|off] (on by default, \(N=60\) frames) projects those frames to a per-pixel mean and maximum, and writes the result into the pixel mask as bit 9, from where it excludes those pixels from every later stage.

The shadow is a place where the background is missing, so it is found by comparing each pixel’s background against the background at the same radius. The mean is pooled over a small box first — a single pixel of a sparse background carries too few counts to tell a shadow from a Poisson hole, and the stop is much wider than the box — and the comparison is against the median of the pixel’s own radius ring, taken over the pixels not already excluded and iterated a few times so the shadow stays out of the baseline it is measured against. A pixel is shadow when the ratio falls below 0.50 and the deficit is significant against its own Poisson scatter, \(\sqrt{2\,(E-N+N\ln(N/E))}\) over the pooled counts. A ratio says nothing when the background behind it is a handful of photons, and it is that significance that makes the comparison scale-free rather than tuned to one exposure: rebuilt from six frames of a low-background sweep, the same ratio without it masks three quarters of the detector. The index of dispersion does not do this job — it is 1.0 inside the shadow and 1.0 outside it, a shadowed pixel being Poisson at a low rate and a lit one Poisson at a high rate — only the rate relative to the ring separates them.

Nothing assumes the stop and the beam are concentric, because only the per-ring comparison is used; a ring lying wholly inside the stop has no unshadowed pixel for its median to find, which is exactly the case where the comparison must fail. Such a ring is judged against what this detector’s background typically is — the median over the rings the walk is willing to judge — and is shadow in its entirety when it falls far below that. Comparing it instead against the largest background further out reads an ordinary background inside a strong ring as blocked, and the walk then runs out to that ring and returns a filled disk of good detector with diffraction rings plainly visible inside it.

The rings are drawn about the centre the data measure, not the one the file claims. Displacing the centre costs nothing for tens of pixels and a great deal beyond: at 150 px out the comparison describes the background’s own radial fall-off rather than the hardware, and returns a tenth of the detector as shadow with nothing blocking it — and header centres are wrong by that much. The centre is therefore fitted from the same per-pixel mean the mask is computed from, before the mask is read, and named to the finder; it costs no frame of its own. That first fit is itself biased by the shadow it has not yet masked (on a sweep with a third of the detector behind a pin it landed 5 px out and called itself 0.40 px), and it does not have to be unbiased: the ring comparison does not notice tens of pixels, and the centre the run reports and consumes is the second fit of §1.4, the one that runs after the mask is loaded with the shadow out of the way. The order is fit, mask, fit, and only the second answer leaves. Two rounds are enough, measured rather than assumed — on clean sweeps the two fits agree to 0.03 px, so a third would draw the same rings.

A ring is only flat once the polarization is divided out. The comparison assumes the background is flat around a ring with nothing in the beam, and it is not: a polarized source suppresses the background in its own plane by the azimuthal factor of §2.2, a factor of three at \(2\theta=55°\) and four at \(70°\) — several times the dip the test is looking for, so on a short-distance geometry the two in-plane lobes of every outer ring read as shadow (measured: 5 % and 11 % of two detectors, with nothing visible under either mask). The mean projection is therefore divided by that factor before the ring comparison, and the Poisson deficit multiplies it back in so the significance is still the significance of the counts that were recorded. The factor is the geometry’s own, evaluated about the centre just fitted rather than the one in the file: polarization is the only correction a ring carries that varies along it — solid angle, detector response and air absorption are functions of \(2\theta\) and the ring median absorbs them — and the geometry is what knows where the polarization plane lies once the detector is tilted, the stored image quarter-turned or the detector rotated in its own plane.

What survives is then shaped into a region, and what makes it specific is size, not connection to the beam. A shadow is cast by something physical and is correspondingly large, while the background wanders a pixel or two at a time, so the connected components holding at least 2000 core pixels are kept and the rest dropped; measured on clean sweeps spanning 0.05 to 9.5 counts/px/frame of background, every one returns exactly one such component — the beam stop — and the largest spurious candidate anywhere is 74 pixels. Requiring the region to touch the direct beam instead is written for the stop and its holder arm; hardware standing in the beam further out — a pin, a loop — casts a shadow that begins some way out in radius, with lit detector between it and the stop and no bridge across the gap, and on one sweep that discarded 950 k correctly-found pixels and sent them into integration as measured-and-near-zero. The kept components are bridged across the module gaps the holder arm crosses, grown outward through the partially shadowed penumbra — much wider for a pin than for a stop edge — closed, and with the interior of the disk filled; the rings found lying wholly inside the stop join the region after the size filter rather than being asked to be large themselves.

Last, any pixel that ever recorded a real reflection — a maximum over the frames well above background, in a cluster, so that a single-frame zinger does not count — is given back, because a beam stop cannot have blocked a reflection that was measured.

1.6 Defective pixels

A pixel that reads high frame after frame, whatever the crystal does, is integrated into whichever reflection’s box it falls in; under rotation that is one pixel collecting a different reflection on every frame that reaches it, and a merged intensity hundreds or thousands of times its shell mean, carried by one or two observations the merge’s outlier test cannot judge. The file’s mask misses such pixels on many detectors, so on rotation data from a counting sensor Rugnux measures them on the pre-scan’s own sample of frames (HotPixelFinder, rugnux/HotPixels.h) and masks them as bit 10. The frames are read a second time for it, once the beam-stop projection has measured where the scattered background puts the beam: the rings have to be drawn about the true beam, and a file’s centre can be far enough out that a ring crosses the background’s radial fall-off.

On each frame a pixel is lit when it exceeds \(b + 3.3\,\max(\sqrt{b}, 1.4826\,\mathrm{MAD}) + 2\), with \(b\) the larger of the median of its 2 px iso-\(2\theta\) ring and of that ring’s sixteenth in azimuth, so ice and powder rings, the polarization dip and partial shadows set their own level. It is persistent when it is lit on at least \(\max(k_1, k_B)\) of the sampled frames: one reflection stays on a pixel for \((\Delta\phi + w)/|\zeta|\) of rotation (\(w = 5°\), a generous rocking width), which covers fewer than \(k_1 = 1 + \lceil (\Delta\phi + w)/(|\zeta|\,\delta) \rceil\) frames sampled \(\delta\) apart; and \(k_B\) is the binomial bound, from the lit rate of the ring’s other pixels, that fewer than 0.01 pixels of the whole detector reach by chance. A persistent pixel is masked if it reads on average at least ten times its ring and its mean excess is above the Poisson bound; a weaker one cannot make an outlier. This holds for a patch of such pixels as for a lone one: a patch lit through the whole sweep is stationary in the lab, so it is no reflection of the rotating crystal, and a reflection crossing it would read the patch. Pixels holding the detector’s error value on most frames (already invalid on every frame, but not in the static mask) are masked with them. A CCD (no sensor depth: read with an offset, not Poisson) is left alone. One log line says how many pixels were masked and why.

2. Azimuthal integration (radial profiles)

Azimuthal integration produces a radial profile \(I(q)\) or \(I(d)\) by histogramming pixels into radial bins. Pixels are not split across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 \(\phi\) sectors (azim_bins, --azim-phi-bins), giving a 2D \(q\times\phi\) profile that exposes azimuthal anisotropy such as detector shadowing or sample texture.

2.1 Histogram estimator

Let bin index \(b(x,y)\) be precomputed from \(q(x,y)\) (or equivalently from \(d(x,y)\)) and, when \(\phi\) sectors are enabled, the azimuth \(\phi(x,y)\) — so \(b = b_q + b_\phi B_q\). For each bin \(b\):

  • accumulate corrected intensity and its square: \( S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, \)

  • and count: \( N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. \)

The profile reports both the mean \(\bar{I}_b = S_b / N_b\) (when \(N_b>0\)) and a per-bin sample standard deviation \(\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}\) (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded.

2.2 Corrections applied

Two standard corrections are available:

(i) Solid angle / geometric correction. A flat pixel’s solid angle falls off with the incidence angle \(\alpha\) between the scattered ray and the detector normal. With the in-plane detector offsets \(u=(x-x_\mathrm{beam})p\) and \(v=(y-y_\mathrm{beam})p\) — measured from the PONI (§1.1), which is what the tilt-invariance below rests on — and detector distance \(D\), \( \cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad C_\Omega = \cos^3\alpha, \) applied — like the polarization term below — as a divisor (intensities are scaled by \(1/\cos^3\alpha\)), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because \(\alpha\) is evaluated in the detector’s own frame it is invariant under detector tilt (\(\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}\)), matching PyFAI’s solidAngleArray and MAX IV azint. It reduces to the commonly quoted \(\cos^3(2\theta)\) form only for an untilted detector, where the incidence angle coincides with the scattering angle.

(ii) Polarization correction. With polarization coefficient \(P\) (beamline dependent) and azimuth \(\phi\): \( C_\mathrm{pol}(2\theta,\phi) = \frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), \) applied as a divisor to intensities (i.e. scale by \(1/C_\mathrm{pol}\)) when enabled. This is the factor of Kahn et al. (1982); \(\phi\) is the azimuth in the lab frame, so the correction follows detector tilt and a swung-out \(2\theta\) arm without further work.

The polarization plane is taken to contain the lab \(x\) axis — a horizontally polarized source. A vertically polarized one is expressed by a negative \(P\): flipping the sign of the \(\cos 2\phi\) term is exactly a \(90^\circ\) rotation of the plane. The plane is not autodetected and is not read from the file — no format Rugnux reads declares one — so a vertical beamline must say so with --polarization. \(P\) is the polarization degree (XDS’s FRACTION_OF_POLARIZATION is \((1+P)/2\)). The default \(P = 0.99\), an undulator value, is applied to every Rugnux run; it is not fitted, because a single dataset does not determine it. Bending-magnet and wiggler beamlines are typically lower (0.8–0.95), and a known value for one should be passed with --polarization.

2.3 Background estimate for profiles

A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid \(Q\) window (default \(2\pi/5\) to \(2\pi/3\) Å\(^{-1}\)). This background is used for monitoring and diagnostics; it is not the same as the local Bragg-spot background used in summation integration (§9.2).

3. Spot finding (strong pixels → Bragg spots)

Spot finding is a two-stage process:

  1. Strong-pixel selection using intensity and/or local signal-to-noise criteria.

  2. Connected-component labeling (CCL) to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction.

3.1 Strong-pixel detection by local statistics

For each pixel \(i\) with value \(v_i\), consider a square window (nominally \(31\times 31\) pixels) around it. Let the window contain \(n\) valid pixels (excluding masked/bad/saturated), and define: \( \Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. \)

To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: \( \Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. \)

A variance-like quantity proportional to \(n'^2\) is formed: \( V = n'\Sigma_2' - (\Sigma')^2, \) and the deviation-from-mean quantity: \( \Delta = v_i n' - \Sigma'. \)

A pixel is considered strong if:

  • it is above a photon/count threshold, and

  • its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and

  • \(\Delta>0\), and

  • the squared deviation exceeds a scaled variance: \( \Delta^2 > V\cdot T^2, \) where \(T\) is the configured signal-to-noise threshold.

This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast.

The test is applied in two passes over the image. The first is as described above. The second repeats it with every pixel found strong by the first excluded from the local background — it is treated exactly like a saturated pixel, so it contributes to no window it falls into and stays strong itself. This matters for any spot wide enough to reach into its own background box: on a single pass such a spot inflates the mean and variance it is then tested against, and its outer pixels fail the criterion. Excluding the core recovers them, so the spot is reported with its true extent rather than its brightest few pixels. Both the CPU and GPU implementations run these two passes and return the same spot list for the same frame.

Special cases:

  • saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots),

  • invalid pixels are never strong.

3.2 Adaptive (self-calibrating) detection

The local-statistics test above needs a fixed photon/count threshold whose correct value depends on the background level, which varies between datasets. The adaptive mode (--adaptive-spots; the default in rugnux and in the viewer for both stills and rotation data, --no-adaptive-spots reverts) derives that threshold from each image’s own noise, per resolution ring, so no per-dataset value is needed. It admits more spots than the fixed threshold, including genuine reflections that belong to no indexed lattice; these are down-weighted rather than filtered in the per-image geometry fit (§7.4).

Pixels are binned into the same resolution rings as the azimuthal integrator (§2). For each ring a robust background is estimated in three passes: one plain pass over all valid pixels, then two \(\sigma\)-clipping passes that keep only pixels within \(\pm 3\sigma\) of the current ring mean (removing the Bragg peaks from the background estimate). This yields a per-ring background mean \(\mu_b\) and scatter \(\sigma_b\).

The ring’s detection threshold is the larger of two arms, \( t_b = \max\!\big(\;\mu_b + z\,\sqrt{\sigma_b^2 + \sigma_\mathrm{read}^2}\;,\;\; k_\mathrm{Poisson}(\mu_b, p)\;\big), \) where \(k_\mathrm{Poisson}(\mu_b,p)\) is the smallest count whose Poisson\((\mu_b)\) upper tail is \(\le p\). The Poisson arm is correct where the background is countable (a bright low-resolution ring gets a high threshold); the Gaussian arm — floored by a detector-level excess-noise constant \(\sigma_\mathrm{read}\) — takes over on near-empty high-resolution rings, where the Poisson arm degenerates to “one photon is significant” and would flood. The operating point \(p = E/N\) is set from a single portable knob \(E\), the expected number of false pixels tolerated per frame (--spot-false-pixels, default 100), with \(N\) the number of valid pixels. Because \(p\) and every \(\mu_b,\sigma_b\) come from the image itself, the same \(E\) lands a sensible photon threshold on strong and weak datasets alike, with no per-dataset tuning. Rings too sparse to characterise (fewer than ~40 pixels) fall back to a whole-frame background. A pixel is strong when \(v_i \ge t_b\) for its ring (saturated pixels are still forced strong); the strong pixels then feed the same CCL stage (§3.4). The signal-to-noise and photon-count criteria of §3.1 are not used in this mode.

Because detection reads the pixel’s ring, a pixel that falls outside the azimuthal-integration \(q\) range has no ring and can never be strong: the integration range bounds what adaptive detection can see. Both upper limits are therefore optional and default to the detector itself — the azimuthal integration runs to the highest \(q\) any pixel of the detector reaches (--azim-max-q unset), and spot finding is not clipped in resolution (--spot-high-resolution unset), for rotation data as well as stills. Setting either one narrows detection accordingly — appropriate for weak, high-background data, where the spots admitted at the detector edge are dominated by noise.

Fused GPU engine. The per-ring reduction the adaptive threshold needs is the same reduction the azimuthal integrator performs. On the GPU path the two are fused into a single image pass (AdaptiveSpotFinderGPU): one reduction accumulates the corrected per-ring sums for the azimuthal profile (§2) and the raw per-ring statistics for the threshold, after which a light kernel flags the strong pixels. One GPU pass therefore replaces both the separate azimuthal-integration pass and the host-side adaptive spot-finding pass, at a small fraction of the CPU finder’s cost per frame and producing the same spot list and azimuthal profile. It is enabled by default in the offline rugnux path, the interactive viewer and the online receiver.

Online. spot_finding_settings in the REST API carries adaptive_threshold and false_pixels_per_frame, so the mode is reachable from the broker and from the web frontend as well as from rugnux and the viewer. It defaults to off online, unlike rugnux and the viewer, because the broker serves both workflows and only one of them can run it: spots are found in software only on the DECTRIS/SIMPLON path, while the JUNGFRAU and EIGER workflows find them on the FPGA at its own fixed threshold. Setting adaptive_threshold on those is refused with an error rather than accepted and ignored, so a detection setting that had no effect cannot be mistaken for one that did.

3.3 Resolution and ice-ring handling

Spot finding can be restricted to a resolution range \([d_\mathrm{high}, d_\mathrm{low}]\) by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6).

A single per-image ice-ring score is derived from a radial profile: for each hexagonal-ice powder ring (see Where the ring positions come from below), the profile intensity at the ring is divided by a smooth background estimated from the whole profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring’s ratio is reported (1 = no ice, \(>1\) = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring’s own edge (offline processing defaults to a fine 0.01 Å⁻¹ spacing — in \(q = 2\pi/d\), like every \(q\) in this document (§1.2) — --azim-q-spacing, so the rings are well resolved). The reported quantity is the ice magnitude rather than a significance: with many photons any real ice ring is statistically significant, so significance does not discriminate.

The profile the score is read off is the peak-excluded one, not the plain azimuthal integration: where adaptive spot finding runs (§3.2 — the offline and viewer default), the score uses the sigma-clipped per-resolution-ring background that finder already computes for its threshold. This matters more than it sounds. A plain azimuthal profile is a per-ring mean, so a few strong low-resolution reflections landing in a ring’s bin raise it exactly as ice would; measured over a rotation battery, that alone ranked ice-free crystals above crystals that really are iced. An ice ring is azimuthally smooth and survives the sigma clip, while Bragg peaks do not, so on the clipped profile ice-free crystals sit near 1 and crystals with confirmed ice above 2. Only where no adaptive finder ran (the FPGA workflow) does the score fall back to the plain profile.

The radial profile sees ice only as a smooth powder ring. Ice in large crystallites diffracts as discrete spots, leaves the profile flat, and is invisible to the score above, so a second channel is read from the spot list itself: the spots found on the ice bands are counted against the spots found in the ice-free flanks \([w,2w)\) either side of each band, rescaled to the bands’ own \(q\) width (a flank landing on another ring is dropped with its width). The indicator is the ratio pooled over the run — per image the control is a handful of spots and the ratio means nothing — and it is taken before the spot-count filter, which orders ice spots last and would discard them first. The two channels barely overlap: smooth ice reads high on the profile and ~1 on the spots, textured ice the reverse, and a clean crystal ~1 on both. Both counts are stored per image (spot_count_ice_rings, spot_count_ice_control; HDF5 /entry/MX/peakCountIceRingRes, /entry/MX/peakCountIceRingControl).

Both channels are used offline as a gate on ice handling: unless the run reaches --ice-min-score (default 1.5) on the profile or --ice-min-spot-ratio (default 2.0) on the spots — 0 disables a channel — ice-ring flagging and the exclusion from the scale fit (§10.10) are skipped. The fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, and more than that on a detector reaching past 1.5 Å, so handling ice on a clean crystal is a pure loss.

Where the ring positions come from. The eleven bands from 3.895 to 1.522 Å are the measured positions of Moreau, Atakisi & Thorne (Acta Cryst D77, 2021, 540–554). That list ends at 1.522 Å by its own scope — the paper states that hexagonal ice “has 11 diffraction rings between 4 and 1.5 Å resolution”, and its subject was detecting ice in deposited data rather than masking it — not because ice stops there. On a detector that reaches further, the rings it does not list are the ones left in the data.

The eight bands below it are calculated, since past that paper there is nothing measured to copy. Enumerating \(hkl\) from the ice Ih cell is not enough: ice Ih is \(P6_3/mmc\) with oxygen on \(4f\), and most of what enumeration emits is extinguished by the oxygen sublattice rather than by the space group — which is why (004) at 1.830 Å and (104) at 1.657 Å are missing from the measured list even though they lie inside its range and its reflection conditions allow them (for even \(l\) the \(4f\) structure factor carries a factor \(\cos 2\pi l z\) at every \(hk\), and \(z \approx 1/16\) kills \(l=4\) — \((104)\) along with \((004)\)). Structure factors are computed instead — oxygen only, the hydrogens being half-occupancy disordered and weak to X-rays — and the lines kept are those reaching 3 % of the strongest. That rule reproduces the measured eleven exactly, and every line it drops inside their range computes to zero, which is what makes it trustworthy below 1.522 Å. The cell is that of Röttger et al. (Acta Cryst B50, 1994, 644–648). The list stops at 1.170 Å because below it the calculated real lines fall to 2–3 % while the extinct ones rise to about 1 %, and an oxygen-only calculation cannot separate them any further.

One consequence is worth stating: the profile score is the strongest ring’s ratio, a maximum over the bands, so a longer list can only raise it. The gate at 1.5 is therefore read against a list of this length, and lengthening it again would need the gate re-checked.

Rings this sample actually shows. Hexagonal ice is the only phase whose rings can be named in advance, and it is not the only thing that powders: a shower of microcrystals around the crystal, or a salt out of the cryoprotectant, leaves the same textured rings at \(d\)-spacings no fixed list carries, and their spots are otherwise handed to the indexer as if they were this crystal’s. The rings are therefore also measured from the run’s own pre-scan spots — read off as what stands above the smooth fall-off of spot density with \(q\) — on every run, and reported whether or not anything acted on them (POWDER_* in the results report: the ring count, the fraction of the pre-scan’s spots they hold, and their \(d\)-spacings). The two lists are not the same list: on one such crystal 16 of the 24 measured rings were ice and the rest were not. A measured ring is set aside from indexing only where a first pass that found no lattice needs it set aside (§6); the fixed ice bands above are what the ice score, the scale fit and the resolution cut read.

A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in \(1/d\) (or \(q\)) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low.

3.4 Connected-component labeling (CCL)

Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with:

  • centroid \((x,y)\) (often intensity-weighted),

  • pixel count (spot size),

  • integrated spot intensity proxy (sum of pixel values),

  • resolution \(d\) at the centroid (or mean over pixels),

  • and quality flags (e.g. ice-ring classification).

Spot-level filters include minimum/maximum pixel count and resolution limits.

The upper bound is on how large a round spot may be, not on how bright one may be. Under the self-calibrating threshold (§3.2) a component’s area above the contour grows as \(\sigma^2\ln(A/T)\) — without bound in the peak amplitude \(A\) — so an upper bound on pixel count alone becomes an intensity ceiling: measured on a strongly diffracting crystal, footprints run 3 px at 30–100 counts to 50 px above 10 000, four times the slope the fixed local-box test gives, and the old bound of 50 discarded the brightest reflections on every image. The bound is therefore 200 (the same as CrystFEL peakfinder8’s --max-pix-count; XDS has no such parameter at all and guards on shape instead, with SPOT_MAXIMUM-CENTROID), and a component above 50 pixels must in addition fill at least a fifth of the square its bounding box fits inside. A Bragg reflection is round and fills about half of that square however bright it is; an ice arc, a cosmic-ray track or a lit detector row fills a fifth or less, which is what an upper bound was ever protecting against. Below 50 pixels no shape is asked for, so everything accepted before still is. The shape test is inert on every dataset it has been measured on — it exists to bound the shape of what the larger size bound now admits, on data carrying arcs or tracks, not because the crystals measured here needed it. The test is integer arithmetic, so the host and the GPU extractor agree by construction.

The host implementation (StrongPixelSet::sparseccl) is the SparseCCL of the ACTS/traccc project: it runs over the strong pixels sorted row-major, uses a sliding window over the previous line and a union-find whose root is each component’s lowest index. On the GPU the same labelling runs on the device (SpotExtractorGPU): the packed strong-pixel bitmask is compacted into that same sorted list without leaving the card, each pixel finds its at most four earlier 8-neighbours by binary search, and a lock-free union-find with path halving labels them. Only the finished spot list — a few hundred entries — comes back to the host, instead of the whole bitmask (2.26 MB per frame at 18 MP). The two implementations produce the same components, in the same order, with the same pixel counts and intensities; tests/SpotExtractorGPUParityTest.cpp holds them to it. The device version is also insensitive to frame content: the host sliding window becomes quadratic when many pixels light up in one detector line — a hot module, or a diffraction ring where it runs tangent to a row — which costs tens to hundreds of milliseconds on such a frame, while the device version stays under a millisecond.

3.5 Adaptive per-image minimum spot size

The minimum-pixels-per-spot filter (§3.4) trades sensitivity against noise: a small value keeps faint one- or two-pixel spots — real signal on strong data, but detector noise on high-background frames — while a larger value keeps only well-formed spots. The best value is dataset-dependent, so for serial-stills indexing it can be chosen per image rather than fixed. The frame is indexed three times, at min-pix 3, 2 and 1, and the setting that maximises

\[ \frac{n_\mathrm{indexed}^2}{n_\mathrm{total}} \quad\text{(indexed-spot count weighted by indexed fraction)} \]

is kept; the frame is then integrated once at that min-pix. The fraction factor discounts the extra spots a smaller min-pix admits unless the lattice actually explains them, so strong frames keep their real weak spots (extending resolution) while noise-flooded frames stay strict. Because min-pix filters the connected components after detection, strong-pixel detection AND the connected-component labelling both run once per frame, and the three attempts only repeat the spot-level filter; the azimuthal profile is the one that single detection pass computed. The winning attempt’s spot list is kept rather than re-extracted, so the frame that is integrated is exactly the frame that was scored. This is a stills-only, indexing-path option — rotation indexing builds one global lattice from all frames and keeps a fixed min-pix. In rugnux it is the default; giving an explicit --min-pix-per-spot pins a fixed value instead.

3.6 Predicting the resolution the merged data will reach

A per-image resolution estimate is read off the finished spot list. It predicts how far the merged data will reach, not how far the furthest spot on this image lies. Each non-ice spot is weighted by \(\sqrt{I}\) — the intensity is a summed photon count, so \(\sqrt{I}\) is its Poisson significance — the \(1/d^2\) is found beyond which a fraction \(f=0.30\) of that weight lies, and the estimate is that resolution taken \(2.25\times\) further in \(1/d\). It is deliberately not limited to what the detector records: the quantile sits in the middle of the fall-off, well inside the recorded range, so it goes on measuring the crystal where the detector stops before the diffraction does, and on such a run it reads finer than the detector corner. The dataset value is the median over images.

Both constants carry a mechanism. A quantile from the middle of the distribution measures the shape of the fall-off, which is the crystal’s own \(\exp(-B/2d^{2})\), where the extreme end of it measures where detection stops — a threshold that moves with the exposure and with how many reflections the unit cell puts on a frame. And merging averages many observations of each reflection, so intensities go on being measurable a fixed factor in \(1/d\) past the point at which one image’s spot finder still detects them; that factor is the \(2.25\). Both are calibrated on rotation data against the resolution at which per-shell CC1/2 falls through 0.30, and the estimate is good to about 0.2 Å there. It is a prediction and not a measurement of what a run achieved: nothing downstream is cut on it, and it is reported alone (Rugnux SPOT_RESOLUTION_ESTIMATE, and per image in the stream, the plots and HDF5).

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS_INDEXING.html b/CPU_DATA_ANALYSIS_INDEXING.html new file mode 100644 index 000000000..46b71c475 --- /dev/null +++ b/CPU_DATA_ANALYSIS_INDEXING.html @@ -0,0 +1 @@ + Data analysis: indexing and geometry (§4–§7) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Data analysis: indexing and geometry (§4–§7)

Part of the CPU/GPU data-analysis reference; the section numbers are continuous across its four parts.

4. Indexing overview

Indexing maps observed reciprocal-space vectors \(\mathbf{s}_i\) to a lattice such that: \( \mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, \) with integer \((h_i,k_i,l_i)\).

Jungfraujoch supports two complementary indexing strategies:

  1. FFT-based indexing (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples.

  2. Fast-feedback indexing (TORO-like): requires an approximate unit cell; optimized for speed and feedback.

Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return more than one lattice per image (multi-lattice indexing; see §5.4).

4.1 Indexed-spot decision (inlier test)

Given a trial lattice with direct basis vectors \(\mathbf{a},\mathbf{b},\mathbf{c}\) (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: \( h_f = \mathbf{s}\cdot\mathbf{a},\quad k_f = \mathbf{s}\cdot\mathbf{b},\quad l_f = \mathbf{s}\cdot\mathbf{c}. \) Let \((h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))\) and define the fractional residual: \( \delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. \) A spot is indexed if \(\delta^2 < \tau^2\), where \(\tau\) is the configured tolerance.

For indexed spots, the reciprocal lattice point \(\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*\) is used to compute \(\Delta_\mathrm{Ewald}(\mathbf{p})\) (stored as a diagnostic and later used in profile-radius estimation).

A frame is taken to be this crystal’s when at least a fraction \(g = 0.20\) of its in-resolution, non-ice spots index. On rotation data that decision is what admits the frame to integration, so its denominator matters: every spot handed to it that is not a reflection of this crystal argues against the frame. Where it cannot do that job — a lattice whose pooled spots fall below \(g\) and which fewer than half the validation frames clear — every frame is integrated instead: the frames that clear \(g\) are then only the upper tail of the same sparse population, not the frames the crystal was in, and admitting them alone dropped most of a small-molecule sweep and its completeness with it.

That test decides a frame. Whether a rotation run has a lattice at all is decided on the spots instead. A frame count comes from serial crystallography, where each image is its own experiment; a rotation sweep is one crystal and one orientation matrix, its frames are not independent of each other, and what such a count mostly measures is how many spots happen to land on a frame — a sweep carrying four spots an image cannot reach a six-spot bar on three frames in four however right the lattice is. The refusal therefore compares the fraction of all spots in the sampled frames that the lattice explains against what the same lattice explains when each frame’s spots are put at another frame’s angle: same lattice, same spots, same detector, same refinement, with only the claim that these spots were seen at these angles removed. That difference is the evidence, and it carries no spots-per-frame number anywhere, so nothing has to be chosen for a crystal that diffracts weakly. Measured over a hundred datasets the permuted level never exceeds 2.6 % and the smallest true margin is seventeen points. --min-indexed-spots (default 6, floor 4 — four is where a lattice stops being fitted by any three spots) still sets the reported indexing rate and the count the first pass ranks candidate lattices by; every rescue and every arbiter still counts frames.

4.2 The spot budget

Only the strongest --max-spots spots of an image are kept (FilterSpotsByCount), and that budget therefore sets the denominator above. Detections are not all reflections — background structure, unlisted ice and detector artefacts are found too — so a budget deeper than an image’s reflections makes the test above a measurement of the background rather than of the crystal, and a larger budget can integrate fewer images.

rugnux measures the budget instead of fixing it. With the sweep’s lattice in hand, the first pass tallies the spots of a sample of frames by their rank in the intensity-ordered list: how many images carried a spot at that rank, and on how many of them it indexed. Weighting each indexed spot by \(1-g\) and each unindexed one by \(-g\) — the same weighting the frame test applies to the list as a whole — the running sum over ranks

\[ E(N) = n_\mathrm{indexed}(N) - g\, n_\mathrm{counted}(N) \]

rises exactly while the spots at that depth lie on the lattice more often than \(g\), and falls after. The budget is \(\arg\max_N E(N)\). Its meaning is “as deep into the list as the image is still showing reflections of this crystal”: deeper spots cannot help the frame test and can only push a frame towards rejection. On crystals whose spot lists are reflections all the way down the maximum is at the end of the list and the budget is unchanged.

The peak has to be one. Under the null — the spots lie on the lattice at the same rate at every depth — \(E\) is a driftless random walk in the counted spots, with per-spot variance \(g(1-g)\), and the maximum of such a walk is positive whatever the data; an \(\arg\max\) taken on its own would shorten every dataset, including one with nothing to shorten. What the budget acts on is the fall from the peak to the end of the list, \(E(N^*) - E(L)\), which is the maximum of the same walk read backwards; by the reflection principle its null law is \(P(\mathrm{fall} > z\sqrt{g(1-g)T}) = 2(1-\Phi(z))\) over \(T\) counted spots in all, so the search over ranks is already accounted for and no further multiple-comparison correction applies. The budget is taken only where the fall clears that bar at \(z = 3.29\), one false shortening in a thousand measurements; otherwise the whole list is kept.

5. FFT indexing (unknown unit cell)

FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms.

5.1 Directional projections and histograms

Choose a set of unit vectors \(\{\mathbf{u}_d\}\) on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction \(d\), form a histogram in the scalar projection: \( t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. \)

Bin width is chosen approximately as: \( \Delta t \approx \frac{1}{2 L_\mathrm{max}}, \) where \(L_\mathrm{max}\) is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum \(q\) used (set by a high-resolution cutoff for indexing).

5.2 FFT peak picking and candidate vectors

For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along \(\mathbf{u}_d\). Each direction yields a candidate real-space length \(L\) chosen not by raw magnitude but by maximum prominence above a running-mean local background (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to \(L\ge L_\mathrm{min}\).

The running-mean background window keeps a constant width and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel’s running sum is still valid.

The longest basis vector the transform can return is fft_max_unit_cell_A, since the histogram is sized from it and its last usable bin is that length; the shortest is fft_min_unit_cell_A (rugnux --fft-min-unit-cell, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with -C moves both bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it.

Candidate vectors are \(\mathbf{v}_d = L_d\,\mathbf{u}_d\).

A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak.

5.3 Lattice reduction and cell candidates

Triples of candidate vectors are combined to form candidate bases \((\mathbf{A},\mathbf{B},\mathbf{C})\), each reduced to its Niggli-reduced cell (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a widened fallback anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses.

A triple whose three vectors are coplanar is rejected before refinement. The length and angle filters cannot see it — any flat combination satisfies them — and a cell that flat has a metric determinant small enough for float to get its sign wrong, after which the guard against a negative argument to the square root places \(\mathbf{c}\) in the \(\mathbf{a}\)-\(\mathbf{b}\) plane, the reciprocal volume diverges and the solver reports a not-a-number Jacobian. The test is the volume fraction \(|V|/(|\mathbf{a}||\mathbf{b}||\mathbf{c}|)\), which must reach 0.02 — about 1.1° off flat, well below the flattest genuine candidate observed and far above where float loses the sign — and it is applied both where triples are produced and at the optimizer’s entry points.

A shortlist confined to one plane cannot close a cell at all, and the row it is missing is the plane normal. That is detected from the eigenvalue ratio of the shortlisted directions’ scatter matrix, and one further transform is then spent with the same direction count inside a narrow cap about the normal. More directions do not substitute for it: at the exact true direction a very long axis can still rank far below the shortlist cut, so for this rescue the obstacle is the ranking rather than the sampling, and a denser grid costs several times the device memory for the same answer.

Sampling has a limit of its own, and it binds well before the 1200 Å the accepted range for fft_max_unit_cell_A admits (§5.2). A direction off a real-space axis of length \(a\) by an angle \(\theta\) smears each projected lattice plane by about \(\theta/d_\mathrm{min}\) in the projection, so the planes (spacing \(1/a\)) stay resolved only while \(\theta \lesssim d_\mathrm{min}/(2a)\). The shipped grid of 16384 directions puts the nearest one within about 0.6° of any axis, which satisfies that bound only up to roughly 120–150 Å at typical indexing resolutions; a longer axis is not refused but returned as a plausible sub-cell or harmonic. Raising the maximum cell alone therefore does not extend the reach — the direction grid has to resolve the axis before the histogram can represent it.

5.4 Robust refinement and best-cell selection

Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked:

  1. more indexed spots wins — unless two candidates index within ~10 % of each other, in which case

  2. the smaller-volume cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then

  3. the smaller refinement score, then the spot count again.

Selection is not limited to a single lattice: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded.

An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant).

5.5 Spindle alignment: the part of the blind cone symmetry cannot repair

A sweep about the spindle \(\hat{\mathbf{e}}\) never brings a reciprocal point closer than \(\theta_\mathrm{max}=\arcsin(\lambda/2d)\) to the axis onto the Ewald sphere, so a double cone of half-angle \(\theta_\mathrm{max}\) is missing from every resolution shell — each shell losing its own \(1-\cos\theta(d)\) — however long the sweep runs. Crystal symmetry normally repairs that loss by mapping the cone onto measured territory. It fails to when a symmetry axis lies inside the cone (the cone maps onto itself) — and, for a 2-fold, equally when the axis is perpendicular to the spindle, because the diad carries the cone onto its opposite lobe, which the sweep leaves just as unmeasured. Friedel never helps: the cone is double-sided. The loss is a coherent cap rather than a scatter of absences, so it costs a map more than the same percentage lost at random.

The per-image score asks how much of that cone the frame’s own orientation makes unrecoverable. The crystal’s short lattice rows are read off the FFT row shortlist of §5.2 (a symmetry axis is always a lattice row, and usually among the short ones), and each plausible direction is scored as if it carried a lone 2-fold:

\[ \text{spindle blind fraction} = \frac{2}{\pi}\left(\arccos x - x\sqrt{1-x^{2}}\right), \qquad x = \min(\beta,\,90^\circ-\beta)\,/\,\theta_\mathrm{max}, \]

where \(\beta\) is the direction’s miss-angle from the spindle. The fold of \(\beta\) about \(45^\circ\) is the diad geometry above: both ends of the range are the bad case, and the closed form reproduces a Monte Carlo of the true double-cone self-overlap to 0.004 at \(\theta_\mathrm{max}=15^\circ\) and 0.008 at \(25^\circ\) (past \(45^\circ\) it under-reports, by 0.05 at \(50^\circ\)). \(\theta_\mathrm{max}\) is taken from the geometric resolution of the setup — the detector corner at the recorded distance and wavelength — an upper bound on any sweep collected without moving the detector; the still’s own spot resolution would understate the cone on exactly the weak frames that mislead. The worst direction wins, and the directions scored are the strong in-window rows and the normals of their pairs — the normal to two lattice rows is itself a reciprocal-lattice row and a symmetry axis is parallel in both bases, so a lone 2-fold on an axis far beyond the length window (a long monoclinic unique axis) is still seen by direction: measured on a synthetic lone-diad crystal with a 300 Å unique axis, the fraction of severe mounts reported severe rises from 0.60 to 1.00 with the pair normals, at no extra engagement on that class’s harmless mounts. Nothing about the goniometer enters: the number describes the problem and leaves the remedy — a second sweep, a reorientation — to the beamline.

This is a worst-case bound under an assumption of no symmetry, not an estimate. A still cannot know the point group, so the nearest plausible row is scored as a lone 2-fold. An axis of order \(\geq 3\) perpendicular to the spindle in fact fully repairs the cone (measured unrepaired fraction 0.000 for orders 3, 4 and 6, against 1.000 for a diad), which a still cannot see, so the bound is deliberately pessimistic on higher-symmetry crystals — that is the intended trade, because the number exists as a trigger for beamline automation, not as a physical quantity a user interprets.

Trigger states. The stored quantity is the continuous score; automation reads it through three fixed states with nothing to tune (SpindleTrigger in SpindleBlindFraction.h): engage at score ≥ 0.5, don’t engage below, and cannot say when there is no value at all — too few spots, no shortlist, the consistency guard refused, the path never computed one. Automation must treat CANNOT SAY as ENGAGE: the error costs are asymmetric — a false negative is unrecoverable (one sweep is collected and the data stay short forever) while a false positive costs minutes of beamtime. Every transport keeps absence distinguishable from a measured zero (an absent CBOR key, a NaN in the HDF5 arrays, an absent optional after read-back). The 0.5 threshold is geometry, not tuning: the score is monotone in the folded miss-angle, so a threshold is a fold-angle gate, and 0.5 gates at \(\min(\beta,90^\circ-\beta) \le 0.404\,\theta_\mathrm{max}\); engaging on any overlap at all would gate at the cone edge, whose perpendicular band alone spans \(\sin\theta_\mathrm{max}\) of orientation space per row (26 % at \(15^\circ\)) and unions over a frame’s rows to well over half of all mountings — a trigger that always fires decides nothing.

Reach and honest rates. The score needs 60 spots (calibrated per crystal — 22 independent mounts — misses triple below it); below that, a frame that still indexed answers from the winning lattice’s shortest rows, and otherwise the state is cannot say. Because the bound is pessimistic by design, it engages on a substantial share of harmless mountings: a single strong row’s perpendicular band alone covers ~11 % of orientation space at the severe level (\(\theta_\mathrm{max}=15^\circ\)), and the union over a frame’s rows and pair normals reaches roughly a quarter to three quarters of random mountings depending on cone width and row count (measured 0.74 on a generic triclinic cell at \(15^\circ\) via the lattice path). That is accepted: the cheap error is the extra wedge. An earlier figure of ~1 % false alarms (AUC 0.948) came from a null of five decoy directions per frame — it shows the estimator does not hallucinate rows near arbitrary directions, which is worth knowing, but it is not a false-alarm rate over harmless mountings, which geometry forbids to be that low.

Offline, the guessing stops. Once rugnux has merged a rotation run it holds the measured point group and the exact indexed orientation, and the run-level number is computed exactly instead: the group’s proper rotations are applied to the blind double cone in the crystal’s actual orientation, and the fraction of unique reflections no operator can recover is reported as SPINDLE_LOST_UNIQUE_FRACTION in the processing report and /entry/MX/spindleLostUniqueFraction in the master file (the results report). That number clears or convicts a mounting the per-image bound can only be pessimistic about: a dihedral crystal with an in-plane diad on the spindle, or any cubic crystal in any orientation, loses nothing at all.

7. Geometry and lattice refinement

Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices.

7.1 Parameterization

The refinement jointly optimizes, depending on mode and constraints:

  • beam center \((x_\mathrm{beam}, y_\mathrm{beam})\),

  • detector distance \(D\),

  • detector tilt angles (two-angle model; third rotation often held at 0),

  • rotation axis direction (for rotation datasets),

  • crystal orientation (a global rotation),

  • unit-cell parameters, with constraints determined by inferred crystal system.

The detector distance is not refined against one crystal’s spots at all: the positional residual leaves it degenerate with the cell scale, so it is fitted elsewhere - by the rotation post-refinement, which frees it alongside the whole crystal and adds the distance-independent rocking-angle residual that breaks the degeneracy, and by the stills --refine-geometry bundle. Per image, the beam centre and the crystal orientation are refined, and the unit cell as well for stills. The first-pass rotation indexing refines the detector tilt and the rotation-axis direction too, against the spots accumulated across the sweep; everywhere else both are held fixed, because on a single crystal a tilt is absorbed almost exactly by the crystal orientation. A lighter orientation-only mode refines just the crystal orientation, for stills whose geometry is already trusted. It carries a weak small-rotation prior penalising the whole angle-axis vector (all three components, at a low weight); what it is there for is the poorly-determined out-of-plane component, which is the one the data barely constrain.

For higher symmetries, constraints are enforced, e.g.

  • cubic: \(a=b=c,\ \alpha=\beta=\gamma=90^\circ\),

  • tetragonal: \(a=b\),

  • hexagonal: \(a=b,\ \gamma=120^\circ\),

  • monoclinic (unique axis \(b\)): \(\alpha=\gamma=90^\circ\), \(\beta\) refined.

7.2 Residuals and objective

For each indexed spot assigned integer \((h,k,l)\), compute:

  • observed reciprocal vector \(\mathbf{s}_\mathrm{obs}\) from its detector position and current geometry,

  • predicted reciprocal vector \(\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})\).

Residual is: \( \mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. \)

A non-linear least squares solver minimizes \(\sum \|\mathbf{r}\|^2\) over all selected inlier spots.

7.3 Rotation datasets: bringing observations to a common reference frame

For oscillation/rotation data, each image corresponds to a rotation angle \(\phi\) about an axis \(\mathbf{m}_2\). Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: \( \mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, \) where \(R(\phi)\) is the rotation by \(+\phi\) about the goniometer axis as stored in the file. The sign is a convention and it is load-bearing: rotating the observations forward by \(+\phi\) means the crystal itself turns by \(-\phi\) about that stored axis, i.e. \(R(\phi)\) is the inverse of the crystal’s own rotation from the reference orientation to frame \(\phi\). The same convention is why the unmerged-MTZ batch headers and the XDS geometry echo carry the axis negated relative to the input file (Rugnux ▸ the unmerged export) — a reimplementation that takes \(R(\phi)\) as the crystal rotation must use \(R(-\phi)\) here instead. The angle \(\phi\) is taken at the centre of each frame’s oscillation (the frame angle plus half the oscillation width).

7.4 Multi-stage tightening of inlier tolerance

Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance \(0.3\to0.2\to0.1\)), which stabilizes convergence when starting from imperfect indexing and approximate geometry.

The loose first stage necessarily admits some spots that are not reflections of this lattice — the fraction of randomly placed spots inside a fractional-Miller tolerance \(t\) is \(\tfrac{4}{3}\pi t^3\), i.e. 11 % at \(t=0.3\) — and an unweighted fit lets them pull the orientation. Each residual is therefore weighted by how strong its spot is for its resolution: the frame’s spots are cut into equal-count resolution shells and each intensity is divided by its shell median, mapped to \(w^2=r/(1+r)\). The shell normalisation is what makes this safe — genuine high-resolution spots are legitimately weaker and carry the cell and distance information, so an un-normalised intensity weight would suppress exactly the spots the fit needs. The weight is a property of the spot and never of the current residual, so it does not depend on how far the geometry is from convergence.

The rotation chain commits its best round, not the round it stops on. After the winning candidate is selected, the refinement is run again — solve, re-accumulate the reciprocal-space cloud under the refined geometry, solve again — up to twenty times, and the loop stops on a test of the detector-tilt step. The chain is a trajectory and its last point is not always its best one: every solve ends by fitting only the spots inside its tightest gate, so a cell with a direction the data barely constrain (which is what a free cell whose metric is near a Bravais class has) can slide along it, pulling a core of spots tighter while the periphery falls out of the fit altogether. Measured on such a chain, the spots inside the tight gate rise over seventeen rounds while the spots inside the widest gate peak at round three and fall away — and round three is the round that merges at ISa 11.0 against 6.3 and \(R_\mathrm{meas}\) 0.148 against 0.213. Nothing the run consulted could see it: indexed fraction, validation frames, validation spots and the tight-gate count all prefer the overfitted end.

So every round is scored on the widest gate — the population the first pass selects on and the last pass does not fit, which makes it the one a converged solve is not optimising — and the best-scoring round is committed. Two conditions keep that from acting on noise. The score is a count of spots, so a lead of fewer than \(\sqrt{\text{count}}\) of them leaves the last round standing. And the round taken has to be the less distorted lattice as well as the better-fitting one: the lattice search (§6) is re-asked every round to measure how far the cell sits from the ideal metric of the class it matches (imposing that class is measured fatal — the snap puts almost everything outside the refinement’s own gate), and an earlier round is taken only when it matched the same class and sits closer to it. Same class is a precondition and not a precaution: the deviation is a fraction of whichever class’s tolerance admitted it, so two classes’ deviations are not the same quantity, and a round that matched no class reports zero, which means “nothing was asserted” rather than “undistorted”. A chain that has settled scores its rounds within a spot or two of each other and a symmetry-constrained solve holds its distortion at zero throughout, so the rule fires on neither: measured over 914 chains, an earlier round scores higher on 44 % of them and the committed round changes on 1 dataset in 54.

A tilt no mounting can have has to prove itself. The detector tilt is refined freely because on a sweep whose spots reach far enough in \(2\theta\) it is a measurement, and restraining it costs those crystals resolution (measured: 0.13–0.16 Å and up to a quarter of ISa on the crystals whose fitted tilt is largest). What makes it a measurement is the keystone — a tilted plane puts one side of the detector nearer and the other further, so the spots move by an amount that grows with their distance from the beam — and a first pass made of spots reaching a few degrees of \(2\theta\) sees a keystone of a pixel or two at most. To such a fit a tilt is a whole-pattern shift the beam centre imitates exactly, its size is whatever the centroids’ own systematics happen to prefer, and the value it commits then mispredicts the far corner of the detector by tens of pixels against an integration disc of a few: a first pass seeded to \(2\theta = 5°\) committed 2.5° and the run collapsed from 2.8 Å to 6.4 Å. For a tilt of a few tenths of a degree nothing in the spots says whether it is real — the held-out positional residual, the rocking angles and a re-fit at the header tilt were all measured unable to, at the same insignificance on crystals whose tilt is real and on the one whose tilt was the artefact — so below what a mounting can be off square by the fit is trusted as before: a detector is mounted square to the beam to a fraction of a degree, and over 211 datasets the fitted tilt left the file’s by more than 0.56° on one. A chain that has walked more than 1° from the tilt it started at has either measured nothing or found a detector the file misdescribes (that one: a \(2\theta\) arm swung out 12.8° that the file records as square), and at that size the spots do tell the two apart, because a real tilt of degrees has a keystone of tens of pixels over the spots the fit is made of and an artefact has none. So the candidate is refined again from where it started with the tilt held there — the beam centre takes the shift the tilt is equivalent to — and the two are judged as the rounds of one chain are, on the spots each indexes inside the wide gate: the walked tilt stands only when it leads by more than the count’s own noise. Held, not bounded — a box the fit lands on is the same wrong answer at a smaller size. The log says what the walked tilt would have moved the far corner by and how the two counts came out; where it was refused, the report’s REFINED_DETECTOR_TILT is the tilt the pass started at and REFUSED_DETECTOR_TILT the tilt the fit had walked to.

7.5 Rotation geometry post-refinement (two-pass)

The refinement above (§7.2) runs per image against that image’s spots. For rotation data an additional post-refinement (on by default; --rotation-no-postrefine disables it) improves the detector distance, beam centre and crystal cell/axis using all frames at once, then re-integrates:

  1. Pass 1 integrates, scales and merges at the header geometry.

  2. From pass-1’s integrated reflections, the crystal and the detector are refined together over all frames (Ceres, robust loss) in one joint fit, against both residuals at once:

    • the positional detector↔reciprocal residual at each partial’s observed spot, and

    • a distance-independent Ewald excitation residual at each reflection’s observed rocking centroid \(\phi_\mathrm{obs}\).

    Free: the crystal orientation, the unit cell (every parameter the crystal system leaves free, not one overall scale), the goniometer-axis direction, the detector distance and the beam centre. The positional residual on its own is degenerate with the cell scale — that is why this used to be split into a cell-scale step and a distance step — but the excitation residual does not involve the detector at all, so it fixes the absolute size of the reciprocal lattice and breaks the degeneracy inside the same problem. Splitting it instead cost accuracy twice over: pass 1 frees the whole lattice against a frozen distance, so the distortion it absorbs is anisotropic and no single scale can undo it; and whatever bias is left in that scale goes straight into the distance, which is only ever determined relative to the cell.

    The fit is cross-validated on a deterministic split of the reflections (an avalanche-mixed \(hkl\) hash, not a frame split and not an \(h+k+l\) parity, which would collide with a centering condition and leave the held-out half empty): fitted on one half, committed only if it lowers the held-out residual by more than that residual’s own noise (the standard errors of the two held-out means combined, the bar a round of the geometry walk has to clear) and its two families agree: the positional residual has to fall by more than its own noise, because the positions are the only evidence of the detector geometry a commit hands the next pass, and the excitation residual must not rise by more than its own noise, because it is the only evidence of the cell scale and the positional values outnumber it about three to one. That noise is the paired standard error — both geometries are read on the same held-out reflections, so what a change has to beat is the scatter of the change each reflection sees; bare signs stood here, and were a coin toss wherever a family did not move — and the move stays inside its bounds: every free cell angle within 1° and the beam centre within 15 px of the nearest centre anything already believes.

    The distance and the cell lengths are bounded one step at a time, not as a whole. One per cent was once a cap on the entire move, and as a cap it was the opposite of its job — a header is most worth correcting when it is most wrong, and a geometry genuinely several per cent out could never be reached (measured: a refused fit of 310.000 → 305.692 mm whose cell landed within 0.06 % of the deposited one). It is a trust region instead. The first solve is asked in the wide box around nominal exactly as before, so a fit that settles within one step commits unchanged; a fit that wants more is re-fitted as a walk of one-per-cent steps, each seeded where the last arrived and each required to lower the held-out residual, stopping where a step stops paying. A walk that uses every step it is allowed has not settled — it stopped because it ran out of steps, not because it arrived — and is refused, which is the runaway the cap stood in for, tested where it can be seen. A move of more than one step is additionally ratified by re-indexing at where it arrived: that is what separates the failure the cap was really aimed at (a second lattice, whose spots bias every cross-validation fold identically) from a wrong header, since a second lattice does not index better at the new geometry and a real distance error does.

    The geometry the run commits is re-fitted on all the reflections once the held-out half has approved it — a walk from where it arrived, one step wide; everything else from nominal in the wide box — and the bounds are asked again of that fit rather than only of the half that earned it. A move outside them leaves the geometry at nominal, as every other refusal does, and the refusal says so in the report (POSTREFINE_REFUSED, with what the fit wanted) rather than passing silently. Detector tilt is held fixed, being gauge-coupled to the crystal orientation on a single crystal.

    Whether the data determine the distance at all is asked, not assumed. At a detector far enough away that no reflection reaches more than a few degrees of \(2\theta\), a longer distance and a larger cell move every spot the same way to first order — the difference is of order \(\sin^2\theta\) of the spot’s own position, about 0.4 px rms per per cent of distance over a 2M detector at 820 mm against 2–3 px at the distances a crystal is usually collected at. The joint fit then finds a distance/cell pair that fits its own spot positions a little better than the header, commits it, and the pass re-integrated there finds the next pair: a walk along the degenerate direction that the realised residual never ratifies (measured on such a sweep: a header at 820 mm walked to 846 mm with the cell 3.3 % too large, the realised held-out residual flat at every round). So the same fit is asked once more with the distance held at the header, every other block as free as before — the nested hypothesis “the header distance is right” — and the two are compared on the one residual family that can tell them apart: the excitation residual. It never involves the detector, so it is blind to the distance itself; what it sees is the cell scale, and a held fit at a wrong header distance is forced into a wrong cell scale by the spot positions, which the rocking angles then refuse (measured: a header 1.4 % long leaves the held fit’s excitation residual seventeen times the free fit’s). Where freeing the distance lowers the held-out excitation residual below the held fit’s by more than that residual’s own standard error, the free fit is committed exactly as before; where it does not, the held fit is — header distance, refined beam, cell, orientation and axis — and the report says so (POSTREFINE_DISTANCE_HELD). The positional residual is deliberately not consulted for this: it is the family whose in-fit gain along the degenerate direction re-integration erases, and pooled with the excitation family it either drowns a decisive excitation gain in its own noise (a 54 % excitation gain read as 9 % pooled against a 9 % noise) or lends the degenerate direction a gain that is not there. Nothing is tuned here: the only input is the standard error of the residual itself, the same noise the geometry walk’s rounds have to beat. The wavelength is never refined on a single crystal for the same reason in its exact form: it scales the spot positions and the rocking angles identically to the cell, so no sweep can tell the two apart at any \(2\theta\).

  3. Pass 2 re-indexes de novo and re-integrates at the committed geometry. Only the detector distance and beam centre carry over: the refined cell, orientation and axis are what make the distance identifiable, but pass 2 re-indexes from scratch, so they are not propagated. Where the re-index finds a different lattice — the two compared on their Niggli-reduced primitive edges, within 2 %, so a symmetric setting is never told apart from its own primitive cell — pass 1’s lattice is scored at the refined geometry as a hypothesis of its own, and integrated when it indexes more validation frames; the same lattice found again is kept as the re-index refined it. The run likewise falls back to pass 1’s lattice where the re-index indexes too few frames, and integrates it at the refined geometry — and the cell is then scaled to the distance it will be used at, since a real-space cell is measured against the distance its spots were seen at, and carrying it across a distance change otherwise scales the whole cell by the ratio of the two. The orientation is untouched.

    Pass 2 measures the post-refinement again, and where it still moves the geometry the run walks: it re-indexes and re-integrates at what the fit asks for, and repeats. A round is kept only for what it realises, not for what the fit predicts, and it can realise a gain in two ways, either of which has to beat its own noise: a lower held-out residual (the standard errors of the two means combined), or a larger share of the validation spots on the lattice beyond chance (the binomial noise of the two shares, z = 3.29). The residual alone misses exactly the errors that cost resolution: it is dominated by the low-resolution reflections, where a distance and the cell scale that compensates it move every spot alike, and its centroids are taken inside a disc centred on the prediction, so they follow the prediction part of the way; the high-resolution validation spots are the first to leave the lattice (measured: 0.6 % of distance read 0.74 of the residual’s noise and 70.4 % against 58.1 % of the validation spots, and cost 0.07 Å of resolution). A move of a trust-region step or more starts the walk outright. A smaller one is first tried as two indexing probes on the validation frames — at the fit’s geometry and at the one in hand, each stopping once the lattice is scored — and pays for a re-integrated round only where the fit’s geometry scores higher. The run keeps the best round it reached.

The space group is determined after pass 2, on the geometry the run refined, and pass 1 does not search at all: a decision taken on the worse of the two passes and then carried forward is a constraint on the better one, and would have to be reconciled with what pass 2 later found. The guard that chooses which pass is written compares each pass’s first merge — \(P1\) on both sides, full resolution range, before the correction surfaces — which both passes produce anyway, so it never compares statistics computed in two different space groups. What it compares there is the signal each pass measured: the count of unique reflections merged at \(I/\sigma \ge 2\), less the count at \(I/\sigma \le -2\). An empty reflection is as likely to land in either tail, so the second count is the merge’s own measure of how much of the first is noise — which is what makes two passes on different lattices comparable: a pass on an \(n\)-fold supercell merges \(n\) times the reflections, most of them empty, and their noise alone once out-counted the crystal’s own lattice. Self-consistency cannot do this job — against an external arbiter the signal count named the more accurate geometry on 23 of 27 arm-dataset pairs where \(R_\mathrm{meas}\), \(CC_{1/2}\) and ISa managed 13, a coin flip — and that merge’s own \(CC_{1/2}\) least of all, being pooled over the whole range, uncut and uncorrected, so the shells with no signal in them dominate it and they are exactly the shells a geometry move disturbs (it reads 0.13 on a crystal whose data merge at 0.995). The refined pass is sent back only where it merges more unique reflections than its cell can hold, where it measured decisively less signal (10 %, and only where it covers no more of reciprocal space either — a wider integration disk pulls weak reflections in and dilutes the strong fraction without measuring less; between two lattices the reflection count of the smaller is scaled by the integer index of one in the other), or where it lost the axial rows the systematic absences are read off. One index-time veto remains and is keyed to pass 1’s lattice rather than its group: a centred pass-1 lattice against a primitive pass-2 one. A pass sent back is pass 1 as it was judged — its whole indexing result is reused, not indexed again de novo at the header geometry, which is a hypothesis nobody had judged.

Only pass 2 is written, as the canonical <prefix>_* output. Pass 1’s merge exists to give the guard something to judge pass 2 against, so it stops short of the parts of the merge that only fill in a file — the correction surfaces, the twinning and radiation-damage analyses, the R-free flags and the amplitudes — and writes no merged files of its own.

Goniometer rotation scale. A stage that turns further than it was commanded to leaves no trace in the file, because the stored \(\omega\) values are the commanded ones; the excess then presents as the crystal drifting, in this program and in others. The excitation residual already measures it without a new degree of freedom: it rotates by \(-\phi\,\mathbf{u}\) with \(\mathbf{u}\) an unnormalised 3-vector, so \(|\mathbf{u}|\) is the factor \(k\) by which the stage actually turned, and normalising the axis throws it away. Pass 1 fits \(k\) as a single parameter on its rocking events, with the crystal and the axis direction held at their committed values and the angle measured from the centre of the sweep. That fit under-reads a real error: it only sees the frames the stored angles still track, and a rate error is exactly what stops them tracking the rest. So it is not acted on directly. Between the passes, at the detector geometry pass 2 runs at, the lattice is indexed under the stored angles and under the fitted \(k\), and each is scored on the validation frames spread over the whole sweep, as the share of their spots it puts on the lattice beyond what it puts there at a wrong spindle angle. The fitted \(k\) is adopted only where it scores higher by more than the binomial noise of the two scores (z = 3.29); the run then integrates and post-refines at it, fits \(k\) again on top of it, and repeats until the next \(k\) no longer scores better - the fixed point of the fit. Otherwise the stored angles stand. The adopted \(k\) drives every later pass (prediction, integration, scaling and the reported oscillation) and is reported as GONIOMETER_ROTATION_SCALE, with GONIOMETER_ROTATION_SCALE_SUSPECT= TRUE. --rotation-scale <k> asserts a calibration and skips all of this.

7.6 Detector geometry from powder rings

Everything above fits the geometry to Bragg data, where the beam centre is the weakest parameter: it is gauge-coupled to the crystal orientation, which is why §7.5 bounds it to within 15 px of a centre something already believes rather than letting the spots place it freely. A powder ring has no orientation to be coupled to. Where it falls on the detector depends on the geometry and on nothing else, which makes a calibrant — LaB₆, silver behenate, CeO₂, silicon — or even ice an independent constraint on exactly the quantity Bragg data cannot pin.

The ring positions are matched to the observed rings and the geometry is refined (Ceres, five parameters: beam centre, distance, and the two detector tilts) so that the \(|s|\) predicted at each observed ring point matches the ring it belongs to. The two tilts can be held fixed (rugnux --no-refine-tilt, the viewer’s Refine detector tilt tick box), leaving a three-parameter fit: a tilt a downstream program cannot express is better left out of the fit than refined and then dropped, since the centre and the distance of a tilted fit have already absorbed it.

Calibrants. LaB₆, silver behenate, CeO₂ and silicon are held as unit cells and their rings enumerated from them. Ice is held as the hexagonal-ice ring positions of §3.3 instead — measured to 1.522 Å, calculated below it — because hexagonal ice is \(P6_3/mmc\) with oxygen on \(4f\) and enumerating \(hkl\) from its cell would emit rings the oxygen sublattice extinguishes. A calibrant is therefore a list of ring \(q\) values throughout, not a cell.

What a ring can and cannot determine. A ring is a conic centred on the beam, so a wrong centre makes its apparent radius oscillate once per turn, \(r(\phi)=R+\delta_x\cos\phi+\delta_y\sin\phi\), with the same amplitude on every ring. A detector tilt \(\beta\) produces a \(\cos\phi\) term too — not the \(\cos2\phi\) one might expect — but one that grows as the ring’s radius squared, \(r(\phi)=R+(R^2/F)(\beta_x\cos\phi+\beta_y\sin\phi)\); the true \(\cos2\phi\) term is \(O(R^3\beta^2/F^2)\), hundredths of a pixel. The two are therefore separated by how the amplitude scales with radius, which needs at least two rings — on a single ring they are exactly degenerate. None of this uses the calibrant’s \(d\)-spacings, so the centre is determined without assuming anything about the standard.

The distance is different: it follows from \(r=F\tan2\theta\) with \(\sin\theta=\lambda/2d\), so a fractional error in the lattice constant passes straight into it, and the \(\lambda\)–\(F\) pair is separated only by the curvature of \(\tan(2\arcsin(\lambda/2d))\) across the rings — \(\partial\ln r/\partial\ln F=1\) at every ring against \(\partial\ln r/\partial\ln\lambda=4\tan\theta/\sin4\theta\), which runs from about 1.05 at low angle to 1.43 at high. That lever collapses as the detector moves back and the rings crowd into small \(2\theta\), so distance is a short-distance measurement and the wavelength is better calibrated by other means.

Reading the rings. The ring points come from one of two measurements, both accumulated over every processed image rather than one. The default reads the azimuthally-binned profile (§2) summed over the run: for each ring and each azimuthal sector, the radial peak is fitted against a locally interpolated background and the measured \((q,\phi)\) mapped back through the current geometry to the pixel it came from. The alternative pools the spot lists, which samples each arc wherever the spot finder’s threshold happens to bite. The accumulated profile is the same size however many images went into it; the pooled spot list is capped, each image contributing an equal share.

A plain radial profile — one azimuthal sector — has averaged the ring over every direction and carries no centre at all, so the profile route requires at least four sectors and uses 32 by default. Sixteen to thirty-two are enough; beyond that the limit is the ring’s own texture, not counting statistics.

The extraction window around a ring is capped at half the gap to its neighbour, because the background under a peak is taken from the ends of that window: hexagonal ice has a triplet of rings (1.947, 1.916 and 1.882 Å) whose neighbours sit only 0.05–0.06 Å⁻¹ apart in \(q = 2\pi/d\), which a fixed window merges into a single peak. Where only one ring is in reach the two tilts are held at their input values rather than fitted, since on a single ring they are degenerate with the centre (above) and the fit would otherwise trade the centre away for them.

\ No newline at end of file diff --git a/CPU_DATA_ANALYSIS_INTEGRATION.html b/CPU_DATA_ANALYSIS_INTEGRATION.html new file mode 100644 index 000000000..53c181e8e --- /dev/null +++ b/CPU_DATA_ANALYSIS_INTEGRATION.html @@ -0,0 +1 @@ + Data analysis: integration, scaling and merging (§8–§12) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Data analysis: integration, scaling and merging (§8–§12)

Part of the CPU/GPU data-analysis reference; the section numbers are continuous across its four parts.

8. Reflection prediction

Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model.

8.1 Enumerating reciprocal lattice points

For a maximum resolution \(d_\mathrm{min}\), accept \((h,k,l)\) such that: \( \lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. \)

8.2 Still prediction (excitation-error cutoff)

For still images, the diffracting condition is approximated by an excitation-error cutoff: \( \left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. \) Accepted reflections are projected to the detector by intersecting the diffracted direction \(\mathbf{S}=\mathbf{S}_0+\mathbf{p}\) with the detector plane, using the current geometry.

When the beam has a finite energy bandwidth, this window is broadened radially per reflection: the cutoff is combined in quadrature with a bandwidth smear, \(\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}\), where \(\sigma_\mathrm{bw}\propto|p_z|\) (the reciprocal-space depth along the beam, growing as \(\sim 1/d^2\)). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same \(\sigma_\mathrm{bw}\) is deconvolved from the measured profile radius (§11.1), so it is not double-counted.

8.3 Rotation prediction (Laue equation + partiality model)

For rotation/oscillation datasets, Jungfraujoch solves for rotation angles \(\phi\) where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define:

  • rotation axis unit vector \(\mathbf{m}_2\),

  • \(\mathbf{S}_0\) incident vector,

  • \(\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)\).

A key quantity is: \( \zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad \mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, \) which also appears in XDS as the Lorentz component linked to the rotation axis.

A Gaussian mosaicity model yields a partiality fraction over an oscillation width \(\Delta\phi\):

\( P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], \)

with mosaicity \(\sigma_M\) in radians.

Reflections are predicted if they meet minimum \(\zeta\) and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area.

8.4 Systematic absences (centering)

Systematic absences are applied at the centering level (prior to full space-group symmetry) when the space group is supplied by the user. With no user-fixed space group, prediction runs in \(P\) regardless of the centering the lattice search inferred: the centering-absent reflections are integrated so that the space-group search (§13) can confirm or disprove the centering from the measured intensities, and so that a missed superstructure shows up. For each centering symbol:

  • \(I\): absent if \(h+k+l\) odd,

  • \(A\): absent if \(k+l\) odd,

  • \(B\): absent if \(h+l\) odd,

  • \(C\): absent if \(h+k\) odd,

  • \(F\): absent if any of \(h+k, h+l, k+l\) is odd,

  • \(R\): absent if \((-h+k+l)\bmod 3 \ne 0\),

  • \(P\): no centering absences.

9. 2D Bragg integration (profile fitting over a three-ring ROI)

Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The default extraction is profile fitting (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection \((I,\sigma,\text{partiality},d)\), so scaling, the rotation combine (§10.6) and merging consume either unchanged.

9.1 Regions of interest

For each predicted reflection at \((x_p,y_p)\), define three radii:

  • \(r_1\): inner signal radius,

  • \(r_2\): inner background radius,

  • \(r_3\): outer background radius.

The defaults are \(4,6,13\) px for rotation data and \(6,8,14\) px for stills, which have a sparser pattern and can afford the wider ring. --integration-radius sets them by hand; on rotation data \(r_1\) is otherwise measured from the crystal’s own spots (§9.5).

Pixels are classified by their squared distance \(r^2=(x-x_p)^2+(y-y_p)^2\):

  • signal region: \(r^2 < r_1^2\),

  • background annulus: \(r_2^2 \le r^2 < r_3^2\).

Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk (\(r<r_2\)) of any other predicted reflection that puts at least 5 % of its flux on this frame (partiality \(\ge 0.05\)) are removed from this reflection’s background annulus, so a neighbouring spot cannot leak into the background estimate. Prediction reaches \(\pm4\sigma\) of the rocking curve, so on a finely sliced frame most predictions are the tails of reflections recorded on the frames either side; left in, they would fill every annulus of a dense pattern while the frame shows no spot there. A reflection whose annulus keeps five or fewer clean pixels is dropped. (Both the annulus and that exclusion become ellipses when the option below is used; with it off, which is the default, they are the circles just described.)

Radially elongated background ring (opt-in, --integration-stencil <k>, default 0). The three radii above are one triple for the whole run, identical for every reflection at every resolution. A reflection is not round, though: a finite bandwidth streaks it radially by \(\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}\), with \(R_\mathrm{px}\) the distance from the beam centre — the same physical smear as §8.2’s and §11.1’s \(\sigma_\mathrm{bw}\), expressed here in detector pixels where those sections use reciprocal units; the two forms are never mixed in one formula. Throughout, \(\text{bandwidth}\) is the rms relative energy spread: the user-facing --bandwidth takes a FWHM (a DMM’s usual specification) and it is divided by 2.355 on input. On a radially smeared spot the fixed \(6\ldots13\) px ring therefore sits only \(\approx1.3\)–\(2.2\) radial \(\sigma\) from the centre — on the reflection’s own tails, which it then measures as background.

With \(k>0\) the background ring becomes an ellipse, elongated along the beam→reflection direction by \(k\sigma_\mathrm{bw}\). The radial semi-axes become \(r_2+k\sigma_\mathrm{bw}\) and \(r_3+k\sigma_\mathrm{bw}\); the tangential half-widths stay \(r_2\) and \(r_3\); and the growth is capped at \(2r_3\), which bounds what a mis-declared bandwidth can do to the bounding box. Pixels are then classified as

  • signal region: \(r^2 < r_1^2\) — a circle, unchanged,

  • background ring: \(r^2-q_\mathrm{in}\rho^2 \ge r_2^2\) and \(r^2-q_\mathrm{out}\rho^2 < r_3^2\),

where \(\rho\) is the pixel’s radial offset (its projection on the beam→reflection direction), \(g=\min(k\sigma_\mathrm{bw},\,2r_3)\) is the capped growth, and \(q=1-\big(r/(r+g)\big)^2\) for the boundary concerned. Written this way \(k=0\) gives \(q=0\) and both tests collapse onto \(r^2\) exactly in floating point, so the default classifies every pixel exactly as the circular stencil did. The neighbour exclusion above follows: each neighbour’s inner ellipse, taken in that neighbour’s own radial frame, is what is masked out of this reflection’s ring.

The width is the bandwidth streak alone, and deliberately not the profile’s full radial variance of §9.3, which also carries the sensor parallax and weak-spot capture terms. Those two are the only terms there are on a monochromatic beam, and widening the ring by them was measured on the rotation battery: it neither helped the crystals with clean high-resolution shells nor left the weak ones alone. The bandwidth streak, by contrast, is a measured elongation of the recorded spot — principal axis along the radius to within a couple of degrees, and azimuth-independent. Keeping only it also makes the option exactly inert on a monochromatic beam, where \(\sigma_\mathrm{bw}\) is zero.

Growing the ring also grows the neighbour exclusion, so on a crowded pattern fewer background pixels survive; a reflection left with too few is rejected outright. On the data this was measured on the loss is under 0.1% of reflections, but it is not structurally zero.

Only the ring moves. The signal disk \(r_1\) stays circular, deliberately: it sets \(n_S\), it sets \(\mathrm{var}(\hat b)\), it is the domain the profile width is learned over (§9.3), and with --integrator boxsum it drives the all-or-nothing “every signal pixel valid” acceptance gate (§9.2), so growing it would reject any box sum carrying a single bad pixel anywhere along a long streak. In the default gaussian mode \(r_1\) does not set the intensity at all — the fit grid, \(\lceil r_2\rceil\), does.

What a circular \(r_1\) loses is flux, and that loss is not a function of resolution alone: measured per reflection, it carries a directional component worth several Ų with a definite principal axis, on top of the isotropic part. Nor is there anything in the merge to absorb it. There is no per-shell scale, and there cannot usefully be one: every scale in §10 is fitted against a reference built from a reflection’s own symmetry equivalents, and equivalents share \(s^2\) exactly, so any function of \(s^2\) lies in the exact null space of the whole scaling model — a per-shell parameter would have zero residual to fit against. (XDS and DIALS have the same null space, for the same reason.) The isotropic part of the loss is instead degenerate with the overall Wilson \(B\) and is silently reported as part of it, so the reported WILSON_B / _reflns.B_iso_Wilson_estimate carries an \(r_1\)-dependent contribution: measured across a constant-ring-area radius sweep it falls monotonically as the disk grows, by 0.5 Ų on sharp strong data and by up to ~10 Ų on weak wide-spot data. What this costs the data is much less than what it costs the flux, because most of the loss is matched by a proportional \(\sigma\): it moves no CC\(_{1/2}\) and no \(R_\text{meas}\), and — to within a few hundredths of an ångström — no resolution cut.

Measured spot footprint (automatic). The radii above are chosen from spots at 5 Å, which at high X-ray energy sit close to the beam. Away from it a spot can grow several times wider — radially from the sensor’s parallax and the obliquity of the incidence, tangentially from the crystal’s azimuthal spread, which rotates the diffracted beam about the incident one and smears the spot along its ring. On small-molecule data at 20–25 keV the standard deviation grows from ~1 px near the beam to ~5 px at the detector edge: the \(r_1 = 4\) disk holds a quarter of the flux there, the \(6\ldots13\) px ring a third of it, and the profile widths learned inside \(r_1\) (§9.3) saturate near \(r_1^2/4\). So the pre-scan measures every spot it finds with a window that follows the spot — three of its own standard deviations, iterated and re-centred — separately along and across the radius, and tabulates the median widths \(\sigma_\rho,\sigma_\tau\) against the distance from the beam. Wherever \(3\max(\sigma_\rho,\sigma_\tau)>r_1\) the integrator then (i) starts the background ring at \(4\sigma\) along each axis, (ii) sums the reflection over the \(r_1\) disk and the \(4\sigma\) footprint ellipse, so the summation — the profile fit’s seed and its fallback — holds the spot rather than its core, and (iii) builds the per-reflection Gaussian at the measured widths on a grid grown to hold them. Where every spot fits the disk nothing is installed and the integration is unchanged bit for bit, which is the case for compact protein spots; like the measured radius, the footprint applies to the canonical pass and not to the geometry pre-pass, and a canonical pass whose wider rings the neighbours starve falls back to the settings without it. The reach is \(4\sigma\) rather than the \(3\sigma\) that decides whether a spot outgrew the disk because wide spots are not Gaussian: mosaic streaks and diffuse halos carry flux past \(3\sigma\), which a ring starting there reads as background. Judged by refining the published structures with SHELXL, it removes the intensity loss that grew with resolution on the small-molecule sets (rugnux/model intensity in the outermost shell 0.81–0.91 → 0.98–1.02).

Split reflections (automatic). A crystal made of slightly misaligned domains — the ferroelastic domains a crystal forms below a phase transition, or a cracked or split crystal — records each reflection as two or more compact spots on either side of the position the averaged lattice predicts, moving apart with resolution. The widths above are measured about each spot and so see compact spots; the \(r_1\) disk then holds the gap between them and the background ring lands on them, and the loss grows to almost everything at the detector edge. Once the geometry pre-pass has a lattice, every spot it indexes is compared with the predicted position of its own reflection on the same frame, and the mean square of that offset, along and across the radius and in the same distance bins, is added to the pre-scan widths. The table then describes the reflection rather than the spot, and the canonical pass integrates with it exactly as above. Where spots sit on their predictions it moves the widths by the prediction error alone (0.2–1 px on protein data, where no reflection then outgrows \(r_1\)); on an inorganic crystal measured below its ferroelectric transition, where every reflection off one zone is a doublet, the offsets reach 8–16 px at the edge, and SHELXL refinement of the published structure goes from \(R_1 = 0.27\) with a spurious extinction parameter to \(R_1 = 0.05\).

9.2 Box summation (seed and fallback)

Let:

  • \(S = \sum I(x,y)\) over signal pixels,

  • \(n_S\) = number of valid signal pixels,

  • \(B = \sum I(x,y)\) over background pixels,

  • \(n_B\) = number of valid background pixels.

Background per pixel and integrated intensity: \( \hat{b} = \frac{B}{n_B},\qquad \hat{I} = S - n_S \hat{b}, \) with a Poisson-like uncertainty \(\sigma(\hat{I})=\max\!\big(1,\ r_\sigma\hat{I},\ \sqrt{S + n_S^2\,\mathrm{var}(\hat{b})}\big)\), i.e. \(\sqrt{S}\) floored both at 1 count (pixel values are photon counts) and at a small fraction \(r_\sigma\) of the intensity. The second term under the root is the uncertainty of the background estimate itself: \(\hat b\) is measured from a finite number of ring pixels, \(\mathrm{var}(\hat b)=\hat b/n_B\), and it is subtracted \(n_S\) times over, so it enters squared. Omitting it understates the variance by \(1+n_S/n_B\) — 1.11 with the shipped circular stencil (\(n_S = 45\), \(n_B = 408\)) — and so understates \(\sigma\) by up to \(\sqrt{1+n_S/n_B} \approx 1.05\), a bound attained on background-limited (weak) reflections and falling towards 1 on strong ones, where \(S\) dominates; with an elongated ring \(n_B\) grows with resolution, so the factor is no longer one number for a run. The same term is carried into the profile fit (§9.3), where it adds \(\big(\sum P/v \,\big/ \sum P^2/v\big)^2\,\mathrm{var}(\hat b)\) — the square of \(\partial I/\partial\hat b\) for that fit; \(n_B\) is the count of pixels behind the final background value, so a clip or trim that discards ring pixels raises it. A box sum is accepted as “observed” only if all signal pixels were valid and \(n_B\) exceeds a minimum — it measures what is in the disk with no model of what should be there, so it cannot renormalise a disk it has lost pixels out of. The profile modes can, and do (§9.3). This box sum is the classical estimator; it is used directly with --integrator boxsum, and otherwise seeds the profile fit below, where \(S\) and \(n_S\) then count only the pixels that were actually read.

High-side clipped background (default on). Because \(\hat{I}=S-n_S\hat{b}\) is a small difference of large numbers for weak reflections, a per-pixel background bias \(\delta\hat{b}\) becomes a fractional intensity bias \(\approx n_S\,\delta\hat{b}/\hat{I}\) that grows as \(\hat{I}\) shrinks — worst at the resolution edge. A plain ring mean reads high there, because neighbour-spot wings that survive the signal-disk mask, tails and zingers are one-sided (positive) contaminants. The ring mean is therefore made robust: pixels above \(\hat{b}+n\sqrt{\hat{b}}\) are rejected and the mean recomputed, with \(n=4\) (--background-clip; \(n=0\) disables), whatever the bandwidth. A clean Poisson ring is essentially unchanged by the cut (measured false-rejection rate 0.04–0.39 % at \(4\sigma\)), while a 40-pixel neighbour core at \(+100\) counts shifts the estimate by \(+0.009\) ct/px.

The clip cuts only the high tail, which matters: the symmetric trimmed mean it replaced (drop the lowest and highest fraction \(f\) of ring pixels, \(f=0.10\); still reachable with --background-trim, which switches the clip off) is not a consistent estimator of the mean of a right-skewed Poisson sample. It sits \(\approx0.1\) ct/px below the true mean at every level, and with \(n_S = 45\) signal pixels in the \(r_1\) disk that under-estimate adds \(\approx4.5\) counts to every partial — negligible at low resolution, but a large fraction of a partial in the outermost shell. The trim also collapses once contamination exceeds \(\approx10\,\%\) of the ring, where the clip does not. Note that removing a positive background bias lowers \(\langle I/\sigma\rangle\) and raises edge \(R_\text{meas}\), because both are inflated by information-free counts — so neither may be read as evidence against the change.

Both estimators are computed in the shared background pass, but only the trim reaches plain box summation: the high-side clip is skipped for --integrator boxsum, which therefore uses the plain ring mean unless --background-trim is given.

Radial background correction (opt-in). A ring mean estimates the background under the signal disk correctly only if the background is flat there. The signal disk and the ring are concentric, so for a background that is linear in position \(\langle B\rangle_\mathrm{ring}=\langle B\rangle_\mathrm{disk}\) identically — a plane or gradient fit buys exactly nothing. The leading error is the curvature of the radial background, which is negligible on a smooth background but reaches tens of counts on a single reflection sitting on a sharp powder ring. That error is a kernel over radial offset, \( \delta \hat b \;=\; \textstyle\sum_k \kappa_k\, \bar B(r_0+k), \) with \(\kappa\) the annulus-minus-disk histogram of the stencil over radial offset, averaged over azimuth, and \(\bar B(r)\) the image’s own radial background curve. With the fixed circular stencil (\(k=0\), §9.1) that single kernel serves every reflection. An elongated ring does not: its radial-offset histogram depends on how far that particular reflection’s ring was grown, so \(\kappa\) becomes a small table of kernels, indexed by the growth rounded to whole pixels. The azimuthal average survives the change unaltered, because the stencil is rebuilt in the reflection’s own frame at each azimuth and so stays radially aligned: what is averaged over is the sub-pixel phase of the detector grid against the radius, which is what genuinely differs between reflections. Applying it costs one short dot product per reflection and no extra pixel reads; correcting the background scalar means the box sum, the profile fit and the variance all pick it up. The curve is accumulated from the same annulus pixels the background pass already reads (a pixel’s radius is the reflection’s radius plus the pixel’s projection on the beam→reflection direction, so no per-pixel square root is needed) and specifically from the clipped pixels, or it would carry neighbour tails and zingers — which is why the correction is inert under --integrator boxsum, that path having no clip pass.

The model is a function of radius alone, so it is applied only where that is true of the background. --background-radial takes on, off or auto. In Rugnux it is auto by default (the broker keeps it off); under auto each image’s peak-excluded ice score (§3.3) is taken after spot detection and before integration, and the correction is applied to that image when the score reaches the same --ice-min-score gate. Smooth powder ice is a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes 43 % of the bands’ excess amplitude, and the improvement is 7× larger inside the bands than outside, which is its stated mechanism; on a crystal whose ice is textured the same correction increased the excess amplitude by half; on a clean crystal it is inert to four decimal places. Auto engages only where a peak-excluded score exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off.

9.3 Profile-fitted extraction (default)

A fixed signal disk captures a width-dependent fraction of each spot, which puts a multiplicative floor on the per-observation precision of strong reflections and weights weak reflections poorly. Profile fitting removes this by extracting each intensity against a fitted spot shape, without needing reference intensities. Per frame:

  1. Seed. Box-sum every reflection (§9.2) to get a rough intensity and observed centroid, and select strong spots (significance \(\ge 5\)).

  2. Build the profile. For gaussian (the default) the width is taken per resolution shell from the measured second moments of the strong spots (shell-dependent because spot size grows with resolution). The moments are anisotropic: each strong spot’s pixels are rotated into its own radial/tangential frame before being accumulated, giving \(\sigma^2_r\) and \(\sigma^2_t\) separately. Stacking the spots in the detector frame instead — they sit at every azimuth — averages the two directions away, leaving only \(\sigma_r^2+\sigma_t^2\), so radial smearing is read back as a wider tangential spot. For empirical the profile is instead the averaged, background-subtracted pixel grid of the shell’s strong spots, accumulated in the detector frame on their rounded predicted positions. For gaussian only, the profile is then rebuilt for each reflection, centred on its sub-pixel predicted position (the noise-free geometric centre, not the observed centroid) and, where needed, elongated only along the radial direction (away from the beam centre) — because two effects stretch a spot radially but not tangentially:

    • a finite energy bandwidth smears each spot by \(\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}\) (\(R_\mathrm{px}\) = distance from the beam centre, large at high resolution), and

    • sensor parallax — the depth over which a photon converts in a thick Si/CdTe sensor — adds a term \(\propto\tan^2(2\theta)\) (material- and energy-dependent), plus a small fixed weak-spot capture term.

    The two enter as a floor on the measured radial excess: \(\sigma^2_\mathrm{radial}=\sigma^2_t+\max\!\left(\sigma^2_r-\sigma^2_t,\ \sigma_\mathrm{bw}^2+c_\mathrm{par}\tan^2(2\theta)\right)\), tangential unchanged at \(\sigma^2_t\). The measured excess is what the signal disk can resolve; the analytic term takes over for a streak too long to be measured there. The Gaussian is built on a grid grown to hold the streak — capturing it without the tangential background an isotropic widening would add. The empirical profile keeps the fixed per-shell grid and gets none of this.

  3. Fit (Kabsch). With profile \(P\), background \(B\) and the shell variance model, the intensity and its uncertainty are \( I = \frac{\sum P\,(c-B)/v}{\sum P^2/v},\qquad \sigma = \sqrt{\frac{1}{\sum P^2/v}},\qquad v = \max\!\left(B + I\,P,\ \tfrac{1}{2}B\right), \) where \(c\) is the pixel value and the de-biased variance \(v\) (background plus model signal, rather than the down-fluctuating observed count) is iterated (a few passes). The plug-in \(I\) enters as it is: half-wave rectifying it, \(v=B+\max(I,0)P\), lets \(v\) — and with it the reported \(1/\sum P^2/v\) — respond only to upward fluctuations of a noisy estimate, which adds \(\approx0.4\,\sigma\sum P^3/(\sum P^2)^2\) to every \(\sigma\) whatever the count rate. That offset is invisible on strong reflections and a large fractional inflation on weak ones; the \(\tfrac12 B\) clamp keeps \(v\) positive without reintroducing it. As a guard, if the profile intensity runs away from the box-sum seed (by more than ~10 box-sum \(\sigma\)) it falls back to the seed, and the background term is floored at \(0.01\) ct/px — enough to keep \(P^2/v\) finite when the ring mean reads exactly zero, which a ring of \(n_B\) pixels cannot distinguish from any background below \(\approx1/n_B\). The rotation/excitation partiality is carried exactly as in the box-sum path.

Pixels the fit cannot use (MINPK). A profile fit is the amplitude of a normalised profile, so a pixel left out of the sum renormalises the estimator by construction: it costs information — \(\sum P^2/v\) shrinks and \(\sigma\) grows — but biases nothing. That is what keeps a reflection whose signal disk is cut by a mask, an untrusted region, a detector gap or an overload: those pixels are simply not read, and the fit is taken over the rest, exactly as the shared pixels of a crowded reflection are (--overlap exclude). The reflection is kept only while enough of the expected profile survives — at least --overlap-minpk of the profile mass that falls on the detector at all, default 0.75, which is XDS’s MINPK and dials’ valid_foreground_threshold. The complete reflections alone teach the profile, its resolution shells and their widths. --integrator boxsum has no profile to renormalise with and keeps the all-or-nothing rule of §9.2.

“Biases nothing” holds only while the profile model is exact. Lose the peak and the amplitude is set by the wings alone, so the result stops being a measurement of the reflection and becomes a measurement of how well the fitted shape describes it. The worst case is a pixel invalidated by the flux it saw — a detector’s per-frame overload marker: that pixel goes missing because the reflection was bright, so the loss concentrates on the strong low-resolution reflections that are the largest terms of \(R_\mathrm{meas}\), where the fit reads \(-50\%\) against the symmetry mates. MINPK cannot catch it, because it cuts on profile mass and the peak of a broad spot is a few percent of the mass. So a second condition applies alongside it, on any unreadable pixel whatever made it unreadable: no unreadable pixel may carry more than 0.9 of the profile’s own peak value. As a fraction of the peak rather than a radius in pixels, that scales with the spot — for a Gaussian it is a cut at \(\sqrt{-2\ln f}\,\sigma = 0.46\sigma\), the peak pixel alone where \(\sigma\) is 0.8 px and the crest of the ridge where the profile is a bandwidth streak — and it costs well under 0.1 % of the recovered observations.

The integrator is selected by --integrator boxsum|gaussian|empirical (default gaussian).

9.4 The prescaling correction

The deterministic per-reflection corrections are carried as three multiplicative factors. prescaling_corr holds the reciprocal Lorentz factor (rotation only — a still’s Lorentz factor is one) times the reciprocal polarization factor from the geometry-based term (§2.2), and nothing else: it is Lorentz x polarization, which is what LP means in every format the field reads. Beside it sit the two terms that describe what happened to the photon between leaving the sample and being counted, and that are kept apart from LP because detector response and beam/crystal geometry are different things: qe_corr, the sensor’s angle-dependent efficiency, and flight_corr, the medium in the flight path (§9.7). The total deterministic correction on a reflection is the product prescaling_corr * qe_corr * flight_corr, and every site that corrects an intensity — the integrator, the scaling fits, the merge ingest, the anisotropy analysis and the unmerged export — multiplies all three. None of them is a scale: the fitted per-image scale and the partiality are separate. The three reach the unmerged MTZ as its LP, QE and FLIGHT columns unchanged (QE and FLIGHT as divisors normalised to 1 at normal incidence), so raw counts are I / LP * QE * FLIGHT.

One term is deliberately not in it. The per-pixel solid angle, which the azimuthal profile does divide out (§2.2), is correctly absent here: a Bragg integration sums all the photons in a reflection, and how many pixels the detector happens to tile that footprint with does not change the count. Detector obliquity does stretch the footprint, and that enters as the parallax term of the spot-width variance rather than as an intensity scale.

9.5 Choosing the signal radius from the crystal’s own spots (rotation)

The three radii are one triple for the whole run, but on rotation data they are no longer a fixed constant: \(r_1\) is measured from how wide this crystal’s spots actually are (--adaptive-integration-radius, on by default for rotation, off for stills, ignored when --integration-radius is given).

Why \(r_1\) matters even though it does not set the intensity. In the default gaussian mode the intensity is a profile-fit amplitude over the grid \(\lceil r_2\rceil\) (§9.3), so \(r_1\) is not the integration domain. It is the aperture the profile width is learned over, and a second moment taken over a disk of radius \(a\) saturates at \(a^2/4\). At \(r_1 = 4\) the learned \(\sigma\) can therefore never exceed 2 px, and a crystal whose spots are broader than that is fitted with a profile the model cannot represent.

The measurement is independent of the integrator. It is made in the pre-scan, on the frames the beam-stop projection already reads, so it costs no extra frame reads and there is no feedback loop. On the spots the spot finder has already found, a spot is used only if it is clear of the detector edge and of the direct beam, has no neighbouring spot within 28 px, is one of the 40 strongest in its resolution band, sits on a fully readable disk, and reaches a signal-to-noise of 15 with its centroid within 2 px of the found position. For each surviving spot the background-subtracted encircled-flux curve is accumulated in 1-px annuli out to a fixed 14 px aperture and normalised at 8 px — an aperture that owes nothing to \(r_1\), \(r_2\) or \(r_3\).

Pooling. Spots are stratified into five resolution bands (2–3, 3–4.5, 4.5–7, 7–12, 12–30 Å), because a weak crystal’s strongest spots sit at high angle and a strong one’s at low angle. Each band with enough members contributes the radius at which its median curve reaches 0.80 of its normalised flux — \(r_{80}\) — at the band’s median \(d\). Those points are fitted by weighted least squares against \(1/d\) (the mosaic contribution to the detector footprint grows as \(1/d\)) and evaluated at a common 5 Å, then clamped to the range the bands actually measured so the fit never extrapolates.

The radius.

\[r_1 = \mathrm{clamp}\!\left(\mathrm{round}(2\,r_{80}),\ 4,\ 6\right),\qquad r_2 = r_1 + 2,\qquad r_3 = \sqrt{r_2^2 + 133}\]

The factor 2 is not fitted: for a Gaussian \(r_{80} = 1.794\,\sigma\), so \(r_1 = 2r_{80} = 3.59\,\sigma\), where the truncated second moment recovers 0.990 of \(\sigma^2\). The expression for \(r_3\) holds the background-ring area constant at its value for the shipped \(4,6,13\) (\(13^2 - 6^2 = 133\)) — a ring that shrank with the disk is what makes a bare --integration-radius worse than the default it replaces. The floor of 4 is that shipped default; the ceiling of 6 is pattern density, since \(r_2\) also drives the neighbour-ownership radius and the ring’s inner edge. At \(r_1 = 4\) the triple is bit-for-bit the shipped default, so a crystal with ordinary spots is left exactly where it was.

The sample grows until the answer settles. The frames are measured in tiers of stride 8, 4, 2, 1, each tier’s sample strictly containing the previous one, and the pooling is redone after each. Measuring stops when the new \(r_{80}\) is within 0.40 px of what the smaller sample said and is at least 0.25 px clear of both radii at which the rounding in \(r_1\) changes answer. Both conditions are load-bearing: clearance alone lets a small sample settle across a switch, and the step test alone lets it settle on one. Every frame of the sample is still read — the beam-stop mask and the beam centre are unchanged; what the tiers save is the decompression, preprocessing and spot finding the width measurement adds on top of the read.

It applies to the final pass only. A rotation run integrates twice (§7.5), and the widened radius is handed to the canonical second pass, not to the geometry pre-pass. The reason is that post-refinement takes its observed positions from the integrator, and an observed position is a first moment over the signal disk with the background still in it: a flat background adds nothing to the numerator but adds \(n\,b\) to the denominator, so every measured offset is pulled toward its prediction by \(I/(I + n b)\), and \(n\) nearly doubles between \(r_1 = 4\) and \(r_1 = 6\). A wider disk therefore under-corrects the geometry — enough, on a crystal whose metric is half a degree off orthorhombic, to flip the second pass’s de-novo Bravais choice.

The density guard. Widening \(r_1\) pushes \(r_2\), the ring’s inner edge, into the neighbours; a reflection whose ring is left with five or fewer clean pixels has no background and is dropped whole. The integrator counts the rings the neighbouring reflections would starve — every predicted neighbour, the rocking-curve tails the background itself does not exclude included, so the count measures the pattern’s density — apart from the ones the detector itself starves (module gaps, the beam stop, the resolution mask), a floor that reaches a couple of percent on some geometries and does not move with \(r_1\). Where the neighbour-driven count exceeds 1.13 % of the predicted reflections, the pattern is too dense for the widened radius and the final pass is integrated again at the fixed \(4,6,13\), reported as pass 3 of 3 with the reason in PASS_DECISION.

Every integration pass, adaptive or not, now logs the radii it used together with the fraction of predicted reflections that lost their background ring, the fraction of rings the neighbouring predictions crowd, and the profile-fit fallback rate.

9.6 Measuring the bandwidth

A finite energy spread \(\sigma\) (\(\Delta\lambda/\lambda\), rms) smears a reflection along its own radius by \(2\tan\theta\,\sigma\) radians of \(2\theta\) and not at all across it. Most files do not state it — a multilayer monochromator is a beamline option, not a header field — so Rugnux reads it off the spots, on the same isolated strong spots the width of §9.5 is measured on (spot_width::EstimateBandwidth). The width settles on fewer spots than this slope needs, so the pre-scan keeps measuring spot shapes for the bandwidth alone until it holds 1000 of them or its sample runs out. For each spot, with \(u\) along the radius and \(v\) across it, the second moments about its centroid are

\[m_u = \tfrac1{12} + p_u + j_r^2\big(s^2 + 4\tan^2\theta\,\sigma^2\big),\qquad m_v = \tfrac1{12} + p_v + j_t^2 s^2,\]

with \(j_r\), \(j_t\) the exact pixels per radian of \(2\theta\) and of the angle across the scattering plane at that spot, \(s\) everything isotropic in angle (divergence, crystal size, mosaic spread), and \(p_u\), \(p_v\) the sensor parallax, fixed from the sensor’s physics: a photon converting at depth \(z\) (exponential with length \(L\cos\psi\) at angle \(\psi\) to the normal, truncated at the thickness) lands \(z\tan\psi\) along the ray’s in-plane direction. Then

\[y = (m_u - \tfrac1{12} - p_u) - (j_r/j_t)^2\,(m_v - \tfrac1{12} - p_v) = a + \sigma^2\,(2 j_r\tan\theta)^2\]

is a straight line whose slope is the bandwidth. It is fitted over eight equal-count bins of the abscissa, each a 20 %-trimmed mean weighted by its own scatter; the slope’s error is the spread of 200 bootstrap re-draws of the spots, inflated by the reduced \(\chi^2\) of the binned fit where the line fits worse than the scatter says, and the log calls the estimate significant at \(z>3\). It is a lower bound — mosaic spread seen along the radius subtracts — and a spread of cell edges is exactly degenerate with it, so what it measures is the effective radial broadening — which is what its consumers need.

A significant estimate is the run’s bandwidth, unless the file states one or --bandwidth is given (--bandwidth 0 forces a monochromatic beam); anything short of \(z>3\) leaves the beam monochromatic. From there it acts continuously, with no broadband mode: prediction and partiality (§8), the profile’s radial width (§9.3) and the background ring’s elongation (--integration-stencil, §9.1) all scale with it and are exactly what they were at zero bandwidth.

9.7 The flight path

A reflection leaving the sample at incidence angle \(\alpha\) to the detector normal reaches its pixel after \(D/\cos\alpha\) of flight rather than \(D\), so it crosses more of whatever fills the flight path than one arriving head-on and arrives attenuated. This is the same \(\cos\alpha\) geometry as the sensor crossing of §9.4, with the opposite sign: the sensor makes an oblique reflection read high, the medium makes it read low.

\[T(\alpha) = \exp\!\left(-\frac{D}{L\cos\alpha}\right), \qquad \text{correction to } I = \frac{T(0)}{T(\alpha)} = \exp\!\left[\frac{D}{L}\left(\frac{1}{\cos\alpha} - 1\right)\right]\]

with \(L = 1/\mu\) the attenuation length of the medium at the photon energy, from the same NIST tabulation the sensor uses. Nothing here is fitted: \(\mu\) is tabulated, and \(D\) and \(\lambda\) are stated by the file. Normalising at \(\alpha = 0\) divides out \(\exp(-D/L)\), a constant for the dataset that the fitted per-image scale absorbs; what is left is the only part of the flight path that is not degenerate with that scale.

For air, \(L\) falls steeply toward low energy — 8.2 m at 18 keV, 3.0 m at 12.4 keV, 0.089 m at 3.8 keV — and the size of the correction follows it:

photon energy

\(L\) (air)

\(D/L\) at 160 mm

correction at \(\alpha = 30°\)

at \(\alpha = 55°\)

18 keV

8.17 m

0.0196

+0.30 %

+1.5 %

12.4 keV

2.99 m

0.0535

+0.83 %

+4.1 %

8 keV

0.84 m

0.191

+3.0 %

+15 %

3.8 keV

0.089 m

1.80

×1.32

×3.8

Helium attenuates about 1/600 of air at 3.8 keV — two electrons an atom against nitrogen and oxygen, and a seventh of the density — which is exactly why long-wavelength stations use it. It is not vacuum, and is modelled rather than treated as one, though at these distances it is worth well under a per cent. Vacuum leaves every intensity untouched.

What it does to merged data

On an untilted detector \(\alpha\) is the scattering angle, so the correction is a pure function of resolution. It therefore cancels within a resolution shell and cannot move \(R_\mathrm{meas}\) or CC\(_{1/2}\) there; its whole effect on merged data is a shift in the Wilson \(B\). Friedel mates share \(2\theta\) and so receive an identical factor, which also means it cannot act on an anomalous difference at all. Both statements are quantitative predictions with no free parameter, and both are confirmed: the Wilson-\(B\) shift is reproduced to within 8 % on three datasets spanning an order of magnitude in \(D/L\), and where a pooled statistic does move, every resolution shell is unchanged and the pooled shift is reproduced by re-weighting alone.

That is what FLIGHT_PATH_WILSON_B in the results report quotes: the correction’s worth, in the one number it can move.

Why the medium is declared and not detected

No field of the NXmx application definition, and no field of any master file rugnux reads, describes the medium in the flight path — there is no air, helium, vacuum, flight-tube or pressure entry anywhere to detect it from.

Inferring it from the physics was considered and refuted. The natural idea is that air becomes unusable at low energy, so an implausibly low implied air transmission would mean helium; but in this corpus a confirmed helium station sits at 51 % implied transmission and a confirmed air station at 63 %. No criterion separates those two without being a threshold fitted between two points, so there is none.

rugnux therefore assumes air — which is what a beamline has unless it was built not to have one — and --flight-path helium|vacuum declares otherwise. Because that is an assumption made on the user’s behalf, the report states it (FLIGHT_PATH) together with what it was worth (FLIGHT_PATH_WILSON_B), and warns where it is worth enough to matter.

10. Scaling and merging

After per-image integration, Jungfraujoch scales observations and merges them into unique reflections. The design is intentionally compatible with XDS/XSCALE concepts, and handles both still and rotation data.

10.1 Observation model

For an observation \(j\) of a unique reflection \(h\) on image (or image group) \(i\), the predicted measured intensity is modeled as: \( I_{ij} \approx G_i \, L_{ij}\, P_{ij}\, I_h, \) where:

  • \(G_i\) is the image scale factor,

  • \(L_{ij}\) is the whole deterministic per-reflection correction of §9.4 - Lorentz x polarization, the sensor’s efficiency and the flight path together, not the LP term alone. Predictions carry its reciprocal split across three fields, so \(L = 1/(\texttt{prescaling\_corr} \cdot \texttt{qe\_corr} \cdot \texttt{flight\_corr})\) and the correction below is applied as a multiplication by their product; the scaling fits, the merge ingest and the anisotropy analysis all use the same three,

  • \(P_{ij}\) is a partiality term (model-dependent),

  • \(I_h\) is the merged (true) intensity parameter for that unique reflection.

A least-squares objective is minimized: \( \sum_{ij} \left(\frac{I_{ij}^{\mathrm{pred}} - I_{ij}^{\mathrm{obs}}}{\sigma_{ij}}\right)^2 \) solved by robust (Cauchy) weighted least squares, with optional post-fit smoothing of the per-frame scales for rotation series (§10.3).

10.2 Partiality models

The partiality applied is fixed by the data type and scaling stage, not chosen from a user menu:

  1. Rotation partiality (XDS-like; see §8.3), used for the per-frame scaling of rotation partials: \( P_{ij} = \frac{1}{2}\left[ \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) \right]. \) Here \(\Delta\phi_{ij}\) is observation \(j\)’s rocking offset from its exact Bragg angle on image \(i\), and the unsubscripted \(\Delta\phi\) is the oscillation width per frame — two different quantities that share a letter. The mosaicity \(\sigma_{M,i}\) is measured once per image at indexing (MLE, §11.2) and held fixed during scaling — only smoothed in frame order (§10.3), never re-refined (it is degenerate with the scale \(G\); §11.2).

  2. Unity (\(P_{ij}=1\)): used for the scale-on-fulls refit (§10.6), where each observation is already a complete reflection.

  3. Fixed: use the per-reflection partiality carried from prediction. Still/serial images are predicted with \(P=1\), so a single-pass stills scale is effectively unity/fixed — which is exactly what --simple-stills keeps. By default the stills path instead post-refines a physical partiality: a small per-crystal orientation tilt \((\delta\psi_x,\delta\psi_y)\) about the two axes perpendicular to the beam is refined against the running merge, and every reflection’s partiality is then recomputed analytically from the refined lattice through its excitation error \(\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|\) and a Gaussian width \(\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2\) — the reciprocal-lattice point’s own radius (resolution-independent), the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature. The fit typically drives \(\gamma_e\to0\), leaving the resolution-independent \(\gamma_0\) as the effective width. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the spatial pattern of partialities — a degree of freedom the per-image scale \(G\) does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with \(G\). Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with \(G\) profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute \(P\) → re-merge, repeated a few times.

Reflections below a minimum partiality can be rejected from merging to avoid unstable corrections.

10.3 Smoothing of per-frame scales

The per-frame scales \(G_i\) are fit by robust (Cauchy) inverse-variance-weighted ratios; there is no explicit \(G\approx1\) prior. For rotation datasets, optional smoothing enforces the expectation that scale and mosaicity vary slowly across a sweep: after the per-frame fit, \(\log G_i\) (and the mosaicity) are replaced by a centred moving average over a window spanning a configurable rotation range (XDS DELPHI-like; --smooth-g, default 5° for rot3d, off otherwise). It is a post-fit smoothing pass, not a curvature penalty inside the least-squares objective. (The per-frame scale refitted on the combined fulls, §10.6, is smoothed differently: by a penalised smoother whose smoothness is chosen by cross-validation.)

The crystal orientation is smoothed the same way, and for the same reason. Geometry is re-refined independently on every frame against that frame’s spots alone — as few as a dozen on a sparse crystal — so the per-frame orientation carries a real slow drift (crystal slippage, up to ~1.3° across a sweep) on top of fit noise that scales with spots per frame. Before scaling, the per-frame lattices are de-rotated to a common reference, averaged in frame order, rotated back, and every partial’s \(\Delta\phi\) — hence its partiality — is recomputed from the smoothed lattice. The window is chosen per dataset by leave-one-out cross-validation (does a frame’s neighbours predict its orientation?) rather than fixed, because drift and noise both vary by two orders of magnitude between crystals; it is capped, because the per-frame fit also absorbs a real per-frame systematic that smoothing too wide destroys. Only frames that actually indexed take part: a frame that did not carries an all-zero lattice, which is finite and so passes a validity check written as a finite test, and would otherwise be both averaged into its neighbours’ orientation and scored in the cross-validation that picks the window. Refining less is not an alternative: with per-image refinement off the space group is lost on several crystals.

A per-frame scale enters every intensity as \(1/G\), so a frame whose fit is not determined by its data can amplify it without bound — and \(\sigma\) is amplified by the same factor, which makes it invisible to any \(\sigma\)-based outlier test. A fitted \(G\) far below the run’s median is therefore treated as undetermined rather than as a successful fit, both here and in the separate refit on the combined fulls (§10.6). The bound is a ratio to the run’s own median because \(G\) is not gauge-fixed: it and the merged means have an exact global multiplicative degeneracy, so no absolute value is meaningful.

10.4 Merging estimator

After refinement, corrected observations are formed: \( I^{\mathrm{corr}}_{ij} = \frac{I^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}},\qquad \sigma^{\mathrm{corr}}_{ij} = \frac{\sigma^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}}. \)

Unique intensities are merged by inverse-variance weighted mean: \( I_h = \frac{\sum_j w_j I^{\mathrm{corr}}_{ij}}{\sum_j w_j},\qquad w_j = \frac{1}{(\sigma^{\mathrm{corr}}_{ij})^2}. \)

The weights use an expected variance: the Poisson signal part of each \(\sigma^{\mathrm{corr}}_{ij}\) is rebuilt at the reflection’s merged \(\langle I\rangle\) rather than at that observation’s own intensity. Weighting by an observation’s own \(\sigma^2\) biases the inverse-variance mean low below about one photon, because an up-fluctuated observation gets a larger sigma and is then down-weighted too hard. The rotation combine already does this; for stills it is on by default, and --no-expected-variance-merge restores the observed-sigma weighting.

An internal-consistency term can inflate uncertainties when multiple observations are present, in the spirit of XSCALE.

10.5 Merging statistics

The shells are nine bins of equal width in \(1/d^2\), laid between the declared low-resolution limit (--scaling-low-resolution, or the whole sphere where it is switched off) and the highest-resolution reflection the merge actually kept — XDS’s rule and XDS’s count, so at the same resolution limits the two programs’ tables have the same shell boundaries and can be read row for row. --resolution-shells changes the count; the binning rule does not change with it.

Per-shell and overall merging statistics are computed on corrected intensities, including:

  • number of observations and of unique reflections, and multiplicity,

  • mean \(I/\sigma(I)\),

  • \(R_\mathrm{meas}\) (the redundancy-independent Diederichs–Karplus form) from within‑HKL deviations,

  • \(\mathrm{CC}_{1/2}\), correlating two half-sets of equal size: an observation’s half is the parity of its rank among its own reflection’s observations, ordered by a key built from the raw Miller index and the peak frame. Every reflection measured more than once therefore contributes (a hash of the image alone leaves \(2^{1-n}\) of the multiplicity-\(n\) reflections entirely in one half, with no second mean to correlate), and because a rank is a property of the set rather than of the order it is walked in, a CUDA build and a JFJOCH_USE_CUDA=OFF build report the same \(\mathrm{CC}_{1/2}\) and the same \(\mathrm{CC}_\mathrm{anom}\). It is the split cctbx’s compute_cc_one_half — and phenix.merging_statistics through it — uses. The stills path balances its halves sequentially instead, in image order. When a reference dataset is supplied, \(\mathrm{CC}_\mathrm{ref}\) is reported beside it,

  • completeness against the reflections the cell and symmetry can give over that same declared range, so low-resolution terms lost to the beam stop, to a detector mask or to the low-resolution limit itself count as missing instead of leaving the denominator along with the data,

  • the anomalous signal-to-noise \(\mathrm{SigAno}\) and the half-set anomalous correlation \(\mathrm{CC}_\mathrm{anom}\) (below).

The error model is refined as \(\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2\), with \(a\) set by the scatter of weak (counting-limited) reflections and \(b\) the intensity-proportional systematic scatter of the strong ones. On the rotation path, ISa is the asymptotic (\(I\to\infty\)) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, Acta Cryst. D66 (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the \(I/\sigma\) threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as \(1/b\) of the whole-range fit, whose \(b\) is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is report-only — nothing downstream reads it, and the merged \(\sigma\) is not floored at \(b|I|\) (that floor was removed). The per-observation \(\sigma_\mathrm{corr}\) (the merge weights) uses the whole-range \(a,b\). The stills path has no asymptotic estimate and reports \(\mathrm{ISa}=1/b\) directly.

\(a\) and \(b\) are reported in XDS’s convention, which is \(\sigma^2 = a(\sigma_0^2 + b I^2)\) with \(\mathrm{ISa}=1/\sqrt{ab}\), so the printed pair can be read straight against a CORRECT.LP. The internal fit keeps the form above; only the report converts, as \(b_\mathrm{XDS} = b^2/a\). Note that \(a\) is the same in both conventions and that the two ISa expressions are the same number, \(1/\sqrt{a\cdot b^2/a} = 1/b\) — so the rotation log prints two ISa, the whole-range \(1/b\) (XDS’s meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: _reflns.jfjoch_diffrn_ISa is the whole-range value, directly comparable with a CORRECT.LP, and the asymptote is written separately as _reflns.jfjoch_diffrn_ISa_asymptotic, with _reflns.jfjoch_error_model_a and _b alongside so the number can be re-derived. Note that a file written before this change carries the asymptote under the plain ISa name. A third, unrelated \(b\) appears in the space-group search (§13.1); it is fitted with the \(\sigma^2\) coefficient held at 1 and its gate constants are calibrated in that convention.

Anomalous signal-to-noise (SigAno). The strength of the anomalous signal is reported per shell and overall as \(\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle\), where \(\Delta I = I(+)-I(-)\) over acentric reflections measured in both Bijvoet hands and \(\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}\). It is computed from the full-multiplicity inverse-variance \(I(+)/I(-)\) split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise \(\mathrm{SigAno}\) approaches the half-normal value \(\sqrt{2/\pi}\approx0.8\), and it rises above \(1\) once a real anomalous difference is present. A half-set anomalous correlation (\(\mathrm{CC}_\mathrm{anom}\)) is reported beside it: \(\Delta I\) is formed once per half-set and the two are correlated over the acentric pairs where both hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic \(\sigma\) cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps \(\mathrm{CC}_{1/2}\) non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS’s CCanom and phenix.merging_statistics’ cc_anom; XDS’s Anomal Corr is a different quantity and is not comparable with it — on the same observations it reads two to three times higher in the low shells. \(\mathrm{CC}_\mathrm{anom}\) is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the CCanom column of the printed merge-statistics table and as CC_ANOM= in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items _reflns.pdbx_absDiff_over_sigma_anomalous (overall) and _reflns_shell.pdbx_absDiff_over_sigma_anomalous (per shell), and appears as the SigAno column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints - and the results report writes no key, which is not the same statement as a value measured to be zero.

10.6 Rotation datasets: combining partials into fulls (3D integration)

In a rotation scan a reflection is recorded as a series of partials spread across the frames its rocking curve crosses. Merging those partials directly would force the merge error model to absorb the rocking-curve slicing as if it were measurement noise, capping the achievable \(I/\sigma\). For rotation data Jungfraujoch instead combines each reflection’s partials into a single full intensity first, then scales and merges the fulls — a 3D integration over the rocking curve.

The combine groups each reflection’s partials into rocking events (contiguous runs of frames) and reduces each event to one full:

  • De-biased weighted sum. Partials are combined by inverse-variance weighting, where each partial’s variance is its background-noise component plus the model signal shared across the event (Kabsch profile-fit form). Using the shared model signal rather than the individual down-fluctuating intensity stops weak partials from being over-weighted, which would otherwise inflate the merged error model. The weights depend on the full, so the estimate is iterated.

  • Captured fraction. The partiality summed over the event, \(f=\min(1,\sum_j p_j)\), measures how completely the rocking curve was sampled. A full whose curve was captured below a threshold (--min-captured-fraction, default 0.7 for rotation) is dropped — an event seen over only a small fraction of its curve is unreliable however many frames it spans. (The per-partial minimum-partiality cut of §10.2 still applies upstream, in the per-frame scaling.)

  • Per-image rejection (opt-in). A frame whose observations correlate poorly with the merged reference is not measuring the crystal being merged — it may be off-crystal, or on a different crystal where two lattices occupy separate regions of the sample. --min-image-cc drops such frames. It has no default: the per-frame correlation measures data quality as much as frame validity, and its typical level varies widely between datasets, so no single absolute bound is generally valid.

  • Capture-aware uncertainty. A full captured incompletely (\(f<1\)) is extrapolated and biased high. The unobserved fraction is charged as an extra systematic uncertainty, \(\sigma^2 \leftarrow \sigma^2 + \big(c\,(1-f)\,I\big)^2\), so the merge down-weights these extrapolated fulls and the error model treats their scatter as expected. The merge rebuilds every full’s variance at the reflection’s mean intensity (§10.4), and the capture term is rebuilt there too, as \(\big(c\,(1-f)\,\langle I\rangle\big)^2\). It is enabled by default for the rotation path.

  • Overloaded events. An event in which any partial had a saturated pixel in its signal disk — or a pixel unreadable on that frame alone, beyond the run’s pixel mask, which is how a detector that writes its error value for a pixel it could not count reports an overload — is dropped whole, as XDS drops an overloaded reflection. The brightest part of such a rocking curve is exactly what is missing, so neither the sum of the remaining partials nor their extrapolation by the partiality model measures the reflection: on a strongly diffracting small-molecule crystal these were the strongest low-order reflections, and they read 2–3× low. The integration keeps an overloaded partial, unfitted and flagged, only so that the event can be recognised; nothing else reads it. The count is OBSERVATIONS_REJECTED_OVERLOAD= in the report.

The fulls are then re-scaled in the XDS sense — a per-image scale refit directly on the complete reflections under the unity partiality model — and merged (§10.4).

Before the combine a per-frame scale can also be fitted on the partials themselves. That needs a frame to hold many rocking events caught at different points of their curves: within one rocking curve a change of scale and an error of the partiality model are the same thing, and on a finely sliced sparse sweep the fit takes one for the other. The rocking events per frame are the prior (the partials are scaled from 50 events per frame), but the counts of small-molecule sweeps (3–43) and proteins (5–900) overlap. So the first merge of a run that is not a space-group search can be made both ways, with the partials scaled and with the scale taken from the fulls alone, and the prior stands unless its merge has no resolved error model (\(b\) not resolved from zero: the strong equivalents do not agree to within a measurable systematic error) while the other merge has one. Where the prior is to scale the partials and that merge resolves its error model, the other cannot change the choice and is not made. The two ISa values are deliberately not compared beyond that: the partiality-model error a partial scale takes up is shared by symmetry mates measured at the same rocking geometry, so their agreement cannot see it — a small-molecule sweep with six events per frame read ISa 10.5 with its partials scaled against 8.7 from the fulls alone, and refined to \(R_1\) 0.105 against 0.062. The log states the choice and the ISa of every arm that was made. Because every merged observation is now a counting-statistics-limited full rather than a partiality-divided slice, the error model reaches a far higher asymptotic \(I/\sigma\).

How smooth that scale is over the rotation is left to the data rather than to a fixed window. Each round fits every frame on its own fulls against the current reference, giving a scale \(G_f\) and its information \(D_f=\sum w^2c^2\); the scale is then the curve \(x=\log G\) minimising \(\sum_f J_f\,(x_f-y_f)^2+\lambda\sum_f(\Delta^2 x)_f^2\), with \(y_f\) the frame’s own fit in log scale and \(J_f\) its information carried there — a penalised (Whittaker–Eilers) smoother, solved as a five-band linear system. \(\lambda\) is chosen by cross-validation: blocks of frames one rocking curve wide are left out in turn and predicted from the curve through the rest (neighbours closer than a rocking curve share their measurement, since a full sums those frames). A frame of hundreds of fulls is then followed frame by frame, a frame of two or three is carried by its neighbours, a stretch with none is bridged by a straight line, and a scale that falls a hundredfold over a few degrees — an absorbing crystal turning edge-on — is followed where a window would average across it. Once the curve settles, one free fit is shrunk toward it frame by frame by how much of each frame’s deviation its neighbour shares (the lag-1 covariance), which hands back a real per-frame systematic and discards fit noise.

After scale-fulls, five correction surfaces are fitted on the combined fulls (rotation path, on by default; disable all with --no-scaling-corrections), each an alternating multiplicative refinement of the per-full scale against the merged reference:

  • Decay. Radiation damage weakens later frames more at higher resolution — a resolution×time (Debye–Waller) systematic the resolution-flat per-image scale cannot capture. A single global relative-\(B\) rate is fitted, \(\ln(I_\mathrm{ref}/I_\mathrm{obs}) = 2\,(\mathrm{d}B/\mathrm{d}n)\,(n-\bar n)\,s^2\) (frame \(n\), \(s^2 = 1/4d^2\)), and folded into the scale. It engages only when the total relative-\(B\) over the run exceeds a physical floor (2 Ų); below that the decay is negligible and “correcting” it only spreads symmetry equivalents (same \(s^2\), different frames). An optional per-batch relative-\(B\) (--relative-b[=deg], off unless requested; 10°-of-rotation batches by default) extends the single global rate to a smooth \(B(n)\) curve — the same \(s^2\)-weighted decay fit solved independently over short frame batches, curvature-penalized so it cannot over-fit and cross-validated like the surfaces below — for crystals whose decay is non-linear in dose. Its cross-validation splits on ASU-group parity, not the frame parity the surfaces below use: a per-batch parameter owns whole frames and so cannot be scored on a held-out frame, whereas splitting the symmetry equivalents tests whether a batch’s \(B\) generalises to reflections it was not fitted on.

  • Absorption. A smooth multiplicative factor over the diffracted-beam direction expressed in the goniometer (crystal) frame: each full’s predicted detector position gives the lab diffracted direction, de-rotated by the spindle so a fixed crystal-frame direction is sampled at many rotation angles and its grid cell is well-determined. Negligible at hard X-rays / thin crystals; it matters at low photon energy.

  • Modulation (detector-plane flat-field). A smooth multiplicative factor over where each reflection lands on the detector (predicted \(x,y\)): symmetry-equivalents land at different positions as the crystal rotates, over-determining the surface. It absorbs detector-response and geometric systematics that inflate \(R_\mathrm{meas}\).

  • Absorption as spherical harmonics. The same crystal-frame absorption as a smooth function instead of a grid: the logarithm of the factor is a sum of real spherical harmonics of the de-rotated diffracted-beam direction, degrees 1 to 6 (48 terms; the incident-beam path depends on the rotation angle alone and is part of the per-image scale). It is fitted through 32 × 64 equal-solid-angle direction cells, one ridge-regularised least-squares step on the coefficients per round, with a prior of width 0.1/l on each degree-l coefficient. It is a candidate like the others and passes the same held-out test; the grid is then tested on what it left.

  • Time-dependent absorption. The same surface as Absorption, but over (rotation angle × detector position) instead of the crystal-frame direction alone. The two agree while the illuminated volume stays put — the incident path then depends only on the spindle angle, which the per-image scale already takes, and the exit path is fixed in the crystal frame. Once the diffracting volume drifts through the beam the exit path becomes a function of the spindle angle as well, and nothing time-independent describes it. Fitted on 12 rotation bins × a 10×10 equal-occupancy detector grid.

The surfaces overlap, so the order decides what is adopted: modulation first (every frame measures the static detector pattern, so its fine grid is the best determined), then time-dependent absorption, then the spherical-harmonic absorption, then the goniometer-frame grid, whose cells collect directions from the whole sweep and the whole resolution range and which, fitted first, takes up part of what the other two describe.

Each cell’s factor is fitted under a prior pull to 1 (a Gaussian prior of width 0.1 on its logarithm): a cell moves off 1 in proportion to the information its observations carry, so a cell with little signal stays near 1 instead of being fitted to its noise. Negative observations enter the fit as measured. On the detector-plane surfaces (modulation, time-dependent absorption) the component that is a function of resolution alone is projected out within resolution shells, since the symmetry equivalents of a reflection share one resolution and such a factor cannot be determined from them.

Each surface is cross-validated on the half-set agreement it is meant to improve: fitted on even-numbered frames and used to merge the odd ones, fitted on the odd frames and used to merge the even ones, and kept only if the correlation between the two half-set means, taken within resolution shells, rises above the same two halves merged with no surface. Each half is corrected by a surface it did not help to fit, so a surface fitted to noise lowers the correlation, and a correlation within a shell is blind to the resolution-dependent scale the surface cannot determine. The change is averaged over the shells on Fisher’s \(z = \operatorname{atanh}(CC)\), not on \(CC\): the strong shells, where a multiplicative error matters, sit at \(CC_{1/2} \approx 0.999\), where even a large reduction of the error moves \(CC\) in the fourth decimal, and averaged on \(CC\) itself the shells of pure noise beyond the reach of the data decide the sign.

Radiation-damage report (rotation, report-only). Independently of whether any decay correction is applied, Rugnux measures and reports the relative Debye–Waller \(B\) across the sweep: the per-image scale’s correlation to the merge and the per-image mosaicity versus frame (dose), together with a per-batch relative-\(B\) curve whose first→last change is a single headline number (measured before any decay correction, against the least-damaged early wedge). It is written to the log and to the merged mmCIF as a data-quality-vs-dose diagnostic and never alters the merged intensities — distinct from the decay correction above, which does fold into the scale.

Each batch’s \(B\) is fitted on resolution-shell means, not on single observations: \(\ln(I_\mathrm{ref}/I_\mathrm{obs})\) of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs \(I_\mathrm{obs} > 0\), which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely dimmer than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as absent rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report (the results report) to name.

Frame disposition (rotation). After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures ΔCC1/2 — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run) with each reflection’s error variance taken from the observed scatter of its own observations rather than from the error model’s σ’s: a bad stretch claims the same σ’s as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group’s own \((n, \sum I, \sum I^2)\) from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic’s own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, \((1-CC^2)/\sqrt{n_\mathrm{refl}-3}\), and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is both several standard errors below zero (Fisher-transformed) and well below the run’s own per-batch distribution (median − 3 robust σ, measured once before anything is removed), and where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame merged / downgraded / rejected ledger is described in the results report.

10.7 R-free test-set flags

A fraction of the unique reflections (rfree_fraction, default 0.05) is flagged as a free (test) set, written to the output (MTZ FreeR_flag, mmCIF _refln.status = f, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection’s orbit under the lattice holohedry — the point group of the cell’s metric, found as twin laws are (Le Page two-folds within 3° obliquity, taking the lattice of the cell’s own basis vectors), which contains the merging group — together with its Friedel mate. Where the cell does not carry the merging group (a space group forced on a metric without it), the Friedel-merged (Laue) ASU index of the merging group is used instead. That gives four properties:

  • all symmetry- and Friedel-equivalent reflections share one flag — in particular a Bijvoet pair \(I(+)/I(-)\), kept as two separate merged rows in anomalous mode, is never split across the work and free sets (which would bias R-free);

  • twin-law mates share one flag too, since a twin law is a lattice symmetry the crystal lacks. Keyed on the merging group instead, nearly every free reflection’s twin mate lands in the working set, and in twin refinement the free reflection’s calculated intensity then carries the working-set fit. The same set is what phenix.refine generates by default (use_lattice_symmetry). It also makes the free set independent of the space group a file is merged in: the merged file, the P1 cross-check and a re-merge in any subgroup carry one free set (nested, where the small-data floor below lifts the fraction of one of them more than another’s);

  • the free/work decision is a deterministic hash of that key, so the same reflection always lands in the same set — reproducible run-to-run and independent of the order in which observations were merged;

  • the hash depends only on the reflection index, not on this dataset’s resolution range or which reflections it happens to contain, so a uniform draw takes ~rfree_fraction of the distinct reflections free and — crucially — every dataset of one crystal form gets the same free set. That cross-dataset consistency is what a multi-dataset campaign (ensemble refinement, PanDDA) requires; a per-shell stratification tied to each dataset’s own \(d_\mathrm{min}\) would break it.

On small data, where rfree_fraction (default 0.05) would give too few test reflections for a statistically stable R-free (Brünger’s ~500–2000 rule), the fraction is floored so at least ~500 distinct reflections are free — capped at 10 % so a large test set never steals working data. For ordinary data this floor is inactive and the fraction stays flat at rfree_fraction, preserving the cross-dataset-identical property above; it only lifts the fraction on genuinely small datasets, where per-dataset R-free stability outweighs cross-dataset identity (and a shared reference set is the way to keep exact identity there).

When a reference (--reference: an MTZ, or a PDB structure-factor mmCIF, which is converted to one with _refln.status f/o becoming flag 0/1) carries a FreeR_flag column, its test set is imported instead: every merged reflection whose Laue-ASU index matches the reference takes the reference’s flag (reflections absent from the reference keep the hash flag). The match is made in the reference’s frame — on rotation data the merge is first put onto the reference’s axes, choosing among every description of the lattice on them (and every alternative indexing) by the intensity correlation with the reference — and only where the merge then matches it: an intensity CC of at least 0.5 over at least half of the merged reflections in the reference’s resolution range. Below that the reference is not the same crystal form on the same axes, its flags would land on unrelated reflections, and they are not imported (REFERENCE_MISMATCH). This lets a whole fragment-screening campaign inherit one shared free set from the apo/reference dataset. The CCP4/refmac convention (test set = flag 0, including the historical 0–19 form) is assumed, with the complement taken automatically if flag 0 would be the majority (a phenix-style file where 1 marks free).

10.8 French–Wilson amplitudes

The last step of the merge estimates a Bayesian structure-factor amplitude \(|F|\) for each unique reflection from its intensity \(I\) and error \(\sigma\), so the output carries amplitudes alongside intensities (a naïve \(\sqrt{\max(I,0)}\) turns every weak or negative measurement into a biased — or zero — amplitude). With the Wilson prior for the true intensity \(J\ge 0\) at that resolution,

\( P_\mathrm{acentric}(J) \propto e^{-J/\Sigma},\qquad P_\mathrm{centric}(J) \propto J^{-1/2}\,e^{-J/2\Sigma}, \)

and a Gaussian likelihood \(\mathcal{N}(I;J,\sigma^2)\), the posterior mean amplitude and its uncertainty are

\( \langle |F|\rangle = \frac{\int_0^\infty \sqrt{J}\,\mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J}{\int_0^\infty \mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J},\qquad \sigma_F = \sqrt{\langle J\rangle - \langle|F|\rangle^2}. \)

The prior mean is \(\Sigma = \varepsilon\,K_\mathrm{shell}\,a(\mathbf{h})\), where \(\varepsilon\) is the reflection’s epsilon (symmetry-enhancement) multiplicity, \(a(\mathbf{h}) = \exp(-\tfrac12\mathbf{s}^\mathsf{T}B\,\mathbf{s})\) carries the deviatoric anisotropy tensor \(B\) of §13.5 along the reflection’s own direction, and \(K_\mathrm{shell} = \sum I/\varepsilon \,/ \sum a\) over its resolution shell, so the priors of a shell still average to its measured Wilson mean. The amplitudes are first made with \(a = 1\) at the end of the merge and made again once the tensor has been fitted; only \(F\)/\(\sigma_F\) change, never an intensity. With an isotropic prior the weak direction of an anisotropic crystal gets a prior set mostly by the strong direction, which turns its noise into amplitude; for isotropic data the tensor is near zero and the prior reduces to the shell mean, so it is used whenever a tensor was fitted. A shell whose \(K\) is not positive takes that of the nearest lower-resolution shell. As in ctruncate, an intensity more than 3.7σ below zero gets no amplitude (the intensity is kept) and does not enter the shell mean. Strong reflections (\(I>20\sigma\)) short-circuit to \(|F|=\sqrt{I}\), where the French–Wilson bias is below 0.3%; a reflection with an unusable \(I/\sigma\) falls back to \(\sqrt{\max(I,0)}\). The integral is evaluated numerically with a log-shift for stability.

Amplitudes are written as MTZ F/SIGF (and F(+)/F(-)) and mmCIF _refln.F_meas_au/F_meas_sigma_au, alongside the intensity columns; the SHELX .hkl holds intensities only. The same \(|F|\) feed the model-validation step (§14), so the reflection file and the maps use one consistent set of amplitudes.

10.9 Reference data: fixing the space group and resolving the indexing ambiguity

A reference dataset (--reference, MTZ or SF-mmCIF) supplies known intensities for the same crystal form, and is used in two ways.

Fix the space group and cell. Unless overridden on the command line (-S for the space group, -C for the cell), the reference’s space group is adopted and its cell is used as the soft reference cell — indexing may still drift the cell within tolerance, so a small mismatch between reference and data is absorbed rather than rejected. This applies to both stills and rotation data.

Resolve the indexing (merohedral) ambiguity. When the lattice symmetry is higher than the crystal’s Laue symmetry (e.g. \(P3\), \(P4\), \(P6\), \(C2\)), more than one indexing of the same lattice is geometrically valid, and the two solutions produce different merged intensities that a self-consistent scale cannot tell apart — only an external reference can. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation \(\mathrm{CC}_\mathrm{ref}\) of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is metric-preserving — only the \(hkl\) labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined, and the whole merge is then repeated in the chosen indexing. For stills it has to be done per image, at integration time: each crystal is indexed independently, so a run resolves the ambiguity image by image (the image’s partiality/Lorentz-corrected intensities are correlated with the reference under each candidate operator, which is scale-invariant, and the best-correlating one is adopted once and for good) — otherwise the merge would average reflections that are not symmetry mates. In neither workflow is the reference a scale target: both scale against their own data (§10.2), so \(\mathrm{ISa}\) and the merging statistics come from the data alone and no cross-dataset systematic is imported. Because the stills choice is made at integration time, a later re-merge of stored reflections cannot repair a dataset integrated without a reference. Where there is no reference dataset but there is a model (--model), the reference intensities are computed from it instead - \(|F_\mathrm{model}|^2\) from the atomic structure factors with a flat bulk-solvent contribution at the standard constants (\(k_\mathrm{sol}=0.35\), \(B_\mathrm{sol}=46\) Å\(^2\)), which are not fitted because there are no observations yet. Nothing is scaled against them; they serve only to rank the candidate indexings, and the correlation that does the ranking is scale-invariant. This needs the cell and space group up front (-C / -S, as serial indexing wants anyway); on rotation data the same job is done after the merge, in §14.5, where a merge exists to fit the model to properly.

10.10 Ice rings at the scale and merge stages

Where the gate of §3.3 has found ice, reflections falling within \(\pm w\) in \(q = 2\pi/d\) (§1.2) of a hexagonal-ice band (\(w=0.03\) Å\(^{-1}\) offline, about the measured ring half-width) are marked. Marked reflections are excluded where a model is fitted — the per-frame scale \(G\), the per-image correlation, and the \(P1\) merge the space-group search runs on — because ice contamination is a positive bias, not extra scatter, and a least-squares scale absorbs it into \(G\) and into the error-model \(b\), where it damages every other reflection on the same frame. They are also left out of the resolution-cut fit (§13.4), which is the one consumer that reads the merged reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge’s own accumulator (host and device alike) as well as on the stills one. They are otherwise kept in the final merge, which is also what the established scaling programs do by default, so the affected shells keep their completeness.

Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge’s own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged \(\mathrm{CC}_{1/2}\) and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by \(-0.001\pm0.018\,\sigma\) — about 2 % of the site height — while removing 1149 unique reflections whose mean \(I/\sigma\) was 3.62 against the dataset’s own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell.

11. Mosaicity and “profile radius” monitoring

11.1 Profile radius (intrinsic excitation-error width)

The “profile radius” is the intrinsic angular width of a reflection — crystal mosaicity plus beam divergence — estimated from the spread of \(\Delta_\mathrm{Ewald}\) over indexed spots, \( R \approx \sqrt{\tfrac{1}{N}\sum_i \Delta_{\mathrm{Ewald},i}^2}. \) When the beam has a finite energy bandwidth, that bandwidth smears each reflection radially by \(\sigma_\mathrm{bw}\approx \mathrm{bandwidth}\cdot\lambda/2d^2\) (largest at high resolution), which also broadens the measured \(\Delta_\mathrm{Ewald}\) spread. Since prediction re-applies the bandwidth term per reflection (§8.2), this contribution is deconvolved from the estimate — \(R^2 = \langle\Delta_\mathrm{Ewald}^2\rangle - \langle\sigma_\mathrm{bw}^2\rangle\) — so that \(R\) is the intrinsic width and bandwidth is not double-counted. Still predictions use an excitation-error cutoff proportional to \(R\).

11.2 Mosaicity from rotation data

For rotation data the mosaicity \(\sigma_M\) is estimated by maximum likelihood from the rocking offsets \(\tau\) of indexed spots, using the XDS reflection-fraction model \(R(\tau;\sigma_M/\zeta)\) (Kabsch 2010): each spot’s exact Bragg angle is located near its frame, \(\zeta\) (the rotation-axis Lorentz component) is computed, and \(\sigma_M\) is chosen to maximize \(\sum_i \log R(\tau_i;\sigma_M/\zeta_i)\).

The \(\phi\) search window for the Bragg angle is set wider than the oscillation, so that reflections recorded at large rocking offset are included. These tail reflections carry most of the information about the mosaic width; a window limited to the oscillation range would truncate the \(\tau\) distribution and bias \(\sigma_M\) low.

The fit uses only the strongest 250 spots of an image, whatever the indexing spot budget (--max-spots) is. A spot is detected when \(I_\mathrm{full}R(\tau)\) clears the finder threshold, so selecting spots by intensity censors on \(R(\tau)\): a deeper list holds proportionally more large-\(\tau\) partially recorded spots and the fit widens with it. Left uncapped, \(\sigma_M\) therefore tracks the spot budget rather than the crystal — and since an over-wide mosaicity mis-states every partiality, the merge degrades sharply with it.

The estimated mosaicity feeds the rotation prediction (how many frames each reflection spans, §8.3) and the rotation partiality (§10.2). It is held fixed during scaling: in the per-image scale fit the mosaicity is degenerate with the scale \(G\) (both rescale the predicted intensity), so refining it there is unstable. A correct mosaicity matters because it controls both how much of each rocking curve is captured and the partiality used to form fulls (§10.6); too small a value truncates the captured curve and over-peaks the partiality, degrading the combined fulls.

12. Auxiliary statistics: ⟨I/σ(I)⟩ and Wilson plot

12.1 Per-shell ⟨I/σ(I)⟩

For monitoring integration quality, Jungfraujoch reports mean \(\langle I/\sigma(I)\rangle\) in a fixed number of resolution shells. Shelling is performed in \(1/d^2\) space (typical of crystallographic practice).

12.2 Wilson plot (B-factor proxy)

A Wilson-type analysis is computed by binning intensities by resolution and fitting: \( \langle I\rangle \propto \exp\!\left(-\frac{B}{2}\frac{1}{d^2}\right), \) i.e. \( \log \langle I\rangle = \mathrm{const} - \frac{B}{2}\left(\frac{1}{d^2}\right). \) A linear regression of \(\log\langle I\rangle\) vs \(1/d^2\) provides an estimate of \(B\), subject to basic quality checks (e.g. \(R^2\) threshold).

A dataset-wide Wilson \(B\) is also estimated over the merged reflections — restricted to the meaningful resolution range (skipping the low-resolution non-linear region below ~4 Å and shells past the signal limit \(\langle I/\sigma\rangle < 1\), so it is insensitive to how far the merged data extend) — and written to the merged mmCIF as _reflns.B_iso_Wilson_estimate (and reported as WILSON_B=), the analogue of XDS’s Wilson-line \(B\). It is diagnostic only and is not fed back into scaling. It is the XDS convention, not the CCP4/Phenix one, and the two are not comparable. The slope here is fitted to \(\log\langle I\rangle\) directly; TRUNCATE, ctruncate and phenix.xtriage fit \(\log(\langle I\rangle/\Sigma)\), dividing out \(\Sigma=\sum_j f_j^2(s)\), the falloff of the atomic form factors for an assumed composition. Leaving \(\Sigma\) in the slope inflates \(B\) by roughly 2 to 8 Ų (measured across in-house merges; the arithmetic gives +6.9 Ų over 4.0-1.5 Å for a generic protein), and the fitted range accounts for more still: against ctruncate and phenix.xtriage on the same merged files this number runs 10 to 36 Ų high, in the same direction every time - though those two disagree with each other by 7 to 16 Ų, so there is a band rather than a right answer. Read it as a relative quantity, comparable between Rugnux runs and against XDS, and do not compare it with a value quoted from a CCP4 or Phenix log. The same caveat applies to _reflns.B_iso_Wilson_estimate in the merged mmCIF, whose deposited values are conventionally the \(\Sigma\)-normalised kind. It is also where the flux the fixed integration disk clips lands: that loss is degenerate with an overall \(B\), so the reported number carries an \(r_1\)-dependent contribution and is not a property of the crystal alone (§9.1). The per-image estimate (used for the live radiation-damage plot) is accepted only when the fit is well-correlated and physically plausible (\(0 < B < 200\) Ų); on a bad frame (an indexing glitch, too few reflections) the Wilson line runs wildly steep, so an implausible \(B\) is reported as NaN rather than a spurious hundreds-of-Ų value.

\ No newline at end of file diff --git a/DEPLOYMENT.html b/DEPLOYMENT.html new file mode 100644 index 000000000..968e8e66a --- /dev/null +++ b/DEPLOYMENT.html @@ -0,0 +1,43 @@ + Deployment — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Deployment

To deploy Jungfraujoch, one needs to follow these steps:

  1. Install main Jungfraujoch code and frontend web interface

  2. Flash the U55C FPGA card with a proper image and install Linux kernel driver

  3. Install Jungfraujoch writer

  4. Install Jungfraujoch image viewer (optional)

  5. Install Python OpenAPI client

rugnux, the offline analysis tool, is installed separately and independently of all of them — see Install Rugnux at the end of this page.

The installation procedure depends a lot on the operating system. For Red Hat Enterprise Linux 8/9, Rocky 8/9, Ubuntu 22.04/24.04 or compatible, installation can be done with prebuilt packages from the package repositories and is relatively straightforward. For other systems one needs to build software from source. Both ways will be presented. What each released package contains, and what it needs on the machine, is described in Release contents.

Install main Jungfraujoch code and frontend web interface

On RHEL 8 systems there is a jfjoch-<version>-1.el8.x86_64.rpm that needs to be installed and contains all the necessary software and web interface.

On other OSes one needs to compile Jungfraujoch from source (from the repo directory):

$ mkdir build
+$ cd build
+$ cmake .. -DCMAKE_INSTALL_PREFIX=<directory to install>
+$ make
+$ sudo make install  
+

For manual installation, we recommend using a non-standard directory (like /opt/jfjoch), to facilitate upgrades and removal. For DKMS to manage kernel module sources it is necessary to copy driver sources to /usr/src/jfjoch-<VERSION> directory. This requires an extra CMake flag -DJFJOCH_INSTALL_DRIVER_SOURCE=ON.

Frontend web user interface has to be built separately with:

$ cd build
+$ make frontend
+

Frontend files (.html and .js) will be placed in frontend/dist (outside of build/ directory!) and have to be copied to a general location, e.g. /usr/local/jfjoch/frontend or /opt/jfjoch/frontend.

Flash the U55C FPGA card with a proper image and install Linux kernel driver

Firmware flashing

  1. Check that the card is detected by OS with “lspci |grep Xilinx” and check the PCIe bus/device/function (BDF) number, 23:00.0 in this case:

$ lspci |grep Xilinx
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2)
+

Note the device number 3450 that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and rev 2 identifying release of the firmware.

  1. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given):

$ sudo lspci -vv -s <PCIe slot number>
+23:00.0 Processing accelerators: Xilinx Corporation Device 3450
+(...)
+LnkSta:     Speed 16GT/s (ok), Width x8 (ok)
+(...)
+

  1. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn’t always reach correct timing).

  2. Flash the card with xbflash.qspi tool (part of Jungfraujoch). For fresh card use:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above> --bar-offset 0x1f06000 
+

For card that was already flashed with Jungfraujoch images:

sudo xbflash.qspi --primary <path to MCS file> --card <PCIe slot from above>
+

It is necessary to confirm the operation by pressing Y key or one can add --force option to avoid confirmation. It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions.

  1. Cold reboot:

sudo ipmitool chassis power cycle
+

Install PCIe driver

For the first run it is recommended to try the driver without installing it into the kernel directory:

$ cd fpga/pcie_driver
+$ make
+$ sudo insmod jfjoch.ko
+

Check with dmesg that the device was properly found:

$ dmesg |grep jfjoch
+[  431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142)
+[  431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a
+

If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. Install the prebuilt jfjoch-driver-dkms package from the Gitea package registry; on other systems follow the procedure in PCIe driver.

DKMS builds the module for the kernel it is being installed for rather than the running one, so a module built during a kernel update loads correctly after the reboot. RHEL 9.5 and later — and their CentOS Stream, Rocky and AlmaLinux equivalents — build unaided; the HAVE_VM_FLAGS_SET workaround earlier releases needed is obsolete.

NOTE: In case the driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk when the driver is updated:

$ sudo dracut -f
+

Configure network

Configure switch according to FPGA network guide - specifically set manual speed and turn off auto-negotiation for the port used to connect U55C card and connect card to switch.

Running Jungfraujoch software

Main Jungfraujoch service is called jfjoch_broker. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. It is recommended to run the service as systemd service.

jfjoch_broker takes two parameters: JSON configuration file and HTTP port (default is 5232). Example JSON files are placed in etc/ folder. JSON file format is also explained in the OpenAPI definition, as jfjoch_settings data structure.

When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring.

Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one jfjoch_broker on a single machine, each confined to a disjoint subset of GPUs, set CUDA_VISIBLE_DEVICES; setting CUDA_DEVICE_ORDER=PCI_BUS_ID keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host:

CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232
+CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233
+

To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. These need to be obtained from the PSI Detector Group.

Card verification

To test that the FPGA board is working properly without access to a JUNGFRAU detector, you can use jfjoch_fpga_test tool. For example, to simulate a 10M pixel system with 4 FPGA cards and 200k images:

jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000
+

Or a 1M pixel system with one FPGA card:

jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000
+

Install Jungfraujoch writer

Jungfraujoch writer is an additional service that connects to the jfjoch_broker ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard.

At the moment it is better to have a separate machine, with access to a distributed file system, for writing images.

Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands:

mkdir build
+cd build
+cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch_writer
+

Install Jungfraujoch image viewer

The Jungfraujoch viewer is an X-ray diffraction image viewer optimized to open Jungfraujoch HDF5 files.

The viewer is a Qt application and it requires a recent version of the library, therefore it is an optional dependency.

To include it in the building of Jungfraujoch use -DJFJOCH_VIEWER_BUILD=ON directive for CMake:

mkdir build
+cd build
+cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX=<directory to install> ..
+make jfjoch_viewer
+

Pre-built viewers for Windows and macOS, and a portable Linux archive, are on the Gitea release page — see Release contents and jfjoch_viewer.

Install Jungfraujoch Python client

Use pip:

pip install jfjoch-client
+

Install Rugnux (offline analysis)

rugnux is not part of the server stack and is installed independently of all of the above. It needs neither the broker, the writer, Qt nor a CUDA toolkit — only an NVIDIA driver if you want to use the GPU — and it does not have to run on the acquisition machine at all.

From the package repositories:

sudo dnf install rugnux          # RHEL / Rocky
+sudo apt install rugnux          # Ubuntu
+

Or, on a machine no repository covers, from the standalone archive:

mkdir -p /opt/rugnux-<version>
+tar xzf rugnux-<version>-linux-x86_64-cuda12.tgz -C /opt/rugnux-<version>
+/opt/rugnux-<version>/bin/rugnux
+

The archive has no top-level directory, so the -C is required. See Installing Rugnux for the Arm, Windows and macOS archives, the driver versions and building from source.

\ No newline at end of file diff --git a/DETECTORS.html b/DETECTORS.html new file mode 100644 index 000000000..c8acc132e --- /dev/null +++ b/DETECTORS.html @@ -0,0 +1 @@ + Supported detectors — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Supported detectors

PSI detectors

Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfraujoch controls the detector via slsDetectorPackage, which is statically compiled into its source code. The detector firmware must match the slsDetectorPackage version used in Jungfraujoch. The default is 8.0.2; 9.2.0 is built with the SLS9=ON CMake option and published in the slsdet9 package repositories. See PSI Detector group website for details.

DECTRIS detectors

Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles the output data stream of the DCU. This mode, called “lite” mode, doesn’t use FPGA boards, but mostly CPUs and GPUs for indexing. The mode is currently experimental and intended for low data rates (100 Hz).

\ No newline at end of file diff --git a/DETECTOR_GEOMETRY.html b/DETECTOR_GEOMETRY.html new file mode 100644 index 000000000..dcf47446d --- /dev/null +++ b/DETECTOR_GEOMETRY.html @@ -0,0 +1,2 @@ + Detector geometry — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Detector geometry

At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location relative to the detector frame. It is not recommended to place detector modules stacked.

The simplest case is a detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance and wavelength.

For a more complex case, one can provide the detector tilt in the PyFAI convention. This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector is defined with three rotation angles: rot1 (rotating detector right), rot2 (rotating detector downwards), rot3 (rotating detector clockwise). See PyFAI documentation for more details.

What a pixel coordinate means: (0, 0) is the centre of the first pixel

Pixel coordinates in Jungfraujoch and Rugnux are 0-based and pixel-centred: an integer coordinate is the centre of that pixel, so pixel i covers [i − 0.5, i + 0.5) and the sensor spans −0.5 … width − 0.5. A beam centre of 948.0 × 546.0 sits in the middle of pixel [546][948], not on any of its corners; 948.5 is the boundary between pixel 948 and 949.

This holds throughout the code: spot and reflection centroids are intensity-weighted sums of the integer pixel indices, the resolution and azimuthal-bin maps evaluate pixel (col, row) at exactly (col, row), and a fractional coordinate is turned back into a pixel index by rounding, not by truncation. The same convention applies to every coordinate the system exposes — the beam centre (beam_x_pxl/beam_y_pxl in the API and broker configuration, --beam-x/--beam-y in Rugnux, beam_center_x/beam_center_y in NXmx and in the CBOR stream), the spot and predicted-reflection positions written to HDF5, and the PONI reported by --mode calibration.

Other programs place the origin differently, and the difference is worth half a pixel — enough to matter when a geometry is copied between programs and then refined:

Convention

Beam centre equivalent to our x = 948.0

Jungfraujoch, Rugnux

948.0

XDS (ORGX/ORGY)

949.0 — also pixel-centred, but pixels are numbered from 1

Measured from the edge of the sensor, in length units — pyFAI (Poni2, fast axis), DIALS/dxtbx

(948.0 + 0.5) × pixel size, because the centre of pixel i is at (i + 0.5) × pixel size from the edge

pyFAI Poni1 (slow axis)

(height − 1 − y + 0.5) × pixel size — pyFAI measures the slow axis from the opposite edge, and the .poni declares orientation: 2 to say so

The .poni file written by rugnux --mode calibration is in pyFAI’s frame and so already carries that half pixel; the pixel values the same run reports are ours. Rot3 in that file is our rot3 negated and turned by 180°: the half turn sets the azimuthal reference, because pyFAI’s in-plane axes are the negatives of ours. It leaves 2θ untouched, so it moves only the azimuth.

Inside: two axis vectors; outside: rot1/rot2/rot3

Internally the detector plane is one orthogonal matrix whose columns are the fast axis (the laboratory direction of a +1 column step), the slow axis (+1 row step) and the normal (the sample→PONI direction). Every geometry calculation — resolution, azimuth, polarization, prediction, refinement — is that matrix applied to the offset of a pixel from the PONI.

rot1/rot2/rot3 remain the way the tilt is stated from outside, and the two views convert both ways: R = Rz(-rot3)·Rx(-rot2)·Ry(+rot1) in the internal frame, and back from the columns as

rot2 = asin(-slow.z)         rot1 = atan2(-fast.z, normal.z)         rot3 = atan2(slow.x, slow.y)
+

with rot2 in [-90°, 90°]. The angles are what is stored and what is written out, so a geometry given as angles comes back exactly as it was given.

What a miniCBF header states about the mounting

A PILATUS miniCBF gives the geometry twice. The # lines every writer produces carry the distance, the beam centre and the angles; some beamlines then append a CBF template block holding a full imgCIF axis table, which states the laboratory direction of the image’s fast and slow pixel directions, of the base goniometer axis, and of a 2theta arm where there is one. Where that table is present it is read, in preference to any assumption - it is the same information NXmx puts in fast_pixel_direction / slow_pixel_direction and the goniometer vector, in the form this format states it.

imgCIF’s laboratory frame has Z from the sample towards the source and Y opposite gravity, so it differs from the internal frame by a half turn about x - a rotation, not a mirror, so an axis carried through it turns the same way by the same angle.

There are two things a header can state that an assumption gets wrong by 90 degrees — an error no refinement recovers, and one the run’s axis-sign rescue cannot reach either, a quarter turn not being a sign:

  • the image mounted a quarter turn round, so its columns run vertically;

  • a spindle that turns about the vertical rather than the horizontal.

Where a header carries no axis table, a +SLOW on its # Oscillation_axis line still says the spindle runs along the image’s slow direction rather than its fast one. The axis name on that line is not usable - one header says X.CW +SLOW where its own table says the axis is Y - but the direction token is, and on the header that states both they agree.

A detector swung out on a 2theta arm

Chemical crystallography reaches high angle by swinging the detector out on a 2theta arm rather than by moving it closer. The arm turns the detector about the sample, so it changes nothing else: the distance is still measured along the detector normal, and the beam centre is still the point of normal incidence, which is where the arm’s own axis meets the detector and does not move. The swing is therefore exactly a PONI rotation, and the direct beam is what moves - by distance * tan(2theta), off the beam centre and often off the detector altogether.

Nothing has to be given for this: Rugnux takes it from the file. An NXmx master states the detector’s position as a depends_on chain of transformations, and the arm is one rotation in that chain - so the chain is followed, rather than a field of one particular name being looked for. A PILATUS miniCBF states it as # Detector_2theta, which turns about the same axis as the base spindle, the two being one axis on the four-circle geometry those headers describe.

Mirrored and quarter-turned detectors

On top of the continuous tilt the detector setup carries a discrete image orientation: whether the stored image is mirrored in Y, and how many multiples of 90° about the beam it is turned by. It is applied to the offset from the PONI before the tilt.

The distinction matters because these two operations are exact pixel remappings — an image can be shown the right way up without resampling anything — while an arbitrary in-plane rotation cannot. rot3 is therefore reserved for the genuinely arbitrary part: an in-plane angle is never decomposed into a quarter turn plus a residual, and the discrete part is set only where something states it (the detector configuration, --detector-mirror-y / --detector-quarter-turns, or the value a Jungfraujoch-written file records).

Both operations leave the distance from the PONI unchanged, so resolution, the solid-angle correction and anything else that needs only a radius are unaffected by them. Polarization is affected, and correctly so: it is computed from the azimuth in the laboratory, and what these operations change is which pixel index lands at which laboratory azimuth.

This is a different setting from mirror_y in the JSON configuration file (described below), which flips the module layout while the image is being assembled and so decides what the stored pixels are. The discrete image orientation changes no pixel at all.

Macromolecular crystallography convention for the vertical direction

One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, with Y values increasing downwards. This is also consistent with computer image formats.

However, other techniques (as well as internal operation of PSI X-ray detectors) might follow a convention where point (0,0) is in the bottom left corner and Y values increase upwards. Such a convention is used, for example, by PyFAI.

In general, the convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows the detector to be mirrored in Y.

The convention in use is worth checking whenever a geometry is carried between programs.

\ No newline at end of file diff --git a/EXTERNAL_TEST_DATA.html b/EXTERNAL_TEST_DATA.html new file mode 100644 index 000000000..74a102bc0 --- /dev/null +++ b/EXTERNAL_TEST_DATA.html @@ -0,0 +1 @@ + External test data — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

External test data

Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeline that only ever sees its own detectors is not tested. The datasets below were collected by other people, on detectors and in file formats we do not produce ourselves, and are used here to check that rugnux reads foreign files correctly and reduces them to sensible results. Most were collected at other facilities, ten on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone else’s detector and someone else’s acquisition system. Their authors published all of these for exactly this kind of reuse, and this page is where we credit them.

None of these data were collected by us. If you use any of them, cite the dataset DOI in the table below; the repositories themselves are cited in ACKNOWLEDGEMENT.

Where the values come from

  • Source is the repository we downloaded from and that repository’s own citable DOI for the archive we took. Every DOI on this page was resolved against DataCite - or, for 6NEN, whose DOI is registered with Crossref, against Crossref - before it was written down, and the identity of each dataset was taken from the repository’s record for the archive - not from our directory names.

  • Beamline, resolution, space group and cell are the values deposited with the PDB entry, read from the RCSB data API. They describe the published experiment. They are not our reprocessing results; no quantity measured by Jungfraujoch appears on this page.

  • Detector is read out of the image files themselves - the NXmx /entry/instrument/detector/description, the miniCBF # Detector: header, the marCCD instrument header or the SMV key block - which is authoritative where the PDB entry names a different detector.

  • Anything that could not be established from one of those sources is left blank.

Datasets

PDB

Source

Facility / beamline

dmin (Å)

Space group

Unit cell a b c α β γ (Å, °)

Detector (from file)

Title

11IF

IRRMC 10.18430/M311IF

NSLS-II 19-ID

1.51

P 43

51.1 51.1 71.9 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of an exported phospholipid binding protein from Bordetella pertussis in complex with Di-palmitoyl-3-sn-phosphatidylethanolamine (DPPE), P43 form 2

36GK

IRRMC 10.18430/M336GK

CLSI 08ID-1

2.28

I 2 2 2

120.6 189.5 199.7 90.0 90.0 90.0

Dectris Eiger 9M

D-GlcNAc-bound structure of Vibrio vulnificus putative carbohydrate binding module and split domain

3INP

IRRMC 10.18430/m33inp

APS 21-ID-F

2.05

F 41 3 2

224.1 224.1 224.1 90.0 90.0 90.0

marCCD, 225 mm plate

2.05 Angstrom Resolution Crystal Structure of D-ribulose-phosphate 3-epimerase from Francisella tularensis.

3KY7

IRRMC 10.18430/m33ky7

APS 21-ID-G

2.35

P 43 3 2

125.2 125.2 125.2 90.0 90.0 90.0

marCCD, 300 mm plate

2.35 Angstrom resolution crystal structure of a putative tRNA (guanine-7-)-methyltransferase (trmD) from Staphylococcus aureus subsp. aureus MRSA252

3MC4

IRRMC 10.18430/M33MC4

Home source, Rigaku MicroMax-007 HF

1.95

H 3

104.0 104.0 105.5 90.0 90.0 120.0

Rigaku Saturn 944+

Crystal structure of WW/RSP5/WWP domain: bacterial transferase hexapeptide repeat: serine O-Acetyltransferase from Brucella Melitensis

3MEB

IRRMC 10.18430/M33MEB

Home source, Rigaku MicroMax-007 HF

1.90

P 1 21 1

58.6 101.2 81.5 90.0 90.6 90.0

Rigaku Saturn 944

Structure of cytoplasmic aspartate aminotransferase from giardia lamblia

3P85

IRRMC 10.18430/M33P85

Home source, Rigaku FR-E+ SuperBright

1.90

P 63 2 2

127.3 127.3 72.9 90.0 90.0 120.0

Rigaku Saturn 944+

Crystal structure enoyl-coa hydratase from mycobacterium avium

3R6O

IRRMC 10.18430/M33R6O

Home source, Rigaku FR-E+ SuperBright

1.95

I 41

90.7 90.7 76.1 90.0 90.0 90.0

Rigaku Saturn 944+

Crystal structure of a probable 2-hydroxyhepta-2,4-diene-1, 7-dioateisomerase from Mycobacterium abscessus

5CC8

IRRMC 10.18430/M35CC8

Home source, Rigaku MicroMax-007 HF

1.75

P 21 21 2

87.1 93.8 72.5 90.0 90.0 90.0

Rigaku Saturn 944+

Structure of thiamine-monophosphate kinase from Acinetobacter baumannii in complex with AMPPNP

5EBI

MXRDR 10.18150/9887707

BESSY 14.2

1.09

P 1 21 1

35.7 44.1 35.7 90.0 120.0 90.0

marCCD, 225 mm plate

Crystal structure of a DNA-RNA chimera in complex with Ba2+ ions: a case of unusual multi-domain twinning

5EPE

IRRMC 10.18430/m3159c

APS 21-ID-G

1.90

F 2 3

157.5 157.5 157.5 90.0 90.0 90.0

Rayonix MX-300

Crystal structure of SAM-dependent methyltransferase from Thiobacillus denitrificans in complex with S-Adenosyl-L-homocysteine

5F6M

SBGrid 10.15785/sbgrid/201

SSRL BL11-1

1.10

P 21 21 21

54.8 58.5 67.4 90.0 90.0 90.0

PILATUS 6M

Isotropic Trypsin Model for Comparison of Diffuse Scattering

5J23

IRRMC 10.18430/M35J23

APS 21-ID-G

2.30

H 3

175.8 175.8 136.8 90.0 90.0 120.0

Rayonix MX-300

Crystal structure of NADPH-dependent glyoxylate/hydroxypyruvate reductase SMc04462 (SmGhrB) from Sinorhizobium meliloti in complex with 2’-phospho-ADP-ribose

5JK4

Zenodo 10.5281/zenodo.49859

ESRF ID14-2

1.10

P 1 21 1

37.7 77.9 56.3 90.0 102.1 90.0

ADSC Quantum 4

Phosphate-Binding Protein from Stenotrophomonas maltophilia.

5JVN

IRRMC 10.18430/m35jvn

ESRF ID29

2.90

P 6 2 2

249.4 249.4 84.1 90.0 90.0 120.0

PILATUS3 6M

C3-type pyruvate phosphate dikinase: intermediate state of the swiveling-domain mechanism

5KY6

MXRDR 10.18150/repod.1494374

BESSY 14.2

1.94

P 1 21 1

84.5 57.3 164.0 90.0 102.6 90.0

marCCD, 225 mm plate

Human muscle fructose-1,6-bisphosphate aldolase

5LZL

Zenodo 10.5281/zenodo.54757

Diamond I02

3.47

P 31 2 1

205.6 205.6 199.2 90.0 90.0 120.0

PILATUS 6M-F

Pyrobaculum calidifontis 5-aminolaevulinic acid dehydratase

5M17

Zenodo 10.5281/zenodo.4300323

Diamond I02

1.03

I 4

108.6 108.6 67.7 90.0 90.0 90.0

PILATUS 6M-F

Structure of the GH99 endo-alpha-mannanase from Bacteroides xylanisolvens

5MLN

IRRMC 10.18430/m35mln

ESRF ID23-2

1.60

P 21 2 21

74.2 80.4 80.5 90.0 90.0 90.0

PILATUS3 2M

The crystal structure of alcohol dehydrogenase 10 from Candida magnoliae

5NW5

SBGrid 10.15785/sbgrid/446

SLS X06DA

6.50

P 21 21 21

92.1 169.8 390.2 90.0 90.0 90.0

PILATUS 2MF

Crystal structure of the Rif1 N-terminal domain (RIF1-NTD) from Saccharomyces cerevisiae in complex with DNA

5REO

Zenodo 10.5281/zenodo.3730956

Diamond I04-1

1.88

C 1 2 1

112.4 52.6 44.4 90.0 103.0 90.0

PILATUS 6M-F

PanDDA analysis group deposition – Crystal Structure of SARS-CoV-2 main protease in complex with PCM-0102578

5SRC

IRRMC 10.18430/M35SRC

ALS 8.3.1

1.05

P 43

88.7 88.7 39.2 90.0 90.0 90.0

PILATUS3 6M

PanDDA analysis group deposition – Crystal structure of SARS-CoV-2 NSP3 macrodomain in complex with Z5198562500 - (R,R) and (R,S) isomers

5T39

SBGrid 10.15785/sbgrid/356

APS 21-ID-F

1.10

P 1 21 1

50.2 41.3 58.5 90.0 98.6 90.0

Rayonix MX-300

Crystal Structure of the N-terminal domain of EvdMO1 in the presence of SAH and D-fucose

5UTH

IRRMC 10.18430/M35UTH

Home source, Rigaku FR-E+ SuperBright

1.95

P 31 2 1

69.3 69.3 153.8 90.0 90.0 120.0

Rigaku Saturn 944+

Crystal structure of thioredoxin reductase from Mycobacterium smegmatis in complex with FAD

5VML

IRRMC 10.18430/M35VML

Home source, Rigaku FR-E+ SuperBright

1.70

P 42 21 2

66.3 66.3 115.3 90.0 90.0 90.0

Rigaku Saturn 944+

Crystal Structure of Acetoacetyl-CoA Reductase from Burkholderia Pseudomallei 1710b with bound NADP

6CDL

IRRMC 10.18430/m36cdl

APS 22-ID

1.25

P 21 21 2

58.3 85.9 46.1 90.0 90.0 90.0

marCCD, 300 mm plate

HIV-1 wild type protease with GRL-03214A, 6-5-5-ring fused umbrella-like tetrahydropyranofuran as the P2-ligand, a cyclopropylaminobenzothiazole as the P2’-ligand and 3,5-difluorophenylmethyl as the P1-ligand

6CEE

IRRMC 10.18430/M36CEE

Home source, Rigaku FR-E SuperBright

1.55

P 21 21 21

40.7 44.1 55.9 90.0 90.0 90.0

Rigaku Saturn A200

Crystal structure of fragment 3-(1-Methyl-2-oxo-1,2-dihydroquinoxalin-3-yl)propionic acid bound in the ubiquitin binding pocket of the HDAC6 zinc-finger domain

6CS9

SBGrid 10.15785/SBGRID/568

Australian Synchrotron MX2

1.85

P 1 21 1

32.9 25.5 40.2 90.0 98.6 90.0

ADSC Quantum 210r

Crystal structure of human beta-defensin 2 in complex with PIP2

6F3P

IRRMC 10.18430/M36F3P

APS 22-ID

1.35

C 1 2 1

142.9 85.7 112.0 90.0 122.2 90.0

marCCD, 300 mm plate

Crystal structure of S-adenosyl-L-homocysteine hydrolase from Pseudomonas aeruginosa in complex with 3’-deoxyadenosine and K+ cation

6FID

SBGrid 10.15785/sbgrid/541

ESRF ID30B

2.20

P 21 21 21

59.9 64.1 69.7 90.0 90.0 90.0

PILATUS3 6M

Bovine trypsin solved by S-SAD on ID30B

6FVZ

IRRMC 10.18430/m36fvz

ESRF ID23-2

1.80

C 2 2 2

131.2 222.8 86.5 90.0 90.0 90.0

PILATUS3 X 2M

Crystal structure of human monoamine oxidase B (MAO B) in complex with an inhibitor

6FWC

IRRMC 10.18430/m36fwc

ESRF MASSIF-3

1.70

C 2 2 2

131.7 222.1 86.3 90.0 90.0 90.0

PILATUS 2MF

Crystal structure of human monoamine oxidase B (MAO B) in complex with fluorophenyl-chromone-carboxamide

6G1F

Zenodo 10.5281/zenodo.1059413

Diamond I03

2.25

C 1 2 1

329.3 83.9 133.4 90.0 111.6 90.0

PILATUS3 6M

Crystal structure of D-phenylglycine aninotransferase (D-PhgAT) from Pseudomonas stutzeri with PLP internal aldimine

6GVK

Zenodo 10.5281/zenodo.1286854

ALBA XALOC

1.55

C 1 2 1

105.6 59.5 42.4 90.0 113.5 90.0

PILATUS 6M

Second pair of Fibronectin type III domains of integrin beta4 (T1663R mutant) bound to the bullous pemphigoid antigen BP230 (BPAG1e)

6H2P

IRRMC 10.18430/m36h2p

BESSY 14.1

1.48

C 2 2 21

103.5 107.1 216.5 90.0 90.0 90.0

PILATUS 6M

Crystal Structure of Arg184Gln mutant of Human Prolidase with Mn ions and Cacodylate ligand

6H5T

IRRMC 10.18430/m36h5t

BESSY 14.3

1.69

I 4 2 2

86.8 86.8 141.8 90.0 90.0 90.0

marCCD, 225 mm plate

Intersectin SH3A short isoform

6HV2

IRRMC 10.18430/m36hv2

SLS X06SA

1.71

P 61 2 2

68.9 68.9 133.6 90.0 90.0 120.0

Dectris Eiger 16M

MMP-13 in complex with the peptide IMISF

6HWJ

SBGrid 10.15785/sbgrid/614

ALBA XALOC

1.98

P 1 21 1

59.8 96.1 80.3 90.0 106.7 90.0

PILATUS 6M

Glucosamine kinase (crystal form A)

6I3J

IRRMC 10.18430/m36i3j

BESSY 14.1

2.59

F 2 2 2

134.4 203.8 226.7 90.0 90.0 90.0

marCCD, 225 mm plate

Bilirubin oxidase from Myrothecium verrucaria in complex with ferricyanide

6IU5

Zenodo 10.5281/zenodo.2532134

SPring-8 BL41XU

2.25

P 31

84.9 84.9 98.2 90.0 90.0 120.0

PILATUS3 6M

Crystal structure of cytoplasmic metal binding domain with zinc ions

6IU6

Zenodo 10.5281/zenodo.2532134

SPring-8 BL41XU

2.90

P 31

84.7 84.7 97.4 90.0 90.0 120.0

PILATUS3 6M

Crystal structure of cytoplasmic metal binding domain with nickel ions

6IU8

Zenodo 10.5281/zenodo.2532134

SPring-8 BL41XU

2.70

P 31

85.5 85.5 98.4 90.0 90.0 120.0

PILATUS3 6M

Crystal structure of cytoplasmic metal binding domain with cobalt

6IU9

Zenodo 10.5281/zenodo.2532134

SPring-8 BL41XU

3.00

P 31

85.3 85.3 97.6 90.0 90.0 120.0

PILATUS3 6M

Crystal structure of cytoplasmic metal binding domain with iron ions

6JGH

IRRMC 10.18430/m36jgh

SPring-8 BL44XU

0.94

P 21 21 21

50.6 62.5 68.2 90.0 90.0 90.0

marCCD, 300 mm plate

Crystal structure of the F99S/M153T/V163A/T203I variant of GFP at 0.94 A

6JGI

IRRMC 10.18430/m36jgi

SPring-8 BL44XU

0.85

P 21 21 21

50.9 62.4 69.2 90.0 90.0 90.0

marCCD, 300 mm plate

Crystal structure of the S65T/F99S/M153T/V163A variant of GFP at 0.85 A

6JGJ

IRRMC 10.18430/m36jgj

SPring-8 BL41XU

0.77

P 21 21 21

50.9 62.3 68.8 90.0 90.0 90.0

PILATUS3 300K

Crystal structure of the F99S/M153T/V163A/E222Q variant of GFP at 0.78 A

6MOJ

SBGrid 10.15785/sbgrid/620

ALS 5.0.1

2.43

I 41 2 2

130.4 130.4 293.5 90.0 90.0 90.0

PILATUS3 6M

Dimeric DARPin A_angle_R5 complex with EpoR

6NEN

UQ eSpace 10.14264/uql.2018.843

Australian Synchrotron MX2

2.15

P 3 1 2

105.5 105.5 35.1 90.0 90.0 120.0

SMV, S/N 928

Catalytic domain of Proteus mirabilis ScsC

6O2H

SBGrid 10.15785/sbgrid/747

CHESS F1

1.21

P 1

27.4 32.1 34.5 88.7 108.5 111.9

PILATUS3 6M

Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset

6OEL

SBGrid 10.15785/sbgrid/652

ALS 8.2.1

3.10

F 41 3 2

328.1 328.1 328.1 90.0 90.0 90.0

SMV, S/N 905

Engineered Fab bound to IL-4 receptor

6P8P

SBGrid 10.15785/sbgrid/673

APS 24-ID-C

1.64

P 4

97.5 97.5 60.1 90.0 90.0 90.0

PILATUS 6M-F

Structure of P. aeruginosa ATCC27853 HORMA1

6PB3

SBGrid 10.15785/sbgrid/681

APS 24-ID-E

2.05

P 6

100.4 100.4 48.9 90.0 90.0 120.0

Dectris Eiger 16M

Structure of Rhizobiales Trip13

6PXB

SBGrid 10.15785/sbgrid/698

APS 24-ID-E

1.75

P 32

64.0 64.0 119.4 90.0 90.0 120.0

PILATUS 6M-F

N-Terminal SH2 domain of the p120RasGAP

6PXC

SBGrid 10.15785/sbgrid/699

APS 24-ID-E

1.60

I 2 2 2

44.2 64.8 87.2 90.0 90.0 90.0

PILATUS 6M-F

N-Terminal SH2 domain of the p120RasGAP bound to a p190RhoGAP phosphotyrosine peptide

6QAJ

SBGrid 10.15785/sbgrid/637

Diamond I03

2.90

C 2 2 21

59.8 169.3 374.5 90.0 90.0 90.0

PILATUS3 6M

Structure of the tripartite motif of KAP1/TRIM28

6R72

Zenodo 10.5281/zenodo.14894181

SOLEIL PROXIMA 2

3.95

P 1 21 1

117.8 110.8 155.6 90.0 93.2 90.0

Dectris Eiger 9M

Crystal structure of BmrA-E504A in an outward-facing conformation

6RLR

Zenodo 10.5281/zenodo.5886687

Diamond I04

2.00

P 1

40.0 40.0 63.6 80.4 76.3 68.2

Eiger 16M

Crystal structure of CD9 large extracellular loop

6RYM

Keele University 10.21252/xbsq-d621

SRS PX10.1 (Daresbury)

1.46

P 43

50.2 50.2 51.9 90.0 90.0 90.0

marCCD 165 mm

Structure of carbohydrate recognition domain with GlcNAc bound

6S1U

MXRDR 10.18150/repod.0005795

BESSY 14.2

1.90

P 1 21 1

51.6 29.4 85.5 90.0 103.8 90.0

marCCD, 225 mm plate

Crystal structure of dimeric M-PMV protease C7A/D26N/C106A mutant in complex with inhibitor

6TOC

Zenodo 10.5281/zenodo.3571040

SLS X06DA

1.85

P 42

31.5 31.5 81.6 90.0 90.0 90.0

PILATUS 2MF

Crystal structure of the oligomerisation domain of the transcription factor PHOSPHATE STARVATION RESPONSE 1 from Arabidopsis (crystal form 3).

6TTN

IRRMC 10.18430/m36ttn

BESSY 14.1

1.12

P 21 21 21

39.9 79.8 104.7 90.0 90.0 90.0

PILATUS 6M

N-terminally truncated hyoscyamine 6-hydroxylase (tH6H) in complex with N-oxalylglycine and hyoscyamine

6U7G

IRRMC 10.18430/m36u7g

APS 23-ID-B

2.35

P 1 21 1

99.6 98.7 147.5 90.0 104.6 90.0

Dectris Eiger 16M

HCoV-229E RBD Class V in complex with human APN

6UKF

IRRMC 10.18430/m36ukf

APS 22-ID

1.00

P 1 21 1

61.0 37.3 69.0 90.0 109.8 90.0

Dectris Eiger 16M

HhaI endonuclease in Complex with DNA at 1 Angstrom Resolution

6V2R

IRRMC 10.18430/m36v2r

Home source, Rigaku FR-E

1.60

P 41 21 2

40.2 40.2 83.1 90.0 90.0 90.0

Rigaku Saturn A200

Crystal Structure of chromodomain of CBX7 mutant V13A in complex with inhibitor UNC3866

6VWW

IRRMC 10.18430/m36vww

APS 19-ID

2.20

P 63

150.5 150.5 111.3 90.0 90.0 120.0

PILATUS3 6M

Crystal Structure of NSP15 Endoribonuclease from SARS CoV-2.

6W4H

IRRMC 10.18430/m36w4h

APS 21-ID-F

1.80

P 31 2 1

167.7 167.7 51.9 90.0 90.0 120.0

Rayonix MX-300

1.80 Angstrom Resolution Crystal Structure of NSP16 - NSP10 Complex from SARS-CoV-2

6W75

IRRMC 10.18430/m36w75

APS 21-ID-F

1.95

P 32 2 1

166.2 166.2 98.3 90.0 90.0 120.0

Rayonix MX-300

1.95 Angstrom Resolution Crystal Structure of NSP10 - NSP16 Complex from SARS-CoV-2

6WZO

SBGrid 10.15785/sbgrid/785

APS 24-ID-E

1.42

P 1

43.7 50.1 69.3 106.5 90.1 97.1

Dectris Eiger 16M

Structure of SARS-CoV-2 Nucleocapsid dimerization domain, P1 form

6YQF

IRRMC 10.18430/m36yqf

Diamond I24

3.33

P 21 21 2

42.7 59.7 156.5 90.0 90.0 90.0

PILATUS3 6M

Crystal structure of the SYCE2-TEX12 delta-Ctip complex in a 4:4 assembly

6Z8O

Zenodo 10.5281/zenodo.3873216

ESRF ID30B

2.20

P 1 21 1

63.7 97.0 121.3 90.0 104.7 90.0

Dectris Eiger 4M

Structure of [NiFeSe] hydrogenase G491A variant from Desulfovibrio vulgaris Hildenborough pressurized with Krypton gas - structure G491A-Kr

6Z9G

Zenodo 10.5281/zenodo.3874714

ESRF ID30B

1.76

P 1 21 1

120.3 93.8 127.0 90.0 105.2 90.0

Dectris Eiger 4M

Structure of [NiFeSe] hydrogenase G491A variant from Desulfovibrio vulgaris Hildenborough pressurized with Oxygen gas - structure G491A-O2

6ZE4

SBGrid 10.15785/sbgrid/806

BESSY 14.1

1.60

P 21 21 21

93.6 109.9 116.1 90.0 90.0 90.0

PILATUS 6M

FAD-dependent oxidoreductase from Chaetomium thermophilum in complex with fragment 4-oxo-N-[(1S)-1-(pyridin-3-yl)ethyl]-4-(thiophen-2-yl)butanamide

6ZQR

Keele University 10.21252/r2nx-0425

Diamond I02

1.93

P 4

113.6 113.6 44.1 90.0 90.0 90.0

SMV, S/N 922

Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with GlcNAc ligand bound

6ZQY

Keele University 10.21252/hx7e-rd04

Diamond I04

1.85

P 4

119.3 119.3 44.2 90.0 90.0 90.0

SMV, S/N 921

Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with Neu5Ac ligand bound

6ZR0

Keele University 10.21252/zcfy-cw20

Diamond I04

1.94

P 4

119.2 119.2 44.2 90.0 90.0 90.0

PILATUS 6M Prosport+

Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with N-acetylalanine ligand bound

7ARR

MXRDR 10.18150/EM87YL

PETRA III, EMBL c/o DESY P13 (MX1)

1.10

P 1

30.9 32.1 43.1 114.2 91.9 109.9

PILATUS 6M-F

The de novo designed hybrid alpha/beta-miniprotein

7ATG

IRRMC 10.18430/m37atg

PETRA III, EMBL c/o DESY P13 (MX1)

0.60

P 21 21 21

18.0 31.0 43.9 90.0 90.0 90.0

PILATUS 6M-F

Crystal structure of Z-DNA in complex with putrescinium and potassium cations at ultrahigh-resolution

7BGT

MXRDR 10.18150/1HQGWO

BESSY 14.2

1.93

P 1

29.3 67.6 69.7 76.8 83.9 83.6

marCCD, 225 mm plate

Mason-Pfizer Monkey Virus Protease mutant C7A/D26N/C106A in complex with peptidomimetic inhibitor

7BGU

MXRDR 10.18150/C9DYSH

EMBL/DESY Hamburg (DORIS) X13

2.43

P 1

29.1 67.9 69.7 77.1 83.3 83.2

marCCD 165 mm

Mason-Pfizer Monkey Virus Protease mutant C7A/D26N/C106A in complex with peptidomimetic inhibitor

7D1M

IRRMC 10.18430/m37brr

SSRF BL17U1

1.35

P 1 21 1

55.5 99.0 59.6 90.0 108.5 90.0

Dectris Eiger 16M

CRYSTAL STRUCTURE OF THE SARS-CoV-2 MAIN PROTEASE COMPLEXED WITH GC376

7DKP

IRRMC 10.18430/M37DKP

ESRF MASSIF-3

1.45

P 1 21 1

49.8 169.5 49.8 90.0 93.5 90.0

Dectris Eiger 4M

Crystal structure of E. coli Grx2 in complex with GSH at 1.45 A resolution

7K1L

IRRMC 10.18430/m37k1l

APS 19-ID

2.25

P 63

150.8 150.8 110.7 90.0 90.0 120.0

PILATUS3 6M

Crystal Structure of NSP15 Endoribonuclease from SARS CoV-2 in the Complex with Uridine-2’,3’-Vanadate

7KCN

IRRMC 10.18430/m37kcn

LNLS W01B-MX2

1.46

P 41 2 2

67.0 67.0 116.9 90.0 90.0 90.0

PILATUS 2M

Reconstructed ancestor of HIUases and Transthyretins

7L6J

IRRMC 10.18430/m37l6j

APS 21-ID-F

1.78

I 41 3 2

171.7 171.7 171.7 90.0 90.0 90.0

Rayonix MX-300

Crystal Structure of the Putative Hydrolase from Stenotrophomonas maltophilia

7L84

SBGrid 10.15785/sbgrid/816

APS 24-ID-C

1.60

P 43 21 2

79.3 79.3 37.8 90.0 90.0 90.0

PILATUS 6M-F

Hen Egg White Lysozyme by Native S-SAD at Room Temperature

7MZT

IRRMC 10.18430/m37mzt

APS 22-ID

4.07

P 21 21 2

113.6 97.0 108.3 90.0 90.0 90.0

Dectris Eiger 16M

Borrelia burgdorferi BBK32-C in complex with an autolytic fragment of human C1r at 4.1A

7N0I

SBGrid 10.15785/sbgrid/835

ALS 5.0.2

2.20

P 21 21 21

75.8 131.6 140.0 90.0 90.0 90.0

PILATUS3 6M

Structure of the SARS-CoV-2 N protein C-terminal domain bound to single-domain antibody E2

7N2S

SBGrid 10.15785/sbgrid/916

SSRL BL12-1

2.37

P 1 21 1

83.2 52.8 106.3 90.0 98.3 90.0

PILATUS 6M

AS3.1-PRPF3-HLA*B27

7ORR

IRRMC 10.18430/M37ORR

MAX IV BioMAX

1.79

I 21 3

105.9 105.9 105.9 90.0 90.0 90.0

Dectris Eiger 16M

Non-structural protein 10 (nsp10) from SARS CoV-2 in complex with fragment VT00022

7OS3

MXRDR 10.18150/74YTYQ

PETRA III, EMBL c/o DESY P13 (MX1)

2.18

P 21 21 21

78.2 91.0 105.8 90.0 90.0 90.0

PILATUS 6M-F

Crystal structure of Rhizobium etli inducible L-asparaginase

7OU1

MXRDR 10.18150/VQQIHQ

BESSY 14.3

1.65

P 1 21 1

77.9 91.3 114.2 90.0 97.1 90.0

marCCD, 225 mm plate

Crystal structure of Rhizobium etli inducible L-asparaginase ReAV (monoclinic form MP2)

7PH1

IRRMC 10.18430/M37PH1

BESSY 14.2

1.18

I 2 2 2

75.0 81.3 124.2 90.0 90.0 90.0

PILATUS3 2M

Trypsin in complex with BPTI mutant (2S)-2-amino-4-monofluorobutanoic acid

7PQ7

IRRMC 10.18430/M3.IRRMC.6072

ELETTRA 11.2C

1.55

C 1 2 1

120.9 51.7 75.5 90.0 125.1 90.0

PILATUS 6M

Crystal structure of Campylobacter jejuni DsbA1

7QIJ

SBGrid 10.15785/sbgrid/907

PETRA III, EMBL c/o DESY P13 (MX1)

4.10

P 21 21 21

143.5 324.9 369.4 90.0 90.0 90.0

PILATUS 6M-F

Complex of the Yersinia enterocolitica Type III secretion export gate YscV with substrate:chaperone complex YscX:YscY

7QIS

IRRMC 10.18430/M37QIS

BESSY 14.2

1.83

P 61

100.3 100.3 206.2 90.0 90.0 120.0

PILATUS3 2M

CRYSTAL STRUCTURE OF THE P1 difluoroethylglycine (DfeGly) BPTI MUTANT- BOVINE CHYMOTRYPSIN COMPLEX

7RAA

SBGrid 10.15785/sbgrid/881

SSRL BL12-2

2.69

P 43 21 2

66.4 66.4 298.3 90.0 90.0 90.0

PILATUS 6M

Designed StabIL-2 seq15

7RIS

IRRMC 10.18430/M37RIS

APS 21-ID-D

1.72

P 32 2 1

44.5 44.5 189.9 90.0 90.0 120.0

Dectris Eiger 9M

Crystal structure of RPA3624, a beta-propeller lactonase from Rhodopseudomonas palustris, with active-site bound phosphate

7RJI

IRRMC 10.18430/M37RJI

LNLS W01B-MX2

1.71

H 3 2

83.0 83.0 124.8 90.0 90.0 120.0

PILATUS 2M

BthTX-II variant b, from Bothrops jararacussu venom, complexed with stearic acid

7T5T

SBGrid 10.15785/sbgrid/864

SSRL BL9-2

1.35

P 42 21 2

95.3 95.3 104.9 90.0 90.0 90.0

PILATUS 6M

Structure of Thauera sp. K11 CapP

7TCD

IRRMC 10.18430/m37tcd

SLS X06SA

1.70

C 1 2 1

138.5 47.9 78.1 90.0 107.6 90.0

Dectris Eiger 16M

LOV2-DARPIN fusion: D13

7YZX

IRRMC 10.18430/M37YZX

Diamond I24

1.90

P 63 2 2

169.4 169.4 141.8 90.0 90.0 120.0

PILATUS3 6M

ScpA from Streptococcus pyogenes, D783A mutant.

8A1A

IRRMC 10.18430/M38A1A

SLS X06SA

2.05

P 65

191.9 191.9 122.4 90.0 90.0 120.0

Dectris Eiger 16M

Structure of a leucinostatin derivative determined by host lattice display : L1F11V1 construct

8AGQ

IRRMC 10.18430/M38AGQ

SLS X06DA

1.09

C 1 2 1

89.9 55.4 54.8 90.0 113.5 90.0

PILATUS 2MF

Crystal structure of anthocyanin-related GSTF8 from Populus trichocarpa in complex with (-)-catechin and glutathione

8DQB

IRRMC 10.18430/m38dqb

NSLS-II 19-ID

2.50

I 2 3

164.1 164.1 164.1 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal structure of 3-dehydroquinate dehydratase I from Klebsiella oxytoca (I23 Form)

8DYZ

SBGrid 10.15785/sbgrid/957

CHESS F1

1.27

P 43 21 2

79.6 79.6 38.3 90.0 90.0 90.0

PILATUS3 6M

Hen lysozyme in tetragonal space group at ambient temperature - diffuse scattering dataset

8DZ7

SBGrid 10.15785/sbgrid/958

CHESS F1

1.34

P 21 21 21

30.5 56.4 73.9 90.0 90.0 90.0

PILATUS3 6M

Hen lysozyme in orthorhombic space group at ambient temperature - diffuse scattering dataset

8EGN

IRRMC 10.18430/M38EGN

CLSI 08B1-1

1.95

P 21 21 21

71.7 75.2 109.8 90.0 90.0 90.0

PILATUS3 6M

Crystal Structure of UDP-N-acetylmuramate-L-alanine ligase (UDP-N-acetylmuramoyl-L-alanine synthetase, MurC) Pseudomonas aeruginosa in complex with ligand AZ-13643701

8IYA

IRRMC 10.18430/m38iya

SSRF BL02U1

2.43

C 1 2 1

102.7 50.1 109.2 90.0 91.8 90.0

Dectris EIGER2 Si 9M

Complex of SETDB1-derived peptide bound to UBE2E1

8K1G

IRRMC 10.18430/M38K1G

PAL/PLS 11C

2.09

I 4 2 2

182.0 182.0 80.7 90.0 90.0 90.0

PILATUS3 6M

Crystal structure of ethylene glycol-bound glycerol dehydrogenase from Klebsiella pneumoniae

8OIC

IRRMC 10.18430/m38oic

Diamond I04

2.80

P 1

73.1 94.7 120.6 105.1 90.0 93.8

Eiger 16M

Trichomonas vaginalis riboside hydrolase (His-tagged)

8OWM

MXRDR 10.18150/II5MT4

PETRA III, EMBL c/o DESY P13 (MX1)

1.70

P 1

95.5 95.6 95.8 90.4 93.6 117.8

Dectris Eiger 16M

Crystal structure of glutamate dehydrogenase 2 from Arabidopsis thaliana binding Ca, NAD and 2,2-dihydroxyglutarate

8PQD

IRRMC 10.18430/m38pqd

ESRF MASSIF-3

1.50

P 21 21 21

59.4 59.4 192.9 90.0 90.0 90.0

Dectris Eiger 4M

c-KIT kinase domain in complex with avapritinib derivative 10

8QAW

MXRDR 10.18150/INUP4Q

PETRA III, EMBL c/o DESY P13 (MX1)

1.55

H 3

137.7 137.7 265.9 90.0 90.0 120.0

Dectris Eiger 16M

Medicago truncatula HISN5 (IGPD) in complex with MN, IMD, EDO, FMT, GOL and TRS

8QJ5

IRRMC 10.18430/m38qj5

ELETTRA 11.2C

1.63

P 1 21 1

57.6 100.6 77.9 90.0 96.1 90.0

PILATUS 6M

Crystal structure of the Levansucrase beta from Pseudomonas syringae pv. actinidiae

8QQ7

Zenodo 10.5281/zenodo.14901515

ESRF MASSIF-1

3.62

P 64 2 2

146.0 146.0 153.6 90.0 90.0 120.0

PILATUS3 2M

Structure of SpNOX: a Bacterial NADPH oxidase

8R5R

IRRMC 10.18430/m38r5r

ESRF ID23-1

3.08

P 21 21 21

91.7 132.9 137.5 90.0 90.0 90.0

Dectris EIGER2 CdTe 16M

Structure of apo TDO with a bound inhibitor

8RUD

MXRDR 10.18150/RBG2F9

PETRA III, EMBL c/o DESY P13 (MX1)

2.10

P 1 21 1

78.1 91.4 114.5 90.0 96.9 90.0

Dectris Eiger 16M

Crystal structure of Rhizobium etli L-asparaginase ReAV K138A mutant

8S38

MXRDR 10.18150/CGLBVH

PETRA III, EMBL c/o DESY P13 (MX1)

1.89

I 21 21 21

95.4 163.1 219.0 90.0 90.0 90.0

PILATUS 6M-F

Crystal structure of Medicago truncatula glutamate dehydrogenase 2 in complex with citrate and NAD

8SA8

IRRMC 10.18430/M38SA8

NSLS-II 19-ID

1.30

I 1 2 1

87.9 131.5 165.4 90.0 104.5 90.0

Dectris EIGER2 Si 9M

Crystal Structure of Cystathionine beta lyase from Klebsiella aerogenes, Covalently bound and free PLP (I2 form)

8SQO

IRRMC 10.18430/m38sqo

NSLS-II 19-ID

1.55

P 4 3 2

112.9 112.9 112.9 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (magnesium bound, F16L mutant)

8SQQ

IRRMC 10.18430/M38SQQ

NSLS-II 19-ID

2.25

F 4 3 2

171.5 171.5 171.5 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (Apo Cubic Form 2, F16L mutant)

8SQT

IRRMC 10.18430/M38SQT

NSLS-II 19-ID

2.20

F 4 3 2

170.7 170.7 170.7 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (iron bound, cubic form 2, F16L mutant)

8T7R

IRRMC 10.18430/M38T7R

APS 22-ID

3.84

C 1 2 1

357.1 259.6 255.4 90.0 133.1 90.0

Dectris Eiger 16M

Crystal structure of human leukocyte antigen A*0101 in complex with the Fab of alloreactive antibody E07

8THA

IRRMC 10.18430/m38tha

SSRL BL9-2

1.68

P 64

69.2 69.2 29.1 90.0 90.0 120.0

PILATUS 6M

1TEL, non-compressed, double-helical crystal form

8TYY

SBGrid 10.15785/sbgrid/1040

APS 24-ID-E

1.68

F 4 3 2

214.9 214.9 214.9 90.0 90.0 90.0

Dectris Eiger 16M

Structure of a bacterial Ubl-deubiquitinase complex (form 2)

8U0I

IRRMC 10.18430/m38u0i

ALS 8.2.1

1.54

P 43 21 2

50.3 50.3 90.6 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal structure of PA0012 complexed with cyclic-di-GMP from Pseudomonas aeruginosa

8V2T

Zenodo 10.5281/zenodo.10201899

NSLS X25

1.40

P 42 21 2

60.9 60.9 92.7 90.0 90.0 90.0

PILATUS 6M

Phosphoheptose isomerase GMHA from Burkholderia pseudomallei bound to inhibitor Mut148591

8V4J

Zenodo 10.5281/zenodo.10222807

NSLS X29A

1.31

P 42 21 2

61.0 61.0 92.4 90.0 90.0 90.0

ADSC Quantum 315

Phosphoheptose isomerase GMHA from Burkholderia pseudomallei bound to inhibitor Mut148233

8V4O

IRRMC 10.18430/m38v4o

NSLS-II 19-ID

2.70

P 61 2 2

139.5 139.5 545.0 90.0 90.0 120.0

Dectris EIGER2 Si 9M

Crystal structure of Acetyl-CoA synthetase 2 in complex with AMP from Candida albicans

8XBP

IRRMC 10.18430/M38XBP

SOLEIL PROXIMA 1

1.99

C 1 2 1

148.3 50.8 60.2 90.0 92.3 90.0

Dectris Eiger 16M

Crystal structure of AtNATA1 bound to Acetyl CoA

8XTE

SBGrid 10.15785/sbgrid/1101

SSRF BL19U1

1.99

P 32

208.8 208.8 67.2 90.0 90.0 120.0

PILATUS3 6M

Crystal structure of methyltransferase MpaG’ in complex with SAH and FDHMP

8XTF

SBGrid 10.15785/sbgrid/1102

SSRF BL02U1

2.13

H 3 2

211.8 211.8 67.4 90.0 90.0 120.0

Dectris EIGER2 Si 9M

Crystal structure of methyltransferase MpaG’ in complex with SAH and FDHMP-3C

8XTG

SBGrid 10.15785/sbgrid/1100

SSRF BL19U1

2.00

P 32

199.5 199.5 67.2 90.0 90.0 120.0

Crystal structure of methyltransferase MpaG’ in complex with SAH and DMMPA

8Y74

XRDa 10.51093/xrd-00227

SSRF BL02U1

1.90

C 1 2 1

125.8 76.6 87.1 90.0 92.4 90.0

Dectris EIGER2 Si 9M

Crystal structure of 9-mer peptide from H9N2 avian influenza virus in complex with BF2*0201

8YS9

IRRMC 10.18430/M38YS9

PAL/PLS 5C (4A)

1.46

P 21 21 21

71.0 77.7 83.2 90.0 90.0 90.0

Dectris Eiger 9M

Crystal structure of Phosphatidylethanolamine N-methyltransferase from R. thermophilum complexed with DMPE and SAH

9B22

IRRMC 10.18430/m39b22

NSLS-II 19-ID

1.30

P 1 21 1

39.8 92.7 57.7 90.0 91.7 90.0

Dectris EIGER2 Si 9M

Crystal structure of ADP-ribose diphosphatase from Klebsiella pneumoniae (ADP Ribose and AMP bound)

9BN8

IRRMC 10.18430/m39bn8

NSLS-II 19-ID

1.35

P 41

65.5 65.5 134.8 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of UDP-N-acetylmuramoylalanine–D-glutamate ligase (MurD) from E. coli in complex with UMA and inhibitor A19

9C18

Zenodo 10.5281/zenodo.11405662

NSLS-II 17-ID-1

1.90

P 1

41.9 42.0 60.2 84.1 87.2 63.7

Dectris EIGER1 Si 9M

Human biliverdin IX beta reductase in complex with NADP

9CHW

SBGrid 10.15785/sbgrid/1124

APS 21-ID-F

2.16

P 61

98.7 98.7 82.1 90.0 90.0 120.0

Rayonix MX-300

Crystal structure of human polymerase eta with incoming dAMPnPP nucleotide opposite threofuranosyl thymidine in DNA template

9CRW

IRRMC 10.18430/m39crw

CLSI 08ID-1

2.49

P 1 21 1

84.0 104.6 118.8 90.0 93.4 90.0

Dectris Eiger 9M

Crystal structure of the Candida albicans kinesin-8 proximal tail domain

9E2T

SBGrid 10.15785/sbgrid/1148

SSRL BL12-1

2.28

P 1

75.5 78.1 101.2 94.6 103.4 114.5

Dectris EIGER2 Si 16M

Structure of a de novo designed interleukin-21 mimetic complex

9EA5

SBGrid 10.15785/sbgrid/1142

SSRL BL9-2

2.00

P 1 21 1

65.9 73.1 98.4 90.0 108.7 90.0

PILATUS 6M

Structure of Citrobacter BubCD D104A mutant

9FCF

MXRDR 10.18150/DGZKW3

PETRA III, EMBL c/o DESY P13 (MX1)

2.36

P 4

91.3 91.3 35.8 90.0 90.0 90.0

Dectris EIGER1 Si 16M

Medicago truncatula 5’-ProFAR isomerase (HISN3) D57N mutant in complex with ProFAR

9FCG

MXRDR 10.18150/LDLSBT

PETRA III, EMBL c/o DESY P13 (MX1)

1.54

P 4

87.8 87.8 35.6 90.0 90.0 90.0

Dectris EIGER1 Si 16M

Medicago truncatula 5’-ProFAR isomerase (HISN3) D57N mutant in complex with PrFAR

9FHC

Zenodo 10.5281/zenodo.11472085

SLS X06SA

2.20

I 2 3

227.5 227.5 227.5 90.0 90.0 90.0

marCCD, 225 mm plate

Crystallographic structure of AcrB V612F with bound minocycline

9GDJ

ESRF 10.15151/ESRF-DC-1848199439

ESRF ID23-1

1.47

P 41 21 2

123.9 123.9 126.4 90.0 90.0 90.0

Dectris EIGER2 CdTe 16M

C-Methyltransferase SgMT from Streptomyces griseoviridis

9GJX

IRRMC 10.18430/M39GJX

Diamond I04

2.40

P 1 21 1

76.8 115.8 103.8 90.0 110.3 90.0

Eiger 16M

Bacillus licheniformis nitroreductase

9GQG

ESRF 10.15151/ESRF-DC-1900353437

ESRF ID30B

2.00

P 32 2 1

48.2 48.2 188.0 90.0 90.0 120.0

Dectris EIGER2 Si 9M

The FK1 domain of FKBP51 in complex with the macrocyclic SAFit analog m5(10,7)-(E)-OH

9H0Q

Zenodo 10.5281/zenodo.13912326

SOLEIL PROXIMA 2

2.55

H 3 2

169.5 169.5 344.0 90.0 90.0 120.0

Dectris EIGER1 Si 9M

N terminal domain of BC2L-C lectin in complex with N-(beta-L-Fucopyranosyl)-biphenyl-3-carboxamide

9HNC

MXRDR 10.60884/0K7B68

PETRA III, EMBL c/o DESY P13 (MX1)

1.88

P 1 2 1

123.8 123.6 187.7 90.0 90.1 90.0

PILATUS 6M-F

Crystal structure of potassium-independent L-asparaginase

9HS7

IRRMC 10.18430/M39HS7

ALBA XALOC

1.70

P 65

65.4 65.4 88.8 90.0 90.0 120.0

PILATUS3 X 6M

Anti-HIV-1 chimeric miniprotein mimicking the N-terminal half of gp41 NHR with an extended region targeting the MPER

9I0A

IRRMC 10.18430/M39I0A

SOLEIL PROXIMA 1

2.22

P 21 21 2

75.2 98.7 208.6 90.0 90.0 90.0

Dectris Eiger 16M

CARM1 in complex with arg-aDMA analog

9I80

Zenodo 10.5281/zenodo.14844040

SOLEIL PROXIMA 1

1.95

P 41

81.2 81.2 165.0 90.0 90.0 90.0

Dectris Eiger 16M

LecA in complex with a tolcapone derivative glycomimetic

9IG7

IRRMC 10.18430/M39IG7

PETRA III, EMBL c/o DESY P13 (MX1)

2.60

P 21 21 2

111.5 153.5 69.0 90.0 90.0 90.0

Dectris EIGER1 Si 16M

KOD-H4 DNA polymerase mutant in a binary complex with DNA:DNA containing two AtNA nucleotides

9IH9

IRRMC 10.18430/M39IH9

ESRF MASSIF-3

1.70

C 1 2 1

78.8 133.9 82.3 90.0 101.4 90.0

Dectris EIGER1 Si 4M

KEAP1 complexed to linear peptide 6

9JQ9

IRRMC 10.18430/M39JQ9

Home source, Excillum MetalJet D2+

1.90

P 21 21 21

48.6 50.5 78.6 90.0 90.0 90.0

PILATUS3 1M

Crystal structure of Plasmoredoxin from Plasmodium falciparum a disulfide oxidoreductase protein unique to Plasmodium species

9JZO

IRRMC 10.18430/m39jzo

PAL/PLS 11C

1.40

P 1

41.6 43.1 54.2 113.0 90.1 118.2

PILATUS3 6M

Crystal structure of PHICD111_20024_EAD.

9KHR

Zenodo 10.5281/zenodo.14070468

RRCAT INDUS-2 PX-BL21

2.00

P 21 21 21

48.7 50.3 78.0 90.0 90.0 90.0

marCCD, 225 mm plate

Crystal structure of Plasmoredoxin, a disulfide oxidoreductase from Plasmodium falciparum crystallized in the presence of Dithiothreitol (DTT)

9LXL

Zenodo 10.5281/zenodo.15005358

SSRF BL17UM

2.19

P 41 21 2

76.8 76.8 225.3 90.0 90.0 90.0

EIGER2 S 16M

Crystal structure of GH29 family alpha-L-fucosidase from Fusarium proliferatum LE1

9MH4

IRRMC 10.18430/M39MH4

NSLS-II 19-ID

3.05

P 21 3

138.7 138.7 138.7 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of Bifunctional protein GlmU from Klebsiella aerogenes

9MIN

SBGrid 10.15785/sbgrid/1151

ALS 8.2.1

2.05

P 21 21 21

95.5 98.5 155.7 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Structure of a designed minibinder to NYESO1-A*02:01

9O0H

IRRMC 10.18430/M39O0H

SSRL BL12-2

2.24

P 21 21 21

55.2 65.5 112.9 90.0 90.0 90.0

Dectris EIGER2 Si 16M

The ubiquitin-associated domain of human thirty-eight negative kinase 1, fused to the 3TEL crystallization chaperone via a 2-glycine linker

9P7Q

IRRMC 10.18430/M39P7Q

SSRL BL12-1

2.21

C 1 2 1

97.0 45.0 72.1 90.0 105.1 90.0

Dectris EIGER2 Si 16M

273K human S-adenosylmethionine decarboxylase

9PBB

IRRMC 10.18430/M39PBB

SSRL BL12-1

2.17

C 1 2 1

97.4 45.9 72.2 90.0 105.0 90.0

Dectris EIGER2 Si 16M

293K human S-adenosylmethionine decarboxylase

9Q41

SBGrid 10.15785/sbgrid/1194

CHESS 7B2

1.95

C 2 2 21

118.6 133.7 82.4 90.0 90.0 90.0

Dectris EIGER2 Si 16M

Crystal Structure of Human Apo Spermidine Synthase

9Q66

SBGrid 10.15785/sbgrid/1208

NSLS-II 17-ID-1

2.01

P 1 21 1

105.9 67.3 158.0 90.0 99.1 90.0

Dectris EIGER1 Si 9M

Human prolyl endopeptidase (PREP) - complex with JP-4-1-7

9QW8

ESRF 10.15151/ESRF-DC-2127908021

ESRF ID23-1

1.80

P 1

35.6 35.6 100.9 86.5 84.2 72.5

Dectris EIGER2 CdTe 16M

FKBP12 in complex with bifunctional ligand 1ad

9RCI

Zenodo 10.5281/zenodo.15615368

SOLEIL PROXIMA 2

1.66

P 1

35.9 39.3 100.9 98.3 90.3 90.1

Dectris Eiger 9M

Crystal Structure of Flap Endonuclease FEN1 with Compound 28

9RCS

XRDa 10.51093/xrd-00383

Diamond I24

3.01

P 1 21 1

70.0 78.8 82.3 90.0 88.6 90.0

Eiger 9M

Cardioderma bat coronavirus KY43 receptor binding domain in complex with human CEACAM6

9RP9

IRRMC 10.18430/M39RP9

SOLEIL PROXIMA 1

2.10

C 1 2 1

73.5 59.8 91.7 90.0 100.8 90.0

Dectris Eiger 16M

Crystal structure of mouse pVHL-ElonginB-ElonginC complex

9S02

MXRDR 10.60884/NRNGS4

MAX IV BioMAX

1.65

P 21 21 2

163.7 88.0 116.7 90.0 90.0 90.0

EIGER2 X 16M

PYCR1 in complex with 3-(2-thiazolyl)propionic acid

9SL0

IRRMC 10.18430/M39SL0

ESRF MASSIF-1

1.60

P 21 21 21

60.2 80.2 111.6 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal structure of HLA-A0201 in complex with peptide LLWNGPMAV

9T6S

SBGrid 10.15785/sbgrid/1260

ESRF ID30B

2.00

P 21 21 21

63.0 64.6 102.7 90.0 90.0 90.0

Dectris EIGER2 Si 9M

Crystal Structure of the Listeria monocytogenes CadC with Cadmium

9UPT

XRDa 10.51093/xrd-00191

NSRRC TPS 05A

2.37

P 6

158.3 158.3 54.0 90.0 90.0 120.0

SMV, S/N 930

Structure of AtBgl1A, a GH1 beta-Glucosidase from Acetivibrio thermocellus

9VX7

IRRMC 10.18430/M39VX7

PAL/PLS 5C (4A)

4.85

P 64

122.5 122.5 118.9 90.0 90.0 120.0

PILATUS3 6M

Transcription factor

9VYB

IRRMC 10.18430/M39VYB

PAL/PLS 5C (4A)

2.12

P 21 21 21

44.4 47.8 48.4 90.0 90.0 90.0

Dectris Eiger 9M

Antitoxin Phd

9W3Y

IRRMC 10.18430/M39W3Y

Photon Factory BL-1A

1.50

P 21 21 21

60.7 70.0 94.2 90.0 90.0 90.0

Dectris EIGER1 Si 4M

X-ray Crystal Structure of Pseudoazurin Met16Gly variant (Tris-HCl pH 7.6)

9YL4

Zenodo 10.5281/zenodo.17298261

APS 17-ID

3.70

P 21 21 21

95.8 111.3 403.0 90.0 90.0 90.0

PILATUS 6M

Crystal structure of PprA S-F filament from Deinococcus radiodurans

9YZK

IRRMC 10.18430/M39YZK

ALS 8.2.2

4.44

I 1 2 1

75.8 163.0 192.3 90.0 98.6 90.0

PILATUS3 S 2M

Isoreticular co-crystal 1 with symmetrical expanded duplex (42mer) containing insert sequence ACCCTTCTATGACCTACTCCA

9Z44

IRRMC 10.18430/M39Z44

ALS 8.2.1

7.20

I 1 2 1

73.5 127.7 141.2 90.0 92.0 90.0

Dectris EIGER2 Si 9M

Isoreticular co-crystal 1 with symmetrical expanded duplex (31mer) containing insert sequence CCCGGCCGGA and loaded with C-clamp domain

9Z72

SBGrid 10.15785/sbgrid/1239

SSRL BL9-2

2.38

P 31 2 1

59.2 59.2 426.2 90.0 90.0 120.0

Dectris EIGER2 Si 16M

Structure of V. cholerae CapS (form 1)

9ZLO

Zenodo 10.5281/zenodo.18652652

Australian Synchrotron MX2

2.00

P 21 21 21

38.4 90.0 107.0 90.0 90.0 90.0

Dectris EIGER1 Si 16M

Crystal structure of Proteus mirabilis UreE

9ZM0

IRRMC 10.18430/M39ZM0

NSLS-II 17-ID-1

2.10

P 1 21 1

50.4 30.1 91.2 90.0 97.1 90.0

Dectris EIGER1 Si 9M

Crystal structure of monomeric Atg23

9ZMU

IRRMC 10.18430/M39ZMU

NSLS-II 19-ID

1.98

P 65 2 2

47.8 47.8 492.6 90.0 90.0 120.0

Dectris EIGER2 Si 9M

Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form)

Seven rows have no PDB code. Six are small-molecule / chemical-crystallography datasets, kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero detector 2θ; the seventh is the second collection in the 6R72 Zenodo record, described below. They have no deposited macromolecular values, so those columns are blank, and their titles are the repository record titles verbatim.

Five datasets are in primitive space groups with no screw axis - 6ZQR, 6ZQY, 6ZR0 and 9FCF in P 4, and 6NEN in P 3 1 2. They are in the battery as negative controls for screw-axis detection: the correct answer for each has no systematic absences.

6Z9G is the collection’s only index-4 superstructure: its deposited cell is four times the sublattice a/2, b, c/2, and the four copies of each of its two entities are related by the XOR-closed trio of near-pure translations (1/2,0,0), (0,0,1/2) and (1/2,0,1/2). Every other pseudo-translation in the collection is index 2 or a setting artefact, so it is the one set that exercises a supercell of index greater than two.

Archives that are not a single sweep

Most rows above are a single continuous rotation. The archives described in this section are not, or needed special handling to obtain the images; their layout is read from the image files themselves, from the repository file listings and from the depositors’ own description of the record. Not every archive in the table has had its layout audited to this depth. Where an archive held more than one collection, only one is kept - the repository’s project page is not a reliable guide to this, because it describes the project rather than the tarball (7TCD’s page lists a 900-frame miniCBF sweep the archive does not contain).

6R72 - two collections on one crystal. The Zenodo record holds two complete 360° sweeps of 3600 × 0.1° frames taken from the same crystal: a helical collection, which produced the deposited structure, and a low-dose collection from a single position, which was not used for a deposition and therefore has no PDB entry. Both are in the table, sharing one DOI; the deposited values belong to the helical collection only. The record also ships the authors’ XDS.INP.

The three CHESS depositions - wedges plus a measured background. Each crystal was rotated in 50° wedges of 500 × 0.1° frames and translated between wedges to spread the dose. Each crystal also has a rotation at 1° per frame taken with the crystal translated out of the beam, which the depositors include as a measured background and say can be matched to the diffraction frames by the phi value in the image header.

PDB

Crystals

Wedges per crystal

Background rotation

8DYZ

1

8

360 frames

8DZ7

2

4

200 frames per crystal

6O2H

4

1, 3, 2, 5 - 11 in all

50, 145, 95, 235 frames, one per crystal

Four archives added for facility coverage hold more than one sweep. One sweep is kept, and the row is pinned to it. 7BGU’s MXRDR record is one directory of 900 marCCD frames that are two sweeps with different oscillation widths: frames 1001-1674 (0.4°) are the row, frames 1675-1900 were moved to sweep2/ so the reader sees one sweep. 5JK4’s archive holds a high-resolution sweep of 185 frames (80 mm, 1°) and a low-resolution one of 93 frames (250 mm, 2°); the row is the high-resolution sweep. 6RYM’s zip holds two sweeps (jmp47a2_1, 70 frames; jmp47a2_2, 60 frames); the row is the first. 6GVK’s Zenodo record has three tarballs (set1-set3); only set1 (1800 miniCBF frames) was downloaded and is the row.

Seven IRRMC archives hold more than one collection. In six of them one sweep is kept and the rest were deleted, so a run over the data directory sees a single collection per dataset. 7RIS is the exception: its two sweeps are at different wavelengths and both are kept.

PDB

What the archive holds

Kept

6UKF

two sweeps on one crystal - 960 x 0.25° (240°) and 1440 x 0.25° (360°)

the 360° sweep

7DKP

two complete 360° sweeps on one crystal, 3° apart in ω

the first

9PBB

two overlapping 135° wedges of one crystal, 90 x 1.5° each

the first

8U0I

a 69-frame screening wedge and three 180° sweeps on three crystals

the first 180° sweep

36GK

two 360° sweeps of 1800 x 0.2° at the same geometry

the one the archive and DOI are named for

9CRW

a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm

the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å

7RIS

two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native)

both

Ten further archives hold more than one collection. Their layout was read from the image files and repository listings; one sweep is kept for a run over the data directory unless noted.

PDB / dataset

What the archive holds

Kept

5JVN

two 360° sweeps of one crystal, 3600 × 0.1° each (w1_3, w1_4)

the w1_3 sweep

6FID

two 360° sweeps of one crystal, 3600 × 0.1° each

the first

6IU8

a two-wavelength MAD pair, 720 × 0.5° each at 1.605 Å (low remote) and 1.740 Å (peak)

both - the pair is the point

7OS3

four 360° sweeps at λ 2.066 Å, 3600 × 0.1° each, from two crystal positions (pos2_1/2, pos3_1/2)

all four are kept as separate sweep directories pos*/

7L84

two ~720° helical sweeps, 1439 × 0.5° each at λ 1.892 Å, room temperature

the 301_helical_1 sweep

5M17

seven crystals in one tar (5M03/5M17/5MEL/5MC8/5M5D/5M3W/5LYR), one 1800-frame sweep each

only the 5M17 tar was downloaded

cytidine

six scans, three ω and three φ, at 2θ = 30° (I19-1 commissioning)

the 1800-frame φ scan

lalanine

four runs of the RODIN L-alanine deposition at 2θ = 20°

the 900-frame pgw240050_01 run

9E2T

one continuous sweep plus screening images

the 2700-frame sweep

8OWM

three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip

the three sweep zips (proc zip skipped)

Three archives needed special handling to obtain the images.

  • 5KY6 is served by MXRDR as 11 separate RAR archives, one folder of frames per archive, 50 frames per archive except the last, 564 frames in all. Reading them needs a RAR reader with RAR3 filter support: the official 7-Zip 7zz reads them, while the unrar-free and p7zip builds of Enterprise Linux 8 cannot.

  • 6ZR0’s zip, as the Keele University repository serves it, is damaged: it has no central directory. Frames 1-1059 of the 1060 were recovered from the zip’s local file headers; the last frame is lost.

  • 6NEN’s University of Queensland eSpace record blocks scripted download, so its archive was downloaded by hand in a browser.

Datasets published as Raw Data Letters

Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a format whose purpose is to make raw images citable and re-processable in their own right. The letters describe the collections and the difficulties in them, and are the reference for what the data are:

  • V. Zampieri, A. Vermot, M. Thepaut, I. Petit-Hartlein, F. Fieschi, P. Falson and V. Chaptal, “X-ray diffraction images for two membrane protein crystals presenting high anisotropy; the B. subtilis ABC transporter BmrA and the S. pneumoniae NADPH oxidase” (2025), IUCrData 10, x250591 doi:10.1107/S2414314625005917 - covers 6R72 and 8QQ7.

  • V. Neviani, M. Lutz, W. Oosterheert, P. Gros and L. Kroon-Batenburg, “Crystal structure of the second extracellular domain of human tetraspanin CD9: twinning and diffuse scattering” (2022), IUCrData 7, x220852 doi:10.1107/S2414314622008525 - covers 6RLR.

The authors of the second letter also published their own reciprocal-space reconstruction of the 6RLR data as a separate Zenodo record, 10.5281/zenodo.6961763.

Detector column for marCCD and SMV files

marCCD and SMV files name the detector differently - or not at all. A marCCD file names no model: its instrument header states the image dimensions and the pixel size, from which the plate size follows (3072 x 73.242 um = 225 mm, 4096 x 73.242 um = 300 mm), and its comment block a serial number; the LS-CAT beamlines additionally write detector='Rayonix MX-300 s/n 023' into the dataset comment. An ADSC-style SMV header names only a serial (DETECTOR_SN=930); a Rigaku d*TREK one names the model (CCD_DETECTOR_DESCRIPTION=Saturn944+), and that is what its row carries. For those rows the Detector column carries what the file itself establishes: the plate size (marCCD, 225 mm plate), the comment’s name where one is present (Rayonix MX-300), or the serial (SMV, S/N 930).

Deposited models and structure factors

184 of the 191 datasets have a released PDB entry, and RCSB reports released structure factors (status_code_sf = REL) for every one of them. A merged result from this pipeline can therefore be checked against the deposited model or against the deposited intensities.

Rows where our reduction and the deposition disagree

Six of the 181 rows are ones where rugnux does not reproduce the deposited space group or cell, and where we have looked at the disagreement closely enough to change how the row is scored. They are collected here because a scoring row that silently disagrees with a published entry is not something a reader should have to discover from the code.

These are open questions, not errors we are attributing to the PDB. A deposited entry was arrived at by someone who had something we do not: a model that had to refine, and usually more knowledge of the crystal than the images carry. Where we describe evidence below, it is evidence about what these images support, which is a narrower thing than what the crystal is. In every one of the symmetry rows the possibility that the crystal really has the lower symmetry, with a pseudo-symmetry too exact for any test available to us to see, remains live - see the limit at the end of this section.

How the manifest records it, in tools/battery/open.json:

  • ref always keeps the deposited values verbatim, so the deposition is never lost.

  • ref_alternatives lists the other answers the row accepts. Each one replaces the reference fields it names - a space group, a cell, or both - and the row passes if our answer matches any of the references, the deposited one included. Each must carry why; an alternative with no stated reason is a schema error, not a silent pass, so the mechanism cannot become a way to turn a failure into a pass quietly. This is how the five knife-edge rows below are recorded: we are not asserting that our answer is right, only that both descriptions are defensible and that picking either one is acceptable. The report counts these rows separately from ordinary passes and prints the reason, so a reader can see how many there are and judge each.

  • ref_override replaces the fields the battery scores against, with ref_override_why. It asserts a corrected reference, so it is for a reference we can show to be wrong about these images - 8XBP below - and not for a disagreement that is open.

  • unscored drops the row from scoring entirely. It is a last resort: it also loses a test that still works, which is why an open question is now recorded as accepted alternatives instead.

8XBP is a question about provenance, not about symmetry

8XBP is different in kind from the other five and should not be read alongside them. Nothing here concerns the deposited model or its space group. The question is whether the raw images uploaded with the entry are the same crystal the deposited cell describes: the master file records data_collection_date 2023-06-21 where the entry records a collection date of 2023-06-23, and the deposited b = 50.78 A is 2.0% away from the b these images give. Two independent signals, one of them nothing to do with our processing. The override replaces the cell with the one DIALS 3.29 indexes de novo on this master and keeps the deposited space group and resolution.

Four trigonal and tetragonal rows where we read a higher point group

PDB

Deposited

rugnux reads

Where it stands

6TOC

P 42

P 42 2 2

both acceptable; the refinement test is not unanimous

8XTE

P 32

P 31 2 1 / P 32 2 1

both acceptable; ours is the better supported

8XTG

P 32

P 31 2 1 / P 32 2 1

both acceptable; the deposition is the better supported

6PXB

P 32

P 31 1 2 / P 32 1 2

both acceptable; unresolved in either direction

All four accept either answer: the deposited group and the one we read both pass. None of them is a claim that the deposited assignment is wrong - each is a question we cannot close, and 8XTE and 8XTG do not lean the same way, so they should not be read in one voice. The two 8XT* rows were for a time scored against our own answer by editing the reference itself, with no reason recorded; the deposition is back in ref verbatim and the disagreement is stated here.

6TOC. The deposited asymmetric unit holds two chains, and they are related by the very two-fold the higher group adds, to 0.16 A C-alpha RMSD over 43 residues - coordinate error at the deposited 1.85 A. Merging in P 42 2 2 costs 0.0006 in Rmeas for 1.75 times the multiplicity, and correlates better with the deposited model than the P 42 merge does. POINTLESS, run independently on our own P1 merge, reads the same point group. The refinement test - refine in each candidate group and compare R-free, which is the one comparison not biased toward the group the deposited model was refined in - does not come out unanimous: ZANUDA 1.097 makes P 42 2 2 the better group at half the parameters and reports the deposited assignment incorrect, while an independent Refmac 5.8.0431 comparison on a symmetry-consistent free set makes P 42 the better one, by less than the spread between refinement protocols - the spread of the test exceeds the effect it is being asked to measure. Both answers are therefore accepted, with the refinement evidence recorded as split.

8XTE. The distinguishing test is the twin-immune centric zone: reflections that the higher group makes centric but the subgroup does not are their own twin mates, so a merohedral twin law cannot make them read centric. They read <|E^2-1|> = 0.946 +/- 0.012 against a centric expectation of 0.968 and an acentric one of 0.736. Re-refinement on a shared free set, with the twin law removed from both sides, favours the higher group. The deposited entry’s published R values are themselves reproducible only with a twin law the entry does not declare, at a twin fraction of 0.50 - and a 0.50-twinned target already has the symmetry in question. Of the four rows this is the one where the evidence most clearly favours what we read; it still cannot be closed, because the centric zone is the only test that speaks to it (see the limit below), so both answers are accepted.

8XTG. This row is genuinely open and is flagged as such in the manifest. Every correlation-based instrument we have - our own operator correlations, and POINTLESS on our P1 merge - reads the higher point group, but the centric-zone test, the only one of them that can separate real symmetry from pseudo-symmetry, reads <|E^2-1|> = 0.869 at -44.9 nats: between the two expectations, and on the wrong side. The L-test indicates a twin fraction near 0.20-0.26. Whether this crystal is partially twinned or purely pseudo-symmetric has not been established. Here the better-supported answer is the deposited one, which is the opposite of 8XTE: the two rows look alike in the table and are not alike in the evidence. Both answers are accepted.

6PXB. Unscored rather than overridden, because the evidence does not settle either way. Our merge and POINTLESS both read a 312 point group, the added two-folds correlate at or above the level of the three-folds nobody disputes, and merging in the higher group lowers Rmeas at twice the multiplicity. Against that, the deposited asymmetric unit’s six chains pair under the added two-fold at 0.3-0.7 A, which is more than coordinate error at 1.75 A, and ZANUDA settles on a different trigonal supergroup - 321 rather than 312 - whose operators these data do not support. Neither answer is established in either direction, so both are accepted and the row still tests everything else about the set.

9RCI: two defensible descriptions of one lattice

The sixth row is not about symmetry but about which cell describes the crystal. The Patterson has an off-origin peak at 62.5% of the origin, so a genuine translational NCS relates the two halves of the cell rugnux reports, and the deposited cell is that supercell’s (0, 1/2, 1/2)-centred sublattice to 0.17%. Both are correct descriptions of the same diffraction: one leaves the near-translation in the contents of a doubled cell, the other absorbs it into the lattice and indexes only the strong sublattice. Which one a program should prefer is a choice, not a measurement, so the row accepts either. The alternative cell recorded in the manifest is computed from the deposited cell alone (c’ = b + 2c, centring removed), not copied from our output, so it stays a statement about the deposition’s lattice.

The limit that applies to all four symmetry rows

A merohedral twin at a twin fraction of exactly 0.5 and a crystal that genuinely has the higher symmetry predict identical intensities. No amount of data and no refinement R separates them, and the same holds, approximately, for a pseudo-symmetry that is merely very exact. Every test described above measures how nearly a symmetry operator holds on these images; none of them can show that it holds exactly. Where the higher symmetry is right, merging in it gains multiplicity and completeness; where it is a pseudo-symmetry that close, merging in it costs nothing measurable either. That is why these rows are described as open questions, and why none of them should be read as a statement that a deposited model is wrong.

Dataset directories whose name is not the PDB code

Directory

PDB code in the table

Why

7brr

7D1M

The IRRMC archive and its DOI are published under 7BRR, which the PDB obsoleted on 2020-10-28 and replaced with 7D1M. The directory and the DOI keep the archive’s own name; the deposited values are 7D1M’s.

An archive that ships placeholder images

8AGQ’s data/ directory contains 30 files named ForBackgroundOnly_000NN.img alongside the 1800-frame sweep. They are not images: each is a 64-byte text file holding a path string. A reader that globs *.img will pick them up, so they are named here rather than silently left.

Datasets with no PDB entry

Dataset

Repository record

Why there is no PDB code

6r72/ld

Zenodo record 10.5281/zenodo.14894181, file prefix V-CK63-8-ld_1_

a second collection in the 6R72 record - a low-dose sweep on the same crystal, not the one the deposited structure was built from

cuhf2

Zenodo record 10.5281/zenodo.6347466

a small-molecule dataset, not a PDB deposition

dnba

Zenodo record 10.5281/zenodo.1036416

a small-molecule dataset, not a PDB deposition

metformin

Zenodo record 10.5281/zenodo.20135265

a small-molecule dataset, not a PDB deposition

nidppe

Zenodo record 10.5281/zenodo.20041091

a small-molecule dataset, not a PDB deposition

cytidine

Zenodo record 10.5281/zenodo.33555

a small-molecule dataset, not a PDB deposition

lalanine

Zenodo record 10.5281/zenodo.11946282

a small-molecule dataset, not a PDB deposition

Five of the six small-molecule sets have a published structure to check a run against. These are reference values from the literature, not results obtained here.

Dataset

Space group

Cell (A, deg)

T

Reference

dnba

C 1 2/c 1 (15)

20.2635 8.7575 9.6697 / 90 109.941 90

30 K

the Zenodo record’s own title and the xia2.html the depositors ship inside it, corroborated by COD 4510614/4510615 - Cryst. Growth Des. 13 (2013) 1861-1871 doi:10.1021/cg300906j

metformin

P 1 21/c 1 (14)

7.9104 13.8794 7.9310 / 90 114.606 90

100 K

the hydrochloride, form I; COD 2108029 - Acta Cryst. B73 (2017) 10-22 doi:10.1107/S2052520616017844

nidppe

P 1 21/c 1 (14)

11.2779 13.3386 15.8739 / 90 98.7953 90

150 K

COD 2012031 - Acta Cryst. C57 (2001) 690-693 doi:10.1107/S0108270101003961

cytidine

P 21 21 21 (19)

13.98 14.788 5.119 / 90 90 90

296 K

β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C49 (1993) 1789-1792 doi:10.1107/S0108270193003464

lalanine

P 21 21 21 (19)

5.791 5.944 12.269 / 90 90 90

100 K

COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B69 (2013) 249-259 doi:10.1107/S2052519213010014

cuhf2 has no confirmed cell. Its space group is published as P 4/n m m (Phys. Rev. B 81, 064422 (2010) doi:10.1103/PhysRevB.81.064422) but no numeric cell was located, so a run on it can be scored on the space group and not on the cell.

The collection in numbers

The collection was chosen to widen the spread of file formats, detectors, facilities and symmetries rather than to be easy to process. The counts below describe where it comes from; like everything else on this page, they are metadata about the depositions and their files, not measurements.

  • Repository: IRRMC 94, SBGrid 36, Zenodo 34, MXRDR 16, Keele University 4, ESRF 3, XRDa 3, UQ eSpace 1.

  • Facility - counted from the facility part of the Facility / beamline column, the beamline ignored so that entries deposited with and without one count the same, over the 180 rows that name one: APS 26, Diamond 20, ESRF 17, NSLS-II 14, BESSY 12, PETRA III 12, SSRL 11, ALS 8, SLS 7, SOLEIL 7, SPring-8 7, SSRF 7, PAL/PLS 5, CHESS 4, CLSI 4, ALBA 3, Australian Synchrotron 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, and one each from NSRRC, Photon Factory, RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other ten rows were collected on laboratory sources: nine on rotating anodes and one on a liquid-metal jet.

  • Crystal system, from the deposited space group of the 184 PDB-coded rows: orthorhombic 46, monoclinic 44, tetragonal 30, trigonal 21, hexagonal 17, cubic 13, triclinic 13.

  • Pink beam: none of these datasets was collected with pink beam. All 184 PDB-coded rows are deposited as SINGLE WAVELENGTH (_diffrn_radiation.pdbx_diffrn_protocol), and all but 5REO, which leaves the field blank, as monochromatic (pdbx_monochromatic_or_laue_m_l M); 9Q41 is the one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The battery’s pink-beam data are in-house SLS measurements (tag pink-beam in tools/battery/inhouse.json), not on this page.

  • Long cell axes: eleven PDB-coded rows have a deposited cell axis longer than 320 Å - 8V4O, 9ZMU, 9Z72, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL.

The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats natively, and accepts the .img and numeric-suffix (.001) file names they arrive with.

Licences

Each dataset carries the licence of its own deposition, stated on the record page linked above. IRRMC and the SBGrid Data Bank both release under CC0 and both ask that the dataset’s own citation - its DOI - be used; the Zenodo records here are CC0 or CC BY 4.0, as each record states. None of these data are redistributed with Jungfraujoch; this page only records where they came from.

\ No newline at end of file diff --git a/FPGA.html b/FPGA.html new file mode 100644 index 000000000..85c27d77a --- /dev/null +++ b/FPGA.html @@ -0,0 +1,16 @@ + FPGA smartNIC — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA smartNIC

See separate document for installation instructions.

Hardware

Currently supported FPGA is only Xilinx Alveo U55C.

See AMD/Xilinx webpage for card user guide (UG1469). According to the user guide:

Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling.
+

The card needs to be placed in a PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C.

Two variants of the card are available:

  • 100g - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch.

  • 8x10g - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

See network documentation for details of network.

Building firmware

The firmware build targets are generated by CMake only when vivado and vitis_hls are detected in the path, and the Vivado version has to match the one below precisely.

Xilinx Vivado

The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version 2022.2 installed on the machine. Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - specifically newer versions of Vivado will not work.

In addition to the Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary:

  • Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see Xilinx website, to build 100g design.

  • A paid license for the 10G/25G Ethernet Subsystem for Ultrascale+ is necessary to build the 8x10g design. PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams generated by PSI continuous integration pipeline for 8x10g is only allowed for non-commercial use.

HLS compilation

Make HLS routines:

mkdir build
+cd build
+cmake ..
+make hls
+

Synthesis

Create PCIe 100g bitstream with the following command:

mkdir build
+cd build
+cmake ..
+make pcie_100g
+

and 8x10g:

mkdir build
+cd build
+cmake ..
+make pcie_8x10g
+

When Vivado is not present

During CMake execution, the following executables: vivado and vitis_hls must be present in the path. If not, build targets will not be generated, and such or similar error message will show up:

$ make pcie_100g
+make: *** No rule to make target 'pcie_100g'.  Stop.
+

Firmware releases

The firmware is stable and is carried from version to version: the MCS files attached to a release are normally the ones from the release before it (see Release contents). When it does need to change, it is rebuilt with the targets above on a machine with Vivado.

Frame generator

The Jungfraujoch card is equipped with a frame generator. It allows simulating a JUNGFRAU detector without having access to such a system. It sits in parallel with the Ethernet MAC, so it is placed before the network stack and before any processing happening on the card. In the future a redirection will be possible to send the simulated stream through the 100G TX network link. Frame generator is written in HLS and controlled with AXI-Lite.

\ No newline at end of file diff --git a/FPGA_DATA_ANALYSIS.html b/FPGA_DATA_ANALYSIS.html new file mode 100644 index 000000000..5d62b8c29 --- /dev/null +++ b/FPGA_DATA_ANALYSIS.html @@ -0,0 +1 @@ + FPGA data analysis — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA data analysis

Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities.

Pixel mask

Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) and will be excluded from subsequent analysis.

ADU histogram

Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature of unconverted values. This is done on a module-basis and works with bins with 32 ADU width.

For EIGER this can be used as just a histogram procedure.

JUNGFRAU conversion

For JUNGFRAU, module images are converted from ADUs to an energy value and divided by a given number to give units of keV. Result of the operation is rounded to integers.

Pixel thresholding

Pixel range can be specified. Pixels below a minimum threshold will be assigned zero. Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled.

Frame summation

Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together.

Azimuthal integration

To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. Corrections were implemented according to formulas developed by Jensen et al. (J. Synchrotron Rad., 29, 1420-1428, 2022).

Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited to 2048 per detector module. This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit fixed point integer implementation and is tuned for standard SAXS/WAXS range.

As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows.

Spot finding

Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels:

  1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run).

  2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding,

  3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation,

  4. Pixel value - pixels above certain threshold value can be marked as strong,

  5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong,

  6. Connected pixels - strong pixels can be discarded if they are “alone”, so their 8 directly neighboring pixels are not counted as strong pixels.

All the above criteria except the bad-pixel criterion are optional (can be turned off); only pixels that fulfill all enabled criteria are selected as strong pixels.

SNR ratio calculation

Signal-to-noise ratio is calculated for a rectangular area. In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. This is dictated by the data flow within the FPGA. In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. Given the very large box size, approximations are made, for example that N ≈ N-1 in calculating standard deviation.

Region-of-interest (ROI) integration

There are 16 ROIs, and the ROI map holds a 16-bit mask per pixel, so a pixel can belong to any subset of them (including none). For each ROI, sum, sum of squares, max count, and number of valid pixels will be calculated. Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis.

ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal (Q-range with an optional φ-sector) — are also evaluated on CPU by the shared image_analysis/roi/ engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path.

Pixel statistics

The following statistics are collected for each module:

  • Number of masked pixels

  • Number of saturated pixels (excl. masked)

  • Number of error pixels (excl. masked)

  • Sum of valid pixels in the module

  • Minimum value of valid pixels in the module

  • Maximum value of valid pixels in the module

Valid pixels are not masked, not saturated, not error pixels.

Square root compression

Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. Scheme was described in Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020. Pixel value X is replaced with round(sqrt(N*N*X)), i.e. round(N*sqrt(X)), where N is integer constant in range 1 to 16. N is what the host writes to the sqrtmult register; the FPGA squares it before multiplying the pixel value.

\ No newline at end of file diff --git a/FPGA_DESIGN.html b/FPGA_DESIGN.html new file mode 100644 index 000000000..3a95344e4 --- /dev/null +++ b/FPGA_DESIGN.html @@ -0,0 +1 @@ + FPGA data flow — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA data flow

The following steps are performed on FPGA (in the order of operation):

  1. UDP header decoding

  2. SLS detector header decoding

  3. State machine that controls data acquisition (start/stop/cancel)

  4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules

  5. ADU histogram for JUNGFRAU

  6. Mask pixels from missing packets with special value

  7. Reorder lines for EIGER to form a proper module

  8. Mask pixels based on provided pixel mask

  9. JUNGFRAU conversion with gain and pedestal corrections

  10. Threshold: pixel values below a set minimum are zeroed, values above a set maximum saturated

  11. Frame summation (up to 256 frames)

  12. Integration according to predefined map (e.g., 1D azimuthal integration)

  13. Spot finding

  14. ROI calculation

  15. Image lossy compression using N*sqrt(pixel) values

  16. Send images, analysis results and metadata to host memory via PCI Express

Each step has a dedicated core, written in high-level synthesis. Exact operation of cores for data analysis is explained in dedicated document.

\ No newline at end of file diff --git a/FPGA_LICENSE.html b/FPGA_LICENSE.html new file mode 100644 index 000000000..7940c91c6 --- /dev/null +++ b/FPGA_LICENSE.html @@ -0,0 +1,7 @@ + FPGA license — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA license

FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. The license is equivalent of GNU Public License with adaptations for hardware. See OHL webpage for details and FAQs.

CERN Open Hardware Licence Version 2 - Strongly Reciprocal

Preamble

CERN has developed this licence to promote collaboration among hardware designers and to provide a legal tool which supports the freedom to use, study, modify, share and distribute hardware designs and products based on those designs. Version 2 of the CERN Open Hardware Licence comes in three variants: CERN-OHL-P (permissive); and two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this licence, CERN-OHL-S (strongly reciprocal).

The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in unmodified form only.

Use of this Licence does not imply any endorsement by CERN of any Licensor or their designs nor does it imply any involvement by CERN in their development.

1 Definitions

1.1 ‘Licence’ means this CERN-OHL-S.

1.2 ‘Compatible Licence’ means

a) any earlier version of the CERN Open Hardware licence, or

b) any version of the CERN-OHL-S, or

c) any licence which permits You to treat the Source to which it applies as licensed under CERN-OHL-S provided that on Conveyance of any such Source, or any associated Product You treat the Source in question as being licensed under CERN-OHL-S.

1.3 ‘Source’ means information such as design materials or digital code which can be applied to Make or test a Product or to prepare a Product for use, Conveyance or sale, regardless of its medium or how it is expressed. It may include Notices.

1.4 ‘Covered Source’ means Source that is explicitly made available under this Licence.

1.5 ‘Product’ means any device, component, work or physical object, whether in finished or intermediate form, arising from the use, application or processing of Covered Source.

1.6 ‘Make’ means to create or configure something, whether by manufacture, assembly, compiling, loading or applying Covered Source or another Product or otherwise.

1.7 ‘Available Component’ means any part, sub-assembly, library or code which:

a) is licensed to You as Complete Source under a Compatible Licence; or

b) is available, at the time a Product or the Source containing it is first Conveyed, to You and any other prospective licensees

i) as a physical part with sufficient rights and information (including any configuration and programming files and information about its characteristics and interfaces) to enable it either to be Made itself, or to be sourced and used to Make the Product; or ii) as part of the normal distribution of a tool used to design or Make the Product.

1.8 ‘Complete Source’ means the set of all Source necessary to Make a Product, in the preferred form for making modifications, including necessary installation and interfacing information both for the Product, and for any included Available Components. If the format is proprietary, it must also be made available in a format (if the proprietary tool can create it) which is viewable with a tool available to potential licensees and licensed under a licence approved by the Free Software Foundation or the Open Source Initiative. Complete Source need not include the Source of any Available Component, provided that You include in the Complete Source sufficient information to enable a recipient to Make or source and use the Available Component to Make the Product.

1.9 ‘Source Location’ means a location where a Licensor has placed Covered Source, and which that Licensor reasonably believes will remain easily accessible for at least three years for anyone to obtain a digital copy.

1.10 ‘Notice’ means copyright, acknowledgement and trademark notices, Source Location references, modification notices (subsection 3.3(b)) and all notices that refer to this Licence and to the disclaimer of warranties that are included in the Covered Source.

1.11 ‘Licensee’ or ‘You’ means any person exercising rights under this Licence.

1.12 ‘Licensor’ means a natural or legal person who creates or modifies Covered Source. A person may be a Licensee and a Licensor at the same time.

1.13 ‘Convey’ means to communicate to the public or distribute.

2 Applicability

2.1 This Licence governs the use, copying, modification, Conveying of Covered Source and Products, and the Making of Products. By exercising any right granted under this Licence, You irrevocably accept these terms and conditions.

2.2 This Licence is granted by the Licensor directly to You, and shall apply worldwide and without limitation in time.

2.3 You shall not attempt to restrict by contract or otherwise the rights granted under this Licence to other Licensees.

2.4 This Licence is not intended to restrict fair use, fair dealing, or any other similar right.

3 Copying, Modifying and Conveying Covered Source

3.1 You may copy and Convey verbatim copies of Covered Source, in any medium, provided You retain all Notices.

3.2 You may modify Covered Source, other than Notices, provided that You irrevocably undertake to make that modified Covered Source available from a Source Location should You Convey a Product in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source. In each case subsection 3.3 shall apply.

  You may only delete Notices if they are no longer applicable to
+  the corresponding Covered Source as modified by You and You may
+  add additional Notices applicable to Your modifications.
+  Including Covered Source in a larger work is modifying the
+  Covered Source, and the larger work becomes modified Covered
+  Source.
+

3.3 You may Convey modified Covered Source (with the effect that You shall also become a Licensor) provided that You:

a) retain Notices as required in subsection 3.2;

b) add a Notice to the modified Covered Source stating that You have modified it, with the date and brief description of how You have modified it;

c) add a Source Location Notice for the modified Covered Source if You Convey in circumstances where the recipient does not otherwise receive a copy of the modified Covered Source; and

d) license the modified Covered Source under the terms and conditions of this Licence (or, as set out in subsection 8.3, a later version, if permitted by the licence of the original Covered Source). Such modified Covered Source must be licensed as a whole, but excluding Available Components contained in it, which remain licensed under their own applicable licences.

4 Making and Conveying Products

You may Make Products, and/or Convey them, provided that You either provide each recipient with a copy of the Complete Source or ensure that each recipient is notified of the Source Location of the Complete Source. That Complete Source is Covered Source, and You must accordingly satisfy Your obligations set out in subsection 3.3. If specified in a Notice, the Product must visibly and securely display the Source Location on it or its packaging or documentation in the manner specified in that Notice.

5 Research and Development

You may Convey Covered Source, modified Covered Source or Products to a legal entity carrying out development, testing or quality assurance work on Your behalf provided that the work is performed on terms which prevent the entity from both using the Source or Products for its own internal purposes and Conveying the Source or Products or any modifications to them to any person other than You. Any modifications made by the entity shall be deemed to be made by You pursuant to subsection 3.2.

6 DISCLAIMER AND LIABILITY

6.1 DISCLAIMER OF WARRANTY – The Covered Source and any Products are provided ‘as is’ and any express or implied warranties, including, but not limited to, implied warranties of merchantability, of satisfactory quality, non-infringement of third party rights, and fitness for a particular purpose or use are disclaimed in respect of any Source or Product to the maximum extent permitted by law. The Licensor makes no representation that any Source or Product does not or will not infringe any patent, copyright, trade secret or other proprietary right. The entire risk as to the use, quality, and performance of any Source or Product shall be with You and not the Licensor. This disclaimer of warranty is an essential part of this Licence and a condition for the grant of any rights granted under this Licence.

6.2 EXCLUSION AND LIMITATION OF LIABILITY – The Licensor shall, to the maximum extent permitted by law, have no liability for direct, indirect, special, incidental, consequential, exemplary, punitive or other damages of any character including, without limitation, procurement of substitute goods or services, loss of use, data or profits, or business interruption, however caused and on any theory of contract, warranty, tort (including negligence), product liability or otherwise, arising in any way in relation to the Covered Source, modified Covered Source and/or the Making or Conveyance of a Product, even if advised of the possibility of such damages, and You shall hold the Licensor(s) free and harmless from any liability, costs, damages, fees and expenses, including claims by third parties, in relation to such use.

7 Patents

7.1 Subject to the terms and conditions of this Licence, each Licensor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in subsections 7.2 and 8.4) patent licence to Make, have Made, use, offer to sell, sell, import, and otherwise transfer the Covered Source and Products, where such licence applies only to those patent claims licensable by such Licensor that are necessarily infringed by exercising rights under the Covered Source as Conveyed by that Licensor.

7.2 If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Covered Source or a Product constitutes direct or contributory patent infringement, or You seek any declaration that a patent licensed to You under this Licence is invalid or unenforceable then any rights granted to You under this Licence shall terminate as of the date such process is initiated.

8 General

8.1 If any provisions of this Licence are or subsequently become invalid or unenforceable for any reason, the remaining provisions shall remain effective.

8.2 You shall not use any of the name (including acronyms and abbreviations), image, or logo by which the Licensor or CERN is known, except where needed to comply with section 3, or where the use is otherwise allowed by law. Any such permitted use shall be factual and shall not be made so as to suggest any kind of endorsement or implication of involvement by the Licensor or its personnel.

8.3 CERN may publish updated versions and variants of this Licence which it considers to be in the spirit of this version, but may differ in detail to address new problems or concerns. New versions will be published with a unique version number and a variant identifier specifying the variant. If the Licensor has specified that a given variant applies to the Covered Source without specifying a version, You may treat that Covered Source as being released under any version of the CERN-OHL with that variant. If no variant is specified, the Covered Source shall be treated as being released under CERN-OHL-S. The Licensor may also specify that the Covered Source is subject to a specific version of the CERN-OHL or any later version in which case You may apply this or any later version of CERN-OHL with the same variant identifier published by CERN.

8.4 This Licence shall terminate with immediate effect if You fail to comply with any of its terms and conditions.

8.5 However, if You cease all breaches of this Licence, then Your Licence from any Licensor is reinstated unless such Licensor has terminated this Licence by giving You, while You remain in breach, a notice specifying the breach and requiring You to cure it within 30 days, and You have failed to come into compliance in all material respects by the end of the 30 day period. Should You repeat the breach after receipt of a cure notice and subsequent reinstatement, this Licence will terminate immediately and permanently. Section 6 shall continue to apply after any termination.

8.6 This Licence shall not be enforceable except by a Licensor acting as such, and third party beneficiary rights are specifically excluded.

\ No newline at end of file diff --git a/FPGA_NETWORK.html b/FPGA_NETWORK.html new file mode 100644 index 000000000..d05e5ff54 --- /dev/null +++ b/FPGA_NETWORK.html @@ -0,0 +1 @@ + FPGA network — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA network

The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 the lower port (when PCIe connector is on the bottom). The card FPGA design is offered in two variants 100g and 8x10g. These have different behavior regarding the network:

100g — this variant operates the QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a switch. QSFP28 transceivers are necessary.

8x10g — this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch.

Transceivers

AMD doesn’t provide transceiver compatibility matrix for Alveo U55C. In our experience operating the card we haven’t seen issues with transceivers from various providers (FS.com, Mellanox, Finisar). We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn’t support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven’t encountered such a situation.

Switch configuration

Special care has to be taken for switch operation, given the FPGA core doesn’t support auto-negotiation. It is necessary to configure switch port to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000).

Network LEDs

Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs mean all ports connected, orange LEDs at least one of the ports connected.

Network stack

Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed.

The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. The card won’t respond to broadcast ICMP pings.

Each link can be put in direct mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. This setting should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any 10 Gbit/s link on the same card. Currently direct mode is turned OFF for 100g design and ON for 8x10g design. This can be manually adjusted for each link.

\ No newline at end of file diff --git a/FPGA_PCIE_DRIVER.html b/FPGA_PCIE_DRIVER.html new file mode 100644 index 000000000..283af03f9 --- /dev/null +++ b/FPGA_PCIE_DRIVER.html @@ -0,0 +1,10 @@ + FPGA PCIe driver — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA PCIe driver

Compilation

To compile the kernel module, type:

make
+

Installation

To install kernel module, you need to have root permissions and run:

sudo make install
+

Loading driver into kernel

After installing the kernel driver, it should be possible to insert it into the kernel via:

modprobe jfjoch
+

Ownership of the character devices

By default, character devices /dev/jfjoch<device number> are owned by root (user/group) and are not accessible by others. This means that jfjoch_broker must be running as superuser, which might not be optimal for security reasons in most cases. The behavior can be changed by creating udev rules. Create a file called /etc/udev/rules.d/99-jfjoch.rules with the following content:

KERNEL=="jfjoch*", OWNER="<UNIX username>", GROUP="<UNIX group>"
+

It is OK to provide only group, for example to make the devices accessible by group jungfrau:

KERNEL=="jfjoch*", GROUP="jungfrau"
+

DKMS

To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. For other systems, it is necessary to follow the procedure below, though it is not well tested.

This first requires installing DKMS - for RHEL it is available via EPEL repository:

sudo dnf install dkms
+

Then use the script provided in the driver directory to copy driver code to DKMS directory:

./install_dkms.sh
+

If upgrading the driver, please first remove the current driver from DKMS system:

dkms remove jfjoch -v <version> --all
+

Driver parameters

Currently, there is one driver parameter nbuffers, that defines count of exchange buffers (see below). This can be adjusted in the modprobe operation, for example:

modprobe jfjoch nbuffers=1024
+

Exchange buffers

The parameter defines number of buffers used to exchange data between card and host application. Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. These buffers are used by both card-to-host and host-to-card operations.

Buffers use special allocation, as they are contiguous in physical address space, which helps the FPGA card to transfer all data associated with detector module in two DMA transfers (one data, one metadata). Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in powers of two, 4 MiB is a safe number for one buffer size. A buffer can be mapped into user space by performing the mmap system call on the /dev/jfjoch<device number> character device.

Buffer count can be adjusted by setting nbuffers parameter. There are two considerations for setting optimal value:

  1. For card-to-host transfers, minimal value is roughly <number of threads in receiver> * <number of modules processed by thread; usually equal to number of modules per card>, this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64.

  2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. So minimal count is <number of modules> * (3 + 3 * <number of storage cells>).

Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration.

Known problems

To avoid inconsistent behavior, this driver won’t load if release number differs between the kernel driver and FPGA card.

CMake file

While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. It is not made for actual compilation of the kernel driver and should not be used for that purpose.

Character device access

For each FPGA device a character device is created called /dev/jfjoch<device number>. When the device is opened, two operations are possible:

  • mmap() to map exchange buffers

  • ioctl() to communicate with the card Interfacing should be done through the JungfraujochDevice class in fpga/host_library directory.

Sysfs access

Certain performance counters can be read through sysfs mechanism in the kernel. One needs to cat files in /sys/class/misc/jfjoch<device number>/ directory.

RHEL 9.5+ virtual memory flags

RedHat Enterprise Linux 9.5 backported the vm_flags_set interface from Linux kernel 6.3 while still reporting kernel version 5.14, so a plain kernel-version test picks the wrong branch and the build fails. This is now detected automatically from RHEL_RELEASE_CODE, so the module builds unaided on RHEL 9.5 and later and on the CentOS Stream, Rocky and AlmaLinux equivalents, as well as on distributions that have not backported it. No user action is needed. The HAVE_VM_FLAGS_SET environment variable that earlier releases required is obsolete; it is still honoured if set, but setting it is no longer necessary and the DKMS packaging never passed it anyway.

Which kernel DKMS builds for

The DKMS package builds the module for the kernel it is being installed for, not the one currently running, so a module built while a kernel update is being applied loads correctly after the reboot. Building by hand in fpga/pcie_driver/ still defaults to the running kernel; pass KDIR=/lib/modules/<version>/build (or KVER=<version>) to target another one.

\ No newline at end of file diff --git a/FPGA_SETTINGS.html b/FPGA_SETTINGS.html new file mode 100644 index 000000000..b7fad2623 --- /dev/null +++ b/FPGA_SETTINGS.html @@ -0,0 +1 @@ + FPGA advanced reference — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

FPGA advanced reference

Register map

FPGA setup can be done via registers:

Address

Bits

Meaning

Mode

Notes

0x000000 - 0x00FFFF

Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory)

0x010000

32

Action Control Register

Bit 0 - Action start

R/W

Bit 1 - Action idle

R

Bit 2 - Action cancel

R/W

cleared on reset or action start

Bit 3 - Clear network counters

R/W

cleared on reset

Bit 12:4 - Debug signals (see action_config.v for details)

R

Bit 16 - AXI Mailbox interrupt 0

R

0x010004

32

Reserved

-

0x010008

32

Reserved

-

0x01000C

32

GIT SHA1

R

0x010010

32

Reserved

R

0x010014

32

Reserved

R

0x010018

32

Jungfraujoch FPGA variant

R

0x01001C

32

Reserved

R

0x010020

32

Max. number of supported detector modules

R

constant

0x010024

32

Reserved

R

constant

0x010028

64

Pipeline stalls before writing to host memory

R

reset on action start

0x010030

64

Pipeline stalls before accessing HBM

R

reset on action start

0x010038

32

FIFO status (see action_config.v for details)

R

0x01003C

32

Size of single HBM channel in bytes (default value for the particular card)

R/W

should not be altered for standard operation

0x010040

64

Packets processed by the action

R

cleared on reset or action start

0x010048

64

Valid ethernet packets

R

cleared on reset

0x010050

64

Valid ICMP packets

R

cleared on reset

0x010058

64

Valid UDP packets

R

cleared on reset

0x010060

64

Valid detector packets processed by the card

R

cleared on reset

0x010068

64

Packets flagged as errors by CMAC

R

cleared on reset

0x010070

64

Pipeline stalls before data processing

R

reset on action start

0x010078

64

AXI-beats before accessing HBM

R

reset on action start

0x010080

64

AXI-beats before data processing

R

reset on action start

0x010088

64

AXI-beats before host writer

R

reset on action start

0x010090

64

Last encountered SwissFEL pulse ID

R

cleared on reset

0x010100

32

Spot finder photon count threshold

R/W

0x010104

32

Spot finder signal-to-noise ratio threshold (single-precision float)

R/W

0x010200

64

MAC address source for internal frame generator

R/W

network byte order

0x010208

32

IPv4 address source for internal frame generator

R/W

network byte order

0x01020C

32

Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.)

R/W

0x010210

32

Data collection mode

R/W

Bit 0 - Conversion to photons

Bit 1 - Output extend to 32-bit

Bit 2 - Output is unsigned integer

Bit 3 - Use sq. root lossy compression

Bit 7 - JUNGFRAU fixed G1 mode

Bit 8 - Set to zero values below threshold

Bit 31:16 - Data collection ID (carried with completions)

0x010214

32

Photon energy in keV (single-precision float)

R/W

0x010218

32

Number of frames expected in the data collection (defines termination condition)

R/W

0x01021C

32

Number of storage cells

R/W

0x010220

32

Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.)

R/W

0x010224

32

Coefficient for sq. root compression (need to set bit in data collection mode to apply)

R/W

0x010228

32

Threshold minimum; values below are set to zero (need to set bit in data collection mode)

R/W

0x01022C

32

Threshold maximum; values above are set to the saturated value

R/W

0x030000 - 0x03FFFF

AXI Mailbox for Work Request / Work Completion

See Xilinx PG114 for register map

0x040000 - 0x04FFFF

QuadSPI flash

See Xilinx PG153 for register map

0x050000 - 0x05FFFF

Interrupt controller

See Xilinx PG099 for register map

0x060000 - 0x06FFFF

Load calibration (HLS)

0x070000 - 0x07FFFF

AXI Firewall

See Xilinx PG293 for register map

0x080000 - 0x08FFFF

Frame generator (HLS)

0x090000 - 0x09FFFF

PCIe DMA control

See Xilinx PG195 for register map

0x0A0000 - 0x0AFFFF

I2C clock generator

See Xilinx PG090 for register map

0x0C0000 - 0x0FFFFF

Xilinx Card Management Solution Subsystem

See Xilinx PG348 for register map

0x100000 - 0x10FFFF

MAC 10G / CMAC 100G

See Xilinx PG210/PG203 for register map

0x110000 - 0x11FFFF

MAC 10G

See Xilinx PG210 for register map

0x120000 - 0x12FFFF

MAC 10G

See Xilinx PG210 for register map

0x130000 - 0x13FFFF

MAC 10G

See Xilinx PG210 for register map

0x140000 - 0x14FFFF

MAC 10G

See Xilinx PG210 for register map

0x150000 - 0x15FFFF

MAC 10G

See Xilinx PG210 for register map

0x160000 - 0x16FFFF

MAC 10G

See Xilinx PG210 for register map

0x170000 - 0x17FFFF

MAC 10G

See Xilinx PG210 for register map

0x200000 - 0x20FFFF

Eth/IPv4 network stack for interface #0

0x210000 - 0x21FFFF

Eth/IPv4 network stack for interface #1

0x220000 - 0x22FFFF

Eth/IPv4 network stack for interface #2

0x230000 - 0x23FFFF

Eth/IPv4 network stack for interface #3

0x240000 - 0x24FFFF

Eth/IPv4 network stack for interface #4

0x250000 - 0x25FFFF

Eth/IPv4 network stack for interface #5

0x260000 - 0x26FFFF

Eth/IPv4 network stack for interface #6

0x270000 - 0x27FFFF

Eth/IPv4 network stack for interface #7

0x400000 - 0x47FFFF

64

Address table: decodes handles used by load_calibration and host_writer to DMA addresses

AXI Mailbox

AXI mailbox is used to send work request from host to action, and receive work completions. Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114).

Work request has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Work completion has the following structure:

Bit start

Bit end

Meaning

0

15

Work request ID (handle)

Special values:

65534 - start of data collection

65535 - end of data collection

16

31

Data collection ID

HBM memory

Interface number

Core

Meaning

0-1

jf_conversion

Gain factor G0

2-3

jf_conversion

Gain factor G1

4-5

jf_conversion

Gain factor G2

6-7

jf_conversion

Pedestal G0

8-9

jf_conversion

Pedestal G1

10-11

jf_conversion

Pedestal G2

12-13

integration

Integration map

14-15

integration

Integration weights

16-17

spot_finder_mask

Spot finder resolution

18-19

roi_calc

ROI calculation

20-21

frame_generator

Frame generator

22-27

load_from_hbm

Frame summation

\ No newline at end of file diff --git a/HARDWARE.html b/HARDWARE.html new file mode 100644 index 000000000..9673b0e03 --- /dev/null +++ b/HARDWARE.html @@ -0,0 +1 @@ + Hardware requirements — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Hardware requirements

Operating Jungfraujoch requires the following:

  1. High performance server

  2. FPGA board(s) installed in the server

  3. (optionally) GPU boards

  4. (optionally) 100G switch to connect FPGA and the detector

Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include AMD FPGA boards. Therefore, the two have to be purchased separately. This might have impact on the warranty for the hardware and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn’t take any responsibility for warranty limitations for operating FPGA boards in the server. Having said this - we didn’t encounter any hardware issues so far.

High performance server

PSI is using HPE DL380 Gen11 servers at the moment to operate Jungfraujoch systems. However, this is because of general preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system from any other vendor with similar specification should work as well.

At PSI, the configuration of HPE DL380 Gen11 used to operate 9M pixel detectors at 2 kHz is as follows:

  • 2 x Intel Xeon 8558P

  • 512 GB RAM

  • 2 x Nvidia L4 GPU (for indexing)

  • 1 x Nvidia ConnectX-6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs)

  • Copper 1G/10G network

PCI slots

When ordering the system, check that it can accommodate enough PCIe cards. In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2 x GPU, 1 x network.

Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK.

FPGA

Jungfraujoch is built for AMD/Xilinx U55C (A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported.

Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires using TX delay functionality in the detector.

GPUs

Operating fast-feedback indexer code requires a graphics processing unit from Nvidia. For practical reasons, i.e. power consumption and cost, we chose the inference-grade Nvidia L4 card. In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work.

Offline processing with rugnux has a requirement of its own, unrelated to the frame rate the acquisition side is sized for: one run wants 3-7 GB on the card and up to 14 GB of host RAM, so a server that also reprocesses data needs headroom beyond the online pipeline’s. The sizing table is in Installing Rugnux ▸ Memory.

Network switch

Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs 8x10g variant of the Jungfraujoch FPGA image, which allows 4 JUNGFRAU modules (two 10 Gbit/s links each) to be connected directly to one U55C card.

Such configuration is however impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujoch server. In this case one needs a network switch.

We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. A switch with only 100G ports must support splitting them into 4x10G ports to connect the detector.

\ No newline at end of file diff --git a/HDF5.html b/HDF5.html new file mode 100644 index 000000000..f459fbecc --- /dev/null +++ b/HDF5.html @@ -0,0 +1,18 @@ + HDF5 / NeXus data format — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

HDF5 / NeXus data format

Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be NXmx-compliant. On top of the NXmx application definition, Jungfraujoch records a substantial amount of derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and reusable.

This page documents the file layout and the data fields. The operational behaviour of the writer (running, republishing, file finalisation) is described in jfjoch_writer. The wire format that feeds the writer is described in CBOR messages; fields below frequently correspond one-to-one to CBOR message fields, and that document is a useful companion for their meaning.

1. Motivation: derived metadata and FAIR data

The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both rotation macromolecular crystallography (single- and multi-crystal, including fine-sliced and helical scans) and serial crystallography (stills, grid scans); the same concerns apply to both:

  • Findability. Raw diffraction images carry almost no descriptive metadata about content. Quantities such as background level, number of diffraction spots, or indexing outcome let a user judge the quality and relevance of a dataset before inspecting the raw images.

  • Accessibility at scale. A single experiment can span tens to hundreds of terabytes. Standard retrieval (e.g. HTTP) makes a dataset available but not inspectable — users would otherwise have to download a large fraction of the data just to decide whether it is useful. Compact derived representations make discovery, assessment and reuse feasible.

Because Jungfraujoch couples acquisition with real-time analysis used to steer experiments, transparency and reproducibility of that analysis matter. As a minimum the writer therefore preserves spot-finding and indexing results together with the filters that were applied, and it can retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse.

Two complementary layouts: per-image spots vs. a reflection table

Jungfraujoch stores analysis products in two shapes, matching how each is accessed.

Per-image spot finding / indexing. Spot finding and indexing are inherently image-centric — the natural query is “give me the spots for image n” — and this holds for serial stills and for rotation frames alike. For these products Jungfraujoch adopts a layout similar to the Coherent X-ray Imaging (CXI) data bank (Maia, 2012) and the convention understood by CrystFEL: spot properties (position, intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, with each image allocated room for up to a predefined maximum number of spots. These dense arrays are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved without traversing variable-length structures. The cost is some storage overhead for unused slots (padded with sentinels), which is acceptable for the access pattern.

Integrated reflections. Integrated intensities are naturally a dataset-wide table, which is exactly the model of the NeXus NXreflections base class. This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration results (see §4.2 below). We deliberately do not force spot finding/indexing into a single experiment-wide table: across the hundreds of thousands of patterns typical of serial — or fine-sliced rotation — experiments, that would require aggregating the whole experiment before the spots of one image can be read. We encourage the community to develop standardised NeXus application definitions for image-centric crystallography products that combine NeXus interoperability with the access patterns and scale of modern high-throughput experiments.

2. File layout

A run is written as one master file plus, depending on the format, one or more data files:

<prefix>_master.h5             # NXmx master file (metadata + links / virtual datasets)
+<prefix>_data_000001.h5        # data file: images + per-image analysis
+<prefix>_data_000002.h5
+...
+

The master file is produced by writer/HDF5NXmx.cpp; data files by writer/HDF5DataFile.cpp and its plugins (writer/HDF5DataFilePlugin*.cpp). Files are written to a temporary *.<random>.tmp name and renamed on successful close.

Three master-file variants exist (set via file_format):

Format

Value

Master ↔ data linking

NXmxLegacy

1

One external link in /entry/data per data file (data_000001, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0.

NXmxVDS (default)

2

A single virtual dataset /entry/data/data spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+.

NXmxIntegrated

3

No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format.

In legacy/VDS mode, image-indexed analysis arrays live in the data files and are exposed in the master file through external links or virtual datasets; in integrated mode they are written directly into the single file.

Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or bitshuffle + Zstd. Signed integer image datasets carry INTx_MIN as the HDF5 fill value (the “masked / no-data” sentinel); unsigned ones are left at HDF5’s own fill of 0, because every unsigned code is a legitimate count. In the master’s virtual dataset the fill is the error marker for both, so a data file missing beside a VDS master reads as masked rather than as zero counts.

Signed images and the Neggia XDS plugin. Neggia dispatches on the size of the pixel in bytes and always casts to an unsigned type, consulting the signedness of the data only for the pixel mask. A signed 16-bit image is therefore read wrongly: a count of -2 reaches XDS as 65534, and the -32768 error marker as 32768. Signed 32-bit happens to degrade safely, because every negative value ends up above INT32_MAX and is mapped to -1. Use the Jungfraujoch XDS plugin, or the Global Phasing build of Durin, for signed data — see Integration with MX data processing software.

Reprocessing output: <prefix>_process.h5

The offline reprocessing tool rugnux (rugnux/rugnux_cli.cpp) re-runs the full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing dataset and writes its results to a master file named <prefix>_process.h5. This file uses the integrated format, but instead of copying the images its /entry/data/data is a virtual dataset that links back to the original image files (hdf5_source_data → NXmx::LinkToData_ProcessingVDS). The result is a compact, self-describing companion file that holds all the derived analysis (everything in §4) plus a virtual view of the raw images — without duplicating terabytes of data.

This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it was processed, while the /entry/data/data VDS still resolves to the original images when they are available. rugnux can also process an equally-spaced subset of images (start/end/stride), producing a down-sampled reference set.

3. NXmx-standard content

The entries below are part of, or valid base classes for, the NXmx application definition. “NXmx” = listed in the application definition; “base” = a valid field of the relevant NeXus base class (NXdetector, NXsample, NXsource) but not in the NXmx required/recommended subset.

/entry (NXentry)

Field

Std

Notes

definition

NXmx

value "NXmx"

start_time

NXmx

arming time

end_time, end_time_estimated

NXmx

approximate end time

File-level HDF5 attributes file_name, file_time, HDF5_Version are also set.

/entry/source (NXsource), /entry/instrument (NXinstrument)

Field

Std

Units

source/name, source/type

NXmx / base

source/current

base

A

instrument/name

NXmx

/entry/instrument/beam (NXbeam)

Field

Std

Units

incident_wavelength

NXmx

angstrom

incident_wavelength_spread

NXmx

angstrom (only if polychromatic)

total_flux

NXmx

Hz

incident_beam_size

NXmx

m (two elements, x then y; written only when both beam sizes are given)

/entry/instrument/attenuator (NXattenuator)

Field

Std

attenuator_transmission

NXmx

/entry/instrument/detector (NXdetector)

Field

Std

Units

depends_on

NXmx

→ transformations/rot3

beam_center_x, beam_center_y

NXmx

pixel (0.0 = centre of the first pixel, see DETECTOR_GEOMETRY). The PONI, not the direct beam - see below

distance

NXmx

m

count_time, frame_time

NXmx

s

sensor_thickness

NXmx

m

sensor_material

NXmx

description

NXmx

threshold_energy

NXmx

eV (EIGER; written only for a single channel)

x_pixel_size, y_pixel_size

base

m

serial_number

base

bit_depth_readout

NXmx

bit depth of the stored image, not of the detector electronics - see below

saturation_value

NXmx

highest valid value. Read inclusively by NXmx, by the DIALS trusted_range and by the XDS OVERLOAD parameter; a saturated pixel carries the value one above it

underload_value

NXmx

lowest valid value: 0 for an unsigned image, INTx_MIN + 1 for a signed one

flatfield_applied

NXmx

pixel_mask, pixel_mask_applied

NXmx

pixel_mask is [y, x], hard-linked from detectorSpecific/pixel_mask

countrate_correction_applied

NXmx

countrate_correction_lookup_table

NXmx

only when the detector sent one (DECTRIS)

virtual_pixel_interpolation_applied

NXmx

only when the detector reported it (DECTRIS)

number_of_cycles

base

frame-summation factor

Why bit_depth_readout is the image depth

NXmx defines only bit_depth_readout, “how many bits the electronics record per pixel”, and has no field for the depth of the image actually stored. The two differ whenever summation is used: the readout stays at the detector’s native width while the summed image must be wider to hold the sum.

Jungfraujoch writes the stored image depth into bit_depth_readout (and the identical value into the non-standard bit_depth_image). The electronic value is a constant of the detector and tells a data consumer nothing, whereas readers do use bit_depth_readout as the width of the stored pixel — DIALS, for instance, derives its masking markers from it and cannot read a 32-bit image without it. Writing the electronic value there would therefore mislead exactly in the case where the two differ.

Note that bit_depth_readout gives the width only. The sign is carried solely by the HDF5 element type of /entry/data/data (and, on the wire, by image_dtype); there is no NXmx field for it.

/entry/instrument/detector/transformations (NXtransformations)

The NXtransformations mechanism (the depends_on chain, transformation_type, vector, offset attributes) is standard. The axis names follow the PyFAI PONI convention chosen by Jungfraujoch (see DETECTOR_GEOMETRY):

Axis

Type

Units

Vector

Depends on

rot3

rotation

rad

(0, 0, -1)

.

rot2

rotation

rad

(1, 0, 0)

rot3

rot1

rotation

rad

(0, -1, 0)

rot2

translation

translation

m

unit vector along the sample→PONI direction

rot1

/entry/instrument/detector/depends_on is translation, and the module’s fast_pixel_direction, slow_pixel_direction and module_offset depend on it in turn. A chain is applied innermost-first, so reading it outwards the detector is placed at its distance and beam centre and then tilted about the sample — which is what makes a tilt pivot about the crystal rather than about the panel corner. The vector values are in NXmx’s McStas frame, which is Jungfraujoch’s internal frame with x and y negated.

The beam centre is encoded in translation (its offset from the sample), not only in the informational beam_center_x/beam_center_y fields. In a _process.h5 written by Rugnux these axes carry the refined detector geometry — the refined beam centre folds into translation and the refined tilt into rot1/rot2/rot3; the broker writes the user-provided geometry unchanged.

beam_center_x/beam_center_y is the PONI; direct_beam_x/direct_beam_y is the beam

beam_center_x/beam_center_y is the PONI - the foot of the perpendicular dropped from the sample onto the detector plane. On a tilted detector that is not where the undeflected beam lands: the two points are distance * tan(tilt) / pixel_size apart, which is around 8 px on a real in-house setup and grows with the distance.

Most programs that ask for “the beam centre” mean the point the beam lands on - XDS’s ORGX/ORGY among them - so Jungfraujoch writes that point out as well:

Dataset

Units

/entry/instrument/detector/detectorSpecific/direct_beam_x

pixel

/entry/instrument/detector/detectorSpecific/direct_beam_y

pixel

They are computed from the same beam centre, distance and rot1/rot2/rot3 written beside them, so the file cannot disagree with itself; on an untilted detector they equal beam_center_x/_y.

The provenance differs by which program wrote the file, though the field does not. In a master written by jfjoch_broker the geometry is the one the user stated, so direct_beam_x/_y is a statement about the user’s geometry; in a _process.h5 written by Rugnux it is the refined geometry, so it is a measurement. A user handing either to XDS should know which of the two they have.

/entry/instrument/detector/module (NXdetector_module)

data_origin, data_size, fast_pixel_direction, slow_pixel_direction, module_offset — all NXmx (fast/slow_pixel_direction and module_offset carry transformation attributes). The two pixel-direction vectors carry the discrete image orientation (mirror in Y, multiples of 90° about the beam); for a detector that this system assembled itself, they are the McStas form of the internal +x and +y, i.e. (-1, 0, 0) and (0, -1, 0).

/entry/sample (NXsample)

Field

Std

Units / notes

name

NXmx

depends_on

NXmx

points at the innermost axis of the sample chain, or . for stills

temperature

NXmx

K

transformations/ (NXtransformations)

NXmx

the sample chain, written in mounting order; hard-linked as /entry/sample/goniometer

unit_cell

base

[a, b, c, α, β, γ]

space_group_number

base

International Tables number

space_group

base

extended Hermann-Mauguin name, e.g. P 43 21 2, R 3:H — this is the field that carries the setting, and the one the reader takes the group from; the number alone always reads back as the reference setting

ub_matrix

base

[1, 3, 3], Angstrom⁻¹

The chain is written from the base outwards, so the innermost axis — the one depends_on names — is the one nearest the sample. It may hold, in that order: the grid-scan translations grid_scan_x and grid_scan_y, the spindle, and a Smargon head’s chi and phi. A grid scan and a goniometer axis are not alternatives; both can be present.

A grid scan is collected at a stationary spindle, and the angle it stood at is stated by sending the goniometer axis with a step of 0 — in dataset_settings.goniometer, or as the goniometer map of the CBOR start message. The angle is then written per image as omega (or whatever the axis is named) in the chain above, and read back by reader/. Send no axis and the spindle is still recorded, at 0: NXmx would allow a sample with no goniometer at all (depends_on = "."), but a sample chain of translations alone is not something readers accept, so the placeholder is written whether or not the angle is known. A 0 there means “nobody said”, not “the spindle was at 0”.

A Smargon head position is told apart from the spindle by the equipment_component attribute, which is "smargon" on chi and phi and absent on the spindle. This is load-bearing: a spindle can itself be named phi, and without the attribute a reader would take a head position for the scan axis. chi and phi are written with one value per image even though neither turns, because a reader takes the image count from the innermost axis of the chain: written as scalars, a still recorded at a head position would read back as a single image however many were collected.

For a rotation scan the goniometer axis carries, beyond the per-image angle array <axis>, the Jungfraujoch conveniences <axis>_end, scalar <axis>_range_average and <axis>_range_total, and for helical scans <axis>_helical_x/_y/_z.

/entry/data (NXdata)

data (3-D image stack, [n_images, y, x]) with image_nr_low / image_nr_high attributes. In legacy mode this group instead contains one external link data_000001, … per data file.

4. Extensions beyond NXmx

Everything in this section is outside the NXmx standard. Each group is declared with NX_class = NXcollection (the NeXus-sanctioned container for non-standardised content) unless noted. The per-image arrays are indexed by image number, padded to the run length and filled with a sentinel (NaN for floats, -1/0 for integer indices) where a quantity is absent.

4.1 /entry/MX — spot finding and indexing (CXI-style)

The flagship extension. Spot (“peak”) properties are stored as fixed-size [n_images, max_spots] arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as [n_images] vectors. In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master.

Per-spot arrays [n_images, max_spots]:

Dataset

Units

Meaning

Indexing only

peakXPosRaw, peakYPosRaw

pixel

spot position (raw detector frame)

peakTotalIntensity

photons

spot intensity

peakIceRingRes

spot lies in an ice-ring resolution band

peakH, peakK, peakL

Miller indices of the (indexed) spot

✓

peakDistEwaldSphere

Å⁻¹

distance of the spot from the Ewald sphere

✓

peakIndexed

spot fits the indexing solution

✓

peakLattice

lattice the spot belongs to (-1 = unindexed)

✓

Per-image vectors [n_images]:

Dataset

Units

Meaning

nPeaks

number of spots stored for the image (CXI)

strongPixels

strong-pixel count (first spot-finding stage)

peakCountUnfiltered

spots found before filtering

peakCountLowRes

low-resolution spots

peakCountIceRingRes

spots inside ice-ring bands

peakCountIceRingControl

spots in the ice-free flanks beside those bands, rescaled to their q width - the control for the count above (their ratio, pooled over the run, is the spot-based ice indicator)

peakCountIndexed

spots fitting the indexing solution

imageIndexed

image was indexed (0/1)

indexingLatticeCount

number of lattices found for the image

niggliClass

Niggli class of the indexed Bravais lattice (see International Tables for Crystallography A (2016), Vol. A, Table 3.1.3.1)

bravaisLattice

Bravais lattice short code, e.g. aP, mC, oF, tI, hP, hR, cF

profileRadius

Å⁻¹

crystal profile radius

mosaicity

deg

mosaicity estimate

bFactor

Ų

per-image B-factor estimate

resolutionEstimate

Å

resolution the merged data are predicted to reach, from this image’s spots alone

integratedReflections

number of integrated reflections

bkgEstimate

photons

mean background in the 3–5 Å resolution band

iceRingScore

ratio

strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice)

spindleBlindFraction

fraction (0-1)

how much of a rotation sweep’s blind cone this orientation makes unrecoverable, as a lone-2-fold worst-case bound; NaN = the frame could not be assessed, which automation must treat like a value at or above the 0.5 trigger, never as 0

beam_corr_x, beam_corr_y

pixel

beam-center correction applied during processing

imageScaleFactor

on-the-fly per-image scale factor g

imageScaleCC

on-the-fly scaling correlation coefficient

imageScaleMosaicity

deg

scaling-model mosaicity

sweepQuality

why this image’s stretch of the sweep was flagged — see below

frameDisposition

what became of this image’s observations in the merged data — see below

Per-image lattices: latticeIndexed [n_images, 9] (Å) — the real-space lattice (flattened 3×3); latticeIndexedExtra [n_images, max_extra_lattices, 9] (Å) — additional orientation variants.

Run-level summaries (written into the master /entry/MX at finalisation):

Dataset

Units

Meaning

indexing_algorithm

FFBIDX / FFT (CUDA) / FFT (FFTW)

geom_refinement_algorithm

e.g. beam_center

rotationLatticeIndexed

Å

whole-run rotation-indexing lattice ([9])

rotationLatticeIndexedExtra

Å

additional whole-run lattices ([m, 9])

rotationLatticeNiggliClass

Niggli class of the run lattice

imageIndexedMean

mean indexing rate over the run

bkgEstimateMean

photons

mean background over the run

spindleBlindFractionMean

fraction (0-1)

mean spindleBlindFraction over the frames that had one

spindleLostUniqueFraction

fraction (0-1)

unique reflections (to the run’s resolution limit) the mounting made unmeasurable, exact under the measured point group and indexed orientation; offline (Rugnux) only

iceRingScoreMean

ratio

mean iceRingScore over the run — the single “how icy was this dataset” number (1 = no ice)

indexedLatticeCount

per-image lattice count summary (master). Note: data files use indexingLatticeCount; readers accept either.

reindexMatrix

change of basis from the setting the per-image data are in to the setting of /entry/sample/unit_cell ([9], int32, flattened 3×3, row major) — see below

Reindex matrix. The per-image h, k, l and latticeIndexed are written as each image is processed, in the setting that image was indexed in. The space group is only settled afterwards, by the merge, and settling it can re-seat the lattice into the group’s conventional setting — so /entry/sample/unit_cell, /entry/sample/space_group_number and rotationLatticeIndexed can be in a different setting from the per-image data beside them. reindexMatrix M is the integral change of basis between the two: hkl_cell = M · hkl_written, and the same M takes each per-image lattice across (latticeIndexedExtra is not re-seated and stays as indexed). It is absent when the two settings are the same one, which is the identity — as it is on every file written before Rugnux recorded it. The Jungfraujoch reader applies it, so everything it hands out is already in the cell’s setting; a third-party reader that ignores it will index the reflections in the wrong frame whenever the dataset is present. Written by the offline rugnux path only — the broker never re-seats a lattice — and not carried on the CBOR stream, in the same way as the other offline-only fields.

A --model run can leave the merged reflection files (.mtz/.cif/.hkl) in a different frame from the _process.h5 beside them: the model settles the alternative indexing where nothing else did, and that choice is applied to the merged reflections as they are written. It also names the enantiomorph, but that is a change of space-group label only and moves no reflection. The process file is not rewritten — its per-image reflections went to disk as they were integrated — so it keeps the space group the run itself determined and stays self-consistent with its own data. The two frames describe the same measurements; an enantiomorphic pair merges identically, having the same Laue class and the same absences.

Sweep quality. sweepQuality [n_images] (uint8) says why the stretch of the sweep this image belongs to was flagged as delivering much less than the rest of the run: 0 means it was not, and any other value is a 1-based index into sweepQualityReasons, a string vector written beside it that carries the whole vocabulary, so the codes can be read without this source. The vocabulary is closed and stable — a code is never renamed and never reused — and currently reads no_diffraction, crystal_out_of_beam, weak_diffraction, loss_of_centring, radiation_damage, inconsistent_with_merge; the Rugnux results report defines what each one means. Both datasets are absent unless the sweep-quality diagnostic ran, which needs scaling and merging; their absence therefore means “not looked for”, not “every image clean”. Written by the offline rugnux path only — the broker does not merge — and not carried on the CBOR stream, in the same way as the other offline-only fields (space_group_number, the refined geometry). The condensed, dataset-wide form of the same finding is in <prefix>_report.txt.

Frame disposition. frameDisposition [n_images] (uint8) says what became of the image’s observations: a 0-based index into frameDispositionCodes, written beside it, which reads merged, downgraded, rejected. Where sweepQuality says what was seen over a stretch, this says what was done about it — a rejected image contributed nothing to the merged intensities. Same availability rule as sweepQuality: absent means the diagnostic never ran.

CrystFEL can read the spots directly with:

peak_list = /entry/MX
+peak_list_type = cxi
+

4.2 /entry/reflections — integrated reflections (NXreflections)

Integrated reflections are stored per image as /entry/reflections/image_NNNNNN groups, each declared NX_class = NXreflections. The columns map mostly onto the standard NXreflections base class:

Dataset

Units

NXreflections

Meaning

h, k, l

standard

Miller indices

d

Å

standard

resolution

int_sum

photons

standard

integrated intensity (summation)

int_err

photons

non-standard name

σ of the intensity (standard equivalent: int_sum_errors)

background_mean

photons

standard

mean background under the peak

background_variance

photons²

non-standard

non-signal part of σ², carried to the merge. Absent in files written before it existed; the reader then recovers it from σ² − I

predicted_x, predicted_y

pixel

name standard, units differ

predicted position. NXreflections predicted_x/_y are physical lengths; the pixel datasets are predicted_px_x/_y

observed_x, observed_y

pixel

name standard, units differ

observed centroid (pixels; standard pixel form is observed_px_x/_y)

observed_frame

standard

image number of the reflection

lp

standard

the Lorentz-polarization factor, stored as the reciprocal of the multiplier that is applied. Lorentz x polarization only, which is what the NXreflections name means; the sensor efficiency is qe, beside it

qe

extension

the sensor efficiency, in the same reciprocal convention as lp, so the whole correction applied to a raw count is 1/lp * 1/qe. Absent in files written before it existed; the reader then takes the whole of lp for Lorentz–polarization, which is what those files mean

flight

extension

the flight path between sample and pixel, in the same reciprocal convention as lp and qe, so the whole correction applied to a raw count is 1/lp * 1/qe * 1/flight. It runs the opposite way to qe — at most 1 where qe is at least 1 — because the medium attenuates an oblique reflection where the sensor favours it. Exactly 1 under --flight-path vacuum, and absent in files written before it existed; the reader then takes it as 1

partiality

standard

recorded fraction of the reflection

delta_phi

deg

extension

XDS Δφ: offset from the centre of the current frame

zeta

extension

Lorentz ζ factor (reciprocal-space geometry term)

image_scale_corr

extension

per-image scale correction; I_true = image_scale_corr · int_sum

In the master file these per-image groups are exposed through /entry/reflections external links (VDS/integrated formats).

4.3 /entry/azint — azimuthal integration

Dataset

Shape

Units

Meaning

bin_to_q

[φ_bins, q_bins]

Å⁻¹

q value of each bin

bin_to_two_theta

[φ_bins, q_bins]

deg

2θ of each bin

bin_to_phi

[φ_bins, q_bins]

deg

azimuthal angle of each bin

image

[n_images, φ_bins, q_bins]

per-image integrated profile (NaN for empty bins)

image_std

[n_images, φ_bins, q_bins]

per-bin standard deviation

image_count

[n_images, φ_bins, q_bins]

pixels contributing per bin

map

[y, x]

pixel→bin mapping (master file only)

4.4 /entry/roi — regions of interest (per-image results)

/entry/roi/<roi_name> has one sub-group per configured ROI, holding the per-image result vectors [n_images]. These are written into the data files; in VDS mode they are exposed from the master file through virtual datasets, and in integrated mode they are in the single file. (In legacy mode they remain only in the data files.)

Dataset

Meaning

max

maximum pixel value in the ROI

sum

sum of pixel values

sum_sq

sum of squared pixel values

npixel

number of valid pixels

x, y

intensity-weighted centroid

4.4.1 /entry/roi_defs — ROI definitions (master file)

The dataset-wide ROI definitions (geometry, fixed for the whole acquisition) live in the master file under a separate /entry/roi_defs group — kept apart from /entry/roi above so that older readers, which iterate /entry/roi, are unaffected by these entries. One sub-group /entry/roi_defs/<roi_name> per ROI:

Dataset

Meaning

bit_index

which bit of roi_map (below) marks this ROI

type

box, circle or azim

min_x_pxl, max_x_pxl, min_y_pxl, max_y_pxl

box bounds (type box)

center_x_pxl, center_y_pxl, radius_pxl

circle (type circle)

q_min_recipA, q_max_recipA

Q range (type azim)

phi_min_deg, phi_max_deg

azimuthal-angle sector (type azim, omitted for a full ring)

/entry/roi_defs/roi_map [y, x] is a uint16 per-pixel bitmask: bit bit_index is set for every pixel belonging to that ROI, so an ROI’s footprint can be recovered exactly.

4.5 /entry/image — per-image pixel statistics

[n_images] vectors: max_value, min_value (viable min/max, excluding error/saturated pixels), error_pixels, saturated_pixels, pixel_sum. Surfaced in the master file under /entry/image.

4.6 /entry/profiling — per-image timing

[n_images] vectors in seconds: spotFindingTime, indexingTime, integrationTime, refinementTime, processingTime, braggPredictionTime, preprocessingTime, compressionTime, azIntTime, indexAnalysisTime, imageScaleTime.

4.7 /entry/detector — acquisition diagnostics (data file)

A convenience NXcollection in the data file (note: distinct from the standard /entry/instrument/detector). In integrated format these datasets are written under /entry/instrument/detector/detectorSpecific instead.

Dataset

Meaning

timestamp, exptime

per-image timestamp and exposure time

number

image number (original number if image rejection was used)

det_info

JUNGFRAU debug field

storage_cell_image

storage-cell number

rcv_delay, rcv_free_send_buffers

receiver internal diagnostics

packets_expected, packets_received

UDP packets per image

data_collection_efficiency_image

received / expected packet ratio

4.8 /entry/xfel — pulsed-source metadata

[n_images] vectors pulseID and eventCode, written for pulsed sources (e.g. SwissFEL).

4.9 Other collections

Path

Class

Content

/entry/instrument/detector/detectorSpecific

NXcollection

Dectris-style detector metadata + Jungfraujoch fields: x_pixels_in_detector, y_pixels_in_detector, nimages, ntrigger, nimages_collected, nimages_written, data_collection_efficiency, max_receiver_delay, storage_cell_number, storage_cell_delay [ns], software_git_commit, software_git_date, jfjoch_release, jfjoch_writer_release, summation_mode, detect_ice_rings, gain_file_names, data_reduction_factor_serialmx, adu_histogram/, data_collection_efficiency_image, direct_beam_x, direct_beam_y

/entry/instrument/detector/calibration

NXcollection

per-channel pedestal / calibration images (bitshuffle-compressed)

/entry/instrument/fluorescence

NXcollection

XRF spectrum: energy [eV], data

/entry/user

NXcollection

scalar values supplied under header_appendix.hdf5

4.10 Non-standard fields inside the NXmx detector group

A few extension scalars are written inside the otherwise-standard /entry/instrument/detector group for compatibility with existing tooling:

Field

Units

Meaning

detector_distance

m

duplicate of distance (Dectris/Neggia compatibility)

detector_number

detector identifier (Dectris convention)

mirror_y (in detectorSpecific)

whether the stored image is mirrored in Y relative to the raw readout; true is the MX convention (row 0 at the top)

detector_orientation_mirror_y (in detectorSpecific)

whether the stored image is mirrored in Y relative to the frame rot1/rot2/rot3 are stated in — a different setting from mirror_y, and one that changes no pixel

detector_orientation_quarter_turns (in detectorSpecific)

multiples of 90° about the beam the stored image is turned by, relative to that same frame (0-3)

direct_beam_x, direct_beam_y (in detectorSpecific)

pixel

where the undeflected beam lands, as opposed to the PONI in beam_center_x/beam_center_y - see above

error_value

masked/error pixel sentinel: UINTx_MAX unsigned, INTx_MIN signed (NXmx has no equivalent). NXmx underload_value is written too: INTx_MIN + 1 for signed, 0 for unsigned

bit_depth_image

stored image bit depth (DECTRIS convention, not NXmx). Equal to bit_depth_readout where that is written, i.e. for unsigned images

acquisition_type

always triggered (Dectris convention)

jungfrau_conversion_applied

JUNGFRAU photon/keV conversion applied

jungfrau_conversion_factor

eV

conversion factor

geometry_transformation_applied

module→full-detector geometry applied

NeXus has no concept of a fill or no-data value — it expects bad pixels to be flagged in pixel_mask, which Jungfraujoch also writes. The in-band sentinel above is a DECTRIS compatibility convention: SIMPLON specifies that masked pixels are flagged with 2^bit_depth_image - 1.

For an unsigned image the sentinel and the saturation code are the same value, so a saturated pixel and a masked one cannot be told apart — the FPGA collapses both onto UINTx_MAX. Signed images keep them separate: INTx_MIN is the marker, INTx_MAX is saturation.

bit_depth_readout is written for unsigned images only. DIALS remaps the top two codes of 2^bit_depth_readout to -1 and -2 whenever the field is present, regardless of the pixel type: for an unsigned image those land below underload_value and are correctly masked, but for a signed one they land inside the trusted range and a saturated pixel would be integrated as a count of -2. Signed images are read correctly without the field; unsigned 32-bit cannot be read at all without it.

4.11 User-supplied metadata: header_appendix and image_appendix

Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two free-form JSON fields in the /start request (broker/jfjoch_api.yaml) provide this without any schema change; both accept any valid JSON:

Field

Carried in

Persisted to HDF5?

header_appendix

the start message, under user_data.user (see CBOR)

no — except the hdf5 sub-object (below)

image_appendix

every image message, as user_data

no

Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, republished analysis, viewers), so they are the recommended channel for facility- or beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has no dedicated API field.

Persisting selected values to HDF5. header_appendix is normally not written to the master file. As an exception, if it contains a key hdf5 whose value is a JSON object of scalars (strings and numbers — no arrays or nested objects), the writer stores each entry under /entry/user/<key>.

For example, a /start request containing:

{
+  "header_appendix": {
+    "proposal": "p20001",
+    "operator": "jdoe",
+    "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 }
+  },
+  "image_appendix": { "trigger_source": "external" }
+}
+

forwards the whole header_appendix as user_data.user on the start message and {"trigger_source": "external"} as user_data on every image message, and writes three scalars into the master file:

/entry/user/beamline          = "X06SA"
+/entry/user/ring_mode         = "top-up"
+/entry/user/attenuator_foils  = 2
+

5. Notes

  • Units are written as the HDF5 units attribute on the dataset (e.g. m, eV, deg, Angstrom, Angstrom^-1, Angstrom^2, pixel, s).

  • Sentinels. Missing per-image values are NaN (floats) or -1/0 (integer indices); image pixels use INTx_MIN / UINTx_MAX.

  • Master vs data file. In legacy/VDS formats the analysis arrays physically live in the data files; the master file links to them (external links in legacy, virtual datasets in VDS). In the integrated format there are no data files and everything is in one place.

  • CXI / CrystFEL. /entry/MX follows the CXI peak-list convention; see CXI file format.

\ No newline at end of file diff --git a/IMAGE_STREAM.html b/IMAGE_STREAM.html new file mode 100644 index 000000000..049442cb2 --- /dev/null +++ b/IMAGE_STREAM.html @@ -0,0 +1,30 @@ + Data streams — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Data streams

The Jungfraujoch process (jfjoch_broker) operates three outputs. All three can be operated/enabled independently. These are:

  • Image - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket)

  • Preview - images with metadata at a reduced frame rate (PUB socket)

  • Metadata - only metadata for all the images, bundled into packages (PUB socket)

Image stream

Images (with metadata) are serialized as CBOR image message. The stream will also include CBOR start message, calibration messages and end message with run metadata.

If file_prefix is not provided for a data collection, images won’t be sent to image stream (or its HDF5/CBOR replacements).

Splitting image stream

Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. All sockets will forward start and end messages. Only the first socket will forward calibration messages and will be marked to write master file.

ZeroMQ image stream

This is using PUSH ZeroMQ socket(s). Multiple receivers must never be connected to one PUSH ZeroMQ socket. ZeroMQ will send the images in a round-robin basis to the receivers. In this case start and end messages will end up only with one receiver. Instead, Jungfraujoch feature of multiple sockets should be used. For ZeroMQ image stream, each writer connects to a different port.

Behavior is as following:

  • Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer.

  • Calibration message is sent to the first socket only, with timeout of 1s.

  • Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped).

  • End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded.

The format is generally interchangeable with DECTRIS Stream2 format.

ZeroMQ configuration

ZeroMQ image stream is configured in the broker JSON configuration file under the zeromq section (schema zeromq_settings):

{
+  "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"],
+  "send_watermark": 100,
+  "send_buffer_size": 67108864,
+  "writer_notification_socket": "tcp://192.168.0.1:*"
+}
+

  • image_socket: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (tcp://, ipc://). 0.0.0.0 binds on all network interfaces.

  • send_watermark (optional): ZeroMQ send high-water mark (number of outstanding messages per socket).

  • send_buffer_size (optional): OS-level send buffer size for the ZeroMQ socket.

  • writer_notification_socket (optional): see Writer notification socket below.

TCP/IP image stream

This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication between the data collection and the writer, and is therefore more robust than ZeroMQ.

For TCP/IP image stream, Jungfraujoch listens on a single TCP port and all writers connect to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic KEEPALIVE frames when no data collection is active to detect dead connections; writers are expected to respond with a KEEPALIVE pong.

Using * as port number (e.g. tcp://127.0.0.1:*) is supported — the OS assigns a free port.

Payloads for PREFLIGHT, START, DATA, CALIBRATION and END frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages.
ACK, CANCEL, KEEPALIVE and BUSY are control frames (no CBOR payload).

The data collection lifecycle on each connection follows: PREFLIGHT → START → CALIBRATION (socket 0 only) → DATA (repeated) → END

If a START ACK fails on any connection, Jungfraujoch sends CANCEL to all already-started connections and rolls back.

For each frame:

  1. Read one TcpFrameHeader (fixed size, 64-byte aligned).

  2. Validate magic (0x4A464A54 / "JFJT") and version (4). Both ends reject a frame of any other version, so writer and broker must be of the same release.

  3. Read payload_size bytes (if non-zero).

Pre-flight

Before a data collection is started - before the detector is armed - Jungfraujoch sends a PREFLIGHT frame on every connection and waits for its ACK. It carries the same CBOR start message a START would, and asks the writer one question: could this run be written? The writer checks the output path, creates the output directory, and checks that no output file is in the way; it opens nothing, writes nothing, and does not change its state. A run that would fail on the first file it wrote is therefore refused while refusing it is free, with the writer’s own message reported to the client by /start and /wait_until_running.

write_master_file is assigned exactly as it is for START (connection index 0), and the writer that owns the master file is the one that answers for the output files - the master and every data file the run will write, its siblings’ included.

The PREFLIGHT payload describes the run but carries none of the per-pixel arrays a START does - no pixel mask, no azimuthal-integration map, no ROI map - so it stays small: about 1.4 kB on a JUNGFRAU 9M, against about 540 kB for the START that follows.

A rejected PREFLIGHT is not fatal: nothing was started, so the connection stays usable and the next attempt (a different file prefix, or overwrite set) proceeds on it. The check cannot be exhaustive - a file created in the moment between the pre-flight and the start still fails at the start, and free space and quota are not inspected - so it lowers how often a run fails on its output, it does not remove the case.

When image stream is split into multiple connections:

  • START and END are sent on all connections,

  • CALIBRATION is sent only on connection 0,

  • DATA frames are distributed by file grouping: connection index = (image_number / images_per_file) % num_connections.

TCP/IP configuration

TCP/IP image stream is configured in the broker JSON configuration file under the tcp section (schema tcp_settings):

{
+  "image_socket": "tcp://192.168.0.1:9100",
+  "nwriters": 2,
+  "send_buffer_size": 67108864
+}
+

  • image_socket: listen address in tcp://<IP>:<port> format. 0.0.0.0 binds on all interfaces. * as port selects a random free port.

  • nwriters (optional): maximum number of simultaneous writer connections accepted.

  • send_buffer_size (optional): OS-level SO_SNDBUF size for accepted connections.

ACK handling

ACK handling is mandatory for correct operation:

  • PREFLIGHT must be acknowledged (ack_for=PREFLIGHT) on each connection within 5 seconds, otherwise the collection is not started. A rejected pre-flight (OK clear) carries the reason as error text and does not break the connection.

  • START must be acknowledged (ACK with ack_for=START) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered.

  • END must be acknowledged (ack_for=END) on each connection within 10 seconds for successful completion.

  • CANCEL should be acknowledged during rollback paths (500ms timeout).

  • DATA should be acknowledged for every frame. A DATA ACK with FATAL flag set reports a downstream error (e.g. disk full) which is propagated to jfjoch_broker via Finalize(). A failed DATA ACK does not break the TCP connection on its own — data continues to flow.

  • CALIBRATION is not acknowledged at this time.

  • KEEPALIVE frames are not acknowledged via ACK; the writer responds with a KEEPALIVE pong frame instead.

Keepalive

When no data collection is active, Jungfraujoch sends KEEPALIVE frames approximately every 5 seconds on each persistent connection. Writers should respond with a KEEPALIVE frame (pong). OS-level TCP keepalive is also enabled (TCP_KEEPIDLE=30s, TCP_KEEPINTVL=10s, TCP_KEEPCNT=3) as a secondary safety net. Dead connections are automatically removed from the pool.

Zero-copy transmission

On Linux, large payload transmission (DATA and CALIBRATION frames) can use kernel TCP zero-copy (SO_ZEROCOPY/MSG_ZEROCOPY) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal send() behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread.

Frame types

Value

Name

Purpose

1

START

Start-of-run metadata

2

DATA

One image payload

3

CALIBRATION

Calibration payload

4

END

End-of-run metadata

5

ACK

Acknowledgement / error reporting

6

CANCEL

Cancel run initialization/stream

7

KEEPALIVE

Connection liveness probe/pong

8

BUSY

Writer alive but stalled; carries its FIFO occupancy

9

PREFLIGHT

Dry run before a collection starts: can this run be written?

TCP frame header (TcpFrameHeader)

Field

Type

Description

magic

uint32_t

Protocol magic (0x4A464A54, "JFJT")

version

uint16_t

Protocol version (4)

type

uint16_t

Frame type (see table above)

image_number

uint64_t

Image index for DATA frames

payload_size

uint64_t

Number of payload bytes after header

socket_number

uint32_t

Connection index in split-stream mode

flags

uint32_t

ACK flags (OK, FATAL, HAS_ERROR_TEXT)

run_number

uint64_t

Run identifier

ack_processed_images

uint32_t

In ACK: number of images processed by receiver

ack_code

uint16_t

In ACK: error/status code

ack_for

uint16_t

In ACK: frame type being acknowledged

ack_fifo_occupancy

uint16_t

In ACK: occupancy of input FIFO in the jfjoch_writer

ack_fifo_max_occupancy

uint64_t

In ACK: max occupancy of input FIFO

The header is 64-byte aligned (alignas(64)).

ACK semantics

  • ACK frames use ack_for to indicate which frame type is acknowledged.

  • flags:

    • OK (bit 0): operation accepted/successful,

    • FATAL (bit 1): receiver reports unrecoverable error (primarily for DATA),

    • HAS_ERROR_TEXT (bit 2): ACK payload contains UTF-8 error text.

  • ack_code can be used to categorize errors:

Code

Name

Meaning

0

None

No error

1

StartFailed

START processing failed

2

DataWriteFailed

Image write failed

3

EndFailed

END processing failed

4

DiskQuotaExceeded

Disk quota exceeded

5

NoSpaceLeft

No space left on device

6

PermissionDenied

Permission denied

7

IoError

General I/O error

8

ProtocolError

Protocol-level error

Image stream replacement

Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting “None” image pusher for all the measurements.

Writer notification socket

The writer notification socket is used only with ZeroMQ image stream. Since ZeroMQ is asynchronous, jfjoch_broker does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back.

For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame.

To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with writer_notification_socket entry:

{
+  "writer_notification_socket":"tcp://192.168.0.1:*"
+}
+

Such entry will create PULL socket on 192.168.0.1 network interface listening on one, random TCP port. When data processing is started, the image stream will send CBOR start message. This message will include information on writer_notification_zmq_addr, which needs to be used by downstream code. Since the start message must reference the address of jfjoch_broker host, notification socket should always listen on a particular network interface, and should not be configured with placeholder address 0.0.0.0. It is, however, OK to use placeholder :* for network port, as it will be substituted for the one chosen by ZeroMQ.

For every image stream socket, downstream code must send the following message to the PULL socket:

{
+  "run_number":135,
+  "run_name": "sample_1",
+  "socket_number": 1,
+  "processed_images":250,
+  "ok": true
+}
+

Here run_number, run_name and socket_number must match information from the start message. ok is boolean confirming if the writing process was OK. processed_images is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. If the writing failed, an error message can be included:

{
+  "run_number":135,
+  "run_name": "sample_1",
+  "socket_number": 1,
+  "processed_images": 0,
+  "ok": false,
+  "error": "Permission error"
+}
+

This way errors from the downstream code are propagated to jfjoch_broker.

If writer notification socket is configured, but downstream code doesn’t send proper notification, jfjoch_broker will time out after 60 seconds producing an error message.

Preview stream

Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. Images are serialized as CBOR image message. The stream will also include CBOR start message and end message with run metadata.

This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope with the messages, it will always receive the last generated message (no backlog). For this reason it is also recommended to use the same option on receiver side.

Given PUB socket properties, it is possible to connect multiple viewers to a single socket — all the viewers should receive all the images sent.

Metadata stream

Jungfraujoch can also send pure metadata for the purpose of archiving such information. Metadata are serialized as CBOR metadata message. This is very similar to the image message, but excludes the actual image array and spot positions. As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. Order of images within bundle, as well as the size of the bundle, are not guaranteed. The stream will also include CBOR start message and end message with run metadata.

This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected.

\ No newline at end of file diff --git a/JFJOCH_BROKER.html b/JFJOCH_BROKER.html new file mode 100644 index 000000000..9af824502 --- /dev/null +++ b/JFJOCH_BROKER.html @@ -0,0 +1,126 @@ + jfjoch_broker — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

jfjoch_broker

jfjoch_broker is the main service for the Jungfraujoch application. It is responsible for:

  • Providing user interface via HTTP and OpenAPI

  • Configuring FPGA firmware

  • Building images from FPGA output and forwarding the results over ZeroMQ

External interfaces

Broker operates four external interfaces.

Image stream ZeroMQ PUSH socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream processing. See details here.

Preview stream ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details here.

Metadata stream ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details here.

Configuration, status and results interface HTTP/REST interface described in the OpenAPI format. Description of the API is presented in the OpenAPI specification.

A dataset can be protected: /start takes an optional tokens list, and while the current dataset has any, its statistics (/statistics/data_collection, /result/scan), buffered images (/image_buffer/*.cbor|jpeg|tiff) and plots (/preview/plot*) need Authorization: Bearer <token> and answer 401 otherwise; /statistics omits its measurement block instead. See Security.

Broker configuration

jfjoch_broker requires JSON configuration files. The file is described by OpenAPI structure jfjoch_settings defined in jfjoch_api.yaml file. It is recommended to go through example files in the etc/.

Example configuration (not every section is shown):

{
+  "pcie": [
+    {
+      "blk": "/dev/jfjoch0",
+      "ipv4": "10.1.1.7"
+    },
+    {
+      "blk": "/dev/jfjoch1",
+      "ipv4": "10.1.1.8"
+    }
+  ],
+  "zeromq": {
+    "send_watermark": 100,
+    "send_buffer_size": 1024,
+    "image_socket": [
+      "tcp://1.2.3.4:5000",
+      "tcp://1.2.3.4:5001"
+    ],
+    "writer_notification_socket": "tcp://1.3.4.6:7000"
+  },
+  "instrument": {
+    "source_name": "Swiss Light Source",
+    "source_type": "Synchrotron X-ray Source",
+    "instrument_name": "X06SA",
+    "pulsed_source": false,
+    "electron_source": false
+  },
+  "detector": [
+    {
+      "description": "EIGER 1M",
+      "serial_number": "E1M-01",
+      "type": "EIGER",
+      "high_voltage_V": 150,
+      "udp_interface_count": 1,
+      "module_sync": true,
+      "sensor_thickness_um": 320,
+      "calibration_file": [
+        "gainMaps.bin"
+      ],
+      "hostname": [
+        "e1m-01",
+        "e1m-02"
+      ],
+      "readout_time_us": 3,
+      "sensor_material": "Si",
+      "tx_delay": [
+        0,1
+      ],
+      "base_data_ipv4_address": "10.10.10.50",
+      "standard_geometry": {
+        "nmodules": 1,
+        "gap_x": 8,
+        "gap_y": 36,
+        "modules_in_row": 1
+      },
+      "custom_geometry": [
+        {
+          "x0": 0,
+          "y0": 0,
+          "fast_axis": "Xp",
+          "slow_axis": "Yp"
+        }
+      ],
+      "mirror_y": true
+    }
+  ],
+  "detector_settings": {
+    "frame_time_us": 450,
+    "count_time_us": 0,
+    "internal_frame_generator": false,
+    "internal_frame_generator_images": 1,
+    "detector_trigger_delay_ns": 0,
+    "timing": "auto",
+    "eiger_threshold_keV": 6.0,
+    "jungfrau_pedestal_g0_frames": 2000,
+    "jungfrau_pedestal_g1_frames": 300,
+    "jungfrau_pedestal_g2_frames": 300,
+    "jungfrau_pedestal_g0_rms_limit": 100,
+    "jungfrau_pedestal_min_image_count": 128,
+    "jungfrau_storage_cell_count": 1,
+    "jungfrau_storage_cell_delay_ns": 5000,
+    "jungfrau_fixed_gain_g1": false,
+    "jungfrau_use_gain_hg0": false
+  },
+  "azim_int": {
+    "polarization_factor": -1,
+    "solid_angle_corr": true,
+    "high_q_recipA": 0,
+    "low_q_recipA": 0,
+    "q_spacing": 0
+  },
+  "image_format": {
+    "summation": true,
+    "geometry_transform": true,
+    "jungfrau_conversion": true,
+    "jungfrau_conversion_factor_keV": 0.001,
+    "bit_depth_image": 16,
+    "signed_output": true,
+    "mask_module_edges": true,
+    "mask_chip_edges": true
+  },
+  "image_buffer_MiB": 2048,
+  "receiver_threads": 64,
+  "frontend_directory": "/usr/share/jfjoch/frontend",
+  "image_pusher": "ZeroMQ",
+  "zeromq_metadata": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4357"
+  },
+  "zeromq_preview": {
+    "enabled": true,
+    "period_ms": 1000,
+    "socket_address": "tcp://0.0.0.0:4356"
+  }
+}
+

Setting up a local test for Jungfraujoch

For development, it is possible to set up a local installation of Jungfraujoch. This will work without FPGA installed in the computer and allows testing the Jungfraujoch software layer, including ZeroMQ streaming and file writing.

The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore very low, as fixed-point calculations have a large performance penalty on the CPU. In the CPU simulation mode, one can simulate using only a single FPGA device.

To run the test:

Compile Jungfraujoch with frontend

mkdir build
+cd build
+cmake ..
+make jfjoch_broker
+make frontend
+

Alternatively, on a RHEL8 system, you can use the RPMs generated by the automated pipeline. The jfjoch package alone is enough. In this case - it is necessary to update etc/broker_local.json file with frontend path in /usr/share/jfjoch/frontend.

Start service

Start broker:

cd build/broker
+./jfjoch_broker ../../etc/broker_local.json 5232
+

Run tests

To run the test, a Python script is provided:

cd tests/test_data
+python jfjoch_broker_test.py
+

The script will initialize Jungfraujoch, import test image and start data collection.

Expected result

You can observe online data analysis by opening the following web page: http://localhost:5232. Also, a dataset with images should be written in the build/broker directory.

\ No newline at end of file diff --git a/JFJOCH_VIEWER.html b/JFJOCH_VIEWER.html new file mode 100644 index 000000000..2ce1c0e3f --- /dev/null +++ b/JFJOCH_VIEWER.html @@ -0,0 +1,9 @@ + jfjoch_viewer — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

jfjoch_viewer

jfjoch_viewer is the interactive desktop application of Jungfraujoch. It opens diffraction datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal integration, per-image statistics), and can follow a live data collection by syncing with a running jfjoch_broker over its HTTP interface.

It is a standalone Qt 6 application, distributed pre-built for Linux, Windows and macOS on the Gitea release page, and for Linux also in the Jungfraujoch RPM/APT repositories — see Release contents for what each package contains and what it requires, and Deployment for how to install it. The macOS build needs macOS 13 (Ventura) or newer on an Apple Silicon Mac (M1 and newer; Intel Macs are not supported) and is CPU-only — see Release contents ▸ macOS, which also covers opening it for the first time, as the release is not yet notarized by Apple.

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

On screen; a processing job can write the same files as rugnux

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Functionality

  • Opens HDF5 files written by jfjoch_writer (*_master.h5) and the *_process.h5 files produced by rugnux. It also opens NXmx files written by DECTRIS detectors, though that path has had only limited testing.

  • Opens PILATUS miniCBF, marCCD and SMV rotation sweeps (the formats listed under What Rugnux reads). These store one frame per file, so naming any frame opens the whole sweep it belongs to. A raw frame carries the images and the geometry but no analysis results, so the spot, reflection and per-image plot panels stay empty until something is computed.

  • Runs an embedded data-processing pipeline — the same analysis code as the rest of Jungfraujoch — performing spot finding, indexing and integration on the displayed image, with the result drawn over it. This interactive analysis is not written anywhere.

  • Runs full processing jobs on the open dataset with Analyze dataset, on the same rugnux engine and off the GUI thread. The settings panel’s MX / AzInt / Calib toggle decides what a run does — full analysis, azimuthal integration only, or a detector calibration — over a chosen image range, optionally writing _process.h5 and the merged .mtz/.cif. A finished run becomes a selectable view of the dataset, so several processing runs can be compared against each other, and its merging statistics (or, for a calibration, its fitted geometry) open in their own window; the Processing panel lists the runs and reopens those results. The equivalent rugnux command line can also be copied out to run the same job on a cluster instead.

  • Detector calibration against a powder standard, on the Calib page: pick the calibrant (LaB6, AgBh, CeO2, Si, ice, or the open dataset’s own unit cell) and fit either the image on screen (Guess / Refine detector calibration) or the whole dataset (Analyze dataset, which writes a pyFAI <output prefix>.poni). The whole-dataset fit measures the rings either from the azimuthally-binned profile summed over the run (Rings, the default) or from the pooled spot lists (Spots), and reports PONI x/y, the two tilts and the distance against the header values. Judge it by the radial rms, not the beam-centre sigma: the sigma shrinks with the number of ring points, so a fit that sits a couple of pixels off every ring can still report a small one. Rings needs the run to be integrated in azimuthal sectors — with the AzInt page’s Azimuthal bins below 4 the calibration run raises it to 32, as rugnux --mode calibration does, and says so. Refine detector tilt is ticked by default and fits the two tilts along with the centre and the distance; unticking it holds them where they are, for a calibration meant for a program that cannot express a tilted detector (rugnux --no-refine-tilt). It applies to both buttons and to Analyze dataset.

  • Settings panel for the geometry, unit cell, spot finding, indexing, azimuthal integration, Bragg integration, scaling, powder calibration and a reference dataset — the same settings the CLI takes.

  • Auxiliary windows: image list, dataset metadata, spot list, reflection list, 2D azimuthal-integration image and calibration-image viewer; plus the Inspector (per-image statistics, image features, resolution rings, ROI statistics), the Magnifier below it (three zoom levels: ×64 and ×32 with the pixel values written on the pixels, ×10 without; Pop out moves it to a window of its own) and dataset-info charts.

  • The Inspector’s Image features section decides what the overlay draws — spots, predictions, saturated and highest pixels, the beam stop — including whether the non-indexed spots and the spots that fall on an ice ring are drawn at all.

  • User-mask editing: build a user mask interactively, load one from TIFF (replacing or adding to the current one), save it as TIFF, clear it, or upload it to a connected server.

  • Mouse-driven navigation of the image, the grid scan and the plots — see Mouse shortcuts below, which the viewer also shows under Help ▸ Mouse Shortcuts.

  • Help shows the mouse shortcuts, the acknowledgements and the third-party licenses.

  • Layout presets (View ▸ Image layout / Processing layout / Reset layout) rearrange the docks for looking at images or at processing results.

  • View ▸ Theme picks the light or the dark colour scheme, or Follow system. Following the system needs a desktop that tells Qt its scheme: macOS and Windows do, and so do GNOME and KDE sessions on Linux (elsewhere, QT_QPA_PLATFORMTHEME=xdgdesktopportal reaches the portal setting); a session Qt cannot read - a bare X server, ssh -X - counts as light. The choice is remembered across restarts, and the half-sun toolbar button toggles light and dark directly.

  • View ▸ Font size (or Ctrl++ / Ctrl+-) enlarges the text to 125 % or 150 %, on top of whatever scaling the desktop already applies, and the choice is remembered across restarts. The viewer also follows the desktop’s own text scaling or display scaling on its own; over ssh -X, where no settings daemon delivers it, launch as QT_SCALE_FACTOR=1.5 jfjoch_viewer instead.

A guided tour

Four views cover most of what the viewer is used for. The screenshots show the lysozyme reference sweep of the in-house test set and a raster scan.

Looking at an image

jfjoch_viewer <file> opens the file and shows its first image; File ▸ Open and the toolbar’s open button do the same, and File ▸ Open HTTP connects to a running jfjoch_broker instead.

The general view: diffraction image, inspector and dataset-info plot

The top toolbars step through the images (slider, first/previous/next/last, Jump and Sum for summing consecutive frames) and set the display (foreground limit, Auto contrast, HDR, colour map, font size, theme). The background limit has a slider of its own, shown from View ▸ Background slider or as soon as B + wheel raises it; Auto sets it back to zero. The image in the middle zooms with the wheel and pans by dragging; hovering shows the pixel position, its value and the resolution on the status bar. The Inspector on the right lists the dataset’s metadata and, once an image has been analysed, its spot count, background, indexing result and resolution estimate; the Magnifier under it follows the cursor while Shift is held. On a window too narrow for both, the inspector (with the magnifier) folds away and comes back when the window is widened again. The Dataset info dock at the bottom plots a per-image quantity over the whole sweep - background here; the combo offers spot counts, indexing results, scale factors and more once they exist - and Shift-hover on it loads the hovered image.

A grid scan

A raster scan opens on its two-dimensional map: each cell is one image, coloured by the metric chosen in the combo (spot count by default for a grid scan), so the crystal shows up as the bright region. Shift-hover or double-click a cell to load that image.

A grid scan on its map, with the spot count per position

The Grid button switches between the map and the per-image line plot; a grid scan remembers its own preferred plot, independently of the one used for rotation data.

Processing settings

The Processing dock on the left (the tab next to Files, or View ▸ Processing layout) holds every setting the analysis takes: geometry and unit cell, the goniometer, spot finding, indexing, Bragg integration, scaling, and the reference dataset (MTZ or structure-factor mmCIF) with an optional atomic model to validate the merged data against. The MX / AzInt / Calib switch selects the kind of analysis and its page.

The Processing dock with the spot finding, indexing and reference sections open

Analyze image re-analyses the current image with these settings, now and on every change while it stays pressed; the inspector and the image overlays update at once.

Processing results

Analyze dataset runs the whole sweep through the same pipeline as rugnux (a job dialog takes the image range, the thread count and which files to write, and Copy command gives the equivalent command line for a cluster). The Jobs dock follows the run.

A finished job: the jobs table and the merge statistics window

When it finishes, the job’s graph button opens the merge statistics (completeness, CC1/2, I/sigma per resolution shell, and the text report). The window names the space group with its screw axes subscripted and lists the crystal pathologies the report checks, one light each: green where the check was made and did not fire, red where it fired (its warning in the tooltip), grey where the data could not answer it. The dataset-info combo gains the per-image indexing result, mosaicity, integrated reflections and scale factors of that run, plotted next to the original file’s values.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. On systems with a GPU, use the CUDA build (a separate package variant everywhere: RPM/APT repository, .tgz and Windows installer) for the embedded indexing and integration; the non-CUDA build runs the same pipeline on the CPU at much lower throughput. The CUDA build also runs on a machine without a GPU — see Release contents ▸ CUDA and non-CUDA builds.

The CUDA build needs an NVIDIA driver on the host but no CUDA toolkit — 525.60.13 or newer for the CUDA 12 artefacts (RHEL 8 packages, portable Linux .tgz), 580.65.06 or newer on Linux and an R580 driver on Windows for the CUDA 13 ones (RHEL 9, Ubuntu, Windows installer). The Windows installer and the .tgz are CUDA 13 and CUDA 12 respectively, which also decides the oldest GPU they run on — a V100 needs the CUDA 12 .tgz. See Release contents ▸ GPU generations and the NVIDIA driver.

On a Mac there is no CUDA: the viewer always runs the same pipeline on the CPU, with the FFTW indexer, like the non-CUDA build elsewhere.

Remote displays

The viewer detects a remote display session (ssh -X and the like) and limits how often panning, zooming and live playback repaint, since on such a link every repaint is shipped as pixels. The detection can be overridden in View ▸ Remote display mode or with JFJOCH_VIEWER_REMOTE=0/1. A VNC- or xpra-based remote desktop still transports the viewer far more efficiently than plain X11 forwarding.

Mouse shortcuts

The same list is available in the application under Help ▸ Mouse Shortcuts.

On macOS, Ctrl below means the Command key (⌘); right click is also Ctrl-click, and on a keyboard without them Home / End are Fn+← / Fn+→ and Page Up / Page Down are Fn+↑ / Fn+↓. The Mouse Shortcuts window shows the Mac keys directly.

Diffraction image

Action

Effect

Wheel

Zoom in / out, centred on the cursor

Shift + wheel

Move the foreground (upper contrast limit) in linear steps

Ctrl + wheel

Move the foreground in multiplicative steps (×1.15 per notch)

F held + wheel

Same as Shift + wheel, for as long as F is held

B held + wheel

Move the background (lower contrast limit) in linear steps, for as long as B is held; shows the background slider

A

Apply auto-contrast once (background back to zero); press it again to switch on continuous Auto

Home / End

Jump to the first / last image in the dataset

Page Up / Page Down

Step one image forward / back

Hover

Status bar shows the pixel position, its value and the resolution

Drag

Pan the image

Shift + move

Move the magnifier panel to the cursor; a frame shows the area it covers

Shift + drag

Draw a rectangular ROI

Shift + Ctrl + drag

Draw a circular ROI

Drag an ROI or its handle

Move or resize the selected ROI

Right click

Copy / save the image, fit to view, clear the ROI

Grid scan

Action

Effect

Hover

Status bar shows the image number, the grid position and its value

Shift + hover

Load the image under the cursor while moving over the grid

Double click

Load the image under the cursor

Other views

Action

Effect

2D azimuthal image: double click

Zoom the diffraction image on the corresponding detector position

Dataset-info plot: hover

Status bar shows the image number and the plotted value

Dataset-info plot: Shift + hover

Load the hovered image

Spot / reflection list: double click

Zoom the diffraction image on that spot or prediction

Image list: double click

Load that image

Opening data

  • File ▸ Open (Ctrl+O) — open a local HDF5 file, or any frame of a miniCBF, marCCD or SMV sweep.

  • File ▸ Open HTTP (Ctrl+H) — connect to a jfjoch_broker HTTP endpoint to follow a live collection. The dialog defaults to host localhost and port 8080; these defaults can be overridden with the environment variables JUNGFRAUJOCH_HTTP_HOST and JUNGFRAUJOCH_HTTP_PORT. The scheme box selects http:// or https:// (the latter for a broker behind a TLS proxy), and the Token field takes the dataset’s bearer token when the collection was started with one. The token can also come from JUNGFRAUJOCH_HTTP_TOKEN or from D-Bus (below); what is typed in the dialog overrides both, and nothing is stored between sessions. Without a valid token for a protected dataset the viewer shows nothing and says so on the status bar — no dialog, since a dataset changing hands is the normal reason.

  • Command line — jfjoch_viewer <file> opens a file (or an http://host:port URL) on start-up. --dbus <true|false> (-d) enables or disables the D-Bus interface (default: enabled); --help and --version behave as usual.

D-Bus interface

When enabled, the viewer registers the D-Bus interface ch.psi.jfjoch_viewer, so other processes can drive it. D-Bus is Linux only: the Windows and macOS builds have no D-Bus interface, and --dbus has no effect there.

  • LoadFile(filename, image_number=0, summation=1, token="") — open a file (or an http://host:port URL) and display the given image; a non-empty token is the bearer token of a protected broker dataset.

  • LoadImage(image_number, summation=1) — navigate to an image in the already-open dataset.

  • SetHttpToken(token) — set the dataset token without reloading; the next request carries it.

summation sums that many consecutive images before display. A repeated LoadFile call naming the file that is already open is cheap (it just navigates, like LoadImage) rather than reopening it, but a client stepping through images of a dataset it opened itself should still prefer LoadImage — it needs no filename and avoids the file-identity comparison.

Building from source on Windows

jfjoch_viewer is cross-platform: it builds on Windows 11 with MSVC and the full CUDA GPU path, and on macOS (see below). (The rest of Jungfraujoch — broker, receiver, FPGA host — is Linux-only.) A pre-built installer is published with every release, so building from source is only needed to develop or to change the build options. On Windows the build is automatically restricted to the viewer and the libraries it needs (JFJOCH_VIEWER_ONLY is forced on), and the remaining dependencies are fetched and built automatically (the first configure needs network access).

Verified toolchain — the same one the released installer is built with:

  • Windows 11

  • Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC

  • CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path

  • Qt 6.11 for MSVC (msvc2022_64), including the Qt Charts module — e.g. C:\Qt\6.11.1\msvc2022_64

  • CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of the box — it comes with the C++ workload, so there is nothing extra to install. Any recent standalone CMake (from cmake.org, or the one bundled with Qt in C:\Qt\Tools\CMake_64) works too.

  • Optional: NSIS to build the .exe installer.

Configure and build from an x64 Native Tools Command Prompt for VS 2026 (so cl, nvcc and ninja are on PATH):

cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^
+  -DCMAKE_PREFIX_PATH="C:/Qt/6.11.1/msvc2022_64"
+cmake --build build-win --target jfjoch_viewer
+

Notes:

  • CMAKE_PREFIX_PATH (Qt) is the only required flag. Every other dependency, zlib and Eigen included, is downloaded and built by the configure itself, so nothing else has to be installed.

  • The CUDA toolchain is located automatically from the CUDA_PATH environment variable that the CUDA installer sets (or from nvcc on PATH). Pass -DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe" only if nvcc is installed in a nonstandard location and is not found.

  • For a machine without an NVIDIA GPU, add -DJFJOCH_USE_CUDA=OFF: the viewer then runs the same pipeline on the CPU (FFTW indexer) at lower throughput.

To produce a self-contained installer (bundles the Qt runtime via windeployqt and — on the CUDA build — the cuFFT runtime DLL, so the target host needs neither Qt nor a CUDA toolkit), with NSIS installed:

cd build-win
+cpack
+

The NSIS generator is selected automatically on Windows (no -G needed). What comes out, and how the CUDA and CPU variants are named and told apart, is described in Release contents ▸ Windows installer.

Building from source on macOS

The viewer also builds on macOS, Apple Silicon only and without CUDA; as on Windows, the build is automatically restricted to the viewer and the libraries it needs (JFJOCH_VIEWER_ONLY is forced on) and fetches every other dependency itself. A pre-built disk image is published with every release, so building from source is only needed to develop or to change the build options.

Verified toolchain — the same one the released .dmg is built with:

  • An Apple Silicon Mac; the resulting app needs macOS 13 or newer, whatever the build machine runs

  • Xcode (Apple Clang)

  • Qt 6.11 for macOS, including the Qt Charts module — e.g. ~/Qt/6.11.2/macos from the Qt online installer

  • CMake (e.g. CMake.app from cmake.org, with /Applications/CMake.app/Contents/bin on PATH)

cmake -S . -B build-mac -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$HOME/Qt/6.11.2/macos
+cmake --build build-mac -j$(sysctl -n hw.ncpu) --target jfjoch_viewer
+open build-mac/viewer/jfjoch_viewer.app
+

As on Windows, CMAKE_PREFIX_PATH (Qt) is the only required flag. To produce the disk image — the Qt frameworks and the license notices copied into the app with macdeployqt, packed into jfjoch-viewer-<version>-macos-arm64.dmg — run cpack in the build directory. What comes out is described in Release contents ▸ macOS.

\ No newline at end of file diff --git a/JFJOCH_WRITER.html b/JFJOCH_WRITER.html new file mode 100644 index 000000000..4d4db3214 --- /dev/null +++ b/JFJOCH_WRITER.html @@ -0,0 +1,69 @@ + jfjoch_writer — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

jfjoch_writer

jfjoch_writer is a NeXus-compliant HDF5 file writer.

Acknowledgements

Thanks to Zdenek Matej (MAX IV) and Felix Engelmann (MAX IV) for testing and multiple improvement suggestions.

Running directory

The writer needs to run in the base directory for writing files - file_prefix is always relative to the writer’s running directory. The writer detects and protects against basic security issues, like file_prefix starting with a slash, or starting with ../, or containing /../.

Usage

Writer needs to be started as a background service, with the following command:

jfjoch_writer {options} <address to connect via ZeroMQ to DCU>
+
+Options:
+-T        | --tcp                      Use raw TCP/IP instead of ZeroMQ
+-j<int>   | --nproc=<int>              Number of forks (only with -T)
+-d<path>  | --root_dir=<path>          Root directory for file writing (-R is a deprecated alias)
+-r<int>   | --zmq_repub_port=<int>     ZeroMQ port for PUSH socket to republish images
+-f<int>   | --zmq_file_port=<int>      ZeroMQ port for PUB socket for notifications on finalized files
+-w<int>   | --rcv_watermark=<int>      Receiving ZeroMQ socket watermark (default = 100)
+-W<int>   | --repub_watermark=<int>    Republish ZeroMQ socket watermark (default = 1000)
+-v        | --verbose                  Verbose output
+-h                                     This message
+

for example:

jfjoch_writer -d /data tcp://dcu-address:5400 
+

Status and cancellation

When a data collection is finalized, each writer reports its outcome back to jfjoch_broker over the writer notification socket — a ZeroMQ address the broker passes in the START message (writer_notification_socket in the broker configuration) — as a JSON message with the socket number, run name and number, processed image count, throughput, and on failure an error string. That is how the broker learns that a writer could not write. On the TCP/IP image stream, failures additionally come back in-band as negative acknowledgements (see Data streams).

To stop a writer, send it SIGINT, SIGQUIT, SIGTERM or SIGHUP: it closes the HDF5 files it is writing and exits. This is only for the case where the broker was terminated or disconnected — it is not the normal way to end a data collection, which the broker finishes on its own.

Republish

Republish creates a PUSH socket on the writer, where all the messages are republished for further use by a data analysis pipeline. Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won’t be republished. In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won’t be retried.

Republish functionality is optional, if republish port number is omitted this functionality is not enabled.

Overwriting files

When jfjoch_writer creates a HDF5 file, it first adds suffix .<random>.tmp. Random value depends on current time-stamp and likely will be different from each file of the particular series. After file is all saved and closed, it is renamed to remove the suffix. By default, renaming won’t happen if this would overwrite existing file. However, this behavior can be changed by setting overwrite parameter to true in the file writer configuration.

When the overwrite conflict is reported

An existing output file is a fatal condition (unless overwrite is true). When it is detected depends on whether the transport between the broker and the writer has a back-channel to report the failure before acquisition starts:

  • Direct HDF5 pusher and TCP writer (back-channel available). The conflict is detected at start: the writer that owns the master file checks whether it already exists and refuses to start. The direct pusher raises the error in-process; the TCP writer returns a START-failure acknowledgement. Either way the broker learns immediately and aborts the data collection before the detector is armed — no images are taken and nothing is written. Only the master file is checked up front: in a multi-writer setup the per-image data files are staggered across writers, and checking them at start would make each writer inspect files it never writes (and race the writers that do). Data-file conflicts are instead caught by their owning writer at the final rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker.

  • ZeroMQ writer (no back-channel). The ZeroMQ image stream is fire-and-forget: the writer has no way to tell the broker to stop, and the broker would keep streaming images regardless. The writer therefore does not fail at start. It writes the whole series to the .<random>.tmp files as usual and only fails at the final rename, leaving the .tmp files on disk. This is deliberate: the acquired images are preserved (in .tmp form) rather than being dropped by a writer that aborted mid-stream. Rename the .tmp files by hand, or re-run with overwrite set, to recover them.

Finalized files information

Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure:

{
+  "filename": <string>: HDF5 data file name (relative to writer root directory),
+  "nimages": <int> number of images in the file (counting from 1!),
+  "file_number": <int> number of file within the acquisition,
+  "sample_name": <string> name of sample,
+  "run_name": <string> name of run,
+  "run_number": <int> number of run,
+  "experiment_group": <string> number of p-group / proposal (optional),
+  "user_data": <any json> user_data,
+  "beam_x_pxl": <float> beam center (X) in pixels,
+  "beam_y_pxl": <float> beam center (Y) in pixels,
+  "detector_distance_m": <float> detector distance in m,
+  "detector_height_pxl": <int> detector size (Y) in pixels,
+  "detector_width_pxl": <int> detector size (X) in pixels,
+  "incident_energy_eV": <float> photon energy of the X-ray beam,
+  "pixel_size_m": <float> pixel size in meter (assuming pixel X == Y),
+  "saturation": <int> this count and higher mean saturation,
+  "space_group_number": <int> space group number (optional),
+  "underload": <int> lowest valid count; anything below it is invalid,
+  "unit_cell": <optional> unit cell dimensions in Angstrom/degree {
+    "a": <float>, "b": <float>, "c": <float>,
+    "alpha": <float>, "beta": <float>, "gamma": <float>
+  },
+}
+

user_data is defined as header_appendix in the /start operation in the jfjoch_broker. Other metadata are also carried over from /start operation.

If the header_appendix is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. For example header_appendix of {"param1": "test1", "param2": ["test1", "test2"]}, then the example message will look as follows:

{
+  "filename": "dataset_name_data_000001.h5",
+  "nimages": 1000,
+  "file_number": 0,
+  "sample_name": "my_sample",
+  "run_name": "my_run",
+  "run_number": 25,
+  "experiment_group": "p00001",
+  "beam_x_pxl": 1200,
+  "beam_y_pxl": 1500,
+  "detector_distance_m": 0.155,
+  "detector_height_pxl": 2164,
+  "detector_width_pxl": 2068,
+  "incident_energy_eV": 12400.0,
+  "pixel_size_m": 7.5e-05,
+  "saturation": 32766,
+  "space_group_number": 96,
+  "underload": -32767,
+  "unit_cell": {
+    "a": 78.0,
+    "alpha": 90.0,
+    "b": 78.0,
+    "beta": 90.0,
+    "c": 39.0,
+    "gamma": 90.0
+  },
+  "user_data": {
+    "param1": "test1", 
+    "param2": ["test1", "test2"]
+  }
+}
+

Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled.

HDF5 file structure

Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, integration, azimuthal integration, per-image statistics and timing) stored beyond the NXmx standard. The complete file layout — master vs data files, the three format variants (NXmxLegacy, NXmxVDS, NXmxIntegrated), every NXmx field that is populated and every Jungfraujoch extension — is documented in HDF5 / NeXus data format.

If data collection was configured with a header_appendix containing a key hdf5 whose value is a JSON object of numbers and strings, those entries are written to /entry/user.

Other formats (CBF and TIFF)

Earlier versions could also write Crystallographic Binary File (CBF, miniCBF) and TIFF images. These writers have been removed: Jungfraujoch now writes only NXmx HDF5. The CBF and TIFF values are retained in the file-format enum for wire back-compatibility, but a request to write either format is rejected.

No file option(s)

There are two options to disable writing of files by the writer:

  • Setting file_prefix to empty string - this will disable sending files on ZeroMQ image socket.

  • Setting file format to NoFile - files are streamed over ZeroMQ socket, but jfjoch_writer will not write anything. This can be useful for debugging purposes, or if you only rely on republishing functionality of the jfjoch_writer

\ No newline at end of file diff --git a/LICENSE.html b/LICENSE.html new file mode 100644 index 000000000..38a812df8 --- /dev/null +++ b/LICENSE.html @@ -0,0 +1,20 @@ + License — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

License

Jungfraujoch software is licensed with GPLv3 license. Jungfraujoch FPGA is licensed with CERN OHL-S license (see FPGA license).

GNU GENERAL PUBLIC LICENSE

Version 3, 29 June 2007

Copyright (C) 2007 Free Software Foundation, Inc. https://fsf.org/ Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.

Preamble

The GNU General Public License is a free, copyleft license for software and other kinds of works.

The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program–to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.

When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.

To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.

For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.

Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.

For the developers’ and authors’ protection, the GPL clearly explains that there is no warranty for this free software. For both users’ and authors’ sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.

Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users’ freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.

Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.

The precise terms and conditions for copying, distribution and modification follow.

TERMS AND CONDITIONS

  1. Definitions.

“This License” refers to version 3 of the GNU General Public License.

“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.

“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.

To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.

A “covered work” means either the unmodified Program or a work based on the Program.

To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.

To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.

An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.

  1. Source Code.

The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.

A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.

The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.

The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work’s System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.

The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.

The Corresponding Source for a work in source code form is that same work.

  1. Basic Permissions.

All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.

You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.

Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.

  1. Protecting Users’ Legal Rights From Anti-Circumvention Law.

No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.

When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work’s users, your or third parties’ legal rights to forbid circumvention of technological measures.

  1. Conveying Verbatim Copies.

You may convey verbatim copies of the Program’s source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.

You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.

  1. Conveying Modified Source Versions.

You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:

a) The work must carry prominent notices stating that you modified it, and giving a relevant date.

b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.

c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.

d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.

A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation’s users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.

  1. Conveying Non-Source Forms.

You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:

a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.

b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.

c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.

d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.

e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.

A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.

A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.

“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.

If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).

The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.

Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.

  1. Additional Terms.

“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.

When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.

Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:

a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or

b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or

c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or

d) Limiting the use for publicity purposes of names of licensors or authors of the material; or

e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or

f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.

All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.

If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.

Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.

  1. Termination.

You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).

However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.

Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.

Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.

  1. Acceptance Not Required for Having Copies.

You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.

  1. Automatic Licensing of Downstream Recipients.

Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.

An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party’s predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.

You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.

  1. Patents.

A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor’s “contributor version”.

A contributor’s “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.

Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor’s essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.

In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.

If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient’s use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.

If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.

A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.

Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.

  1. No Surrender of Others’ Freedom.

If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.

  1. Use with the GNU Affero General Public License.

Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.

  1. Revised Versions of this License.

The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.

Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.

If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy’s public statement of acceptance of a version permanently authorizes you to choose that version for the Program.

Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.

  1. Disclaimer of Warranty.

THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.

  1. Limitation of Liability.

IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

  1. Interpretation of Sections 15 and 16.

If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.

END OF TERMS AND CONDITIONS

How to Apply These Terms to Your New Programs

If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.

To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.

<one line to give the program's name and a brief idea of what it does.>
+Copyright (C) <year>  <name of author>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+

Also add information on how to contact you by electronic and paper mail.

If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:

<program>  Copyright (C) <year>  <name of author>
+This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+This is free software, and you are welcome to redistribute it
+under certain conditions; type `show c' for details.
+

The hypothetical commands show w and show c should show the appropriate parts of the General Public License. Of course, your program’s commands might be different; for a GUI interface, you would use an “about box”.

You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see https://www.gnu.org/licenses/.

The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read https://www.gnu.org/licenses/why-not-lgpl.html.

Jungfraujoch exceptions to GPL

As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL.

We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose of simulating FPGA design on CPU.

If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch API it may be distributed under terms of your choosing without being subject to GPL requirements.

\ No newline at end of file diff --git a/NAMING.html b/NAMING.html new file mode 100644 index 000000000..c46abb575 --- /dev/null +++ b/NAMING.html @@ -0,0 +1 @@ + Naming — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Naming

The software is Swiss, and so are its names: both halves of the system are named after places in the Alps that are, in one way or another, about moving a lot of something up a steep mountain as efficiently as possible — usually by train. Throughput, in other words.

Part

Name

What it does

Streaming / acquisition

Jungfraujoch

Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing.

Data processing

Rugnux

Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the rugnux tool).

Jungfraujoch

The Jungfraujoch is a high mountain col in the Bernese Alps, the saddle (Joch is German for “yoke” or “col”) between the peaks Jungfrau and Mönch, at 3,466 m. It is the site of the High Altitude Research Station Jungfraujoch, whose long-running atmospheric measurements are co-operated by the Paul Scherrer Institute — the same institute that develops this software and the JUNGFRAU detector.

The name is also a small piece of word-play. PSI’s JUNGFRAU detector and DECTRIS’s EIGER detector are both named after Bernese Alps peaks (the famous trio is Eiger, Mönch, Jungfrau). The Jungfraujoch — the pass between Jungfrau and Mönch — is where those two detector worlds meet.

And it fits the theme of the whole project: the Jungfraujoch is reached by the Jungfraubahn, whose terminus is the highest railway station in Europe (3,454 m, the “Top of Europe”). It is the closest you can get to that summit in a genuinely high-throughput way — by train, moving crowds up the mountain — which is exactly what the streaming side of this software does with detector frames.

Pronunciation (German): Jungfraujoch ≈ YUNG-frow-yokh. “Jung” as in young, “frau” rhymes with cow, and the final “joch” ends in the guttural ch of Scottish loch or German Bach — not a hard k.

Rugnux

Piz Rugnux is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern Switzerland. (Piz is the Romansh word for “peak”.) It rises above the Albula line of the Rhaetian Railway (Rhätische Bahn), part of the “Rhaetian Railway in the Albula / Bernina Landscapes” — a UNESCO World Heritage Site (Welterbe).

That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude in very little horizontal distance, it corkscrews through a series of helical (spiral) tunnels looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a steep mountain efficiently — the same idea the data-processing side of this software is built around: pushing a large volume of diffraction data through the analysis pipeline.

So the theme is consistent — Swiss mountains, trains, and throughput — while keeping the two subsystems clearly distinct: Jungfraujoch streams, Rugnux processes.

Pronunciation (Romansh): Piz Rugnux ≈ peets roo-NYOOKS. The “gn” is a soft palatal ñ, as in canyon or Italian gnocchi, not two separate sounds.

What is Romansh?

Romansh (Rumantsch) is the fourth national language of Switzerland, alongside German, French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the canton of Graubünden. It survives in several regional idioms, brought together in a standard form called Rumantsch Grischun. Naming the processing engine with a Romansh mountain is a small nod to the least-spoken but no-less-Swiss corner of the country.

\ No newline at end of file diff --git a/OPENAPI.html b/OPENAPI.html new file mode 100644 index 000000000..dea1bd73f --- /dev/null +++ b/OPENAPI.html @@ -0,0 +1,2 @@ + OpenAPI — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

OpenAPI

OpenAPI specs

See document with detailed OpenAPI specs. The spec declares one security scheme, bearerAuth, on the endpoints that expose the current dataset; it applies only while the dataset was started with tokens (see Security).

Python client

Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. For convenience, we provide a Python client as the jfjoch-client PyPI package. To install the client you can use pip tool:

pip install jfjoch-client
+

See API reference from the OpenAPI generator.

\ No newline at end of file diff --git a/OPENAPI_SPECS.html b/OPENAPI_SPECS.html new file mode 100644 index 000000000..abe529dca --- /dev/null +++ b/OPENAPI_SPECS.html @@ -0,0 +1 @@ + OpenAPI specification — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

OpenAPI specification

See document with detailed OpenAPI specs generated with Redocly.

\ No newline at end of file diff --git a/PIXEL_MASK.html b/PIXEL_MASK.html new file mode 100644 index 000000000..d92f403c8 --- /dev/null +++ b/PIXEL_MASK.html @@ -0,0 +1,13 @@ + Pixel mask — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Pixel mask

Mask format

Jungfraujoch generally follows the NXmx format for the pixel mask. The pixel mask is a 32-bit unsigned integer array of the same size as the image. The conditions for masking a pixel are encoded by setting a particular bit to one. This makes it possible to record the reason why a pixel is included in the mask, and several reasons can be recorded for one pixel at the same time.

Bit values are set as follows:

Bit 0 - gap (pixel with no sensor)

Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn’t set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask)

Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization)

Bit 8 - user defined mask

Bit 9 - beam stop shadow (found by rugnux --detect-beam-stop, on by default; see Rugnux). Unlike the other bits this one belongs to the run that found it, not to the dataset: Rugnux clears it at the start of every run, so a mask read back from a file that carries one starts clear. The user mask (bit 8) is left alone.

Bit 10 - defective pixel found on the run’s own frames (rotation data from a counting sensor; see CPU data analysis §1.6): lit above its resolution ring on more frames than one reflection or chance explains, or holding the detector’s error value on most frames.

Bit 30 - module edge (only for PSI systems)

Bit 31 - chip edge interpolated pixel (multipixel)

Custom user mask

Jungfraujoch allows a custom user mask to be uploaded. This happens in two steps. First create the mask in TIFF format:

import numpy as np
+import tifffile as tiff
+
+# Create an array matching a 2068 x 2164 (width x height) image: 2164 rows, 2068 columns
+array = np.zeros((2164, 2068), dtype=np.uint32)
+
+# Mark the pixel at column 400, row 300 with the value 1
+array[300, 400] = 1
+
+# Save the array as a TIFF file
+tiff.imwrite('mask.tiff', array)
+

Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8).

Then upload the mask to Jungfraujoch server:

curl -v http://<jfjoch_broker http address>/config/user_mask.tiff -XPUT --data-binary @mask.tiff
+
\ No newline at end of file diff --git a/PYTHON_CLIENT.html b/PYTHON_CLIENT.html new file mode 100644 index 000000000..1e765f618 --- /dev/null +++ b/PYTHON_CLIENT.html @@ -0,0 +1,6 @@ + OpenAPI Python client — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

OpenAPI Python client

The broker’s REST API has a generated Python client, published on PyPI as jfjoch-client and regenerated from broker/jfjoch_api.yaml by update_version.sh — the YAML is the single source of truth (see OpenAPI).

  • Client README — installation, quick start, and the index of every endpoint and model.

  • DefaultApi — the full endpoint reference, with a generated example per call.

A protected dataset (one started with tokens) is read with the token as the client’s bearer credential:

import jfjoch_client
+api = jfjoch_client.DefaultApi(jfjoch_client.ApiClient(
+    jfjoch_client.Configuration(host="http://localhost:5232", access_token="<token>")))
+api.post_start(jfjoch_client.DatasetSettings(..., tokens=["<token>"]))
+api.get_statistics_data_collection()   # sends Authorization: Bearer <token>
+

The per-model pages are generated as well and are linked from the two pages above. They are built with the site but kept out of the navigation sidebar on purpose — sixty generated reference pages would otherwise be most of it.

\ No newline at end of file diff --git a/RELEASE_CONTENTS.html b/RELEASE_CONTENTS.html new file mode 100644 index 000000000..c55794a61 --- /dev/null +++ b/RELEASE_CONTENTS.html @@ -0,0 +1,2 @@ + Release contents — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Release contents

This page describes what a Jungfraujoch release ships and what each artefact needs on the target machine — which CPU instruction set the binaries were compiled for, which CUDA toolkit they were built against, and which runtime libraries are bundled rather than expected from the host.

The artefacts in the table below are built and published by the continuous-integration pipeline (.gitea/workflows/build_and_test.yml) when a tag is pushed. For how to install and configure the result see Deployment; for the package-repository URLs see Linux package repositories.

Artefacts

Artefact

Distributed via

Contains

.rpm / .deb packages

package repositories

The full server stack: jfjoch (broker, frontend, FPGA and detector tools), jfjoch-writer, jfjoch-viewer (incl. the XDS plugin), jfjoch-driver-dkms, and rugnux (offline analysis, independent of the rest)

jfjoch_viewer-<version>-linux-cuda<major>.tgz, ...-linux-cpu.tgz

Gitea release page

Portable Linux viewer: jfjoch_viewer, its desktop entry, icon and D-Bus service, and the license notices

jfjoch-viewer-<version>-win64-cuda<major>.exe, ...-win64-cpu.exe

Gitea release page

Windows installer for jfjoch_viewer, plus the Qt runtime

jfjoch-viewer-<version>-macos-arm64.dmg

Gitea release page

macOS disk image with jfjoch_viewer.app (Apple Silicon), the Qt runtime and the license notices inside the app

rugnux-<version>-linux-x86_64-cuda<major>.tgz

Gitea release page

Portable Linux rugnux, the offline analysis CLI, and the license notices. One executable

rugnux-<version>-linux-aarch64-cuda<major>.tgz

Gitea release page

The same, cross-built for 64-bit Arm — NVIDIA GH200 and DGX Spark

rugnux-<version>-win64-cuda<major>.zip

Gitea release page

The same for Windows, plus the cuFFT DLL

rugnux-<version>-macos-arm64-cpu.tgz

Gitea release page

The same for macOS on Apple Silicon, CPU only

jfjoch-writer .rpm / .deb

Gitea release page

The writer alone, for a file-writing machine without the rest of the stack

libjfjoch_xds_plugin.so.<version>

Gitea release page

XDS HDF5 read plugin (built on RHEL 8); see Integration with MX software

jfjoch-client

PyPI and the Gitea PyPI index

Generated Python OpenAPI client

Documentation

Read the Docs and the gitea-pages branch

This documentation set

The FPGA firmware (.mcs) images are attached to the release as well. The firmware is stable and is carried from version to version, and is rebuilt with Vivado (see FPGA smartNIC) when it needs to change — so a card keeps its image across a software upgrade unless the release notes say otherwise.

CPU instruction set

The architecture flags live in the CI configuration rather than in CMakeLists.txt, so a site building from source picks its own (x86-64-v4 on an AVX-512 cluster, -march=native, or the plain baseline the compiler defaults to). The released binaries are compiled to a fixed floor:

Release

Flags

Minimum CPU

Linux (all packages, and the portable .tgz)

-march=x86-64-v3 -flto=auto

AVX2 + FMA + BMI2 — Intel Haswell (2013) / AMD Zen (2017) and newer

Windows installer

/arch:AVX

AVX — Intel Sandy Bridge (2011) / AMD Bulldozer and newer

macOS .dmg and rugnux .tgz

none (the compiler’s default Apple Silicon target)

Any Apple Silicon Mac — M1 and newer

The Windows floor is lower because MSVC has no spelling for the x86-64-v2 level; /arch:AVX is the nearest one and implies SSE4.1/4.2, which is what actually matters — without it Eigen has no vectorised round and falls back to a libm call per element. Link-time optimisation is applied on Linux only. The macOS artefacts need no flag: the Apple compiler’s default target is already the M1, the oldest Apple Silicon chip.

A binary will fault with an illegal instruction on a CPU below its floor. If you must run on older hardware, build from source without the flags.

Operating-system floor

The .rpm / .deb packages are built per distribution (RHEL/Rocky 8 and 9, Ubuntu 22.04 and 24.04) and are tied to it. The portable viewer .tgz and the x86_64 rugnux .tgz are built on RHEL 8, the oldest supported distribution, so their glibc floor is low enough to run on any newer Linux — that is what they are for, and why they replace the per-distro packaging of those programs on the release page. The aarch64 rugnux .tgz is the exception: it is cross-built against Ubuntu 24.04, so it needs glibc 2.39 or newer (which DGX OS 7 and any current Arm server distribution have). The Windows installer is built and verified on Windows 11. The macOS artefacts need macOS 13 (Ventura) or newer — the floor of the Qt 6.11 they bundle — and an Apple Silicon Mac; see macOS below.

The portable archives have no top-level directory. They unpack straight into bin/ and share/, so always extract them into a directory of their own (tar xzf … -C /opt/rugnux-<version>) rather than into a working directory.

CUDA and non-CUDA builds

Every Linux and Windows binary artefact is released in two variants, cuda<major> and cpu (the macOS ones exist only as cpu: there is no CUDA on macOS). The CUDA variant adds the GPU fast-feedback indexer (ffbidx), the GPU FFT indexer and GPU image processing; the CPU-only variant runs the same pipeline on the CPU with the FFTW indexer, at much lower throughput.

The CUDA toolkit used is the one on the corresponding build machine: CUDA 12 for the RHEL 8 packages, CUDA 13 for RHEL 9, Ubuntu and Windows. The major version is part of the artefact and repository name, so a download is self-identifying. Building from source needs CUDA 12.8 or newer.

A CUDA build does not require a CUDA machine. Jungfraujoch asks how many CUDA devices are present at start-up and treats “none” (including “no driver installed”) as zero GPUs, falling back to the CPU path. So a CUDA build starts and runs correctly on a machine with no NVIDIA GPU at all. What each artefact has to find at run time differs:

  • Portable Linux .tgz — nothing. The CUDA runtime, the fast-feedback indexer and cuFFT are all linked statically, so each archive is a single executable that depends on nothing but the C and C++ runtimes. On a GPU machine the NVIDIA driver is the only NVIDIA component needed.

  • Windows installer and .zip — the CUDA toolkit ships no static cuFFT for Windows, so the cuFFT DLL is part of the distribution, next to the executable. No CUDA toolkit is needed.

  • .rpm / .deb — these deliberately keep cuFFT dynamic, so that one dependency is managed centrally with the rest of CUDA. Install the cuFFT package alongside, or use the nocuda repositories on a machine where CUDA is not wanted.

Static CUDA linkage makes the Linux CUDA artefacts substantially bigger than the CPU ones, and the Windows cuFFT DLL is ~256 MB — which is the other reason for shipping both variants.

On a machine with an NVIDIA GPU, take the CUDA variant: only that one uses the GPU.

GPU generations and the NVIDIA driver

A CUDA variant carries compiled device code for a fixed set of GPU generations, and which generations those are follows from the CUDA toolkit it was built with. The CUDA runtime is linked statically, so the only NVIDIA component the target machine has to supply is the driver — there is no CUDA-toolkit version requirement on the host.

Artefact

CUDA toolkit

GPU generations

Minimum driver

RHEL 8 packages, portable viewer .tgz, x86_64 rugnux .tgz

12.9

Volta (V100) through Blackwell: sm_70, 75, 80, 86, 89, 90, 100, 120, 121

525.60.13

RHEL 9 and Ubuntu packages, Windows installer, Windows rugnux .zip

13.x

Turing (T4) through Blackwell: the same list without sm_70

580.65.06 (Linux), R580 (Windows)

aarch64 rugnux .tgz

13.x

sm_90 (GH200) and sm_121 (DGX Spark) only

580.65.06

any cpu / nocuda variant, and everything for macOS

—

—

none

The aarch64 build is cross-compiled and verified in CI to be Arm, self-contained and to carry both GPU targets, but it is not exercised on hardware — CI has no GH200 or Spark runner.

A V100 needs the CUDA 12 build. CUDA 13 dropped offline compilation for Volta, and the PTX that a fatbin also carries only ever JIT-compiles forwards, so a CUDA 13 artefact contains nothing a V100 can execute: every kernel launch fails with no kernel image is available for execution on the device. On a V100 host take the RHEL 8 packages or the portable Linux .tgz. Nothing older than Volta is supported.

Newer GPUs never need a newer build — the highest generation in the list ships PTX as well as SASS, which the driver JIT-compiles for a GPU that came out after the release.

The minimum driver above is the floor for the whole CUDA major version, which is what applies here because the CUDA runtime is statically linked (CUDA minor version compatibility). Newer drivers are always fine; they are backward compatible. A driver from the same release as the build toolkit (575.57.08 for the CUDA 12.9 build, 610.43.02 for a CUDA 13.3 one) additionally rules out the single caveat of minor version compatibility — a call into a driver API newer than the installed driver, which fails with cudaErrorCallRequiresNewerDriver.

Windows installer

The Windows artefacts are the jfjoch_viewer installer and the separate rugnux .zip; the rest of Jungfraujoch (broker, receiver, FPGA host, detector control) is Linux-only.

The toolchain bounds of the released installer are:

  • Visual Studio 2026 with the C++ (MSVC) toolset. MSVC is not optional — CUDA on Windows builds through it — and it is what the release is compiled with.

  • CUDA Toolkit 13.3 for the cuda13 variant.

  • Qt 6.11 for MSVC (msvc2022_64), including Qt Charts.

  • Ninja as the generator; every third-party library is fetched and built by the configure.

The installer is generated with NSIS and bundles the Qt runtime (via windeployqt) and, on the CUDA variant, the cuFFT DLL — so the end user installs neither Qt nor a CUDA toolkit. The two variants share an install directory and Start Menu group and replace each other (CUDA is a strict superset); they are told apart by the installer filename and the Add/Remove Programs entry:

Build

Installer file

Add/Remove Programs

CUDA (default)

jfjoch-viewer-<version>-win64-cuda<major>.exe

Jungfraujoch (CUDA)

CPU-only

jfjoch-viewer-<version>-win64-cpu.exe

Jungfraujoch (CPU)

To build the viewer yourself on Windows, see jfjoch_viewer ▸ Building from source on Windows.

macOS

The macOS artefacts are the jfjoch_viewer disk image and the rugnux .tgz; as on Windows, the rest of Jungfraujoch is Linux-only. Both are CPU-only — macOS has no CUDA, so indexing uses the FFTW indexer and the whole pipeline runs on the CPU — and both are built for Apple Silicon only. They need macOS 13 (Ventura) or newer.

Intel Macs are not supported. No Intel (x86_64) code is built for macOS, and Rosetta cannot help here: it runs Intel programs on Apple Silicon, not the other way round.

jfjoch-viewer-<version>-macos-arm64.dmg holds jfjoch_viewer.app; open the image and drag the app onto the Applications shortcut next to it. The app is self-contained: the Qt frameworks are inside it, and so are the license notices (Contents/Resources), which is where a macOS app keeps them.

The release is not notarized yet (that needs an Apple Developer account), so macOS refuses to open a downloaded copy the first time. On macOS 15 and newer: try to open it once, then allow it in System Settings ▸ Privacy & Security with Open Anyway. On macOS 13 and 14, Control-click the app and choose Open. Alternatively, clear the download flag in Terminal:

xattr -dr com.apple.quarantine /Applications/jfjoch_viewer.app
+

The same applies to the rugnux binary from the .tgz — see Installing Rugnux.

The toolchain of the released macOS artefacts is Xcode (Apple Clang) and Qt 6.11 for macOS, including Qt Charts; to build the viewer yourself see jfjoch_viewer ▸ Building from source on macOS.

Licenses

Every package variant carries the project license, the third-party manifest and the verbatim license texts of the bundled dependencies, each package under a directory of its own — share/doc/jfjoch_broker, jfjoch_writer, jfjoch_viewer, jfjoch_rugnux, jfjoch_driver_dkms — so that no two packages claim the same path and they can be upgraded independently. The macOS viewer is the exception: its notices are inside the app, in jfjoch_viewer.app/Contents/Resources. See Third-party software notices.

\ No newline at end of file diff --git a/REPOSITORIES.html b/REPOSITORIES.html new file mode 100644 index 000000000..65a3296f8 --- /dev/null +++ b/REPOSITORIES.html @@ -0,0 +1,5 @@ + Linux package repositories — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Linux package repositories

For convenience, we are providing package repositories, in versions including and excluding CUDA linking. We recommend installing the Jungfraujoch viewer from a nocuda repository (published in the slsdet8 flavour only) and the remaining packages from a cuda12/cuda13 repository.

The repository name encodes two choices: the slsDetectorPackage version the packages were built against (slsdet8 = 8.0.2, slsdet9 = 9.2.0 — it must match the detector firmware) and whether CUDA is linked in. What ends up inside each package, and what it needs on the target machine, is described in Release contents.

RHEL based systems

For RHEL systems we provide the following repositories:

RHEL version

slsDetectorPackage

CUDA

Repository file

8.x

8.0.2

12.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo

8.x

9.2.0

12.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12.repo

8.x

8.0.2

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo

9.x

8.0.2

13.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13.repo

9.x

9.2.0

13.x

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13.repo

9.x

8.0.2

-

https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda.repo

To install the repository, run:

dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo
+

RPMs are signed by the Gitea package registry as they are uploaded. If your system cannot verify the signature, set gpgcheck=0 in the repository file or install with --nogpgcheck.

We provide the following packages in the repository:

  • jfjoch — broker, web frontend, FPGA and detector command-line tools

  • jfjoch-driver-dkms — PCIe kernel-module source, built by DKMS

  • jfjoch-writer — HDF5 writer service

  • jfjoch-viewer — desktop viewer and the XDS plugin

  • Rugnux — offline analysis CLI, independent of the acquisition stack

Note that rugnux is named without the jfjoch- prefix, matching the program and the release archive. Up to 1.0.0-rc.163 it was part of jfjoch-viewer; the package declares that move, so installing it upgrades an older viewer rather than colliding with it.

Ubuntu based systems

For Ubuntu systems, we also provide the following repositories:

sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc
+echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list
+sudo apt update
+

$distribution uses Ubuntu names jammy (22.04) and noble (24.04). $component can be set to either cuda13 or nocuda. Only slsDetectorPackage 8.0.2 is built for Ubuntu.

The same five packages as above are provided: jfjoch, jfjoch-driver-dkms, jfjoch-writer, jfjoch-viewer and rugnux. Up to 1.0.0-rc.160 the first of them was misnamed jfjoch-jfjoch; the current package replaces it, so apt upgrade handles the rename.

Ubuntu packages currently undergo only very limited testing.

\ No newline at end of file diff --git a/RUGNUX.html b/RUGNUX.html new file mode 100644 index 000000000..ff818d342 --- /dev/null +++ b/RUGNUX.html @@ -0,0 +1,24 @@ + Rugnux — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Rugnux

rugnux is the offline crystallographic data-analysis tool of Jungfraujoch — the data-processing half of the system (see Naming for where the name comes from). It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the results to a _process.h5 file, plus reflection files (.mtz/.cif/.hkl) when merging is requested.

It runs the same analysis code as the online and interactive tools, just driven from the command line over a file rather than a live detector stream.

rugnux {<options>} <input.h5>
+

Run it with no arguments to print the usage.

Note. rugnux is under very active development. This page describes the tool and its options at a high level; the authoritative, always-current list of options is the program’s own usage message — run rugnux with no arguments.

Quick start

Four commands cover most of what people ask of rugnux. Each takes the master file of the dataset — one written by Jungfraujoch, a DECTRIS EIGER master, or an NXmx master written by another facility’s toolchain; a PILATUS miniCBF, marCCD or SMV sweep works too (see What Rugnux reads) — and names its output files from -o:

# 1. everything from the data - index, integrate, scale and merge with the defaults
+rugnux -o myrun dataset_master.h5
+
+# 2. with a reference dataset of the same crystal form: it fixes the space group and the cell,
+#    resolves the indexing ambiguity, and hands over its R-free set
+rugnux -o myrun -z reference.mtz dataset_master.h5
+
+# 3. with a known structure: R-work / R-free and sigma_A-weighted 2mFo-DFc / mFo-DFc maps
+rugnux -o myrun --model model.pdb dataset_master.h5
+
+# 4. with the space group and the cell pinned (-S takes either spelling: P43212 or 96)
+rugnux -o myrun -S P43212 -C 79,79,38,90,90,90 dataset_master.h5
+

Parallelism needs no asking for: a run already uses the machine’s threads. -N is there to limit that, or to lift the per-image loop’s default ceiling of 16 workers per GPU.

Nothing more is needed to pick the workflow: a dataset carrying a goniometer axis is processed as a rotation sweep, one without as independent stills, and scaling and merging run by default in both. A rotation run that merges — the default — leaves eight files next to each other:

myrun.mtz            merged intensities + French-Wilson amplitudes, for CCP4 / phenix
+myrun.cif            the same, as mmCIF - the self-describing format, and what to deposit
+myrun.hkl            the scaled observations unmerged, as SHELX HKLF 4 - for SHELXL, SHELXC / ANODE
+myrun_unmerged.mtz   every observation before scaling, for pointless / aimless / careless -
+                     the largest file of the run (--no-export-unmerged skips it)
+myrun_P1.mtz         the same observations merged in P1, so a wrong space-group call can be
+                     re-merged or re-refined without reprocessing (--no-p1-crosscheck skips it)
+myrun_report.txt     what the run determined: cell, space group, statistics, warnings
+myrun_plot.txt       one row per image, for plotting how the crystal behaved over the sweep
+myrun_detector.jpg   the detector with the pixel mask and the beam-stop shadow drawn on it
+

The two MTZ extras are most of the bytes a run writes — worth knowing when sizing a scratch directory for a campaign, and both have off switches.

Read myrun_report.txt first: it says which space group was chosen and on what evidence, how far the data go, and anything that needs attention.

A few things worth knowing before reaching for more flags:

  • The written reflections stop where CC1/2 falls through 0.30. Every reflection file is resolution-trimmed automatically (--resolution-cutoff cc-logistic, one shell past the crossing); --resolution-cutoff off keeps the full measured range, --scaling-high-resolution fixes the limit by hand. A Rugnux file reaching less far than another program’s on the same data is usually this default at work, not lost data.

  • Rotation data are best left de novo. Pinning the cell and space group (recipe 4) is the normal thing to do for serial stills, where the ffbidx indexer needs a cell; on a rotation sweep it tends to degrade low-symmetry cases, so prefer recipe 1 and let the run determine both (see Rotation data). -S takes a Hermann-Mauguin symbol (P43212) or a space-group number (96), whichever is to hand.

  • Anomalous data are there without -A. A rotation merge always keeps the Bijvoet split: a default run’s .mtz carries I(+)/I(-) beside IMEAN, and its .hkl every observation at the index it was measured at — FRIEDELS_LAW= TRUE in the report says how the statistics were counted, not that the signal was averaged away. What -A changes is the counting basis and the error model: each hand becomes a merged observation of its own, so multiplicity, completeness and ⟨I/σ⟩ are counted anomalously and the sigmas are refitted on the Bijvoet-separated merge. Reach for it for anomalous statistics; the signal itself is in the file either way. (Stills merges carry no split by default — there -A is what creates one.)

  • A model names the enantiomorph. Where the data accept the model — it is tested against a null of the same model in random orientations, and MODEL_FIT= in the report says the verdict — --model settles which of P41212 and P43212 the merged reflections are labelled with — a choice no merged intensity can make. It is a label and nothing more: the two groups have the same rotation operations, so no reflection moves, and in particular I(+) and I(-) are left exactly as measured. Whether the model agrees with the data about the hand is then a real question, and the anomalous difference map answers it — a run says so when the density at the model’s atoms comes out inverted.

  • -z and --model overlap but are not the same. A reference MTZ steers the processing from the start; a model scores the merge and settles the frame it is written in — where the data accept it; a model they reject changes nothing. Either resolves an indexing ambiguity, which on serial data decides whether the merged intensities are usable at all.

  • Small molecules need no flag either. Short axes, spots wider than the integration disk and glide-plane space groups are handled by the default run, and myrun.hkl goes straight to SHELXT / SHELXL (Small-molecule data).

  • --scaling-high-resolution <d>, where the resolution is already known, sharpens both the space-group search and the error model.

  • A run wants memory in proportion to what it integrates, not to the detector: 2.5-14 GB of host RAM and 3-7 GB on the card over the datasets measured, both peaking in scaling and merging. Installing Rugnux ▸ Memory has the table and the two flags that lower it. A very large cell needs far more, and a CPU-only build most of all — tens of GB of host RAM (Very large unit cells).

  • Everything else is in Running Rugnux and the full Command-line options.

The rest of the manual

One page per job, so the answer needed is near the top of a short page:

  • What Rugnux does — the pipeline from images to merged reflections, in order. Read this one first.

  • Installing Rugnux — packages, the release archive, GPU drivers, building from source, hardware.

  • What Rugnux reads — will it open your data: NXmx / EIGER masters, PILATUS miniCBF, marCCD and SMV (ADSC, Rigaku d*TREK) sweeps, one sweep per input.

  • Running Rugnux — a first run in detail, rotation, serial and small-molecule data, and every file a run writes.

  • Rugnux with other programs — the reflection-file conventions, the unmerged export, and worked command lines for phenix, REFMAC, POINTLESS / AIMLESS, careless, Phaser, SHELXC/D/E and, for small molecules, SHELXT / SHELXL.

  • The results report — the KEY= value interface, sweep quality and the anisotropy section.

  • Advanced usage — reference data and the indexing ambiguity, model validation, re-merging, and the full command-line option tables.

  • Detector calibration — the geometry from a calibrant’s powder rings (--mode calibration).

  • CPU/GPU data analysis — the algorithms behind all of it.

Where it fits among the three analysis tools

Tool

Mode

Driven by

Output

jfjoch_broker

Online, real-time streaming analysis on FPGA + GPU

HTTP/REST + ZeroMQ

Live results and statistics, images streamed to jfjoch_writer

jfjoch_viewer

Interactive, on-screen exploration

Qt desktop application

On screen; a processing job can write the same files as rugnux

rugnux

Offline batch processing of a stored dataset

Command-line interface

_process.h5, and .mtz/.cif/.hkl when merging

Use rugnux to re-analyse data after acquisition, to experiment with processing parameters, or to produce merged intensities for downstream structure solution.

\ No newline at end of file diff --git a/RUGNUX_ADVANCED.html b/RUGNUX_ADVANCED.html new file mode 100644 index 000000000..bac1ecb0d --- /dev/null +++ b/RUGNUX_ADVANCED.html @@ -0,0 +1,5 @@ + Advanced Rugnux — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Advanced Rugnux

The machinery behind a run that needs more than the defaults: external reference data, model validation, re-merging without re-integration, and the full option tables.

Reference data and the indexing ambiguity

What a reference does (-z)

-z reference.mtz supplies known intensities of the same crystal form — a previously merged dataset, or F-model amplitudes computed from a structure. The reference can equally be a PDB structure-factor file (-z 1abc-sf.cif, SF-mmCIF), gzipped or not; the format is taken from the file’s content. From an mmCIF the first merged reflection block with a usable column is read (the log names it), its columns under the MTZ labels gemmi cif2mtz gives them (IMEAN, FP, FC, …), and its R-free set from _refln.status (f free, o working), or from _refln.pdbx_r_free_flag where the status marks no test set. It is read once, before processing starts, and used for four things:

  • It fixes the space group and the unit cell the run works in, unless -S / -C override them. The cell is a soft reference: indexing may still drift within tolerance, so a small mismatch between reference and data is absorbed rather than rejected.

  • It fixes the axes the files are written on. The group is taken in the setting the reference is written in (P 2 21 21 stays P 2 21 21, not only “number 18”), and once the run has merged, every description of its lattice on the reference’s axes - axis permutations, sign flips, I- against C-centred - and every alternative indexing on top is correlated with the reference; the best one is re-merged and written. REFERENCE_OPERATOR= names the operator (h,k,l where the data were already on the reference’s axes), REFERENCE_CC= and REFERENCE_MATCHED_FRACTION= how well the result matches.

  • It resolves the indexing ambiguity (below) — the one thing the data cannot settle for themselves.

  • It hands over its R-free test set, where the file carries one, so every dataset of a campaign is scored on the same free reflections - in the reference’s frame, and only where the merge matches the reference (CC at least 0.5 over at least half of the merged reflections in its resolution range). A reference that does not is reported (REFERENCE_MISMATCH), and the merge keeps its own test set; REFERENCE_FREE_FLAGS_INHERITED= says which happened.

  • It reports CCref, the correlation of the merged intensities against the reference, in the statistics table. Stills only — the rotation merge never scores itself against the reference, and its table shows - in that column.

A reference is not a scale anchor. Both workflows scale against their own data — scaling images against a foreign dataset injects that dataset’s systematics — so -z never puts the reference’s errors into the intensities. --reference-column picks the column to read where the automatic choice (F-model, else IMEAN/I, else FP/FOBS/F) is not the right one.

For the second of those four jobs — and only that one — an atomic model does as well: --model computes the intensities it needs from the structure. Where a reference dataset exists, prefer it; where only a model does, it resolves the ambiguity just the same.

The indexing ambiguity

Some crystals can be indexed in more than one way, each equally valid geometrically, and each giving different merged intensities. This happens whenever the lattice is more symmetric than the crystal: in P3, P4, P6, P31, C2 and their relatives (merohedral), and also where the cell is metrically more symmetric than the Laue class by accident (pseudo-merohedral, up to 2° of obliquity). The alternatives are related by the crystal’s twin laws — reindexing operators such as k,h,-l.

Nothing in the data breaks the tie: the merge is equally self-consistent either way, so which solution comes out is arbitrary. What that costs depends on the workflow:

  • Rotation. The whole sweep is one lattice, so the whole dataset lands in one indexing, picked at random. The merge itself is sound; it may simply be the other solution from an earlier dataset of the same crystal form, and the two cannot be combined, compared or phased against the same model.

  • Serial stills. Every crystal is indexed independently, so a run mixes both indexings into one merge. That is not a labelling matter — reflections that are not symmetry mates get averaged together, and CC1/2, Rmeas and the anomalous signal all degrade.

Every run that merges tests for the ambiguity, and where it exists and nothing resolves it, says so in the log and in the report’s warnings:

Indexing ambiguity: this cell / space group admits alternative indexing (reindex operator(s):
+-h,-k,l). Serial-stills crystals are indexed in one hand at random, and rugnux can only break this
+against an external reference. WITHOUT one the merge mixes the hands and CC1/2 is degraded - supply
+a reference MTZ (-z) or a model (--model, which needs -C and -S here) to resolve it.
+

Where something does resolve it, the run says so instead — Indexing ambiguity present (reindex operator(s): -h,-k,l); resolved against the supplied model.

How to resolve it:

Situation

What to do

Rotation, a reference dataset exists

-z reference.mtz. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the hkl labels change; the cell does not

Serial stills, a reference dataset exists

-z reference.mtz. Resolved per image, at integration time, by correlating each crystal’s intensities with the reference — so the merge never mixes hands in the first place

Rotation, only a model

--model model.pdb. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (MODEL_FIT= ACCEPTED) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (MODEL_INDEXING_MARGIN_SIGMA). It is the data that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model’s indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand

Serial stills, only a model

--model model.pdb, with the cell and space group given (-C / -S). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be — the ambiguity has to be broken at integration time, and a model can supply the intensities to break it with

Neither

The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame, for stills a degraded one

Two things to know about where the choice lands:

  • --mode scale cannot repair a stills run after the fact. The per-image test happens at integration time, so a _process.h5 whose images were integrated without a reference — an MTZ or a model — has already lost the distinction, and no re-merge brings it back. On rotation data, where the ambiguity is one choice for the whole dataset, --mode scale --model model.pdb does resolve it.

  • The choice reaches the written reflections, not only the R-factors and the maps: the merged .mtz / .cif / .hkl (and _unmerged.mtz, where it is asked for) are written in the indexing the model or the reference settled on, so the file can be refined against that model as it stands.

Two things the indexing ambiguity is not:

  • Not the enantiomorph. P41212 versus P43212 (or P31 versus P32) leaves the merged intensities unchanged, so no amount of data can choose between them and Rugnux never tries — the run reports the pair it cannot separate. A model the data accept names it (MODEL_FIT= ACCEPTED in the report), and the naming is a label only: the written reflections take the model’s space group, no reflection moves, and I(+)/I(-) stay exactly as measured — reindexing by the change of hand would flip every anomalous difference, so it is never done. Whether the model and the data really agree about the hand is answered afterwards by the anomalous difference map, which warns when the density at the model’s atoms comes out inverted. The _process.h5, whose per-image reflections were written as they were integrated, keeps the group the run determined and is left alone.

  • Not twinning. The twin laws are the same operators, but twinning is a property of the crystal — two orientations diffracting at once — and is reported separately in the report’s twinning section. A crystal can have an indexing ambiguity without being twinned, and usually is not.

The algorithms behind both are in CPU/GPU data analysis ▸ Reference data.

The setting the files are written in

The space-group search decides which axes carry the screws and the centring on the cell as it was indexed, whose axes are ordered by length. The files are then written in the standard setting, as XDS and POINTLESS write it (SETTING_SOURCE=STANDARD): P2221 and P21212 with the unique axis on c and a<b; the other orthorhombic groups a<=b<=c; C222 and C2221 with a<b; monoclinic b-unique, C-centred (never I2), least oblique with beta >= 90; triclinic keeps the reduced cell. So a crystal indexed as 52.51 87.87 137.72 in P 2 21 21 is written as 87.87 137.72 52.51 in P 21 21 2.

What the user gives takes precedence, in this order: a reference MTZ (-z, its own setting), a model that fits (--model, its setting), the axis order of a cell given with -C (SETTING_SOURCE=CELL), and the setting of a group given with -S by a non-standard symbol (-S "P 2 21 21", SETTING_SOURCE=SPACE_GROUP). A group given with -S is also placed on the axes its absences name before anything is merged, so -S 18 on a cell whose pure two-fold is its shortest axis puts the screws on the right rows.

The change is a relabelling made after every decision - nothing measured, and no decision, depends on it - and it is applied to everything the run writes: the merged .mtz/.cif/.hkl, _unmerged.mtz, _P1.mtz and the _process.h5 (whose per-image reflections carry the reindex matrix). SETTING_OPERATOR= gives it as a reindexing operator on the indices the space group was determined on, CCP4 style (k,l,h is h’=k, k’=l, l’=h); h,k,l where nothing moved.

Validating against a model (rugnux --model)

Given an atomic model of the same structure, --model model.pdb scales the model structure factors to the merged amplitudes — fitting a flat bulk-solvent contribution and an overall anisotropic B — and reports R-work / R-free and the mean 2mFo−DFc density at the atom centres. It also writes the σA-weighted maps <prefix>_2fofc.ccp4 (2mFo−DFc) and <prefix>_fofc.ccp4 (mFo−DFc), and the map-coefficient MTZ <prefix>_maps.mtz, next to the merged reflections — and, where the merge kept the Bijvoet split (a rotation merge always does), <prefix>_anom.ccp4, the anomalous difference map whose strongest sites the report names (ANOMALOUS_SITE_01…10), and the map’s mean height at the model’s anomalous scatterers - every atom from phosphorus up - as one number for the anomalous signal (ANOMALOUS_SCATTERER_MEAN_SIGMA). The structure itself is not refined; the model is re-fractionalized into the data cell and then placed as one rigid body, so a deposited model from a crystal that is not quite isomorphous still sits where the density is. The placement runs on the GPU where there is one and on the CPU otherwise, with nothing to set; the two agree to rounding, not bit for bit.

Because the model moves, the input file no longer describes these maps, so the model as placed is written as <prefix>_model.cif — the input’s chains, residues, ligands, waters, B-factors, occupancies and anisotropic Us, at the coordinates the maps were computed from, in the same unit cell and space group as <prefix>.mtz beside it. The same coordinates are written as <prefix>_model.pdb as well, because the fragment-screening tools this file exists to feed take a PDB: PanDDA’s per-dataset input is <name>.pdb beside <name>.mtz, and dimple produces that same pair. (The PDB is skipped, with a log line, for a cell its fixed-width format cannot hold; the mmCIF is unconditional.) That is the file to open with the maps, and the one to hand to REFMAC5 or phenix.refine; its starting R-free is the R_FREE= the report quotes. (<prefix>.cif is the merged reflections — hence the separate name.) Both are written whenever the maps are, including for a model the data rejected: the rejection is a result, and it is exactly the case where someone wants to look at the model in the density.

The model may be PDB or mmCIF, gzipped or not, and the format is taken from the file’s own content rather than from its name — a model downloaded as .cif, .pdb, .ent or with no useful extension at all is read the same way. A model that cannot be read, or that has no atoms, no unit cell or no usable space group, does not fail the run: it is logged, and the results report carries a WARNING: Model validation did not run: … line, so a run that silently produced no R-free and no maps cannot be mistaken for one that was never given --model.

Either way the results report carries a 5. MODEL VALIDATION section: R_WORK= / R_FREE= with their reflection counts, R_MODEL_SHELL_SCALED= and MODEL_RADIAL_MISFIT= beside them (the R that compares between two runs, and why the other two do not — see RUGNUX_REPORT), the bulk-solvent and overall scale parameters, the mean 2mFo−DFc density at the atom centres, the reindexing operators the written reflections were brought into the model’s frame with, and MAPS_PREFIX=; or MODEL_VALIDATION= NOT_PERFORMED with MODEL_VALIDATION_REASON= when the model could not be used. A run given no --model has no such section at all.

It is a data-quality lens, independent of the internal statistics: R-free measures the merged intensities against external truth, where CC1/2 and Rmeas only measure them against themselves. It also settles the two things merged intensities alone cannot, in two different ways — though only where the data accept the model first: the same model is refitted, and re-placed, from random orientations about its own centroid, and the real fit has to beat that null (MODEL_FIT= in the report; §14.5). A model that fits no better than its own random placements decides nothing, and the reflection files are byte for byte what a run with no model would have written. The enantiomorph is a relabelling: data merged in P41212 against a P43212 model take the model’s space group as the label they are written under, with no reflection moved and I(+)/I(-) exactly as measured. A merohedral indexing ambiguity — when no reference MTZ has already fixed it — is a reindexing: the candidate with the lowest R-free is kept — where its lead over the runner-up beats the lead a random placement of the same model takes — and applied to the written reflections as well as to the R-factors and the maps; otherwise the data keep the indexing they were merged in. This is distinct from a model written in another description of the lattice — other axes, a different unique axis, I-centred where the run indexed C-centred — or in another point group: there the model is first put into the data’s description (MODEL_CHANGE_OF_BASIS=) to be scored. A model that asserts such a setting is then tested against its random placements like any other claim, and where it fits, the data move instead: the files are written in the model’s setting (SETTING_SOURCE=MODEL), and the validation is made once more on those axes so the maps and _model.pdb match them. Where it does not fit, the files stay in the standard setting. Validation runs before the reflection files, so the .mtz / .cif / .hkl come out in the model’s frame either way — reindexed where the ambiguity decided, and carrying the model’s space group where the hand was adopted. The log names the operator in each case, and for the indexing choice gives the winning R-free together with the runner-up.

Re-scaling and re-merging (rugnux --mode scale)

The scale mode re-scales and merges the already-integrated reflections stored in a _process.h5 file, without re-running spot finding or integration. Use it to re-merge quickly with a different space group, resolution limit, anomalous setting or outlier rejection. It reuses the same -o/-N/-s/-e/-S/-A/-z/--scaling-* options as the full run, and (unlike the full pipeline) does not run a space-group search: it merges in the space group and unit cell the file records, and -S / -C override them. A _process.h5 written before the group was stored carries none, and merges in P1 unless -S says otherwise.

A reference MTZ (-z) is accepted here on stills data, where it fixes the space group and cell, reports CCref and hands over its R-free flags; on rotation data the rotation scaler declines it and the run stops with a message saying so. --model works here exactly as in a full run: on rotation data, where the indexing ambiguity is one choice for the whole dataset, it can settle the enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. What no re-merge can repair is a stills _process.h5 integrated without a reference: there the ambiguity was decided per image, at integration time, and the distinction is gone from the stored reflections.

Where the full run re-seated the lattice — the space group it settled on is in a different setting from the one each image was indexed in — the file records the change of basis as /entry/MX/reindexMatrix, and rugnux applies it on read, so the reflections and the stored cell describe the same frame. An older file that was affected by this cannot be repaired (the matrix is not recoverable after the fact); such a file now stops with a message naming both cells and the exact -S/-C override to merge it in its own setting, instead of failing inside the merge.

Command-line options

General:

Option

Description

-o, --output-prefix <txt>

Output file prefix (default: output)

-N, --threads <num>

Number of worker threads (default, and for any value ≤ 0: all hardware threads). Some stages take fewer, because past a point more workers make them slower: the per-image loop of --mode mx uses at most 16 per GPU unless -N was given a positive value, and first-pass spot finding and the beam-stop pre-scan have ceilings of their own that -N does not lift. Scaling, merging and the space-group search use the full count

-s, --start-image <num>

First image to process (default: 0)

-e, --end-image <num>

Last image to process (default: all)

-t, --stride <num>

Process every n-th image (default: 1)

-v, --verbose

Verbose output

Mode — --mode <name> (default mx):

Value

Description

mx

Full analysis — spot finding, indexing, integration and merging

azint

Only azimuthal integration (no spot finding/indexing); writes <prefix>_process.h5

scale

Only re-scale/merge the already-integrated reflections in the input _process.h5 (no re-integration)

calibration

Determine the detector geometry from powder rings; writes <prefix>.poni and <prefix>.json

Calibration (--mode calibration):

Option

Description

--calibrant <name>

Powder standard: lab6 | agbh | ceo2 | si | ice (default lab6, case-insensitive)

--calibration <txt>

How the rings are measured: rings | spots (default rings; see above). rings defaults --azim-phi-bins to 32

--no-refine-tilt

Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance, for a calibration handed to a program that cannot express a tilted detector (XDS)

Detector mask:

Option

Description

--detect-beam-stop[=N|off]

Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. On by default (60 images); =off disables. Reflections behind the stop are attenuated but not flagged, so they integrate low with a plausible sigma and no existing rejection catches them

Geometry:

Option

Description

--beam-center-check[=off]

Measure the beam centre from the isotropy of the scattered background on every run, report how far the file’s value is from it (against how right this geometry needs it to be), and on rotation data index a second first pass at the measured centre. The fit reads the projection --detect-beam-stop already builds, so it costs no extra frames. On by default; =off disables it, and with it the measured centre’s part in the post-refinement bound. The measured centre is adopted where the file’s centre indexes nothing and the measured one indexes a majority; where the two return cells related by an integer volume factor (2 to 4) and the measured centre’s cell, larger or smaller, carries materially more of the pooled validation spots against its own chance level; and, on a two-pass run, where the first pass is run at both centres and the measured one merges better — done where the two give the same cell in different metric symmetries, where they agree only once less of each frame is read, and where they agree but lie further apart than the geometry absorbs and the file’s centre merges inconsistently in the lowest-resolution shell. Otherwise the file’s centre is kept. Still runs when --beam-x/--beam-y are given. See §1.4

--beam-center-search[=N|off]

After a rotation first pass that indexes fewer than half the validation frames, step the centre a pixel at a time out to N px along each detector axis, re-finding the spots at every rung, and keep the first rung that indexes a majority. On by default (12 px); =off disables. It runs only after a pass that has already failed, so a run that indexes never pays for it. Both detector directions are searched: a centre error across the spindle collapses the indexed fraction and announces itself, while one along it holds the frame count up and quietly returns an axis harmonic, so a rung whose cell is an integer or √3 volume multiple of the starting one is refused. Skipped where the background places the centre further away than N px

--estimate-beam-center

Measure the beam centre before indexing and use it in place of the file’s: from the symmetry of the spot positions where the sweep reaches half a turn, from the scattered background otherwise. Adopted only where its sigma is within the larger of 1 px and what this geometry needs, and the move is over three sigma; otherwise the file’s value is kept. Ignored with --beam-x/--beam-y. Off by default

--no-fit-spindle

With --estimate-beam-center, keep the rotation axis given in the file instead of fitting its skew about the beam

Spot finding:

Option

Description

--spot-sigma <num>

Noise sigma level for spot finding (default: 4.0)

--spot-threshold <num>

Photon-count threshold for spot finding (default: 10)

--adaptive-spots

Self-calibrating detection (default, stills and rotation alike): the strong-pixel threshold comes from each image’s own per-resolution-ring noise instead of the fixed --spot-threshold, so one setting adapts across datasets (no per-dataset --spot-threshold/--spot-sigma tuning)

--no-adaptive-spots

Turn adaptive detection off and use the fixed --spot-threshold / --spot-sigma finder

--spot-false-pixels <num>

Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies --adaptive-spots)

--spot-high-resolution <num>

High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills

--spot-low-resolution <num>

Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit)

--min-pix-per-spot <num>

Minimum connected strong pixels per spot. If omitted, min-pix is chosen per image (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead.

--max-spots <num>

Maximum spots kept per image (the strongest ones) and handed to indexing. If omitted, the budget is measured on rotation data: the first pass reads how deep into an image’s spot list its spots still lie on the lattice it found, and the run keeps that many (never more than 1000). Give a value to pin it. Stills always use the fixed 1000.

--detect-ice-rings[=on|off]

Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file’s detect_ice_rings, or — where the file carries no such key — on for rotation and off for stills

Azimuthal integration (the radial profile behind the per-image ice-ring score). Every q here is q = 2π/d, in Å⁻¹:

Option

Description

-q, --azim-q-spacing <num>

Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings)

--azim-min-q <num>

Minimum Q, 1/Å

--azim-max-q <num>

Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see

--azim-phi-bins <num>

Number of azimuthal (phi) bins (default: 1)

--polarization-correction <on|off>

Enable/disable the azimuthal polarization correction

--solid-angle-correction <on|off>

Enable/disable the azimuthal solid-angle correction

Indexing:

A dataset with a rotation goniometer axis is processed as rotation data (two-pass rotation indexing) by default; a dataset without one is processed as independent stills. --force-still overrides the former; the -R / --single-pass-rotation / --force-rotation-lattice flags request rotation explicitly and pick the pass or lattice.

The FFT search looks for cell axes between 10 Å (--fft-min-unit-cell) and a default longest axis of 500 Å, which has no flag of its own — a reference cell (-C) moves both bounds to cover the cell it names. A de-novo rotation run also tries a second first-pass hypothesis with the short end lowered to 5 Å, so a small-molecule cell below the 10 Å floor is indexed on its true axes rather than as a supercell of them; the standard pass’s answer stands unless that evidence takes it. An axis beyond the search’s reach is not refused: the run returns a plausible shorter sub-cell or an axis harmonic and processes it happily, so a cell that comes out at a half or a third of the expected long axis should be read as this limit, not as the crystal. For very long axes the direction grid’s angular resolution binds well below 500 Å — see the analysis reference on FFT indexing.

Option

Description

--force-still

Treat a rotation (goniometer) dataset as independent stills instead of rotation

-X, --indexing-algorithm <txt>

FFBIDX | FFT | FFTW | Auto | None

-C, --unit-cell <cell>

Reference unit cell "a,b,c,alpha,beta,gamma" (required by ffbidx). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and --fft-min-unit-cell is lowered to admit it

--fft-min-unit-cell <num>

Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a second first-pass hypothesis with the floor lowered to 5 Å by default, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. -C lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered

--min-indexed-spots <num>

Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead (§4.1), and integration is not gated by it

-S, --space-group <num|symbol>

Space group number (96) or Hermann-Mauguin symbol (P43212) — for indexing and scaling

-r, --refine <txt>

Geometry refinement: none | orientation | beam_and_lattice (default) | flex (try all three per image, keep whichever indexes the most spots; alias multi)

-R, --two-pass-rotation[=num]

Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100)

--single-pass-rotation[=num]

Online-like single-pass rotation indexing (optional min angular range, deg)

--force-rotation-lattice <vec>

Force rotation lattice (9 floats, Å), skipping the first pass

--rotation-no-postrefine

Rotation: disable the default-on two-pass geometry post-refine (see the rotation section)

--refine-geometry[=N|off]

Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (-C / -z), =off disables

--index-ice-rings[=on|off]

Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: off; no effect without --detect-ice-rings, which does the flagging)

Indexer choice in brief: ffbidx (GPU) refines toward a known cell and is best for sparse serial stills; fft (GPU) / fftw (CPU) index de novo and suit strong rotation data. See the CPU/GPU data-analysis reference for the algorithms.

Scaling and merging:

Option

Description

--no-merge

Skip scaling and merging, which are on by default; the per-image _process.h5 is then written instead of the merged files (_unmerged.mtz and the results report are still written)

-A, --anomalous

Anomalous mode: merge each Bijvoet hand as its own unique reflection, so multiplicity, completeness, ⟨I/σ⟩ and the error model are counted anomalously. A default rotation merge already writes I(+)/I(-) (see Reflection-file conventions); -A changes the counting basis, not whether the anomalous signal is in the file

--scale-fulls / --no-scale-fulls

rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills

--smooth-g[=deg]

rot3d: smooth the per-frame scale G over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off)

--no-scaling-corrections

rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below)

--relative-b[=deg]

rot3d: fit a per-batch relative-B beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise)

--simple-stills

Stills: treat every reflection as a full (p = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement

--no-expected-variance-merge

Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation’s signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting

--capture-uncertainty <num>

rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise)

--min-captured-fraction <num>

rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off)

--scaling-high-resolution <num>

High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below)

--scaling-low-resolution <num>

Low-resolution limit for scaling and merging, Å (default: 50, which is also XDS’s own default; 0 removes the limit). The beam stop suppresses air scatter well beyond the shadow it casts, so the background stays depressed across pixels the beam-stop mask leaves open — measured at 30–44% of the field value, recovering only around 50 Å. A background ring in that zone over-estimates the background, so reflections coarser than the limit mostly integrate negative. Beam-stop masking therefore does not make this redundant. A large cell (a few hundred Å) does have real reflections coarser than 50 Å, but on the geometries measured here they fall in that zone and are not usable as integrated; raise the limit only if the low-resolution shell statistics justify it

--resolution-cutoff <txt>

Automatic high-resolution cutoff for the written reflections and reported shells: cc-logistic | off (default: cc-logistic; ignored when --scaling-high-resolution is set)

--resolution-cc-target <num>

CC1/2 target defining the cc-logistic fall-off (default: 0.30)

--resolution-shells <num>

Number of resolution shells in the reported statistics table (default: 9). The bins are equal steps in 1/d² between the lowest- and highest-resolution reflection merged, which is XDS’s rule, and 9 is XDS’s count — so at the same resolution limits the two tables have the same shells and can be read row for row

--report-resolution <dmin>[,<dmax>]

Also report the merging statistics over this resolution range, Å (either order; dmax defaults to the run’s own low-resolution limit), as a second table beside the run’s own — the REFRES_* keys of the results report — binned from the same merged reflections, so a run can be read against another program’s table at that program’s range. Report-only: the cut, the scaling, the error model and every decision stay the run’s own, and shells finer than the run’s own limit are reported as not merged rather than filled from data the run did not keep

--min-partiality <num>

Minimum partiality to accept a reflection (default: 0.02)

--ice-min-score <num>

Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before any ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness

--ice-min-spot-ratio <num>

The second ice-presence channel: found spots on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so --ice-min-score alone is blind to it (default: 2.0; 0 disables this channel)

--reject-outliers <num>

Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rot3d, off otherwise)

--min-image-cc <num>

Per-image CC limit, percent (default: no limit)

--search-min-zeta <num>

De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry |ζ| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences

--mosaicity <num>

Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed

--scaling-iterations <num>

Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle, and a merge that hits the cap is flagged SCALING_NOT_CONVERGED (default: 100)

-z, --reference <file>

Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (-sf.cif), gzipped or not, recognised by content; --reference-mtz is the same option: fixes the space group, its setting and the cell, resolves the indexing ambiguity, hands over the R-free set and reports CCref. Not a scale anchor

--reference-column <label>

Reference column to use, by MTZ label — an mmCIF under the labels gemmi cif2mtz gives it (default: auto — F-model, else IMEAN/I/…, else FP/FOBS/F)

--model <file>

Validate the merged intensities against this atomic model (PDB or mmCIF, gzipped or not; the format is taken from the file’s content) — R-work / R-free and maps (see Validating against a model). It also settles the frame the reflections are written in: the enantiomorph, and the indexing ambiguity where no -z did. For serial stills given -C / -S, the model’s structure factors become the per-image reference

--write-process-h5

Also write the (large) _process.h5 when merging (default: only .mtz/.cif)

--developer

Write the full <prefix>_report.txt: the pipeline-internal keys and the long explanations the default report leaves out. Nothing is computed differently — the same report, rendered in full (see The results report)

--finalist-ledger

Report the full-resolution evidence for each space group the search considered, not only the one it adopted (report-only; the decision is unchanged)

--export-unmerged

Write <prefix>_unmerged.mtz, an unmerged MTZ (POINTLESS column layout) of the integrated observations, for aimless / pointless / careless. On by default whenever there is an output prefix. Rotation partials are summed into one full per reflection. Intensities carry the deterministic per-reflection corrections and nothing else — Lorentz and polarization in LP, sensor efficiency in QE, flight path in FLIGHT — while the partiality is not divided out and the per-image scale is not applied. Written in --mode mx and --mode scale, and with --no-merge. See The unmerged export

--no-export-unmerged

Do not write <prefix>_unmerged.mtz. It is the largest file a run produces, so a run whose observations are not going to another scaling program can skip it

--no-p1-crosscheck

Do not write <prefix>_P1.mtz, the P1 cross-check merge described under Output files. Every rotation run that determines its own space group writes it; a run given -S writes none either way

--export-unmerged-partials

Write <prefix>_unmerged_partials.mtz, the same observations with each partial as its own row (one batch per image) for the reading program to sum. Off by default, and independent of --export-unmerged

Integration:

Option

Description

--integrator <txt>

Spot integrator: gaussian (profile-fit, default) | empirical | boxsum (classical fallback)

--integration-radius <r>

Signal-box radius r1, or r1,r2,r3 (px). One value ⇒ r2=r1+2, r3=r1+4

--adaptive-integration-radius[=on|off]

Set the signal radius r1 from how wide this crystal’s spots actually are (default: on for rotation, off for stills). r1 is not the integration domain — that is the profile-fit grid — but it is the aperture the profile width is learned over, and a second moment over a disk of radius a saturates at a²/4, so at the shipped r1 = 4 the learned σ can never exceed 2 px and a broader spot is fitted with a profile the model cannot represent. The pre-scan reads r80, the radius holding 80 % of a spot’s flux, off isolated strong spots over a fixed 14 px aperture that owes nothing to r1, fits it against 1/d and evaluates it at 5 Å; then r1 = clamp(round(2·r80), 4, 6), r2 = r1 + 2, and r3 is taken so the r2..r3 background ring keeps the area it has at the default 4,6,13. The ceiling of 6 is pattern density: r2 also drives the neighbour-ownership radius and the ring’s inner edge, and past it a dense pattern starts losing reflections whose ring falls below six clean pixels. Ignored when --integration-radius is given. The widened radius applies to the final integration pass only — the two-pass geometry pre-pass keeps the radius the run started with, because the post-refinement fits the detector distance and beam to the observed reflection positions and those move with the signal disk (33 µm and 0.02 px between r1 = 4 and r1 = 6 on one crystal, enough for the second pass’s de-novo lattice search to settle on a different lattice and index a fifth fewer frames). And where the widened radius leaves more than 1.1 % of the predicted reflections without a background ring — a pattern too dense for it — the final pass is re-integrated at the fixed 4 px radius, and the log says so. Over the rotation regression battery it moves 12 of 38 crystals and leaves the merged intensities of the other 26 unchanged; where it moves them, per-shell ⟨I/σ⟩ improves by up to 31 % and R_meas by up to 24 %

--integration-stencil <k>

Push the r2..r3 background ring out by k times the beam’s radial streak bandwidth·R_px, per reflection (default 0 = the fixed circular ring). A fixed ring otherwise ends up on a streaked reflection’s own tails at high resolution and measures them as background. Only the ring moves, and only radially — the r1 signal box stays a circle — and the growth is capped at 2·r3. The neighbour exclusion grows with it, so on a crowded pattern a few reflections can be left with too little background and dropped. On a monochromatic beam the streak is zero and this does nothing

--background-clip <n>

High-side clip of the background ring at mean + n·√mean (default 4, whatever the bandwidth; 0 = off). The default background estimator — it rejects neighbour cores and zingers without the symmetric trim’s Poisson skew bias. Ignored by --integrator boxsum

--background-trim <f>

Use the old symmetric trimmed mean for the background ring instead of the clip, 0≤f<0.5 (0.10 was the former default). Switches --background-clip off. A symmetric trim is biased low on Poisson data and adds ~5 counts to every partial, so this is for back compatibility only; 0 = plain ring mean. Rings holding more than 512 pixels fall back to the plain mean (the GPU sorts the ring in shared memory and the CPU now matches it), which the default radii never reach but wide ones do

--background-radial[=on|off|auto]

Correct the background ring for the curvature of the radial background (default auto). Disk and ring are concentric, so a background linear in position cancels between them and only curvature survives — which on a smooth ice ring reaches +26 counts on a single reflection. auto applies it per image where that image’s ice score shows a smooth powder ring, since the model is a function of radius alone: on ice made of discrete crystallite spots there is no smooth ring and the correction makes the bias worse. Ignored by --integrator boxsum (no clip pass to take the curve from)

--integration-high-resolution <num>

High-resolution limit for prediction and integration. Omitted (or 0) means integration extends as far as the detector reaches — which is what the predictor can place on the detector anyway, since it rejects reflections that miss it. Set a value to integrate less than the detector offers

--max-hkl <n>

Predict reflections with |h|,|k|,|l| ≤ n (max 511). By default this is derived per crystal from the refined cell as ceil(max(a,b,c)/d_min) + 1, which is the exact bound: the predictor keeps only |q| ≤ 1/d_min and h = a·q, so no reflection can lie outside it and no candidate inside it is wasted on a shorter axis. Set it only to override that

--bandwidth <num>

Relative X-ray bandwidth FWHM (e.g. 0.01 for a 1% DMM). Default: the file’s value; where the file states none, the one the pre-scan measures from how much longer strong spots are along their radius than across it, used where it is significant; otherwise 0 (monochromatic). --bandwidth 0 forces a monochromatic beam

--overlap <txt>

What to do where two predicted reflections share signal pixels: off | reject | exclude (default exclude). A shared pixel belongs to the nearer centre; without this a crowded reflection reads high on a dense pattern. exclude drops the shared pixels from the profile fit, which renormalises itself, and keeps the reflection; reject instead drops the whole reflection when too little of its profile is cleanly its own. --integrator boxsum has no profile to renormalise, so only reject acts there

--overlap-minpk <f>

Least fraction of a reflection’s expected profile that must be usable for it to be kept (default 0.75, XDS MINPK). Governs both the fraction that must be readable — not masked, untrusted, in a gap or overloaded — in every profile mode, and, under --overlap reject, the fraction that must be cleanly its own. Under --integrator boxsum any unreadable pixel discards the disk outright and the reject fraction goes by disk area, which cuts harder

--flight-path <txt>

What the diffracted beam crosses between the sample and the detector: air (default) | helium | vacuum. A reflection arriving at angle α to the detector normal flies D/cos α instead of D, so it crosses more of the medium and reads low; the factor is exp(D/L·(1/cos α − 1)), with L the attenuation length of the medium from the NIST tabulation and D and λ as stated by the file — nothing in it is fitted. Nothing in NXmx, and no field of any master file Rugnux reads, describes the medium, so it cannot be auto-detected and is assumed; the report states which medium was assumed and what it was worth (FLIGHT_PATH, FLIGHT_PATH_WILSON_B). Nor can it be inferred from the implied transmission — in this corpus a confirmed helium station sits at 51 % implied air transmission and a confirmed air station at 63 %. In air the term is +1.5 % at α = 55° for 18 keV over 160 mm and several-fold below 5 keV, where stations use helium precisely because air is unusable; helium attenuates ~1/600 of air, and vacuum leaves intensities untouched

--prediction-mosaicity <num>

Diagnostic: fix the rocking width (deg) the prediction window opens to, leaving partiality on the per-image σ_M. The two are one number by default, so a σ_M that moves takes the integrated reflection population with it

Geometry overrides (defaults are taken from the input file; override them to reprocess with a corrected geometry):

Option

Description

--beam-x <num>

Beam centre X (pixel). Switches --estimate-beam-center off; --beam-center-check still runs and can adopt its measured centre (§1.4)

--beam-y <num>

Beam centre Y (pixel). Switches --estimate-beam-center off; --beam-center-check still runs and can adopt its measured centre (§1.4)

--detector-distance <num>

Detector distance (mm)

--wavelength <num>

Wavelength (Å)

--rot1 <num>

PONI detector rotation 1 (rad)

--rot2 <num>

PONI detector rotation 2 (rad)

--rot3 <num>

PONI detector rotation 3, about the beam (rad) — the one rugnux --mode calibration writes into its .poni, so a refined third rotation can be given straight back

--detector-mirror-y

The stored image is mirrored in Y relative to the frame the PONI angles are stated in

--detector-quarter-turns <0-3>

The stored image is turned by this many multiples of 90° about the beam relative to that same frame

--polarization <num>

Degree of polarization p (XDS FRACTION_OF_POLARIZATION = (1+p)/2), signed: positive for a horizontally polarized source, negative for a vertically polarized one. Default 0.99, an undulator value; bending-magnet and wiggler sources are typically lower (0.8–0.95), and no format Rugnux reads states it reliably, so pass the beamline’s known value here

--rotation-scale <k>

Goniometer rotation scale: the stage turned k times the angle stored in the file (the commanded one). Applied to both passes, and overrides the scale Rugnux fits for itself

\ No newline at end of file diff --git a/RUGNUX_CALIBRATION.html b/RUGNUX_CALIBRATION.html new file mode 100644 index 000000000..e1f3e79ca --- /dev/null +++ b/RUGNUX_CALIBRATION.html @@ -0,0 +1,4 @@ + Detector calibration from powder rings (rugnux --mode calibration) — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Detector calibration from powder rings (rugnux --mode calibration)

The calibration mode determines the detector geometry — PONI x/y, the two tilts rot1/rot2 and the distance — from the powder rings of a calibrant, and writes it as a pyFAI <prefix>.poni file and a machine-readable <prefix>.json, alongside a printed report of how far each parameter moved from the header. Bragg data pin the beam centre worst (it is gauge-coupled to the crystal orientation); a powder ring has no orientation to be coupled to, so this is the measurement that fixes it.

rugnux --mode calibration --calibrant lab6 -o det LaB6_master.h5
+

What it writes

<prefix>.poni is for pyFAI and the tools that read its format. <prefix>.json is for everything else, and has two members:

  • dataset_settings — the geometry under the property names dataset_settings gives them in jfjoch_broker’s OpenAPI schema, and nothing else. It is a valid dataset_settings body as it stands, so it can be POSTed or merged into one without translating a field:

    curl -X POST -H 'Content-Type: application/json' \
+     -d "$(jq -c .dataset_settings det.json)" http://broker:5232/start
+

    beam_x_pxl/beam_y_pxl is the PONI, as everywhere in this system — on a tilted detector it is not where the direct beam lands. The three poni_rot*_rad are written whenever any of them is non-zero, and left out when all are zero: a body that omits them states a flat detector, so they travel together or not at all.

  • calibration — what the run knows about that geometry: the residual, the fit’s own sigmas and the correlation between the tilt and the beam centre, whether the tilt cleared its significance test or was declined and pinned, where the direct beam lands, and where the spots independently put the beam. A calibration that has gone wrong looks exactly like one that has not until those are read.

--calibrant takes lab6, agbh (silver behenate), ceo2, si or ice, case-insensitively. ice calibrates a real experiment against its own ice rings — no calibrant exposure needed — and is the reason a calibrant is a list of ring positions rather than a unit cell: hexagonal ice is P63/mmc, so rings enumerated from its cell would include systematically absent ones.

--calibration picks how the rings are measured, and both aggregate over every processed image — -s/-e/-t select which images:

  • rings (default) sums the (q × azimuth) azimuthal profile over every processed image into one map and fits the ring arcs in it. A powder ring is an arc, not a set of spots, and the summed profile measures it at every azimuth with all the run’s counts behind it. It needs the profile to be binned in azimuth, so this mode defaults --azim-phi-bins to 32.

  • spots pools the found spots of every processed image and fits those. It determines the centre from scratch (a Hough circle vote, which quantises it to a whole pixel) and then refines.

Both routes read the ring position out of a binned profile or a spot centroid, so the radial sampling matters: at a long detector distance the default 0.01 Å⁻¹ q bin is several pixels wide and quantises the rings route accordingly — pass a finer --azim-q-spacing there (the total q × azimuth bin count must stay under 65534).

The report prints the fitted geometry, the scatter of the ring points about the fitted rings and the standard error that implies on the centre. That error is formal: it measures the scatter of the points, not whether the rings themselves are trustworthy, so it stays small when a fit goes wrong for a structural reason — one visible ring, or ice that is textured rather than smooth.

A calibration is run because the header is in doubt, so a fit that quietly hands part of that header back has answered nothing — and it looks exactly like a fit that worked, down to the residual and the sigmas around it. calibration.converged in the JSON says which of the two a file is. It is false when the tilt was declined and pinned at a non-zero header value — the rings said they could not tell a tilt from a shift of the beam centre, and the angle written in its place has no more support than the one refused — or when the fit’s covariance never conditioned, so it cannot say what it determined. In either case Rugnux prints the reason, writes the .json with converged false and not_converged_reason beside it, exits non-zero, and writes no .poni: a PONI file states where the detector is and has no field in which to say that it does not know. A declined tilt over a header that states no tilt is not this — reporting no tilt is then exactly what the fit measured.

A .poni is refused for a second reason, whatever the fit found: a detector whose image orientation is not the identity — the stored image mirrored in Y, or turned by a multiple of 90° about the beam. That comes either from --detector-mirror-y / --detector-quarter-turns or from the file itself (a PILATUS miniCBF axis table, an NXmx module’s pixel directions). A PONI states the detector in five numbers — two offsets, a distance and three rotations — and has no field for how the image is stored, so one written here would describe a different geometry from the one that was measured. The run says so and exits non-zero.

--no-refine-tilt holds rot1/rot2 where the header put them and fits only the centre and the distance. The tilt is real and worth measuring, but a program that has nowhere to put one — XDS takes a detector normal to the beam — is better given a geometry that was measured with the tilt pinned than one that was measured tilted and then flattened, because in the tilted fit the centre and the distance have already absorbed the tilt.

Both the PONI (the point of normal incidence, which is what a .poni file stores) and the direct beam (where the beam lands, which is what most other programs call the beam centre) are printed. They differ by distance × tan(rot) once the detector is tilted, which on a 0.3° tilt at 300 mm is several pixels — enough to look like a disagreement with another program when there is none.

\ No newline at end of file diff --git a/RUGNUX_FORMATS.html b/RUGNUX_FORMATS.html new file mode 100644 index 000000000..5f804d2af --- /dev/null +++ b/RUGNUX_FORMATS.html @@ -0,0 +1 @@ + What Rugnux reads — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

What Rugnux reads

Which data rugnux opens, before anything is typed. The short answer: an HDF5 master (NXmx or DECTRIS, from any facility), a PILATUS miniCBF sweep, a marCCD sweep or an SMV sweep (ADSC or Rigaku d*TREK) — nothing else is read, so any other format has to be converted to one of these first.

Input is an HDF5 master file, or a directory of PILATUS miniCBF, marCCD or SMV frames. One input is one sweep of one crystal — Rugnux does not combine sweeps or crystals in a run; process each sweep to its own _unmerged.mtz and merge them downstream (see Taking the data onward).

  • HDF5 master (NXmx-based) — a file written by jfjoch_writer, a DECTRIS EIGER master, or an NXmx master written by another facility’s toolchain. Lengths are taken in the unit the file declares, the image size from the image array’s own shape, and each data file’s images at the path the master’s own link names, so masters that state single values as one-element arrays, compose their images as a virtual dataset over the master itself, or keep them somewhere other than /entry/data/data all open. Where the NXmx spellings are absent the pre-NXmx ones are tried, so a DECTRIS firmware 1.x master opens too — including its goniometer, which used to be missed and the sweep read as stills. Images may be bitshuffle+LZ4/Zstd or the HDF Group’s plain LZ4 (filter 32004); any other filter is named in the error.

  • PILATUS miniCBF sweep — one frame per file, read natively with no conversion and no libcbf. Name any frame of the sweep, or the directory holding it, and the whole sweep is processed: the frames are the ones matching that frame’s template (prefix plus digit count), so a directory holding two sweeps is not spliced into one crystal, and naming a directory takes the sweep with the most frames in it. The geometry, the rotation axis and the detector mounting come from the header, including the imgCIF axis table where the header carries one (see Detector geometry). A raw CBF carries no analysis results, so --mode scale — which re-scales the reflections stored in a _process.h5 — does not accept one. Gzipped frames (.cbf.gz) are read directly, with no decompression step: EMBL Hamburg’s beamlines write that form by default, and expanding a sweep first would double the disk it needs. The two forms count as separate sweeps.

  • marCCD sweep — what Rayonix MX-series and mar Mosaic detectors write, and what a decade of deposited CCD data is archived as: an uncompressed TIFF with the instrument header in the gap before the pixels. One frame per file, read natively. Naming a frame or its directory selects the sweep exactly as for miniCBF, under either naming scheme these detectors use — a numbered stem (xtal_1_00042.mccd) or the frame number as the file extension (D1.042). The distance, beam centre, pixel size, wavelength and the circle that turned come from the header; the rotation axis’ direction does not, because the format has nowhere to state it, so the run settles its sign from the data as it does for a miniCBF that carries no axis table. A CCD frame marks no untrusted pixels, so the sweep starts with nothing masked. Like a raw CBF, it carries no analysis results and --mode scale does not accept one.

  • SMV sweep — what ADSC Quantum detectors wrote and what Rayonix and others still write, so most archived CCD data from the 2000s is in this form: an ASCII KEY=value; block between braces, then the pixels. One frame per file, read natively, the sweep selected exactly as above. The beam centre it states is in millimetres and is converted here; its convention varies between writers, and a file that states it transposed indexes nothing until the run’s own beam-centre measurement replaces it, which it does automatically. The saturation value is read where the header states one (CCD_IMAGE_SATURATION or SATURATED_VALUE); where it does not, overloads are judged on the 16-bit container alone and the reader says so once per sweep. Rigaku d*TREK headers — what Saturn CCDs on rotating-anode home sources write, recognised by their DETECTOR_NAMES key — are read in their own vocabulary: the pixel size and the point of normal incidence from the spatial-distortion record, the image directions from the detector and distortion vectors (so a mirrored or turned image keeps its hand), the distance and the detector circles, including a swung 2θ arm, from the detector goniometer, the spindle and its angles from the rotation record, and the wavelength from the scan record. Pixel values above 32767 are the format’s encoded overflows and are expanded with the header’s compression ratio.

Support for the two CCD formats — marCCD and SMV — is very limited and provided as-is; they are read so that a CCD sweep does not need a conversion step to be processed. Only the header fields named above are read, and the headers vary by site and vintage, so no promise is made that every writer’s dialect opens. The HDF5 and miniCBF paths are the maintained ones.

XFEL data saved as individual panels is not read. The detectors of serial XFEL endstations — CSPAD, AGIPD, ePix and their kin — write each panel as its own array, with the panel positions kept in a separate geometry file rather than in the image file. Rugnux has no reader for those per-panel containers and no way to take an external geometry, so such data cannot be opened; it belongs in the pipelines built around panel geometries, such as CrystFEL or cctbx.xfel. An XFEL dataset that has already been assembled into single images in an NXmx master opens like any other HDF5 input.

Spots are always found by rugnux itself, including for the two-pass rotation first pass — the spot lists a dataset may already carry were found online, at the acquisition’s threshold and with its ice-band spots already discarded, so reusing them would hide the spot-finding settings from the lattice search.

\ No newline at end of file diff --git a/RUGNUX_INSTALL.html b/RUGNUX_INSTALL.html new file mode 100644 index 000000000..f6e2d899c --- /dev/null +++ b/RUGNUX_INSTALL.html @@ -0,0 +1,15 @@ + Installing Rugnux — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Installing Rugnux

rugnux is a single self-contained executable. It needs no CUDA toolkit, no Qt, and no Jungfraujoch service running anywhere; on a machine with an NVIDIA GPU it needs the NVIDIA driver, and without one it still runs on the CPU.

From the package repositories (RHEL / Rocky / Ubuntu)

On a distribution covered by the package repositories, rugnux is a package of its own:

sudo dnf install rugnux          # RHEL / Rocky 8 and 9
+sudo apt install rugnux          # Ubuntu 22.04 / 24.04
+

It installs /usr/bin/rugnux and depends on nothing from the acquisition side — no broker, no detector libraries, no Qt — so it can go on a machine that only processes data.

Upgrading from rc.163 or earlier. /usr/bin/rugnux used to belong to the jfjoch-viewer package. The rugnux package declares that the file has moved, so installing it upgrades an old jfjoch-viewer in the same transaction instead of failing on the duplicate path. If your jfjoch-viewer is pinned to an old version, unpin it or remove it first.

From the release archive

For a machine no package manager covers — or for Windows, macOS and Arm, which have no repository — take the archive for your architecture from the Gitea release page:

Archive

For

rugnux-<version>-linux-x86_64-cuda12.tgz

64-bit Intel/AMD Linux. Built on RHEL 8, so it runs on any newer Linux

rugnux-<version>-linux-aarch64-cuda13.tgz

64-bit Arm Linux — NVIDIA GH200 and DGX Spark. Built on Ubuntu 24.04, so it needs glibc 2.39 or newer. Cross-compiled and not yet exercised on Arm hardware

rugnux-<version>-win64-cuda13.zip

64-bit Windows

rugnux-<version>-macos-arm64-cpu.tgz

macOS 13 (Ventura) or newer on Apple Silicon (M1 and newer). CPU only — there is no CUDA on macOS. Intel Macs are not supported

The archive has no top-level directory — it unpacks straight into bin/ and share/. Always give tar a destination of its own, or it will scatter those into whatever directory you are in:

mkdir -p /opt/rugnux-1.0.0
+tar xzf rugnux-1.0.0-linux-x86_64-cuda12.tgz -C /opt/rugnux-1.0.0
+/opt/rugnux-1.0.0/bin/rugnux            # prints the usage
+

What you get is:

bin/rugnux                                  the program
+share/doc/jfjoch_rugnux/LICENSE             GPLv3
+share/doc/jfjoch_rugnux/THIRD_PARTY_NOTICES.md
+share/doc/jfjoch_rugnux/licenses/           verbatim licence texts of the bundled dependencies
+

Nothing is written outside that directory, nothing needs root, and several versions can sit side by side. To remove it, delete the directory. Put bin/ on your PATH if you want to type rugnux rather than the full path.

macOS: the first run is blocked. The release is not yet notarized by Apple, and a .tgz downloaded with a browser hands its download flag on to everything tar extracts from it, so macOS refuses to run rugnux (“cannot be opened because the developer cannot be verified”). Clear the flag once for the whole directory:

xattr -dr com.apple.quarantine ~/rugnux-1.0.0
+

An archive fetched with curl carries no such flag. Unpacking into a directory under your home directory (~/rugnux-<version>) rather than /opt avoids needing sudo on a Mac.

Mixing the two. If a rugnux package is also installed, /usr/bin/rugnux will normally win on PATH. Put the archive’s bin/ first, or call it by its full path, to be sure which one you are running — rugnux prints its version on every run.

GPU support

The released Linux and Windows archives are CUDA builds (the macOS one is CPU-only). They need only an NVIDIA driver on the host — 525.60.13 or newer for the CUDA 12 archive, 580.65.06 or newer for the CUDA 13 ones — and no CUDA toolkit, because everything CUDA is linked statically. With no GPU or no driver, rugnux reports zero CUDA devices and falls back to the CPU path, which works but is far slower and offers only the fftw indexer. A V100 needs the CUDA 12 archive; which generations each build covers is in Release contents ▸ GPU generations and the NVIDIA driver.

Building from source

rugnux alone, without the server stack or Qt:

cmake -S . -B build -DJFJOCH_RUGNUX_ONLY=ON -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_CXX_FLAGS="-march=x86-64-v3" -DCMAKE_C_FLAGS="-march=x86-64-v3"
+cmake --build build -j$(nproc) --target rugnux
+

The binary lands in build/rugnux/rugnux. No library has to come from the system: every dependency is downloaded and built during the first configure, which therefore needs network access. cmake --build build --target package produces the same .tgz the release ships. The -march flag is not set by the build system on purpose, so a plain build is slower than the released one on the CPU-bound stages — see the note in CMakeLists.txt — but it is set up to give the same results: the build turns floating-point contraction off, so the x86 level a binary targets changes its speed and not its answer.

On a Mac (Apple Silicon), leave the two -march flags out — they name an x86 level; the Apple compiler’s default target is already the M1 — and use sysctl -n hw.ncpu for the job count. The build has no CUDA there and needs nothing installed beyond Xcode and CMake.

Hardware

As with the rest of Jungfraujoch, serious performance requires an NVIDIA GPU. The CUDA build provides the GPU fast-feedback indexer (ffbidx) and the GPU FFT indexer (fft); without CUDA only the CPU fftw indexer is available. With a GPU present most of the per-image pipeline runs on the device — bitshuffle+LZ4 decompression, image preprocessing, azimuthal integration, spot finding, prediction and Bragg integration — as do the pre-scan (beam-stop mask, background beam-centre fit, spot widths, defective pixels), rotation scaling and merging with its correction surfaces, and the --model rigid-body placement, with CPU implementations as the fallback where there is no GPU. The choice is automatic; a CUDA build runs on the CPU when the GPU is hidden from it (CUDA_VISIBLE_DEVICES= rugnux ...). The thread count (-N) governs the CPU side of all of it.

The released CUDA builds need only an NVIDIA driver on the host, no CUDA toolkit: 525.60.13 or newer for the CUDA 12 artefacts (RHEL 8 packages, the x86_64 rugnux archive) and 580.65.06 or newer for the CUDA 13 ones (RHEL 9, Ubuntu, the aarch64 and Windows rugnux archives). Which GPU generations each artefact supports — a V100 in particular works only with the CUDA 12 build — is in Release contents ▸ GPU generations and the NVIDIA driver.

Memory

A run’s memory is set by how many reflection observations it integrates, not by how many pixels the detector has. Measured on rc.169 at -N 6, over rotation sweeps of 1200-1800 frames:

Detector

Observations integrated

Host (peak RSS)

Device

1679 x 1475 (2.5 MP)

25 M

7.8 GB

3.8 GB

2527 x 2463 (6.2 MP)

13 M

4.9 GB

3.1 GB

3262 x 3108 (10.1 MP)

7 M

2.5 GB

3.0 GB

4371 x 4150 (18.1 MP)

13 M

4.3 GB

3.2 GB

4371 x 4150 (18.1 MP)

20 M

6.0 GB

4.4 GB

3262 x 3108 (10.1 MP)

50 M

14.2 GB

7.0 GB

Two sweeps on the same detector differ by a factor of five, so size the machine on the observation count rather than on the detector. As a rule, 0.6 GB + 0.3 GB of host memory per million observations, and 2.4 GB + 0.1 GB of device memory per million. The run reports what it actually integrated — RotationScaleMerge: ingested ... partial observations. 32 GB of host RAM and an 8 GB card cover every dataset measured here at -N 6; an ordinary sweep needs 16 GB and 6 GB.

Both peaks fall in scaling and merging, not in the per-image loop: the loop alone stays under 3.2 GB on the device at -N 6, whatever the detector. The stage that needs the most is the 3D combine.

-N multiplies the per-image loop only — about 8 bytes of device memory per detector pixel per worker (150 MB per worker at 18 MP, 87 MB at 6 MP) on top of a fixed ~2.6 GB, and it is linear with no ceiling: the same 18 MP sweep takes 2.7 GB of device memory at -N 1, 3.2 GB at -N 6 and 8.0 GB at -N 32. Host memory does not move with -N, and neither does the merge’s own device memory. Where several runs share one card, -N is the knob that decides how many fit.

To use less:

  • -N — the only flag that lowers the per-image loop’s device memory. Lowering it is what rescues a run on a card that is otherwise full.

  • --no-merge — on the heaviest sweep measured it took the host peak from 14.2 to 8.3 GB and the device peak from 7.0 to 2.9 GB. It also gives up the merged output.

  • --no-export-unmerged does not save memory. <prefix>_unmerged.mtz is the largest file a run writes, but its observations are resident either way; the flag saves disk, not RAM.

Nothing measures the free memory on the card before allocating. A run that does not fit degrades where it can — device decompression falls back to the host, the beam-stop projection falls back to the host, an indexer thread that cannot allocate reports the frame as not indexed, the --model rigid-body placement moves to the CPU — and fails the run otherwise, with Processing failed: CUDA (GPU) error (Failed to allocate device memory) and a non-zero exit status. That is deliberate: a CUDA out-of-memory says nothing about the frame in flight and everything about the machine, so skipping frames would leave a run that looks complete with quietly different merged numbers. A run is never silently short; it either completes or fails. On a GPU shared with another job, lower -N or wait for the card.

Very large unit cells

The rule above holds for the cells in that table, and a very large cell sits far beyond it. A long axis, a dense pattern and many frames multiply together: each frame predicts more reflections, each reflection is split over more frames, and tens of millions of partial observations are normal for a big cell. One sweep of a crystal with an axis of about 640 Å predicted 236 million reflections and peaked at 17.5 GB of host memory on a GPU build and 72.5 GB on a CPU-only build. A CPU-only build keeps in host RAM what a GPU build keeps on the card, so it is the one that needs the large machine. When such a set is too large for GPU scaling on the card, the run stops and says to run it on the CPU (CUDA_VISIBLE_DEVICES= rugnux ..., or a CPU-only build) — which then wants that much host memory.

When host memory runs out

Nothing estimates the host memory a run will need before it starts, so a set that does not fit runs until the memory is gone, and then ends in one of two ways.

Where the allocation is refused — under an address-space limit (ulimit -v, which some batch systems set), with strict overcommit, or for one request larger than the machine could ever grant — the run stops with

Processing failed: out of host memory (std::bad_alloc) - this data set needs more RAM than is available
+

and a non-zero exit status. The CUDA driver reserves a large address space of its own, so a GPU build under a tight ulimit -v can stop before reading anything, with CUDA (GPU) error (out of memory).

More often on Linux the allocation succeeds and the memory runs out as it is used. The kernel’s OOM killer then ends the process with SIGKILL, which no program can catch: the run stops with no message of its own, the shell prints Killed (exit status 137), and dmesg or the batch system’s log has an Out of memory: Killed process line. A job capped by a cgroup memory limit ends the same way. A run that disappears like this needs more RAM than the machine or the job was given, exactly as if it had said so. --no-merge lowers the host peak, as above; otherwise the run needs a larger machine.

\ No newline at end of file diff --git a/RUGNUX_INTEGRATION.html b/RUGNUX_INTEGRATION.html new file mode 100644 index 000000000..c324a5b87 --- /dev/null +++ b/RUGNUX_INTEGRATION.html @@ -0,0 +1,83 @@ + Rugnux with other programs — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Rugnux with other programs

What the reflection files promise to a reading program, and the minimum commands that get each downstream suite running on Rugnux output.

Reflection-file conventions

mmCIF. Standard items carry their standard meanings — _refln.intensity_meas / _intensity_sigma, the pdbx_I_plus/pdbx_I_minus and pdbx_F_plus/pdbx_F_minus anomalous pairs, _reflns.* and _reflns_shell.* for the merging statistics, _reflns.B_iso_Wilson_estimate for the Wilson B, and _cell.* / _diffrn_radiation_wavelength.wavelength for the geometry.

Anything Rugnux reports that has no standard item is written under a jfjoch_ prefix, inside the standard category it belongs to. That is a deliberate choice: a reader that does not know these items ignores them, and one that does can find them without guessing.

item

meaning

_reflns.jfjoch_diffrn_ISa

Asymptotic I/σ in XDS’s sense: the whole-range 1/√(a·b) of the error model, so it can be read directly against a CORRECT.LP

_reflns.jfjoch_diffrn_ISa_asymptotic

The strong-reflection tier — the counting-subtracted scatter of well-measured groups. XDS has no equivalent, and it can only ever be the more optimistic of the two. Rotation path only

_reflns.jfjoch_error_model_a, _b

The error model in XDS’s convention, σ² = a(σ₀² + b·I²), so the ISa above is re-derivable from the file rather than taken on trust

_reflns.jfjoch_second_moment_I

Twinning second moment ⟨I²⟩/⟨I⟩² — 2.00 untwinned, 1.50 for a perfect twin

_reflns.jfjoch_L_test_mean_abs_L, _L_test_mean_L_squared

Padilla–Yeates L-test. ⟨|L|⟩ is 0.500 untwinned / 0.375 for a perfect twin; ⟨L²⟩ is 0.333 / 0.200. Written only when the test found pairs

_reflns.jfjoch_radiation_damage_relative_B

Relative B from the first to the last rotation batch (Ų); positive is the usual direction, high-resolution intensity fading with dose

_jfjoch_radiation_damage_batch.*

Per-batch loop: id, rotation_start_deg, relative_B

_diffrn_detector.jfjoch_distance_mm, _jfjoch_beam_center_x_pxl, _jfjoch_beam_center_y_pxl

The refined detector geometry actually used, which is not otherwise recoverable from the reflection file

_reflns.pdbx_aniso_B_tensor_eigenvalue_1..3, _pdbx_aniso_B_tensor_eigenvector_*

The anisotropy tensor, eigen-decomposed. Eigenvalues are relative to the weakest direction (so the third is 0 and the first is the anisotropic ΔB), because only the deviatoric part is determined; eigenvectors are in the PDB orthogonalisation convention. Not written for a cubic Laue class, where symmetry forces ΔB to be zero

_reflns.jfjoch_aniso_delta_B, _jfjoch_aniso_delta_B_linear

The anisotropic ΔB, and the part of it that actually follows exp(−½ sᵀBs). The second is what the verdict is gated on

_reflns.jfjoch_aniso_d_min_1..3

Diffraction limit (Å) along each principal direction. A comment marks a value that is the edge of the measured data rather than the crystal’s own limit

_reflns.jfjoch_aniso_shape, _jfjoch_aniso_floor, _jfjoch_aniso_significance, _jfjoch_aniso_verdict

The resolution signature of the deficit, the data set’s own systematic-error floor, ΔBlinear over that floor, and the resulting verdict. Each carries its vocabulary as a comment

Compatibility note. Before rc.161, _reflns.jfjoch_diffrn_ISa carried the asymptote, not the whole-range value. There is no version marker inside the file, so a number taken from an older .cif is not comparable with one taken from a newer one.

SHELX HKLF 4 (<prefix>.hkl). Fixed-format 3I4,2F8.2 — h k l I σ(I), terminated by a 0 0 0 record — which is what SHELXL, SHELXC, SHELXD and ANODE expect. Two properties worth knowing before using it:

  • On rotation data it is unmerged: one record per full reflection (its partials summed), with the per-frame scale and every correction applied and the σ(I) the merge weighted it with, but not averaged with its symmetry equivalents, and at the index it was measured at — the chemical crystallographer’s convention, so SHELXL computes Rint and Rsigma itself and Friedel mates keep their own records. Observations the merge rejected as outliers or left out for an overloaded pixel, and those beyond its resolution cut, are not in the file. There is no batch number column (in HKLF 4 that selects a BASF scale factor). A stills run writes the merged reflections instead, Bijvoet mates at +hkl and -hkl.

  • Intensities are rescaled by a single global factor so the largest value fits the F8.2 field. I and σ(I) share that factor, so every ratio — and therefore the anomalous signal — is untouched, but the absolute scale is not meaningful. This matters only if you intend to compare magnitudes with another file; SHELXC and ANODE use ratios alone.

MTZ (<prefix>.mtz, and <prefix>_P1.mtz beside it). The CCP4 anomalous layout, with the column types CCP4 programs dispatch on:

H K L IMEAN SIGIMEAN I(+) SIGI(+) I(-) SIGI(-) F SIGF F(+) SIGF(+) F(-) SIGF(-) FreeR_flag
+H H H    J        Q    K       M    K       M  F    Q    G       L    G       L          I
+

F is the French–Wilson amplitude. The header carries the determined space group, the refined cell and the wavelength, on a dataset of its own behind the reserved HKL_base, which is where the MTZ format puts them. Older Rugnux wrote the data on dataset 0, the id reserved for HKL_base, and CCP4’s mtzinfo then reported its 1.54187 Å (Cu Kα) default instead of the real wavelength — every other reader tried, mtzdmp, truncate, ctruncate, gemmi, iotbx and phenix.xtriage, recovered the true value from those files as well, so the effect was confined to that one report. Note that <prefix>_unmerged.mtz still reads 1.54187 under mtzinfo and is not wrong: its columns sit on HKL_base deliberately, as POINTLESS expects, and the wavelength AIMLESS and POINTLESS read is the per-batch one, which is correct. The Bijvoet columns are present on any rotation merge, with or without -A; a stills merge has no Bijvoet split and the file then stops after F SIGF FreeR_flag.

There is deliberately no DANO/SIGDANO pair, the anomalous difference columns a CCP4 merged file usually carries. They are a restatement rather than a measurement: checked column against column on a ctruncate file, DANO is F(+) − F(-) to the last bit and SIGDANO is √(σ(+)² + σ(−)²) to the last bit, on every reflection — the quadrature sum is the convention whether or not the two mates came from one scale model, and no correlation correction is applied by anybody. Every program in the phasing routes below reads the Bijvoet columns directly and forms the difference itself, and CCP4’s own phasing engines prefer them: bp3 and afro want F+/SF+/F-/SF- and tell a user holding F/DANO to convert to that form, and mtz2sca ranks I(+/-) over F(+/-) over F/DANO. Where the pair is genuinely wanted — fft’s anomalous-difference Fourier takes a DANO label and has no other spelling — one command makes it, with the ISYM column that belongs beside it:

ctruncate -hklin myrun.mtz -hklout myrun_ct.mtz \
+    -colin '/*/*/[IMEAN,SIGIMEAN]' -colano '/*/*/[I(+),SIGI(+),I(-),SIGI(-)]'
+

The unmerged export

<prefix>_unmerged.mtz holds every integrated observation, before scaling and merging, in the column layout POINTLESS writes and aimless, pointless, careless and iotbx.merging_statistics read. It is written by default, in --mode mx and --mode scale alike and with --no-merge as well, and it replaces nothing — Rugnux still writes its own merged files in the same run. It needs an output prefix (-o). It is the largest file a run produces, larger on a dense rotation dataset than the merged .mtz, .cif and .hkl put together, so a run that only wants the merged numbers — a regression battery, or a throughput pipeline — turns it off with --no-export-unmerged.

Use it to scale the data with a different program, to have pointless give an independent opinion on the space group, or to compare Rugnux’s merge against another one on identical input. Each sweep’s file is self-contained, so several of them can be handed to pointless and aimless as separate HKLINs to merge sweeps Rugnux does not combine itself.

Trap when combining a wild-carded series. For an HKLIN given with wild-cards, POINTLESS accepts the files in order and terminates acceptance at the first file out of chronological order, then merges what it kept and prints a plausible result. Its own keyword lifts the check — ALLOW OUTOFSEQUENCEFILES — or name each file as its own HKLIN, which is not a series; either way, check the file count in its log against the number you meant to give.

Columns. H K L M/ISYM BATCH I SIGI FRACTIONCALC XDET YDET ROT LP QE FLIGHT FLAG — POINTLESS’s own set down to FLAG, plus QE and FLIGHT (the sensor-efficiency and flight-path divisors described above; QE is DIALS’s column) — then four Rugnux extras, DELPHI (offset from the centre of the rocking curve), ZETA (the Lorentz geometry of that curve), BGMEAN and BGVAR (the background that was subtracted, and its variance). BATCH is the image ordinal plus one, and a batch header is written for every batch that carries an observation. M/ISYM records both the symmetry operation and the Friedel hand, so the index as measured is recoverable from the index as stored.

Header symmetry and order. The file’s MTZ header carries the space group the run determined (P1 where none was), and the rows are sorted on H K L M/ISYM BATCH — the order POINTLESS leaves an unmerged file in, and the order AIMLESS requires of its input — so both programs take the file directly.

What has been applied to the intensities, and what has not. I and SIGI carry the deterministic per-reflection corrections and nothing else. Three columns record them: LP is Lorentz x polarization; QE is the sensor’s quantum efficiency at the angle the diffracted beam meets the detector; and FLIGHT is the attenuation in the medium the reflection crossed on its way there. QE and FLIGHT are both divisors normalised to 1 at normal incidence, so raw counts are I / LP * QE * FLIGHT. They are applied because they are per-observation geometry that varies by more than two orders of magnitude across a sweep and no reader can reconstruct them. QE is at least 1 and FLIGHT at most 1: an oblique reflection crosses more sensor, which makes it read high, and more of the medium, which makes it read low. FLIGHT is a column of ones under --flight-path vacuum.

QE is kept out of LP because that is what the field means by LP: XDS’s RLP is Lorentz x polarization alone (its own column is flat to 0.1% across a detector over which the efficiency term spans 7%), and DIALS fills LP from lorentz/polarization only and writes QE as a separate column — a column of ones where it has no correction. Rugnux normalises QE to normal incidence where DIALS stores the un-normalised absorbed fraction; the two differ by a per-dataset constant, i.e. by an overall scale. Deliberately not applied: the partiality is not divided out (it is reported in FRACTIONCALC), and the per-image scale is not applied at all — those programs fit their own scale model, and handing them pre-scaled data would have them fit a correction to a correction. No resolution cut, outlier rejection or ice-ring filtering is applied either.

Partials. On a rotation run the partials of each reflection are summed into one full, using the same rule Rugnux’s own 3D combine uses — consecutive frames no more than two apart — and the full is written at the batch its rocking curve is centred on, with the summed rocking-curve fraction in FRACTIONCALC. An event that caught less of its rocking curve than --min-captured-fraction (or --min-partiality) is not written, and neither is one with an overloaded pixel on any of its frames, exactly as in the merge. Summing is the default because a downstream program’s own partial handling is far more conservative than Rugnux’s: given raw partials, aimless accepted a small fraction of the file and merged at a fraction of the multiplicity; given summed fulls it uses essentially all of it. --export-unmerged-partials writes the unsummed form to <prefix>_unmerged_partials.mtz for a program that would rather sum them itself. Stills have no rocking events and are the same either way.

Systematic absences. Lattice-centring absences are not written; screw and glide absences are. Prediction runs in a primitive setting so that the space-group search can test the centring, but the interstitial reflections that leaves make a reading program take the lattice for primitive and demote the group. Screw and glide absences are kept because they are the evidence the space group was chosen on — deleting them would turn a reading program’s test into an assumption. XDS and DIALS draw the line in the same place.

Scan axis. The batch headers carry the goniometer axis negated relative to the one in the input file. This is not a correction to the file: Rugnux brings an observation made at angle φ back to zero by rotating it by +φ, so the crystal itself turns by −φ, and an MTZ batch header records the axis a batch’s own increasing PHI turns the crystal about. With the sign as exported, pointless’s independently determined orientation matrix agrees with Rugnux’s to well under a degree.

Taking the data onward

The reflection files are inputs to other suites, and the handover has a few conventions worth one line each. These are the minimum commands that get each program running on Rugnux output.

phenix. The merged files carry both the mean intensity and the Bijvoet pairs, and a phenix program that has not said which it wants stops on the pair of them — from the MTZ and from the mmCIF alike, each listing its own format’s labels:

Sorry: Multiple equally suitable arrays of observed xray data found.
+
+Possible choices:
+  myrun.mtz:IMEAN,SIGIMEAN
+  myrun.mtz:I(+),SIGI(+),I(-),SIGI(-)
+

Two things are worth knowing before reading that as a fault in the file. The tie is between the two intensity arrays and nothing else: iotbx scores F/SIGF and F(+)/F(-) below them, so they are never in the running and writing amplitudes as well as intensities is not what causes this. And ctruncate’s own output ties in the same place — put any merged data through CCP4’s truncate step and phenix asks the same question of the result, because a mean intensity array and an anomalous one score equally whenever the calling program has expressed no preference. The only file change that removes the tie is dropping one of the two, and dropping the Bijvoet columns would take the anomalous signal — and the whole SHELX route — with it.

So the answer is a label. The parameter name differs by program, which is the part that catches people out:

phenix.xtriage myrun.mtz xray_data.obs_labels=IMEAN
+phenix.xtriage myrun.mtz "xray_data.obs_labels=I(+)"        # the Bijvoet array instead
+phenix.refine  model.pdb myrun.mtz miller_array.labels.name=IMEAN
+

IMEAN on its own is enough — the match is on a substring — and IMEAN,SIGIMEAN and the fully-qualified scaling.input.xray_data.obs_labels= work equally. Quote the anomalous one: the parentheses are shell syntax otherwise. The same behaviour appears on the mmCIF in that format’s own vocabulary, and a label from one format does not work on the other (Sorry: No matching array):

phenix.xtriage myrun.cif xray_data.obs_labels=intensity_meas
+phenix.xtriage myrun.cif xray_data.obs_labels=pdbx_I_plus
+

A program that states a preference needs none of this. phenix.hyss, phenix.find_peaks_holes, phenix.molprobity and the data import behind phenix.autosol ask for anomalous data by preference, which breaks the tie for them. phenix.hyss myrun.mtz n_sites=6 scattering_type=S opens the file with no labels given, reports Miller array info: myrun.mtz:I(+),SIGI(+),I(-),SIGI(-), and forms the anomalous differences itself.

The R-free convention. FreeR_flag is 0 = free, 1 = work — the CCP4 convention the column’s own name belongs to (5 % free by default). REFMAC5’s default FREE 0 reads it directly and phenix.refine detects the numbering on its own, so neither needs a keyword:

refmac5 XYZIN model.pdb HKLIN myrun.mtz XYZOUT refined.pdb HKLOUT refined.mtz <<eof
+LABIN FP=F SIGFP=SIGF FREE=FreeR_flag
+NCYC 10
+END
+eof
+

(A merged MTZ written before rc.166 carried the opposite, phenix/CNS numbering — 0 = work — under the same column name; REFMAC5 stops on such a file with more than half of reflections are in free R set and Cannot switch free R flag, and the keyword FREE 1 is the cure for those files only.)

POINTLESS / AIMLESS. myrun_unmerged.mtz opens in both directly — it is sorted the way AIMLESS requires and its header carries the determined space group (see The unmerged export). Running pointless first remains the safe route, and its independent space-group opinion is what the file exists for:

pointless HKLIN myrun_unmerged.mtz HKLOUT sorted.mtz
+aimless   HKLIN sorted.mtz         HKLOUT scaled.mtz
+

Several sweeps of one crystal form go in as separate HKLINs to the same pointless run — that is how sweeps Rugnux does not combine itself are merged.

careless wants exactly what the unmerged export is — unmerged, unscaled intensities carrying only the deterministic per-reflection corrections, with the partiality reported and not divided out. Against its published examples, two renames: BG/SIGBG are called BGMEAN/BGVAR here and BGVAR is a variance, not a sigma. A QE column is present, as in DIALS output (normalised to 1 at normal incidence where DIALS stores the absorbed fraction — a per-dataset overall scale). Hobs/Kobs/Lobs are reconstructed from M/ISYM by reciprocalspaceship, and dHKL careless computes from the cell, so the metadata string that names this file’s columns is

careless mono --anomalous "BATCH,dHKL,Hobs,Kobs,Lobs,XDET,YDET,BGMEAN,BGVAR,LP,FRACTIONCALC" \
+    myrun_unmerged.mtz out/myrun
+

Molecular replacement and experimental phasing each get a section of their own below — Phaser and SHELXC/D/E. Both are where Rugnux stops and the next program starts, and both meet the one thing the merged intensities could not decide: which of several space groups the data are in.

iotbx.merging_statistics myrun_unmerged.mtz needs no arguments or label choices at all.

Molecular replacement with Phaser

rugnux does not do molecular replacement, so Phaser is the next program for anyone who has a search model. Both CCP4 and phenix ship it — phaser and phenix.phaser, the same 2.8.3 build in the versions this was checked against — and either takes the merged myrun.mtz as it is written.

No LABIN, no label choices. Phaser reads the cell, the space group and the resolution range out of the file and picks the intensity columns itself. Where phenix stops on a merged file because it cannot choose between two equally usable observation arrays (see above), Phaser simply announces what it took:

   Data read from mtz file: myrun.mtz
+   Space-Group Name (Hall Symbol): P 41 21 2 ( P 4abw 2nw)
+   Unit Cell:   78.06   78.06   37.70   90.00   90.00   90.00
+   Column Labels Selected: IMEAN SIGIMEAN
+   Resolution on Mtz file:  0.99 39.03
+

So the whole run is the model and the cell contents:

phaser <<eof
+MODE MR_AUTO
+HKLIN myrun.mtz
+ENSEMBLE model PDBFILE model.pdb IDENTITY 1.0
+COMPOSITION PROTEIN MW 14300 NUMBER 1
+SEARCH ENSEMBLE model NUMBER 1
+ROOT myrun_mr
+eof
+

On a 1.0 Å dataset in a tetragonal point group that run placed one copy at TFZ 11.1, refining to TFZ== 80.3 and LLG 10247, in 54 s of wall clock, with no warnings about the file. The one trap in that script has nothing to do with Rugnux: COMPOSITION PROTEIN SEQUENCE wants a file name, and given a chain identifier instead it fails with FILE OPENING ERROR: X before it reads anything. Use MW unless you have the sequence file to hand.

The space group is the interesting part. SPACE_GROUP_NAME in the results report is a scalar and reads like a determination, but it is one of the groups the absences allow, chosen by convention — section 2 says which others it could not separate, as SPACE_GROUP_ALTERNATIVES, and whether the hand is open, as SPACE_GROUP_ENANTIOMORPH= UNDETERMINED. Merged intensities never name a hand: an enantiomorphic pair has the same absences and the same Laue class. Phaser is one of the few programs that can settle it, because a wrong hand simply fails to place the model.

It does this without being asked. MODE MR_AUTO defaults to SGALTERNATIVE SELECT HAND, so the run above listed

   Space Group(s) to be tested:
+     P 43 21 2
+     P 41 21 2
+

and returned a single solution in P 43 21 2 — the hand opposite the one in the MTZ header. Nothing in the command asked for that. The space group of the solution is the answer, whichever hand the file happened to carry, and it is on the SOLU SPAC line of the .sol file and in the CRYST1 of the placed model.

When the alternative is not the hand, name it. SPACE_GROUP_ALTERNATIVES also carries screw variants that share a point group — I 2 3 and I 21 3 on a body-centred cubic lattice is the common one — and SGALTERNATIVE SELECT ALL searches every group Phaser derives from the input one by translation symmetry. On a P 41 21 2 input that is all eight of P 4 2 2 … P 43 21 2, and it took the run above from 54 s to 65 s; on an I 2 3 input it is I 2 3, I 21 3 and an origin-shifted I 2 3. To see the list a given file would produce without searching it, MODE CCA prints it and stops:

phaser <<eof
+MODE CCA
+HKLIN myrun.mtz
+COMPOSITION PROTEIN MW 14300 NUMBER 1
+ROOT myrun_cca
+eof
+

What Phaser cannot repair from this file is a wrong point group. SGALTERNATIVE moves within one, so a run whose report carries a non-NONE SPACE_GROUP_REFUSED_POINT_GROUP, or a point group you suspect is too high, has to be merged again rather than searched again — myrun_P1.mtz is written for exactly that, and myrun_unmerged.mtz will do it through pointless.

mmCIF is not a route into Phaser. HKLIN myrun.cif stops at FILE OPENING ERROR: myrun.cif, in both the CCP4 and the phenix build — 2.8.3 reads MTZ only. Convert rather than look for a keyword:

gemmi cif2mtz myrun.cif fromcif.mtz
+

That file gives the same solution — same space group, same placement to a hundredth of a degree, LLG 10248 against 10247. Its amplitude columns come out as FP/SIGFP where Rugnux’s own MTZ writes F/SIGF, which matters only if you were naming columns by hand; the automatic choice is IMEAN/SIGIMEAN either way. Since Rugnux writes the MTZ and the mmCIF in the same run, the conversion is only worth knowing about for a file that arrived without its .mtz.

Small-molecule structures with SHELXT and SHELXL

A small-molecule sweep (see Small-molecule data) goes to SHELX as xtal.hkl, the rotation run’s scaled reflections unmerged, in HKLF 4. shelxt and shelxl come with CCP4 as well as with the SHELX distribution. HKLF 4 carries no metadata, so the instruction file is written by hand: the wavelength and the cell from the report, the lattice and symmetry of the space group the report names, and the cell contents, which only the user knows.

cp xtal.hkl struct.hkl
+w=$(grep '^WAVELENGTH= ' xtal_report.txt | cut -d' ' -f2)
+cell=$(grep '^UNIT_CELL_CONSTANTS= ' xtal_report.txt | cut -d' ' -f2-)
+cat > struct.ins <<eof
+TITL struct
+CELL $w $cell
+ZERR 4 0.001 0.001 0.001 0.01 0.01 0.01
+LATT 1
+SYMM -X, 0.5+Y, 0.5-Z
+SFAC C H N O
+UNIT 40 48 4 8
+END
+eof
+shelxt struct
+

The LATT and SYMM lines here are those of P2₁/c (SPACE_GROUP_NAME= P 1 21/c 1); write the ones of the group in the report, from International Tables or from a program that writes them. SHELXT takes the Laue group and the lattice from them and determines the space group itself, so where the report’s SPACE_GROUP_CENTRE= is NOT_DETERMINED its choice is a second opinion on the centre of symmetry. ZERR carries Z and the cell’s standard uncertainties; the values above are placeholders, since the report gives the cell without them. SHELXT writes its best solution as struct_a.res, with struct_a.hkl beside it in the setting it chose, and refinement continues from those:

cp struct_a.res struct_a.ins
+shelxl struct_a
+

What SHELXL makes of the file:

  • It merges the equivalents itself (MERG 2, its default) and prints R(int) and R(sigma) from them, which a merged file would hide. In a group without a centre of symmetry it keeps the Friedel opposites apart, so the absolute structure (the Flack parameter) comes from the same file, with no -A at processing.

  • The scale is arbitrary: the intensities were multiplied by one factor so the largest fits the F8.2 field. SHELXL refines its own overall scale, so nothing has to be done about it.

  • There is no batch column, which in HKLF 4 would select a BASF scale factor; the reflections are on one scale already.

  • What the merge rejected is not in the file: the outliers, and the measurements left out for an overloaded pixel (OBSERVATIONS_REJECTED_OVERLOAD= in the report). A crystal whose strongest low-order reflections were overloaded is missing them, and SHELXL cannot say so; the report can.

  • The resolution stops at the run’s cut, as in every written file. Run with --resolution-cutoff off to hand SHELXL everything the detector recorded.

Experimental phasing with SHELX

shelxc, shelxd and shelxe come with CCP4 (phenix does not ship them). The input is myrun.hkl, and it is the only one of the three reflection files that works: SHELXC 2016/1 reads XDS and SHELX formats, not MTZ, and SAD myrun.mtz gets ** Cannot open file myrun.mtz ** — after which SHELXC exits 0 and writes nothing, so a script has to check for the _fa.hkl it should have produced rather than trust the exit status.

Nothing has to be switched on to get the anomalous signal. A default rotation merge keeps the Bijvoet split, whether or not -A was given: myrun.mtz carries I(+)/I(-) and F(+)/F(-) beside the means, and myrun.hkl holds every observation unmerged at the index it was measured at, so SHELXC sees both hands. -A changes what the merging statistics are counted over, not whether the signal is in the file. The one case with no anomalous signal at all is a stills run, which computes no Bijvoet split; there myrun.hkl holds means only and there is nothing for SHELXC to work with. myrun_unmerged.mtz is not part of this chain (it is unscaled), so a run with --no-export-unmerged is not missing a file SHELX needs.

HKLF 4 carries no metadata, so the cell and the space group have to be repeated on the SHELXC command — take them from UNIT_CELL_CONSTANTS and SPACE_GROUP_NAME in section 2 of the report, with the spaces taken out of the group’s name. (SHELXC also puts a wavelength in the CELL line of the .ins files it writes; that is its own 0.98 Å default, not anything read from the data, and neither SHELXD nor SHELXE uses it.) The whole chain, for a sulfur substructure — the cell and group here are tetragonal lysozyme’s, so substitute your own report’s:

shelxc sad <<eof
+SAD myrun.hkl
+CELL 79.0 79.0 38.0 90 90 90
+SPAG P41212
+FIND 10
+SFAC S
+MAXM 2
+eof
+shelxd sad_fa
+

SHELXC’s own table is the first honest look at whether this is worth continuing — <d"/σ> should be about 0.80 where there is no anomalous signal. Two sweeps are quoted below, both collected at 5 keV for the sulfur signal: a cubic one that went all the way, and a tetragonal one that did not. The cubic one, 2.5 Å at 95 % completeness and multiplicity 30, reads:

 Resl.   Inf. 13.02  8.01  6.03  4.93  4.22  3.71  3.33  3.04  2.80  2.60  2.43
+ <I/sig>   108.8  91.4  63.0  64.7  70.6  61.8  45.5  34.7  23.7  12.4   5.0
+ %Complete  96.2 100.0 100.0 100.0 100.0 100.0 100.0 100.0 100.0  99.0  72.7
+ <d"/sig>   2.58  5.06  3.97  2.92  2.36  1.68  1.50  1.33  1.48  1.38  1.79
+

SHELXD will separate space groups the merged intensities could not. That sweep’s report named a body-centred cubic pair as indistinguishable, so SHELXC and SHELXD were run once per candidate — same reflections, same FIND, only SPAG different. One gave CC 37.93 / CC(weak) 14.05 / CFOM 51.98 and the other CC 46.76 / CC(weak) 22.61 / CFOM 69.37. The substructure is where the screw axis shows itself, and the second group is the right one. This is the same handover as Phaser’s arrived at from the other side, and it is worth doing whenever SPACE_GROUP_ALTERNATIVES is not NONE — SHELXD takes seconds, and the pair of runs costs less than reprocessing anything.

SHELXE decides the hand, and says so. Run it twice, -i inverting the substructure. -s is the solvent fraction, -h says the substructure atoms belong to the native structure, as sulfur does, and -a turns on autotracing, which is what actually makes the two hands separate. The two runs write sad.pdb and sad_i.pdb, so they can share a directory:

shelxe sad sad_fa -h -s0.62 -m20 -a15 -q
+shelxe sad sad_fa -h -s0.62 -m20 -a15 -q -i
+

At 63 % solvent the two hands came out at 42.93 % and 15.28 % for the autotrace CC against the native data — pseudo-free CC 66.49 against 37.12, map contrast 0.87 against 0.44, 215 traced atoms — which is a solved structure, from myrun.hkl and nothing else. Where the group is one of the 22 that come in enantiomorphic pairs, SHELXE makes the group change itself: the inverted run prints ** Space group converted to enantiomorph ** and writes the changed group into the CRYST1 of its traced model, so the answer is readable off the output file the same way it is off Phaser’s.

A negative result, for calibration. A tetragonal dataset at the same wavelength with the same kind of substructure, but 87 % complete at multiplicity 20 rather than 95 % at 30, gave a plausible SHELXD CFOM 47.62 and then failed at the hand: 15.33 % against 15.60 % autotrace CC, map contrast 0.33 either way. That is not a discrimination and it is not a solution. Nothing about the file was the limit — the anomalous signal SHELXC measured on it was real, <d"/σ> reaching 4.2 — so the reading is that sulfur phasing wants the completeness and the multiplicity, and a .hkl from a sweep that does not have them will get this far and no further.

Comparing the geometry with XDS

Every run logs the detector geometry a second time in XDS’s convention, so it can be read straight across against the IDXREF.LP / CORRECT.LP of an XDS run on the same data:

XDS convention: ORGX= 1091.00 ORGY= 1137.00 DETECTOR_DISTANCE= 75.0000
+XDS convention: DIRECTION_OF_DETECTOR_X-AXIS= 1.000000 0.000000 0.000000
+XDS convention: DIRECTION_OF_DETECTOR_Y-AXIS= 0.000000 1.000000 0.000000
+XDS convention: INCIDENT_BEAM_DIRECTION= 0 0 1 X-RAY_WAVELENGTH= 1.000000 QX= QY= 0.075000
+XDS convention: ROTATION_AXIS= -1.000000 0.000000 0.000000
+

XDS is never given this geometry — the XDS plugin supplies image data only, and XDS refines its own from XDS.INP — which is what makes the comparison worth having. The two laboratory frames coincide (x along increasing detector column, y along increasing row, z along the beam), so the numbers are directly comparable, and a tilt appears as the two detector axis vectors rather than as angles, which is how XDS reports it after refinement. Two things to keep in mind: ORGX/ORGY are 1-based, because XDS counts pixels from 1 and Jungfraujoch from 0; and they are the PONI, the same quantity Jungfraujoch’s beam centre is — so no correction is needed — but not the direct beam once the detector is tilted (see above).

\ No newline at end of file diff --git a/RUGNUX_OVERVIEW.html b/RUGNUX_OVERVIEW.html new file mode 100644 index 000000000..6b1706272 --- /dev/null +++ b/RUGNUX_OVERVIEW.html @@ -0,0 +1 @@ + What Rugnux does — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

What Rugnux does

The map of a run, in the order it happens — one paragraph per stage, each linking into the data-analysis reference where the algorithm lives. The walk-through is a rotation run with the defaults; stills differences are at the end.

Open the dataset. The geometry, wavelength and goniometer come from the file (What Rugnux reads). A goniometer axis makes it a rotation run, none makes it serial stills — nothing is asked of the user.

Pre-scan. A projection of frames spread over the sweep (60 by default) finds the beam-stop shadow and masks it (§1.5), masks pixels its frames show to be defective (§1.6), measures the beam centre from the isotropy of the scattered background and compares it with the file’s (§1.4), and reads how wide this crystal’s spots are, which sets the integration radius (§9.5), and how much longer they are along their radius than across it, which is the beam’s bandwidth where the file does not state one (§9.6), and how much wider still they grow away from the beam, which sets the integration footprint (§9.1). In a GPU build the pre-scan runs on the card.

Spots. Every image is decoded — on the GPU straight from the compressed chunk (§0) — and one fused pass computes the azimuthal profile and finds the spots against each image’s own per-resolution-ring noise (§2–§3). The ice-ring score is read off the same profile.

Indexing. The spots of a sample of frames are rotated back to a common crystal frame and the FFT search looks for periodicity over thousands of directions; candidate cells are Niggli-reduced, classified by Bravais lattice, refined both constrained and triclinic, and decided on how many validation frames each actually indexes (§4–§7). A de-novo run also tries a second hypothesis with the shortest accepted axis lowered from 10 to 5 Å, for small-molecule cells. The first pass is indexed a second time at the beam centre the pre-scan measured; the file’s centre is kept unless it indexes nothing, gives an axis harmonic of the measured centre’s lattice, or loses to it when both first passes are merged (§1.4). A failed pass triggers the discrete rescues — the rotation-axis sign, the beam-centre search — before anything is given up on.

First integration pass. At the geometry in the file, every frame is predicted (§8) and profile-fit integrated (§9); partials are combined into fulls, scaled and merged (§10).

Geometry post-refinement. From those reflections the detector distance, beam centre and the cell scale / rotation axis are refined over all frames at once, each step committed only if it improves a held-out residual (§7.5).

Second pass. The sweep is re-indexed de novo and re-integrated at the refined geometry; this pass is the canonical output, and a guard compares the two passes and keeps the better one (reported as PASS= / PASS_DECISION= in the report).

Space group. On the P1 merge of the final pass, the point group is scored operator by operator on resolution-normalised intensities and the screw axes, glide planes and centring are read from the systematic absences, with the centre of symmetry from the intensity distribution where the absences leave it open (§13.1); twinning and translational pseudo-symmetry are checked beside it (§13.2). CANNOT_DETERMINE and an enantiomorphic pair are real answers here, not evasions.

Scale and merge. In the determined group: per-frame scales, the cross-validated correction surfaces (decay, absorption, modulation), the error model and ISa, outlier rejection, the CC1/2-based resolution cut, the anisotropy description, French–Wilson amplitudes and the R-free flags (§10, §13.3–§13.5).

Write. The merged .mtz / .cif / .hkl, the unmerged MTZ, the P1 cross-check and the results report land next to each other (Output files); with --model, validation runs first and the maps and the placed model are written too (§14).

Stills instead. Serial data skip the two-pass machinery: each image is indexed independently (with the known-cell ffbidx indexer where a cell is given), partiality comes from a per-crystal orientation-tilt post-refinement rather than a rocking curve, and a merohedral indexing ambiguity has to be broken per image, at integration time, against a reference or a model (Advanced ▸ the indexing ambiguity).

\ No newline at end of file diff --git a/RUGNUX_REPORT.html b/RUGNUX_REPORT.html new file mode 100644 index 000000000..9e1595e00 --- /dev/null +++ b/RUGNUX_REPORT.html @@ -0,0 +1,32 @@ + The results report — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

The results report

<prefix>_report.txt records what the run determined, next to the reflection files. It is written on every --mode mx and --mode scale run that has an output prefix — there is no option to enable or disable it. Two cases follow from that:

  • An empty output prefix (-o "", the “compute the statistics, persist nothing” mode) writes nothing, the report included.

  • --no-merge still writes a report. It determined an indexing and a geometry result, and those are recorded; the merging section then says MERGE= NOT_PERFORMED rather than being omitted, so the absence is a statement and not something a reader has to infer.

The report is never allowed to fail a run: if it cannot be written (unwritable path, full disk) the failure is logged as a warning and the run finishes normally.

Exit status. Rugnux exits 0 when the run completed — everything it determined, declined to determine (CANNOT_DETERMINE) or warned about is in the report — and non-zero when the run stopped: unreadable input, no usable lattice, a -S group the indexed lattice cannot host, an indexer that could not run. The reason goes to the terminal, and the report may not exist in that case — so a script branches on the exit status first and greps the report second.

Format

The model is XDS’s CORRECT.LP: prose and tables a crystallographer reads top to bottom, with a structure a script can consume without parsing prose. Every line is one of three kinds — a KEY= value data line, a # comment, or blank — so grep -v '^#' leaves the data alone (since REPORT_VERSION= 8; before that, comment lines had no prefix).

  • KEY= value assignment lines. Every number worth extracting is one, so a consumer gets it with a single grep '^ISA= ' and never has to read a sentence. Key names are stable.

  • # comment lines — everything else: the prose, the section banners, and the fixed-width tables (stable header row; the resolution shells, the space-group candidates, the sweep-quality ranges).

  • # WARNING: lines, one per finding, in plain English: # WARNING: Frames 500-600 out of beam (10.1 deg, scale 0.12 and CC 0.30 of the run, 2% scaled). grep '^# WARNING:' finds every one.

The worked examples on this page are shown with the leading # stripped for readability; in the file itself every such line starts with #.

A quantity the run did not measure writes no key at all, and the fixed-width tables print - in its place. There is one rule and no placeholders — no nan, and no 0.0% that reads as a measured total failure — so a consumer must treat an absent key as not measured rather than assume every key it knows about is present. A measured value always prints, including a negative one. The keys a script meets this on first are SIGANO= and CC_ANOM=: a rotation merge measures them whether or not -A was given, but where no Bijvoet pair could be split in both hands — a stills merge without -A, or too few pairs — the quantity does not exist and the key is absent; COMPLETENESS=, MULTIPLICITY=, I_OVER_SIGMA=, R_MEAS=, CC_HALF= and WILSON_B= follow the same rule.

R_MEAS= is the ordinary R_meas, every observation counted once, as XDS and AIMLESS count it - the one to set beside theirs. R_MEAS_WEIGHTED= (and REFRES_R_MEAS_WEIGHTED=) weights each observation as the merge weights it - by 1/sigma^2 under the error model - with each reflection’s weights normalised to their effective number, so where a reflection’s observations share one sigma it equals R_MEAS=. A stretch the crystal barely diffracted over is in the merge at the small weight its scaled-up counting error gives it, and is in R_MEAS_WEIGHTED= at that weight too, rather than setting the number with its noise; a large gap between the two says weak frames are kept at low weight. MULTIPLICITY= stays a count of observations.

OBSERVATIONS_REJECTED= counts the observations the merge’s outlier tests removed, and OBSERVATIONS_REJECTED_WILSON= the part of them the Wilson test took (each listed in the developer report). OBSERVATIONS_REJECTED_OVERLOAD= is separate from both: the rotation measurements left out because a pixel of the spot was overloaded on one of its frames, as XDS leaves out an overloaded reflection. Outlier rejection takes observations out of R_MEAS= and CC_HALF= too, so a run that rejects much scores better on both; read the counts beside them.

The last blocks before END OF REPORT are the authorship and the acknowledgement: who wrote rugnux, its licence (GPLv3 — free to use for academic institutions and commercial companies alike) and where releases are published, then the credit to the X-ray research community whose methods rugnux implements and the open-source projects it is built on — both credited in ACKNOWLEDGEMENT.md beside LICENSE and THIRD_PARTY_NOTICES.md in the installed package. rugnux prints the acknowledgement at startup as well.

REPORT_VERSION= is the format’s own version. Key names, table columns and the reason vocabulary below are an interface other software may depend on: they do not change without that number moving. Adding a key does not move it — a consumer that greps for what it needs is unaffected by one more line. It counts releases, not changes: it moves at most once per release, however many format changes that release carries, because a reader only ever meets the format that was released.

The header block above section 1 records how the result was produced: RUGNUX_VERSION= and RUGNUX_DOWNLOAD= (the release page of exactly that version), DATE=, INPUT_FILE= and OUTPUT_PREFIX=, plus

  • RUGNUX_GIT= — the commit the binary was built from, stamped at build time so it cannot go stale in a reconfigured tree; a -dirty suffix marks a build from uncommitted changes.

  • BUILD_CXX_FLAGS= — the compiler flags of the build (NONE for a plain configure). The build is set up so that the CPU level it targets (-march) does not change the results, only the speed; where two reports of one commit still disagree, this is the first line to compare.

  • COMMAND_LINE= — the invocation as one shell-ready line, arguments containing spaces quoted.

  • WALL_TIME= — the whole invocation in seconds. It covers everything the process did, opening the file and setting up included, so it is a little larger than the Processing time printed on stdout, which starts once the analysis does.

  • GPU_COUNT= and GPU= — how many GPUs were visible and what they are, e.g. GPU= 4x NVIDIA A100-SXM4-80GB; several models on one machine are listed as separate groups. GPU_COUNT= 0 appears on its own, with no GPU= line, when nothing was visible — which is the first thing to check when a run took far longer than expected. Rugnux prints the same line at startup, before the run, so a missing GPU can be caught while there is still time to stop.

Rates, per-image costs and progress remain on stdout only.

Sections, in order: the SUMMARY, then 1. DATA SET AND GEOMETRY, 2. CRYSTAL, 3. MERGED DATA, 4. DIAGNOSTICS, and 5. MODEL VALIDATION only with --model (see The summary below). The numbering is contiguous, and the fifth section appearing renumbers nothing; a stage that did not run states that inside its section — MERGE= NOT_PERFORMED — rather than the section disappearing.

SPOT_RESOLUTION_ESTIMATE= in section 1 is how far the merged data are expected to reach, read off the found spots alone — no lattice, no integration, no merge — so it is there on a run that never merges, and on a run that does it can be read against INCLUDE_RESOLUTION_RANGE in section 3. It is a prediction, good to about 0.2 Å on rotation data; nothing is cut on it. It is not limited to what the detector records: where it reads finer than the high-resolution end of INCLUDE_RESOLUTION_RANGE, the crystal diffracts past the corner and the run was detector-limited.

JFJOCH_DATASET_SETTINGS= in section 1 is the geometry the run integrated at — on a rotation run the post-refined one — written as the object jfjoch_broker takes it in: the four required properties of dataset_settings in broker/jfjoch_api.yaml, joined by the three poni_rot*_rad angles whenever any of them is non-zero (a body without them states a flat detector), on one line of valid JSON, so a refined beam centre and distance can go back to the instrument for the next collection without anyone retyping them.

JFJOCH_DATASET_SETTINGS= {"beam_x_pxl": 2078.24, "beam_y_pxl": 2233.92, "detector_distance_mm": 190.311, "incident_energy_keV": 12.4000}
+
grep '^JFJOCH_DATASET_SETTINGS=' out_report.txt | cut -d' ' -f2- > geometry.json
+

Which pass. A rotation run integrates twice — once at the geometry in the input file, then again at the post-refined geometry — and can integrate a third time if a guard rejects the second pass. There is one report, for the pass that became the canonical output, and PASS= / PASS_DECISION= (--developer) in section 1 say which pass that is and on what evidence, so no number in the file is ambiguous about which geometry produced it.

Not in the report: timing, frame rates, thread counts, per-image progress and library banners. Those are process, not result, and stay on stdout.

The summary, and what the run decided

The file opens with a SUMMARY section, above everything it summarises. It exists because the report used to have no evaluative line anywhere until its last section: a run that produced garbage and a run that produced a textbook data set read identically for their first three hundred lines.

  • VERDICT= is a closed vocabulary — OK, WARNINGS, UNUSABLE, FAILED. FAILED means no lattice was determined or the run was cancelled (a sample with no crystal in the beam ends here, with NO_LATTICE, and the run exits with status 1); UNUSABLE means the data merged but carry no usable signal; WARNINGS means something else needs attention; OK means nothing did. It is decided from the warnings the rest of the report produced, so it introduces no new analysis and cannot disagree with the sections below it.

  • VERDICT_TEXT= is one to three sentences of free text saying the same thing in English.

  • PATHOLOGY_FLAGS= is the type of each condition that fired, from a closed vocabulary, so a consumer switches on a code rather than parsing a sentence: NO_LATTICE, INDEXING_AMBIGUITY, SYMMETRY_AMBIGUITY, CENTERING_UNTESTED, UNUSABLE_MERGE, LOW_COMPLETENESS, SWEEP_GAPS, GONIO_SCALE, SPINDLE_CAP, ANISOTROPY, TWINNING, PSEUDO_TRANSLATION, LATTICE_TRANSLATION, MODEL_HAND, MODEL_NOT_VALIDATED, REFERENCE_MISMATCH, CANCELLED, RESOLUTION_FIT, FLIGHT_PATH, GEOMETRY_NOT_CONVERGED, SCALING_NOT_CONVERGED, HARMONIC_CONTAMINATION, SUPERCELL_POSSIBLE, MULTIPLE_LATTICES, ICE_RINGS, POWDER_RINGS. NONE when nothing fired. A code appears if and only if its warning fired, so the flags and the WARNING: lines are two renderings of one list. Thresholds are deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on data that turn out fine — the closed type for machinery, the open sentence for a person.

  • Below them, one line each, the facts a reader needs before reading further: space group, cell, mosaicity, the powder and ice rings, multiple lattices, resolution, completeness, signal, anomalous signal, anisotropy, pseudo-symmetry, twinning, supercell, radiation damage and the sweep. A line states a finding that is not a condition too - weak powder rings, a possible supercell no stronger than correct cells show, an arbitrary indexing choice (Indexing choice, a rotation sweep whose cell admits alternative indexing: one sweep is indexed consistently, so it matters only against other data; the operators are INDEXING_AMBIGUITY_OPERATORS= in section 2). On serial stills the same ambiguity mixes the hands in the merge, and is an INDEXING_AMBIGUITY warning. INDEXING_RATE= - the share of frames the per-image indexer took - is a stills measure: a rotation run indexes the sweep as a whole, so there it is written only with --developer, and NO_LATTICE rests on whether the sweep was indexed.

  • WARNING_COUNT= and the WARNING: lines follow, in the same section. They are what they always were; they have moved from the bottom of the file to the top.

Then the numbered sections: 1. DATA SET AND GEOMETRY, 2. CRYSTAL, 3. MERGED DATA, 4. DIAGNOSTICS, and 5. MODEL VALIDATION only with --model.

The reference-range table

--report-resolution <dmin>[,<dmax>] adds a second block of merging statistics to section 3, the REFRES_* keys and a second shell table, over the resolution range it is given rather than the range the run chose. It exists for comparison: another program’s table is at that program’s range, and running rugnux at that range (--scaling-high-resolution) is not the same run — the cut moves, and with it the symmetry decision and everything downstream of it. The reference table is instead the same merged reflections binned again, so nothing is processed differently whether or not it is asked for; the merged files and every decision are byte-for-byte the run’s own.

  • REFRES_RANGE= is the requested range (dmax dmin, as INCLUDE_RESOLUTION_RANGE; INF when the run has no low-resolution limit and none was given). REFRES_MEASURED_RANGE= is the coarsest and finest merged reflection that actually landed in it.

  • REFRES_COMPLETENESS=, REFRES_MULTIPLICITY=, REFRES_I_OVER_SIGMA=, REFRES_R_MEAS=, REFRES_CC_HALF=, REFRES_SIGANO=, REFRES_CC_ANOM=, REFRES_UNIQUE_REFLECTIONS= and REFRES_TOTAL_OBSERVATIONS= are the overall numbers over that range, under the same absent-when-unmeasured rule as their section-3 namesakes. REFRES_ISA= is the error model refitted on the reflections of this table alone, in XDS’s convention, so it reads against an ISa produced at that range; the table itself is merged under the run’s own model.

  • The table holds only what the run kept. Where the reference range is finer than the run’s own limit, the shells past that limit are empty by the run’s decision — it judged them to carry no signal and did not merge them — and are printed as past the run's own limit of X A: not merged (N possible) rather than as zeros; the shell the limit falls inside is marked. REFRES_SHELLS_PAST_LIMIT= counts those shells (0 when the range lies within the run’s own), so a consumer can tell not merged from a measured zero. REFRES_COMPLETENESS counts their reflections as missing; every other overall number is over the shells the run reached.

The shells are equal steps in 1/d² between the two bounds, as XDS’s are, so at XDS’s INCLUDE_RESOLUTION_RANGE the two tables read row for row.

The developer report

--developer renders the same report in full. The default report carries what a person deciding keep or recollect acts on; --developer adds the pipeline’s own internals — the anisotropy detection gate’s parameters, the space-group operator and candidate tables, the model-fit null, the sweep-quality internals, the twinning statistics measured before the search, and the long explanatory passages — plus advisories about the cut’s own behaviour that no user can act on.

Nothing is computed differently and nothing is lost by leaving the flag off: the report is built once, in full, and the flag selects how much of it is written. Every key the default report writes, --developer writes too.

The space group, and the Sohncke answer beside it

The space group lives in section 2: SPACE_GROUP_NAME= / SPACE_GROUP_NUMBER=, with SPACE_GROUP_ALTERNATIVES= naming the candidates the data could not separate — an enantiomorphic partner among them — and, on a group that has such a partner, SPACE_GROUP_ENANTIOMORPH= saying whether the hand is open (UNDETERMINED), asserted by the user (GIVEN), or taken from an accepted model (ASSUMED_FROM_MODEL).

SPACE_GROUP_SCREW_UNDETERMINED= names the axes whose screw these data could not decide at all, as one or more of a, b, c; when every screw the data could show was judged it reads NONE and, like the other keys whose answer is “nothing to report”, is written only with --developer. It fires when the axial row a screw lives on was never recorded — it lay in the spindle’s blind cone, or outside the resolution range — so nothing was measured that could confirm or refuse the screw. On such an axis SPACE_GROUP_NAME= is not an answer: the answer is that group or any of SPACE_GROUP_ALTERNATIVES, and no measurement in the run chooses between them. What is written to the .mtz/.cif/.hkl is unchanged — the member claiming no screw, because a reflection file must carry one group — and the report prints the axis, why the row could not be judged, and what the alternatives are. To settle it, record the missing row: a different crystal orientation, or a sweep that reaches it. With --developer the space-group section names the same axes in prose beside the per-zone screw table.

SETTING_OPERATOR= and SETTING_SOURCE= in section 2 say which axes the files are on: the reindexing operator from the axes the space group was determined on (CCP4 style, h,k,l where nothing moved) and what chose the setting - STANDARD, CELL (-C), SPACE_GROUP (a non-standard -S), MODEL (a --model that fits) or REFERENCE (-z). SPACE_GROUP_NAME, the cell and every file the run writes are in that setting (see RUGNUX_ADVANCED.md, “The setting the files are written in”).

SOHNCKE_SPACE_GROUP= in section 2 is written on every run whose space group was determined by the search; a run given its group with -S has no Sohncke candidate to name and omits the key. Where the search found a glide plane, SPACE_GROUP_NAME= names the group with it and this names the best group without - a crystal of chiral molecules, which is any protein, cannot have a glide plane or an inversion centre, so a reader who knows their sample is a protein reads this key and needs no second run. Where no glide was found the two keys read the same, deliberately: greppability is the point, and a key that appears only sometimes has to be tested for before it can be read.

SPACE_GROUP_CENTRE= says how the centre of symmetry of a glide group was settled; it is written only where a non-Sohncke group was adopted. IMPLIED_BY_ABSENCES - the group is centrosymmetric and no group without a centre predicts its absences (P2_1/c, Pbca, Ia-3d), so the screws and glides found imply the centre. ABSENT_BY_ABSENCES - the reverse (I-42d, Fdd2): no centrosymmetric group predicts these absences. NOT_DETERMINED - a group with exactly the same absences and the other answer exists (C2/c and Cc, Pnma and Pna2_1), and it is in SPACE_GROUP_ALTERNATIVES. Friedel’s law hides the centre from the symmetry of merged intensities, so the centrosymmetric group is written by default: a missed centre is the common error in small-molecule space-group assignment. ABSENT_BY_STATISTICS - the same situation, but the intensity distribution read acentric and the lattice ruled twinning out, so the group without a centre was written; CENTRE_TWINNING_EXCLUDED= says, for these two, whether the lattice ruled twinning out. Two non-centrosymmetric groups can also share absences (I-42d and I4_1md); the lowest-numbered is written and the other listed as the alternative.

CENTRE_STATISTICS_* are the intensity statistics behind that, read on every searched run on the general reflections - those no operator of the Laue class but the identity maps to themselves, which are centric exactly when the crystal is centrosymmetric; CENTRE_STATISTICS_REFLECTIONS= is how many there were. CENTRE_STATISTICS_L= is <|L|> (Padilla & Yeates; 0.5 acentric, 2/pi centric), CENTRE_STATISTICS_E2_MINUS_1= <|E^2-1|> (0.736 / 0.968) and CENTRE_STATISTICS_NZ01= N(0.1), the fraction with E^2 < 0.1 (0.095 / 0.248). Each has an _F beside it (CENTRE_STATISTICS_L_F=, _E2_MINUS_1_F=, _NZ01_F=) that places the statistic between the values the same reflections read when simulated acentric (0) and centric (1) with their own shell means and sigmas; CENTRE_STATISTICS_CONTROL_F= does the same for the reflections centric in every candidate, which must read centric. The statistics are written only where L-test pairs and control reflections were found. CENTRE_STATISTICS_VERDICT= is ACENTRIC when <|L|> reads _F <= 0.3 and the other two <= 0.5, NOT_APPLICABLE when any of the three falls outside -0.3..1.3 (a structure with a few heavy atoms on special positions reads beyond centric) or the control reads below 0.5, and INCONCLUSIVE otherwise. A twinned crystal reads acentric whether or not it has a centre, which is why the verdict decides only with twinning excluded by the lattice.

SPACE_GROUP_ENANTIOMORPH= in section 2 reads ASSUMED_FROM_MODEL when the hand written in the files is the model’s. Assumed, not determined: merged intensities cannot see the hand at all — |F| is invariant under the change of hand — so an accepted model asserts it out of prior chemical knowledge. It is only ever written where MODEL_FIT= ACCEPTED, and the anomalous difference map vetoes it outright where the map says the model and the data are in opposite hands. (Before REPORT_VERSION= 6 this value was spelled DETERMINED_FROM_MODEL and was emitted whenever a model file merely parsed.)

Sweep quality, the disposition, and their vocabularies

Section 4 lists the stretches of the sweep over which the crystal delivered much less than the rest of the run — the feedback a beamline control system needs to tell an operator that a crystal should be recentred or recollected — says what became of each of them, measures what keeping each one costs the merged intensities, and drops the stretches that cost too much.

SWEEP_QUALITY_STATUS= COMPUTED
+SWEEP_QUALITY_COUNT= 2
+SWEEP_QUALITY_REASONS= no_diffraction crystal_out_of_beam weak_diffraction loss_of_centring radiation_damage inconsistent_with_merge
+SWEEP_DISPOSITIONS= merged downgraded rejected
+FRAMES_MERGED= 1663
+FRAMES_DOWNGRADED= 101
+FRAMES_REJECTED= 36
+FRAMES_REJECTED_PCT= 2.00
+ROTATION_REJECTED_DEG= 3.6
+SWEEP_ROTATION= 180.0
+FLUX_PEAK_TO_TROUGH= 1.03
+SCALE_MODULATION_PEAK_TO_TROUGH= 1.00
+
+  FIRST_IMAGE   LAST_IMAGE   N_IMAGES  ROTATION  REASON                   SEVERITY   SCALE      CC   INDEXED  DISPOSITION  DELTA_CC_HALF  DELTA_CC_HALF_SE
+  -----------  -----------  ---------  --------  -----------------------  --------  ------  ------  --------  -----------  -------------  ----------------
+          500          600        101      10.1  crystal_out_of_beam          0.83    0.12    0.30      0.02  downgraded         +0.0004            0.0031
+          612          630         19       1.9  no_diffraction               0.98    0.02    0.00      0.00  rejected           -0.0481            0.0110
+  -----------  -----------  ---------  --------  -----------------------  --------  ------  ------  --------  -----------  -------------  ----------------
+

Of those keys, SWEEP_QUALITY_COUNT and the five disposition keys (FRAMES_MERGED, FRAMES_DOWNGRADED, FRAMES_REJECTED, FRAMES_REJECTED_PCT, ROTATION_REJECTED_DEG) are in the default report; SWEEP_QUALITY_STATUS, SWEEP_QUALITY_REASONS, SWEEP_DISPOSITIONS, SWEEP_ROTATION, FLUX_PEAK_TO_TROUGH and SCALE_MODULATION_PEAK_TO_TROUGH appear with --developer (the default report states in prose whether the diagnostic ran).

The three frame counts partition the sweep — every processed image is exactly one of them and they add up to the frame count — so FRAMES_REJECTED_PCT is the answer to “how much of this experiment was useless”. It is reported beside ROTATION_REJECTED_DEG on purpose: a percentage of frames moves when the same experiment is re-sliced, and a percentage of the rotation does not. The prose headline above the table states both.

A SWEEP_GAPS warning is given, one per range, where the degraded ranges together cover at least 1 % of the sweep’s rotation; below that - a frame or two - they are in the table and the summary’s Sweep line, and the warning lines in --developer.

SWEEP_QUALITY_STATUS distinguishes COMPUTED (the diagnostic ran; a count of 0 means the sweep was clean throughout) from NOT_COMPUTED (it did not run — no scaling and merging, or stills data). A consumer must not read a missing table or a zero count as “clean” without checking it. SWEEP_QUALITY_REASONS lists the whole vocabulary this version can emit, so an unknown code is distinguishable from a missing one.

Reason code

Meaning

no_diffraction

The range recorded essentially no diffraction from the indexed lattice.

crystal_out_of_beam

Frames were lost: over the range a per-image scale could be fitted far less often than over the run.

weak_diffraction

The frames all still index, but with much less intensity — the cause was not determined.

loss_of_centring

One cycle of modulation per revolution: the crystal is off the rotation axis.

radiation_damage

The range runs to the end of a sweep whose quality was already decaying.

inconsistent_with_merge

The frames diffract as the run does, but their intensities do not agree with it — the only evidence is DELTA_CC_HALF, so the cause is not named.

The vocabulary is closed and stable: a code is never renamed, and never reused for a different meaning. New codes are only ever added, and adding one moves REPORT_VERSION at the next release.

The columns are: FIRST_IMAGE/LAST_IMAGE — inclusive, in processed-image ordinals (the numbering of <prefix>_plot.txt and of every other per-image array rugnux writes; with -s/--stride the source image is start + ordinal * stride); ROTATION — the width of the range in degrees; SEVERITY — the fraction of the run’s typical diffracting power missing over the range, 0 (as good as the run) to 1 (nothing at all); SCALE and CC — the range’s mean per-image scale and CC-to-merge relative to the run median; INDEXED — the fraction of the range’s frames that were scaled at all; DISPOSITION — what became of it; DELTA_CC_HALF and DELTA_CC_HALF_SE — what keeping it costs the merged intensities, and how precisely that is known. Every range also appears as a WARNING: sentence in the SUMMARY, with the same cost in words. A range is split where its disposition changes, so each row is wholly kept or wholly rejected.

What the disposition means, and what decides it

Disposition

Meaning

merged

The frame’s observations are in the merged data at their own weight.

downgraded

They are in the merged data, but over a stretch the run itself flagged, carried at the reduced weight the frame’s own scale and sigmas give it. Nothing extra is subtracted: for weak-but-consistent data that reduced weight is the honest weight, and a second, invented per-frame weight would double-count with the σ’s.

rejected

Nothing of the frame reached the merge — because ΔCC1/2 convicted it, because an earlier guard dropped a frame whose scale had collapsed to an unusable number, or because the frame recorded nothing to drop in the first place. To a user asking how much of the experiment was useless these are the same answer, and the REASON column separates them.

DELTA_CC_HALF is ΔCC1/2: the overall CC1/2 of the merged data with the range minus the CC1/2 without it, evaluated over the reflections the range touches. Negative means keeping the range makes the merged intensities worse. It is computed in the σ-τ form — no random half-dataset split, so the same input gives the same answer every run — with each reflection’s error variance taken from the observed scatter of its own observations, not from the error model’s σ’s (a bad stretch claims the same σ’s as a good one, so an error-model estimate would read a stretch that adds noise as one that adds precision). It is a CC1/2 over the range’s own reflections, not over the whole dataset — a range that touches a few hundred reflections can carry a large ΔCC1/2 without the dataset’s headline CC1/2 moving by anything like as much. DELTA_CC_HALF_SE is the standard error of a CC1/2 on that many reflections, in the same units, and a ΔCC1/2 smaller than it says nothing.

A range is rejected only where ΔCC1/2 is both well below the rest of this run’s own batches and several standard errors below zero, and only where every frame in it is one the per-image channels call worse than the run’s typical frame — the scale and the CC to the merge, frame by frame and never as an average over the stretch, because an average cannot tell a uniformly bad stretch from a healthy arc lying beside a dead one. All three are needed: a healthy crystal merges at CC1/2 ≈ 0.999, where a harm of 0.001 in CC is already many standard errors, so significance alone convicts frames on clean data whose removal moves nothing; and ΔCC1/2 is itself measured against the merge, so removing whichever frames disagree with it most improves every agreement statistic whether or not anything was wrong with them — the decision has to be triggered by a channel that owes nothing to the merge, and only then confirmed by what the merge does. The test runs last, after the per-frame scale, the decay slope and the per-batch relative-B have been fitted, so it judges corrected data. The decision is taken over 10° batches — the same batches as the radiation-damage curve — and over those batches doubled, and doubled again, up to a quarter of the sweep: a defect much longer than a batch is invisible one batch at a time, because each batch inside it is judged against a merge that still contains the rest of the defect. Where the harm lies is then settled finely: each edge of the convicted stretch is slid frame by frame with the whole stretch re-measured at every position, so the range is reported where it actually lies rather than at the batch grid, and the reflection count the verdict rests on never shrinks with the edge. Each edge is then pulled back off any frame the per-image channels call normal, and what is left has to carry the verdict again on its own: a stretch that cannot be taken without healthy frames holds more than one thing and is not removed at all. A rejected stretch is never narrower than one rocking event, because the partials of one event are combined into the same intensities and inside it no frame can be judged apart from its neighbours. Never more than a quarter of the sweep is removed.

What ΔCC1/2 cannot do, because the report must not imply otherwise:

  • it says nothing about the cause: a shutter fault and a crystal that slipped have the identical signature, both integrating background, so the cause comes from the REASON column and never from the ΔCC1/2 itself;

  • a second lattice entering is invisible to it: those spots were never integrated, so they are not in the merged intensities it measures;

  • a centring drift that is pure attenuation reads ≈ 0. That is the right answer, not a blind spot: the data are weak but consistent, the σ’s already say so, and their ΔCC1/2 is the evidence that discarding them would cost completeness for nothing;

  • it is attributed to the frame carrying a rocking event’s peak partial, so it cannot resolve a single frame: a stretch narrower than one rocking event is never rejected, and the edges of a rejected stretch are soft to within half an event;

  • loss_of_centring needs ≥ 350° of sweep to be named at all. On a 90° sweep the same drift is still detected, only as crystal_out_of_beam or weak_diffraction — “cause not determined” here means this sweep cannot determine it, not that it is undeterminable.

The same finding is written per image into the _process.h5 as /entry/MX/sweepQuality and /entry/MX/frameDisposition, when one is written — see HDF5.

SPINDLE_SYMMETRY_AXIS_ANGLE_DEG= and SPINDLE_SYMMETRY_AXIS_ORDER= (--developer) in section 4 say how the crystal sat on the goniometer: the angle between the spindle and the nearest symmetry axis, and that axis’s order. They are descriptive: neither convicts nor clears the mounting on its own, because an aligned axis of any order maps the sweep’s blind cone onto itself while an axis near perpendicular does the same only when it is a lone 2-fold, and only the nearest axis is reported. (New in REPORT_VERSION= 7.)

SPINDLE_LOST_UNIQUE_FRACTION= in section 4 is the exact verdict the angle cannot give: the fraction (0-1, so 0.0300 means 3%) of unique reflections, to this run’s resolution limit, that the mounting made unmeasurable - the part of the sweep’s blind double cone that no operator of the measured point group maps onto measured territory, computed in the crystal’s actual indexed orientation. 0.0000 means the mounting cost nothing; the run warns when the group recovers less than half of the cone’s content. The same number is written to the master file as /entry/MX/spindleLostUniqueFraction, so a pipeline can read it from either output without parsing prose. Written on every rotation run that determined a space group and merged reflections.

Powder contamination

A crystalline phase other than the crystal, diffracting as rings among its reflections — hexagonal ice, a shower of microcrystals, a salt out of the cryoprotectant. It is measured on every run, in the pre-scan, from the spots found there, and reported whether or not anything acted on it: a user whose crystal sat in a powder is told so even where the run indexed perfectly well. Only hexagonal ice has rings that can be named in advance, so POWDER_RINGS_A is what this sample showed.

POWDER_RINGS_DETECTED= TRUE
+POWDER_RING_COUNT= 24
+POWDER_SPOT_FRACTION= 0.412
+POWDER_RINGS_SEPARABLE_TO= 2.31
+POWDER_RINGS_A= 3.897 3.671 3.447 ...
+POWDER_EXCLUDED_FROM_INDEXING= TRUE
+POWDER_INDEXING_D_MIN= 4.91
+

POWDER_RINGS_DETECTED is written on every merging run and is FALSE on nearly all of them; the rest of the keys appear only where rings were found. POWDER_SPOT_FRACTION is the share of the pre-scan’s spots the rings hold over the smooth fall-off around them — what the contaminant contributes, not what happens to lie in a ring band. POWDER_RINGS_SEPARABLE_TO is the resolution past which the rings crowd together too tightly to be told apart, and so the finest an indexing pass can be asked to trust on such a pattern; it is absent where they stay separable over the whole range, which is the ordinary case. POWDER_EXCLUDED_FROM_INDEXING says whether the run needed them left out to index at all (with POWDER_INDEXING_D_MIN the resolution the retried first pass used). Rings are detected far more often than they are excluded: exclusion happens only where a first pass found no usable lattice. The rings are split between those on a hexagonal-ice position (POWDER_ICE_RING_COUNT=, POWDER_ICE_SPOT_FRACTION=) and the rest - a salt, microcrystals, another phase (POWDER_NON_ICE_SPOT_FRACTION=); the two fractions add up to POWDER_SPOT_FRACTION=. The summary’s Powder line gives both. A warning under the ICE_RINGS flag is given where the ice rings hold at least 5 % of the spots, and one under POWDER_RINGS where the other rings do.

Hexagonal ice is also measured by the merge itself, which leaves reflections on the ice rings out of scaling (and keeps them in the merge) where it finds ice:

ICE_RINGS_DETECTED= TRUE
+ICE_RING_SCORE= 2.69
+ICE_SPOT_RATIO= 1.08
+ICE_REFLECTIONS_ON_RINGS_PCT= 25.7
+

ICE_RING_SCORE= is the strongest ice ring over the smooth radial background (fine-grained ice that makes a smooth ring), ICE_SPOT_RATIO= the pile-up of found spots on the ring positions against the ice-free flanks beside them (ice in large crystallites); 1 is no ice on either. ICE_RINGS_DETECTED= is TRUE where either passes its gate (1.5 and 2.0), and ICE_REFLECTIONS_ON_RINGS_PCT= is then the share of the integrated reflections set aside from scaling. The summary has an Ice line, and where the merge found ice an ICE_RINGS warning prompts a look at the ice-ring shells (one warning line, whichever of the two measurements raised it). The measurement is described in CPU/GPU data analysis ▸ Resolution and ice-ring handling.

Index-2 superstructure

On rotation data, section 4 reports whether the lattice the run adopted has intensity at half-integer positions it does not index. On 60 frames spread over the sweep, after each frame’s own integration, the lattice is predicted doubled along all three primitive axes and integrated to 3 Å; the reflections split into eight parity classes of h, k and l, of which 0 0 0 is the lattice itself and each of the other seven is one index-2 superstructure. Nothing is decided on it: whether a superstructure belongs in the cell is as much the depositor’s call as the data’s, and several crystals whose accepted cell is the sub-cell carry one.

SUPERCELL_CLASS= is the parity class with the most intensity over 20–3 Å. SUPERCELL_OCCUPANCY_PCT= is its mean intensity against the lattice’s own reflections on the same frames, and SUPERCELL_ROCK_PCT= (± SUPERCELL_ROCK_SE_PCT=) the part of it that follows the partiality the way a Bragg reflection does, from a fit I = a + b p over the class, on the same scale. SUPERCELL_I_OVER_SIGMA= is its mean I/σ. A class near zero on both is empty. One that is occupied and rocks is a superstructure whose reflections this run did not integrate; SUPERCELL_DOUBLED_CELL= is the Niggli-reduced cell the lattice would double to, to give with -C to process on it. One that is occupied but hardly rocks is diffuse or disordered intensity rather than Bragg reflections.

SUPERCELL_POSSIBLE= is TRUE where the class is measured (SUPERCELL_I_OVER_SIGMA= at least 0.5) and part of it rocks like Bragg reflections (SUPERCELL_ROCK_PCT= at least 2 %, three standard errors clear of zero). It is advice to check, not a finding: the same numbers come from a real doubled cell and from a correct cell with weak ordered intensity between its reflections - or with further lattice domains whose spots land on the half-integer positions - and which of the two a structure is decided by refinement. Process both settings - the run’s cell, and SUPERCELL_DOUBLED_CELL= given with -C - and compare them there. A TRUE is also a warning under the SUPERCELL_POSSIBLE flag, worded as a prompt to check: on a battery of rotation data sets most of the crystals it named refine normally in the sub-cell (rocking parts of 2-16 %), but a real doubled cell can read inside that range too. The summary’s Supercell line carries the same advice, and says where further lattice domains were found that may put spots on the half-integer positions.

Further lattices

On rotation data, once the first pass has its lattice, section 4 reports what the spots that lattice leaves over index to. Every spot of the spread and validation frames the main lattice takes is set aside; the rest, ice left out, go to a fresh rotation indexer. A lattice it finds is kept when the leftover validation spots sit on it far more often than they do with each frame’s spots displaced to another frame’s spindle angle - the null the first pass judges its own lattice against - by at least five standard deviations (EXTRA_LATTICE_<n>_Z=). Its spots are set aside in turn and the search goes on, up to three lattices. A lattice of the same cell within 1° of the main one is the main lattice’s own spots its tolerance missed, and is not listed. Nothing is decided on any of this: only the main lattice is integrated, whatever is found.

MAIN_LATTICE_SPOTS_PCT= and MAIN_LATTICE_INTENSITY_PCT= are the main lattice’s share of the non-ice spots of those frames and of their summed intensity; EXTRA_LATTICE_COUNT= is the number of lattices listed. For each, numbered from 1 in the order found:

  • EXTRA_LATTICE_<n>_KIND= — DOMAIN: the same cell, misoriented, over the whole sweep (a split crystal; more than 10° from the main lattice, the report calls it a second crystal); TWIN_DOMAIN: the same, turned by 180° (within 3°) — a non-merohedral twin domain; SEGMENTED: the same cell, with at least 60 % of its spots in 2 of 8 equal blocks of the sweep — the crystal in the beam changes along the rotation; RELATED_CELL: a cell whose volume is 1 to 8 times, or a 1/2 to 1/8 of, the main one’s — a question about the main cell (see the supercell keys above and the harmonic) rather than a second crystal; FOREIGN: an unrelated cell — a second crystal, or a main lattice that is not the crystal’s.

  • EXTRA_LATTICE_<n>_LATTICE= and EXTRA_LATTICE_<n>_CELL= — its crystal system, centring and conventional cell.

  • EXTRA_LATTICE_<n>_VOLUME_RATIO= — its reduced cell volume over the main one’s.

  • EXTRA_LATTICE_<n>_MISORIENTATION_DEG= — for the same cell only: the smallest rotation taking the main lattice onto it, over every basis whose metric matches, so a symmetry of the lattice is not read as a misorientation.

  • EXTRA_LATTICE_<n>_SPOTS_PCT=, EXTRA_LATTICE_<n>_INTENSITY_PCT= — its share of the non-ice spots and of their intensity.

  • EXTRA_LATTICE_<n>_SWEEP_CONCENTRATION= — the share of its spots in the 2 densest of 8 blocks of the sweep; 0.25 is uniform.

EXTRA_LATTICE_INTENSITY_PCT= sums the intensity share of the domains (DOMAIN, TWIN_DOMAIN, SEGMENTED); a RELATED_CELL or FOREIGN lattice is listed but not counted, since it is as often a wrong main lattice or tNCS as a second crystal, and MULTIPLE_LATTICES= is TRUE, with a warning under the MULTIPLE_LATTICES flag, where that sum is at least 10 % - the domains together, not the strongest alone. It is information, not a failure: a split crystal whose second domain is half as strong as the main one can still give data as good as XDS’s, since the domains’ reflections are only a problem where they overlap the main lattice’s. Look at the crystal, and at the result. The summary at the top of the report has a Multiple lattices line, none found where nothing was listed.

Mosaicity

MOSAICITY_DEG= (section 2, and a Mosaicity line in the summary) is the mosaic spread the merge computed partiality from, on rotation data: Kabsch’s σ_M, the standard deviation (not the FWHM) of a reflection’s rocking curve in degrees, in the same partiality model as XDS - so it is the number to set beside XDS’s REFLECTING_RANGE_E.S.D.: the value under SUGGESTED VALUES FOR INPUT PARAMETERS at the end of INTEGRATE.LP, which CORRECT.LP repeats. On in-house rotation data it reads 0.90× that value (median over 34 data sets). XDS’s per-image SIGMAR column and the per-block CRYSTAL MOSAICITY in INTEGRATE.LP are a different number - they run about 1.35× higher - and are not the comparison. It is fitted on every frame by maximum likelihood from the rocking offsets of that frame’s 250 strongest indexed spots, with the energy bandwidth taken out where the beam has one (XDS keeps it in), and smoothed in frame order; the value is the median over the frames, with MOSAICITY_DEG_P10= and MOSAICITY_DEG_P90= for its spread. The per-frame values are the sigma_M_deg column of _plot.txt.

Translational pseudo-symmetry

Two copies of the contents of the asymmetric unit related by a pure translation that is not a lattice vector. It is the pathology that most reliably breaks molecular replacement, because the modulation it puts on the intensities is not in the search model. Section 4 reports it beside twinning, because the two interact. The algorithm is in CPU/GPU data analysis ▸ Twinning and translational pseudo-symmetry.

TNCS_DETECTED= is TRUE, FALSE, INCONCLUSIVE or NOT_MEASURED. The last two are not FALSE: NOT_MEASURED means the merge has too few reflections in 20–5 Å to compute the statistic at all, INCONCLUSIVE means the Patterson was measured but too few acentric reflections remain to test whether the vector it names modulates the intensities. Neither is a statement that the crystal has no pseudo-symmetry.

A warning under the PSEUDO_TRANSLATION flag is given wherever TNCS_DETECTED= TRUE. Below a Patterson peak of 20 % of the origin (the height phenix.xtriage flags) the detection is significant but its modulation is small; the warning and the summary call it weak and ask to check whether molecular replacement needs it.

TRUE requires both of two tests, because either alone over-calls by about a factor of two:

  • TNCS_PATTERSON_PEAK_PCT= — the largest off-origin peak of the native Patterson, as a percentage of the origin peak, counting only peaks farther than 15 Å from any origin-equivalent lattice point. TNCS_PATTERSON_PEAK_Z= scores it against TNCS_PATTERSON_NULL_PCT=, the same map recomputed with the intensities permuted within resolution shells. The null is per dataset and not a table: the noise floor of this statistic runs from about 1% on a large merge to about 18% on a small one, so no fixed percentage separates the two populations.

  • TNCS_MODULATION= — the ratio of the strongest to the weakest bin mean of ⟨E²⟩ over the phase frac(h·u), with u the refined peak vector (TNCS_VECTOR=, fractional, and TNCS_VECTOR_LENGTH= in Å). TNCS_MODULATION_NULL= is the same search started from random vectors, so the contrast the search itself can manufacture is measured rather than assumed.

The vector is good to about 0.05 fractional. It is a starting point for a program that refines it, not a refined result: the refinement maximises the modulation, not the accuracy of the vector.

TNCS_PSEUDO_CENTRED= and TNCS_SUBLATTICE= are separate claims and are deliberately not merged. A vector that is a rational translation 1/q of the cell means the crystal is pseudo-centred; the cell itself is not in question, because the suppressed class is weak rather than absent and a smaller cell would contradict it. Only TNCS_SUBLATTICE= NEAR_EXTINCT_CLASS — the suppressed class almost gone — says the reported cell may be a supercell.

UNDECLARED_LATTICE_TRANSLATION= is a different finding, and is reported instead of a pseudo-symmetry rather than as one: a translation at which the Patterson reaches at least 75 % of the origin, UNDECLARED_LATTICE_TRANSLATION_PCT=. Near 100 % the merged data are invariant under it, and a translation the data are invariant under is a lattice vector by definition — so the centring or the cell is wrong, not the packing; an undeclared centring reads 83-102 %. Every one is a warning under the LATTICE_TRANSLATION flag, asking to check the centring and the cell; below 90 % the warning also names the other reading, a very strong pseudo-translation that molecular replacement needs declared, which refinement in the cell tells apart. It is what a centred lattice merged in P1 looks like, which --mode scale on a file with no space group produces by design. The pseudo-symmetry search continues underneath it, so a real pseudo-translation sitting under an undeclared centring is still found.

L_TEST_VS_TNCS= says how the twinning L-test beside it coped. A pseudo-translation u biases ⟨|L|⟩ upwards unless the partner reflection at h + s shares its class, which happens exactly when s·u is an integer; a half-integer u — a pseudo-centering — is preserved by the ordinary axis step of 2 and reads UNAFFECTED. Where it is not, the steps are restricted to those that do preserve the class (REPAIRED), and where no step does, UNREADABLE says the statistic was dropped from the twin verdict in both directions: it can no longer indicate a twin, and it can no longer be read as proof that there is none. The second moment then decides alone.

TWINNING_VERDICT= is NO_INDICATION, INDICATED (⟨|L|⟩ below 0.44, the phenix.xtriage convention, or the second moment low, in a Laue class that admits a twin law), SYMMETRY_SUSPECT (the same low reading in a holohedral class, where no twin law exists and a false adopted operator gives the same distribution) or NOT_READABLE. ⟨|L|⟩ has a physical range: 0.5 untwinned, 0.375 a perfect twin, whatever the law. Where it reads below 0.375, or above 0.55 (on this merge or on the one before the space-group search), no twin fraction explains it - the statistic is distorted, by tNCS, anisotropy, overlapping reflections of a very long cell, or too few reflections - and the verdict is NOT_READABLE, with no warning: it says nothing about twinning, nor about the space group.

Diffraction anisotropy

Section 4 also reports how much the fall-off with resolution depends on direction, and whether that is established above the data set’s own systematic error. It runs automatically on every merging run — there is no flag — and it is a description only: no intensity is corrected, no reflection is removed on a directional criterion, and the merged data and the written reflection files do not depend on direction at all. The algorithm is in CPU/GPU data analysis ▸ Diffraction anisotropy.

Two different quantities are reported and they are not interchangeable. ANISOTROPY_DELTA_B is a rate — the range of the principal components of the anisotropy tensor, on the ordinary crystallographic B scale, so it is directly comparable with phenix.xtriage’s B_cart, ctruncate’s anisotropic B and AIMLESS’s anisotropic ΔB. ANISOTROPY_D_MIN_PRINCIPAL is where the signal actually runs out along each principal direction. A crystal can have a large ΔB and almost no spread in directional limit, or the reverse.

key

meaning

ANISOTROPY_VERDICT

DETECTED | NOT_DETECTED | CANNOT_DETERMINE

ANISOTROPY_FREE_DIRECTIONS

Deviatoric directions the Laue class allows — 5 triclinic, 3 monoclinic, 2 orthorhombic, 1 tetragonal/trigonal/hexagonal, 0 cubic

ANISOTROPY_DELTA_B

The anisotropic ΔB (Ų), fitted on intensities with nothing dropped

ANISOTROPY_DELTA_B_LINEAR

The part of it that follows exp(−½ sᵀBs). This is the number the verdict is gated on, and the report says which of the two it is quoting

ANISOTROPY_PRINCIPAL_B

The three principal components, relative to the weakest

ANISOTROPY_D_MIN_PRINCIPAL

Diffraction limit (Å) along each principal direction — where ⟨I/σ(I)⟩ in a 20° cone about it falls through 2

ANISOTROPY_D_MIN_CENSORED

One flag per direction. 1 means ⟨I/σ(I)⟩ never fell through 2, so the limit is the edge of the measured data, a bound and not a measurement. The prose marks it with a <

ANISOTROPY_D_MIN_SPREAD

Range of the three limits — itself a lower bound if any is censored

ANISOTROPY_SHAPE

LINEAR (a real Debye–Waller B) | FLAT (the deficit does not follow a B at all, so ΔB may be an under-estimate) | CONVEX (grows faster than a B can) | UNDETERMINED (the verdict moved on rebinning)

ANISOTROPY_FLOOR, ANISOTROPY_SIGNIFICANCE

The data set’s own systematic-error floor (Ų) and ΔBlinear over it. Banded: below 2 not established, 2–3.5 marginal, above 3.5 established, above 5 strong

ANISOTROPY_DETECTION_LIMIT

The smallest ΔB that could have been established on these data. It is set by systematic error, not by counting, so it does not improve with more reflections or a longer exposure

ANISOTROPY_N_OBSERVATIONS, ANISOTROPY_FORBIDDEN_Z, ANISOTROPY_SIGMA_SYSTEMATIC

The unmerged observations the floor was measured on, that measurement against its own counting noise, and the floor before the counting part is added back

The default report carries ANISOTROPY_VERDICT, ANISOTROPY_DELTA_B, ANISOTROPY_D_MIN_PRINCIPAL and ANISOTROPY_D_MIN_SPREAD; the rest of this table — the detection gate’s own parameters — is written with --developer.

CANNOT_DETERMINE is a real answer, not an evasion. The verdict is not measured against counting statistics — real data carry systematic error far larger than that, and gating on counting error reports anisotropy on data sets that have none. Instead the data set measures its own systematic error in the tensor directions its Laue class forbids, where the true value is exactly zero whatever the crystal is. Where that measurement cannot be made, the run says so and gives the reason: a triclinic Laue class (no forbidden direction exists), an observed rotation under about 90°, merged data at the noise floor, a scale model carrying no dose term (--no-scaling-corrections), or no unmerged observations. A cubic Laue class is different again — symmetry forces ΔB to be exactly zero, and the run says that rather than reporting a measurement.

Where anisotropy is detected and the directional limits differ by more than 0.5 Å, a WARNING: line says so, since refinement and map interpretation should allow for it.

Model validation

Section 5 appears only with --model. It reports the supplied model against the merged data — R-factors, maps, anomalous sites — and, separately, whether the data accepted the model at all.

The two are not the same question, and the report keeps them apart. The R-factors, the maps and the rigid-body placement describe the model: they are computed and reported whatever the answer, because a model that does not belong to this crystal still has an R against it, and that is the negative result. MODEL_FIT= is the answer, and it is what governs whether the model was allowed to change anything about the written reflections.

There is no threshold on R behind it. What a model that explains nothing reaches against a given data set depends on its atom count and B-factors as much as on the data, so the same model is refitted — and re-placed as a rigid body, exactly as the real one is — from MODEL_FIT_NULL_REPLICATES random orientations about its own centroid, and MODEL_FIT_SIGMA is how far the real fit sits above that distribution. The statistic is R-work, not R-free - not because nothing is refined against the working set (the placement’s six parameters are), but because every null replicate is placed the same way, so what they buy is bought on both sides and cancels; and it is decided on an order of magnitude more reflections than R-free.

That null is only built where the model claims one of the two things it could change — the enantiomorph, or an indexing other than the one the data were merged in. A model already in the data’s space group on a crystal with no merohedral ambiguity, which is the isomorphous case a screening campaign is made of, claims neither: MODEL_FIT= NOT_TESTED, MODEL_DECISIONS_TAKEN= NONE, and the run does not pay for a null that would gate nothing. NOT_TESTED is not REJECTED — it says the question was never put, not that the data answered it badly — and the three values are distinguishable by grepping the one key. The MODEL_FIT_NULL_* and MODEL_FIT_SIGMA keys are absent in that case, since there is no null to report; R_WORK, R_FREE, the maps and the rigid-body shift are all there as usual.

key

meaning

MODEL_VALIDATION

PERFORMED | NOT_PERFORMED (with MODEL_VALIDATION_REASON, and no R-factors)

MODEL_FIT

ACCEPTED | REJECTED | NOT_TESTED — whether the model may decide anything, or had nothing to decide

MODEL_FIT_STATISTIC

What the verdict was taken on; R_WORK

MODEL_FIT_VALUE, MODEL_FIT_NULL_MEAN, MODEL_FIT_NULL_SD, MODEL_FIT_NULL_REPLICATES

The real fit, and the null of the same model in random orientations. Absent when NOT_TESTED

MODEL_FIT_SIGMA

The real fit above that null, in its standard deviations. Signed. Absent when NOT_TESTED

MODEL_DECISIONS_TAKEN

NONE | ENANTIOMORPH | INDEXING | ENANTIOMORPH+INDEXING

MODEL_ENANTIOMORPH_ADOPTED, MODEL_INDEXING_OPERATOR

The two decisions individually; x,y,z is no reindexing

MODEL_CHANGE_OF_BASIS, MODEL_SETTING_AS_READ

Present only where the model was written in another description of the lattice (other axes, other centring, or another point group) and was put into the data’s to be scored. Where such a model fits, the data are then written in the model’s setting instead (SETTING_SOURCE=MODEL) and these keys are absent, since the model no longer moves. An alternative indexing of the same point group is settled here only where a reference MTZ (-z) has already fixed the data’s indexing; otherwise the data are reindexed into the model’s, and that is MODEL_INDEXING_OPERATOR

MODEL_INDEXING_MARGIN, MODEL_INDEXING_MARGIN_NULL, MODEL_INDEXING_MARGIN_SIGMA

Present only where a merohedral ambiguity was probed. The winner’s lead over the runner-up in R-free, against the lead a random placement of the same model produces

The null’s own numbers — MODEL_FIT_STATISTIC, MODEL_FIT_VALUE, the MODEL_FIT_NULL_* keys and the MODEL_INDEXING_MARGIN* keys — are written with --developer; the default report carries the verdict (MODEL_FIT, MODEL_FIT_SIGMA, MODEL_DECISIONS_TAKEN and the two decisions).

Comparing two runs: R_MODEL_SHELL_SCALED

R_WORK and R_FREE describe this dataset against this model, and that is all they describe. They are not comparable with another run’s. The model is scaled to the data by an overall factor and a symmetry-constrained anisotropic B — a shape that can only bend one way with resolution, kept that way on purpose so that a batch of maps stays on one scale (see CPU_DATA_ANALYSIS_DECISIONS). Whatever the amplitudes’ own radial profile does that this shape cannot follow is then reported as R. Two reductions of one crystal whose merged amplitudes have different radial profiles therefore differ in R_FREE for a reason that has nothing to do with either fitting the model better, and by more than a real change in the data moves it.

R_MODEL_SHELL_SCALED is the same sum with that taken out: one free scale per resolution shell — the merge table’s own shells — fitted on the shell’s reflections, so only the agreement inside each shell is left. R_MODEL is the same sum without the per-shell scale, so the gap between the two is what the radial profile cost, and MODEL_RADIAL_MISFIT is the size of the rescale that closed it (the RMS of ln kshell about its mean). When the misfit moves between two runs, R_FREE between those two runs cannot be read; R_MODEL_SHELL_SCALED can.

Both R values are over all the reflections, not the free 5%. Nothing here is refined — the coordinates, the B-factors and the occupancies are the model’s own, and only the scale and the placement are fitted — so work and free estimate the same quantity (they differ by a median 0.001 over our corpus) and the split buys no cross-validation while costing a factor of √20 in precision.

None of this touches the maps: the map coefficients, the .ccp4 files and R_WORK / R_FREE are computed from the scale described above and are unchanged by these keys.

key

meaning

R_MODEL, R_MODEL_REFLECTIONS

R over every reflection, on the scale the maps use

R_MODEL_SHELL_SCALED

The same, with one free scale per resolution shell — the number to compare between runs

MODEL_RADIAL_MISFIT

How large that per-shell rescale had to be: RMS of its logarithm. Near zero means the maps’ own scale already described the radial profile, and then the two R values agree

CC(model, data)

Beside the R-factors, section 5 carries the correlation of the merged intensities with the placed, scaled model, |Fmodel|², by resolution shell. The shells are the merge table’s own, so a row here can be read straight across from that shell’s CC1/2 and Rmeas in section 3. It is a correlation of intensities, like CC1/2 and CCref beside it, and the observed value is the merged intensity itself rather than the French–Wilson |F|² the R-factors use — that amplitude is a posterior mean under a Wilson prior, which pulls a weak reflection towards its shell mean and would show up as correlation in exactly the outer shells this number is read in.

Nothing here was refined against these reflections — the model is placed and scaled with eleven parameters — so there is no work/free distinction to draw: the correlation is unbiased on all the reflections of a shell, not only the few hundred free ones, and SIGMA is correspondingly sharp.

key

meaning

CC_MODEL_OVERALL, CC_MODEL_REFLECTIONS

The correlation over every reflection in the table, and how many — the N column sums to it. Like any overall correlation it is shell-weighted and can take any value between the best shell and the worst; the table is what to read

CC_MODEL_CONFIRMED_TO_D_MIN

The finest shell whose correlation reaches 3 σ, or NONE. A lower bound on the useful resolution

the D_MIN / CC_MODEL / N / SIGMA table

Per shell: the correlation, the reflections it was formed on, and how far above zero it sits (Fisher’s transform, atanh(CC)·√(N−3))

Read it in one direction only. A shell whose correlation is significantly above zero carries signal — a model cannot agree by accident with measurements it was never fitted to — so CC_MODEL_CONFIRMED_TO_D_MIN is evidence for keeping more data. A shell whose correlation is near zero says nothing about the data: the model may be incomplete, in the wrong hand, or simply wrong for this crystal, and cutting on it would be cutting because the model is poor. Nothing in the pipeline acts on these numbers; they are reported and no more. This is the same asymmetry cryo-EM works under, where the half-map FSC sets the resolution and the model–map FSC only validates it.

A significantly negative correlation in a shell is worth chasing rather than ignoring: it cannot be signal, so it points at a systematic error — an indexing the model disagrees with, or an outer shell the scaling has mistreated.

MODEL_DECISIONS_TAKEN= NONE — whether the model was rejected or never tested — means the reflection files are byte for byte what a run with no model would have written — same space group, same indexing, same .mtz, .cif, .hkl and _unmerged.mtz. A rejected model is therefore safe to try: it costs the null’s compute and changes nothing else.

\ No newline at end of file diff --git a/RUGNUX_TUTORIAL.html b/RUGNUX_TUTORIAL.html new file mode 100644 index 000000000..e9bd312ab --- /dev/null +++ b/RUGNUX_TUTORIAL.html @@ -0,0 +1,37 @@ + Running Rugnux — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Running Rugnux

A first run in detail

The Quick start command is the whole of it:

rugnux -o myrun /path/to/dataset_master.h5
+

-o myrun is the prefix every output file is named from and the last argument is the master file of the dataset — any of the formats under What Rugnux reads, not only one written by Jungfraujoch; -N would set the worker-thread count, which otherwise follows the machine. Nothing is assumed about the crystal — the goniometer axis in the file tells Rugnux this is a rotation sweep, the unit cell comes from indexing the data, the space group from its systematic absences, and the resolution limit from where CC1/2 falls off. Progress, statistics and timing go to the terminal, and the output files land next to each other.

myrun_report.txt is written for a person, top to bottom: it says which space group was chosen and on what evidence, how far the data go, and anything that needs attention. To pull one number out of it in a script, every value is a KEY= value line:

grep '^SPACE_GROUP_NUMBER= '        myrun_report.txt
+grep '^SPACE_GROUP_NAME= '         myrun_report.txt   # the setting, which the number does not name
+grep '^UNIT_CELL_CONSTANTS= '      myrun_report.txt
+grep '^INCLUDE_RESOLUTION_RANGE= ' myrun_report.txt
+grep '^ISA= '                      myrun_report.txt
+grep '^WARNING:'                   myrun_report.txt
+

One caveat for a script reading those keys: where the space group is one of an enantiomorphic pair, SPACE_GROUP_NUMBER= / SPACE_GROUP_NAME= carry one member of the pair by convention, not by determination — the report says so beside them, in section 2, as SPACE_GROUP_ALTERNATIVES= naming the other hand and SPACE_GROUP_ENANTIOMORPH= UNDETERMINED. Treat the two hands as interchangeable until a model settles it: with --model, where the data accept the model (MODEL_FIT= ACCEPTED), the report carries MODEL_ENANTIOMORPH_ADOPTED= TRUE and the model’s group instead, and refining an isomorphous model against the wrong hand of the pair costs nothing subtler than an R factor near 0.55.

Useful variations, each independent of the others:

# tell it where the data really stop, if you already know - this sharpens the
+# space-group search and the error model
+rugnux -o myrun --scaling-high-resolution 1.4 dataset_master.h5
+
+# keep Friedel pairs apart, for anomalous work
+rugnux -o myrun -A dataset_master.h5
+
+# a quick look at the first 200 images only
+rugnux -o quicklook -e 200 dataset_master.h5
+
+# merge as usual, but also keep the per-image file so the data can be re-merged later
+rugnux -o myrun --write-process-h5 dataset_master.h5
+
+# skip the unmerged MTZ (written by default), when only the merged data is wanted
+rugnux -o myrun --no-export-unmerged dataset_master.h5
+
+# check the merged data against a known structure: R-work / R-free and maps
+rugnux -o myrun --model model.pdb dataset_master.h5
+

Re-merging is cheap and does not re-read the images. Ask the full run to keep its per-image file with --write-process-h5, and --mode scale will then re-scale and re-merge the reflections already integrated in it — seconds rather than minutes:

rugnux -o myrun --write-process-h5 dataset_master.h5   # integrate and merge once
+rugnux --mode scale -o remerged -A myrun_process.h5          # re-merge, here anomalously
+

Use it to try a different resolution limit, anomalous setting or outlier rejection without paying for integration again. --mode scale merges in the space group and cell the file records, so the second command needs no -S. Note that --no-merge also writes a _process.h5, but a run that never merged never determined a space group either, so re-merging that file lands in P1 unless you pass -S yourself — --write-process-h5 is the one to use.

Rotation data

Index, integrate, scale and merge a rotation sweep, fully de novo:

rugnux rotation_master.h5 \
+    -o rotation_run \
+    --scaling-high-resolution 1.4
+

Because the dataset carries a rotation goniometer axis, it is processed as rotation data by default: two-pass rotation indexing (index the sweep once, then process every frame against that lattice) with the rot3d partiality model (rotation partials combined into 3D fulls). Scaling and merging run by default (for both rotation and stills; --no-merge turns them off); the unit cell is taken from the rotation indexer and the space group is determined from systematic absences, and both are written into the merged files.

Run fully de novo (no -C/-S) for the best result — supplying a cell or space group up front tends to degrade low-symmetry cases. A -S group whose Bravais lattice the crystal turns out not to have stops the run and names the cell that was indexed, rather than merging in a frame the reflections are not in; where the lattice does have that group’s setting, the reflections are reindexed into it. --scaling-high-resolution (set it to your expected resolution) sharpens both the space-group search and the error model. To tune the first pass use --two-pass-rotation=100 (or -R100 — the first-pass image count); to force the sweep to be treated as independent stills use --force-still.

By default a rotation run also post-refines the geometry in a second pass: the first pass integrates and merges at the header geometry, then the detector distance + beam centre and the crystal cell / rotation-axis are refined against the merged fulls (cross-validated; the distance moves 1 % a step and a larger move is ratified by re-indexing, and the gauge-weak beam centre stays within 15 px of the centre the first pass ran at or of the run’s own measured centre, whichever is nearer), and the second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical <prefix>_* output; the header-geometry pass merges only to choose the space group and to judge the refined pass against, and writes no merged files of its own — no <prefix>_01.mtz, .cif or .hkl, and no _01_plot.txt or _01_detector.jpg. (Where a process file is asked for at all, with --no-merge or --write-process-h5, each pass still writes its own, so <prefix>_01_process.h5 appears beside <prefix>_process.h5.) Disable it with --rotation-no-postrefine.

After the per-frame scale-fulls step, rotation scaling applies three kinds of correction surface, on by default (--no-scaling-corrections disables all):

  • Decay — a global Debye–Waller relative-B over the run, for the radiation damage that weakens later frames more at high resolution (a resolution×time systematic the resolution-flat per-frame scale cannot remove). It only engages when the total relative-B exceeds a physical floor (2 Ų). An optional --relative-b[=deg] extends this single global rate to a smooth per-batch relative-B curve (default 10°-of-rotation batches when bare, off otherwise), cross-validated like the surfaces here, for crystals whose decay is non-linear in dose.

  • Absorption — a smooth multiplicative factor over the diffracted-beam direction in the goniometer frame (path length through the crystal), offered as a grid, as spherical harmonics and as a surface that also varies with the rotation angle. Negligible at hard X-rays / thin crystals; it matters at low photon energy and on strongly absorbing crystals. Its benefit shows up most on model-based metrics: a smooth absorption error largely cancels among symmetry mates (little effect on the error model / ISa) but still biases the intensities, so it measurably lowers Rfree.

  • Modulation — a smooth multiplicative factor over the position where a reflection lands on the detector (a flat-field: detector-response and geometric systematics that vary across the detector plane). Symmetry-equivalents of one reflection land at different detector positions as the crystal rotates, which over-determines the surface. Because it lives in the detector frame (not the rotation) the same correction concept applies to stills. This is the largest of the three on JUNGFRAU data — it lowers Rmeas by several to tens of percent on datasets that carry a detector systematic, while holding or improving CC1/2 and the anomalous signal.

All of them are cross-validated — fitted on even-numbered frames and used to merge the odd ones, and the other way round, and kept only if the correlation between the two half-set means, within resolution shells, rises over the same halves merged without the surface. The score does not involve the sigmas, so a surface can never pass by merely reshaping them; where the systematic is absent the surface is a no-op rather than a source of added noise, which is why they are safe to leave on (§10.6).

Independently of any correction, a rotation run prints a radiation-damage report — the per-image scale correlation-to-merge and mosaicity versus dose, and the relative B-factor change over the run (first→last) together with a per-batch relative-B curve, also written to the merged mmCIF. It is a data-quality-vs-dose diagnostic and never alters the merged intensities. A batch whose data cannot support a measurement prints - instead of a value, and the first→last number is printed only where a straight line describes the curve — damage is progressive, so a curve that dips and recovers is a disturbance of the sweep, not dose, and the report says so and points at the sweep-quality section (RADIATION_DAMAGE_RELATIVE_B= NOT_A_TREND).

Still / serial data

A dataset with no goniometer axis (e.g. a serial grid scan) is processed as independent stills automatically — no flag needed. Known-cell indexing with the GPU fast-feedback indexer, then merge against a reference structure:

rugnux serial_master.h5 \
+    -o serial_run \
+    -X ffbidx -C 79,79,38,90,90,90 -S 96 \
+    -z reference.mtz \
+    --scaling-high-resolution 1.8
+

A crystal form with an indexing ambiguity — P3, P4, P6 and their relatives — needs either the -z above or --model model.pdb, and needs it on the run that integrates: every crystal is indexed in its own hand, and the two are averaged together in the merge unless each image is put into the same hand as it is integrated.

ffbidx requires a known cell (-C) and is the indexer of choice for sparse serial stills. The self-calibrating spot finder is on by default for both workflows (--no-adaptive-spots turns it off), and for serial stills leave --min-pix-per-spot unset so it is chosen per image — across the still-target battery this combination raises the indexing rate and typically extends resolution over a fixed threshold and fixed min-pix, at equal or better CC1/2. (You can still pin a fixed threshold with --spot-sigma / --spot-threshold and a fixed min-pix with --min-pix-per-spot.) If a dataset does carry a goniometer axis but you want per-frame stills processing anyway, add --force-still.

Small-molecule data

A rotation sweep of a small-molecule crystal is processed with the same command as any other, and needs no flag to say what it is:

rugnux -o xtal sweep_master.h5
+

What differs from a protein is handled by the run itself:

  • Short cell axes. The FFT search’s floor is 10 Å, but a de-novo rotation run also tries a second first-pass hypothesis with the floor at 5 Å and takes it where the standing cell is an integer supercell of what it finds, so a cell with short axes is indexed on its true axes. An axis shorter than 5 Å needs --fft-min-unit-cell lowered, or the cell given with -C.

  • Wide and split spots. Spots far from the beam, at the high energies small-molecule data are often taken at, grow several times wider than the integration disk, and a crystal of slightly misaligned domains records each reflection as two or more spots. The pre-scan measures how wide the spots and their offsets from the predicted positions are across the detector, and the integration follows that footprint wherever it outgrows the disk (§9.1); on compact spots nothing changes.

  • Sparse patterns. A lattice that explains too few spots on most frames to pass the per-frame test is integrated on every frame rather than only on the frames that happen to carry more spots (§4.1). Where a sweep has few reflections per frame, the per-frame scale may be taken from the full reflections alone instead of from the partials; the merge decides that from the data and the log says which it took (§10.6).

  • Overloads. A measurement of a reflection with an overloaded pixel on any of its frames is left out, as XDS leaves it out: its brightest part is the part that was not measured. On a strongly diffracting crystal these are the strongest low-order reflections, and OBSERVATIONS_REJECTED_OVERLOAD= in the report counts them. Many of them mean the sweep wants a weaker beam, not different processing.

  • Space group. The search names glide planes, and groups without a centre of symmetry as well as centrosymmetric ones. Where the absences cannot separate the two (C2/c and Cc, Pnma and Pna2₁), the centrosymmetric group is written and the other is listed in SPACE_GROUP_ALTERNATIVES=, unless the intensity statistics read acentric and the lattice rules twinning out; SPACE_GROUP_CENTRE= says which of these happened (see The results report). Structure solution and refinement settle it.

Two settings are worth a look before running. The default polarization (--polarization 0.99) is an undulator’s; a laboratory source is close to unpolarized, and no file format states it, so give the instrument’s value. And the written reflections stop where CC1/2 falls off (the default --resolution-cutoff cc-logistic, the same cut for every file, .hkl included); --resolution-cutoff off keeps everything to the edge of the detector, for a refinement that weights the weak data itself.

The file for SHELX is xtal.hkl: on a rotation sweep it holds the scaled reflections unmerged, each at the index it was measured at, so SHELXL computes R(int) and R(sigma) itself and the Friedel opposites it needs for the absolute structure are all there — no -A is needed. How to take it into SHELXT and SHELXL is under Small-molecule structures with SHELXT and SHELXL. Absorption is corrected only by the empirical surface a rotation run fits and keeps where it helps (Rotation data); there is no correction from the crystal’s measured faces. --model is for macromolecular models and plays no part here.

Output files

Output (controlled by -o, --output-prefix, default output):

  • <prefix>_process.h5 — NXmx-compliant HDF5 with derived metadata (spots, indexing, integration, azimuthal integration, per-image statistics). See HDF5 / NeXus data format for the layout. Written by default only when not merging (i.e. under --no-merge); add --write-process-h5 to also write it when merging. It does not copy the images: /entry/data/data is a virtual dataset over the input files, so the input has to stay where it was for the pictures to be readable, and the pixel metadata (bit_depth_readout, underload_value, the dataset type) describes those files rather than the signed 32-bit container Rugnux processes in.

  • Merging is on by default (--no-merge disables it). The merged reflections are written in three formats — each has its uses downstream:

    • <prefix>.mtz — CCP4 MTZ for the CCP4 / phenix reflection tools. The columns are H K L IMEAN SIGIMEAN I(+) SIGI(+) I(-) SIGI(-) F SIGF F(+) SIGF(+) F(-) SIGF(-) FreeR_flag (F is the French–Wilson amplitude); the anomalous columns are present whenever any reflection carries a Bijvoet split — a rotation run always does, a stills run only with -A.

    • <prefix>.cif — mmCIF, for deposition and as the self-describing native format (also carries the merging statistics, ISa, twinning and radiation-damage indicators).

    • <prefix>.hkl — SHELX HKLF 4 text (h k l I σ(I), fixed 3I4,2F8.2), the direct input for SHELXL and SHELXC / ANODE / SHELXD. On rotation data it holds the scaled full reflections unmerged — every correction applied, symmetry equivalents not averaged, each at the index it was measured at — so SHELXL reports Rint and Rsigma itself and the anomalous signal is all there; stills runs write the merged reflections (Bijvoet mates at +hkl and -hkl). Intensities are put on a common scale so the largest value fits the fixed-width field, and the file ends with the 0 0 0 terminator record.

    All three carry the refined unit cell (from rotation indexing) and the space group determined from systematic absences (constrained to the indexed lattice symmetry).

  • <prefix>_unmerged.mtz — the integrated observations before merging, as an unmerged MTZ in POINTLESS’s column layout, so the data can be scaled and merged by aimless, pointless, careless or iotbx.merging_statistics instead of by Rugnux. Written by default, alongside the merged files and with --no-merge too; --no-export-unmerged skips it. See The unmerged export. --export-unmerged-partials writes <prefix>_unmerged_partials.mtz, one row per image, instead of summing.

  • <prefix>_P1.mtz — the P1 cross-check dataset: the same observations merged in P1 instead of the space group the run determined, so a wrong call can be re-merged, re-solved or re-refined without processing the images again. It is a full merge, not the degraded one the search itself runs on, and it is what rugnux --mode scale -S P1 would make from a _process.h5. Every rotation run that determines its own space group writes it — including one that determined P1, where it simply repeats the merged output — so a script harvesting results can always expect the file rather than having to reproduce the search’s decision to know whether it exists. It is not written when -S fixed the space group: prediction then rejects that group’s centring absences, so those reflections were never integrated and a P1 merge of them would be missing whole classes of reflections. --no-p1-crosscheck declines it. Stills are not covered yet. Its FreeR_flag is drawn by the same index hash but over P1’s own asymmetric unit, so it is not the merged file’s test set: a re-refinement in P1 is a fresh cross-validation, and its R-free is not comparable with one against <prefix>.mtz.

  • With --model, the validation outputs land beside the files above: the σA-weighted maps <prefix>_2fofc.ccp4 and <prefix>_fofc.ccp4, the map-coefficient MTZ <prefix>_maps.mtz, the anomalous difference map <prefix>_anom.ccp4 (where the merge kept the Bijvoet split), and the model as placed against the data as <prefix>_model.cif and <prefix>_model.pdb — the coordinates that go with the maps, in the cell and space group of <prefix>.mtz. Mind the names: <prefix>.cif is merged reflections, <prefix>_model.cif is coordinates. The PDB is there because fragment-screening tools want a <name>.pdb beside a <name>.mtz — PanDDA’s per-dataset input layout, and the pair dimple produces — and it is skipped, with a log line, for a cell the PDB format cannot hold. See Validating against a model.

  • <prefix>_report.txt — the results report: what the run determined, in a form both a person and a beamline script can read. Always written, next to the files above. See The results report.

  • <prefix>_plot.txt — the per-image table, one row per processed image, whitespace-separated, with a single # legend line over its own columns so gnuplot plots it as it stands (plot '<prefix>_plot.txt' using 1:4). The columns are:

    Column

    Meaning

    image

    Processed-image ordinal, the numbering every per-image array rugnux writes uses; with -s/--stride the source image is start + ordinal * stride.

    angle_deg

    Spindle angle at mid-exposure.

    bkg

    Background estimate from the azimuthal profile (the bkgEstimate of the HDF5 file).

    resolution_A

    How far the image’s spots reach, from spot finding alone.

    spots

    Spots found.

    scale_G

    Per-image scale factor fitted by scaling.

    sigma_M_deg

    Mosaicity as the rocking-curve standard deviation σM in degrees — not a FWHM, and not XDS’s REFLECTING_RANGE. See the note below.

    cc_to_merge

    Correlation of the image’s own observations with the merged data.

    cc_n

    Observations that correlation was computed over; 0 where none was.

    merged

    1 where the image’s observations are in the merged data, 0 where they are not — an image that never indexed, one an earlier guard dropped, and one ΔCC1/2 convicted all read 0. nan on a run that merged nothing, --no-merge included, where the question has no answer.

    A quantity nothing measured for an image is written nan, which gnuplot and numpy both skip; the columns never shift and a row is never left out. A stills run writes the table too, with nan in the columns a sweep would fill; --no-merge writes it with the scaling columns nan.

    Reading sigma_M_deg. It is fitted per image — a maximum-likelihood Gaussian rocking width over that image’s own indexed spots — but it is then smoothed in frame order over the same rotation window as the per-image scale (--smooth-g), and any frame too sparse to fit one of its own is given the run’s median. So the column varies only on the smoothing window’s scale, and on a weakly diffracting crystal it is close to, or exactly, a constant: that flatness is the honest statement that the run measured no per-frame variation, not a placeholder. The value is the one scaling used to recompute partialities. σM here is the width of a Gaussian rocking curve and nothing more — it is not a physical crystal mosaic spread, it absorbs beam divergence and every other source of angular spread, and it reads systematically low against XDS’s SIGMAR (about 0.9× on the regression corpus). Use it to see whether the run’s rocking width moves across the sweep, not as a number to quote.

  • <prefix>_detector.jpg — the detector as the run saw it, so the beam stop and the module gaps can be checked by eye. Rotation runs only, and only where --detect-beam-stop was left on (it is by default). The picture is the mean projection over the frames the beam-stop detection sampled — the very image it tested — coloured with jfjoch_viewer’s default map: white to indigo for the counts, grey for the module and chip gaps, coral for the shadow the run detected, and magenta for everything else the pixel mask holds. The scale saturates at eight times the median measured pixel, not at the brightest pixels, so the background spans the ramp and the Bragg spots clip: it is the background the picture is read for, since a shadow is a place the background is missing. A shadow drawn where the background is unbroken, or a dark patch the run left uncoloured, is visible at a glance; read it against the Beam stop shadow: N pixels ... line in the log.

Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. By default the written resolution is trimmed automatically where CC1/2 falls off (--resolution-cutoff cc-logistic, CC1/2 target 0.30); set --scaling-high-resolution to fix the limit by hand, or --resolution-cutoff off to keep the full range.

\ No newline at end of file diff --git a/SECURITY.html b/SECURITY.html new file mode 100644 index 000000000..0d5207d28 --- /dev/null +++ b/SECURITY.html @@ -0,0 +1 @@ + Security — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Security

Jungfraujoch is a data-acquisition and analysis system for X-ray detectors, designed to run inside a controlled facility network. This document describes what the software does and does not protect against, the current known limitations, and the authentication work in progress.

Threat model and scope

The security model targets a semi-trusted internal facility network. The concern is a peer on that network reaching a Jungfraujoch service with little or no effort — a mistyped host/port, a curious colleague, a mis-pointed script, a stray browser tab — not a determined attacker and not passive wire capture (which is the responsibility of the network layer: 802.1x, VLANs, facility infrastructure).

The asset that matters most is the confidentiality of live analysis data: the diffraction images and derived metadata (unit cell, resolution, spot counts, sample name) that reveal which sample is being measured. This matters for industrial and proprietary experiments. By contrast, acquisition control (start / stop / configure) is treated as low risk — scientists operate their own experiments and there is little to gain from restricting it.

Security is best-effort: measures that materially impede normal operation get turned off, so the design favours a few high-value, low-friction controls over comprehensive lockdown.

Out of scope. Jungfraujoch is not designed to be exposed to an untrusted network or the public internet. Do not do this.

1. Good practice — what is and is not protected

What you can secure (and should)

These controls work and a deployment should apply them (see also DEPLOYMENT.md):

  • Network isolation. Keep the broker and its data streams on a controlled segment. The broker ↔ writer ↔ receiver traffic should run on a dedicated back-end network, with the data-socket addresses pinned to that interface and the ports firewalled.

  • Reverse proxy for TLS. The broker speaks plain HTTP. To get HTTPS, put a reverse proxy (Apache / nginx) in front that terminates TLS and pin the broker to localhost behind it. The desktop viewer supports https:// endpoints — choose the scheme in the Open HTTP Connection dialog.

  • Filesystem confinement of written data. The writer creates NXmx HDF5 files on shared storage. Confidentiality of that data at rest is enforced by the filesystem: run the writer under a dedicated identity and use directory ownership / ACLs (and setgid) so that only the owning experiment can read its files.

  • Firewall the ZeroMQ ports. The image / preview / metadata / republish streams have no access control of their own (see below), so restrict who can reach those ports at the network layer.

What the software does NOT provide

The gaps are listed explicitly so a deployment does not assume protection that is not there:

  • No authentication or authorization in the broker. The HTTP/REST API currently has no login, token, or access control. Anyone who can reach the broker’s host and port has full read access (live images, unit cell, resolution, sample metadata) and full write access (start, cancel, reconfigure). Confidentiality currently depends entirely on network/firewall isolation. This is being addressed — see §3.

  • No transport encryption in the broker. The broker serves plain HTTP; there is no built-in TLS. Encryption must be provided by a reverse proxy.

  • No access control or encryption on the ZeroMQ streams. The preview, metadata, image, and republish streams are unauthenticated sockets. Any peer that can connect can subscribe to live data. For the image PUSH stream specifically, an accidental extra consumer does not merely eavesdrop — a PULL peer is load-balanced into the stream and will divert images away from the real writer.

  • No per-user isolation. The broker has no concept of users; it cannot separate one operator’s access from another’s.

  • No application-level audit trail of who accessed or changed what.

2. Known issues

#

Issue

Impact

Mitigation today

1

Broker HTTP API has no authentication for control, and read access is protected only per dataset

Anyone who can reach it can start / stop / configure; a dataset started without tokens is readable by anyone

Per-dataset bearer tokens (§3) for the read endpoints; network / firewall isolation for the rest

2

Broker binds all interfaces, plain HTTP

Reachable from anywhere routable; no encryption

Expose only on the trusted segment; TLS via reverse proxy

3

ZeroMQ preview / metadata / image / republish streams are unauthenticated and unencrypted

Live-data exfiltration; a rogue PULL on the image stream diverts/steals images

Firewall the ports; run only on the back-end network

4

Web frontend has no login

The bundled UI takes a dataset token (key button) but nothing identifies its user

Serve and reach it only on the trusted network

Input robustness. Services parse framed data from peers on the (trusted) data path. Hardening of untrusted-frame handling (size caps, overflow guards) is ongoing; these paths are not intended to face an untrusted network.

3. Authenticated read access — per-dataset bearer tokens

The confidential read endpoints are protected with a best-effort, low-friction scheme; acquisition control stays open.

How it works. /start accepts an optional list of tokens - plain strings, any number, all equivalent. A typical pair is one constant beamline secret and one secret minted for the experiment, so both the beamline staff and the experiment’s own users can open the data. While the current dataset has tokens, the endpoints below answer 401 unless the request carries Authorization: Bearer <one of them>; the 401 says nothing about the dataset. Every accepted /start replaces the previous tokens, so a run started without them is open, and the next run’s users cannot read this one. No endpoint returns the tokens; the broker only compares strings (constant-time) and keeps them in memory. Whoever runs /start hands the token to the viewers. An accepted /start also clears what the previous run left readable - its statistics, plots and buffered images - in the same step that installs the new tokens, so the previous run is never served under the new tokens, and the new run’s name never under the old ones. A refused /start (wrong state, invalid settings) changes nothing.

Endpoint

With tokens set

/statistics/data_collection (dataset name, unit cell, …)

401 without a token

/result/scan (dataset name, cell of a grid scan / rotation)

401 without a token

/image_buffer/start.cbor, /image_buffer/image.cbor, /image_buffer/image.jpeg, /image_buffer/image.tiff (the images and the start message)

401 without a token

/preview/plot, /preview/plot.bin (per-image plots, unit cell)

401 without a token

/statistics (the aggregate the web UI polls)

200, but the measurement block is omitted

everything else (/status, /config/*, /start, /cancel, masks, pedestal, …)

open

Clients.

  • jfjoch_viewer - the token field of File ▸ Open HTTP (password echo), the JUNGFRAUJOCH_HTTP_TOKEN environment variable, or D-Bus (LoadFile(url, image, sum, token) / SetHttpToken(token)); the dialog overrides both. A 401 clears the display and puts a note on the status bar - no dialog, since a changed dataset is the normal reason.

  • Web frontend - the key button in the top bar; the token lives in the tab’s sessionStorage and is sent with the protected calls only. The start form has a field for the tokens of a new run.

  • Python client - Configuration(host=..., access_token="<token>").

  • Anything else - curl -H "Authorization: Bearer <token>" ....

What it does not do. The broker still speaks plain HTTP, so the token crosses the network in clear unless a TLS reverse proxy fronts the broker (§1); on a facility network this raises the bar from “type the IP” to “capture packets”, which is the aim. It does not authenticate users or control, does not touch the ZeroMQ streams (issue #3), and /status’s free-text message may still quote a path. Datasets of different users within one session are separated by their tokens alone; there is no long-lived login.

\ No newline at end of file diff --git a/SOFTWARE.html b/SOFTWARE.html new file mode 100644 index 000000000..4b931a730 --- /dev/null +++ b/SOFTWARE.html @@ -0,0 +1 @@ + Software requirements — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Software requirements

Operating system

Recommended operating system is Red Hat Enterprise Linux (RHEL) / Rocky Linux versions 8 or 9. For these operating systems we provide RPMs with pre-built binaries to simplify deployment. On an experimental basis we also build repositories for Ubuntu 22.04 and 24.04.

Running Jungfraujoch on Red Hat Enterprise Linux 7 is currently not tested and not recommended, but likely possible by providing some packages from external repositories.

Two programs additionally run on Windows 11: the desktop viewer jfjoch_viewer, shipped as a pre-built installer, and rugnux, the offline analysis CLI, shipped as a .zip. Both can be built from source with Visual Studio 2026 (MSVC) and CUDA 13.3 — the viewer additionally needs Qt 6.11; see jfjoch_viewer ▸ Building from source on Windows. The Windows artefacts bundle the Qt runtime (viewer only) and, on the CUDA builds, the cuFFT DLL, so end users need neither Qt nor a CUDA toolkit installed — only an NVIDIA GPU driver for the GPU path. On Linux the portable archives link CUDA entirely statically and so need nothing but the driver. rugnux is also built for 64-bit Arm Linux (GH200, DGX Spark). The rest of Jungfraujoch is Linux-only and x86-64-only. See Release contents for the CPU baseline and CUDA requirements of each released package.

Software dependencies

Required:

  • C++20 compiler and C++20 standard library; recommended GCC 11+ or clang 14+ (Intel OneAPI and AMD AOCC also work)

  • CMake version 3.26 or newer + a build tool (GNU make or Ninja)

HDF5, libtiff, libjpeg-turbo, zlib and Eigen used to be required system packages; they are now downloaded and built automatically by CMake (see the note below), so none of them needs to be installed. Beyond the optional dependencies listed below, no third-party library has to be provided by the host any more.

Optional:

  • CUDA compiler version 12.8 or newer - required for the MX fast feedback indexer and GPU analysis

  • FFTW library - for indexing if GPU/CUDA is absent (also auto-downloaded by CMake)

  • Node.js - to build the frontend

  • Qt version 6 (for jfjoch_viewer)

  • OpenSSL 3.0 or newer - required to build jfjoch_viewer on Linux, where the fetched libcurl uses it for TLS; a build on OpenSSL 1.1 fails in libcurl

Many further dependencies (spdlog, Zstandard, HDF5, slsDetectorPackage, libzmq, libtiff, libjpeg-turbo, Ceres, the fast feedback indexer, Catch2, …) are downloaded automatically by CMake and statically linked; building therefore requires network access on the first configure. zlib (as zlib-ng in its zlib-compatible mode) and Eigen are among them: a copy already on the machine is used instead if the configure is given -DZLIB_ROOT=<prefix> or -DEigen3_DIR=<dir>. Others are vendored directly in the source tree. The complete list of third-party components, with copyright holders, licenses and verbatim license texts, is in Third-party software notices and the licenses/ directory.

\ No newline at end of file diff --git a/SOFTWARE_INTEGRATION.html b/SOFTWARE_INTEGRATION.html new file mode 100644 index 000000000..d6de302b3 --- /dev/null +++ b/SOFTWARE_INTEGRATION.html @@ -0,0 +1,4 @@ + Integration with MX data processing software — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Integration with MX data processing software

Jungfraujoch writes NXmx HDF5 in three layouts (see HDF5 / NeXus data format), and not every downstream program reads all three. NXmxVDS is the default and the one to use unless a program specifically needs another.

NXmxLegacy

NXmxVDS (default)

NXmxIntegrated

Jungfraujoch XDS plugin

yes

yes

yes

Durin (Global Phasing)

yes

yes

yes

Durin (Diamond, original)

yes

known bugs

known bugs

Neggia

yes

no — no virtual-dataset support

not tested

DIALS / xia2

only one data file

yes

yes

CrystFEL

yes

yes

yes

NXmxLegacy joins the data files to the master with external links, which is what DECTRIS’s filewriter-1 format did. Use it only for a program that needs it, and then keep the whole run in a single data file — see the DIALS section below.

XDS

XDS reads HDF5 through a plugin, named in XDS.INP:

LIB="/opt/xds/libjfjoch_xds_plugin.so.1.0.0"
+

Use the Jungfraujoch plugin. It is Linux-only, is downloadable from the Gitea release directory (built on RHEL 8), and also ships inside the jfjoch_viewer RPM/APT packages. The three numbers are the plugin version and change over time.

The alternatives, in order of preference:

  • Durin, Global Phasing build — github.com/CV-GPhL/durin. Prefer it over the original from Diamond Light Source, which has known bugs with non-DECTRIS files (virtual datasets and the single-file layout). It is the only third-party plugin that reads signed Jungfraujoch images correctly.

  • Neggia — github.com/dectris/neggia. No virtual-dataset support, so it cannot read the default layout. It also mis-reads signed 16-bit images: it dispatches on the pixel size in bytes and always casts to an unsigned type, so a count of -2 reaches XDS as 65534 and the -32768 error marker as 32768. Signed 32-bit degrades safely. Since JUNGFRAU in photon-counting conversion writes signed images by default, this affects the ordinary PSI case — do not use Neggia for it.

OVERLOAD must be set by hand

No XDS plugin — ours, Durin or Neggia — reads saturation_value from the file. XDS therefore takes its overload from OVERLOAD= in XDS.INP, and you must set it to the master file’s /entry/instrument/detector/saturation_value. XDS treats it inclusively: a pixel is overloaded when it exceeds OVERLOAD.

Signed images

MINIMUM_VALID_PIXEL_VALUE= may not be negative in current XDS, so a genuinely negative photon count — the reason signed output exists — cannot be declared valid. xia2 clamps the value to 0. There is no header field that changes this: if XDS is the target, consider collecting unsigned.

Which pixels are masked

The plugins do not all act on the same mask bits, so XDS and DIALS do not mask the same pixels:

mask bit

meaning

jfjoch plugin

Durin / Neggia

DIALS

0

module gap

yes

yes

yes

1, 4

error, noisy

yes

yes

yes

8, 9

user mask, beam stop

yes

no

yes

30

module edge

yes

no

yes

31

chip gap

no

no

yes

DIALS masks a pixel whenever any pixel_mask bit is set; Durin and Neggia look only at bits 0–4. On a JUNGFRAU with the default edge masking this is a difference of order 2% of the detector. Bits 30 and 31 mark pixels that are larger than normal rather than bad, which is why the Jungfraujoch plugin passes chip-gap pixels through — but be aware that a dataset processed by XDS and by DIALS will not have used exactly the same pixels.

DIALS

Tested regularly against DIALS (currently 3.27.0), including the xia2.ssx pipeline for serial crystallography.

  • Use NXmxVDS or NXmxIntegrated. With NXmxLegacy, DIALS reads only the first data file and reports a correspondingly short image count, without an error; if a goniometer is present it then fails on the frames past the first file. A legacy run that fits in one data file is read correctly — set images_per_file to cover the whole run.

  • Unsigned 32-bit images require bit_depth_readout, which Jungfraujoch writes. For signed images the field is deliberately omitted: DIALS remaps the top two codes of 2^bit_depth_readout, and on signed data those land inside the trusted range.

  • trusted_range is inclusive at both ends, and is taken from underload_value and saturation_value.

pyFAI

rugnux --mode calibration writes a .poni file describing the detector geometry — see Detector geometry and Rugnux.

  • It declares orientation, so it needs pyFAI 2024.01 or newer. An older pyFAI ignores the key and places the beam centre wrongly along the slow axis.

  • A .poni file carries geometry only. pyFAI does not learn the saturation value, the error marker or the pixel mask from it, and will happily integrate a masked pixel at UINTx_MAX as a count. Pass the marker and the mask at integration time:

    ai = pyFAI.load("calibration.poni")
+res = ai.integrate1d(image, 1000, dummy=65535, delta_dummy=0.5, mask=pixel_mask != 0)
+

    with dummy set to the master file’s /entry/instrument/detector/error_value for the stored pixel type, and pixel_mask read from /entry/instrument/detector/pixel_mask.

CrystFEL

Jungfraujoch files are compatible with CrystFEL. max_adu is inclusive — a pixel is bad when it exceeds the value — so set it from saturation_value.

\ No newline at end of file diff --git a/TESTS.html b/TESTS.html new file mode 100644 index 000000000..7fffbb2ef --- /dev/null +++ b/TESTS.html @@ -0,0 +1,12 @@ + Tests — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Tests

The unit and integration tests are written with Catch2 and collected into a single binary, tests/jfjoch_test. Build and run it with:

make -j$(nproc) jfjoch_test
+cd tests
+./jfjoch_test                 # everything
+./jfjoch_test "<test name>"   # one test case
+./jfjoch_test "[tag]"         # by tag
+

There are also benchmark and hardware routines, each printing its own usage:

  • jfjoch_hdf5_test to measure HDF5 dataset writing speed (single threaded). It doubles as the generator of the HDF5 files used by the external-software tests below.

  • jfjoch_lite_perf_test to measure the CPU/GPU (“lite”) analysis path - indexing, integration and optional file writing.

  • jfjoch_fpga_test to test quality/performance of FPGA card(s) and software routines. With -H it runs the high-level-synthesis C model on the CPU, so no FPGA device is needed.

Out-of-space handling is covered separately by jfjoch_hdf5_enospc_test, run under the enospc_shim LD_PRELOAD module that makes writes fail with ENOSPC.

In addition, tests are executed to verify that datasets written by Jungfraujoch are readable by other MX software (see Integration with MX data processing software) - XDS through the Jungfraujoch, Durin and Neggia plugins, and DIALS xia2.ssx - for each of the NXmx layouts. Input files for these programs are placed in the tests/xds, tests/xds_durin, tests/xds_neggia and tests/crystfel folders. See .gitea/workflows/build_and_test.yml for the exact commands; the CrystFEL fixtures are run by hand rather than in the pipeline.

Judging a change to the analysis itself

The harnesses below run rugnux over stored datasets and score the result. None is part of CI - run them when a change plausibly moves merged results, not as a reflex. The public datasets the pipeline is exercised on, and the DOI to cite for each, are listed in External test data.

  • tools/battery/battery.py - the rotation battery, the one canonical way to judge a rotation change: public PDB depositions scored against the deposition, in-house standard crystals and no-crystal controls scored against XDS, and a local-only private arm. Its manifests, the run and compare protocol and the report are described in tools/battery/README.md. Each set runs in several variants back to back: plain, with the deposited model (--model, R-free against the published one) on the open arm, and with XDS’s settings on the XDS arms.

  • rugnux_stills_ab.py - the stills analogue of the battery: scores a change on a serial dataset by what it does to the merge. Takes its dataset list from outside the repository.

  • rugnux_anomalous.py - the anomalous-peak-height arbiter, below.

The anomalous-peak-height arbiter

A change that touches partiality - a mosaicity estimator, a rocking-curve model, a background change, anything that alters how partial reflections are weighted - cannot be judged by the statistics we normally reach for:

statistic

why it fails for this class of change

ISa, R_meas, error-model b

one measurement, not three; dominated by the low-resolution shells; not invariant to the uniform intensity rescale a partiality change produces

last-shell R_meas

moves with its denominator, i.e. the wrong way by construction

rugnux --model R-free

tracks its own zero-information floor, which moves ~22x more than R-free itself over the same sweep

per-shell agreement with XDS_ASCII.HKL

XDS never divides by partiality, so “divide less” moves us toward it mechanically; measured to put the optimum ~1.4x too low

Anomalous difference density at known scatterer sites has none of these problems. It is read in units of the map’s own sigma, so a uniform intensity rescale cancels exactly, and it is referenced to the structure rather than to another program’s partiality model.

rugnux_anomalous.py measures it: shelxc + anode -a (CCP4) on each arm’s merged reflections, against a model that is placed once and then held fixed. It reports, per dataset, the mean site height and the off-site noise floor, and, between arms, the paired per-site change.

# compare two arms (each a directory of <id>/<id>.hkl + .mtz)
+./rugnux_anomalous.py --config <table>.json  base=<dir-A>  test=<dir-B>
+
+# a parameter scan: numeric labels turn the arms into a curve with a per-dataset optimum
+./rugnux_anomalous.py --config <table>.json \
+    0.85='<scan>/{name}/s0p85.hkl' 1.00='<scan>/{name}/s1.hkl' 1.20='<scan>/{name}/s1p2.hkl'
+

An arm is a Rugnux output directory or a path template containing {name}. --place does the one-off model placement, --write-config-template prints the config skeleton, and ANODE results are cached under the config’s workdir (a full 9-dataset x 11-arm scan takes under a minute).

The gate. A dataset counts only if its reference arm shows top peak > 1.5x the highest off-site peak and at least 3 sites over 5 sigma. A dataset that fails is reported as EXCLUDED, never as a zero - the difference between two noise measurements is not a measurement.

Standing dataset set (2026-08): 8 datasets from 7 crystals - 114 site-measurements across the datasets, 108 distinct sulfur sites - all judged on native sulfur signal.

crystals

space group

photon energy

sites each

2

P41212

12.4, 16.0 keV

18

2 (lysozyme)

P43212

13.0, 5.0 keV

27

3 (4 datasets - one crystal contributes two energies)

cubic, I-centred

13.0, 6.0, 5.0, 5.0 keV

6

Report n as crystals, not datasets: two energies of one crystal are not two independent votes, and the tool prints both counts for that reason.

Traps this tool exists to encapsulate. Every one of them has already cost a working day:

  1. The phasing space group comes from the config, never from the merged file. I23 and I213 have identical systematic absences (I-centring already forces the screw condition), so no data can separate them, and phaser’s automatic space-group test only tries the enantiomorph - which for I23 is itself. Phasing an I-centred cubic case in the I23 that both Rugnux and XDS report gives TFZ 7-11 where the other member gives 30-50, and drops the mean site height by a factor 3-10 - enough to make four good datasets look signal-free. Thirteen classes of chiral space group are indistinguishable this way; --place tries every member of the class and reports each one’s LLG/TFZ.

  2. Place the model once, from a reference arm, and reuse it unchanged. Re-phasing per arm lets the model move and contaminates the comparison. Refining the placed model against the dataset’s own amplitudes is allowed (it lifts the peaks another 4-10%) as long as the same refined model is then used for every arm.

  3. The gate and the measurement must use the same model. Gating on one model and scoring the curve with another silently changes which datasets are in the set.

  4. The off-site floor skips special positions. A peak on the cell origin is a ripple of the calculated phases, not a sample of the background; leaving it in inflates the floor by several sigma and can turn a passing dataset into a failing one. Such peaks are reported in their own spec column rather than dropped silently.

Reading the result. Judge the paired per-site change, with its standard error, pooled over crystals. A per-dataset optimum whose arm does not beat the reference on the paired test is flagged not significant vs ref and must not be quoted as a preference; so must one sitting on the edge of the scanned grid (grid edge) - extend the grid instead.

\ No newline at end of file diff --git a/THIRD_PARTY_NOTICES.html b/THIRD_PARTY_NOTICES.html new file mode 100644 index 000000000..efbc95635 --- /dev/null +++ b/THIRD_PARTY_NOTICES.html @@ -0,0 +1,2 @@ + Third-party software notices — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Third-party software notices

Jungfraujoch is licensed under GPL-3.0 (see LICENSE); the FPGA design is licensed under CERN-OHL-S-2.0 (see fpga/LICENSE). It builds on a number of third-party components, acknowledged below as required by their licenses.

This file is the human-readable manifest. The verbatim license texts live in the licenses/ directory (regenerate with bash licenses/COLLECT.sh). The frontend’s bundled JavaScript dependencies are listed separately in frontend/dist/THIRD_PARTY_LICENSES.txt, generated at build time (npm run licenses).

All licenses below are GPL-3.0-compatible, with one exception: the NVIDIA CUDA Toolkit is used under its own EULA (see the notes at the end of this file).

Fetched at build time and statically linked into the C++ binaries

These are downloaded by CMake (FetchContent / ExternalProject) during the first configure and linked into the Jungfraujoch executables.

Component

Version

Copyright

License (SPDX)

License text

spdlog

1.17.0

Gabi Melman

MIT

spdlog.txt

Zstandard

1.5.7

Meta Platforms, Inc.

BSD-3-Clause

zstd.txt

HDF5

2.2.0

The HDF Group; UIUC

BSD-3-Clause-style + Apache-2.0

hdf5.txt

slsDetectorPackage

8.0.2 / 9.2.0

PSI

LGPL-3.0-or-later

LGPL, GPL

cpp-httplib

0.56.0

Yuji Hirose

MIT

cpp-httplib.txt

libzmq (ZeroMQ)

4.3.5

iMatix and contributors

MPL-2.0

libzmq.txt

libtiff

4.7.2

Sam Leffler; SGI

libtiff (BSD-like)

libtiff.txt

FFTW

3.3.10

Matteo Frigo; MIT

GPL-2.0-or-later

fftw.txt

Ceres Solver

(pinned)

Google Inc. and contributors

BSD-3-Clause

ceres-solver.txt

Abseil

20250127 (required by Ceres)

Google Inc.

Apache-2.0

abseil.txt

fast-feedback-indexer

(pinned)

PSI

BSD-3-Clause

fast-feedback-indexer.txt

libjpeg-turbo

3.2.0

D. R. Commander and others; IJG

IJG + BSD-3-Clause + Zlib

libjpeg-turbo.txt

curl

8.22.0

Daniel Stenberg and contributors

curl (MIT-like)

curl.txt

Catch2

3.16.0

Catch2 Authors

BSL-1.0

catch2.txt

zlib-ng

2.3.3

Jean-loup Gailly and Mark Adler; the zlib-ng contributors

Zlib

zlib.txt

Eigen

3.4.1

Benoit Jacob, Gael Guennebaud and contributors

MPL-2.0 (+ BSD parts)

eigen.txt, README

libcurl is fetched and statically linked only for viewer builds (JFJOCH_VIEWER_BUILD / JFJOCH_VIEWER_ONLY), where it is jfjoch_viewer’s HTTP client; the broker and writer never link it. Its TLS and Kerberos backends are the OS-native ones (Schannel/SSPI on Windows, OpenSSL and system krb5 on Linux), so no TLS stack is vendored with it.

Catch2 is used only to build the test binary (jfjoch_test) and is not part of any shipped artifact; it is listed here for completeness.

zlib is supplied by zlib-ng built in its zlib-compatible mode (same zlib.h, same API and symbol names), so it is the zlib that HDF5, libtiff, cpp-httplib, libcurl and GEMMI all link. It is built during the configure rather than added as a subproject; the licence is the zlib licence either way. Eigen is header-only: only its headers reach the binaries, and no Eigen CMake runs.

Vendored directly in the repository

These live in the source tree (see the path) rather than being fetched; traccc and the Ceres-derived minimiser are the exceptions - code adapted into first-party files rather than a vendored directory, see the notes at the end of this file.

Component

Path

Copyright

License (SPDX)

License text

nlohmann/json

include/nlohmann/

Niels Lohmann

MIT

nlohmann-json.txt

Macaron Base64

include/base64/

tomykaira

MIT

base64-macaron.txt

TinyCBOR

frame_serialize/tinycbor/

Intel Corporation

MIT

tinycbor.txt

Bitshuffle

compression/bitshuffle/

Kiyoshi Masui

MIT

bitshuffle.txt

Bitshuffle (h-perf)

compression/bitshuffle_hperf/

Kal Conley

Apache-2.0

bitshuffle-hperf.txt

LZ4

compression/lz4/

Yann Collet

BSD-2-Clause

lz4.txt

HLS arbitrary-precision types

fpga/include/

Xilinx, Inc.

Apache-2.0

xilinx-hls-headers.txt

GEMMI

gemmi_gph/

Global Phasing Ltd.

MPL-2.0

gemmi.txt

PEGTL

gemmi_gph/gemmi/third_party/tao/

Dr. Colin Hirsch and Daniel Frey

MIT

pegtl.txt

sajson

gemmi_gph/gemmi/third_party/sajson.h

Chad Austin

MIT

sajson.txt

fast_float

gemmi_gph/gemmi/third_party/fast_float.h

The fast_float authors (Daniel Lemire et al.)

Apache-2.0 OR MIT OR BSL-1.0

fast-float.txt

half

gemmi_gph/gemmi/third_party/half.hpp

Christian Rau

MIT

half.txt

pocketfft

gemmi_gph/gemmi/third_party/pocketfft_hdronly.h

Max-Planck-Society; Peter Bell; MIT (FFTW-derived parts)

BSD-3-Clause

pocketfft.txt

tinydir

gemmi_gph/gemmi/third_party/tinydir.h

Cong Xu, Lautis Sun, Baudouin Feildel, Andargor

BSD-2-Clause

tinydir.txt

traccc (ACTS)

image_analysis/spot_finding/StrongPixelSet.cpp, SpotExtractorGPU.cu

CERN, for the benefit of the ACTS project

MPL-2.0

traccc.txt

Ceres Solver (adapted)

image_analysis/geom_refinement/LMSolver.h, LMSolver.cpp

Google Inc.

BSD-3-Clause

ceres-solver.txt

xbflash.qspi

tools/xbflash.qspi/

Xilinx / AMD

Apache-2.0

xbflash-qspi.txt

wingetopt

tools/wingetopt/

Todd C. Miller; The NetBSD Foundation

ISC AND BSD-2-Clause

wingetopt.txt

Runtime libraries and SDKs (shipped in binaries, not in the source tree)

Component

Used by

License

Notice

Qt 6

jfjoch_viewer

LGPL-3.0

notice, LGPL-3.0

NVIDIA CUDA Toolkit (cudart, cuFFT)

CUDA builds

NVIDIA CUDA EULA

notice, EULA

Frontend (npm) dependencies

The React/TypeScript frontend (frontend/) bundles a large transitive tree of npm packages, overwhelmingly MIT/ISC/BSD/Apache-2.0 licensed. Their full notices are generated automatically:

cd frontend && npm run licenses     # writes dist/THIRD_PARTY_LICENSES.txt
+

The generated file is produced as part of the frontend build target and installed alongside the served frontend, so the shipped web UI carries its own attribution.

Notes on weak-copyleft and attribution-sensitive components

  • MPL-2.0 (Eigen, GEMMI, libzmq, traccc): file-level copyleft. GEMMI is vendored in gemmi_gph/ in trimmed form; libzmq and Eigen are fetched at build time (Eigen is header-only). The corresponding source is available from each project upstream.

  • GEMMI’s own bundled third-party headers — PEGTL, sajson, fast_float, half, pocketfft and tinydir, all under gemmi_gph/gemmi/third_party/ — are listed separately above rather than being absorbed into GEMMI’s row: they are other authors’ code under other licences (MIT, BSD and Apache/MIT/BSL), and GEMMI’s MPL-2.0 does not speak for them. Their terms are carried in the headers themselves rather than in LICENSE files, so the licenses/*.txt copies are kept by hand; only PEGTL ships a LICENSE, which COLLECT.sh copies.

  • traccc is the one entry that is not a vendored directory. Its sparse connected-component labelling enters two otherwise first-party files: StrongPixelSet.cpp adapts the SparseCCL source, and SpotExtractorGPU.cu follows the design of its GPU counterpart. MPL-2.0 is file-level, so both files name the origin at the top and are covered by licenses/traccc.txt. See ACKNOWLEDGEMENT.md for the citation.

  • Ceres Solver is also fetched and linked (table above); separately, LMSolver.h/.cpp re-implement its trust-region Levenberg-Marquardt minimiser, projected line search, polynomial step choice and sphere manifold for the crystal refinement, following its source. Both files name the origin at the top and are covered by licenses/ceres-solver.txt.

  • FFTW is GPL-2.0-or-later — compatible with, and absorbed by, this project’s GPL-3.0 license.

  • Apache-2.0 components: where upstream ships a NOTICE file, it is reproduced in the corresponding licenses/ text.

  • Qt (LGPL-3.0) and NVIDIA CUDA (EULA) carry redistribution conditions beyond a copyright notice; see their dedicated notice files. The verbatim LGPL-3.0 and CUDA EULA texts are bundled (licenses/Qt6-LGPL-3.0.txt, licenses/NVIDIA-CUDA-EULA.txt); the CUDA EULA is the one shipped with CUDA Toolkit 12.8 — replace it if you build against a different toolkit version.

\ No newline at end of file diff --git a/TOOLS.html b/TOOLS.html new file mode 100644 index 000000000..a41f81dea --- /dev/null +++ b/TOOLS.html @@ -0,0 +1,13 @@ + Tools — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Tools

Besides the main services (jfjoch_broker, jfjoch_writer, jfjoch_viewer), the repository ships a number of command-line tools. Each prints its own usage when run with -h or without arguments.

Data analysis

Rugnux

Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, integration, scaling/merging) on a stored HDF5 dataset, producing a _process.h5 file and, when merging, reflection files. Merging is on by default (--no-merge disables it). --mode picks what a run does: mx (the above, the default), azint (only azimuthal integration, no spot finding/indexing), scale (re-scale/merge the already-integrated reflections in a _process.h5 without re-integrating) or calibration (detector geometry from a calibrant’s powder rings, written as <prefix>.poni and <prefix>.json). See Rugnux.

rugnux installs on its own, as the rugnux package or as a standalone archive — see Installing Rugnux.

jfjoch_extract_hkl

Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across neighbouring images and compare against an XDS INTEGRATE.HKL reference. A developer utility: it is built from source but is not installed into any package.

FPGA / PCIe card management

jfjoch_pcie_status

Prints detailed status information about the card. Safe to run during data collection:

./jfjoch_pcie_status /dev/jfjoch0
+

jfjoch_pcie_net_cfg

Reads and modifies the network configuration of the card’s interfaces:

jfjoch_pcie_net_cfg <device name>
+     Read configuration for all network interfaces of a device
+jfjoch_pcie_net_cfg <device name> <if number>|fgen
+     Read configuration for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen ipv4 <IPv4 address>
+     Set IPv4 address for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen direct 0|1
+     Set direct mode for a particular network interface / internal frame generator
+jfjoch_pcie_net_cfg <device name> <if number>|fgen clear
+     Clear Ethernet counters for a particular network interface / internal frame generator
+

jfjoch_pcie_clear_net_counters

Resets the card’s Ethernet, UDP and ICMP packet counters (which otherwise run from power-on):

./jfjoch_pcie_clear_net_counters /dev/jfjoch0
+

Testing, benchmarking and simulation

jfjoch_udp_simulator

UDP packet simulator used to test the Jungfraujoch FPGA receiver.

jfjoch_fpga_test

Exercises and benchmarks the FPGA data path and receiver. With -H it runs the high-level synthesis C model on the CPU, so no FPGA device is required.

jfjoch_lite_perf_test

Performance test of the lite (CPU/GPU) analysis path — indexing, integration and optional file writing.

jfjoch_hdf5_test

Tests single-threaded HDF5 writer performance.

jfjoch_simplon_test

Minimal test client for a DECTRIS SIMPLON detector API.

\ No newline at end of file diff --git a/VERSIONING.html b/VERSIONING.html new file mode 100644 index 000000000..bfecc997b --- /dev/null +++ b/VERSIONING.html @@ -0,0 +1 @@ + Semantic versioning — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Semantic versioning

Jungfraujoch is following semantic versioning. For this purpose we define public API as following:

  • OpenAPI configuration interface

  • CBOR serialization ZeroMQ stream

  • HDF5 file format

This means that a change in any of these formats must be accompanied by a version change - major version in case of breaking changes, minor version in case of feature expansion.

NOTE: FPGA design, PCIe driver, and internal libraries are not part of the public API and are considered internals of Jungfraujoch. Breaking changes in these components can happen without incrementing major version of the whole package. It will be marked in changelog.

\ No newline at end of file diff --git a/WEB_FRONTEND.html b/WEB_FRONTEND.html new file mode 100644 index 000000000..657f770a1 --- /dev/null +++ b/WEB_FRONTEND.html @@ -0,0 +1 @@ + Web frontend — Jungfraujoch 1.0.0-rc.174 documentation Skip to content

Web frontend

Jungfraujoch is equipped with a React-based web frontend for a user-friendly experience. The frontend has the following features:

  • Presenting current state of the detector

  • Plotting results of online quality calculations

  • Showing live view images from the detector

  • JUNGFRAU calibration numbers

  • Configuring the detector, as well as pedestal/initialization operations

When the current dataset was started with tokens, the plots, the live preview and the measurement statistics need one of them: the key button in the top bar takes it (kept in the tab’s session storage only, cleared when the tab closes) and the start form has a field for the tokens of a new run. See Security.

The frontend is written in TypeScript. For details see the frontend/ directory.

\ No newline at end of file diff --git a/_images/battery_dmin.png b/_images/battery_dmin.png new file mode 100644 index 000000000..192eace2d Binary files /dev/null and b/_images/battery_dmin.png differ diff --git a/_images/battery_rfree.png b/_images/battery_rfree.png new file mode 100644 index 000000000..5bb00ffe7 Binary files /dev/null and b/_images/battery_rfree.png differ diff --git a/_images/battery_time.png b/_images/battery_time.png new file mode 100644 index 000000000..2daf733bf Binary files /dev/null and b/_images/battery_time.png differ diff --git a/_images/battery_time_inhouse.png b/_images/battery_time_inhouse.png new file mode 100644 index 000000000..075faca17 Binary files /dev/null and b/_images/battery_time_inhouse.png differ diff --git a/_images/battery_time_zoom.png b/_images/battery_time_zoom.png new file mode 100644 index 000000000..39ca2e2c5 Binary files /dev/null and b/_images/battery_time_zoom.png differ diff --git a/_images/jfjoch.png b/_images/jfjoch.png new file mode 100644 index 000000000..aa75a7ea8 Binary files /dev/null and b/_images/jfjoch.png differ diff --git a/_images/viewer_general.png b/_images/viewer_general.png new file mode 100644 index 000000000..070689ec2 Binary files /dev/null and b/_images/viewer_general.png differ diff --git a/_images/viewer_grid_scan.png b/_images/viewer_grid_scan.png new file mode 100644 index 000000000..b890b640c Binary files /dev/null and b/_images/viewer_grid_scan.png differ diff --git a/_images/viewer_processing_results.png b/_images/viewer_processing_results.png new file mode 100644 index 000000000..c72547919 Binary files /dev/null and b/_images/viewer_processing_results.png differ diff --git a/_images/viewer_processing_settings.png b/_images/viewer_processing_settings.png new file mode 100644 index 000000000..cdf90427d Binary files /dev/null and b/_images/viewer_processing_settings.png differ diff --git a/_sources/ACKNOWLEDGEMENT.md.txt b/_sources/ACKNOWLEDGEMENT.md.txt new file mode 100644 index 000000000..ca82c3cbc --- /dev/null +++ b/_sources/ACKNOWLEDGEMENT.md.txt @@ -0,0 +1,486 @@ +# Acknowledgements + +Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). + +## Funding and support + +The project is supported by: +* Innosuisse via Innovation Project "NextGenDCU high data rate acquisition system for X-ray detectors in structural biology applications" (101.535.1 IP-ENG; Apr 2023 - Sep 2025). +* ETH Domain via Open Research Data Contribute project (Jan - Dec 2023). +* AMD University Program with donation of licenses of Ethernet IP cores and Vivado software. + +## Crystallographic methods adopted from other packages + +The analysis pipeline reimplements methods first published, and in most cases first implemented, by +other crystallographic software. The code below is Jungfraujoch's own; the methods are theirs, and +are acknowledged here. Where a package's source was consulted this is said explicitly. Most of these +packages are neither linked nor vendored; the three that are - GEMMI, traccc and +fast-feedback-indexer - also carry a licence obligation, recorded in +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). + +### Spot finding + +**[CrystFEL](https://www.desy.de/~twhite/crystfel/)** — spot finding, the three-ring integration +region, the serial/stills processing model, and the per-frame indexing acceptance test +(`indexing_peak_check()` in `peaks.c`). T. A. White, R. A. Kirian, A. V. Martin, A. Aquila, K. Nass, +A. Barty and H. N. Chapman, "CrystFEL: a software suite for snapshot serial crystallography" (2012), +J. Appl. Cryst. 45, 335-341 [doi:10.1107/S0021889812002312](https://doi.org/10.1107/S0021889812002312). +The self-calibrating spot finder's per-resolution-ring background statistics, with the Bragg peaks +excluded by iterated clipping, follow Cheetah's peakfinder8: A. Barty, R. A. Kirian, +F. R. N. C. Maia, M. Hantke, C. H. Yoon, T. A. White and H. N. Chapman, "Cheetah: software for +high-throughput reduction and analysis of serial femtosecond X-ray diffraction data" (2014), +J. Appl. Cryst. 47, 1118-1131 +[doi:10.1107/S1600576714007626](https://doi.org/10.1107/S1600576714007626). + +Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the +ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, +A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP +experiments" (2025), [arXiv:2505.22822](https://arxiv.org/abs/2505.22822); +[traccc](https://github.com/acts-project/traccc). The CPU spot extractor adapts its SparseCCL source, +and the CUDA spot extractor follows the design of its GPU counterpart - a backward-neighbour graph +over a sorted hit list, resolved by a parallel union-find. traccc is MPL-2.0; see +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). The SparseCCL algorithm itself is A. Hennequin, +B. Couturier, V. V. Gligorov and L. Lacassagne, "SparseCCL: Connected Components Labeling and +Analysis for sparse images" (2019), DASIP 2019, 65-70 +[doi:10.1109/DASIP48288.2019.9049184](https://doi.org/10.1109/DASIP48288.2019.9049184). + +### Indexing + +**[MOSFLM](https://www.mrc-lmb.cam.ac.uk/mosflm/)** — the Rossmann FFT autoindexing algorithm and +post-refinement practice, including which parameters are safe to refine per image and which must be +refined over a wedge. The autoindexing algorithm itself — projecting the reciprocal-space points +onto many directions and Fourier-transforming the 1D projection histograms — is I. Steller, +R. Bolotovsky and M. G. Rossmann, "An algorithm for automatic indexing of oscillation images using +Fourier analysis" (1997), J. Appl. Cryst. 30, 1036-1040 +[doi:10.1107/S0021889897008777](https://doi.org/10.1107/S0021889897008777); MOSFLM is the +implementation whose practice is followed. A. G. W. Leslie and H. R. Powell, "Processing diffraction data with MOSFLM" +(2007), in *Evolving Methods for Macromolecular Crystallography*, NATO Science Series II, vol. 245, +41-51 [doi:10.1007/978-1-4020-6316-9_4](https://doi.org/10.1007/978-1-4020-6316-9_4); +T. G. G. Battye, L. Kontogiannis, O. Johnson, H. R. Powell and A. G. W. Leslie, "iMOSFLM: a new +graphical interface for diffraction-image processing with MOSFLM" (2011), Acta Cryst. D67, 271-281 +[doi:10.1107/S0907444910048675](https://doi.org/10.1107/S0907444910048675); H. R. Powell, +T. G. G. Battye, L. Kontogiannis, O. Johnson and A. G. W. Leslie, "Integrating macromolecular X-ray +diffraction data with the graphical user interface iMosflm" (2017), Nat. Protoc. 12, 1310-1325 +[doi:10.1038/nprot.2017.037](https://doi.org/10.1038/nprot.2017.037). + +**[fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer)** — the +known-cell indexer for serial stills (`-X ffbidx`) is PSI's fast-feedback-indexer library, linked at +build time (BSD-3-Clause; see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)), which implements +the TORO algorithm: P. Gasparotto, L. Barba, H.-C. Stadler, G. Assmann, H. Mendonça, A. W. Ashton, +M. Janousch, F. Leonarski and B. Béjar, "TORO Indexer: a PyTorch-based indexing algorithm for +kilohertz serial crystallography" (2024), J. Appl. Cryst. 57, 931-944 +[doi:10.1107/S1600576724003182](https://doi.org/10.1107/S1600576724003182). + +### Cell reduction and lattice symmetry + +**[GEMMI](https://github.com/project-gemmi/gemmi)** — symmetry operations, unit-cell and +structure-factor machinery, and MTZ / XDS_ASCII I/O. Vendored in `gemmi_gph/`, so it also carries a +licence obligation. M. Wojdyr, "GEMMI: A library for structural biology" (2022), J. Open Source +Softw. 7, 4200 [doi:10.21105/joss.04200](https://doi.org/10.21105/joss.04200). + +**Křivý & Gruber's Niggli reduction, and the lattice-character table** — the reduction that puts +every candidate cell in a comparable form is I. Křivý and B. Gruber, "A unified algorithm for +determining the reduced (Niggli) cell" (1976), Acta Cryst. A32, 297-298 +[doi:10.1107/S0567739476000636](https://doi.org/10.1107/S0567739476000636), used through GEMMI's +implementation; the table of lattice characters that maps a reduced cell to Bravais lattices and +centrings follows International Tables for Crystallography Vol. A, Table 9.2.5.1. + +**Grosse-Kunstleve, Sauter & Adams's numerically stable cell reduction** - the magnitude-scaled +tolerance that decides the sign of a structurally-zero scalar product, and with it the Niggli type a +reduced cell is presented in. R. W. Grosse-Kunstleve, N. K. Sauter and P. D. Adams, "Numerically +stable algorithms for the computation of reduced unit cells" (2004), Acta Cryst. A60, 1-6 +[doi:10.1107/S010876730302186X](https://doi.org/10.1107/S010876730302186X). + +**Le Page's metric-symmetry search** - the obliquity of each of the 81 candidate two-folds of a +reduced cell, which is what tells a run that its lattice metric hosts more rotational symmetry than +the group its intensities supported, and the derivation of the conventional axes from that +rotation group, which is what the run then offers to the space-group search as a second lattice +candidate. The two-fold search is used through GEMMI's implementation of it. Y. Le Page, "The +derivation of the axes of the conventional unit cell from the dimensions of the Buerger-reduced +cell" (1982), J. Appl. Cryst. 15, 255-259 +[doi:10.1107/S0021889882011959](https://doi.org/10.1107/S0021889882011959). + +### Integration and rotation geometry + +**[XDS](https://xds.mr.mpg.de/)** — rotation geometry and notation, the reciprocal Lorentz and +partiality treatment, the maximum-likelihood mosaicity estimate, the `MINPK` criterion for rejecting +a reflection whose predicted profile is not cleanly its own, the intensity-based test for a +centred lattice, the recognition of shaded detector regions by comparing a pixel's background +against the background at its own resolution (`DEFPIX`), and the scaling correction surfaces +indexed by image number and detector region. W. Kabsch, "XDS" (2010), Acta Cryst. D66, 125-132 +[doi:10.1107/S0907444909047337](https://doi.org/10.1107/S0907444909047337); W. Kabsch, "Integration, +scaling, space-group assignment and post-refinement" (2010), Acta Cryst. D66, 133-144 +[doi:10.1107/S0907444909047374](https://doi.org/10.1107/S0907444909047374). + +**Profile fitting** with reweighted, de-biased variances is the Kabsch/Otwinowski iteration, from the +second XDS paper above and from Z. Otwinowski and W. Minor, "Processing of X-ray diffraction data +collected in oscillation mode" (1997), Methods Enzymol. 276, 307-326 +[doi:10.1016/S0076-6879(97)76066-X](https://doi.org/10.1016/S0076-6879%2897%2976066-X). + +**The two-dimensional integration architecture** — integrating each image in the detector plane +and only afterwards assembling a reflection's partials into a full across images, as against +three-dimensional profile fitting through the image stack — is the architecture of DENZO/SCALEPACK +and MOSFLM, and it is the one Rugnux's rotation pipeline follows (per-image profile-fitted +integration, then partials combined into fulls; §9 and §10.6 of the [data-analysis reference](CPU_DATA_ANALYSIS.md)). The +Otwinowski & Minor citation above and the MOSFLM citations below carry the credit for the paradigm +as well as for the specifics taken from each. + +### Space group, twinning and pseudo-symmetry + +**[POINTLESS](https://www.ccp4.ac.uk/)** (CCP4) — the space-group search. Stage A scores each +candidate rotation operator by the correlation of I(h) with I(Rh) on **resolution-normalised** +intensities (E²), as POINTLESS does — both arms of a symmetry pair sit at the same |s|, so on raw +intensities the resolution fall-off is variance shared between them and lifts a false operator's +correlation as much as a true one's; the screw-axis test scores a +predicted-absent class against the rest of its own axial row rather than against a global mean or a +fixed cut, and lets confidence fall away with the number of axial reflections instead of refusing +below a count; the glide-plane test is that same test applied to a zone, scoring the extinguished +class against the rest of its own plane, as POINTLESS scores zonal absences. P. Evans, "Scaling and assessment of data quality" (2006), Acta Cryst. D62, 72-82 +[doi:10.1107/S0907444905036693](https://doi.org/10.1107/S0907444905036693); P. R. Evans, "An +introduction to data reduction: space-group determination, scaling and intensity statistics" (2011), +Acta Cryst. D67, 282-292 [doi:10.1107/S090744491003982X](https://doi.org/10.1107/S090744491003982X); +P. R. Evans and G. N. Murshudov, "How good are my data and what is the resolution?" (2013), Acta +Cryst. D69, 1204-1214 [doi:10.1107/S0907444913000061](https://doi.org/10.1107/S0907444913000061); +J. Agirre, M. Atanasova, H. Bagdonas et al., "The CCP4 suite: integrative software for macromolecular +crystallography" (2023), Acta Cryst. D79, 449-461 +[doi:10.1107/S2059798323003595](https://doi.org/10.1107/S2059798323003595). + +**Baur and Kassner** — the convention for which of two absence-equivalent glide groups the space-group +search writes (P2/c over Pc, C2/c over Cc): the centrosymmetric one, because a missed inversion centre is +the common error among published small-molecule space-group assignments. W. H. Baur and D. Kassner, +"The perils of Cc: comparing the frequencies of falsely assigned space groups with their general +population" (1992), Acta Cryst. B48, 356-369 +[doi:10.1107/S0108768191014726](https://doi.org/10.1107/S0108768191014726). + +**The centre-of-symmetry statistics** are Wilson's and Howells, Phillips & Rogers's: the intensity +distributions of acentric and centric structures and the cumulative N(z) test built on them, read here +on the general reflections of the Laue class beside <|E^2-1|> and Padilla and Yeates's L test (taken at +its centric value, 2/pi). A. J. C. Wilson, "The probability distribution of X-ray intensities" (1949), +Acta Cryst. 2, 318-321 [doi:10.1107/S0365110X49000813](https://doi.org/10.1107/S0365110X49000813); +E. R. Howells, D. C. Phillips and D. Rogers, "The probability distribution of X-ray intensities. II. +Experimental investigation and the X-ray detection of centres of symmetry" (1950), Acta Cryst. 3, +210-214 [doi:10.1107/S0365110X50000513](https://doi.org/10.1107/S0365110X50000513). + +**The twinning L test** is Padilla and Yeates's: pairing each acentric reflection with a +symmetry-independent neighbour and reading the first and second moments of +L = (I1−I2)/(I1+I2) against their untwinned and perfect-twin values. J. E. Padilla and +T. O. Yeates, "A statistic for local intensity differences: robustness to anisotropy and +pseudo-centering and utility for detecting twinning" (2003), Acta Cryst. D59, 1124-1130 +[doi:10.1107/S0907444903007947](https://doi.org/10.1107/S0907444903007947). Their title claims +robustness to pseudo-centering, and this program's partner steps deliver it: a step of 2 along an +axis preserves the class of a half-integer pseudo-translation, which is what a pseudo-centering is. +That robustness does not extend to a pseudo-translation which is not half-integer, and the partner +steps are restricted when one is detected - see the translational-pseudo-symmetry note below. + +**Translational pseudo-symmetry** is detected from the native Patterson computed from the merged +intensities, and the interpretation of an off-origin peak as a pseudo-translation between copies of +the contents of the asymmetric unit - together with the modulation it puts on the intensities, which +is the second half of the test here - is Read, Adams and McCoy's. Their fitted peak-height table is +not used: the peak is scored against a per-dataset within-shell permutation null instead, because the +noise floor of the statistic depends strongly on how many reflections a dataset has. The same +modulation is what the axial systematic-absence test scores against, so that a reflection class a +pseudo-translation merely suppresses is not read as extinct and does not buy a screw axis; the +estimate of its depth there is our own, measured per axial row from the merged intensities rather +than from the Patterson vector. R. J. Read, +P. D. Adams and A. J. McCoy, "Intensity statistics in the presence of translational +noncrystallographic symmetry" (2013), Acta Cryst. D69, 176-183 +[doi:10.1107/S0907444912045374](https://doi.org/10.1107/S0907444912045374). + +### Scaling, merging and data quality + +**[DIALS](https://dials.github.io/)** — the resolution cutoff from the CC1/2 fall-off, per-observation +outlier rejection at merge, the scaling error model, and the treatment of a reflection whose +background is contaminated. Its published behaviour, and in places its source, settled several +choices here. G. Winter, D. G. Waterman, J. M. Parkhurst et al., "DIALS: implementation and +evaluation of a new integration package" (2018), Acta Cryst. D74, 85-97 +[doi:10.1107/S2059798317017235](https://doi.org/10.1107/S2059798317017235); D. G. Waterman, +G. Winter, R. J. Gildea et al., "Diffraction-geometry refinement in the DIALS framework" (2016), +Acta Cryst. D72, 558-575 [doi:10.1107/S2059798316002187](https://doi.org/10.1107/S2059798316002187); +J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in the DIALS software +package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399 +[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198); J. M. Parkhurst, +G. Winter, D. G. Waterman et al., "Robust background modelling in DIALS" (2016), J. Appl. Cryst. 49, +1912-1921 [doi:10.1107/S1600576716013595](https://doi.org/10.1107/S1600576716013595). + +**Wilson outlier test** — judging an observation that has no symmetry mates against the acentric and +centric intensity distributions of its resolution shell, with the symmetry enhancement factor, is +Wilson's statistics; rejecting only observations that are also significant, and keeping a reflection +whose observations are all large, follows AIMLESS's EMAX test. A. J. C. Wilson, "The probability +distribution of X-ray intensities" (1949), Acta Cryst. 2, 318-321 +[doi:10.1107/S0365110X49000813](https://doi.org/10.1107/S0365110X49000813); +P. Evans, "Scaling and assessment of data quality" (2006), Acta Cryst. D62, 72-82 +[doi:10.1107/S0907444905036693](https://doi.org/10.1107/S0907444905036693). + +**Amplitudes from intensities** — the posterior-mean amplitude of each merged intensity under the +acentric and centric Wilson priors is French and Wilson's; giving no amplitude to an intensity more +than 3.7 sigma below zero, and leaving such intensities out of the prior, follows CCP4's +[ctruncate](https://www.ccp4.ac.uk/) (C. Ballard and N. Stein). So does scaling each reflection's +Wilson prior by the anisotropy tensor along its direction, which ctruncate has done by default since +its version 1.7 ("use anisotropy in prior for truncate procedure"); rugnux uses its own tensor for it +(see Diffraction anisotropy below). S. French and K. Wilson, "On the treatment of negative intensity +observations" (1978), Acta Cryst. A34, 517-525 +[doi:10.1107/S0567739478001114](https://doi.org/10.1107/S0567739478001114); ctruncate is cited through +the CCP4 suite: M. D. Winn, C. C. Ballard, K. D. Cowtan et al., +"Overview of the CCP4 suite and current developments" (2011), Acta Cryst. D67, 235-242 +[doi:10.1107/S0907444910045749](https://doi.org/10.1107/S0907444910045749). + +**Absorption as spherical harmonics** — describing an empirical absorption correction as a series of +real spherical harmonics of the beam directions in the crystal frame is Blessing's; its use as a +restrained scaling surface of the diffracted-beam direction follows SCALA and AIMLESS. R. H. Blessing, +"An empirical correction for absorption anisotropy" (1995), Acta Cryst. A51, 33-38 +[doi:10.1107/S0108767394005726](https://doi.org/10.1107/S0108767394005726); P. Evans, "Scaling and +assessment of data quality" (2006), Acta Cryst. D62, 72-82 +[doi:10.1107/S0907444905036693](https://doi.org/10.1107/S0907444905036693). + +**Diffraction anisotropy** — the description of the overall fall-off by a single anisotropic +displacement tensor, its symmetry constraints, and the fact that only its deviatoric part is +determined (the isotropic part being degenerate with the overall scale) are Sheriff and Hendrickson's. +The estimator fits that tensor to the observed intensity distribution, taking sigma(I) into account, +in the sense of Popov and Bourenkov. The directional diffraction limits - in a cone about +each principal direction, and the reporting of the anisotropic deltaB as the range of the principal +components - follow AIMLESS. Rugnux reports these; it corrects no intensity and removes no reflection +on a directional criterion. S. Sheriff and W. A. Hendrickson, "Description of overall anisotropy in +diffraction from macromolecular crystals" (1987), Acta Cryst. A43, 118-121 +[doi:10.1107/S010876738709977X](https://doi.org/10.1107/S010876738709977X); A. N. Popov and +G. P. Bourenkov, "Choice of data-collection parameters based on statistic modelling" (2003), Acta +Cryst. D59, 1145-1153 [doi:10.1107/S0907444903008163](https://doi.org/10.1107/S0907444903008163); +P. R. Evans and G. N. Murshudov, "How good are my data and what is the resolution?" (2013), Acta +Cryst. D69, 1204-1214 [doi:10.1107/S0907444913000061](https://doi.org/10.1107/S0907444913000061). + +**Data-quality statistics** follow the established conventions rather than any one program: R_meas +and R_pim, CC1/2 and CC\*, the per-shell CC(model, data) between F^2_calc and F^2_obs, and the +reporting of I/sigma(I). K. Diederichs and P. A. Karplus, "Improved +R-factors for diffraction data analysis in macromolecular crystallography" (1997), Nat. Struct. Biol. +4, 269-275 [doi:10.1038/nsb0497-269](https://doi.org/10.1038/nsb0497-269); P. A. Karplus and +K. Diederichs, "Linking crystallographic model and data quality" (2012), Science 336, 1030-1033 +[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231); K. Diederichs and +P. A. Karplus, "Better models by discarding data?" (2013), Acta Cryst. D69, 1215-1222 +[doi:10.1107/S0907444913001121](https://doi.org/10.1107/S0907444913001121). + +**The Whittaker smoother** — the per-frame scale of the rotation fulls is a penalised least-squares +curve (a second-difference penalty, the smoothness chosen by cross-validation) +in the form P. H. C. Eilers gave Whittaker's graduation: P. H. C. Eilers, "A perfect smoother" (2003), +Anal. Chem. 75, 3631-3636 [doi:10.1021/ac034173t](https://doi.org/10.1021/ac034173t); E. T. Whittaker, +"On a new method of graduation" (1923), Proc. Edinburgh Math. Soc. 41, 63-75 +[doi:10.1017/S0013091500077853](https://doi.org/10.1017/S0013091500077853). + +**Fisher's z-transformation** — the cross-validation of the scaling correction surfaces averages the +change of the half-set CC1/2 over resolution shells on atanh(CC), so that the shells near CC = 1, where +a multiplicative error shows, are not outweighed by the noise of the shells without signal. R. A. +Fisher, "Frequency distribution of the values of the correlation coefficient in samples from an +indefinitely large population" (1915), Biometrika 10, 507-521 +[doi:10.2307/2331838](https://doi.org/10.2307/2331838). + +**The frame disposition** - which stretches of a rotation sweep are kept, carried at reduced weight or +dropped from the merge - decides every exclusion on delta-CC1/2, the change in the overall CC1/2 when a +group of images is left out, measured in the sigma-tau form so that no random half-dataset split is +involved. The statistic, the Fisher transformation used to compare it across CC1/2 values, its standard +error going as the inverse square root of the reflection count, and the rejection discipline (never +remove a group whose delta-CC1/2 is positive or near zero; remove a little, re-form the reference and +repeat) are all taken from its authors, whose XDSCC12 is the reference implementation. +G. Assmann, W. Brehm and K. Diederichs, "Identification of rogue datasets in serial crystallography" +(2016), J. Appl. Cryst. 49, 1021-1028 +[doi:10.1107/S1600576716005471](https://doi.org/10.1107/S1600576716005471); G. M. Assmann, M. Wang and +K. Diederichs, "Making a difference in multi-data-set crystallography: simple and deterministic +data-scaling/selection methods" (2020), Acta Cryst. D76, 636-652 +[doi:10.1107/S2059798320006348](https://doi.org/10.1107/S2059798320006348). That the same statistic +belongs at scaling, applied to groups of images rather than to whole datasets, follows +[DIALS](https://dials.github.io/) (`dials.scale`, delta-CC1/2 image-group filtering): +J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in the DIALS software +package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399 +[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198). + +**Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature: +D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography: +Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75 +[doi:10.1107/S0108767388009596](https://doi.org/10.1107/S0108767388009596); D. Schwarzenbach, +S. C. Abrahams, H. D. Flack, E. Prince and A. J. C. Wilson, "Statistical descriptors in +crystallography. II. Report of a Working Group on Expression of Uncertainty in Measurement" (1995), +Acta Cryst. A51, 565-569 [doi:10.1107/S0108767395002340](https://doi.org/10.1107/S0108767395002340). + +### Physical corrections and calibration + +**Sensor absorption at oblique incidence, and the flight path** — the angle-dependent quantum +efficiency of a flat sensor, the radial parallax variance that comes from the same integral, and the +attenuation of a reflection in the air between the sample and its pixel are all the Beer-Lambert law +taken along a ray that crosses t/cos(alpha) of sensor, or D/cos(alpha) of air, and converts at a +random depth. The attenuation coefficients, for silicon, CdTe, dry air and helium alike, are the +NIST tabulation: J. H. Hubbell and S. M. Seltzer, "Tables of X-Ray +Mass Attenuation Coefficients and Mass Energy-Absorption Coefficients from 1 keV to 20 MeV for +Elements Z = 1 to 92 and 48 Additional Substances of Dosimetric Interest" (1995, data updated 2004), +NIST Standard Reference Database 126 +[doi:10.18434/T4D01F](https://doi.org/10.18434/T4D01F). + +**Polarization correction** — the azimuthal polarization factor applied to the azimuthally +integrated profile, to the integrated Bragg intensities and to the ring background the beam-stop +shadow test compares a pixel against is the one derived for a partially polarized +synchrotron source by R. Kahn, R. Fourme, A. Gadet, J. Janin, C. Dumas and D. Andre, "Macromolecular +crystallography with synchrotron radiation: photographic data collection and polarization +correction" (1982), J. Appl. Cryst. 15, 330-337 +[doi:10.1107/S0021889882012060](https://doi.org/10.1107/S0021889882012060). + +**Hexagonal-ice ring positions** — the eleven ring $d$ spacings from 3.895 to 1.522 Å that the +ice-ring score, the ice-ring flagging and the ice calibrant are all built on are taken from the +measurements of Moreau and co-workers, not enumerated from a cell. D. W. Moreau, H. Atakisi and R. E. Thorne, "Ice in +biomolecular cryocrystallography" (2021), Acta Cryst. D77, 540-554 +[doi:10.1107/S2059798321001170](https://doi.org/10.1107/S2059798321001170). + +That list ends at 1.522 Å by its own scope, so the eight bands below it are calculated here rather +than taken from anyone: ice Ih structure factors on the oxygen sublattice, kept where they reach 3% of +the strongest line, which reproduces the eleven measured positions exactly. The lattice constants are +Röttger and co-workers'. K. Röttger, A. Endriss, J. Ihringer, S. Doyle and W. F. Kuhs, "Lattice +constants and thermal expansion of H2O and D2O ice Ih between 10 and 265 K" (1994), Acta Cryst. B50, +644-648 [doi:10.1107/S0108768194004933](https://doi.org/10.1107/S0108768194004933). + +### Model-based analysis and maps + +**Bulk-solvent correction and overall scaling** — the model's structure factors are put on the +observed scale with an overall factor, an anisotropic B and a flat bulk-solvent term, the flat-mask +model of A. Fokine and A. Urzhumtsev, "Flat bulk-solvent model: obtaining optimal parameters" +(2002), Acta Cryst. D58, 1387-1392 +[doi:10.1107/S0907444902010284](https://doi.org/10.1107/S0907444902010284), which is also the source +of the starting values and of the range those two parameters are physically meaningful over. The +procedure that fits them — a grid search over that range for the solvent pair, with the overall +scale and the anisotropic B refitted at every grid point — follows P. V. Afonine, +R. W. Grosse-Kunstleve and P. D. Adams, "A robust bulk-solvent correction and anisotropic scaling +procedure" (2005), Acta Cryst. D61, 850-855 +[doi:10.1107/S0907444905007894](https://doi.org/10.1107/S0907444905007894). The fit is unweighted, +as in both that procedure and REFMAC5: G. N. Murshudov, P. Skubak, A. A. Lebedev, N. S. Pannu, +R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long and A. A. Vagin, "REFMAC5 for the refinement of +macromolecular crystal structures" (2011), Acta Cryst. D67, 355-367 +[doi:10.1107/S0907444911001314](https://doi.org/10.1107/S0907444911001314). + +**sigma_A map coefficients** — the maps written by `--model` are weighted by a maximum-likelihood +sigma_A estimated per resolution shell, giving 2mFo-DFc and mFo-DFc rather than 2Fo-Fc and Fo-Fc. +What is taken is the formalism itself: the Rice and Woolfson likelihoods of |Fo| given |Fc| and +sigma_A, the figure of merit m and the scale D that follow from it, and the result that the +bias-corrected coefficient is 2mFo-DFc for an acentric reflection and mFo for a centric one. +R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with +errors" (1986), Acta Cryst. A42, 140-149 +[doi:10.1107/S0108767386099622](https://doi.org/10.1107/S0108767386099622). + +**[ANODE](https://doi.org/10.1107/S0021889811041768)** — reading the anomalous difference map at the +atoms of a supplied model and reporting the strongest sites by name, instead of searching the map for +blobs. The map itself is the textbook anomalous difference Fourier; what is taken from ANODE is that +reading: A. Thorn and G. M. Sheldrick, "ANODE: anomalous and heavy-atom density calculation" (2011), +J. Appl. Cryst. 44, 1285-1287 +[doi:10.1107/S0021889811041768](https://doi.org/10.1107/S0021889811041768). + +**Uniform random rotations** — the null a supplied model is scored against re-orients that model at +random about its own centroid, and the rotations are drawn uniformly from SO(3) through a uniform +random unit quaternion. K. Shoemake, "Uniform Random Rotations", in *Graphics Gems III*, ed. D. Kirk, +Academic Press (1992), 124-132 (no DOI). + +**Variable projection** — the rigid-body placement of a supplied model re-fits the overall scale at +every step, and its Jacobian folds that re-fit in by projecting the scale's own derivatives out of the +placement's, in Kaufman's simplified form of Golub and Pereyra's derivative of the reduced problem. +G. H. Golub and V. Pereyra, "The Differentiation of Pseudo-Inverses and Nonlinear Least Squares +Problems Whose Variables Separate" (1973), SIAM J. Numer. Anal. 10, 413-432 +[doi:10.1137/0710036](https://doi.org/10.1137/0710036). L. Kaufman, "A variable projection method +for solving separable nonlinear least squares problems" (1975), BIT 15, 49-57 +[doi:10.1007/BF01932995](https://doi.org/10.1007/BF01932995). + +## Software and computing methods + +Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading +the result, follows Jon Wright (ESRF): "Experiences with GPU decompression for bitshuffle + LZ4 +data", HDF5 User Group meeting (2021), and [bslz4decoders](https://github.com/jonwright/bslz4decoders). +The CUDA kernels in Jungfraujoch are its own, but the approach is his. + +Removing the small islands of solvent from the bulk-solvent mask on the GPU labels the connected +components with a parallel union-find, following D. P. Playne and K. Hawick, "A New Algorithm for +Parallel Connected-Component Labelling on GPUs" (2018), IEEE Trans. Parallel Distrib. Syst. 29, +1217-1230 [doi:10.1109/TPDS.2018.2799216](https://doi.org/10.1109/TPDS.2018.2799216). + +This software uses the Viridis, Magma and Inferno colormaps from Matplotlib under its +BSD-compatible license. J. D. Hunter, "Matplotlib: A 2D graphics environment" (2007), Comput. Sci. +Eng. 9, 90-95 [doi:10.1109/MCSE.2007.55](https://doi.org/10.1109/MCSE.2007.55). + +## File formats read from a published specification + +**CBF / imgCIF** - the native miniCBF reader implements the `x-CBF_BYTE_OFFSET` compression scheme +and reads the imgCIF `_axis` table (the laboratory directions of the image's fast and slow pixel +directions, of the goniometer axes and of a 2theta arm) from the specification alone; no CBFlib or +other CBF code is used, so there is no licence obligation, only this credit. +H. J. Bernstein and A. P. Hammersley, "Specification of the Crystallographic Binary File +(CBF/imgCIF)" (2006), International Tables for Crystallography Vol. G, 37-43 +[doi:10.1107/97809553602060000729](https://doi.org/10.1107/97809553602060000729); +A. P. Hammersley, H. J. Bernstein and J. D. Westbrook, "Image dictionary (imgCIF)" (2006), +International Tables for Crystallography Vol. G, 444-458 +[doi:10.1107/97809553602060000746](https://doi.org/10.1107/97809553602060000746). + +**d\*TREK SMV** - the SMV reader reads the d\*TREK header vocabulary written by Rigaku's +CrystalClear (Saturn and R-AXIS detectors: detector and spatial-distortion vectors, detector +circles, 2theta arm, encoded overflows) from the headers themselves, interpreting the detector +vectors as dxtbx does for these detectors; no d\*TREK or dxtbx code is used. +J. W. Pflugrath, "The finer things in X-ray diffraction data collection" (1999), Acta Cryst. D55, +1718-1725 [doi:10.1107/S090744499900935X](https://doi.org/10.1107/S090744499900935X); +[dxtbx](https://github.com/cctbx/dxtbx): J. M. Parkhurst, A. S. Brewster, L. Fuentes-Montero, +D. G. Waterman et al., "dxtbx: the diffraction experiment toolbox" (2014), J. Appl. Cryst. 47, +1459-1465 [doi:10.1107/S1600576714011996](https://doi.org/10.1107/S1600576714011996). + +## Public diffraction data used for testing + +In addition to in-house datasets collected at SLS 2.0, Jungfraujoch is tested against public +diffraction data collected on other people's beamlines, on detectors and in file formats we do not +produce ourselves - most of it at other facilities, a few sets at the Swiss Light Source but not by +this system. That data was collected and published by other people. Every dataset used, the DOI to +cite for it, and the deposition it belongs to are listed in +[EXTERNAL_TEST_DATA](EXTERNAL_TEST_DATA.md); we thank the depositors, and the repositories that make +the data findable and citable. + +**[IRRMC](https://proteindiffraction.org/)**, the Integrated Resource for Reproducibility in +Macromolecular Crystallography (Minor lab, University of Virginia), is the source of most of them. +IRRMC releases its data under CC0 and asks that the DOI of the dataset be +cited. M. Grabowski, K. M. Langner, M. Cymborowski, P. J. Porebski, P. Sroka, H. Zheng, +D. R. Cooper, M. D. Zimmerman, M.-A. Elsliger, S. K. Burley and W. Minor, "A public database of +macromolecular diffraction experiments" (2016), Acta Cryst. D72, 1181-1193 +[doi:10.1107/S2059798316014716](https://doi.org/10.1107/S2059798316014716); M. Grabowski, +M. Cymborowski, P. J. Porebski, T. Osinski, I. G. Shabalin, D. R. Cooper and W. Minor, "The +Integrated Resource for Reproducibility in Macromolecular Crystallography: Experiences of the first +four years" (2019), Struct. Dyn. 6, 064301 +[doi:10.1063/1.5128672](https://doi.org/10.1063/1.5128672). + +**[SBGrid Data Bank](https://data.sbgrid.org/)**, the structural biology community's data +publication service (SBGrid Consortium, Harvard Medical School). P. A. Meyer, +S. Socias, J. Key, E. Ransey, E. C. Tjon, A. Buschiazzo et al., "Data publication with the +structural biology data grid supports live analysis" (2016), Nat. Commun. 7, 10882 +[doi:10.1038/ncomms10882](https://doi.org/10.1038/ncomms10882). + +**[Zenodo](https://zenodo.org/)**, CERN's open repository, hosts datasets deposited there directly +by the groups that collected them. European Organization for Nuclear Research and OpenAIRE, +"Zenodo" (2013), CERN [doi:10.25495/7GXK-RD71](https://doi.org/10.25495/7GXK-RD71). Some of those +deposits are described in IUCrData Raw Data Letters; the letters are cited on the +[EXTERNAL_TEST_DATA](EXTERNAL_TEST_DATA.md) page, beside the datasets they describe. + +**[MXRDR](https://mxrdr.icm.edu.pl/)**, the Macromolecular Xtallography Raw Data Repository +(ICM, University of Warsaw), releases its data under CC0 and asks that the DOI of the dataset be +cited. + +**[XRDa](https://xrda.pdbj.org/)**, the Xtal Raw Data Archive (Protein Data Bank Japan), which +publishes raw diffraction images - X-ray, electron and neutron - and mints a DOI for each; it asks +that the DOI of the dataset be cited. It has no canonical citation paper. + +**The [ESRF data portal](https://data.esrf.fr/)**, through which the European Synchrotron +publishes raw data under its data policy (CC BY 4.0, with the dataset DOI to be cited). +R. Dimper, A. Götz, A. De Maria, V. A. Solé, M. Chaillet and B. Lebayle, "ESRF Data Policy, +Storage, and Services" (2019), Synchrotron Rad. News 32, 7-12 +[doi:10.1080/08940886.2019.1608119](https://doi.org/10.1080/08940886.2019.1608119). + +**The [Keele University research data repository](https://researchdata.keele.ac.uk/)** and +**[UQ eSpace](https://espace.library.uq.edu.au/)** (The University of Queensland), which host the +raw images of datasets deposited there by the groups that collected them; the dataset DOIs are +cited on the [EXTERNAL_TEST_DATA](EXTERNAL_TEST_DATA.md) page. + +The beamline, resolution, space group and unit cell quoted for each dataset are the values +deposited with the corresponding PDB entry, read from the RCSB PDB data API. H. M. Berman, +J. Westbrook, Z. Feng, G. Gilliland, T. N. Bhat, H. Weissig, I. N. Shindyalov and P. E. Bourne, +"The Protein Data Bank" (2000), Nucleic Acids Res. 28, 235-242 +[doi:10.1093/nar/28.1.235](https://doi.org/10.1093/nar/28.1.235). + +## Generative AI usage declaration + +Large language models were used extensively in developing this code. Jungfraujoch development was +supported with JetBrains AI (mostly GPT models) to refactor and verify particular code fragments. +Rugnux was developed with the assistance of Claude Code (mostly the Opus model). This documentation +was written with the assistance of Claude Opus and Fable models. diff --git a/_sources/BATTERY_REPORT.md.txt b/_sources/BATTERY_REPORT.md.txt new file mode 100644 index 000000000..b378d1c71 --- /dev/null +++ b/_sources/BATTERY_REPORT.md.txt @@ -0,0 +1,929 @@ +# Rugnux battery - 20261005-2102_3770f4_rc174-final-refmac + +- rugnux 1.0.0-rc.174, sha256 fee1c90710c43704, build flags: Release CXX_FLAGS='-march=x86-64-v3' CUDA=ON +- runner 3770f42c9, host mpc2898.psi.ch, 2026-10-05T21:02:33 -> 2026-10-06T06:58:55 +- hardware: AMD Ryzen 9 5950X 16-Core Processor, 3.40 GHz nominal (acpi_cppc), 5.08 GHz max boost, 16C/32T, 1 socket(s), 125 GiB RAM, GPU: NVIDIA GeForce RTX 5080 (16303 MiB) +- arms: open, inhouse; 249 sets (full arm) +- timing: NOT a reference - the GPU was shared +- command options: (none beyond output selection) +- command, open arm: `rugnux --model ` (without --model where there is no deposited model: small molecules, unpublished sets) +- command, inhouse arm: `rugnux --report-resolution , [--model ] `: the second statistics table at XDS's range is report-only; where XDS kept Friedel mates apart the scorer reads the run's own per-hand table, which needs no flag. --model where the set names a PDB entry of its crystal form (the standard proteins) + +## Commentary + +This is the validation battery of release 1.0.0-rc.174 (the binary built from `3770f42c9` with the CI +flags), run once per set with the REFMAC model check, against the rc.173 battery as baseline. The +private arm ran too; as always, only its aggregate is stated here: 23 pass, 2 fail, 2 not scored, with +no verdict changed from rc.173. + +- **Verdicts.** Open arm 188 pass / 11 fail / 3 not scored over 202 sets, in-house 46 / 1 over 47. + Every failure was known before this run. Seven are point groups called too high on twinned or + pseudo-symmetric crystals (2wnq, 4bwl, 5ebi, 6oww, 6p8j, 8c3e, 3r6o), two are screw axes (5cc8, + 9hnc), two are halved lattices (6z9g, 9min), and KDP is scored against I-42d while Rugnux reports + the group with the same absences, I4_1md. On the 171 open sets both runs have, 164 of 168 scored + pass against 166 of 169 at rc.173 (5ebi and 9hnc lost, 7mzt now not scored); the in-house sets + both runs have all pass. The 31 open sets added since rc.173 (twinned crystals, home-source and + other facilities) carry most of the failures. +- **R-free.** The deposited model refined by REFMAC against Rugnux's data comes within 0.05 of the + same refinement against the depositor's data on 174 of 182 structures (rc.173: 154 of 159). On the + 160 structures both runs scored, the median change is -0.001, 33 better and 15 worse by more than + 0.005. Two losses are worth a look: 9qw8 (0.264 to 0.338, with a coarser resolution cut, 1.71 + against 1.59 A) and 9i80 (0.234 to 0.279 at the same resolution, a twinned P4_1 crystal). +- **Resolution.** Rugnux's own cut is finer than the deposited one on 173 of 184 structures. +- **Processing time.** The median over all 249 sets is 28 s (open arm 33 s, rc.173 44 s); on the + sets both runs have, the median time ratio is 0.88 on the open arm and 0.78 on the in-house arm. + The battery times include reading the images from disk, so they understate the gain in + computation. The in-house arm alone (standard proteins, small molecules and the two no-crystal + controls; the proteins now also run the model check) has a median of 13.7 s against 17.4 s at + rc.173; its slowest sets are KDP (93 s), L-cystine at 20 keV (57 s) and the two controls (44-48 s), + which search for a lattice that is not there. The header line "the GPU was shared" only records that the run used `--gpulock`; no + set saw another process on the GPU. +- **Small molecules.** SHELXL R1 against the published structures is at or below XDS on the + organic sets (aspirin 0.036, citric acid 0.037, HEPES 0.031) and well below it on KDP (0.048 + against 0.112); YAG (0.102) and L-cystine (0.14-0.16) remain the weak cases. + +![Processing time of every open and in-house set](images/battery_time.png) + +![The same, over the first 60 s](images/battery_time_zoom.png) + +![Processing time of the in-house sets, over the first 60 s](images/battery_time_inhouse.png) + +![R-free of the deposited model refined against the depositor's data and against Rugnux's data](images/battery_rfree.png) + +![High-resolution limit: deposition against Rugnux](images/battery_dmin.png) + +## Summary + +| arm | sets | pass | of which alt | fail | unscored | not run | pass rate | median res. gain | median ISa | median R_meas | median time s | total time min | +|:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:| +| open | 202 | 188 | 5 | 11 | 3 | 0 | 188/199 (94%) | +12.1% | 13.6 | 13.8% | 33 | 176.2 | +| inhouse | 47 | 46 | 0 | 1 | 0 | 0 | 46/47 (98%) | +6.3% | 21.8 | 9.8% | 14 | 14.0 | + +_Plot in report.html: verdicts per arm_ + +Pass rate is over the scored sets (pass + fail). `of which alt` counts the passes that are already in `pass`: rows where the answer matched an accepted alternative reference rather than the deposition, listed under Accepted alternatives below. In the bar chart they are drawn as their own segment. Every number in this table is rugnux's own run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own resolution range, so they describe a run and do not rank two; the like-for-like comparison with XDS is the reference-range table below. Each set runs once, so every time includes reading its images from disk (unless they were still in the page cache from before the run). + +### By population + +| arm | population | sets | pass rate | median res. gain | +|:--|:--|--:|:--|--:| +| open | SMV | 2 | 2/2 (100%) | +11.2% | +| open | cbf | 97 | 90/95 (95%) | +12.2% | +| open | cdte | 3 | 3/3 (100%) | +5.1% | +| open | cubic | 13 | 13/13 (100%) | +14.7% | +| open | diffuse | 4 | 4/4 (100%) | +9.9% | +| open | h5 | 57 | 56/57 (98%) | +12.6% | +| open | hexagonal | 16 | 15/15 (100%) | +13.9% | +| open | home-source | 11 | 8/11 (73%) | +13.2% | +| open | long-axis | 2 | 2/2 (100%) | +16.2% | +| open | long-wavelength | 1 | 1/1 (100%) | - | +| open | low-resolution | 1 | 1/1 (100%) | -8.8% | +| open | marCCD | 15 | 14/15 (93%) | +12.4% | +| open | marccd | 8 | 8/8 (100%) | +11.5% | +| open | monoclinic | 49 | 43/49 (88%) | +12.1% | +| open | nxs | 2 | 1/1 (100%) | +8.5% | +| open | orthorhombic | 48 | 45/47 (96%) | +12.0% | +| open | pseudo-merohedral | 4 | 2/4 (50%) | +10.3% | +| open | pseudo-symmetry | 7 | 5/7 (71%) | +9.9% | +| open | small-molecule | 6 | 5/5 (100%) | -12.9% | +| open | smv | 21 | 17/21 (81%) | +10.9% | +| open | tetragonal | 24 | 23/24 (96%) | +10.3% | +| open | tncs | 5 | 3/5 (60%) | +12.6% | +| open | triclinic | 13 | 13/13 (100%) | +9.5% | +| open | trigonal | 19 | 18/19 (95%) | +13.7% | +| open | twin | 14 | 10/14 (71%) | +10.6% | +| open | two-wavelength | 2 | 2/2 (100%) | +10.8% | +| inhouse | control | 2 | 2/2 (100%) | - | +| inhouse | cubic | 1 | 1/1 (100%) | +2.2% | +| inhouse | cytochrome-c | 3 | 3/3 (100%) | +9.2% | +| inhouse | h5 | 47 | 46/47 (98%) | +6.3% | +| inhouse | hexagonal | 2 | 2/2 (100%) | - | +| inhouse | insulin | 9 | 9/9 (100%) | +5.2% | +| inhouse | iodine | 2 | 2/2 (100%) | +16.9% | +| inhouse | long-wavelength | 7 | 7/7 (100%) | +1.1% | +| inhouse | lysozyme | 12 | 12/12 (100%) | +6.7% | +| inhouse | monoclinic | 3 | 3/3 (100%) | +2.6% | +| inhouse | myoglobin | 6 | 6/6 (100%) | -2.8% | +| inhouse | orthorhombic | 1 | 1/1 (100%) | +2.6% | +| inhouse | pink-beam | 3 | 3/3 (100%) | +16.1% | +| inhouse | small-molecule | 8 | 7/8 (88%) | +2.6% | +| inhouse | tetragonal | 1 | 0/1 (0%) | +10.3% | +| inhouse | thaumatin | 7 | 7/7 (100%) | +2.7% | +| inhouse | twin | 4 | 4/4 (100%) | -22.9% | + +### Distributions (sets that merged, controls left out) + +| arm | metric | n | min | q1 | median | q3 | max | +|:--|:--|--:|--:|--:|--:|--:|--:| +| open | ISa | 202 | 4.1 | 9.5 | 13.6 | 19.8 | 45.5 | +| open | R_meas | 202 | 2.5% | 8.8% | 13.8% | 22.2% | 366.1% | +| open | CC1/2 | 202 | 0.917 | 0.994 | 0.998 | 0.999 | 1.000 | +| open | res. gain % | 198 | -68.3 | +8.1 | +12.1 | +16.8 | +34.3 | +| open | low-res shell R_meas | 202 | 1.7% | 4.1% | 5.5% | 8.2% | 33.8% | +| open | R_model, shell-scaled | 196 | 0.096 | 0.180 | 0.209 | 0.247 | 0.580 | +| open | R_free (placement, within-run only) | 196 | 0.121 | 0.188 | 0.218 | 0.255 | 0.575 | +| open | radial misfit | 196 | 0.03 | 0.09 | 0.12 | 0.16 | 1.10 | +| open | R_free ratio | 196 | 0.813 | 0.961 | 1.030 | 1.118 | 3.380 | +| open | anomalous map at the scatterers, sigma | 196 | -3.42 | 0.70 | 1.51 | 3.51 | 29.13 | +| open | REFMAC R_free ratio | 192 | 0.807 | 0.981 | 0.998 | 1.027 | 1.393 | +| open | dCC vs depositor, all | 193 | -0.317 | -0.018 | -0.004 | +0.003 | +0.194 | +| open | dCC vs depositor, outer 2 shells | 193 | -0.664 | -0.032 | -0.000 | +0.030 | +0.318 | +| open | CC(Fc^2) past the deposited limit | 178 | -0.013 | 0.229 | 0.297 | 0.367 | 0.926 | +| open | time s | 202 | 5 | 18 | 33 | 71 | 401 | +| inhouse | ISa | 44 | 1.9 | 7.7 | 21.8 | 32.9 | 53.4 | +| inhouse | R_meas | 45 | 2.6% | 6.3% | 9.8% | 25.2% | 125.6% | +| inhouse | CC1/2 | 45 | 0.702 | 0.998 | 0.999 | 1.000 | 1.000 | +| inhouse | res. gain % | 43 | -42.2 | +1.8 | +6.3 | +11.1 | +21.5 | +| inhouse | ref-range ISa / XDS | 43 | 0.209 | 0.838 | 1.104 | 1.264 | 6.410 | +| inhouse | ref-range R_meas / XDS | 43 | 0.207 | 0.838 | 0.942 | 1.054 | 2.123 | +| inhouse | low-res shell R_meas | 45 | 2.2% | 3.7% | 4.8% | 9.3% | 41.8% | +| inhouse | ref-range low-res R_meas / XDS | 43 | 0.273 | 0.871 | 1.021 | 1.149 | 2.681 | +| inhouse | ref-range CC1/2 noise / XDS | 43 | 0.03 | 0.33 | 0.54 | 0.80 | 8.58 | +| inhouse | R_model, shell-scaled | 37 | 0.183 | 0.227 | 0.253 | 0.287 | 0.520 | +| inhouse | R_free (placement, within-run only) | 37 | 0.175 | 0.234 | 0.287 | 0.325 | 0.572 | +| inhouse | radial misfit | 37 | 0.02 | 0.07 | 0.10 | 0.18 | 0.27 | +| inhouse | R_free ratio | 37 | 0.940 | 1.226 | 1.531 | 1.724 | 2.624 | +| inhouse | anomalous map at the scatterers, sigma | 37 | 0.35 | 1.45 | 3.11 | 6.19 | 11.50 | +| inhouse | time s | 45 | 5 | 9 | 13 | 18 | 93 | + +### Per set, against the reference + +_Plot in report.html: open: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 = finer than the reference_ + +_Plot in report.html: inhouse: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 = finer than the reference_ + +_Plot in report.html: inhouse: ISa of the reference-range table (the error model refitted on it) / XDS's ISa; above 1 = rugnux's error model is the better one. One point per set, sorted._ + +_Plot in report.html: inhouse: R_meas over the reference range / XDS's R_meas; below 1 = rugnux's merge is the more consistent one. One point per set, sorted._ + +_Plot in report.html: inhouse: R_meas of the lowest-resolution shell of the reference-range table / of XDS's table; below 1 = rugnux's strong reflections agree better. One point per set, sorted._ + +_Plot in report.html: inhouse: half-set noise over the reference range read off CC1/2 (1 / CC1/2 - 1) / XDS's; 2 = as noisy as XDS's merge would be with half its observations (not scored). One point per set, sorted._ + +_Plot in report.html: R_free of the deposited model against our merge, as rugnux reports it with --model (a rigid-body placement, scored on rugnux's own free set: a trend number, not a refinement) / the R_free the depositor published, one point per set, sorted; below 1 = lower than published_ + +_Plot in report.html: REFMAC check (--model-check): R_free of the deposited model against our merge / against the deposited structure factors, both on the depositor's free set with the same protocol, one point per set, sorted; below 1 = our data fit the model better_ + +### Against the depositor's data + +Per resolution shell, the rank correlation of our merged intensities with |Fc|^2 minus that of the depositor's own data, on the reflections both carry (d < 4 A, eight shells of equal count), |Fc| from the deposited model as it is (no bulk solvent, no refinement; depdata_check.py). The model was refined against the depositor's data, so zero is already a good result: across a corpus the median sits near -0.01. 'Past the limit' is CC(I, |Fc|^2) of our reflections in the outermost of up to three shells beyond the deposited data's limit, which the model never saw: clearly above zero there is signal. Reported, never scored. + +| set | dep. data | our d_min | dep. d_min | common refl. | dCC all | dCC outer 2 | CC past limit | to d A | REFMAC ratio | ISa | tags | +|:--|:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--| +| 3r6o | F | 1.53 | 1.95 | 16829 | -0.314 | -0.664 | 0.076 | 1.54 | 1.393 | 4.5 | smv, tetragonal, home-source | +| 2wnq | F | 1.65 | 1.80 | 101895 | -0.148 | -0.626 | 0.039 | 1.65 | 1.066 | 10.3 | smv, monoclinic, twin, pseudo-merohedral | +| 4bwl | F | 1.68 | 2.00 | 72820 | -0.317 | -0.520 | 0.117 | 1.67 | 1.030 | 14.3 | smv, monoclinic, twin, pseudo-merohedral | +| 2xfw | F | 1.55 | 1.65 | 147572 | -0.127 | -0.515 | 0.258 | 1.55 | 1.158 | 14.1 | smv, monoclinic, twin, pseudo-merohedral | +| 3p85 | F | 1.62 | 1.90 | 24299 | -0.162 | -0.404 | 0.261 | 1.62 | 0.879 | 8.1 | smv, hexagonal, home-source | +| 2wnz | F | 1.85 | 1.85 | 96083 | -0.106 | -0.396 | - | - | 1.113 | 14.8 | smv, monoclinic, pseudo-symmetry | +| 9qw2 | F | 1.76 | 1.92 | 43869 | -0.172 | -0.360 | 0.234 | 1.75 | 1.185 | 7.8 | cbf, monoclinic, pseudo-symmetry | +| 2wnn | F | 1.44 | 1.65 | 123469 | -0.055 | -0.351 | 0.204 | 1.44 | 0.807 | 14.1 | smv, monoclinic, twin, pseudo-merohedral | +| 3mc4 | F | 1.79 | 1.95 | 24530 | -0.161 | -0.333 | 0.280 | 1.79 | 1.365 | 12.5 | smv, trigonal, home-source | +| 9qw8 | I | 1.71 | 1.80 | 38358 | -0.192 | -0.326 | 0.192 | 1.70 | 1.272 | 9.2 | h5, triclinic | +| 9lxl | I | 2.06 | 2.19 | 18039 | -0.277 | -0.270 | 0.068 | 2.05 | 1.058 | 6.2 | h5, tetragonal | +| 7bgu | I | 2.30 | 2.43 | 14571 | -0.094 | -0.246 | 0.009 | 2.30 | 1.140 | 10.0 | marCCD, triclinic | +| 9hnc | F | 1.63 | 1.88 | 404705 | -0.062 | -0.237 | 0.376 | 1.63 | 0.923 | 13.6 | cbf, monoclinic | +| 6yqf | I | 3.02 | 3.33 | 933 | -0.134 | -0.233 | 0.015 | 3.01 | 1.001 | 4.2 | cbf, orthorhombic | +| 6r72 | F | 4.39 | 3.95 | 18989 | -0.092 | -0.232 | - | - | 1.038 | 17.9 | h5, monoclinic | +| 9rcs | I | 3.27 | 3.01 | 2052 | -0.208 | -0.171 | - | - | 0.928 | 7.3 | h5, monoclinic, cdte | +| 6zqy | F | 1.70 | 1.85 | 47653 | -0.029 | -0.144 | 0.242 | 1.70 | 1.071 | 12.7 | smv | +| 5ky6 | F | 1.54 | 1.94 | 98442 | -0.074 | -0.142 | 0.159 | 1.54 | 1.076 | 6.1 | marccd | +| 8qq7 | I | 3.16 | 3.62 | 5507 | -0.104 | -0.107 | -0.013 | 3.15 | 1.065 | 6.4 | cbf, hexagonal | +| 8k1g | F | 1.63 | 2.09 | 34025 | -0.077 | -0.107 | 0.281 | 1.62 | 0.974 | 11.7 | cbf, tetragonal | +| 3meb | F | 1.53 | 1.90 | 35703 | -0.026 | -0.107 | 0.330 | 1.54 | 0.993 | 15.6 | smv, monoclinic, home-source | +| 7arr | I | 0.92 | 1.10 | 47848 | -0.036 | -0.103 | 0.293 | 0.92 | 1.016 | 18.9 | cbf, triclinic | +| 6zqr | F | 1.76 | 1.93 | 37169 | -0.051 | -0.098 | 0.297 | 1.76 | 1.064 | 9.1 | smv | +| 6u7g | I | 1.88 | 2.35 | 87246 | -0.055 | -0.097 | 0.243 | 1.88 | 0.980 | 13.3 | h5, monoclinic | +| 5epe | F | 1.77 | 1.90 | 22792 | -0.015 | -0.093 | 0.322 | 1.77 | 1.039 | 9.9 | marCCD, cubic | +| 9pbb | I | 1.78 | 2.16 | 11031 | -0.048 | -0.087 | 0.160 | 1.78 | 0.980 | 17.0 | cbf, monoclinic | +| 9hs7 | F | 1.70 | 1.70 | 9595 | -0.131 | -0.083 | -0.008 | 1.69 | 0.958 | 13.3 | cbf, hexagonal | +| 8c3e | I | 1.79 | 2.10 | 4079 | -0.038 | -0.076 | 0.162 | 1.80 | 1.031 | 5.8 | cbf, trigonal, home-source, twin | +| 6jgh | I | 0.87 | 0.94 | 139810 | -0.014 | -0.070 | 0.333 | 0.87 | 1.069 | 6.6 | marccd | +| 6qaj | I | 2.70 | 2.63 | 4163 | -0.132 | -0.069 | - | - | 0.951 | 13.2 | cbf | +| 5f6m | I | 1.09 | 1.10 | 68757 | -0.006 | -0.059 | 0.364 | 1.09 | 1.032 | 23.0 | cbf, orthorhombic, diffuse | +| 7tcd | I | 1.65 | 1.70 | 27140 | -0.043 | -0.057 | 0.083 | 1.65 | 0.968 | 18.4 | h5, monoclinic | +| 7os3 | I | 1.96 | 2.18 | 32881 | -0.004 | -0.054 | 0.355 | 1.96 | 0.985 | 24.4 | cbf, orthorhombic | +| 5t39 | I | 1.01 | 1.10 | 92595 | -0.010 | -0.053 | 0.400 | 1.01 | 1.003 | 16.2 | marccd | +| 6cdl | I | 1.13 | 1.24 | 56384 | -0.008 | -0.051 | 0.353 | 1.13 | 1.039 | 11.6 | marccd | +| 8dyz | I | 1.14 | 1.27 | 24752 | -0.007 | -0.050 | 0.625 | 1.14 | 0.958 | 37.8 | cbf, tetragonal, diffuse | +| 6oww | F | 2.72 | 3.84 | 3373 | +0.031 | -0.049 | 0.053 | 2.71 | 0.938 | 11.8 | cbf, monoclinic, pseudo-symmetry | +| 9upt | F | 2.03 | 2.37 | 24477 | -0.036 | -0.046 | 0.324 | 2.03 | 1.056 | 6.5 | SMV, hexagonal | +| 6z8o | F | 2.21 | 2.20 | 22416 | -0.136 | -0.045 | - | - | 1.043 | 13.9 | h5, monoclinic | +| 9b22 | I | 1.14 | 1.30 | 96597 | -0.014 | -0.044 | 0.422 | 1.14 | 1.007 | 16.4 | h5, monoclinic | +| 7brr | I | 1.24 | 1.40 | 102801 | -0.023 | -0.041 | 0.274 | 1.25 | 0.981 | 16.6 | h5, monoclinic | +| 5vml | F | 1.92 | 1.70 | 14226 | -0.015 | -0.038 | - | - | 1.053 | 10.9 | smv, tetragonal, home-source | +| 9p7q | I | 1.76 | 2.21 | 11981 | -0.031 | -0.037 | 0.282 | 1.76 | 1.017 | 10.8 | cbf, monoclinic | +| 7raa | I | 2.58 | 2.69 | 7523 | -0.065 | -0.036 | 0.046 | 2.58 | 0.990 | 11.0 | cbf | +| 8a1a | F | 1.93 | 2.05 | 101602 | -0.009 | -0.036 | 0.355 | 1.94 | 0.991 | 18.8 | h5, hexagonal | +| 5reo | F | 1.65 | 1.88 | 18392 | -0.026 | -0.036 | 0.338 | 1.65 | 0.991 | 23.7 | cbf, monoclinic | +| 5m17 | I | 0.98 | 1.03 | 185105 | -0.006 | -0.034 | 0.311 | 0.98 | 1.024 | 16.2 | cbf, tetragonal | +| 8y74 | F | 1.68 | 1.90 | 58040 | -0.010 | -0.032 | 0.288 | 1.69 | 0.962 | 8.6 | h5, monoclinic | +| 6s1u | I | 1.75 | 1.90 | 17747 | -0.003 | -0.031 | 0.344 | 1.75 | 0.975 | 13.8 | marccd | +| 9qvv | I | 2.49 | 2.72 | 16358 | -0.014 | -0.030 | 0.141 | 2.50 | 0.947 | 37.3 | nxs, orthorhombic, pseudo-symmetry | +| 7atg | I | 0.60 | 0.60 | 54556 | -0.004 | -0.026 | - | - | - | 22.9 | cbf, orthorhombic | +| 5lzl | F | 2.86 | 3.47 | 21840 | -0.014 | -0.026 | 0.231 | 2.87 | 1.027 | 15.2 | cbf, trigonal | +| 9zlo | I | 1.56 | 2.00 | 22408 | -0.011 | -0.024 | 0.242 | 1.56 | 1.011 | 24.1 | h5, orthorhombic | +| 9fhc | F | 1.92 | 2.20 | 76515 | -0.002 | -0.024 | 0.353 | 1.92 | 1.042 | 12.3 | marCCD, cubic | +| 6iu8 | I | 2.32 | 2.70 | 15286 | +0.004 | -0.024 | 0.175 | 2.32 | 0.984 | 8.1 | cbf, trigonal | +| 8r5r | F | 2.80 | 3.08 | 12960 | -0.032 | -0.024 | 0.211 | 2.80 | 1.003 | 19.3 | h5, orthorhombic | +| 6oel | I | 2.85 | 3.10 | 13233 | -0.013 | -0.024 | 0.272 | 2.85 | 0.965 | 9.9 | SMV, cubic | +| 9yl4 | I | 3.61 | 3.70 | 1130 | -0.010 | -0.023 | 0.175 | 3.61 | 0.987 | 9.5 | cbf, orthorhombic | +| 9gdj | I | 1.40 | 1.47 | 157444 | -0.007 | -0.023 | 0.299 | 1.40 | 1.021 | 12.9 | h5, tetragonal, cdte | +| 8agq | I | 0.97 | 1.09 | 98576 | -0.013 | -0.022 | 0.298 | 0.97 | 1.153 | 14.1 | cbf, monoclinic | +| 9ih9 | F | 1.40 | 1.70 | 78425 | -0.005 | -0.021 | 0.381 | 1.40 | 1.007 | 12.7 | h5, monoclinic | +| 9q66 | F | 2.03 | 2.01 | 87912 | -0.017 | -0.021 | - | - | 1.017 | 13.1 | h5, monoclinic | +| 7mzt | I | 3.12 | 4.05 | 8489 | -0.074 | -0.020 | 0.083 | 3.11 | 0.991 | 4.1 | cbf, orthorhombic | +| 9zm0 | I | 1.80 | 2.10 | 13858 | -0.014 | -0.020 | 0.115 | 1.80 | 0.984 | 9.0 | h5, monoclinic | +| 7kcn | I | 1.39 | 1.46 | 43289 | -0.011 | -0.020 | 0.569 | 1.39 | 1.055 | 11.7 | cbf, tetragonal | +| 6ze4 | I | 1.30 | 1.60 | 136778 | -0.054 | -0.020 | 0.323 | 1.29 | 1.040 | 9.3 | cbf, orthorhombic | +| 9e2t | I | 2.29 | 2.28 | 32088 | -0.110 | -0.019 | - | - | 0.980 | 6.8 | cbf, triclinic | +| 9gjx | I | 2.06 | 2.40 | 52052 | -0.001 | -0.019 | 0.268 | 2.06 | 1.033 | 40.1 | h5, monoclinic | +| 7q6j | I | 1.98 | 2.20 | 41720 | +0.005 | -0.019 | 0.215 | 1.98 | 0.996 | 11.0 | cbf, orthorhombic, pseudo-symmetry | +| 6fvz | F | 1.49 | 1.80 | 105993 | -0.019 | -0.018 | 0.386 | 1.50 | 0.998 | 20.5 | cbf, orthorhombic | +| 6ttn | I | 1.08 | 1.12 | 126153 | -0.001 | -0.018 | 0.223 | 1.08 | 1.020 | 14.5 | cbf, orthorhombic | +| 6iu5 | I | 2.12 | 2.25 | 30820 | -0.010 | -0.018 | 0.231 | 2.12 | 1.008 | 8.0 | cbf, trigonal, twin | +| 7n2s | I | 2.56 | 2.37 | 10827 | -0.009 | -0.018 | - | - | 0.968 | 9.6 | cbf, monoclinic | +| 6fid | F | 1.98 | 2.20 | 10527 | -0.005 | -0.018 | 0.819 | 1.97 | 0.977 | 13.6 | cbf, orthorhombic | +| 6p8p | I | 1.46 | 1.64 | 64791 | -0.015 | -0.017 | 0.289 | 1.45 | 1.036 | 15.5 | cbf, tetragonal | +| 6iu6 | I | 2.36 | 2.90 | 10758 | +0.003 | -0.016 | 0.252 | 2.36 | 1.063 | 7.2 | cbf, trigonal, twin | +| 6w75 | F | 1.69 | 1.95 | 99979 | -0.010 | -0.016 | 0.320 | 1.69 | 0.991 | 15.8 | marccd | +| 9h0q | I | 2.10 | 2.55 | 45699 | -0.005 | -0.016 | 0.343 | 2.11 | 0.980 | 18.6 | h5 | +| 6h5t | F | 1.48 | 1.69 | 28184 | -0.007 | -0.015 | 0.356 | 1.48 | 1.057 | 9.3 | marCCD, tetragonal | +| 9crw | I | 2.28 | 2.49 | 53873 | -0.021 | -0.014 | 0.130 | 2.28 | 1.010 | 15.8 | h5, monoclinic | +| 6rlr | I | 1.92 | 2.00 | 19974 | +0.002 | -0.014 | 0.189 | 1.92 | 1.022 | 16.4 | h5, triclinic, twin | +| 5uth | F | 1.72 | 1.95 | 27663 | -0.015 | -0.012 | 0.436 | 1.73 | 1.009 | 9.5 | smv, trigonal, home-source | +| 6w4h | F | 1.62 | 1.80 | 70331 | -0.003 | -0.012 | 0.419 | 1.62 | 0.996 | 18.5 | marCCD, trigonal | +| 9jq9 | I | 1.65 | 1.90 | 13865 | +0.001 | -0.011 | 0.509 | 1.65 | 1.005 | 19.5 | cbf, orthorhombic, home-source | +| 6rym | I | 1.45 | 1.46 | 17018 | -0.005 | -0.009 | - | - | 1.022 | 22.9 | smv, tetragonal | +| 6hv2 | F | 1.41 | 1.71 | 19192 | -0.035 | -0.008 | 0.102 | 1.41 | 0.829 | 13.7 | h5, hexagonal | +| 8dz7 | I | 1.19 | 1.34 | 13603 | -0.002 | -0.008 | 0.809 | 1.19 | 1.009 | 35.1 | cbf, orthorhombic, diffuse | +| 9chw | I | 1.60 | 2.16 | 20358 | -0.005 | -0.008 | 0.648 | 1.60 | 1.006 | 21.3 | marCCD, hexagonal | +| 7l84 | I | 1.70 | 1.71 | 11375 | -0.001 | -0.008 | - | - | 0.976 | 11.4 | cbf, tetragonal | +| 5mln | F | 1.26 | 1.59 | 60800 | -0.012 | -0.007 | 0.409 | 1.26 | 1.017 | 21.5 | cbf | +| 5ojv | I | 1.82 | 2.06 | 63705 | -0.030 | -0.005 | 0.267 | 1.82 | 0.998 | 13.2 | cbf, orthorhombic, pseudo-symmetry | +| 6pxb | I | 1.39 | 1.75 | 50825 | +0.008 | -0.005 | 0.097 | 1.40 | 0.991 | 12.1 | cbf, trigonal | +| 7ris | I | 1.51 | 1.72 | 22161 | -0.001 | -0.004 | 0.221 | 1.51 | 0.987 | 31.0 | h5, trigonal | +| 6o2h | I | 1.10 | 1.21 | 11612 | -0.002 | -0.003 | 0.926 | 1.10 | 1.157 | 23.9 | cbf, triclinic, diffuse | +| 8xtf | I | 1.84 | 2.13 | 27230 | +0.000 | -0.003 | 0.420 | 1.83 | 0.956 | 7.6 | h5, trigonal | +| 7l6j | F | 1.51 | 1.78 | 37485 | -0.002 | -0.001 | 0.414 | 1.51 | 1.013 | 10.5 | marCCD, cubic | +| 7bgt | I | 1.78 | 1.93 | 33462 | -0.003 | -0.000 | 0.308 | 1.79 | 0.999 | 17.4 | marCCD, triclinic | +| 9sl0 | I | 1.36 | 1.60 | 67358 | +0.002 | +0.001 | 0.243 | 1.37 | 0.996 | 20.2 | h5, orthorhombic | +| 6cee | I | 1.38 | 1.55 | 14202 | -0.001 | +0.001 | 0.678 | 1.38 | 0.993 | 23.9 | smv, orthorhombic, home-source | +| 3ky7 | F | 1.92 | 2.35 | 11401 | +0.002 | +0.001 | 0.182 | 1.92 | 0.992 | 12.2 | marCCD, cubic | +| 6i3j | I | 2.32 | 2.59 | 34974 | -0.006 | +0.002 | 0.366 | 2.32 | 1.154 | 6.8 | marCCD, orthorhombic | +| 8tyy | I | 1.28 | 1.68 | 44790 | -0.001 | +0.002 | 0.476 | 1.28 | 0.989 | 16.3 | cbf, cubic | +| 9i80 | I | 1.59 | 1.95 | 68449 | -0.011 | +0.002 | 0.351 | 1.59 | 1.024 | 6.8 | h5, tetragonal, twin | +| 3inp | I | 1.70 | 2.05 | 26251 | -0.005 | +0.003 | 0.276 | 1.70 | 0.984 | 13.2 | marCCD, cubic | +| 8sqt | I | 1.88 | 2.20 | 9296 | -0.011 | +0.004 | 0.320 | 1.88 | 1.044 | 29.9 | h5, cubic | +| 6wzo | I | 1.04 | 1.42 | 91585 | -0.010 | +0.004 | 0.355 | 1.04 | 0.965 | 16.8 | cbf, triclinic | +| 6vww | F | 1.99 | 2.20 | 58979 | -0.003 | +0.004 | 0.345 | 1.98 | 1.025 | 8.0 | cbf, hexagonal, twin | +| 6zr0 | F | 1.66 | 1.94 | 40650 | -0.006 | +0.005 | 0.345 | 1.65 | 1.014 | 24.5 | cbf | +| 9mh4 | I | 2.78 | 3.05 | 9490 | +0.006 | +0.005 | 0.206 | 2.79 | 1.017 | 13.8 | h5, cubic | +| 8dqb | I | 2.05 | 2.50 | 19165 | -0.002 | +0.006 | 0.382 | 2.05 | 1.005 | 22.1 | h5, cubic | +| 6nen | I | 1.77 | 2.15 | 10259 | +0.003 | +0.007 | 0.363 | 1.77 | 1.008 | 6.3 | smv | +| 6gvk | I | 1.42 | 1.55 | 32916 | +0.002 | +0.007 | 0.227 | 1.42 | 0.989 | 19.8 | cbf, monoclinic | +| 8v4j | I | 1.10 | 1.30 | 41379 | +0.001 | +0.008 | 0.580 | 1.10 | 0.983 | 22.1 | smv, tetragonal | +| 9zmu | I | 1.72 | 1.98 | 21684 | -0.002 | +0.009 | 0.189 | 1.71 | 0.997 | 11.6 | h5, hexagonal | +| 6v2r | I | 1.38 | 1.57 | 9330 | -0.000 | +0.009 | 0.458 | 1.38 | 0.996 | 24.4 | smv, tetragonal, home-source | +| 6cs9 | I | 1.72 | 1.85 | 5090 | -0.000 | +0.010 | 0.328 | 1.72 | 1.018 | 13.1 | smv, monoclinic | +| 9vyb | I | 1.67 | 2.12 | 5191 | -0.007 | +0.012 | 0.352 | 1.68 | 0.995 | 22.1 | h5, orthorhombic | +| 7yzx | F | 1.88 | 1.90 | 83105 | -0.031 | +0.013 | 0.319 | 1.88 | 1.025 | 10.4 | cbf, hexagonal | +| 8v2t | I | 1.17 | 1.40 | 33174 | +0.001 | +0.014 | 0.369 | 1.16 | 0.975 | 11.9 | cbf, tetragonal | +| 9gqg | I | 1.81 | 2.00 | 15522 | -0.004 | +0.014 | 0.241 | 1.81 | 1.036 | 12.8 | h5, trigonal | +| 9ig7 | F | 2.02 | 2.60 | 26626 | -0.007 | +0.015 | 0.258 | 2.02 | 1.030 | 11.1 | cbf, orthorhombic | +| 8egn | F | 1.63 | 1.95 | 38629 | -0.011 | +0.015 | 0.276 | 1.62 | 0.997 | 22.9 | cbf, orthorhombic | +| 6f3p | F | 1.13 | 1.35 | 235840 | -0.001 | +0.015 | 0.386 | 1.13 | 0.999 | 9.4 | marccd | +| 8iya | F | 1.91 | 2.43 | 16262 | -0.002 | +0.016 | 0.239 | 1.92 | 0.998 | 5.8 | h5, monoclinic | +| 6g1f | I | 1.93 | 2.25 | 131738 | +0.001 | +0.017 | 0.237 | 1.93 | 1.018 | 21.0 | cbf | +| 6fwc | F | 1.41 | 1.70 | 126460 | -0.001 | +0.018 | 0.394 | 1.41 | 0.980 | 27.3 | cbf, orthorhombic | +| 9jzo | F | 1.15 | 1.40 | 47773 | +0.002 | +0.018 | 0.769 | 1.15 | 1.045 | 8.1 | cbf, triclinic | +| 8sqq | I | 1.90 | 2.25 | 8697 | +0.006 | +0.018 | 0.331 | 1.91 | 0.937 | 17.6 | h5, cubic | +| 5jk4 | I | 1.02 | 1.01 | 133760 | -0.001 | +0.018 | - | - | 0.992 | 14.8 | smv, monoclinic | +| 9bn8 | I | 1.21 | 1.35 | 119114 | -0.001 | +0.019 | 0.488 | 1.21 | 0.975 | 19.8 | h5, tetragonal | +| 7dkp | F | 1.18 | 1.45 | 131351 | +0.004 | +0.020 | 0.679 | 1.18 | 0.984 | 26.1 | h5, monoclinic | +| 8qaw | F | 1.30 | 1.55 | 256134 | +0.001 | +0.022 | 0.356 | 1.30 | 0.957 | 11.0 | cbf, trigonal, long-axis | +| 9fcf | F | 1.75 | 2.36 | 9898 | -0.018 | +0.022 | 0.120 | 1.75 | 0.976 | 7.0 | cbf | +| 11if | I | 1.36 | 1.51 | 27273 | -0.005 | +0.022 | 0.286 | 1.36 | 0.982 | 25.7 | h5, tetragonal | +| 6ukf | I | 0.96 | 1.00 | 143260 | +0.004 | +0.023 | 0.327 | 0.96 | 0.957 | 9.8 | cbf, monoclinic | +| 5ebi | F | 0.85 | 1.09 | 39168 | +0.010 | +0.024 | 0.192 | 0.85 | 0.996 | 10.9 | marCCD, monoclinic, twin | +| 36gk | F | 2.09 | 2.28 | 84276 | -0.011 | +0.024 | 0.313 | 2.09 | 1.014 | 11.8 | h5, orthorhombic | +| 8u0i | I | 1.38 | 1.54 | 16737 | +0.005 | +0.025 | 0.346 | 1.38 | 1.008 | 17.0 | cbf, tetragonal | +| 7t5t | I | 1.24 | 1.35 | 101432 | -0.002 | +0.025 | 0.251 | 1.24 | 1.017 | 16.5 | cbf, tetragonal | +| 9c18 | I | 1.71 | 1.90 | 25042 | +0.010 | +0.025 | 0.261 | 1.72 | 0.957 | 10.8 | h5, triclinic | +| 6h2p_1p89A | F | 1.76 | 1.88 | 85465 | +0.004 | +0.026 | 0.525 | 1.76 | 1.043 | 22.8 | cbf, orthorhombic, long-wavelength, two-wavelength | +| 6jgj | F | 0.65 | 0.77 | 250738 | -0.049 | +0.026 | 0.387 | 0.65 | 1.127 | 15.1 | cbf, orthorhombic, cdte | +| 6moj | I | 2.43 | 2.43 | 13971 | -0.050 | +0.029 | - | - | 0.977 | 8.3 | cbf, tetragonal | +| 9o0h | I | 2.02 | 2.24 | 15448 | +0.030 | +0.029 | 0.266 | 2.03 | 0.987 | 5.8 | cbf, orthorhombic | +| 8qj5 | I | 1.29 | 1.63 | 101763 | +0.003 | +0.029 | 0.214 | 1.29 | 1.058 | 8.4 | cbf, monoclinic | +| 8s38 | F | 1.63 | 1.89 | 121216 | -0.003 | +0.030 | 0.356 | 1.63 | 0.999 | 22.3 | cbf, orthorhombic | +| 8sqo | I | 1.32 | 1.55 | 33951 | +0.002 | +0.033 | 0.369 | 1.32 | 0.987 | 14.2 | h5, cubic | +| 9z72 | I | 2.00 | 2.38 | 28228 | +0.009 | +0.033 | 0.272 | 2.00 | 0.994 | 11.8 | cbf, trigonal, long-axis | +| 5src | I | 0.97 | 1.05 | 138681 | +0.015 | +0.035 | 0.247 | 0.97 | 0.988 | 19.6 | cbf, tetragonal | +| 6pxc | F | 1.41 | 1.60 | 15636 | +0.004 | +0.035 | 0.300 | 1.41 | 1.019 | 10.2 | cbf, orthorhombic | +| 5cc8 | F | 1.53 | 2.00 | 28144 | +0.038 | +0.036 | 0.447 | 1.53 | 0.950 | 11.2 | smv, orthorhombic, home-source, tncs | +| 6iu9 | I | 2.74 | 3.00 | 9191 | +0.011 | +0.036 | 0.152 | 2.73 | 0.983 | 5.4 | cbf, trigonal, twin | +| 8pqd | F | 1.30 | 1.50 | 102729 | +0.004 | +0.036 | 0.241 | 1.30 | 0.995 | 15.0 | h5, orthorhombic | +| 9s02 | F | 1.42 | 1.65 | 187371 | -0.006 | +0.036 | 0.235 | 1.42 | 0.969 | 26.0 | h5, orthorhombic | +| 8xtg | I | 1.54 | 2.00 | 177630 | -0.004 | +0.037 | 0.285 | 1.54 | 1.001 | 7.3 | cbf, trigonal | +| 5nw5 | I | 7.07 | 6.50 | 9834 | -0.050 | +0.038 | - | - | 1.053 | 7.7 | cbf, orthorhombic, low-resolution | +| 9yzk | I | 3.87 | 4.50 | 13466 | +0.003 | +0.038 | 0.017 | 3.88 | 0.995 | 9.3 | cbf, monoclinic | +| 9ea5 | I | 1.64 | 2.00 | 51543 | +0.008 | +0.040 | 0.315 | 1.64 | 0.986 | 26.7 | cbf, monoclinic | +| 6h2p_native | F | 1.32 | 1.48 | 188243 | -0.007 | +0.041 | 0.295 | 1.32 | 0.959 | 20.0 | cbf, orthorhombic, two-wavelength | +| 7pq7 | F | 1.37 | 1.55 | 51889 | -0.001 | +0.041 | 0.152 | 1.38 | 1.007 | 14.0 | cbf, monoclinic | +| 9khr | I | 1.38 | 2.00 | 11649 | +0.008 | +0.042 | 0.277 | 1.38 | 0.984 | 9.5 | marCCD, orthorhombic | +| 8xte | I | 1.64 | 1.99 | 195244 | +0.005 | +0.043 | 0.307 | 1.63 | 0.999 | 12.4 | cbf, trigonal | +| 8tha | I | 1.33 | 1.68 | 8018 | +0.001 | +0.044 | 0.316 | 1.33 | 0.980 | 27.7 | cbf, hexagonal | +| 6pb3 | I | 1.84 | 2.05 | 15267 | -0.002 | +0.045 | 0.230 | 1.84 | 1.002 | 25.4 | cbf, hexagonal | +| 8owm | F | 1.48 | 1.70 | 293908 | +0.004 | +0.046 | 0.388 | 1.48 | 0.981 | 22.2 | cbf, triclinic | +| 7ph1 | F | 1.08 | 1.18 | 120936 | -0.007 | +0.050 | 0.379 | 1.08 | 0.991 | 16.0 | cbf, orthorhombic | +| 8sa8 | I | 1.10 | 1.30 | 409824 | +0.008 | +0.051 | 0.484 | 1.10 | 0.977 | 22.0 | h5, monoclinic | +| 9w3y | I | 1.19 | 1.50 | 61343 | +0.008 | +0.051 | 0.454 | 1.19 | 0.995 | 20.2 | h5, orthorhombic | +| 7n0i | I | 1.68 | 1.80 | 117513 | +0.018 | +0.051 | 0.154 | 1.68 | 1.041 | 10.9 | cbf, orthorhombic, tncs | +| 8rud | I | 1.57 | 2.10 | 77721 | +0.003 | +0.052 | 0.297 | 1.58 | 0.927 | 11.9 | cbf, monoclinic | +| 6hwj | I | 1.71 | 1.98 | 52132 | +0.007 | +0.052 | 0.370 | 1.71 | 0.967 | 39.0 | cbf, monoclinic | +| 8t7r | I | 3.23 | 3.84 | 13698 | +0.042 | +0.054 | 0.284 | 3.23 | 0.998 | 7.8 | cbf, monoclinic | +| 7orr | I | 1.62 | 1.79 | 16877 | +0.002 | +0.056 | 0.357 | 1.62 | 1.020 | 26.5 | h5, cubic | +| 8oic | I | 2.34 | 2.51 | 78747 | -0.018 | +0.056 | 0.249 | 2.34 | 0.956 | 20.0 | h5, triclinic | +| 8v4o | I | 2.10 | 2.70 | 59576 | +0.012 | +0.058 | 0.262 | 2.10 | 0.997 | 15.5 | h5, hexagonal | +| 9q41 | I | 1.68 | 1.95 | 42195 | +0.001 | +0.059 | 0.396 | 1.67 | 0.972 | 7.9 | h5, orthorhombic | +| 9z44 | I | 6.73 | 7.20 | 1893 | +0.013 | +0.059 | 0.089 | 6.81 | 1.031 | 8.5 | cbf, monoclinic | +| 7qij | I | 3.59 | 4.10 | 135378 | +0.027 | +0.061 | 0.082 | 3.60 | 0.996 | 9.4 | cbf, orthorhombic | +| 9fcg | F | 1.38 | 1.54 | 37974 | +0.002 | +0.062 | 0.454 | 1.38 | 1.004 | 9.7 | cbf, tetragonal | +| 7qis | F | 1.75 | 1.83 | 93162 | +0.016 | +0.064 | 0.276 | 1.75 | 1.024 | 17.4 | cbf, hexagonal | +| 6jgi | I | 0.75 | 0.85 | 187106 | +0.003 | +0.069 | 0.408 | 0.75 | 0.980 | 8.5 | marCCD, orthorhombic | +| 9rp9 | I | 1.90 | 2.10 | 19429 | +0.024 | +0.073 | 0.358 | 1.91 | 1.033 | 32.7 | h5, monoclinic | +| 7ou1 | F | 1.40 | 1.65 | 176468 | +0.008 | +0.085 | 0.392 | 1.40 | 0.966 | 9.0 | marccd | +| 7rji | I | 1.48 | 1.71 | 16700 | +0.004 | +0.087 | 0.327 | 1.48 | 0.977 | 8.4 | cbf, trigonal | +| 5jvn | I | 2.24 | 2.90 | 21045 | +0.030 | +0.092 | 0.235 | 2.24 | 1.031 | 15.6 | cbf, hexagonal | +| 7k1l | F | 1.91 | 2.25 | 54383 | +0.012 | +0.093 | 0.336 | 1.91 | 1.016 | 9.9 | cbf, hexagonal | +| 8ys9 | I | 1.31 | 1.46 | 75646 | +0.007 | +0.096 | 0.379 | 1.31 | 0.982 | 15.4 | h5, orthorhombic | +| 9t6s | I | 1.75 | 2.00 | 25144 | +0.017 | +0.102 | 0.237 | 1.75 | 0.948 | 29.9 | h5, orthorhombic, tncs | +| 6p8j | I | 1.28 | 1.47 | 191664 | +0.004 | +0.120 | 0.387 | 1.28 | 1.062 | 5.3 | cbf, monoclinic, pseudo-symmetry | +| 6toc | I | 1.64 | 1.85 | 6080 | +0.015 | +0.132 | 0.362 | 1.64 | 0.917 | 24.8 | cbf, tetragonal, twin | +| 9i0a | F | 1.81 | 2.22 | 60317 | +0.018 | +0.164 | 0.200 | 1.80 | 0.984 | 14.1 | h5, orthorhombic | +| 8xbp | F | 1.66 | 2.00 | 24156 | +0.013 | +0.245 | 0.121 | 1.65 | 1.063 | 18.6 | h5, monoclinic | +| 5j23 | F | 2.17 | 2.30 | 56769 | +0.194 | +0.318 | 0.387 | 2.17 | 1.012 | 11.6 | marCCD, trigonal, twin | + +No comparison: 9min (the merge does not match the model in any setting); 9rci (the merge does not match the model in any setting); 6z9g (the merge does not match the model in any setting) + +### Like for like with XDS: the reference-range table + +The same merge, binned a second time over XDS's range (--report-resolution; report-only, nothing was processed differently for it), against XDS's CORRECT.LP totals. Where rugnux's own cut is coarser than the reference ('coverage' in the last column), the shells past it are not merged at all: the completeness there is coverage of XDS's range, not a quality loss, and the other rugnux numbers are over the shells it reached. XDS's totals are over its own merged range, which is the reference range except where the reference d_min was derived (see below). + +| set | arm | range A | own d_min | compl % (rugnux / XDS) | mult | I/sigma | R_meas | low-res R_meas (to d A) | CC1/2 | CC1/2 noise ratio | ISa | reading | +|:--|:--|:--|--:|:--|:--|--:|:--|:--|:--|--:|:--|:--| +| aspirin_x10sa_20keV | inhouse | 50.000 0.680 | 0.66 | 85.2 / 84.0 | 6.0 / 3.1 | 43.5 | 3.0% / 3.3% | 2.2% / 3.3% (2.04 / 2.01) | 0.9997 / 0.999 | 0.20 | 36.4 / 27.4 | like for like | +| aspirin_x10sa_25keV | inhouse | 50.000 0.550 | 0.53 | 87.6 / 86.6 | 6.0 / 3.1 | 35.5 | 3.1% / 3.4% | 2.2% / 3.2% (1.65 / 1.63) | 0.9996 / 0.999 | 0.27 | 36.4 / 29.2 | like for like | +| citricacid_x10sa_20keV | inhouse | 50.000 0.680 | 0.67 | 83.2 / 82.0 | 5.8 / 3.1 | 45.4 | 3.6% / 4.1% | 3.8% / 5.0% (2.04 / 2.02) | 0.9986 / 0.997 | 0.40 | 24.0 / 20.4 | like for like | +| cytc_x06da_1 | inhouse | 50.000 1.875 | 1.70 | 100.0 / 99.9 | 10.5 / 7.7 | 14.8 | 8.4% / 8.4% | 3.7% / 3.3% (5.59 / 5.58) | 0.9996 / 0.999 | 0.27 | 17.8 / 24.5 | like for like | +| cytc_x06da_2 | inhouse | 50.000 1.690 | 1.57 | 100.0 / 99.9 | 10.2 / 7.8 | 12.9 | 9.1% / 9.7% | 3.5% / 3.2% (5.05 / 5.03) | 0.9996 / 0.999 | 0.27 | 21.5 / 27.0 | like for like | +| cytc_x10sa | inhouse | 50.000 2.039 | 1.95 | 99.9 / 99.8 | 10.6 / 10.7 | 9.5 | 18.0% / 23.2% | 3.9% / 3.6% (6.08 / 6.05) | 0.9990 / 0.999 | 0.67 | 26.6 / 31.8 | like for like | +| hepes_x10sa_20keV | inhouse | 50.000 0.680 | 0.66 | 93.0 / 91.8 | 10.4 / 5.7 | 82.7 | 2.6% / 3.1% | 3.5% / 3.7% (2.04 / 2.00) | 0.9994 / 0.999 | 0.40 | 39.7 / 28.5 | like for like | +| insu_H_x06da_notwin | inhouse | 50.000 1.544 | 1.42 | 98.7 / 97.4 | 4.8 / 3.5 | 15.5 | 5.9% / 6.8% | 4.3% / 4.5% (4.61 / 4.61) | 0.9988 / 0.998 | 0.48 | 20.6 / 17.7 | like for like | +| insu_H_x06da_twin | inhouse | 50.000 1.455 | 1.38 | 96.8 / 95.0 | 4.5 / 3.6 | 8.0 | 12.5% / 11.8% | 10.3% / 10.6% (4.35 / 4.34) | 0.9869 / 0.988 | 1.05 | 6.2 / 6.8 | like for like | +| insu_I_x06da_13keV | inhouse | 50.000 1.635 | 1.47 | 100.0 / 100.0 | 20.2 / 17.2 | 9.4 | 26.8% / 27.9% | 8.1% / 9.3% (4.88 / 4.85) | 0.9982 / 0.998 | 0.72 | 26.2 / 18.8 | like for like | +| insu_I_x06da_5keV | inhouse | 50.000 2.450 | 2.43 | 96.0 / 91.7 | 15.4 / 11.2 | 33.9 | 6.2% / 7.3% | 4.4% / 4.8% (7.28 / 7.21) | 0.9995 / 0.999 | 0.33 | 28.1 / 17.5 | like for like | +| insu_I_x06da_5keV_2 | inhouse | 50.000 2.450 | 2.42 | 96.3 / 91.8 | 15.9 / 12.7 | 27.1 | 7.9% / 7.6% | 5.7% / 4.8% (7.28 / 7.21) | 0.9991 / 0.999 | 0.60 | 25.3 / 20.0 | like for like | +| insu_I_x06da_6keV | inhouse | 50.000 2.040 | 2.03 | 95.0 / 92.4 | 15.5 / 12.4 | 34.7 | 5.8% / 6.5% | 4.8% / 4.7% (6.08 / 6.03) | 0.9995 / 0.999 | 0.33 | 20.6 / 17.9 | like for like | +| insu_I_x06da_low_isa | inhouse | 50.000 1.300 | 1.44 | 74.3 / 90.8 | 16.6 / 13.3 | 7.7 | 22.6% / 22.0% | 16.1% / 16.7% (3.89 / 3.92) | 0.9949 / 0.998 | 2.04 | 5.6 / 4.2 | coverage: own cut 1.44 A is coarser than the reference, 1 shell(s) not merged | +| insu_I_x06da_ref | inhouse | 50.000 1.621 | 1.40 | 100.0 / 100.0 | 16.1 / 13.8 | 17.3 | 25.8% / 34.9% | 6.2% / 14.3% (4.84 / 4.82) | 0.9994 / 0.998 | 0.24 | 33.4 / 25.1 | like for like | +| insu_I_x06da_weak | inhouse | 999.000 1.080 | 1.64 | 28.9 / 92.4 | 40.1 / 29.0 | 15.8 | 16.7% / 40.1% | 6.2% / 5.4% (3.24 / 3.22) | 0.9997 / 0.999 | 0.20 | 15.0 / 18.9 | coverage: own cut 1.64 A is coarser than the reference, 5 shell(s) not merged | +| kdp_x10sa_20keV | inhouse | 50.000 0.740 | 0.66 | 99.7 / 99.2 | 10.4 / 6.1 | 48.0 | 5.0% / 24.1% | 4.7% / 17.2% (2.22 / 2.06) | 0.9991 / 0.969 | 0.03 | 26.3 / 4.1 | like for like | +| lysoI_micromax_mono | inhouse | 50.000 1.650 | 1.36 | 100.0 / 100.0 | 6.5 / 6.8 | 22.2 | 5.8% / 6.8% | 3.1% / 2.8% (4.93 / 4.99) | 0.9994 / 0.999 | 0.40 | 39.5 / 31.4 | like for like | +| lysoI_micromax_pink | inhouse | 50.000 1.650 | 1.38 | 100.0 / 100.0 | 6.5 / 6.8 | 20.1 | 6.6% / 7.9% | 3.2% / 2.8% (4.93 / 4.99) | 0.9993 / 0.999 | 0.47 | 36.4 / 29.2 | like for like | +| lyso_micromax_mono | inhouse | 50.000 1.500 | 1.18 | 100.0 / 100.0 | 6.0 / 6.2 | 24.4 | 4.7% / 4.7% | 2.5% / 2.3% (4.48 / 4.54) | 0.9996 / 1.000 | 0.80 | 40.9 / 39.9 | like for like | +| lyso_micromax_pink | inhouse | 50.000 1.450 | 1.20 | 99.9 / 99.9 | 5.8 / 6.0 | 21.4 | 4.8% / 5.1% | 2.4% / 2.2% (4.34 / 4.39) | 0.9996 / 1.000 | 0.80 | 41.2 / 37.4 | like for like | +| lyso_x06da_5keV | inhouse | 50.000 2.450 | 2.43 | 87.7 / 86.4 | 11.1 / 8.8 | 34.0 | 5.7% / 6.4% | 5.4% / 4.7% (7.28 / 7.22) | 0.9991 / 0.998 | 0.36 | 28.2 / 19.4 | like for like | +| lyso_x06da_atten_wedge | inhouse | 50.000 1.264 | 1.19 | 100.0 / 99.9 | 12.8 / 11.0 | 8.8 | 35.8% / 23.9% | 6.9% / 6.9% (3.78 / 3.76) | 0.9981 / 0.998 | 0.76 | 13.3 / 16.6 | like for like | +| lyso_x06da_half_image | inhouse | 50.000 1.650 | 1.57 | 100.0 / 99.6 | 5.5 / 5.3 | 3.8 | 110.9% / 59.5% | 33.0% / 25.8% (4.93 / 4.91) | 0.7920 / 0.958 | 5.92 | 7.0 / 6.6 | like for like | +| lyso_x06da_ice | inhouse | 50.000 1.431 | 1.34 | 100.0 / 100.0 | 10.5 / 11.9 | 9.4 | 16.6% / 21.9% | 4.6% / 5.5% (4.28 / 4.26) | 0.9986 / 0.998 | 0.56 | 23.3 / 23.3 | like for like | +| lyso_x06da_ref | inhouse | 50.000 1.200 | 0.99 | 100.0 / 100.0 | 13.8 / 13.8 | 36.0 | 4.3% / 4.5% | 2.7% / 2.9% (3.59 / 3.58) | 0.9998 / 1.000 | 0.40 | 30.3 / 28.3 | like for like | +| lyso_x10sa_90deg_1 | inhouse | 50.000 1.966 | 1.83 | 100.0 / 99.3 | 3.5 / 3.5 | 6.9 | 15.2% / 18.2% | 4.5% / 4.9% (5.86 / 5.83) | 0.9956 / 0.995 | 0.80 | 20.7 / 20.8 | like for like | +| lyso_x10sa_90deg_2 | inhouse | 50.000 1.973 | 1.85 | 100.0 / 99.3 | 3.5 / 3.5 | 6.8 | 15.1% / 18.0% | 4.6% / 5.1% (5.88 / 5.86) | 0.9956 / 0.994 | 0.68 | 19.5 / 18.1 | like for like | +| lyso_x10sa_strong | inhouse | 999.000 1.180 | 1.36 | 66.0 / 65.8 | 21.5 / 11.5 | 9.1 | 23.8% / 11.2% | 11.4% / 7.7% (3.54 / 3.52) | 0.9966 / 0.998 | 1.36 | 5.5 / 8.6 | coverage: own cut 1.36 A is coarser than the reference, 2 shell(s) not merged | +| myob_x06da | inhouse | 50.000 1.422 | 1.23 | 99.8 / 99.4 | 3.5 / 3.5 | 7.5 | 12.5% / 23.2% | 4.2% / 10.5% (4.25 / 4.23) | 0.9963 / 0.987 | 0.27 | 26.0 / 7.6 | like for like | +| myob_x06da_powder_1 | inhouse | 50.000 1.495 | 1.73 | 64.8 / 74.2 | 5.3 / 2.9 | 1.5 | 76.4% / 49.3% | 27.7% / 12.5% (4.47 / 4.44) | 0.7653 / 0.966 | 8.58 | 1.9 / 5.5 | coverage: own cut 1.73 A is coarser than the reference, 2 shell(s) not merged | +| myob_x06da_powder_2 | inhouse | 999.000 0.990 | 1.41 | 35.0 / 58.0 | 5.5 / 4.7 | 0.9 | 125.6% / 110.6% | 45.9% / 70.6% (2.97 / 2.98) | 0.9076 / 0.655 | 0.19 | 2.5 / 2.2 | coverage: own cut 1.41 A is coarser than the reference, 4 shell(s) not merged | +| myob_x06da_sparse | inhouse | 50.000 2.000 | 1.77 | 100.0 / 73.7 | 5.5 / 5.2 | 3.1 | 38.6% / 32.6% | 17.2% / 14.6% (5.96 / 5.92) | 0.9004 / 0.972 | 3.77 | 3.2 / 5.5 | like for like | +| myob_x06da_split | inhouse | 50.000 1.506 | 1.96 | 45.9 / 77.7 | 6.0 / 4.4 | 2.0 | 71.2% / 57.7% | 24.4% / 9.1% (4.50 / 4.48) | 0.9199 / 0.984 | 5.19 | 2.6 / 12.4 | coverage: own cut 1.96 A is coarser than the reference, 3 shell(s) not merged | +| myob_x10sa | inhouse | 50.000 1.742 | 1.57 | 98.1 / 97.7 | 3.2 / 3.3 | 5.7 | 16.0% / 24.7% | 7.6% / 18.6% (5.20 / 5.16) | 0.9900 / 0.965 | 0.27 | 10.3 / 5.2 | like for like | +| thau_bl1a_3p8keV | inhouse | 100.000 3.100 | 3.02 | 93.5 / 92.2 | 9.1 / 7.9 | 29.7 | 6.5% / 5.7% | 6.3% / 3.9% (9.26 / 9.23) | 0.9966 / 0.997 | 0.97 | 23.6 / 30.8 | like for like | +| thau_bl1a_4p6keV | inhouse | 100.000 2.530 | 2.47 | 93.2 / 92.0 | 9.0 / 7.9 | 27.2 | 6.4% / 6.2% | 5.2% / 3.9% (7.57 / 7.56) | 0.9979 / 0.998 | 0.84 | 38.2 / 35.6 | like for like | +| thau_bl1a_6p5keV | inhouse | 100.000 1.780 | 1.73 | 93.2 / 91.8 | 8.9 / 8.0 | 19.6 | 7.3% / 7.2% | 4.6% / 4.1% (5.33 / 5.33) | 0.9988 / 0.999 | 0.80 | 43.4 / 34.5 | like for like | +| thau_micromax_pink | inhouse | 50.000 1.400 | 1.24 | 99.0 / 98.5 | 9.5 / 9.4 | 15.3 | 8.7% / 8.0% | 4.8% / 4.0% (4.19 / 4.17) | 0.9991 / 0.999 | 0.60 | 16.0 / 21.2 | like for like | +| thau_x10sa_0p1deg | inhouse | 50.000 2.197 | 1.91 | 99.5 / 99.0 | 11.4 / 11.4 | 12.4 | 13.6% / 23.0% | 6.9% / 10.4% (6.54 / 6.51) | 0.9981 / 0.997 | 0.54 | 10.8 / 9.2 | like for like | +| thau_x10sa_16keV | inhouse | 50.000 1.300 | 1.22 | 94.3 / 94.3 | 18.5 / 18.4 | 24.9 | 7.0% / 7.2% | 2.8% / 2.7% (3.89 / 3.89) | 0.9998 / 1.000 | 0.40 | 53.5 / 44.5 | like for like | +| thau_x10sa_injection | inhouse | 50.000 1.280 | 1.26 | 86.3 / 86.2 | 10.1 / 10.0 | 37.3 | 3.9% / 3.7% | 2.6% / 2.2% (3.83 / 3.83) | 0.9998 / 1.000 | 0.40 | 33.1 / 36.2 | like for like | +| yag_x10sa_20keV | inhouse | 50.000 0.680 | 0.67 | 99.6 / 99.0 | 36.5 / 28.4 | 21.8 | 39.4% / 43.2% | 41.8% / 43.6% (2.04 / 1.99) | 0.9298 / 0.947 | 1.34 | 3.0 / 3.2 | like for like | + +Data quality is not scored; this table is for a human to judge. CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the half-set noise; the noise ratio is rugnux's over XDS's, with XDS's CC1/2 taken at the bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS's merge with half its observations. The low-resolution R_meas is the lowest shell of each table (the shells are XDS's, so both cover the same reflections); reported like everything here. + +### Small molecules: SHELXL refinement of the published structure + +The set's structure from the Crystallography Open Database (COD id) refined against our p.hkl by SHELXL with one fixed recipe (shelx_check.py: data reindexed into the COD setting, non-H anisotropic, H fixed at the COD positions, EXTI refined, three rounds with the weights SHELXL suggests, MERG 2), the way a chemical crystallographer judges data. WGHT a near its 0.2 ceiling and a large EXTI mean the strong reflections or the sigmas are off; K top is / of SHELXL's strongest analysis-of-variance bin (below 1: strong reflections read low). Fixed-model R1(F) scores the data against |Fc| of the COD model as published, one scale, nothing refined. R(int) means something only for unmerged data. Reported, never scored. + +| set | arm | COD | own d_min | R1 >4sig (n) | wR2 | GooF | EXTI | WGHT a / b | peak / hole e/A3 | R(int) / R(sigma) | K top | fixed-model R1(F) | status | +|:--|:--|:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--| +| cytidine | open | 2001311 | 0.58 | 0.0613 (3657) | 0.1762 | 1.061 | 0.039 | 0.047 / 0.10 | 0.27 / -0.24 | 0.0865 / 0.0883 | 1.009 | 0.1084 | ok | +| dnba | open | 4510615 | 0.81 | 0.0266 (1171) | 0.0706 | 1.072 | 0.002 | 0.030 / 1.87 | 0.21 / -0.21 | 0.0202 / 0.0225 | 1.008 | 0.0795 | ok | +| lalanine | open | 2311261 | 0.65 | 0.0311 (1070) | 0.0939 | 1.150 | 0.000 | 0.055 / 0.07 | 0.25 / -0.25 | 0.0204 / 0.0258 | 1.018 | 0.0439 | ok | +| metformin | open | 2108029 | 0.51 | 0.0322 (4453) | 0.0960 | 1.101 | 0.000 | 0.041 / 0.40 | 0.59 / -0.38 | 0.0284 / 0.0206 | 1.012 | 0.0373 | ok | +| nidppe | open | 2012031 | 0.51 | 0.0414 (9289) | 0.1022 | 1.036 | 0.004 | 0.037 / 0.55 | 0.58 / -0.65 | 0.0726 / 0.0525 | 0.993 | 0.1541 | ok | +| aspirin_x10sa_20keV | inhouse | 7050897 | 0.66 | 0.0363 (2288) | 0.1095 | 1.100 | 0.009 | 0.058 / 0.29 | 0.42 / -0.39 | 0.0274 / 0.0171 | 1.005 | 0.0819 | ok | +| aspirin_x10sa_25keV | inhouse | 7050897 | 0.53 | 0.0358 (4379) | 0.1165 | 1.077 | 0.012 | 0.069 / 0.11 | 0.54 / -0.45 | 0.0288 / 0.0180 | 1.005 | 0.0908 | ok | +| citricacid_x10sa_20keV | inhouse | 5000063 | 0.67 | 0.0374 (2092) | 0.1026 | 1.092 | 0.149 | 0.052 / 0.36 | 0.43 / -0.26 | 0.0329 / 0.0233 | 1.009 | 0.2755 | ok | +| hepes_x10sa_20keV | inhouse | 2224210 | 0.66 | 0.0310 (3358) | 0.0895 | 1.061 | 0.053 | 0.052 / 0.80 | 0.44 / -0.54 | 0.0246 / 0.0118 | 1.015 | 0.0404 | ok | +| kdp_x10sa_20keV | inhouse | 9009999 | 0.66 | 0.0475 (305) | 0.1208 | 1.363 | 0.028 | 0.000 / 6.44 | 0.82 / -0.76 | 0.0494 / 0.0238 | 0.990 | 0.2658 | ok | +| lcystine_x10sa_20keV | inhouse | 1513328 | 0.67 | 0.1575 (1216) | 0.3812 | 1.457 | 0.000 | 0.200 / 0.00 | 2.21 / -1.17 | 0.3066 / 0.1109 | 1.396 | 0.2102 | ok | +| lcystine_x10sa_25keV | inhouse | 1513328 | 0.53 | 0.1397 (2305) | 0.3903 | 1.465 | 0.000 | 0.200 / 0.00 | 2.20 / -2.01 | 0.2667 / 0.0919 | 1.256 | 0.2363 | ok | +| yag_x10sa_20keV | inhouse | 2003066 | 0.67 | 0.1021 (247) | 0.2472 | 1.142 | 0.584 | 0.181 / 18.31 | 2.69 / -6.79 | 0.4973 / 0.1053 | 0.789 | 0.1740 | ok | + +XDS merged past its own signal (CC1/2 of its finest shell not significant), so the reference d_min is where XDS's CC1/2 falls through 0.30, and that is also the d_min of the reference-range table: cytc_x10sa (XDS range 2.04 A, reference 2.27 A); insu_I_x06da_weak (XDS range 1.08 A, reference 1.81 A); lyso_x10sa_strong (XDS range 1.18 A, reference 1.24 A). XDS's pooled R_meas, CC1/2 and completeness are over its whole range. + +### Failures + +| set | arm | cause | reason | +|:--|:--|:--|:--| +| 3r6o | open | sym_over | I 41 2 2 vs reference I 41 | +| 5cc8 | open | sym_screw | P 21 21 21 vs reference P 21 21 2 | +| 5ebi | open | sym_over | C 2 2 21 vs reference P 1 21 1 | +| 9hnc | open | sym_screw | P 1 21 1 vs reference P 1 2 1 | +| 9min | open | lattice_halved | primitive volume ratio 0.49 | +| 6z9g | open | lattice_halved | primitive volume ratio 0.50 | +| 8c3e | open | sym_over | P 6 2 2 vs reference P 31 2 1 | +| 4bwl | open | sym_over | C 2 2 21 vs reference P 1 21 1 | +| 2wnq | open | sym_over | C 2 2 21 vs reference P 1 21 1 | +| 6oww | open | sym_over | P 41 21 2 vs reference P 1 21 1 | +| 6p8j | open | sym_over | P 21 21 2 vs reference P 1 21 1 | +| kdp_x10sa_20keV | inhouse | sym_other | I 41 m d vs reference I -4 2 d | + +Not scored: 7k1l (P 6 vs reference P 63: the screw along c is undeterminable from these data (offered P 61 | P 65 | P 62 | P 64 | P 63)); 7mzt (P 21 21 21 vs reference P 21 21 2: the screw along c is undeterminable from these data (offered P 21 21 2)); cuhf2 (no reference to score against) + +### Accepted alternatives + +5 of the passes above are rows that accept more than one reference: the deposition and our reduction disagree, the disagreement is real, and no test available to us settles it, so either answer passes as long as the program picks one of them. These are open questions, not errors attributed to the deposition; the manifest's `ref` keeps the deposited values verbatim in every one of them. + +- 6pxb (open): we report P 31 1 2, the reference is P 32; accepted as P 32 1 2. The deposition merged in point group 3 (its 55502 unique reflections to 1.747 A are what Laue class -3 holds, twice what -3 1 m would) and refined six chains in P 32. The intensities read point group 312 instead: the three added two-folds correlate at 0.98-0.99, at or above the three-folds nobody disputes (0.98), against 0.37-0.40 for the 321 and 622 operators, POINTLESS on our P1 merge picks P -3 1 m (likelihood 1.000), and the reflections centric in 312 but not in 3 are distributed as centric (+435 nats), which a twin law cannot produce. Against that, the deposited chains pair under the added two-fold at 0.3-0.7 A, more than coordinate error at 1.75 A, and the model tells the two indexings apart (R 0.22 against 0.25), so the two-fold may be a near-exact non-crystallographic one; ZANUDA settles on P 32 2 1, whose operators these data do not support. Neither answer is established, so both are accepted; P 31 1 2, which the data cannot separate from P 32 1 2, is accepted as its hand +- 6toc (open): we report P 42 2 2, the reference is P 42; accepted as P 42 2 2. Our reduction and an independent POINTLESS run on our own P1 merge both read point group 422, and the deposited asymmetric unit's two chains are related by the very two-fold the higher group adds, to 0.16 A CA RMSD over 43 residues - coordinate error at 1.85 A. Merging in P 42 2 2 costs 0.0006 in R_meas for 1.75x the multiplicity and correlates better with the deposited model (0.9802 vs 0.9764). The refinement test is NOT unanimous: ZANUDA 1.097 refines P 42 2 2 to R-free 0.2561 against P 42's 0.2636 at half the parameters and reports the deposited assignment incorrect, while an independent Refmac 5.8.0431 comparison on a symmetry-consistent free set puts P 42 ahead by 0.004-0.020 depending on cycle count - less than the spread between refinement protocols. Neither answer is established: a pseudo-symmetry too exact for any test we have remains a live explanation, and so does the deposited assignment. Either is accepted +- 8xte (open): we report P 31 2 1, the reference is P 32; accepted as P 31 2 1. The evidence favours the higher group here. The twin-immune centric zone of the added two-folds - reflections the higher group makes centric, which are their own twin mates and so cannot be made to read centric by a merohedral twin law - gives <|E^2-1|> = 0.946 +/- 0.012 against a centric expectation of 0.968 and an acentric 0.736, at +707 nats. Re-refinement on a shared free set with the twin law removed from both sides favours P 3_2 2 1 (0.2177/0.2322) over P 3_2 (0.2460/0.2612), and the deposited entry's published R values reproduce only with an undeclared twin law h,-h-k,-l at alpha = 0.50, which is itself a 321-symmetric target. No refinement R can close the question in principle, because a merohedral twin at exactly alpha = 0.5 and true 321 predict identical intensities; the case rests on the centric zone. Both answers are accepted, ours being the better supported +- 8xtg (open): we report P 31 2 1, the reference is P 32; accepted as P 31 2 1. The evidence favours the DEPOSITION here, and this row must not be read as the 8xte one. Every correlation-based instrument we have - our own operator correlations, POINTLESS (0.85 on our P1 merge) - reads point group 321, but our own twin-immune centric-zone test, the only one that separates real symmetry from pseudo-symmetry, read <|E^2-1|> = 0.869 at -44.9 nats against the promotion when it assumed an untwinned zone; read at the fraction of the crystal's other twin laws (rc174) it now favours the promotion, with an L-test twin fraction of 0.20-0.26. Whether this crystal is partially twinned or purely pseudo-symmetric is not established. Both answers are accepted, the deposition being the better supported +- 9rci (open): we report P 1, the reference is P 1; accepted as cell 35.869 39.297 199.976. The deposited cell is the (0,1/2,1/2)-centred sublattice of the supercell we report, to 0.17%, and both descriptions of this lattice are defensible. The Patterson has an off-origin peak at 62.5% of the origin, so a real translational NCS relates the two halves of our cell: describing the crystal by the doubled cell with the near-translation left in the content, or by its sublattice with the near-translation absorbed into the lattice, is a choice, not a measurement. The alternative cell is the deposited one doubled along c with the centring removed (c' = b + 2c), computed from the deposited cell alone - not from our output + +### Per set + +Space group: on the open arm the data's own determination. Where --model put the model's enantiomorph on the label, the label is in the note and the group scored is the search's. + +R_free and R_work describe one run against its model and do not carry across runs: the model is scaled to the data by an overall factor and an anisotropic B, so whatever the amplitudes' radial profile does that this shape cannot follow is reported as R, and a change in the profile alone moves R_free further than a real change in the data does. R_model (shell-scaled) is the same R with one free scale per resolution shell removing exactly that, over every reflection rather than the free 5%, and it is the column to read in a delta table. Radial misfit is how big that per-shell rescale had to be (the RMS of its logarithm): when it moves between two runs, R_free between them means little. Both are reported for a human to judge; nothing is scored on them. + +Anom. sigma is the anomalous difference map (F+ - F- on the model's phases) read at the model's anomalous scatterers - every atom from phosphorus up: the S of Met and Cys, metals, Cl, I - and averaged, in units of the map's r.m.s.: how much anomalous signal the merge carries, on the model's yardstick. Reported, never scored. + +| set | arm | verdict | space group | ref | cell dev % | V ratio | d_min | ref d_min | gain | compl % | mult | R_meas | low-res R_meas | CC1/2 | ISa | ref ISa | idx | R_model (shell-scaled) | radial misfit | CC model | R_free (within-run) | R_work | dep R_free | R_free ratio | anom. sigma | REFMAC R_free | REFMAC dep data | REFMAC ratio | dCC dep. outer | model fit | time s | note | +|:--|:--|:--|:--|:--|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--| +| 11if | open | pass | P 41 | P 43 | 0.12 | 0.997 | 1.36 | 1.51 | +10.0% | 88.5 | 11.1 | 4.8% | 2.8% | 1.000 | 25.7 | - | - | 0.192 | 0.134 | 0.966 | 0.188 | 0.197 | 0.210 | 0.898 | 2.920 | 0.229 | 0.233 | 0.982 | 0.022 | ACCEPTED | 12 | P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model | +| 2wnn | open | pass | P 1 21 1 | P 1 21 1 | 0.11 | 1.003 | 1.44 | 1.65 | +12.5% | 73.7 | 3.6 | 7.2% | 4.6% | 0.997 | 14.1 | - | - | 0.279 | 0.174 | 0.913 | 0.286 | 0.284 | 0.249 | 1.148 | 1.000 | 0.253 | 0.313 | 0.807 | -0.351 | NOT_TESTED | 76 | | +| 2wnq | open | fail | C 2 2 21 | P 1 21 1 | 68.55 | 0.997 | 1.65 | 1.80 | +8.1% | 97.1 | 7.1 | 10.8% | 6.0% | 0.997 | 10.3 | - | - | 0.536 | 0.275 | 0.397 | 0.545 | 0.549 | 0.253 | 2.153 | 0.020 | 0.278 | 0.261 | 1.066 | -0.626 | NOT_TESTED | 63 | C 2 2 21 vs reference P 1 21 1 | +| 2wnz | open | pass | P 1 21 1 | P 1 21 1 | 0.13 | 1.001 | 1.85 | 1.85 | -0.2% | 99.7 | 3.7 | 12.2% | 4.0% | 0.996 | 14.8 | - | - | 0.207 | 0.220 | 0.936 | 0.232 | 0.237 | 0.225 | 1.031 | 0.940 | 0.255 | 0.229 | 1.113 | -0.396 | NOT_TESTED | 61 | | +| 2xfw | open | pass | P 1 21 1 | P 1 21 1 | 0.11 | 1.000 | 1.55 | 1.65 | +5.8% | 99.7 | 3.7 | 14.8% | 4.4% | 0.996 | 14.1 | - | - | 0.200 | 0.157 | 0.951 | 0.220 | 0.218 | 0.182 | 1.209 | 1.110 | 0.215 | 0.186 | 1.158 | -0.515 | ACCEPTED | 93 | | +| 36gk | open | pass | I 2 2 2 | I 2 2 2 | 0.09 | 0.999 | 2.09 | 2.28 | +8.2% | 99.7 | 13.8 | 15.7% | 7.3% | 0.998 | 11.8 | - | - | 0.203 | 0.109 | 0.950 | 0.203 | 0.212 | 0.223 | 0.913 | 2.800 | 0.255 | 0.252 | 1.014 | 0.024 | NOT_TESTED | 38 | | +| 3inp | open | pass | F 41 3 2 | F 41 3 2 | 0.04 | 1.001 | 1.70 | 2.05 | +16.9% | 99.7 | 18.8 | 10.8% | 5.5% | 0.999 | 13.2 | - | - | 0.239 | 0.070 | 0.946 | 0.239 | 0.240 | 0.177 | 1.351 | 2.980 | 0.240 | 0.244 | 0.984 | 0.003 | NOT_TESTED | 46 | | +| 3ky7 | open | pass | P 43 3 2 | P 43 3 2 | 0.03 | 1.001 | 1.92 | 2.35 | +18.3% | 99.7 | 25.7 | 10.7% | 5.9% | 0.999 | 12.2 | - | - | 0.339 | 0.043 | 0.899 | 0.335 | 0.341 | 0.255 | 1.312 | 0.400 | 0.352 | 0.355 | 0.992 | 0.001 | NOT_TESTED | 58 | | +| 3mc4 | open | pass | R 3:H | H 3 | 0.06 | 0.998 | 1.79 | 1.95 | +8.1% | 79.9 | 3.3 | 10.0% | 8.4% | 0.994 | 12.5 | - | - | 0.252 | 0.061 | 0.900 | 0.256 | 0.254 | 0.156 | 1.642 | 1.400 | 0.261 | 0.191 | 1.365 | -0.333 | ACCEPTED | 10 | | +| 3meb | open | pass | P 1 21 1 | P 1 21 1 | 0.43 | 0.992 | 1.53 | 1.90 | +19.4% | 31.7 | 2.2 | 13.9% | 9.5% | 0.989 | 15.6 | - | - | 0.201 | 1.105 | 0.954 | 0.203 | 0.205 | 0.214 | 0.950 | -0.190 | 0.229 | 0.230 | 0.993 | -0.107 | NOT_TESTED | 6 | | +| 3p85 | open | pass | P 63 2 2 | P 63 2 2 | 0.50 | 1.007 | 1.62 | 1.90 | +14.7% | 86.8 | 10.8 | 12.1% | 10.2% | 0.998 | 8.1 | - | - | 0.194 | 0.117 | 0.953 | 0.201 | 0.201 | 0.216 | 0.930 | 7.760 | 0.203 | 0.232 | 0.879 | -0.404 | NOT_TESTED | 10 | | +| 3r6o | open | fail | I 41 2 2 | I 41 | 0.43 | 0.990 | 1.53 | 1.95 | +21.4% | 43.1 | 7.2 | 19.1% | 17.2% | 0.985 | 4.5 | - | - | 0.302 | 0.725 | 0.807 | 0.307 | 0.304 | 0.185 | 1.657 | 1.580 | 0.310 | 0.223 | 1.393 | -0.664 | NOT_TESTED | 5 | I 41 2 2 vs reference I 41 | +| 4bwl | open | fail | C 2 2 21 | P 1 21 1 | 72.04 | 1.004 | 1.68 | 2.00 | +16.1% | 99.4 | 7.0 | 14.1% | 5.7% | 0.998 | 14.3 | - | - | 0.520 | 0.130 | 0.383 | 0.524 | 0.524 | 0.235 | 2.230 | 0.300 | 0.234 | 0.227 | 1.030 | -0.520 | NOT_TESTED | 59 | C 2 2 21 vs reference P 1 21 1 | +| 5cc8 | open | fail | P 21 21 21 | P 21 21 2 | 0.06 | 1.000 | 1.53 | 1.75 | +12.6% | 48.5 | 3.9 | 8.0% | 6.7% | 0.997 | 11.2 | - | - | 0.171 | 0.165 | 0.959 | 0.184 | 0.178 | 0.194 | 0.950 | 4.010 | 0.203 | 0.214 | 0.950 | 0.036 | ACCEPTED | 11 | P 21 21 21 vs reference P 21 21 2 | +| 5ebi | open | fail | C 2 2 21 | P 1 21 1 | 40.37 | 1.001 | 0.85 | 1.09 | +21.6% | 99.4 | 6.7 | 12.3% | 7.2% | 0.998 | 10.9 | - | - | 0.580 | 0.135 | 0.340 | 0.571 | 0.584 | 0.169 | 3.380 | -0.150 | 0.181 | 0.181 | 0.996 | 0.024 | REJECTED | 46 | C 2 2 21 vs reference P 1 21 1 | +| 5epe | open | pass | F 2 3 | F 2 3 | 0.00 | 1.000 | 1.77 | 1.90 | +7.0% | 99.7 | 15.1 | 16.5% | 8.4% | 0.998 | 9.9 | - | - | 0.173 | 0.109 | 0.951 | 0.181 | 0.182 | 0.167 | 1.086 | 11.950 | 0.208 | 0.200 | 1.039 | -0.093 | ACCEPTED | 36 | | +| 5f6m | open | pass | P 21 21 21 | P 21 21 21 | 0.08 | 1.002 | 1.09 | 1.10 | +1.0% | 78.8 | 4.0 | 5.1% | 3.7% | 0.998 | 23.0 | - | - | 0.152 | 0.132 | 0.972 | 0.165 | 0.160 | 0.154 | 1.065 | 5.990 | 0.163 | 0.158 | 1.032 | -0.059 | NOT_TESTED | 8 | | +| 5j23 | open | pass | R 3:H | H 3 | 0.15 | 0.998 | 2.17 | 2.30 | +5.8% | 99.7 | 5.2 | 15.4% | 6.0% | 0.996 | 11.6 | - | - | 0.226 | 0.145 | 0.939 | 0.234 | 0.237 | 0.169 | 1.386 | 0.860 | 0.188 | 0.185 | 1.012 | 0.318 | NOT_TESTED | 34 | | +| 5jk4 | open | pass | P 1 21 1 | P 1 21 1 | 0.12 | 1.000 | 1.02 | 1.10 | +7.5% | 84.9 | 4.0 | 8.9% | 4.1% | 0.998 | 14.8 | - | - | 0.109 | 0.092 | 0.980 | 0.123 | 0.121 | 0.125 | 0.990 | 3.690 | 0.152 | 0.154 | 0.992 | 0.018 | NOT_TESTED | 25 | | +| 5jvn | open | pass | P 6 2 2 | P 6 2 2 | 0.03 | 0.999 | 2.24 | 2.90 | +22.8% | 98.7 | 16.8 | 16.3% | 5.4% | 0.999 | 15.6 | - | - | 0.228 | 0.137 | 0.884 | 0.249 | 0.249 | 0.235 | 1.059 | 1.360 | 0.250 | 0.242 | 1.031 | 0.092 | NOT_TESTED | 42 | | +| 5ky6 | open | pass | P 1 21 1 | P 1 21 1 | 0.64 | 0.994 | 1.54 | 1.94 | +20.5% | 96.8 | 3.2 | 25.4% | 10.6% | 0.986 | 6.1 | - | - | 0.233 | 0.123 | 0.937 | 0.243 | 0.241 | 0.223 | 1.091 | 0.550 | 0.246 | 0.229 | 1.076 | -0.142 | NOT_TESTED | 91 | | +| 5lzl | open | pass | P 31 2 1 | P 31 2 1 | 0.29 | 0.992 | 2.86 | 3.47 | +17.6% | 94.9 | 8.9 | 17.1% | 4.8% | 0.998 | 15.2 | - | - | 0.222 | 0.139 | 0.878 | 0.229 | 0.233 | 0.250 | 0.915 | 1.260 | 0.264 | 0.257 | 1.027 | -0.026 | NOT_TESTED | 18 | | +| 5m17 | open | pass | I 4 | I 4 | 0.08 | 0.998 | 0.98 | 1.03 | +4.6% | 92.6 | 6.0 | 5.5% | 5.1% | 0.996 | 16.2 | - | - | 0.132 | 0.220 | 0.961 | 0.158 | 0.159 | 0.130 | 1.217 | 7.090 | 0.175 | 0.171 | 1.024 | -0.034 | NOT_TESTED | 97 | | +| 5mln | open | pass | P 21 21 2 | P 21 2 21 | 0.10 | 0.999 | 1.26 | 1.60 | +21.3% | 99.7 | 8.1 | 10.4% | 3.5% | 0.999 | 21.5 | - | - | 0.181 | 0.144 | 0.949 | 0.188 | 0.197 | 0.180 | 1.046 | 2.170 | 0.194 | 0.190 | 1.017 | -0.007 | ACCEPTED | 22 | | +| 5nw5 | open | pass | P 21 21 21 | P 21 21 21 | 0.19 | 0.997 | 7.07 | 6.50 | -8.8% | 99.7 | 6.3 | 33.3% | 16.7% | 0.927 | 7.7 | - | - | 0.395 | 0.254 | 0.806 | 0.448 | 0.419 | 0.277 | 1.618 | 0.090 | 0.342 | 0.325 | 1.053 | 0.038 | NOT_TESTED | 46 | | +| 5ojv | open | pass | P 21 21 2 | P 21 21 2 | 0.41 | 0.993 | 1.82 | 2.06 | +11.7% | 98.5 | 6.5 | 14.5% | 4.6% | 0.998 | 13.2 | - | - | 0.173 | 0.158 | 0.960 | 0.187 | 0.192 | 0.194 | 0.964 | 1.990 | 0.213 | 0.214 | 0.998 | -0.005 | NOT_TESTED | 237 | | +| 5reo | open | pass | C 1 2 1 | C 1 2 1 | 0.35 | 1.009 | 1.65 | 1.88 | +12.1% | 99.4 | 3.1 | 22.2% | 8.1% | 0.990 | 23.7 | - | - | 0.202 | 0.068 | 0.960 | 0.199 | 0.206 | 0.227 | 0.879 | 0.750 | 0.231 | 0.233 | 0.991 | -0.036 | NOT_TESTED | 9 | | +| 5src | open | pass | P 41 | P 43 | 0.09 | 0.999 | 0.97 | 1.05 | +7.7% | 99.0 | 6.2 | 5.7% | 3.1% | 0.999 | 19.6 | - | - | 0.161 | 0.190 | 0.962 | 0.183 | 0.180 | 0.175 | 1.042 | 6.050 | 0.208 | 0.211 | 0.988 | 0.035 | ACCEPTED | 38 | P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model | +| 5t39 | open | pass | P 1 21 1 | P 1 21 1 | 0.13 | 1.003 | 1.01 | 1.10 | +8.5% | 90.9 | 4.2 | 7.3% | 4.4% | 0.998 | 16.2 | - | - | 0.151 | 0.088 | 0.956 | 0.156 | 0.153 | 0.154 | 1.014 | 6.470 | 0.183 | 0.182 | 1.003 | -0.053 | NOT_TESTED | 71 | | +| 5uth | open | pass | P 31 2 1 | P 31 2 1 | 0.21 | 0.995 | 1.72 | 1.95 | +11.7% | 82.8 | 8.4 | 12.4% | 7.5% | 0.997 | 9.5 | - | - | 0.194 | 0.165 | 0.952 | 0.217 | 0.210 | 0.202 | 1.074 | 2.800 | 0.222 | 0.220 | 1.009 | -0.012 | ACCEPTED | 10 | | +| 5vml | open | pass | P 42 21 2 | P 42 21 2 | 0.02 | 1.000 | 1.92 | 1.70 | -12.6% | 81.0 | 7.4 | 10.2% | 9.8% | 0.996 | 10.9 | - | - | 0.156 | 0.050 | 0.964 | 0.165 | 0.158 | 0.171 | 0.966 | 3.400 | 0.180 | 0.171 | 1.053 | -0.038 | NOT_TESTED | 6 | | +| 6cdl | open | pass | P 21 21 2 | P 21 21 2 | 1.22 | 1.023 | 1.13 | 1.25 | +9.5% | 71.7 | 6.4 | 7.8% | 6.4% | 0.997 | 11.6 | - | - | 0.147 | 0.233 | 0.949 | 0.159 | 0.159 | 0.181 | 0.881 | 4.600 | 0.195 | 0.187 | 1.039 | -0.051 | NOT_TESTED | 49 | | +| 6cee | open | pass | P 21 21 21 | P 21 21 21 | 0.04 | 1.001 | 1.38 | 1.55 | +10.9% | 87.8 | 6.1 | 5.4% | 3.6% | 0.999 | 23.9 | - | - | 0.166 | 0.137 | 0.952 | 0.174 | 0.182 | 0.169 | 1.028 | 9.050 | 0.181 | 0.182 | 0.993 | 0.001 | NOT_TESTED | 7 | | +| 6cs9 | open | pass | P 1 21 1 | P 1 21 1 | 0.06 | 0.999 | 1.72 | 1.85 | +7.1% | 91.3 | 3.8 | 9.9% | 4.3% | 0.998 | 13.1 | - | - | 0.211 | 0.143 | 0.902 | 0.205 | 0.226 | 0.227 | 0.903 | 0.770 | 0.257 | 0.252 | 1.018 | 0.010 | NOT_TESTED | 29 | | +| 6f3p | open | pass | C 1 2 1 | C 1 2 1 | 0.12 | 0.997 | 1.13 | 1.35 | +16.1% | 99.3 | 3.6 | 10.0% | 5.4% | 0.997 | 9.4 | - | - | 0.143 | 0.214 | 0.968 | 0.169 | 0.169 | 0.127 | 1.331 | -3.420 | 0.173 | 0.174 | 0.999 | 0.015 | NOT_TESTED | 128 | | +| 6fid | open | pass | P 21 21 21 | P 21 21 21 | 0.41 | 1.008 | 1.98 | 2.20 | +10.1% | 72.9 | 12.4 | 9.4% | 10.4% | 0.998 | 13.6 | - | - | 0.193 | 0.054 | 0.943 | 0.196 | 0.198 | 0.220 | 0.892 | 11.260 | 0.237 | 0.242 | 0.977 | -0.018 | NOT_TESTED | 28 | | +| 6fvz | open | pass | C 2 2 2 | C 2 2 2 | 0.43 | 0.989 | 1.49 | 1.80 | +17.2% | 99.5 | 6.7 | 24.2% | 5.1% | 0.996 | 20.5 | - | - | 0.198 | 0.094 | 0.954 | 0.202 | 0.206 | 0.199 | 1.017 | 1.330 | 0.207 | 0.207 | 0.998 | -0.018 | NOT_TESTED | 38 | | +| 6fwc | open | pass | C 2 2 2 | C 2 2 2 | 0.04 | 0.999 | 1.41 | 1.70 | +16.9% | 94.7 | 4.0 | 16.4% | 4.3% | 0.995 | 27.3 | - | - | 0.188 | 0.098 | 0.959 | 0.196 | 0.198 | 0.189 | 1.037 | 1.440 | 0.197 | 0.201 | 0.980 | 0.018 | NOT_TESTED | 26 | | +| 6g1f | open | pass | C 1 2 1 | C 1 2 1 | 0.04 | 1.000 | 1.93 | 2.25 | +14.1% | 99.6 | 3.8 | 11.8% | 3.9% | 0.997 | 21.0 | - | - | 0.208 | 0.120 | 0.954 | 0.222 | 0.220 | 0.211 | 1.054 | 1.040 | 0.231 | 0.226 | 1.018 | 0.017 | NOT_TESTED | 116 | | +| 6gvk | open | pass | C 1 2 1 | C 1 2 1 | 0.09 | 0.999 | 1.42 | 1.55 | +8.5% | 99.6 | 6.7 | 5.2% | 3.3% | 0.999 | 19.8 | - | - | 0.200 | 0.179 | 0.950 | 0.215 | 0.212 | 0.210 | 1.021 | 1.880 | 0.225 | 0.227 | 0.989 | 0.007 | NOT_TESTED | 94 | | +| 6h2p_1p89A | open | pass | C 2 2 21 | C 2 2 21 | 0.03 | 1.000 | 1.76 | - | - | 88.1 | 10.2 | 8.5% | 5.2% | 0.999 | 22.8 | - | - | 0.143 | 0.076 | 0.971 | 0.149 | 0.147 | 0.170 | 0.873 | 9.200 | 0.170 | 0.163 | 1.043 | 0.026 | NOT_TESTED | 217 | | +| 6h2p_native | open | pass | C 2 2 21 | C 2 2 21 | 0.04 | 0.999 | 1.32 | 1.48 | +10.8% | 99.5 | 6.6 | 13.7% | 3.9% | 0.999 | 20.0 | - | - | 0.161 | 0.067 | 0.973 | 0.165 | 0.165 | 0.170 | 0.968 | 3.330 | 0.175 | 0.182 | 0.959 | 0.041 | NOT_TESTED | 113 | | +| 6h5t | open | pass | I 4 2 2 | I 4 2 2 | 0.40 | 0.991 | 1.48 | 1.69 | +12.4% | 99.4 | 7.4 | 14.6% | 8.3% | 0.996 | 9.3 | - | - | 0.196 | 0.192 | 0.941 | 0.222 | 0.218 | 0.191 | 1.164 | 9.000 | 0.227 | 0.214 | 1.057 | -0.015 | NOT_TESTED | 35 | | +| 6hv2 | open | pass | P 61 2 2 | P 61 2 2 | 0.04 | 1.001 | 1.41 | 1.71 | +17.6% | 99.8 | 29.2 | 19.8% | 5.5% | 1.000 | 13.7 | - | - | 0.219 | 0.279 | 0.947 | 0.239 | 0.242 | 0.263 | 0.910 | 7.830 | 0.271 | 0.326 | 0.829 | -0.008 | NOT_TESTED | 32 | | +| 6hwj | open | pass | P 1 21 1 | P 1 21 1 | 0.10 | 1.001 | 1.71 | 1.98 | +13.4% | 97.5 | 3.5 | 8.6% | 2.4% | 0.999 | 39.0 | - | - | 0.187 | 0.129 | 0.960 | 0.192 | 0.198 | 0.214 | 0.894 | 1.600 | 0.215 | 0.223 | 0.967 | 0.052 | NOT_TESTED | 14 | | +| 6i3j | open | pass | F 2 2 2 | F 2 2 2 | 0.08 | 0.999 | 2.32 | 2.59 | +10.5% | 98.7 | 7.2 | 24.6% | 16.3% | 0.977 | 6.8 | - | - | 0.203 | 0.172 | 0.914 | 0.237 | 0.234 | 0.226 | 1.049 | 1.730 | 0.219 | 0.190 | 1.154 | 0.002 | NOT_TESTED | 75 | | +| 6iu5 | open | pass | P 31 | P 31 | 0.02 | 1.000 | 2.12 | 2.25 | +6.0% | 99.1 | 4.8 | 18.2% | 11.4% | 0.993 | 8.0 | - | - | 0.212 | 0.079 | 0.932 | 0.216 | 0.222 | 0.250 | 0.863 | 7.630 | 0.261 | 0.258 | 1.008 | -0.018 | ACCEPTED | 88 | | +| 6iu6 | open | pass | P 31 | P 31 | 0.33 | 0.996 | 2.36 | 2.90 | +18.6% | 97.4 | 4.2 | 11.5% | 7.9% | 0.996 | 7.2 | - | - | 0.221 | 0.165 | 0.950 | 0.233 | 0.238 | 0.211 | 1.102 | 3.540 | 0.223 | 0.209 | 1.063 | -0.016 | ACCEPTED | 91 | | +| 6iu8 | open | pass | P 31 | P 31 | 0.02 | 1.000 | 2.32 | 2.70 | +14.1% | 99.7 | 10.2 | 14.1% | 7.5% | 0.998 | 8.1 | - | - | 0.270 | 0.173 | 0.910 | 0.282 | 0.282 | 0.215 | 1.315 | 1.230 | 0.211 | 0.215 | 0.984 | -0.024 | NOT_TESTED | 13 | | +| 6iu9 | open | pass | P 31 | P 31 | 0.22 | 1.005 | 2.74 | 3.00 | +8.8% | 99.7 | 4.7 | 14.4% | 9.8% | 0.993 | 5.4 | - | - | 0.310 | 0.096 | 0.811 | 0.327 | 0.312 | 0.265 | 1.234 | 1.430 | 0.250 | 0.254 | 0.983 | 0.036 | ACCEPTED | 86 | | +| 6jgh | open | pass | P 21 21 21 | P 21 21 21 | 0.39 | 1.008 | 0.87 | 0.94 | +7.1% | 99.3 | 7.2 | 23.4% | 10.9% | 0.990 | 6.6 | - | - | 0.136 | 0.170 | 0.964 | 0.156 | 0.157 | 0.129 | 1.206 | 2.070 | 0.174 | 0.163 | 1.069 | -0.070 | NOT_TESTED | 104 | | +| 6jgi | open | pass | P 21 21 21 | P 21 21 21 | 0.20 | 0.997 | 0.75 | 0.85 | +12.0% | 98.5 | 7.0 | 14.3% | 8.5% | 0.995 | 8.5 | - | - | 0.112 | 0.144 | 0.975 | 0.128 | 0.127 | 0.112 | 1.145 | 6.270 | 0.145 | 0.148 | 0.980 | 0.069 | NOT_TESTED | 115 | | +| 6jgj | open | pass | P 21 21 21 | P 21 21 21 | 0.28 | 0.992 | 0.65 | 0.77 | +15.7% | 85.0 | 7.2 | 7.4% | 4.6% | 0.999 | 15.1 | - | - | 0.164 | 0.057 | 0.962 | 0.167 | 0.166 | 0.125 | 1.333 | 2.600 | 0.188 | 0.167 | 1.127 | 0.026 | NOT_TESTED | 49 | | +| 6moj | open | pass | I 41 2 2 | I 41 2 2 | 0.08 | 0.998 | 2.43 | 2.43 | +0.1% | 99.7 | 26.7 | 52.4% | 8.9% | 0.998 | 8.3 | - | - | 0.248 | 0.210 | 0.917 | 0.257 | 0.262 | 0.250 | 1.028 | 1.050 | 0.276 | 0.282 | 0.977 | 0.029 | NOT_TESTED | 125 | | +| 6nen | open | pass | P 3 1 2 | P 3 1 2 | 0.07 | 0.998 | 1.77 | 2.15 | +17.7% | 99.6 | 21.6 | 28.5% | 10.2% | 0.997 | 6.3 | - | - | 0.211 | 0.054 | 0.936 | 0.218 | 0.215 | 0.208 | 1.050 | 1.000 | 0.212 | 0.210 | 1.008 | 0.007 | ACCEPTED | 19 | | +| 6o2h | open | pass | P 1 | P 1 | 0.74 | 0.982 | 1.10 | 1.21 | +9.5% | 30.3 | 1.1 | 7.0% | 8.3% | 0.977 | 23.9 | - | - | 0.096 | 0.132 | 0.934 | 0.129 | 0.127 | 0.117 | 1.103 | 0.110 | 0.179 | 0.155 | 1.157 | -0.003 | ACCEPTED | 13 | | +| 6oel | open | pass | F 41 3 2 | F 41 3 2 | 0.00 | 1.000 | 2.85 | 3.10 | +8.1% | 99.7 | 41.4 | 36.5% | 8.2% | 0.998 | 9.9 | - | - | 0.248 | 0.136 | 0.815 | 0.255 | 0.259 | 0.256 | 0.998 | 0.700 | 0.278 | 0.288 | 0.965 | -0.024 | NOT_TESTED | 40 | | +| 6oww | open | fail | P 41 21 2 | P 1 21 1 | 0.19 | 0.998 | 2.72 | 3.84 | +29.3% | 99.7 | 42.0 | 203.3% | 14.0% | 0.994 | 11.8 | - | - | 0.363 | 0.076 | 0.857 | 0.376 | 0.368 | 0.340 | 1.105 | 2.030 | 0.321 | 0.342 | 0.938 | -0.049 | ACCEPTED | 222 | P 41 21 2 vs reference P 1 21 1 | +| 6p8j | open | fail | P 21 21 2 | P 1 21 1 | 0.33 | 0.992 | 1.28 | 1.47 | +13.1% | 99.6 | 6.6 | 23.7% | 12.0% | 0.990 | 5.3 | - | - | 0.241 | 0.099 | 0.907 | 0.251 | 0.247 | 0.229 | 1.094 | 1.180 | 0.257 | 0.242 | 1.062 | 0.120 | NOT_TESTED | 143 | P 21 21 2 vs reference P 1 21 1 | +| 6p8p | open | pass | P 4 | P 4 | 0.14 | 1.003 | 1.46 | 1.64 | +10.9% | 99.7 | 6.7 | 13.2% | 5.1% | 0.998 | 15.5 | - | - | 0.189 | 0.143 | 0.950 | 0.204 | 0.203 | 0.198 | 1.030 | 1.890 | 0.210 | 0.203 | 1.036 | -0.017 | NOT_TESTED | 20 | | +| 6pb3 | open | pass | P 6 | P 6 | 0.17 | 0.995 | 1.84 | 2.05 | +10.1% | 93.6 | 10.3 | 6.5% | 3.0% | 1.000 | 25.4 | - | - | 0.214 | 0.108 | 0.956 | 0.229 | 0.219 | 0.252 | 0.908 | 10.600 | 0.270 | 0.269 | 1.002 | 0.045 | ACCEPTED | 32 | | +| 6pxb | open | pass | P 31 1 2 | P 32 | 0.23 | 0.994 | 1.39 | 1.75 | +20.3% | 99.7 | 10.0 | 8.8% | 4.6% | 0.999 | 12.1 | - | - | 0.240 | 0.258 | 0.959 | 0.279 | 0.261 | 0.263 | 1.059 | 0.620 | 0.291 | 0.294 | 0.991 | -0.005 | NOT_TESTED | 16 | P 31 1 2 vs reference P 32: accepted alternative P 32 1 2 - a knife-edge this battery does not decide | +| 6pxc | open | pass | I 2 2 2 | I 2 2 2 | 0.23 | 0.996 | 1.41 | 1.60 | +11.9% | 96.0 | 5.7 | 8.1% | 5.6% | 0.998 | 10.2 | - | - | 0.210 | 0.164 | 0.960 | 0.214 | 0.217 | 0.210 | 1.022 | 0.690 | 0.226 | 0.222 | 1.019 | 0.035 | NOT_TESTED | 32 | | +| 6qaj | open | pass | C 2 2 21 | C 2 2 21 | 0.41 | 0.993 | 2.70 | 2.90 | +6.8% | 99.7 | 9.2 | 24.4% | 5.1% | 0.997 | 13.2 | - | - | 0.321 | 0.230 | 0.858 | 0.332 | 0.331 | 0.291 | 1.139 | 4.460 | 0.311 | 0.327 | 0.951 | -0.069 | NOT_TESTED | 65 | | +| 6r72 | open | pass | P 1 21 1 | P 1 21 1 | 1.32 | 0.979 | 4.39 | 3.95 | -11.2% | 99.7 | 7.1 | 16.5% | 3.5% | 1.000 | 17.9 | - | - | 0.362 | 0.045 | 0.586 | 0.367 | 0.364 | 0.321 | 1.143 | 0.130 | 0.394 | 0.380 | 1.038 | -0.232 | NOT_TESTED | 26 | | +| 6rlr | open | pass | P 1 | P 1 | 0.03 | 1.001 | 1.92 | 2.00 | +3.9% | 97.6 | 3.5 | 12.9% | 4.1% | 0.998 | 16.4 | - | - | 0.250 | 0.067 | 0.896 | 0.254 | 0.254 | 0.279 | 0.909 | 0.680 | 0.287 | 0.281 | 1.022 | -0.014 | NOT_TESTED | 18 | | +| 6rym | open | pass | P 41 | P 43 | 0.06 | 0.998 | 1.45 | 1.46 | +0.5% | 79.7 | 3.9 | 4.2% | 4.1% | 0.998 | 22.9 | - | - | 0.170 | 0.246 | 0.940 | 0.190 | 0.183 | 0.184 | 1.033 | 14.400 | 0.198 | 0.194 | 1.022 | -0.009 | ACCEPTED | 16 | P 41 vs reference P 43 (hand only (needs anomalous)); labelled P 43 from the model | +| 6s1u | open | pass | P 1 21 1 | P 1 21 1 | 0.13 | 1.002 | 1.75 | 1.90 | +7.8% | 95.0 | 3.8 | 22.3% | 5.6% | 0.989 | 13.8 | - | - | 0.202 | 0.089 | 0.951 | 0.200 | 0.210 | 0.235 | 0.849 | 0.600 | 0.244 | 0.251 | 0.975 | -0.031 | NOT_TESTED | 13 | | +| 6toc | open | pass | P 42 2 2 | P 42 | 0.30 | 0.991 | 1.64 | 1.85 | +11.6% | 99.7 | 24.3 | 9.4% | 2.3% | 1.000 | 24.8 | - | - | 0.276 | 0.136 | 0.973 | 0.270 | 0.281 | 0.265 | 1.018 | 0.540 | 0.245 | 0.267 | 0.917 | 0.132 | ACCEPTED | 11 | P 42 2 2 vs reference P 42: accepted alternative P 42 2 2 - a knife-edge this battery does not decide | +| 6ttn | open | pass | P 21 21 21 | P 21 21 21 | 0.41 | 1.012 | 1.08 | 1.12 | +3.2% | 98.5 | 11.9 | 10.7% | 4.5% | 0.999 | 14.5 | - | - | 0.132 | 0.155 | 0.975 | 0.154 | 0.149 | 0.146 | 1.051 | 8.200 | 0.175 | 0.172 | 1.020 | -0.018 | NOT_TESTED | 45 | | +| 6u7g | open | pass | P 1 21 1 | P 1 21 1 | 0.13 | 1.003 | 1.88 | 2.35 | +20.1% | 78.4 | 3.3 | 8.0% | 4.1% | 0.998 | 13.3 | - | - | 0.198 | 0.110 | 0.933 | 0.200 | 0.204 | 0.218 | 0.916 | 1.570 | 0.232 | 0.237 | 0.980 | -0.097 | NOT_TESTED | 71 | | +| 6ukf | open | pass | P 1 21 1 | P 1 21 1 | 0.08 | 1.001 | 0.96 | 1.00 | +4.3% | 91.5 | 6.8 | 9.5% | 5.8% | 0.998 | 9.8 | - | - | 0.165 | 0.074 | 0.954 | 0.161 | 0.167 | 0.166 | 0.970 | 2.260 | 0.183 | 0.191 | 0.957 | 0.023 | NOT_TESTED | 48 | | +| 6v2r | open | pass | P 41 21 2 | P 41 21 2 | 0.02 | 0.999 | 1.38 | 1.60 | +13.6% | 92.6 | 11.3 | 5.2% | 2.9% | 1.000 | 24.4 | - | - | 0.205 | 0.207 | 0.945 | 0.209 | 0.230 | 0.235 | 0.889 | 12.960 | 0.262 | 0.263 | 0.996 | 0.009 | NOT_TESTED | 8 | | +| 6vww | open | pass | P 63 | P 63 | 0.20 | 1.005 | 1.99 | 2.20 | +9.7% | 99.6 | 5.6 | 16.4% | 8.9% | 0.993 | 8.0 | - | - | 0.240 | 0.118 | 0.925 | 0.243 | 0.249 | 0.178 | 1.370 | 0.670 | 0.256 | 0.249 | 1.025 | 0.004 | NOT_TESTED | 15 | | +| 6w4h | open | pass | P 31 2 1 | P 31 2 1 | 0.02 | 1.000 | 1.62 | 1.80 | +9.8% | 97.8 | 6.9 | 7.6% | 3.9% | 0.999 | 18.5 | - | - | 0.163 | 0.160 | 0.967 | 0.169 | 0.176 | 0.163 | 1.036 | 4.710 | 0.195 | 0.196 | 0.996 | -0.012 | ACCEPTED | 71 | | +| 6w75 | open | pass | P 31 2 1 | P 32 2 1 | 0.01 | 1.000 | 1.69 | 1.95 | +13.5% | 99.7 | 10.5 | 12.0% | 4.7% | 0.999 | 15.8 | - | - | 0.175 | 0.170 | 0.957 | 0.188 | 0.187 | 0.174 | 1.075 | 3.590 | 0.192 | 0.194 | 0.991 | -0.016 | ACCEPTED | 94 | P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model | +| 6wzo | open | pass | P 1 | P 1 | 0.04 | 1.000 | 1.04 | 1.42 | +26.8% | 67.0 | 3.8 | 6.2% | 3.7% | 0.999 | 16.8 | - | - | 0.174 | 0.120 | 0.959 | 0.175 | 0.175 | 0.173 | 1.013 | 2.680 | 0.178 | 0.184 | 0.965 | 0.004 | NOT_TESTED | 54 | | +| 6yqf | open | pass | P 21 21 2 | P 21 21 2 | 0.71 | 1.017 | 3.02 | 3.33 | +9.3% | 99.7 | 5.7 | 70.0% | 11.0% | 0.982 | 4.2 | - | - | 0.434 | 0.156 | 0.742 | 0.442 | 0.448 | 0.367 | 1.204 | -0.200 | 0.420 | 0.420 | 1.001 | -0.233 | NOT_TESTED | 23 | | +| 6z8o | open | pass | P 1 21 1 | P 1 21 1 | 0.63 | 1.016 | 2.21 | 2.20 | -0.4% | 90.7 | 3.6 | 17.4% | 5.8% | 0.995 | 13.9 | - | - | 0.276 | 0.069 | 0.905 | 0.281 | 0.280 | 0.275 | 1.023 | 1.240 | 0.298 | 0.286 | 1.043 | -0.045 | NOT_TESTED | 64 | | +| 6z9g | open | fail | P 1 21 1 | P 1 21 1 | 22.53 | 0.501 | 1.59 | 1.76 | +9.4% | 81.5 | 4.3 | 11.4% | 4.9% | 0.997 | 14.8 | - | - | 0.530 | 0.184 | 0.142 | 0.542 | 0.536 | 0.240 | 2.260 | 0.050 | - | - | - | - | NOT_TESTED | 94 | primitive volume ratio 0.50 | +| 6ze4 | open | pass | P 21 21 21 | P 21 21 21 | 0.59 | 1.009 | 1.30 | 1.60 | +18.9% | 96.2 | 8.0 | 20.2% | 6.7% | 0.993 | 9.3 | - | - | 0.213 | 0.079 | 0.966 | 0.218 | 0.217 | 0.202 | 1.077 | 1.790 | 0.193 | 0.185 | 1.040 | -0.020 | NOT_TESTED | 75 | | +| 6zqr | open | pass | P 4 | P 4 | 0.40 | 1.010 | 1.76 | 1.93 | +8.8% | 99.7 | 8.2 | 19.3% | 8.1% | 0.996 | 9.1 | - | - | 0.197 | 0.168 | 0.949 | 0.204 | 0.213 | 0.191 | 1.066 | 1.350 | 0.214 | 0.201 | 1.064 | -0.098 | NOT_TESTED | 17 | | +| 6zqy | open | pass | P 4 | P 4 | 0.12 | 0.997 | 1.70 | 1.85 | +8.2% | 98.8 | 7.9 | 23.3% | 5.6% | 0.994 | 12.7 | - | - | 0.213 | 0.158 | 0.948 | 0.222 | 0.227 | 0.196 | 1.131 | 1.360 | 0.220 | 0.205 | 1.071 | -0.144 | ACCEPTED | 32 | | +| 6zr0 | open | pass | P 4 | P 4 | 0.08 | 1.001 | 1.66 | 1.94 | +14.7% | 98.0 | 3.7 | 13.1% | 4.1% | 0.993 | 24.5 | - | - | 0.216 | 0.068 | 0.960 | 0.218 | 0.218 | 0.210 | 1.039 | 1.800 | 0.222 | 0.219 | 1.014 | 0.005 | ACCEPTED | 57 | | +| 7arr | open | pass | P 1 | P 1 | 0.28 | 0.993 | 0.92 | 1.10 | +16.7% | 70.3 | 3.5 | 3.7% | 3.0% | 0.999 | 18.9 | - | - | 0.154 | 0.197 | 0.964 | 0.166 | 0.165 | 0.161 | 1.033 | 0.000 | 0.203 | 0.200 | 1.016 | -0.103 | ACCEPTED | 35 | | +| 7atg | open | pass | P 21 21 21 | P 21 21 21 | 0.08 | 1.002 | 0.60 | 0.60 | +0.5% | 86.8 | 3.8 | 5.2% | 7.1% | 0.995 | 22.9 | - | - | 0.127 | 0.201 | 0.442 | 0.176 | 0.175 | 0.095 | 1.857 | 6.470 | - | - | - | -0.026 | NOT_TESTED | 28 | | +| 7bgt | open | pass | P 1 | P 1 | 0.32 | 0.991 | 1.78 | 1.93 | +7.6% | 97.9 | 2.2 | 13.7% | 5.6% | 0.993 | 17.4 | - | - | 0.195 | 0.118 | 0.955 | 0.207 | 0.206 | 0.212 | 0.977 | 0.230 | 0.225 | 0.226 | 0.999 | -0.000 | NOT_TESTED | 12 | | +| 7bgu | open | pass | P 1 | P 1 | 0.17 | 0.996 | 2.30 | 2.43 | +5.6% | 94.7 | 2.9 | 12.4% | 4.5% | 0.937 | 10.0 | - | - | 0.286 | 0.108 | 0.801 | 0.304 | 0.304 | 0.236 | 1.288 | 0.260 | 0.268 | 0.235 | 1.140 | -0.246 | NOT_TESTED | 41 | | +| 7brr | open | pass | P 1 21 1 | P 1 21 1 | 0.09 | 0.998 | 1.24 | 1.35 | +7.9% | 96.8 | 6.3 | 6.6% | 3.8% | 0.999 | 16.6 | - | - | 0.192 | 0.095 | 0.963 | 0.191 | 0.196 | 0.197 | 0.966 | 3.070 | 0.205 | 0.209 | 0.981 | -0.041 | NOT_TESTED | 17 | | +| 7dkp | open | pass | P 1 21 1 | P 1 21 1 | 0.06 | 1.002 | 1.18 | 1.45 | +18.3% | 66.3 | 7.3 | 11.9% | 4.6% | 0.998 | 26.1 | - | - | 0.158 | 0.080 | 0.969 | 0.161 | 0.163 | 0.160 | 1.001 | 2.830 | 0.170 | 0.172 | 0.984 | 0.020 | NOT_TESTED | 29 | | +| 7k1l | open | unscored | P 6 | P 63 | 0.27 | 0.995 | 1.91 | 2.25 | +15.1% | 98.9 | 6.5 | 16.1% | 8.3% | 0.994 | 9.9 | - | - | 0.261 | 0.098 | 0.909 | 0.268 | 0.268 | 0.192 | 1.401 | 0.780 | 0.271 | 0.266 | 1.016 | 0.093 | NOT_TESTED | 20 | P 6 vs reference P 63: the screw along c is undeterminable from these data (offered P 61 / P 65 / P 62 / P 64 / P 63) | +| 7kcn | open | pass | P 41 2 2 | P 41 2 2 | 0.07 | 1.000 | 1.39 | 1.46 | +4.7% | 92.8 | 20.7 | 7.8% | 7.4% | 0.999 | 11.7 | - | - | 0.172 | 0.154 | 0.961 | 0.189 | 0.190 | 0.182 | 1.041 | 8.440 | 0.209 | 0.198 | 1.055 | -0.020 | NOT_TESTED | 33 | | +| 7l6j | open | pass | I 41 3 2 | I 41 3 2 | 0.00 | 1.000 | 1.51 | 1.78 | +15.3% | 99.7 | 29.0 | 20.9% | 7.5% | 0.999 | 10.5 | - | - | 0.172 | 0.179 | 0.963 | 0.187 | 0.188 | 0.154 | 1.211 | 29.130 | 0.181 | 0.178 | 1.013 | -0.001 | NOT_TESTED | 107 | | +| 7l84 | open | pass | P 41 21 2 | P 43 21 2 | 0.06 | 0.999 | 1.70 | 1.70 | +0.0% | 91.6 | 33.5 | 9.1% | 13.0% | 0.999 | 11.4 | - | - | 0.150 | 0.090 | 0.911 | 0.169 | 0.163 | 0.162 | 1.040 | 17.580 | 0.179 | 0.183 | 0.976 | -0.008 | ACCEPTED | 17 | P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model | +| 7mzt | open | unscored | P 21 21 21 | P 21 21 2 | 0.56 | 0.998 | 3.12 | 4.07 | +23.4% | 99.7 | 10.6 | 366.1% | 33.8% | 0.971 | 4.1 | - | - | 0.414 | 0.104 | 0.556 | 0.426 | 0.418 | 0.372 | 1.144 | -0.040 | 0.394 | 0.398 | 0.991 | -0.020 | ACCEPTED | 33 | P 21 21 21 vs reference P 21 21 2: the screw along c is undeterminable from these data (offered P 21 21 2) | +| 7n0i | open | pass | P 21 21 21 | P 21 21 21 | 0.22 | 0.996 | 1.68 | 2.20 | +23.8% | 99.7 | 6.6 | 14.5% | 5.7% | 0.998 | 10.9 | - | - | 0.267 | 0.259 | 0.916 | 0.295 | 0.294 | 0.271 | 1.085 | 0.530 | 0.296 | 0.284 | 1.041 | 0.051 | NOT_TESTED | 27 | | +| 7n2s | open | pass | P 1 21 1 | P 1 21 1 | 0.07 | 1.000 | 2.56 | 2.37 | -8.1% | 97.6 | 3.4 | 56.7% | 8.7% | 0.917 | 9.6 | - | - | 0.299 | 0.111 | 0.822 | 0.312 | 0.308 | 0.311 | 1.002 | 0.340 | 0.315 | 0.325 | 0.968 | -0.018 | NOT_TESTED | 12 | | +| 7orr | open | pass | I 2 3 | I 21 3 | 0.03 | 0.999 | 1.62 | 1.79 | +9.3% | 99.6 | 15.7 | 6.7% | 4.0% | 1.000 | 26.5 | - | - | 0.183 | 0.164 | 0.965 | 0.196 | 0.194 | 0.184 | 1.062 | 5.080 | 0.184 | 0.181 | 1.020 | 0.056 | NOT_TESTED | 20 | I 2 3 vs reference I 21 3 (UNDECIDABLE from intensities) | +| 7os3 | open | pass | P 21 21 21 | P 21 21 21 | 0.10 | 1.002 | 1.96 | 2.18 | +10.1% | 83.6 | 11.4 | 8.4% | 3.8% | 0.999 | 24.4 | - | - | 0.181 | 0.153 | 0.955 | 0.194 | 0.193 | 0.224 | 0.866 | 6.150 | 0.233 | 0.236 | 0.985 | -0.054 | NOT_TESTED | 33 | | +| 7ou1 | open | pass | P 1 21 1 | P 1 21 1 | 0.04 | 0.999 | 1.40 | 1.65 | +15.0% | 87.9 | 3.4 | 15.9% | 7.7% | 0.991 | 9.0 | - | - | 0.202 | 0.098 | 0.955 | 0.209 | 0.210 | 0.222 | 0.944 | 1.520 | 0.229 | 0.237 | 0.966 | 0.085 | NOT_TESTED | 22 | | +| 7ph1 | open | pass | I 2 2 2 | I 2 2 2 | 0.12 | 0.998 | 1.08 | 1.18 | +8.1% | 99.7 | 7.1 | 11.3% | 5.4% | 0.998 | 16.0 | - | - | 0.155 | 0.158 | 0.968 | 0.174 | 0.172 | 0.166 | 1.051 | 3.210 | 0.207 | 0.209 | 0.991 | 0.050 | NOT_TESTED | 77 | | +| 7pq7 | open | pass | C 1 2 1 | C 1 2 1 | 0.22 | 0.993 | 1.37 | 1.55 | +11.4% | 98.1 | 3.7 | 6.5% | 4.3% | 0.998 | 14.0 | - | - | 0.188 | 0.193 | 0.946 | 0.215 | 0.209 | 0.200 | 1.074 | 2.320 | 0.230 | 0.229 | 1.007 | 0.041 | NOT_TESTED | 14 | | +| 7q6j | open | pass | P 21 21 21 | P 21 21 21 | 0.25 | 0.998 | 1.98 | 2.20 | +9.9% | 99.7 | 7.3 | 14.6% | 5.4% | 0.996 | 11.0 | - | - | 0.224 | 0.137 | 0.945 | 0.236 | 0.240 | 0.236 | 1.000 | 1.620 | 0.258 | 0.260 | 0.996 | -0.019 | NOT_TESTED | 126 | | +| 7qij | open | pass | P 21 21 21 | P 21 21 21 | 0.36 | 0.993 | 3.59 | 4.10 | +12.4% | 99.7 | 6.8 | 24.1% | 5.6% | 0.996 | 9.4 | - | - | 0.357 | 0.065 | 0.831 | 0.364 | 0.362 | 0.325 | 1.121 | 0.180 | 0.363 | 0.364 | 0.996 | 0.061 | NOT_TESTED | 92 | | +| 7qis | open | pass | P 61 | P 61 | 0.05 | 0.999 | 1.75 | 1.83 | +4.3% | 99.6 | 9.1 | 20.9% | 6.2% | 0.997 | 17.4 | - | - | 0.166 | 0.195 | 0.965 | 0.192 | 0.191 | 0.189 | 1.019 | 1.410 | 0.214 | 0.209 | 1.024 | 0.064 | NOT_TESTED | 31 | | +| 7raa | open | pass | P 41 21 2 | P 43 21 2 | 0.04 | 0.999 | 2.58 | 2.69 | +4.1% | 98.7 | 35.1 | 17.2% | 5.5% | 1.000 | 11.0 | - | - | 0.296 | 0.217 | 0.902 | 0.335 | 0.304 | 0.294 | 1.140 | 0.530 | 0.305 | 0.308 | 0.990 | -0.036 | ACCEPTED | 182 | P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model | +| 7ris | open | pass | P 31 2 1 | P 32 2 1 | 0.08 | 1.000 | 1.51 | 1.72 | +12.0% | 96.3 | 17.9 | 10.0% | 3.3% | 1.000 | 31.0 | - | - | 0.188 | 0.047 | 0.961 | 0.189 | 0.190 | 0.201 | 0.942 | 2.260 | 0.206 | 0.209 | 0.987 | -0.004 | ACCEPTED | 24 | P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model | +| 7rji | open | pass | R 3 2:H | H 3 2 | 0.26 | 1.008 | 1.48 | 1.71 | +13.3% | 99.3 | 28.9 | 13.1% | 8.2% | 0.999 | 8.4 | - | - | 0.206 | 0.119 | 0.947 | 0.198 | 0.212 | 0.224 | 0.883 | 6.960 | 0.227 | 0.232 | 0.977 | 0.087 | NOT_TESTED | 27 | | +| 7t5t | open | pass | P 42 21 2 | P 42 21 2 | 0.04 | 0.999 | 1.24 | 1.35 | +8.2% | 98.4 | 12.6 | 6.5% | 4.4% | 0.999 | 16.5 | - | - | 0.163 | 0.246 | 0.960 | 0.192 | 0.188 | 0.168 | 1.145 | 11.230 | 0.193 | 0.189 | 1.017 | 0.025 | NOT_TESTED | 60 | | +| 7tcd | open | pass | C 1 2 1 | C 1 2 1 | 0.20 | 1.004 | 1.65 | 1.70 | +2.8% | 69.3 | 7.1 | 8.6% | 4.3% | 0.999 | 18.4 | - | - | 0.191 | 0.195 | 0.956 | 0.211 | 0.208 | 0.252 | 0.835 | 1.170 | 0.248 | 0.257 | 0.968 | -0.057 | NOT_TESTED | 28 | | +| 7yzx | open | pass | P 63 2 2 | P 63 2 2 | 0.25 | 0.994 | 1.88 | 1.90 | +1.3% | 99.7 | 14.1 | 20.5% | 6.2% | 0.998 | 10.4 | - | - | 0.212 | 0.106 | 0.951 | 0.215 | 0.215 | 0.229 | 0.938 | 1.380 | 0.236 | 0.230 | 1.025 | 0.013 | NOT_TESTED | 49 | | +| 8a1a | open | pass | P 61 | P 65 | 0.42 | 0.987 | 1.93 | 2.05 | +5.9% | 99.7 | 41.8 | 52.0% | 8.1% | 0.997 | 18.8 | - | - | 0.177 | 0.029 | 0.962 | 0.179 | 0.178 | 0.185 | 0.968 | 1.820 | 0.178 | 0.179 | 0.991 | -0.036 | ACCEPTED | 94 | P 61 vs reference P 65 (hand only (needs anomalous)); labelled P 65 from the model | +| 8agq | open | pass | C 1 2 1 | C 1 2 1 | 0.30 | 0.991 | 0.97 | 1.09 | +11.6% | 95.6 | 6.0 | 9.2% | 4.3% | 0.999 | 14.1 | - | - | 0.181 | 0.212 | 0.945 | 0.204 | 0.203 | 0.149 | 1.373 | 6.460 | 0.201 | 0.174 | 1.153 | -0.022 | NOT_TESTED | 26 | | +| 8c3e | open | fail | P 6 2 2 | P 31 2 1 | 0.24 | 0.994 | 1.79 | 2.10 | +14.5% | 98.0 | 7.0 | 29.4% | 11.9% | 0.980 | 5.8 | - | - | 0.359 | 0.290 | 0.758 | 0.400 | 0.388 | 0.249 | 1.607 | 0.960 | 0.381 | 0.370 | 1.031 | -0.076 | ACCEPTED | 5 | P 6 2 2 vs reference P 31 2 1 | +| 8dqb | open | pass | I 2 3 | I 2 3 | 0.08 | 0.998 | 2.05 | 2.50 | +18.1% | 97.5 | 10.3 | 14.3% | 4.4% | 0.996 | 22.1 | - | - | 0.230 | 0.112 | 0.939 | 0.237 | 0.243 | 0.239 | 0.990 | 3.080 | 0.254 | 0.252 | 1.005 | 0.006 | NOT_TESTED | 14 | | +| 8dyz | open | pass | P 41 21 2 | P 43 21 2 | 0.08 | 0.999 | 1.14 | 1.27 | +10.4% | 64.2 | 3.8 | 3.1% | 2.4% | 0.999 | 37.8 | - | - | 0.120 | 0.046 | 0.973 | 0.121 | 0.122 | 0.133 | 0.907 | 4.220 | 0.171 | 0.178 | 0.958 | -0.050 | ACCEPTED | 12 | P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model | +| 8dz7 | open | pass | P 21 21 21 | P 21 21 21 | 0.08 | 0.998 | 1.19 | 1.34 | +11.2% | 37.2 | 3.2 | 3.0% | 2.7% | 0.999 | 35.1 | - | - | 0.118 | 0.068 | 0.968 | 0.128 | 0.122 | 0.136 | 0.942 | 5.140 | 0.172 | 0.170 | 1.009 | -0.008 | NOT_TESTED | 10 | | +| 8egn | open | pass | P 21 21 21 | P 21 21 21 | 0.07 | 1.001 | 1.63 | 1.95 | +16.6% | 91.2 | 5.4 | 6.3% | 3.3% | 0.999 | 22.9 | - | - | 0.197 | 0.154 | 0.953 | 0.215 | 0.205 | 0.219 | 0.979 | 2.530 | 0.230 | 0.231 | 0.997 | 0.015 | NOT_TESTED | 14 | | +| 8iya | open | pass | C 1 2 1 | C 1 2 1 | 0.16 | 0.996 | 1.91 | 2.43 | +21.3% | 97.9 | 5.5 | 21.6% | 11.5% | 0.989 | 5.8 | - | - | 0.247 | 0.033 | 0.946 | 0.234 | 0.249 | 0.247 | 0.948 | 0.400 | 0.254 | 0.254 | 0.998 | 0.016 | NOT_TESTED | 8 | | +| 8k1g | open | pass | I 4 2 2 | I 4 2 2 | 0.82 | 1.019 | 1.63 | 2.09 | +22.1% | 99.7 | 20.6 | 25.6% | 6.8% | 0.999 | 11.7 | - | - | 0.206 | 0.200 | 0.956 | 0.227 | 0.226 | 0.207 | 1.093 | 2.260 | 0.209 | 0.215 | 0.974 | -0.107 | NOT_TESTED | 43 | | +| 8oic | open | pass | P 1 | P 1 | 0.02 | 1.000 | 2.34 | 2.80 | +16.4% | 98.2 | 3.6 | 24.4% | 4.6% | 0.988 | 20.0 | - | - | 0.238 | 0.112 | 0.933 | 0.248 | 0.246 | 0.251 | 0.988 | 0.410 | 0.249 | 0.260 | 0.956 | 0.056 | ACCEPTED | 42 | | +| 8owm | open | pass | P 1 | P 1 | 0.02 | 1.000 | 1.48 | 1.70 | +13.2% | 96.3 | 3.6 | 10.2% | 3.4% | 0.998 | 22.2 | - | - | 0.164 | 0.127 | 0.971 | 0.177 | 0.176 | 0.172 | 1.031 | 1.370 | 0.183 | 0.186 | 0.981 | 0.046 | NOT_TESTED | 67 | | +| 8pqd | open | pass | P 21 21 21 | P 21 21 21 | 0.03 | 1.000 | 1.30 | 1.50 | +13.0% | 94.5 | 13.5 | 9.0% | 5.5% | 0.999 | 15.0 | - | - | 0.184 | 0.216 | 0.961 | 0.204 | 0.200 | 0.194 | 1.051 | 3.570 | 0.209 | 0.210 | 0.995 | 0.036 | NOT_TESTED | 63 | | +| 8qaw | open | pass | R 3:H | H 3 | 0.04 | 0.999 | 1.30 | 1.55 | +16.3% | 99.4 | 9.9 | 12.2% | 7.6% | 0.998 | 11.0 | - | - | 0.143 | 0.163 | 0.958 | 0.153 | 0.157 | 0.161 | 0.950 | 11.400 | 0.175 | 0.183 | 0.957 | 0.022 | NOT_TESTED | 401 | | +| 8qj5 | open | pass | P 1 21 1 | P 1 21 1 | 0.45 | 0.987 | 1.29 | 1.63 | +20.9% | 98.1 | 6.1 | 17.7% | 7.7% | 0.996 | 8.4 | - | - | 0.192 | 0.121 | 0.941 | 0.206 | 0.203 | 0.190 | 1.083 | 1.500 | 0.204 | 0.193 | 1.058 | 0.029 | NOT_TESTED | 20 | | +| 8qq7 | open | pass | P 62 2 2 | P 64 2 2 | 0.65 | 1.017 | 3.16 | 3.62 | +12.7% | 99.4 | 24.9 | 19.0% | 10.5% | 0.997 | 6.4 | - | - | 0.432 | 0.370 | 0.330 | 0.429 | 0.436 | 0.320 | 1.340 | 0.750 | 0.351 | 0.330 | 1.065 | -0.107 | ACCEPTED | 13 | P 62 2 2 vs reference P 64 2 2 (hand only (needs anomalous)); labelled P 64 2 2 from the model | +| 8r5r | open | pass | P 21 21 21 | P 21 21 21 | 0.04 | 1.001 | 2.80 | 3.08 | +9.0% | 99.7 | 10.1 | 22.7% | 4.7% | 0.998 | 19.3 | - | - | 0.263 | 0.144 | 0.884 | 0.284 | 0.271 | 0.252 | 1.127 | 0.430 | 0.253 | 0.252 | 1.003 | -0.024 | NOT_TESTED | 30 | | +| 8rud | open | pass | P 1 21 1 | P 1 21 1 | 0.40 | 0.989 | 1.57 | 2.10 | +25.1% | 99.7 | 6.6 | 37.8% | 6.3% | 0.991 | 11.9 | - | - | 0.253 | 0.081 | 0.924 | 0.261 | 0.258 | 0.262 | 0.997 | 0.690 | 0.242 | 0.262 | 0.927 | 0.052 | NOT_TESTED | 236 | | +| 8s38 | open | pass | I 2 2 2 | I 21 21 21 | 0.16 | 0.996 | 1.63 | 1.89 | +13.8% | 99.5 | 6.7 | 8.6% | 3.5% | 0.999 | 22.3 | - | - | 0.180 | 0.133 | 0.962 | 0.185 | 0.188 | 0.187 | 0.990 | 2.280 | 0.204 | 0.204 | 0.999 | 0.030 | NOT_TESTED | 125 | I 2 2 2 vs reference I 21 21 21 (UNDECIDABLE from intensities) | +| 8sa8 | open | pass | I 1 2 1 | I 1 2 1 | 0.02 | 1.000 | 1.10 | 1.30 | +15.0% | 85.7 | 7.2 | 11.7% | 3.7% | 0.999 | 22.0 | - | - | 0.155 | 0.144 | 0.972 | 0.169 | 0.169 | 0.152 | 1.109 | 3.790 | 0.181 | 0.185 | 0.977 | 0.051 | ACCEPTED | 94 | | +| 8sqo | open | pass | P 4 3 2 | P 4 3 2 | 0.13 | 0.996 | 1.32 | 1.55 | +14.7% | 99.7 | 70.4 | 23.4% | 6.1% | 1.000 | 14.2 | - | - | 0.183 | 0.180 | 0.957 | 0.206 | 0.199 | 0.177 | 1.159 | 7.020 | 0.190 | 0.193 | 0.987 | 0.033 | NOT_TESTED | 72 | | +| 8sqq | open | pass | F 4 3 2 | F 4 3 2 | 0.17 | 0.995 | 1.90 | 2.25 | +15.5% | 99.7 | 39.2 | 25.1% | 5.4% | 0.999 | 17.6 | - | - | 0.211 | 0.105 | 0.963 | 0.206 | 0.217 | 0.245 | 0.839 | 3.100 | 0.242 | 0.258 | 0.937 | 0.018 | NOT_TESTED | 63 | | +| 8sqt | open | pass | F 4 3 2 | F 4 3 2 | 0.11 | 0.997 | 1.88 | 2.20 | +14.7% | 99.7 | 21.6 | 20.1% | 3.5% | 0.999 | 29.9 | - | - | 0.223 | 0.109 | 0.967 | 0.226 | 0.232 | 0.262 | 0.860 | 2.080 | 0.269 | 0.257 | 1.044 | 0.004 | NOT_TESTED | 28 | | +| 8t7r | open | pass | C 1 2 1 | C 1 2 1 | 0.28 | 0.991 | 3.23 | 3.84 | +15.8% | 99.1 | 4.0 | 31.2% | 8.3% | 0.984 | 7.8 | - | - | 0.283 | 0.078 | 0.755 | 0.285 | 0.290 | 0.263 | 1.086 | 0.430 | 0.282 | 0.283 | 0.998 | 0.054 | NOT_TESTED | 63 | | +| 8tha | open | pass | P 62 | P 64 | 0.17 | 0.995 | 1.33 | 1.68 | +21.1% | 99.1 | 19.0 | 16.3% | 3.2% | 0.999 | 27.7 | - | - | 0.212 | 0.091 | 0.961 | 0.220 | 0.217 | 0.215 | 1.026 | 4.370 | 0.215 | 0.220 | 0.980 | 0.044 | ACCEPTED | 19 | P 62 vs reference P 64 (hand only (needs anomalous)); labelled P 64 from the model | +| 8tyy | open | pass | F 4 3 2 | F 4 3 2 | 0.06 | 1.002 | 1.28 | 1.68 | +23.6% | 98.0 | 38.5 | 16.6% | 5.2% | 0.999 | 16.3 | - | - | 0.160 | 0.151 | 0.970 | 0.180 | 0.175 | 0.163 | 1.104 | 13.520 | 0.161 | 0.162 | 0.989 | 0.002 | NOT_TESTED | 184 | | +| 8u0i | open | pass | P 41 21 2 | P 43 21 2 | 0.06 | 1.001 | 1.38 | 1.54 | +10.6% | 98.1 | 11.0 | 8.1% | 3.4% | 0.999 | 17.0 | - | - | 0.179 | 0.054 | 0.964 | 0.183 | 0.182 | 0.210 | 0.871 | 3.370 | 0.212 | 0.210 | 1.008 | 0.025 | ACCEPTED | 18 | P 41 21 2 vs reference P 43 21 2 (hand only (needs anomalous)); labelled P 43 21 2 from the model | +| 8v2t | open | pass | P 42 21 2 | P 42 21 2 | 0.24 | 1.005 | 1.17 | 1.40 | +16.8% | 90.9 | 9.7 | 8.8% | 5.1% | 0.999 | 11.9 | - | - | 0.166 | 0.213 | 0.969 | 0.181 | 0.180 | 0.163 | 1.110 | 10.860 | 0.161 | 0.165 | 0.975 | 0.014 | NOT_TESTED | 31 | | +| 8v4j | open | pass | P 42 21 2 | P 42 21 2 | 0.04 | 1.000 | 1.10 | 1.31 | +16.0% | 89.5 | 11.4 | 5.4% | 3.1% | 1.000 | 22.1 | - | - | 0.170 | 0.112 | 0.965 | 0.177 | 0.176 | 0.172 | 1.026 | 12.350 | 0.169 | 0.172 | 0.983 | 0.008 | NOT_TESTED | 110 | | +| 8v4o | open | pass | P 61 2 2 | P 61 2 2 | 0.06 | 1.001 | 2.10 | 2.70 | +22.2% | 99.7 | 20.4 | 31.9% | 6.0% | 0.998 | 15.5 | - | - | 0.236 | 0.157 | 0.929 | 0.253 | 0.254 | 0.240 | 1.056 | 0.960 | 0.247 | 0.247 | 0.997 | 0.058 | NOT_TESTED | 63 | | +| 8xbp | open | pass | C 1 2 1 | C 1 2 1 | 0.15 | 0.999 | 1.66 | 1.99 | +16.7% | 81.0 | 6.6 | 11.2% | 4.1% | 0.999 | 18.6 | - | - | 0.313 | 0.177 | 0.892 | 0.329 | 0.320 | 0.288 | 1.141 | 1.210 | 0.329 | 0.310 | 1.063 | 0.245 | NOT_TESTED | 21 | | +| 8xte | open | pass | P 31 2 1 | P 32 | 0.06 | 1.001 | 1.64 | 1.99 | +17.8% | 99.5 | 9.4 | 25.2% | 7.7% | 0.996 | 12.4 | - | - | 0.275 | 0.121 | 0.913 | 0.287 | 0.284 | 0.219 | 1.307 | 0.910 | 0.290 | 0.290 | 0.999 | 0.043 | NOT_TESTED | 34 | P 31 2 1 vs reference P 32: accepted alternative P 31 2 1 - a knife-edge this battery does not decide | +| 8xtf | open | pass | R 3 2:H | H 3 2 | 0.22 | 1.006 | 1.84 | 2.13 | +13.7% | 99.6 | 18.9 | 73.3% | 16.7% | 0.989 | 7.6 | - | - | 0.191 | 0.044 | 0.964 | 0.186 | 0.193 | 0.210 | 0.884 | 1.400 | 0.208 | 0.218 | 0.956 | -0.003 | NOT_TESTED | 31 | | +| 8xtg | open | pass | P 31 2 1 | P 32 | 0.05 | 1.002 | 1.54 | 2.00 | +22.8% | 99.7 | 9.9 | 27.3% | 10.7% | 0.994 | 7.3 | - | - | 0.275 | 0.134 | 0.903 | 0.284 | 0.284 | 0.211 | 1.345 | 0.880 | 0.280 | 0.280 | 1.001 | 0.037 | ACCEPTED | 50 | P 31 2 1 vs reference P 32: accepted alternative P 31 2 1 - a knife-edge this battery does not decide | +| 8y74 | open | pass | C 1 2 1 | C 1 2 1 | 0.34 | 0.992 | 1.68 | 1.90 | +11.4% | 97.6 | 6.2 | 13.0% | 7.8% | 0.997 | 8.6 | - | - | 0.214 | 0.104 | 0.939 | 0.225 | 0.220 | 0.240 | 0.935 | 1.230 | 0.257 | 0.267 | 0.962 | -0.032 | NOT_TESTED | 11 | | +| 8ys9 | open | pass | P 21 21 21 | P 21 21 21 | 0.16 | 0.998 | 1.31 | 1.46 | +10.5% | 99.7 | 13.4 | 13.3% | 4.6% | 0.999 | 15.4 | - | - | 0.176 | 0.104 | 0.962 | 0.188 | 0.182 | 0.197 | 0.955 | 3.110 | 0.209 | 0.213 | 0.982 | 0.096 | NOT_TESTED | 29 | | +| 9b22 | open | pass | P 1 21 1 | P 1 21 1 | 0.07 | 1.002 | 1.14 | 1.30 | +12.1% | 77.1 | 6.1 | 6.2% | 3.5% | 0.999 | 16.4 | - | - | 0.159 | 0.121 | 0.968 | 0.166 | 0.164 | 0.164 | 1.014 | 0.160 | 0.196 | 0.194 | 1.007 | -0.044 | NOT_TESTED | 17 | | +| 9bn8 | open | pass | P 41 | P 41 | 0.09 | 0.997 | 1.21 | 1.35 | +10.6% | 95.4 | 11.6 | 8.1% | 3.8% | 1.000 | 19.8 | - | - | 0.151 | 0.104 | 0.967 | 0.159 | 0.157 | 0.158 | 1.011 | 3.770 | 0.177 | 0.182 | 0.975 | 0.019 | ACCEPTED | 27 | | +| 9c18 | open | pass | P 1 | P 1 | 0.61 | 0.985 | 1.71 | 1.90 | +9.8% | 97.4 | 3.6 | 27.3% | 8.5% | 0.986 | 10.8 | - | - | 0.221 | 0.067 | 0.947 | 0.221 | 0.224 | 0.229 | 0.965 | 0.710 | 0.224 | 0.234 | 0.957 | 0.025 | ACCEPTED | 11 | | +| 9chw | open | pass | P 61 | P 61 | 0.04 | 1.001 | 1.60 | 2.16 | +25.9% | 72.5 | 3.1 | 5.8% | 3.3% | 0.998 | 21.3 | - | - | 0.180 | 0.140 | 0.962 | 0.184 | 0.186 | 0.210 | 0.879 | 1.730 | 0.216 | 0.214 | 1.006 | -0.008 | NOT_TESTED | 79 | | +| 9crw | open | pass | P 1 21 1 | P 1 21 1 | 0.23 | 0.994 | 2.28 | 2.49 | +8.4% | 97.2 | 7.0 | 8.7% | 3.7% | 0.999 | 15.8 | - | - | 0.247 | 0.132 | 0.955 | 0.262 | 0.260 | 0.278 | 0.943 | 0.690 | 0.296 | 0.293 | 1.010 | -0.014 | NOT_TESTED | 16 | | +| 9e2t | open | pass | P 1 | P 1 | 0.06 | 1.001 | 2.29 | 2.28 | -0.4% | 91.6 | 5.7 | 29.1% | 7.3% | 0.992 | 6.8 | - | - | 0.230 | 0.047 | 0.937 | 0.232 | 0.233 | 0.242 | 0.958 | 0.420 | 0.254 | 0.259 | 0.980 | -0.019 | NOT_TESTED | 67 | | +| 9ea5 | open | pass | P 1 21 1 | P 1 21 1 | 0.85 | 0.998 | 1.64 | 2.00 | +17.9% | 90.9 | 6.7 | 17.9% | 4.2% | 0.997 | 26.7 | - | - | 0.196 | 0.111 | 0.954 | 0.202 | 0.204 | 0.207 | 0.976 | 1.200 | 0.212 | 0.215 | 0.986 | 0.040 | ACCEPTED | 94 | | +| 9fcf | open | pass | P 4 | P 4 | 0.04 | 1.001 | 1.75 | 2.36 | +25.8% | 99.6 | 13.1 | 34.5% | 10.0% | 0.994 | 7.0 | - | - | 0.292 | 0.064 | 0.923 | 0.295 | 0.295 | 0.257 | 1.147 | 0.850 | 0.269 | 0.276 | 0.976 | 0.022 | ACCEPTED | 146 | | +| 9fcg | open | pass | P 4 | P 4 | 0.07 | 0.999 | 1.38 | 1.54 | +10.2% | 89.9 | 11.3 | 14.1% | 8.3% | 0.997 | 9.7 | - | - | 0.180 | 0.111 | 0.956 | 0.189 | 0.191 | 0.198 | 0.952 | 2.420 | 0.202 | 0.201 | 1.004 | 0.062 | ACCEPTED | 48 | | +| 9fhc | open | pass | I 2 3 | I 2 3 | 0.25 | 0.993 | 1.92 | 2.20 | +12.9% | 95.2 | 18.4 | 17.0% | 4.7% | 0.998 | 12.3 | - | - | 0.239 | 0.072 | 0.917 | 0.244 | 0.244 | 0.238 | 1.024 | 1.200 | 0.250 | 0.240 | 1.042 | -0.024 | ACCEPTED | 107 | | +| 9gdj | open | pass | P 41 21 2 | P 41 21 2 | 0.16 | 0.996 | 1.40 | 1.47 | +5.1% | 99.7 | 13.1 | 11.2% | 6.1% | 0.999 | 12.9 | - | - | 0.154 | 0.199 | 0.968 | 0.172 | 0.176 | 0.175 | 0.983 | 2.890 | 0.199 | 0.195 | 1.021 | -0.023 | NOT_TESTED | 289 | | +| 9gjx | open | pass | P 1 21 1 | P 1 21 1 | 0.13 | 0.997 | 2.06 | 2.40 | +14.1% | 96.3 | 6.7 | 16.3% | 3.1% | 0.997 | 40.1 | - | - | 0.193 | 0.146 | 0.952 | 0.224 | 0.216 | 0.219 | 1.020 | 0.840 | 0.234 | 0.227 | 1.033 | -0.019 | NOT_TESTED | 28 | | +| 9gqg | open | pass | P 31 2 1 | P 32 2 1 | 0.01 | 1.000 | 1.81 | 2.00 | +9.6% | 99.6 | 7.8 | 11.4% | 6.0% | 0.998 | 12.8 | - | - | 0.232 | 0.116 | 0.912 | 0.243 | 0.244 | 0.261 | 0.932 | 1.790 | 0.276 | 0.267 | 1.036 | 0.014 | ACCEPTED | 57 | P 31 2 1 vs reference P 32 2 1 (hand only (needs anomalous)); labelled P 32 2 1 from the model | +| 9h0q | open | pass | R 3 2:H | H 3 2 | 0.41 | 0.990 | 2.10 | 2.55 | +17.6% | 99.7 | 10.6 | 17.4% | 4.5% | 0.998 | 18.6 | - | - | 0.205 | 0.111 | 0.945 | 0.214 | 0.219 | 0.222 | 0.965 | 1.100 | 0.231 | 0.236 | 0.980 | -0.016 | NOT_TESTED | 44 | | +| 9hnc | open | fail | P 1 21 1 | P 1 2 1 | 0.09 | 0.999 | 1.63 | 1.88 | +13.0% | 90.9 | 7.0 | 12.8% | 5.3% | 0.998 | 13.6 | - | - | 0.246 | 0.149 | 0.948 | 0.255 | 0.254 | 0.211 | 1.208 | 0.880 | 0.227 | 0.246 | 0.923 | -0.237 | NOT_TESTED | 66 | P 1 21 1 vs reference P 1 2 1 | +| 9hs7 | open | pass | P 61 | P 65 | 0.08 | 0.999 | 1.70 | 1.70 | +0.2% | 99.7 | 10.2 | 11.7% | 5.0% | 0.999 | 13.3 | - | - | 0.213 | 0.206 | 0.965 | 0.236 | 0.229 | 0.239 | 0.988 | 1.260 | 0.244 | 0.255 | 0.958 | -0.083 | ACCEPTED | 26 | P 61 vs reference P 65 (hand only (needs anomalous)); labelled P 65 from the model | +| 9i0a | open | pass | P 21 21 2 | P 21 21 2 | 0.43 | 1.008 | 1.81 | 2.22 | +18.6% | 98.9 | 12.7 | 16.9% | 5.1% | 0.999 | 14.1 | - | - | 0.224 | 0.116 | 0.955 | 0.233 | 0.235 | 0.237 | 0.984 | 0.950 | 0.255 | 0.259 | 0.984 | 0.164 | NOT_TESTED | 59 | | +| 9i80 | open | pass | P 41 | P 41 | 0.07 | 1.002 | 1.59 | 1.95 | +18.2% | 99.7 | 13.0 | 25.1% | 10.7% | 0.992 | 6.8 | - | - | 0.219 | 0.117 | 0.905 | 0.221 | 0.225 | 0.203 | 1.092 | 1.890 | 0.212 | 0.207 | 1.024 | 0.002 | NOT_TESTED | 116 | | +| 9ig7 | open | pass | P 21 21 2 | P 21 21 2 | 0.17 | 0.996 | 2.02 | 2.60 | +22.3% | 99.7 | 13.7 | 21.7% | 7.3% | 0.991 | 11.1 | - | - | 0.234 | 0.081 | 0.931 | 0.249 | 0.239 | 0.259 | 0.963 | 0.800 | 0.286 | 0.277 | 1.030 | 0.015 | NOT_TESTED | 82 | | +| 9ih9 | open | pass | C 1 2 1 | C 1 2 1 | 0.21 | 0.997 | 1.40 | 1.70 | +17.6% | 96.0 | 2.4 | 11.2% | 5.0% | 0.996 | 12.7 | - | - | 0.199 | 0.075 | 0.952 | 0.207 | 0.204 | 0.203 | 1.020 | 1.020 | 0.204 | 0.202 | 1.007 | -0.021 | NOT_TESTED | 20 | | +| 9jq9 | open | pass | P 21 21 21 | P 21 21 21 | 0.12 | 0.997 | 1.65 | 1.90 | +13.2% | 84.9 | 10.2 | 6.9% | 4.3% | 0.999 | 19.5 | - | - | 0.212 | 0.151 | 0.936 | 0.242 | 0.223 | 0.237 | 1.022 | 4.820 | 0.241 | 0.239 | 1.005 | -0.011 | NOT_TESTED | 6 | | +| 9jzo | open | pass | P 1 | P 1 | 0.22 | 0.996 | 1.15 | 1.40 | +18.0% | 61.1 | 3.3 | 5.8% | 4.5% | 0.997 | 8.1 | - | - | 0.179 | 0.047 | 0.960 | 0.176 | 0.180 | 0.195 | 0.906 | 4.390 | 0.203 | 0.194 | 1.045 | 0.018 | NOT_TESTED | 11 | | +| 9khr | open | pass | P 21 21 21 | P 21 21 21 | 0.12 | 0.997 | 1.38 | 2.00 | +30.9% | 96.9 | 6.5 | 18.8% | 11.3% | 0.994 | 9.5 | - | - | 0.244 | 0.116 | 0.942 | 0.259 | 0.254 | 0.235 | 1.104 | 2.020 | 0.245 | 0.249 | 0.984 | 0.042 | NOT_TESTED | 13 | | +| 9lxl | open | pass | P 41 21 2 | P 41 21 2 | 0.49 | 1.013 | 2.06 | 2.19 | +6.1% | 99.7 | 14.7 | 41.9% | 10.2% | 0.996 | 6.2 | - | - | 0.306 | 0.095 | 0.896 | 0.321 | 0.304 | 0.234 | 1.371 | 0.690 | 0.302 | 0.286 | 1.058 | -0.270 | NOT_TESTED | 60 | | +| 9mh4 | open | pass | P 21 3 | P 21 3 | 0.31 | 0.991 | 2.78 | 3.05 | +8.8% | 99.7 | 40.5 | 20.2% | 4.9% | 1.000 | 13.8 | - | - | 0.229 | 0.150 | 0.922 | 0.232 | 0.237 | 0.215 | 1.079 | 0.980 | 0.242 | 0.238 | 1.017 | 0.005 | NOT_TESTED | 24 | | +| 9min | open | fail | P 21 21 2 | P 21 21 21 | 36.97 | 0.494 | 1.86 | 2.05 | +9.2% | 99.6 | 23.4 | 26.5% | 6.8% | 0.999 | 10.5 | - | - | 0.566 | 0.207 | 0.221 | 0.575 | 0.577 | 0.267 | 2.150 | -0.000 | - | - | - | - | NOT_TESTED | 73 | primitive volume ratio 0.49 | +| 9o0h | open | pass | P 21 21 21 | P 21 21 21 | 0.25 | 0.994 | 2.02 | 2.24 | +9.7% | 99.6 | 12.7 | 65.1% | 12.8% | 0.985 | 5.8 | - | - | 0.242 | 0.079 | 0.941 | 0.249 | 0.247 | 0.249 | 0.999 | 0.110 | 0.263 | 0.267 | 0.987 | 0.029 | NOT_TESTED | 59 | | +| 9p7q | open | pass | C 1 2 1 | C 1 2 1 | 0.14 | 1.003 | 1.76 | 2.21 | +20.3% | 88.0 | 2.6 | 23.8% | 8.8% | 0.985 | 10.8 | - | - | 0.269 | 0.036 | 0.950 | 0.282 | 0.270 | 0.259 | 1.090 | 0.870 | 0.267 | 0.262 | 1.017 | -0.037 | NOT_TESTED | 12 | | +| 9pbb | open | pass | C 1 2 1 | C 1 2 1 | 0.15 | 0.997 | 1.78 | 2.17 | +17.8% | 73.7 | 2.3 | 19.1% | 6.0% | 0.993 | 17.0 | - | - | 0.233 | 0.035 | 0.953 | 0.231 | 0.234 | 0.223 | 1.037 | 0.670 | 0.233 | 0.238 | 0.980 | -0.087 | NOT_TESTED | 16 | | +| 9q41 | open | pass | C 2 2 21 | C 2 2 21 | 0.20 | 1.004 | 1.68 | 1.95 | +14.1% | 99.7 | 6.9 | 29.0% | 12.9% | 0.984 | 7.9 | - | - | 0.188 | 0.099 | 0.959 | 0.194 | 0.195 | 0.213 | 0.908 | 0.480 | 0.208 | 0.214 | 0.972 | 0.059 | NOT_TESTED | 23 | | +| 9q66 | open | pass | P 1 21 1 | P 1 21 1 | 0.34 | 0.990 | 2.03 | 2.01 | -0.9% | 99.7 | 7.1 | 32.1% | 8.2% | 0.990 | 13.1 | - | - | 0.204 | 0.084 | 0.903 | 0.216 | 0.216 | 0.266 | 0.813 | 0.660 | 0.280 | 0.275 | 1.017 | -0.021 | NOT_TESTED | 30 | | +| 9qvv | open | pass | I 2 2 2 | I 2 2 2 | 0.42 | 0.989 | 2.49 | 2.72 | +8.5% | 87.8 | 13.8 | 11.2% | 2.3% | 1.000 | 37.3 | - | - | 0.235 | 0.204 | 0.867 | 0.239 | 0.246 | 0.279 | 0.854 | 1.640 | 0.259 | 0.273 | 0.947 | -0.030 | NOT_TESTED | 110 | | +| 9qw2 | open | pass | P 1 21 1 | P 1 21 1 | 0.77 | 1.012 | 1.76 | 1.92 | +8.5% | 99.6 | 3.4 | 18.5% | 6.6% | 0.958 | 7.8 | - | - | 0.240 | 0.040 | 0.930 | 0.245 | 0.242 | 0.212 | 1.155 | 0.540 | 0.260 | 0.220 | 1.185 | -0.360 | NOT_TESTED | 98 | | +| 9qw8 | open | pass | P 1 | P 1 | 0.46 | 1.000 | 1.71 | 1.80 | +5.2% | 97.0 | 2.9 | 20.0% | 7.6% | 0.989 | 9.2 | - | - | 0.294 | 0.148 | 0.890 | 0.312 | 0.302 | 0.243 | 1.286 | 0.330 | 0.317 | 0.249 | 1.272 | -0.326 | NOT_TESTED | 66 | | +| 9rci | open | pass | P 1 | P 1 | 97.58 | 1.990 | 1.76 | 1.66 | -5.8% | 85.2 | 2.6 | 29.6% | 12.3% | 0.951 | 7.0 | - | - | 0.561 | 0.084 | 0.284 | 0.555 | 0.564 | 0.283 | 1.959 | -0.020 | - | - | - | - | NOT_TESTED | 22 | P 1 vs reference P 1: accepted alternative cell 35.869 39.297 199.976 - a knife-edge this battery does not decide | +| 9rcs | open | pass | P 1 21 1 | P 1 21 1 | 0.93 | 1.022 | 3.27 | 3.01 | -8.5% | 99.7 | 7.0 | 23.4% | 9.9% | 0.994 | 7.3 | - | - | 0.318 | 0.124 | 0.502 | 0.341 | 0.325 | 0.317 | 1.078 | 0.380 | 0.336 | 0.363 | 0.928 | -0.171 | ACCEPTED | 51 | | +| 9rp9 | open | pass | C 1 2 1 | C 1 2 1 | 0.16 | 0.996 | 1.90 | 2.10 | +9.3% | 99.6 | 6.1 | 17.4% | 4.8% | 0.997 | 32.7 | - | - | 0.204 | 0.138 | 0.931 | 0.233 | 0.225 | 0.222 | 1.051 | 2.470 | 0.235 | 0.228 | 1.033 | 0.073 | NOT_TESTED | 25 | | +| 9s02 | open | pass | P 21 21 2 | P 21 21 2 | 0.06 | 0.999 | 1.42 | 1.65 | +13.8% | 98.7 | 12.4 | 10.2% | 3.3% | 0.999 | 26.0 | - | - | 0.180 | 0.109 | 0.961 | 0.189 | 0.189 | 0.193 | 0.977 | 2.250 | 0.195 | 0.202 | 0.969 | 0.036 | ACCEPTED | 127 | | +| 9sl0 | open | pass | P 21 21 21 | P 21 21 21 | 0.37 | 0.990 | 1.36 | 1.60 | +14.9% | 99.3 | 12.3 | 8.0% | 3.7% | 1.000 | 20.2 | - | - | 0.245 | 0.156 | 0.912 | 0.255 | 0.256 | 0.264 | 0.967 | 2.200 | 0.269 | 0.270 | 0.996 | 0.001 | NOT_TESTED | 45 | | +| 9t6s | open | pass | P 21 21 21 | P 21 21 21 | 0.06 | 1.000 | 1.75 | 2.00 | +12.6% | 95.7 | 5.1 | 11.2% | 3.6% | 0.999 | 29.9 | - | - | 0.215 | 0.044 | 0.970 | 0.232 | 0.216 | 0.241 | 0.960 | 2.510 | 0.239 | 0.252 | 0.948 | 0.102 | NOT_TESTED | 12 | | +| 9upt | open | pass | P 6 | P 6 | 0.20 | 0.994 | 2.03 | 2.37 | +14.4% | 99.7 | 5.7 | 24.4% | 10.8% | 0.989 | 6.5 | - | - | 0.218 | 0.088 | 0.942 | 0.218 | 0.225 | 0.215 | 1.014 | 1.310 | 0.228 | 0.216 | 1.056 | -0.046 | NOT_TESTED | 77 | | +| 9vyb | open | pass | P 21 21 21 | P 21 21 21 | 0.40 | 0.989 | 1.67 | 2.12 | +21.3% | 86.9 | 10.0 | 8.2% | 3.6% | 0.999 | 22.1 | - | - | 0.246 | 0.082 | 0.934 | 0.253 | 0.249 | 0.269 | 0.938 | 0.420 | 0.262 | 0.264 | 0.995 | 0.012 | NOT_TESTED | 34 | | +| 9w3y | open | pass | P 21 21 21 | P 21 21 21 | 0.25 | 1.004 | 1.19 | 1.50 | +20.4% | 99.7 | 6.8 | 24.7% | 5.2% | 0.997 | 20.2 | - | - | 0.187 | 0.088 | 0.973 | 0.198 | 0.197 | 0.198 | 0.998 | 4.930 | 0.214 | 0.215 | 0.995 | 0.051 | NOT_TESTED | 11 | | +| 9yl4 | open | pass | P 21 21 21 | P 21 21 21 | 0.08 | 1.001 | 3.61 | 3.70 | +2.5% | 99.7 | 13.4 | 31.8% | 9.4% | 0.996 | 9.5 | - | - | 0.293 | 0.066 | 0.942 | 0.303 | 0.297 | 0.283 | 1.069 | 5.750 | 0.284 | 0.288 | 0.987 | -0.023 | NOT_TESTED | 90 | | +| 9yzk | open | pass | I 1 2 1 | I 1 2 1 | 0.20 | 0.996 | 3.87 | 5.10 | +24.1% | 98.5 | 3.3 | 30.9% | 4.9% | 0.997 | 9.3 | - | - | 0.355 | 0.294 | 0.867 | 0.406 | 0.412 | 0.304 | 1.335 | 0.180 | 0.312 | 0.314 | 0.995 | 0.038 | ACCEPTED | 12 | | +| 9z44 | open | pass | I 1 2 1 | I 1 2 1 | 1.35 | 0.964 | 6.73 | 7.20 | +6.5% | 98.2 | 3.3 | 22.3% | 8.9% | 0.981 | 8.5 | - | - | 0.334 | 0.163 | 0.782 | 0.324 | 0.343 | 0.345 | 0.939 | -0.030 | 0.361 | 0.350 | 1.031 | 0.059 | ACCEPTED | 27 | | +| 9z72 | open | pass | P 31 2 1 | P 31 2 1 | 0.15 | 1.004 | 2.00 | 2.38 | +16.1% | 99.7 | 9.9 | 80.3% | 12.6% | 0.991 | 11.8 | - | - | 0.240 | 0.048 | 0.912 | 0.255 | 0.243 | 0.269 | 0.948 | 0.570 | 0.275 | 0.277 | 0.994 | 0.033 | NOT_TESTED | 44 | | +| 9zlo | open | pass | P 21 21 21 | P 21 21 21 | 0.39 | 0.995 | 1.56 | 2.00 | +21.9% | 99.7 | 12.7 | 12.3% | 3.9% | 0.999 | 24.1 | - | - | 0.224 | 0.065 | 0.942 | 0.231 | 0.228 | 0.219 | 1.053 | 1.750 | 0.225 | 0.222 | 1.011 | -0.024 | NOT_TESTED | 25 | | +| 9zm0 | open | pass | P 1 21 1 | P 1 21 1 | 0.16 | 0.997 | 1.80 | 2.10 | +14.5% | 99.7 | 6.5 | 25.0% | 9.9% | 0.996 | 9.0 | - | - | 0.255 | 0.098 | 0.955 | 0.275 | 0.259 | 0.298 | 0.923 | 4.590 | 0.297 | 0.302 | 0.984 | -0.020 | NOT_TESTED | 9 | | +| 9zmu | open | pass | P 61 2 2 | P 65 2 2 | 0.25 | 1.007 | 1.72 | 1.98 | +13.4% | 99.7 | 28.0 | 30.1% | 7.3% | 0.999 | 11.6 | - | - | 0.278 | 0.153 | 0.893 | 0.284 | 0.289 | 0.294 | 0.966 | 0.890 | 0.303 | 0.304 | 0.997 | 0.009 | ACCEPTED | 31 | P 61 2 2 vs reference P 65 2 2 (hand only (needs anomalous)); labelled P 65 2 2 from the model | +| cuhf2 | open | unscored | P 4 2 2 | - | - | - | 0.51 | - | - | 82.1 | 13.9 | 2.8% | 2.1% | 1.000 | 45.5 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 32 | no reference to score against | +| cytidine | open | pass | P 21 21 21 | P 21 21 21 | 0.31 | 0.995 | 0.58 | - | - | 91.1 | 3.6 | 10.3% | 9.7% | 0.993 | 8.7 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 24 | | +| dnba | open | pass | C 1 2/c 1 | C 1 2/c 1 | 0.06 | 0.999 | 0.81 | 0.48 | -68.3% | 75.8 | 2.8 | 2.5% | 1.8% | 1.000 | 36.8 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 6 | | +| lalanine | open | pass | P 21 21 21 | P 21 21 21 | 0.07 | 0.998 | 0.65 | - | - | 77.3 | 2.6 | 2.6% | 1.7% | 1.000 | 42.4 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 9 | | +| metformin | open | pass | P 1 21/c 1 | P 1 21/c 1 | 0.17 | 1.001 | 0.51 | 0.45 | -12.9% | 73.5 | 4.8 | 3.1% | 2.8% | 1.000 | 30.8 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 5 | | +| nidppe | open | pass | P 1 21/c 1 | P 1 21/c 1 | 0.28 | 0.997 | 0.51 | 0.77 | +34.3% | 66.6 | 5.3 | 7.9% | 4.0% | 0.999 | 32.9 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 5 | | +| aspirin_x10sa_20keV | inhouse | pass | P 1 21/c 1 | P 1 21/c 1 | 0.05 | 1.001 | 0.66 | 0.68 | +2.6% | 79.5 | 5.9 | 3.0% | 2.2% | 1.000 | 36.4 | 27.4 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 9 | | +| aspirin_x10sa_25keV | inhouse | pass | P 1 21/c 1 | P 1 21/c 1 | 0.05 | 1.001 | 0.53 | 0.55 | +3.6% | 79.1 | 6.0 | 3.1% | 2.2% | 1.000 | 36.4 | 29.2 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 11 | | +| citricacid_x10sa_20keV | inhouse | pass | P 1 21/c 1 | P 1 21/c 1 | 0.10 | 1.001 | 0.67 | 0.68 | +1.8% | 79.2 | 5.8 | 3.6% | 3.8% | 0.999 | 24.1 | 20.4 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 11 | | +| cytc_x06da_1 | inhouse | pass | P 31 2 1 | P 31 2 1 | 0.04 | 0.999 | 1.70 | 1.88 | +9.2% | 98.7 | 18.0 | 9.3% | 3.9% | 1.000 | 17.5 | 24.5 | - | 0.264 | 0.273 | 0.927 | 0.335 | 0.293 | 0.296 | 1.132 | 5.380 | - | - | - | - | ACCEPTED | 14 | labelled P 32 2 1 from the model | +| cytc_x06da_2 | inhouse | pass | P 31 2 1 | P 31 2 1 | 0.11 | 1.002 | 1.57 | 1.69 | +7.3% | 99.5 | 17.4 | 9.8% | 3.5% | 1.000 | 21.2 | 27.0 | - | 0.253 | 0.221 | 0.924 | 0.306 | 0.279 | 0.286 | 1.069 | 6.320 | - | - | - | - | ACCEPTED | 15 | labelled P 32 2 1 from the model | +| cytc_x10sa | inhouse | pass | P 31 2 1 | P 31 2 1 | 0.13 | 1.003 | 1.95 | 2.27 | +13.9% | 99.6 | 20.1 | 20.0% | 4.0% | 0.999 | 26.2 | 31.8 | - | 0.262 | 0.263 | 0.938 | 0.357 | 0.287 | 0.340 | 1.050 | 2.690 | - | - | - | - | ACCEPTED | 23 | labelled P 32 2 1 from the model | +| hepes_x10sa_20keV | inhouse | pass | P b c a | P b c a | 0.07 | 0.999 | 0.66 | 0.68 | +2.6% | 86.8 | 10.3 | 2.6% | 3.5% | 1.000 | 39.6 | 28.5 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 12 | | +| insu_H_x06da_notwin | inhouse | pass | R 3:H | R 3 | 0.06 | 0.999 | 1.42 | 1.54 | +8.4% | 91.3 | 8.6 | 6.2% | 4.2% | 0.999 | 20.4 | 17.7 | - | 0.229 | 0.110 | 0.935 | 0.215 | 0.233 | 0.229 | 0.940 | 6.050 | - | - | - | - | ACCEPTED | 9 | | +| insu_H_x06da_twin | inhouse | pass | R 3:H | R 3 | 0.04 | 1.001 | 1.38 | 1.46 | +5.2% | 90.6 | 8.5 | 12.1% | 10.3% | 0.995 | 6.2 | 6.8 | - | 0.264 | 0.088 | 0.885 | 0.253 | 0.267 | 0.229 | 1.108 | 4.400 | - | - | - | - | NOT_TESTED | 7 | | +| insu_I_x06da_13keV | inhouse | pass | I 2 3 | I 2 3 | 0.27 | 1.008 | 1.47 | 1.64 | +9.9% | 99.7 | 38.9 | 32.1% | 8.4% | 0.999 | - | 18.8 | - | 0.234 | 0.029 | 0.940 | 0.259 | 0.234 | 0.161 | 1.606 | 1.330 | - | - | - | - | ACCEPTED | 15 | | +| insu_I_x06da_5keV | inhouse | pass | I 2 3 | I 2 3 | 0.03 | 0.999 | 2.43 | 2.45 | +0.9% | 95.4 | 28.1 | 6.6% | 4.4% | 1.000 | 28.1 | 17.5 | - | 0.221 | 0.105 | 0.923 | 0.245 | 0.228 | 0.161 | 1.523 | 7.950 | - | - | - | - | ACCEPTED | 6 | | +| insu_I_x06da_5keV_2 | inhouse | pass | I 2 3 | I 2 3 | 0.05 | 0.999 | 2.42 | 2.45 | +1.1% | 93.9 | 29.5 | 8.4% | 5.8% | 0.999 | 25.3 | 20.0 | - | 0.219 | 0.263 | 0.899 | 0.294 | 0.274 | 0.161 | 1.828 | 9.120 | - | - | - | - | NOT_TESTED | 5 | | +| insu_I_x06da_6keV | inhouse | pass | I 2 3 | I 2 3 | 0.04 | 0.999 | 2.03 | 2.04 | +0.7% | 95.6 | 28.4 | 6.1% | 4.8% | 1.000 | 20.6 | 17.9 | - | 0.213 | 0.037 | 0.933 | 0.236 | 0.213 | 0.161 | 1.464 | 11.500 | - | - | - | - | ACCEPTED | 6 | | +| insu_I_x06da_low_isa | inhouse | pass | I 2 3 | I 2 3 | 0.08 | 1.002 | 1.44 | 1.30 | -10.4% | 99.9 | 31.9 | 22.4% | 16.1% | 0.998 | 5.6 | 4.2 | - | 0.226 | 0.095 | 0.933 | 0.261 | 0.229 | 0.161 | 1.619 | 3.110 | - | - | - | - | NOT_TESTED | 11 | | +| insu_I_x06da_ref | inhouse | pass | I 2 3 | I 2 3 | 0.13 | 1.004 | 1.40 | 1.62 | +13.4% | 99.7 | 24.8 | 30.6% | 6.2% | 0.999 | 32.2 | 25.1 | - | 0.226 | 0.077 | 0.937 | 0.245 | 0.228 | 0.161 | 1.522 | 3.330 | - | - | - | - | NOT_TESTED | 10 | | +| insu_I_x06da_weak | inhouse | pass | I 2 3 | I 2 3 | 0.79 | 1.024 | 1.64 | 1.81 | +9.2% | 99.5 | 40.1 | 16.7% | 4.9% | 1.000 | 15.0 | 18.9 | - | 0.286 | 0.147 | 0.919 | 0.315 | 0.289 | 0.161 | 1.958 | 1.920 | - | - | - | - | NOT_TESTED | 16 | | +| kdp_x10sa_20keV | inhouse | fail | I 41 m d | I -4 2 d | 0.07 | 1.002 | 0.66 | 0.74 | +10.3% | 90.9 | 16.7 | 5.2% | 4.6% | 0.999 | 25.1 | 4.1 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 93 | I 41 m d vs reference I -4 2 d | +| lcystine_x10sa_20keV | inhouse | pass | P 61 2 2 | P 61 2 2 | 0.49 | 0.991 | 0.67 | - | - | 92.9 | 19.5 | 30.6% | 18.4% | 1.000 | 5.7 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 57 | | +| lcystine_x10sa_25keV | inhouse | pass | P 61 2 2 | P 61 2 2 | 0.52 | 0.990 | 0.53 | - | - | 92.1 | 20.9 | 26.7% | 13.4% | 1.000 | 5.5 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 23 | | +| lysoI_micromax_mono | inhouse | pass | P 41 21 2 | P 43 21 2 | 0.06 | 1.001 | 1.36 | 1.65 | +17.6% | 98.9 | 9.6 | 7.0% | 2.9% | 1.000 | 37.9 | 31.4 | - | 0.270 | 0.080 | 0.906 | 0.290 | 0.274 | 0.177 | 1.638 | 3.080 | - | - | - | - | ACCEPTED | 21 | P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lysoI_micromax_pink | inhouse | pass | P 41 21 2 | P 43 21 2 | 0.09 | 1.002 | 1.38 | 1.65 | +16.1% | 99.3 | 9.8 | 8.0% | 3.1% | 1.000 | 35.1 | 29.2 | - | 0.275 | 0.084 | 0.900 | 0.297 | 0.279 | 0.177 | 1.677 | 2.470 | - | - | - | - | ACCEPTED | 13 | P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_micromax_mono | inhouse | pass | P 41 21 2 | P 43 21 2 | 0.16 | 1.003 | 1.18 | 1.50 | +21.5% | 79.1 | 8.3 | 5.0% | 2.5% | 1.000 | 39.9 | 39.9 | - | 0.234 | 0.151 | 0.932 | 0.249 | 0.243 | 0.177 | 1.407 | 4.530 | - | - | - | - | ACCEPTED | 20 | P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_micromax_pink | inhouse | pass | P 41 21 2 | P 43 21 2 | 0.16 | 1.004 | 1.20 | 1.45 | +17.0% | 83.6 | 8.3 | 5.1% | 2.5% | 1.000 | 40.6 | 37.4 | - | 0.236 | 0.180 | 0.930 | 0.260 | 0.253 | 0.177 | 1.468 | 4.650 | - | - | - | - | ACCEPTED | 14 | P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_x06da_5keV | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.29 | 1.009 | 2.43 | 2.45 | +0.7% | 86.2 | 19.7 | 6.5% | 5.4% | 0.999 | 28.2 | 19.4 | - | 0.282 | 0.175 | 0.825 | 0.297 | 0.297 | 0.177 | 1.675 | 5.830 | - | - | - | - | ACCEPTED | 6 | labelled P 43 21 2 from the model | +| lyso_x06da_atten_wedge | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.25 | 1.006 | 1.19 | 1.26 | +6.0% | 99.7 | 23.1 | 42.5% | 7.2% | 0.998 | 13.1 | 16.6 | - | 0.295 | 0.073 | 0.896 | 0.306 | 0.296 | 0.177 | 1.729 | 1.930 | - | - | - | - | ACCEPTED | 17 | labelled P 43 21 2 from the model | +| lyso_x06da_half_image | inhouse | pass | P 41 21 2 | P 43 21 2 | 0.71 | 1.021 | 1.57 | 1.65 | +5.1% | 99.7 | 9.8 | 118.0% | 33.5% | 0.969 | 6.8 | 6.6 | - | 0.278 | 0.061 | 0.889 | 0.289 | 0.282 | 0.177 | 1.629 | 0.830 | - | - | - | - | ACCEPTED | 26 | P 41 21 2 vs reference P 43 21 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_x06da_ice | inhouse | pass | P 41 21 2 | P 4 2 2 | 0.09 | 1.002 | 1.34 | 1.43 | +6.4% | 99.7 | 19.4 | 18.8% | 4.7% | 0.999 | 22.5 | 23.3 | - | 0.287 | 0.103 | 0.902 | 0.313 | 0.292 | 0.177 | 1.765 | 2.660 | - | - | - | - | ACCEPTED | 13 | P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_x06da_ref | inhouse | pass | P 41 21 2 | P 4 2 2 | 0.04 | 1.000 | 0.99 | 1.20 | +17.3% | 85.7 | 22.2 | 4.8% | 2.7% | 1.000 | 29.9 | 28.3 | - | 0.272 | 0.089 | 0.902 | 0.287 | 0.275 | 0.177 | 1.622 | 9.660 | - | - | - | - | ACCEPTED | 15 | P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS); labelled P 43 21 2 from the model | +| lyso_x10sa_90deg_1 | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.12 | 0.998 | 1.83 | 1.97 | +6.9% | 99.7 | 6.4 | 17.5% | 4.6% | 0.998 | 20.7 | 20.8 | - | 0.380 | 0.099 | 0.817 | 0.369 | 0.386 | 0.177 | 2.083 | 0.350 | - | - | - | - | ACCEPTED | 8 | labelled P 43 21 2 from the model | +| lyso_x10sa_90deg_2 | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.16 | 0.997 | 1.85 | 1.97 | +6.3% | 99.7 | 6.4 | 17.0% | 4.8% | 0.998 | 19.0 | 18.1 | - | 0.378 | 0.104 | 0.819 | 0.372 | 0.385 | 0.177 | 2.100 | 0.560 | - | - | - | - | ACCEPTED | 8 | labelled P 43 21 2 from the model | +| lyso_x10sa_strong | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.63 | 0.984 | 1.36 | 1.24 | -9.5% | 99.4 | 21.5 | 23.8% | 10.3% | 0.997 | 5.5 | 8.6 | - | 0.375 | 0.197 | 0.844 | 0.368 | 0.379 | 0.177 | 2.076 | 1.560 | - | - | - | - | ACCEPTED | 36 | labelled P 43 21 2 from the model | +| myob_x06da | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.12 | 1.003 | 1.23 | 1.42 | +13.4% | 99.7 | 6.7 | 17.0% | 4.1% | 0.998 | 24.7 | 7.6 | - | 0.229 | 0.105 | 0.953 | 0.233 | 0.236 | 0.218 | 1.070 | 3.410 | - | - | - | - | NOT_TESTED | 8 | | +| myob_x06da_powder_1 | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.02 | 1.000 | 1.73 | 1.50 | -15.6% | 99.2 | 5.3 | 76.4% | 25.3% | 0.765 | 1.9 | 5.5 | - | 0.520 | 0.234 | 0.280 | 0.572 | 0.524 | 0.218 | 2.624 | 0.380 | - | - | - | - | ACCEPTED | 12 | | +| myob_x06da_powder_2 | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.34 | 0.995 | 1.41 | 0.99 | -42.2% | 99.4 | 5.5 | 125.6% | 33.4% | 0.908 | 2.5 | 2.2 | - | 0.447 | 0.181 | 0.092 | 0.490 | 0.464 | 0.218 | 2.247 | 1.090 | - | - | - | - | NOT_TESTED | 12 | | +| myob_x06da_sparse | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.44 | 0.993 | 1.77 | 2.00 | +11.7% | 99.7 | 5.5 | 45.7% | 18.3% | 0.902 | 3.0 | 5.5 | - | 0.398 | 0.133 | 0.770 | 0.365 | 0.402 | 0.218 | 1.674 | 0.890 | - | - | - | - | NOT_TESTED | 16 | | +| myob_x06da_split | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.24 | 0.998 | 1.96 | 1.51 | -30.1% | 99.7 | 6.0 | 71.2% | 20.4% | 0.920 | 2.6 | 12.4 | - | 0.387 | 0.120 | 0.219 | 0.375 | 0.391 | 0.218 | 1.718 | 0.480 | - | - | - | - | NOT_TESTED | 12 | | +| myob_x10sa | inhouse | pass | P 1 21 1 | P 1 21 1 | 0.31 | 1.004 | 1.57 | 1.74 | +10.0% | 97.5 | 5.7 | 17.2% | 7.7% | 0.995 | 10.2 | 5.2 | - | 0.244 | 0.079 | 0.936 | 0.246 | 0.248 | 0.218 | 1.131 | 2.130 | - | - | - | - | NOT_TESTED | 14 | | +| nothing_1 | inhouse | pass | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 48 | no lattice reported, as expected | +| nothing_2 | inhouse | pass | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 44 | no lattice reported, as expected | +| thau_bl1a_3p8keV | inhouse | pass | P 41 21 2 | P 4 2 2 | 0.02 | 1.000 | 3.02 | 3.10 | +2.7% | 88.7 | 15.5 | 7.3% | 6.5% | 0.998 | 23.6 | 30.8 | - | 0.185 | 0.031 | 0.935 | 0.188 | 0.186 | 0.151 | 1.241 | 8.910 | - | - | - | - | NOT_TESTED | 9 | P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS) | +| thau_bl1a_4p6keV | inhouse | pass | P 41 21 2 | P 4 2 2 | 0.08 | 0.998 | 2.47 | 2.53 | +2.5% | 88.4 | 15.8 | 6.8% | 5.3% | 0.999 | 38.2 | 35.6 | - | 0.183 | 0.027 | 0.943 | 0.175 | 0.185 | 0.151 | 1.154 | 8.530 | - | - | - | - | NOT_TESTED | 8 | P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS) | +| thau_bl1a_6p5keV | inhouse | pass | P 41 21 2 | P 4 2 2 | 0.04 | 0.999 | 1.73 | 1.78 | +2.6% | 88.5 | 16.1 | 7.5% | 4.7% | 0.999 | 42.8 | 34.5 | - | 0.191 | 0.052 | 0.951 | 0.183 | 0.194 | 0.151 | 1.212 | 6.690 | - | - | - | - | NOT_TESTED | 11 | P 41 21 2 vs reference P 4 2 2 (screws not judged against XDS) | +| thau_micromax_pink | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.13 | 0.997 | 1.24 | 1.40 | +11.1% | 95.1 | 13.8 | 8.8% | 5.0% | 0.999 | 15.9 | 21.2 | - | 0.190 | 0.202 | 0.962 | 0.213 | 0.215 | 0.151 | 1.407 | 4.020 | - | - | - | - | NOT_TESTED | 16 | | +| thau_x10sa_0p1deg | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.23 | 0.993 | 1.91 | 2.20 | +13.0% | 87.1 | 17.3 | 14.4% | 7.3% | 0.998 | 10.8 | 9.2 | - | 0.236 | 0.027 | 0.932 | 0.232 | 0.238 | 0.151 | 1.531 | 1.260 | - | - | - | - | NOT_TESTED | 22 | | +| thau_x10sa_16keV | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.02 | 1.000 | 1.22 | 1.30 | +6.3% | 82.4 | 17.6 | 7.1% | 2.8% | 1.000 | 53.4 | 44.5 | - | 0.228 | 0.119 | 0.942 | 0.231 | 0.235 | 0.151 | 1.525 | 2.340 | - | - | - | - | NOT_TESTED | 18 | | +| thau_x10sa_injection | inhouse | pass | P 41 21 2 | P 41 21 2 | 0.02 | 1.000 | 1.26 | 1.28 | +1.6% | 82.5 | 19.0 | 3.9% | 2.6% | 1.000 | 33.1 | 36.2 | - | 0.227 | 0.024 | 0.939 | 0.223 | 0.229 | 0.151 | 1.474 | 9.340 | - | - | - | - | NOT_TESTED | 25 | | +| yag_x10sa_20keV | inhouse | pass | I a -3 d | I a -3 d | 0.03 | 1.001 | 0.67 | 0.68 | +2.2% | 96.6 | 34.9 | 39.4% | 41.8% | 0.702 | 3.0 | 3.2 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | 17 | | + +## Delta vs baseline 20260929-2003_cca7bf_r12-rc173-refmac + +Baseline rugnux 1.0.0-rc.173. Pass rates on the sets both runs have: + +| arm | common sets | baseline | this run | +|:--|--:|:--|:--| +| open | 171 | 166/169 (98%) | 164/168 (98%) | +| inhouse | 39 | 39/39 (100%) | 39/39 (100%) | + +Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with `compare --rerun-changed` before believing a single-set change): + +| set | arm | verdict | space group | d_min | ISa | R_meas | CC1/2 | cell dev % | R_model (shell-scaled) | R_free | radial misfit | anom. sigma | ref-range R_meas | ref-range low-res R_meas | SHELXL R1 | SHELXL wR2 | SHELXL GooF | SHELXL EXTI | time s | beyond noise | +|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--|:--| +| insu_H_x06da_twin | inhouse | pass | R 3:H | 1.38 | 6.3 -> 6.2 | 12.8% -> 12.1% | 0.994 -> 0.995 | 0.04 | - -> 0.264 | - -> 0.253 | - -> 0.09 | - -> 4.40 | 13.2% -> 12.5% | 11.5% -> 10.3% | - | - | - | - | 12 -> 7 | r_meas, refres_r_meas, refres_lowres_r_meas | +| insu_I_x06da_ref | inhouse | pass | I 2 3 | 1.40 | 31.1 -> 32.2 | 27.8% -> 30.6% | 0.999 | 0.13 | - -> 0.226 | - -> 0.245 | - -> 0.08 | - -> 3.33 | 24.0% -> 25.8% | 5.9% -> 6.2% | - | - | - | - | 15 -> 10 | r_meas, refres_r_meas, refres_lowres_r_meas | +| lysoI_micromax_pink | inhouse | pass | P 41 21 2 | 1.42 -> 1.38 | 33.5 -> 35.1 | 7.4% -> 8.0% | 1.000 | 0.09 | - -> 0.275 | - -> 0.297 | - -> 0.08 | - -> 2.47 | 6.3% -> 6.6% | 3.1% -> 3.2% | - | - | - | - | 17 -> 13 | d_min, isa, r_meas, refres_r_meas | +| lyso_micromax_mono | inhouse | pass | P 41 21 2 | 1.20 -> 1.18 | 39.0 -> 39.9 | 4.9% -> 5.0% | 1.000 | 0.16 | - -> 0.234 | - -> 0.249 | - -> 0.15 | - -> 4.53 | 4.6% -> 4.7% | 2.5% | - | - | - | - | 24 -> 20 | completeness | +| lyso_micromax_pink | inhouse | pass | P 41 21 2 | 1.26 -> 1.20 | 35.7 -> 40.6 | 5.2% -> 5.1% | 1.000 | 0.16 | - -> 0.236 | - -> 0.260 | - -> 0.18 | - -> 4.65 | 4.9% -> 4.8% | 2.5% -> 2.4% | - | - | - | - | 17 -> 14 | d_min, isa, completeness, refres_isa | +| lyso_x06da_5keV | inhouse | pass | P 41 21 2 | 2.43 | 25.6 -> 28.2 | 6.6% -> 6.5% | 0.999 | 0.29 | - -> 0.282 | - -> 0.297 | - -> 0.17 | - -> 5.83 | 5.9% -> 5.7% | 5.4% | - | - | - | - | 8 -> 6 | isa, refres_isa | +| lyso_x06da_atten_wedge | inhouse | pass | P 41 21 2 | 1.16 -> 1.19 | 13.0 -> 13.1 | 35.8% -> 42.5% | 0.999 -> 0.998 | 0.25 | - -> 0.295 | - -> 0.306 | - -> 0.07 | - -> 1.93 | 30.9% -> 35.8% | 7.1% -> 6.9% | - | - | - | - | 19 -> 17 | d_min, r_meas, completeness, refres_r_meas | +| lyso_x06da_half_image | inhouse | pass | P 41 21 2 | 1.56 -> 1.57 | 7.3 -> 6.8 | 101.0% -> 118.0% | 0.972 -> 0.969 | 0.71 | - -> 0.278 | - -> 0.289 | - -> 0.06 | - -> 0.83 | 94.7% -> 110.9% | 25.6% -> 33.0% | - | - | - | - | 47 -> 26 | isa, r_meas, refres_r_meas, refres_isa, refres_lowres_r_meas, time | +| lyso_x10sa_90deg_1 | inhouse | pass | P 41 21 2 | 1.86 -> 1.83 | 18.8 -> 20.7 | 18.1% -> 17.5% | 0.997 -> 0.998 | 0.12 | - -> 0.380 | - -> 0.369 | - -> 0.10 | - -> 0.35 | 16.3% -> 15.2% | 4.7% -> 4.5% | - | - | - | - | 10 -> 8 | isa, refres_r_meas, refres_isa | +| lyso_x10sa_90deg_2 | inhouse | pass | P 41 21 2 | 1.88 -> 1.85 | 17.7 -> 19.0 | 17.7% -> 17.0% | 0.998 | 0.16 | - -> 0.378 | - -> 0.372 | - -> 0.10 | - -> 0.56 | 16.2% -> 15.1% | 4.8% -> 4.6% | - | - | - | - | 9 -> 8 | isa, refres_r_meas, refres_isa | +| lyso_x10sa_strong | inhouse | pass | P 41 21 2 | 1.37 -> 1.36 | 4.9 -> 5.5 | 24.0% -> 23.8% | 0.996 -> 0.997 | 0.63 | - -> 0.375 | - -> 0.368 | - -> 0.20 | - -> 1.56 | 24.0% -> 23.8% | 12.9% -> 11.4% | - | - | - | - | 40 -> 36 | isa, refres_isa, refres_lowres_r_meas | +| myob_x06da_powder_1 | inhouse | pass | P 1 2 1 -> P 1 21 1 | 1.38 -> 1.73 | 2.4 -> 1.9 | 98.7% -> 76.4% | 0.929 -> 0.765 | 0.02 | - -> 0.520 | - -> 0.572 | - -> 0.23 | - -> 0.38 | 83.4% -> 76.4% | 17.0% -> 27.7% | - | - | - | - | 16 -> 12 | space group, d_min, isa, r_meas, cc_half, refres_r_meas, refres_isa, refres_lowres_r_meas | +| myob_x06da_powder_2 | inhouse | pass | P 1 2 1 -> P 1 21 1 | 1.41 | 2.8 -> 2.5 | 151.9% -> 125.6% | 0.888 -> 0.908 | 0.34 | - -> 0.447 | - -> 0.490 | - -> 0.18 | - -> 1.09 | 151.9% -> 125.6% | 58.8% -> 45.9% | - | - | - | - | 18 -> 12 | space group, isa, r_meas, cc_half, refres_r_meas, refres_isa, refres_lowres_r_meas | +| myob_x06da_sparse | inhouse | pass | P 1 2 1 -> P 1 21 1 | 1.77 | 3.0 | 49.1% -> 45.7% | 0.893 -> 0.902 | 0.44 | - -> 0.398 | - -> 0.365 | - -> 0.13 | - -> 0.89 | 41.8% -> 38.6% | 19.9% -> 17.2% | - | - | - | - | 40 -> 16 | space group, r_meas, cc_half, refres_r_meas, refres_lowres_r_meas, time | +| myob_x06da_split | inhouse | pass | P 1 21 1 | 1.84 -> 1.96 | 3.1 -> 2.6 | 69.9% -> 71.2% | 0.525 -> 0.920 | 0.21 -> 0.24 | - -> 0.387 | - -> 0.375 | - -> 0.12 | - -> 0.48 | 69.9% -> 71.2% | 20.6% -> 24.4% | - | - | - | - | 19 -> 12 | d_min, isa, cc_half, refres_isa, refres_lowres_r_meas | +| myob_x10sa | inhouse | pass | P 1 21 1 | 1.62 -> 1.57 | 9.1 -> 10.2 | 18.4% -> 17.2% | 0.994 -> 0.995 | 0.31 | - -> 0.244 | - -> 0.246 | - -> 0.08 | - -> 2.13 | 17.7% -> 16.0% | 7.9% -> 7.6% | - | - | - | - | 18 -> 14 | d_min, isa, r_meas, refres_r_meas, refres_isa | +| thau_bl1a_3p8keV | inhouse | pass | P 41 21 2 | 3.02 | 17.4 -> 23.6 | 8.1% -> 7.3% | 0.997 -> 0.998 | 0.02 | - -> 0.185 | - -> 0.188 | - -> 0.03 | - -> 8.91 | 7.4% -> 6.5% | 6.9% -> 6.3% | - | - | - | - | 10 -> 9 | isa, r_meas, refres_r_meas, refres_isa, refres_lowres_r_meas | +| thau_bl1a_4p6keV | inhouse | pass | P 41 21 2 | 2.47 | 33.8 -> 38.2 | 7.0% -> 6.8% | 0.999 | 0.08 | - -> 0.183 | - -> 0.175 | - -> 0.03 | - -> 8.53 | 6.6% -> 6.4% | 5.2% | - | - | - | - | 11 -> 8 | isa, refres_isa | +| thau_bl1a_6p5keV | inhouse | pass | P 41 21 2 | 1.73 | - -> 42.8 | 7.2% -> 7.5% | 0.999 | 0.04 | - -> 0.191 | - -> 0.183 | - -> 0.05 | - -> 6.69 | 7.1% -> 7.3% | 4.6% | - | - | - | - | 15 -> 11 | isa, refres_isa | +| thau_micromax_pink | inhouse | pass | P 41 21 2 | 1.28 -> 1.24 | 15.4 -> 15.9 | 8.6% -> 8.8% | 0.999 | 0.13 | - -> 0.190 | - -> 0.213 | - -> 0.20 | - -> 4.02 | 8.6% -> 8.7% | 4.9% -> 4.8% | - | - | - | - | 21 -> 16 | d_min, completeness | +| thau_x10sa_0p1deg | inhouse | pass | P 41 21 2 | 2.04 -> 1.91 | 7.0 -> 10.8 | 18.7% -> 14.4% | 0.999 -> 0.998 | 0.23 | - -> 0.236 | - -> 0.232 | - -> 0.03 | - -> 1.26 | 18.4% -> 13.6% | 9.1% -> 6.9% | - | - | - | - | 23 -> 22 | d_min, isa, r_meas, completeness, refres_r_meas, refres_isa, refres_lowres_r_meas | +| 11if | open | pass | P 41 | 1.38 -> 1.36 | 25.5 -> 25.7 | 4.6% -> 4.8% | 1.000 | 0.12 | 0.191 -> 0.192 | 0.200 -> 0.188 | 0.19 -> 0.13 | - -> 2.92 | - | - | - | - | - | - | 14 -> 12 | completeness, radial_misfit, rfree | +| 5ebi | open | pass -> fail | P 1 21 1 -> C 2 2 21 | 0.90 -> 0.85 | 14.4 -> 10.9 | 10.6% -> 12.3% | 0.997 -> 0.998 | 0.05 -> 40.37 | 0.549 -> 0.580 | 0.535 -> 0.571 | 0.07 -> 0.14 | - -> -0.15 | - | - | - | - | - | - | 52 -> 46 | verdict pass->fail, space group, d_min, isa, r_meas, cell_dev_pct, rmodel_shell_scaled, radial_misfit, rfree | +| 5epe | open | pass | F 2 3 | 1.78 -> 1.77 | 10.0 -> 9.9 | 14.3% -> 16.5% | 0.998 | 0.00 | 0.166 -> 0.173 | 0.175 -> 0.181 | 0.08 -> 0.11 | - -> 11.95 | - | - | - | - | - | - | 67 -> 36 | r_meas, rmodel_shell_scaled, rfree, time | +| 5j23 | open | pass | R 3:H | 2.17 | 11.5 -> 11.6 | 15.2% -> 15.4% | 0.996 | 0.15 | 0.227 -> 0.226 | 0.234 | 0.13 -> 0.14 | - -> 0.86 | - | - | - | - | - | - | 62 -> 34 | time | +| 5jvn | open | pass | P 6 2 2 | 2.24 | 15.6 | 16.3% | 0.998 -> 0.999 | 0.03 | 0.234 -> 0.228 | 0.250 -> 0.249 | 0.14 | - -> 1.36 | - | - | - | - | - | - | 40 -> 42 | rmodel_shell_scaled | +| 5ky6 | open | pass | P 1 21 1 | 1.55 -> 1.54 | 7.2 -> 6.1 | 27.3% -> 25.4% | 0.987 -> 0.986 | 0.64 | 0.245 -> 0.233 | 0.249 -> 0.243 | 0.11 -> 0.12 | - -> 0.55 | - | - | - | - | - | - | 92 -> 91 | isa, r_meas, rmodel_shell_scaled, rfree | +| 5m17 | open | pass | I 4 | 0.98 | 16.0 -> 16.2 | 6.0% -> 5.5% | 0.994 -> 0.996 | 0.08 | 0.132 | 0.159 -> 0.158 | 0.23 -> 0.22 | - -> 7.09 | - | - | - | - | - | - | 102 -> 97 | r_meas | +| 5mln | open | pass | P 21 21 2 | 1.26 | 21.4 -> 21.5 | 10.4% | 0.999 | 0.10 | 0.184 -> 0.181 | 0.189 -> 0.188 | 0.14 | - -> 2.17 | - | - | - | - | - | - | 45 -> 22 | time | +| 5nw5 | open | pass | P 21 21 21 | 7.10 -> 7.07 | 8.8 -> 7.7 | 32.0% -> 33.3% | 0.936 -> 0.927 | 0.19 | 0.401 -> 0.395 | 0.440 -> 0.448 | 0.36 -> 0.25 | - -> 0.09 | - | - | - | - | - | - | 47 -> 46 | isa, cc_half, rmodel_shell_scaled, radial_misfit, rfree | +| 5t39 | open | pass | P 1 21 1 | 1.01 | 16.0 -> 16.2 | 7.0% -> 7.3% | 0.998 | 0.13 | 0.152 -> 0.151 | 0.165 -> 0.156 | 0.17 -> 0.09 | - -> 6.47 | - | - | - | - | - | - | 75 -> 71 | r_meas, radial_misfit, rfree | +| 6cdl | open | pass | P 21 21 2 | 1.15 -> 1.13 | 11.2 -> 11.6 | 8.2% -> 7.8% | 0.996 -> 0.997 | 1.22 | 0.150 -> 0.147 | 0.174 -> 0.159 | 0.28 -> 0.23 | - -> 4.60 | - | - | - | - | - | - | 57 -> 49 | r_meas, completeness, rfree | +| 6f3p | open | pass | C 1 2 1 | 1.13 | 9.4 | 10.6% -> 10.0% | 0.997 | 0.12 | 0.144 -> 0.143 | 0.167 -> 0.169 | 0.18 -> 0.21 | - -> -3.42 | - | - | - | - | - | - | 130 -> 128 | r_meas | +| 6fwc | open | pass | C 2 2 2 | 1.41 | 27.2 -> 27.3 | 16.4% | 0.995 | 0.04 | 0.189 -> 0.188 | 0.197 -> 0.196 | 0.10 | - -> 1.44 | - | - | - | - | - | - | 48 -> 26 | time | +| 6h2p_1p89A | open | pass | C 2 2 21 | 1.76 | 22.5 -> 22.8 | 8.3% -> 8.5% | 0.999 | 0.03 | 0.144 -> 0.143 | 0.154 -> 0.149 | 0.11 -> 0.08 | - -> 9.20 | - | - | - | - | - | - | 172 -> 217 | rfree, time | +| 6h2p_native | open | pass | C 2 2 21 | 1.32 | 19.9 -> 20.0 | 13.7% | 0.999 | 0.04 | 0.170 -> 0.161 | 0.174 -> 0.165 | 0.08 -> 0.07 | - -> 3.33 | - | - | - | - | - | - | 110 -> 113 | rmodel_shell_scaled, rfree | +| 6h5t | open | pass | I 4 2 2 | 1.48 | 9.2 -> 9.3 | 15.2% -> 14.6% | 0.996 | 0.40 | 0.203 -> 0.196 | 0.218 -> 0.222 | 0.13 -> 0.19 | - -> 9.00 | - | - | - | - | - | - | 35 | rmodel_shell_scaled, radial_misfit | +| 6hv2 | open | pass | P 61 2 2 | 1.42 -> 1.41 | 13.6 -> 13.7 | 18.1% -> 19.8% | 1.000 | 0.04 | 0.235 -> 0.219 | 0.247 -> 0.239 | 0.46 -> 0.28 | - -> 7.83 | - | - | - | - | - | - | 37 -> 32 | r_meas, rmodel_shell_scaled, radial_misfit, rfree | +| 6i3j | open | pass | F 2 2 2 | 2.32 | 7.0 -> 6.8 | 23.5% -> 24.6% | 0.983 -> 0.977 | 0.08 | 0.202 -> 0.203 | 0.233 -> 0.237 | 0.16 -> 0.17 | - -> 1.73 | - | - | - | - | - | - | 83 -> 75 | cc_half | +| 6iu6 | open | pass | P 31 | 2.38 -> 2.36 | 7.0 -> 7.2 | 14.2% -> 11.5% | 0.992 -> 0.996 | 0.33 | 0.221 | 0.234 -> 0.233 | 0.19 -> 0.17 | - -> 3.54 | - | - | - | - | - | - | 87 -> 91 | r_meas | +| 6iu9 | open | pass | P 31 | 2.73 -> 2.74 | 5.3 -> 5.4 | 18.0% -> 14.4% | 0.989 -> 0.993 | 0.22 | 0.306 -> 0.310 | 0.316 -> 0.327 | 0.11 -> 0.10 | - -> 1.43 | - | - | - | - | - | - | 129 -> 86 | r_meas, rfree, time | +| 6jgh | open | pass | P 21 21 21 | 0.87 | 6.7 -> 6.6 | 22.7% -> 23.4% | 0.989 -> 0.990 | 0.39 | 0.148 -> 0.136 | 0.169 -> 0.156 | 0.19 -> 0.17 | - -> 2.07 | - | - | - | - | - | - | 105 -> 104 | rmodel_shell_scaled, rfree | +| 6moj | open | pass | I 41 2 2 | 2.43 | 8.3 | 52.6% -> 52.4% | 0.998 | 0.08 | 0.275 -> 0.248 | 0.285 -> 0.257 | 0.37 -> 0.21 | - -> 1.05 | - | - | - | - | - | - | 124 -> 125 | rmodel_shell_scaled, radial_misfit, rfree | +| 6nen | open | pass | P 3 1 2 | 1.72 -> 1.77 | 6.3 | 26.9% -> 28.5% | 0.997 | 0.07 | 0.210 -> 0.211 | 0.212 -> 0.218 | 0.08 -> 0.05 | - -> 1.00 | - | - | - | - | - | - | 24 -> 19 | d_min, r_meas, rfree | +| 6o2h | open | pass | P 1 | 1.10 | 20.8 -> 23.9 | 7.2% -> 7.0% | 0.978 -> 0.977 | 0.74 | 0.099 -> 0.096 | 0.131 -> 0.129 | 0.21 -> 0.13 | - -> 0.11 | - | - | - | - | - | - | 16 -> 13 | isa, radial_misfit | +| 6oel | open | pass | F 41 3 2 | 2.85 | 9.9 | 36.6% -> 36.5% | 0.998 | 0.00 | 0.248 | 0.255 | 0.14 | - -> 0.70 | - | - | - | - | - | - | 57 -> 40 | time | +| 6p8p | open | pass | P 4 | 1.47 -> 1.46 | 15.3 -> 15.5 | 12.4% -> 13.2% | 0.998 | 0.14 | 0.192 -> 0.189 | 0.209 -> 0.204 | 0.18 -> 0.14 | - -> 1.89 | - | - | - | - | - | - | 25 -> 20 | r_meas | +| 6pxb | open | pass | P 31 1 2 | 1.39 | 12.0 -> 12.1 | 8.8% | 0.999 | 0.23 | 0.245 -> 0.240 | 0.277 -> 0.279 | 0.31 -> 0.26 | - -> 0.62 | - | - | - | - | - | - | 38 -> 16 | radial_misfit, time | +| 6pxc | open | pass | I 2 2 2 | 1.43 -> 1.41 | 10.1 -> 10.2 | 8.5% -> 8.1% | 0.996 -> 0.998 | 0.23 | 0.209 -> 0.210 | 0.215 -> 0.214 | 0.17 -> 0.16 | - -> 0.69 | - | - | - | - | - | - | 36 -> 32 | completeness | +| 6qaj | open | pass | C 2 2 21 | 2.70 | 13.1 -> 13.2 | 24.6% -> 24.4% | 0.997 | 0.41 | 0.376 -> 0.321 | 0.390 -> 0.332 | 0.48 -> 0.23 | - -> 4.46 | - | - | - | - | - | - | 67 -> 65 | rmodel_shell_scaled, radial_misfit, rfree | +| 6r72 | open | pass | P 1 21 1 | 4.39 | 15.7 -> 17.9 | 14.8% -> 16.5% | 0.999 -> 1.000 | 1.32 | 0.374 -> 0.362 | 0.383 -> 0.367 | 0.06 -> 0.04 | - -> 0.13 | - | - | - | - | - | - | 29 -> 26 | isa, r_meas, rmodel_shell_scaled, rfree | +| 6rlr | open | pass | P 1 | 2.07 -> 1.92 | 13.2 -> 16.4 | 11.0% -> 12.9% | 0.998 | 0.03 | 0.249 -> 0.250 | 0.258 -> 0.254 | 0.08 -> 0.07 | - -> 0.68 | - | - | - | - | - | - | 18 | d_min, isa, r_meas | +| 6s1u | open | pass | P 1 21 1 | 1.75 | 13.7 -> 13.8 | 22.4% -> 22.3% | 0.993 -> 0.989 | 0.13 | 0.204 -> 0.202 | 0.208 -> 0.200 | 0.09 | - -> 0.60 | - | - | - | - | - | - | 36 -> 13 | rfree, time | +| 6toc | open | pass | P 42 2 2 | 1.64 | 24.3 -> 24.8 | 9.4% | 1.000 | 0.30 | 0.276 | 0.269 -> 0.270 | 0.14 | - -> 0.54 | - | - | - | - | - | - | 49 -> 11 | time | +| 6u7g | open | pass | P 1 21 1 | 1.92 -> 1.88 | 13.2 -> 13.3 | 8.3% -> 8.0% | 0.998 | 0.13 | 0.201 -> 0.198 | 0.206 -> 0.200 | 0.14 -> 0.11 | - -> 1.57 | - | - | - | - | - | - | 70 -> 71 | d_min, completeness, rfree | +| 6ukf | open | pass | P 1 21 1 | 0.95 -> 0.96 | 9.4 -> 9.8 | 10.7% -> 9.5% | 0.997 -> 0.998 | 0.07 -> 0.08 | 0.166 -> 0.165 | 0.162 -> 0.161 | 0.09 -> 0.07 | - -> 2.26 | - | - | - | - | - | - | 54 -> 48 | r_meas | +| 6vww | open | pass | P 63 | 2.00 -> 1.99 | 8.6 -> 8.0 | 15.9% -> 16.4% | 0.993 | 0.20 | 0.239 -> 0.240 | 0.241 -> 0.243 | 0.10 -> 0.12 | - -> 0.67 | - | - | - | - | - | - | 24 -> 15 | isa | +| 6wzo | open | pass | P 1 | 1.06 -> 1.04 | 16.9 -> 16.8 | 5.8% -> 6.2% | 0.998 -> 0.999 | 0.04 | 0.179 -> 0.174 | 0.186 -> 0.175 | 0.24 -> 0.12 | - -> 2.68 | - | - | - | - | - | - | 57 -> 54 | d_min, r_meas, completeness, radial_misfit, rfree | +| 6yqf | open | pass | P 21 21 2 | 3.05 -> 3.02 | 3.9 -> 4.2 | 50.3% -> 70.0% | 0.988 -> 0.982 | 0.71 | 0.432 -> 0.434 | 0.470 -> 0.442 | 0.33 -> 0.16 | - -> -0.20 | - | - | - | - | - | - | 24 -> 23 | isa, r_meas, cc_half, radial_misfit, rfree | +| 6z8o | open | pass | P 1 21 1 | 2.23 -> 2.21 | 13.8 -> 13.9 | 17.1% -> 17.4% | 0.995 | 0.63 | 0.290 -> 0.276 | 0.295 -> 0.281 | 0.09 -> 0.07 | - -> 1.24 | - | - | - | - | - | - | 62 -> 64 | rmodel_shell_scaled, rfree | +| 6ze4 | open | pass | P 21 21 21 | 1.30 | 9.2 -> 9.3 | 20.4% -> 20.2% | 0.992 -> 0.993 | 0.59 | 0.232 -> 0.213 | 0.237 -> 0.218 | 0.11 -> 0.08 | - -> 1.79 | - | - | - | - | - | - | 73 -> 75 | rmodel_shell_scaled, rfree | +| 6zqr | open | pass | P 4 | 1.79 -> 1.76 | 8.7 -> 9.1 | 19.6% -> 19.3% | 0.996 | 0.40 | 0.190 -> 0.197 | 0.198 -> 0.204 | 0.12 -> 0.17 | - -> 1.35 | - | - | - | - | - | - | 45 -> 17 | rmodel_shell_scaled, rfree, time | +| 6zqy | open | pass | P 4 | 1.72 -> 1.70 | 13.8 -> 12.7 | 18.5% -> 23.3% | 0.997 -> 0.994 | 0.12 | 0.212 -> 0.213 | 0.226 -> 0.222 | 0.20 -> 0.16 | - -> 1.36 | - | - | - | - | - | - | 62 -> 32 | isa, r_meas, time | +| 6zr0 | open | pass | P 4 | 1.73 -> 1.66 | 24.3 -> 24.5 | 11.1% -> 13.1% | 0.997 -> 0.993 | 0.08 | 0.217 -> 0.216 | 0.224 -> 0.218 | 0.14 -> 0.07 | - -> 1.80 | - | - | - | - | - | - | 61 -> 57 | d_min, r_meas, completeness, radial_misfit, rfree | +| 7arr | open | pass | P 1 | 0.92 | 17.3 -> 18.9 | 4.7% -> 3.7% | 0.993 -> 0.999 | 0.28 | 0.155 -> 0.154 | 0.175 -> 0.166 | 0.27 -> 0.20 | - -> 0.00 | - | - | - | - | - | - | 41 -> 35 | isa, r_meas, cc_half, radial_misfit, rfree | +| 7atg | open | pass | P 21 21 21 | 0.60 | 22.5 -> 22.9 | 5.1% -> 5.2% | 0.995 | 0.08 | 0.135 -> 0.127 | 0.199 -> 0.176 | 0.23 -> 0.20 | - -> 6.47 | - | - | - | - | - | - | 41 -> 28 | rmodel_shell_scaled, rfree, time | +| 7bgt | open | pass | P 1 | 1.77 -> 1.78 | 14.7 -> 17.4 | 15.0% -> 13.7% | 0.989 -> 0.993 | 0.32 | 0.197 -> 0.195 | 0.207 | 0.12 | - -> 0.23 | - | - | - | - | - | - | 32 -> 12 | isa, r_meas, time | +| 7brr | open | pass | P 1 21 1 | 1.27 -> 1.24 | 15.8 -> 16.6 | 6.5% -> 6.6% | 0.999 | 0.09 | 0.191 -> 0.192 | 0.195 -> 0.191 | 0.15 -> 0.10 | - -> 3.07 | - | - | - | - | - | - | 25 -> 17 | d_min, completeness, radial_misfit | +| 7mzt | open | fail -> unscored | P 21 21 21 | 3.25 -> 3.12 | 3.4 -> 4.1 | 269.6% -> 366.1% | 0.968 -> 0.971 | 0.56 | 0.452 -> 0.414 | 0.466 -> 0.426 | 0.16 -> 0.10 | - -> -0.04 | - | - | - | - | - | - | 37 -> 33 | verdict fail->unscored, d_min, isa, r_meas, rmodel_shell_scaled, radial_misfit, rfree | +| 7n0i | open | pass | P 21 21 21 | 1.65 -> 1.68 | 10.7 -> 10.9 | 14.0% -> 14.5% | 0.998 | 0.22 | 0.290 -> 0.267 | 0.310 -> 0.295 | 0.36 -> 0.26 | - -> 0.53 | - | - | - | - | - | - | 58 -> 27 | rmodel_shell_scaled, radial_misfit, rfree, time | +| 7n2s | open | pass | P 1 21 1 | 2.57 -> 2.56 | 9.6 | 52.8% -> 56.7% | 0.903 -> 0.917 | 0.07 | 0.315 -> 0.299 | 0.334 -> 0.312 | 0.14 -> 0.11 | - -> 0.34 | - | - | - | - | - | - | 42 -> 12 | r_meas, cc_half, rmodel_shell_scaled, rfree, time | +| 7orr | open | pass | I 2 3 | 1.65 -> 1.62 | 25.5 -> 26.5 | 6.4% -> 6.7% | 1.000 | 0.03 | 0.181 -> 0.183 | 0.190 -> 0.196 | 0.14 -> 0.16 | - -> 5.08 | - | - | - | - | - | - | 23 -> 20 | r_meas, rfree | +| 7ou1 | open | pass | P 1 21 1 | 1.41 -> 1.40 | 9.5 -> 9.0 | 16.1% -> 15.9% | 0.991 | 0.04 | 0.206 -> 0.202 | 0.211 -> 0.209 | 0.09 -> 0.10 | - -> 1.52 | - | - | - | - | - | - | 44 -> 22 | time | +| 7ph1 | open | pass | I 2 2 2 | 1.08 | 15.9 -> 16.0 | 11.4% -> 11.3% | 0.998 | 0.12 | 0.160 -> 0.155 | 0.178 -> 0.174 | 0.17 -> 0.16 | - -> 3.21 | - | - | - | - | - | - | 87 -> 77 | rmodel_shell_scaled | +| 7pq7 | open | pass | C 1 2 1 | 1.38 -> 1.37 | 13.7 -> 14.0 | 7.1% -> 6.5% | 0.998 | 0.22 | 0.189 -> 0.188 | 0.209 -> 0.215 | 0.18 -> 0.19 | - -> 2.32 | - | - | - | - | - | - | 16 -> 14 | r_meas, rfree | +| 7qij | open | pass | P 21 21 21 | 3.59 | 9.4 | 24.5% -> 24.1% | 0.996 | 0.36 | 0.374 -> 0.357 | 0.381 -> 0.364 | 0.07 | - -> 0.18 | - | - | - | - | - | - | 94 -> 92 | rmodel_shell_scaled, rfree | +| 7ris | open | pass | P 31 2 1 | 1.51 | 30.2 -> 31.0 | 10.0% | 1.000 | 0.08 | 0.196 -> 0.188 | 0.195 -> 0.189 | 0.03 -> 0.05 | - -> 2.26 | - | - | - | - | - | - | 29 -> 24 | rmodel_shell_scaled, rfree | +| 7tcd | open | pass | C 1 2 1 | 1.72 -> 1.65 | 15.0 -> 18.4 | 10.0% -> 8.6% | 0.999 | 0.20 | 0.205 -> 0.191 | 0.211 | 0.14 -> 0.20 | - -> 1.17 | - | - | - | - | - | - | 32 -> 28 | d_min, isa, r_meas, completeness, rmodel_shell_scaled, radial_misfit | +| 8a1a | open | pass | P 61 | 1.95 -> 1.93 | 17.9 -> 18.8 | 41.5% -> 52.0% | 0.998 -> 0.997 | 0.42 | 0.171 -> 0.177 | 0.177 -> 0.179 | 0.06 -> 0.03 | - -> 1.82 | - | - | - | - | - | - | 100 -> 94 | r_meas, rmodel_shell_scaled | +| 8agq | open | pass | C 1 2 1 | 0.97 | 14.0 -> 14.1 | 9.8% -> 9.2% | 0.998 -> 0.999 | 0.30 | 0.180 -> 0.181 | 0.204 | 0.21 | - -> 6.46 | - | - | - | - | - | - | 29 -> 26 | r_meas | +| 8dqb | open | pass | I 2 3 | 2.06 -> 2.05 | 22.3 -> 22.1 | 13.5% -> 14.3% | 0.996 | 0.08 | 0.230 | 0.240 -> 0.237 | 0.13 -> 0.11 | - -> 3.08 | - | - | - | - | - | - | 26 -> 14 | r_meas, time | +| 8dz7 | open | pass | P 21 21 21 | 1.19 | 33.7 -> 35.1 | 3.2% -> 3.0% | 0.998 -> 0.999 | 0.08 | 0.119 -> 0.118 | 0.129 -> 0.128 | 0.07 | - -> 5.14 | - | - | - | - | - | - | 15 -> 10 | r_meas | +| 8egn | open | pass | P 21 21 21 | 1.64 -> 1.63 | 22.7 -> 22.9 | 6.1% -> 6.3% | 0.999 | 0.07 | 0.200 -> 0.197 | 0.214 -> 0.215 | 0.16 -> 0.15 | - -> 2.53 | - | - | - | - | - | - | 15 -> 14 | completeness | +| 8k1g | open | pass | I 4 2 2 | 1.63 | 11.6 -> 11.7 | 25.7% -> 25.6% | 0.999 | 0.82 | 0.219 -> 0.206 | 0.236 -> 0.227 | 0.23 -> 0.20 | - -> 2.26 | - | - | - | - | - | - | 48 -> 43 | rmodel_shell_scaled, rfree | +| 8oic | open | pass | P 1 | 2.37 -> 2.34 | 19.6 -> 20.0 | 20.4% -> 24.4% | 0.991 -> 0.988 | 0.02 | 0.234 -> 0.238 | 0.250 -> 0.248 | 0.15 -> 0.11 | - -> 0.41 | - | - | - | - | - | - | 42 | r_meas | +| 8qj5 | open | pass | P 1 21 1 | 1.35 -> 1.29 | 8.4 | 17.1% -> 17.7% | 0.996 | 0.45 | 0.189 -> 0.192 | 0.203 -> 0.206 | 0.14 -> 0.12 | - -> 1.50 | - | - | - | - | - | - | 59 -> 20 | d_min, completeness, time | +| 8qq7 | open | pass | P 62 2 2 | 3.16 | 6.3 -> 6.4 | 18.8% -> 19.0% | 0.993 -> 0.997 | 0.65 | 0.457 -> 0.432 | 0.464 -> 0.429 | 0.50 -> 0.37 | - -> 0.75 | - | - | - | - | - | - | 17 -> 13 | rmodel_shell_scaled, radial_misfit, rfree | +| 8r5r | open | pass | P 21 21 21 | 2.89 -> 2.80 | 15.1 -> 19.3 | 27.8% -> 22.7% | 0.997 -> 0.998 | 0.11 -> 0.04 | 0.274 -> 0.263 | 0.290 -> 0.284 | 0.14 | - -> 0.43 | - | - | - | - | - | - | 35 -> 30 | d_min, isa, r_meas, rmodel_shell_scaled, rfree | +| 8rud | open | pass | P 1 21 1 | 1.70 -> 1.57 | 12.3 -> 11.9 | 29.5% -> 37.8% | 0.992 -> 0.991 | 0.40 | 0.244 -> 0.253 | 0.255 -> 0.261 | 0.11 -> 0.08 | - -> 0.69 | - | - | - | - | - | - | 284 -> 236 | d_min, r_meas, rmodel_shell_scaled, rfree | +| 8sa8 | open | pass | I 1 2 1 | 1.11 -> 1.10 | 22.0 | 11.2% -> 11.7% | 0.999 | 0.02 | 0.158 -> 0.155 | 0.175 -> 0.169 | 0.17 -> 0.14 | - -> 3.79 | - | - | - | - | - | - | 99 -> 94 | rfree | +| 8sqo | open | pass | P 4 3 2 | 1.33 -> 1.32 | 14.2 | 22.1% -> 23.4% | 1.000 | 0.13 | 0.183 | 0.211 -> 0.206 | 0.20 -> 0.18 | - -> 7.02 | - | - | - | - | - | - | 72 | r_meas, rfree | +| 8sqt | open | pass | F 4 3 2 | 1.89 -> 1.88 | 29.6 -> 29.9 | 18.9% -> 20.1% | 0.999 | 0.11 | 0.221 -> 0.223 | 0.224 -> 0.226 | 0.12 -> 0.11 | - -> 2.08 | - | - | - | - | - | - | 30 -> 28 | r_meas | +| 8t7r | open | pass | C 1 2 1 | 3.22 -> 3.23 | 9.9 -> 7.8 | 35.3% -> 31.2% | 0.984 | 0.28 | 0.288 -> 0.283 | 0.288 -> 0.285 | 0.06 -> 0.08 | - -> 0.43 | - | - | - | - | - | - | 66 -> 63 | isa, r_meas | +| 8tha | open | pass | P 62 | 1.34 -> 1.33 | 25.9 -> 27.7 | 13.7% -> 16.3% | 1.000 -> 0.999 | 0.17 | 0.208 -> 0.212 | 0.225 -> 0.220 | 0.16 -> 0.09 | - -> 4.37 | - | - | - | - | - | - | 23 -> 19 | isa, r_meas, radial_misfit | +| 8u0i | open | pass | P 41 21 2 | 1.40 -> 1.38 | 16.1 -> 17.0 | 7.9% -> 8.1% | 0.999 | 0.06 | 0.176 -> 0.179 | 0.180 -> 0.183 | 0.07 -> 0.05 | - -> 3.37 | - | - | - | - | - | - | 24 -> 18 | isa | +| 8v4o | open | pass | P 61 2 2 | 2.10 | 15.5 | 31.9% | 0.998 | 0.06 | 0.241 -> 0.236 | 0.253 | 0.16 | - -> 0.96 | - | - | - | - | - | - | 92 -> 63 | time | +| 8xbp | open | pass | C 1 2 1 | 1.72 -> 1.66 | 16.2 -> 18.6 | 10.8% -> 11.2% | 0.999 | 0.15 | 0.315 -> 0.313 | 0.325 -> 0.329 | 0.15 -> 0.18 | - -> 1.21 | - | - | - | - | - | - | 22 -> 21 | d_min, isa, completeness | +| 8xtf | open | pass | R 3 2:H | 1.84 | 7.5 -> 7.6 | 85.8% -> 73.3% | 0.987 -> 0.989 | 0.22 | 0.191 | 0.185 -> 0.186 | 0.04 | - -> 1.40 | - | - | - | - | - | - | 38 -> 31 | r_meas | +| 8y74 | open | pass | C 1 2 1 | 1.71 -> 1.68 | 8.8 -> 8.6 | 12.3% -> 13.0% | 0.997 | 0.34 | 0.212 -> 0.214 | 0.224 -> 0.225 | 0.12 -> 0.10 | - -> 1.23 | - | - | - | - | - | - | 24 -> 11 | r_meas, time | +| 8ys9 | open | pass | P 21 21 21 | 1.35 -> 1.31 | 15.4 | 11.0% -> 13.3% | 0.999 | 0.16 | 0.173 -> 0.176 | 0.191 -> 0.188 | 0.14 -> 0.10 | - -> 3.11 | - | - | - | - | - | - | 31 -> 29 | d_min, r_meas | +| 9b22 | open | pass | P 1 21 1 | 1.18 -> 1.14 | 16.3 -> 16.4 | 6.0% -> 6.2% | 0.999 | 0.07 | 0.161 -> 0.159 | 0.181 -> 0.166 | 0.20 -> 0.12 | - -> 0.16 | - | - | - | - | - | - | 21 -> 17 | d_min, completeness, radial_misfit, rfree | +| 9bn8 | open | pass | P 41 | 1.20 -> 1.21 | 19.8 | 7.5% -> 8.1% | 0.999 -> 1.000 | 0.09 | 0.151 | 0.172 -> 0.159 | 0.17 -> 0.10 | - -> 3.77 | - | - | - | - | - | - | 30 -> 27 | r_meas, radial_misfit, rfree | +| 9c18 | open | pass | P 1 | 1.76 -> 1.71 | 10.2 -> 10.8 | 23.4% -> 27.3% | 0.988 -> 0.986 | 0.61 | 0.222 -> 0.221 | 0.221 | 0.06 -> 0.07 | - -> 0.71 | - | - | - | - | - | - | 13 -> 11 | d_min, isa, r_meas | +| 9crw | open | pass | P 1 21 1 | 2.29 -> 2.28 | 15.5 -> 15.8 | 8.7% | 0.999 | 0.23 | 0.257 -> 0.247 | 0.271 -> 0.262 | 0.14 -> 0.13 | - -> 0.69 | - | - | - | - | - | - | 19 -> 16 | rmodel_shell_scaled, rfree | +| 9e2t | open | pass | P 1 | 2.33 -> 2.29 | 6.9 -> 6.8 | 28.1% -> 29.1% | 0.990 -> 0.992 | 0.06 | 0.248 -> 0.230 | 0.254 -> 0.232 | 0.11 -> 0.05 | - -> 0.42 | - | - | - | - | - | - | 70 -> 67 | rmodel_shell_scaled, radial_misfit, rfree | +| 9fcf | open | pass | P 4 | 1.81 -> 1.75 | 7.8 -> 7.0 | 28.6% -> 34.5% | 0.995 -> 0.994 | 0.04 | 0.285 -> 0.292 | 0.293 -> 0.295 | 0.06 | - -> 0.85 | - | - | - | - | - | - | 142 -> 146 | d_min, isa, r_meas, rmodel_shell_scaled | +| 9fcg | open | pass | P 4 | 1.39 -> 1.38 | 9.3 -> 9.7 | 15.4% -> 14.1% | 0.996 -> 0.997 | 0.10 -> 0.07 | 0.179 -> 0.180 | 0.184 -> 0.189 | 0.09 -> 0.11 | - -> 2.42 | - | - | - | - | - | - | 138 -> 48 | r_meas, time | +| 9fhc | open | pass | I 2 3 | 1.97 -> 1.92 | 11.6 -> 12.3 | 22.4% -> 17.0% | 0.995 -> 0.998 | 0.25 | 0.245 -> 0.239 | 0.250 -> 0.244 | 0.04 -> 0.07 | - -> 1.20 | - | - | - | - | - | - | 104 -> 107 | d_min, isa, r_meas, completeness, rmodel_shell_scaled, rfree | +| 9gdj | open | pass | P 41 21 2 | 1.40 | 12.8 -> 12.9 | 10.9% -> 11.2% | 0.999 | 0.16 | 0.153 -> 0.154 | 0.178 -> 0.172 | 0.24 -> 0.20 | - -> 2.89 | - | - | - | - | - | - | 355 -> 289 | rfree | +| 9gjx | open | pass | P 1 21 1 | 2.15 -> 2.06 | 35.2 -> 40.1 | 12.5% -> 16.3% | 0.999 -> 0.997 | 0.13 | 0.188 -> 0.193 | 0.225 -> 0.224 | 0.18 -> 0.15 | - -> 0.84 | - | - | - | - | - | - | 33 -> 28 | d_min, isa, r_meas, completeness | +| 9h0q | open | pass | R 3 2:H | 2.13 -> 2.10 | 18.7 -> 18.6 | 15.4% -> 17.4% | 0.998 | 0.41 | 0.203 -> 0.205 | 0.233 -> 0.214 | 0.18 -> 0.11 | - -> 1.10 | - | - | - | - | - | - | 72 -> 44 | r_meas, radial_misfit, rfree, time | +| 9hnc | open | pass -> fail | P 1 2 1 -> P 1 21 1 | 1.65 -> 1.63 | 1.4 -> 13.6 | 52.8% -> 12.8% | 0.882 -> 0.998 | 0.13 -> 0.09 | 0.461 -> 0.246 | 0.465 -> 0.255 | 0.16 -> 0.15 | - -> 0.88 | - | - | - | - | - | - | 80 -> 66 | verdict pass->fail, space group, isa, r_meas, cc_half, completeness, rmodel_shell_scaled, rfree | +| 9hs7 | open | pass | P 61 | 1.68 -> 1.70 | 13.3 | 11.8% -> 11.7% | 0.999 | 0.08 | 0.251 -> 0.213 | 0.279 -> 0.236 | 0.58 -> 0.21 | - -> 1.26 | - | - | - | - | - | - | 27 -> 26 | rmodel_shell_scaled, radial_misfit, rfree | +| 9i0a | open | pass | P 21 21 2 | 1.81 | 13.8 -> 14.1 | 16.8% -> 16.9% | 0.999 | 0.43 | 0.228 -> 0.224 | 0.239 -> 0.233 | 0.17 -> 0.12 | - -> 0.95 | - | - | - | - | - | - | 59 | radial_misfit, rfree | +| 9i80 | open | pass | P 41 | 1.61 -> 1.59 | 6.7 -> 6.8 | 23.2% -> 25.1% | 0.991 -> 0.992 | 0.07 | 0.216 -> 0.219 | 0.221 | 0.15 -> 0.12 | - -> 1.89 | - | - | - | - | - | - | 158 -> 116 | r_meas, time | +| 9ig7 | open | pass | P 21 21 2 | 2.03 -> 2.02 | 11.0 -> 11.1 | 18.7% -> 21.7% | 0.991 | 0.17 | 0.235 -> 0.234 | 0.255 -> 0.249 | 0.18 -> 0.08 | - -> 0.80 | - | - | - | - | - | - | 85 -> 82 | r_meas, radial_misfit, rfree | +| 9jzo | open | pass | P 1 | 1.15 | 8.4 -> 8.1 | 9.2% -> 5.8% | 0.994 -> 0.997 | 0.22 | 0.175 -> 0.179 | 0.175 -> 0.176 | 0.08 -> 0.05 | - -> 4.39 | - | - | - | - | - | - | 17 -> 11 | r_meas | +| 9khr | open | pass | P 21 21 21 | 1.38 | 9.4 -> 9.5 | 18.9% -> 18.8% | 0.994 | 0.12 | 0.248 -> 0.244 | 0.261 -> 0.259 | 0.11 -> 0.12 | - -> 2.02 | - | - | - | - | - | - | 33 -> 13 | time | +| 9min | open | fail | P 21 21 2 | 1.87 -> 1.86 | 10.4 -> 10.5 | 26.6% -> 26.5% | 0.998 -> 0.999 | 36.97 | 0.573 -> 0.566 | 0.578 -> 0.575 | 0.20 -> 0.21 | - -> -0.00 | - | - | - | - | - | - | 77 -> 73 | rmodel_shell_scaled | +| 9o0h | open | pass | P 21 21 21 | 2.01 -> 2.02 | 6.1 -> 5.8 | 52.6% -> 65.1% | 0.985 | 0.25 | 0.247 -> 0.242 | 0.255 -> 0.249 | 0.04 -> 0.08 | - -> 0.11 | - | - | - | - | - | - | 64 -> 59 | isa, r_meas, rmodel_shell_scaled, rfree | +| 9p7q | open | pass | C 1 2 1 | 1.81 -> 1.76 | 10.6 -> 10.8 | 25.7% -> 23.8% | 0.988 -> 0.985 | 0.14 | 0.270 -> 0.269 | 0.285 -> 0.282 | 0.03 -> 0.04 | - -> 0.87 | - | - | - | - | - | - | 15 -> 12 | d_min, r_meas, completeness | +| 9pbb | open | pass | C 1 2 1 | 1.83 -> 1.78 | 14.5 -> 17.0 | 20.7% -> 19.1% | 0.995 -> 0.993 | 0.15 | 0.236 -> 0.233 | 0.236 -> 0.231 | 0.04 | - -> 0.67 | - | - | - | - | - | - | 19 -> 16 | d_min, isa, r_meas, completeness | +| 9q41 | open | pass | C 2 2 21 | 1.69 -> 1.68 | 7.8 -> 7.9 | 27.3% -> 29.0% | 0.984 | 0.20 | 0.187 -> 0.188 | 0.191 -> 0.194 | 0.09 -> 0.10 | - -> 0.48 | - | - | - | - | - | - | 57 -> 23 | r_meas, time | +| 9q66 | open | pass | P 1 21 1 | 2.04 -> 2.03 | 13.0 -> 13.1 | 31.7% -> 32.1% | 0.990 | 0.34 | 0.204 | 0.216 | 0.09 -> 0.08 | - -> 0.66 | - | - | - | - | - | - | 68 -> 30 | time | +| 9qw8 | open | pass | P 1 | 1.59 -> 1.71 | 11.3 -> 9.2 | 14.6% -> 20.0% | 0.994 -> 0.989 | 0.12 -> 0.46 | 0.237 -> 0.294 | 0.248 -> 0.312 | 0.10 -> 0.15 | - -> 0.33 | - | - | - | - | - | - | 73 -> 66 | d_min, isa, r_meas, cc_half, cell_dev_pct, rmodel_shell_scaled, radial_misfit, rfree | +| 9rci | open | pass | P 1 | 1.82 -> 1.76 | 6.0 -> 7.0 | 29.4% -> 29.6% | 0.942 -> 0.951 | 97.59 -> 97.58 | 0.576 -> 0.561 | 0.571 -> 0.555 | 0.09 -> 0.08 | - -> -0.02 | - | - | - | - | - | - | 25 -> 22 | d_min, isa, cc_half, rmodel_shell_scaled, rfree | +| 9rcs | open | pass | P 1 21 1 | 3.37 -> 3.27 | 6.0 -> 7.3 | 24.3% -> 23.4% | 0.992 -> 0.994 | 0.93 | 0.383 -> 0.318 | 0.442 -> 0.341 | 0.19 -> 0.12 | - -> 0.38 | - | - | - | - | - | - | 55 -> 51 | d_min, isa, rmodel_shell_scaled, radial_misfit, rfree | +| 9rp9 | open | pass | C 1 2 1 | 1.91 -> 1.90 | 28.8 -> 32.7 | 16.1% -> 17.4% | 0.997 | 0.16 | 0.206 -> 0.204 | 0.224 -> 0.233 | 0.08 -> 0.14 | - -> 2.47 | - | - | - | - | - | - | 23 -> 25 | isa, r_meas, radial_misfit, rfree | +| 9sl0 | open | pass | P 21 21 21 | 1.38 -> 1.36 | 20.0 -> 20.2 | 7.6% -> 8.0% | 1.000 | 0.37 | 0.245 | 0.256 -> 0.255 | 0.15 -> 0.16 | - -> 2.20 | - | - | - | - | - | - | 50 -> 45 | r_meas | +| 9t6s | open | pass | P 21 21 21 | 1.76 -> 1.75 | 24.9 -> 29.9 | 10.5% -> 11.2% | 0.999 | 0.06 | 0.220 -> 0.215 | 0.236 -> 0.232 | 0.03 -> 0.04 | - -> 2.51 | - | - | - | - | - | - | 28 -> 12 | isa, r_meas, time | +| 9upt | open | pass | P 6 | 2.03 | 6.4 -> 6.5 | 25.9% -> 24.4% | 0.988 -> 0.989 | 0.20 | 0.227 -> 0.218 | 0.229 -> 0.218 | 0.08 -> 0.09 | - -> 1.31 | - | - | - | - | - | - | 76 -> 77 | r_meas, rmodel_shell_scaled, rfree | +| 9vyb | open | pass | P 21 21 21 | 1.71 -> 1.67 | 16.7 -> 22.1 | 8.0% -> 8.2% | 1.000 -> 0.999 | 0.40 | 0.245 -> 0.246 | 0.258 -> 0.253 | 0.07 -> 0.08 | - -> 0.42 | - | - | - | - | - | - | 41 -> 34 | d_min, isa, completeness, rfree | +| 9w3y | open | pass | P 21 21 21 | 1.20 -> 1.19 | 19.9 -> 20.2 | 23.4% -> 24.7% | 0.997 | 0.25 | 0.186 -> 0.187 | 0.200 -> 0.198 | 0.11 -> 0.09 | - -> 4.93 | - | - | - | - | - | - | 17 -> 11 | r_meas | +| 9yl4 | open | pass | P 21 21 21 | 3.61 | 9.5 | 32.0% -> 31.8% | 0.985 -> 0.996 | 0.08 | 0.295 -> 0.293 | 0.306 -> 0.303 | 0.07 | - -> 5.75 | - | - | - | - | - | - | 85 -> 90 | cc_half | +| 9yzk | open | pass | I 1 2 1 | 3.86 -> 3.87 | 7.8 -> 9.3 | 29.7% -> 30.9% | 0.996 -> 0.997 | 0.20 | 0.375 -> 0.355 | 0.394 -> 0.406 | 0.29 | - -> 0.18 | - | - | - | - | - | - | 15 -> 12 | isa, rmodel_shell_scaled, rfree | +| 9z44 | open | pass | I 1 2 1 | 6.86 -> 6.73 | 5.9 -> 8.5 | 26.6% -> 22.3% | 0.953 -> 0.981 | 1.35 | 0.336 -> 0.334 | 0.325 -> 0.324 | 0.17 -> 0.16 | - -> -0.03 | - | - | - | - | - | - | 32 -> 27 | isa, r_meas, cc_half | +| 9z72 | open | pass | P 31 2 1 | 1.97 -> 2.00 | 11.8 | 76.6% -> 80.3% | 0.991 | 0.15 | 0.241 -> 0.240 | 0.256 -> 0.255 | 0.06 -> 0.05 | - -> 0.57 | - | - | - | - | - | - | 152 -> 44 | time | +| 9zlo | open | pass | P 21 21 21 | 1.58 -> 1.56 | 23.0 -> 24.1 | 12.0% -> 12.3% | 0.999 | 0.39 | 0.235 -> 0.224 | 0.241 -> 0.231 | 0.05 -> 0.07 | - -> 1.75 | - | - | - | - | - | - | 26 -> 25 | rmodel_shell_scaled, rfree | +| 9zm0 | open | pass | P 1 21 1 | 1.81 -> 1.80 | 8.6 -> 9.0 | 23.7% -> 25.0% | 0.996 | 0.16 | 0.262 -> 0.255 | 0.283 -> 0.275 | 0.12 -> 0.10 | - -> 4.59 | - | - | - | - | - | - | 11 -> 9 | r_meas, rmodel_shell_scaled, rfree | +| 9zmu | open | pass | P 61 2 2 | 1.71 -> 1.72 | 10.8 -> 11.6 | 34.0% -> 30.1% | 0.999 | 0.25 | 0.276 -> 0.278 | 0.285 -> 0.284 | 0.17 -> 0.15 | - -> 0.89 | - | - | - | - | - | - | 39 -> 31 | isa, r_meas | +| cuhf2 | open | unscored | P 2 2 2 -> P 4 2 2 | 0.52 -> 0.51 | 3.8 -> 45.5 | 17.4% -> 2.8% | 0.988 -> 1.000 | - | - | - | - | - | - | - | - | - | - | - | 56 -> 32 | space group, isa, r_meas, cc_half, completeness, time | +| cytidine | open | pass | P 21 21 21 | 0.58 | 4.5 -> 8.7 | 20.6% -> 10.3% | 0.980 -> 0.993 | 0.31 | - | - | - | - | - | - | 0.1018 -> 0.0613 | 0.3180 -> 0.1762 | 1.124 -> 1.061 | 0.096 -> 0.039 | 29 -> 24 | isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL goof, SHELXL exti | +| dnba | open | pass | C 1 2/c 1 | 0.81 | 4.3 -> 36.8 | 16.7% -> 2.5% | 0.979 -> 1.000 | 0.06 | - | - | - | - | - | - | 0.0607 -> 0.0266 | 0.1531 -> 0.0706 | 1.036 -> 1.072 | 0.004 -> 0.002 | 10 -> 6 | isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL goof | +| lalanine | open | pass | P 21 21 21 | 0.65 | 7.5 -> 42.4 | 12.1% -> 2.6% | 0.994 -> 1.000 | 0.07 | - | - | - | - | - | - | 0.0759 -> 0.0311 | 0.2201 -> 0.0939 | 1.147 -> 1.150 | 0.027 -> 0.000 | 14 -> 9 | isa, r_meas, cc_half, SHELXL r1, SHELXL wr2, SHELXL exti | +| metformin | open | pass | P 1 21/c 1 | 0.51 | 13.6 -> 30.8 | 6.4% -> 3.1% | 0.998 -> 1.000 | 0.17 | - | - | - | - | - | - | 0.0527 -> 0.0322 | 0.1569 -> 0.0960 | 1.073 -> 1.101 | 0.000 | 8 -> 5 | isa, r_meas, SHELXL r1, SHELXL wr2 | +| nidppe | open | pass | P 1 21/c 1 | 0.51 | 25.6 -> 32.9 | 8.2% -> 7.9% | 0.998 -> 0.999 | 0.28 | - | - | - | - | - | - | 0.0441 -> 0.0414 | 0.1171 -> 0.1022 | 1.046 -> 1.036 | 0.006 -> 0.004 | 9 -> 5 | isa, SHELXL wr2 | +| aspirin_x10sa_20keV | inhouse | - -> pass | - -> P 1 21/c 1 | - -> 0.66 | - -> 36.4 | - -> 3.0% | - -> 1.000 | - -> 0.05 | - | - | - | - | - -> 3.0% | - -> 2.2% | - -> 0.0363 | - -> 0.1095 | - -> 1.100 | - -> 0.009 | - -> 9 | only in B | +| aspirin_x10sa_25keV | inhouse | - -> pass | - -> P 1 21/c 1 | - -> 0.53 | - -> 36.4 | - -> 3.1% | - -> 1.000 | - -> 0.05 | - | - | - | - | - -> 3.1% | - -> 2.2% | - -> 0.0358 | - -> 0.1165 | - -> 1.077 | - -> 0.012 | - -> 11 | only in B | +| citricacid_x10sa_20keV | inhouse | - -> pass | - -> P 1 21/c 1 | - -> 0.67 | - -> 24.1 | - -> 3.6% | - -> 0.999 | - -> 0.10 | - | - | - | - | - -> 3.6% | - -> 3.8% | - -> 0.0374 | - -> 0.1026 | - -> 1.092 | - -> 0.149 | - -> 11 | only in B | +| hepes_x10sa_20keV | inhouse | - -> pass | - -> P b c a | - -> 0.66 | - -> 39.6 | - -> 2.6% | - -> 1.000 | - -> 0.07 | - | - | - | - | - -> 2.6% | - -> 3.5% | - -> 0.0310 | - -> 0.0895 | - -> 1.061 | - -> 0.053 | - -> 12 | only in B | +| kdp_x10sa_20keV | inhouse | - -> fail | - -> I 41 m d | - -> 0.66 | - -> 25.1 | - -> 5.2% | - -> 0.999 | - -> 0.07 | - | - | - | - | - -> 5.0% | - -> 4.7% | - -> 0.0475 | - -> 0.1208 | - -> 1.363 | - -> 0.028 | - -> 93 | only in B | +| lcystine_x10sa_20keV | inhouse | - -> pass | - -> P 61 2 2 | - -> 0.67 | - -> 5.7 | - -> 30.6% | - -> 1.000 | - -> 0.49 | - | - | - | - | - | - | - -> 0.1575 | - -> 0.3812 | - -> 1.457 | - -> 0.000 | - -> 57 | only in B | +| lcystine_x10sa_25keV | inhouse | - -> pass | - -> P 61 2 2 | - -> 0.53 | - -> 5.5 | - -> 26.7% | - -> 1.000 | - -> 0.52 | - | - | - | - | - | - | - -> 0.1397 | - -> 0.3903 | - -> 1.465 | - -> 0.000 | - -> 23 | only in B | +| yag_x10sa_20keV | inhouse | - -> pass | - -> I a -3 d | - -> 0.67 | - -> 3.0 | - -> 39.4% | - -> 0.702 | - -> 0.03 | - | - | - | - | - -> 39.4% | - -> 41.8% | - -> 0.1021 | - -> 0.2472 | - -> 1.142 | - -> 0.584 | - -> 17 | only in B | +| 2wnn | open | - -> pass | - -> P 1 21 1 | - -> 1.44 | - -> 14.1 | - -> 7.2% | - -> 0.997 | - -> 0.11 | - -> 0.279 | - -> 0.286 | - -> 0.17 | - -> 1.00 | - | - | - | - | - | - | - -> 76 | only in B | +| 2wnq | open | - -> fail | - -> C 2 2 21 | - -> 1.65 | - -> 10.3 | - -> 10.8% | - -> 0.997 | - -> 68.55 | - -> 0.536 | - -> 0.545 | - -> 0.28 | - -> 0.02 | - | - | - | - | - | - | - -> 63 | only in B | +| 2wnz | open | - -> pass | - -> P 1 21 1 | - -> 1.85 | - -> 14.8 | - -> 12.2% | - -> 0.996 | - -> 0.13 | - -> 0.207 | - -> 0.232 | - -> 0.22 | - -> 0.94 | - | - | - | - | - | - | - -> 61 | only in B | +| 2xfw | open | - -> pass | - -> P 1 21 1 | - -> 1.55 | - -> 14.1 | - -> 14.8% | - -> 0.996 | - -> 0.11 | - -> 0.200 | - -> 0.220 | - -> 0.16 | - -> 1.11 | - | - | - | - | - | - | - -> 93 | only in B | +| 3mc4 | open | - -> pass | - -> R 3:H | - -> 1.79 | - -> 12.5 | - -> 10.0% | - -> 0.994 | - -> 0.06 | - -> 0.252 | - -> 0.256 | - -> 0.06 | - -> 1.40 | - | - | - | - | - | - | - -> 10 | only in B | +| 3meb | open | - -> pass | - -> P 1 21 1 | - -> 1.53 | - -> 15.6 | - -> 13.9% | - -> 0.989 | - -> 0.43 | - -> 0.201 | - -> 0.203 | - -> 1.10 | - -> -0.19 | - | - | - | - | - | - | - -> 6 | only in B | +| 3p85 | open | - -> pass | - -> P 63 2 2 | - -> 1.62 | - -> 8.1 | - -> 12.1% | - -> 0.998 | - -> 0.50 | - -> 0.194 | - -> 0.201 | - -> 0.12 | - -> 7.76 | - | - | - | - | - | - | - -> 10 | only in B | +| 3r6o | open | - -> fail | - -> I 41 2 2 | - -> 1.53 | - -> 4.5 | - -> 19.1% | - -> 0.985 | - -> 0.43 | - -> 0.302 | - -> 0.307 | - -> 0.72 | - -> 1.58 | - | - | - | - | - | - | - -> 5 | only in B | +| 4bwl | open | - -> fail | - -> C 2 2 21 | - -> 1.68 | - -> 14.3 | - -> 14.1% | - -> 0.998 | - -> 72.04 | - -> 0.520 | - -> 0.524 | - -> 0.13 | - -> 0.30 | - | - | - | - | - | - | - -> 59 | only in B | +| 5cc8 | open | - -> fail | - -> P 21 21 21 | - -> 1.53 | - -> 11.2 | - -> 8.0% | - -> 0.997 | - -> 0.06 | - -> 0.171 | - -> 0.184 | - -> 0.17 | - -> 4.01 | - | - | - | - | - | - | - -> 11 | only in B | +| 5jk4 | open | - -> pass | - -> P 1 21 1 | - -> 1.02 | - -> 14.8 | - -> 8.9% | - -> 0.998 | - -> 0.12 | - -> 0.109 | - -> 0.123 | - -> 0.09 | - -> 3.69 | - | - | - | - | - | - | - -> 25 | only in B | +| 5ojv | open | - -> pass | - -> P 21 21 2 | - -> 1.82 | - -> 13.2 | - -> 14.5% | - -> 0.998 | - -> 0.41 | - -> 0.173 | - -> 0.187 | - -> 0.16 | - -> 1.99 | - | - | - | - | - | - | - -> 237 | only in B | +| 5uth | open | - -> pass | - -> P 31 2 1 | - -> 1.72 | - -> 9.5 | - -> 12.4% | - -> 0.997 | - -> 0.21 | - -> 0.194 | - -> 0.217 | - -> 0.17 | - -> 2.80 | - | - | - | - | - | - | - -> 10 | only in B | +| 5vml | open | - -> pass | - -> P 42 21 2 | - -> 1.92 | - -> 10.9 | - -> 10.2% | - -> 0.996 | - -> 0.02 | - -> 0.156 | - -> 0.165 | - -> 0.05 | - -> 3.40 | - | - | - | - | - | - | - -> 6 | only in B | +| 6cee | open | - -> pass | - -> P 21 21 21 | - -> 1.38 | - -> 23.9 | - -> 5.4% | - -> 0.999 | - -> 0.04 | - -> 0.166 | - -> 0.174 | - -> 0.14 | - -> 9.05 | - | - | - | - | - | - | - -> 7 | only in B | +| 6cs9 | open | - -> pass | - -> P 1 21 1 | - -> 1.72 | - -> 13.1 | - -> 9.9% | - -> 0.998 | - -> 0.06 | - -> 0.211 | - -> 0.205 | - -> 0.14 | - -> 0.77 | - | - | - | - | - | - | - -> 29 | only in B | +| 6gvk | open | - -> pass | - -> C 1 2 1 | - -> 1.42 | - -> 19.8 | - -> 5.2% | - -> 0.999 | - -> 0.09 | - -> 0.200 | - -> 0.215 | - -> 0.18 | - -> 1.88 | - | - | - | - | - | - | - -> 94 | only in B | +| 6oww | open | - -> fail | - -> P 41 21 2 | - -> 2.72 | - -> 11.8 | - -> 203.3% | - -> 0.994 | - -> 0.19 | - -> 0.363 | - -> 0.376 | - -> 0.08 | - -> 2.03 | - | - | - | - | - | - | - -> 222 | only in B | +| 6p8j | open | - -> fail | - -> P 21 21 2 | - -> 1.28 | - -> 5.3 | - -> 23.7% | - -> 0.990 | - -> 0.33 | - -> 0.241 | - -> 0.251 | - -> 0.10 | - -> 1.18 | - | - | - | - | - | - | - -> 143 | only in B | +| 6rym | open | - -> pass | - -> P 41 | - -> 1.45 | - -> 22.9 | - -> 4.2% | - -> 0.998 | - -> 0.06 | - -> 0.170 | - -> 0.190 | - -> 0.25 | - -> 14.40 | - | - | - | - | - | - | - -> 16 | only in B | +| 6v2r | open | - -> pass | - -> P 41 21 2 | - -> 1.38 | - -> 24.4 | - -> 5.2% | - -> 1.000 | - -> 0.02 | - -> 0.205 | - -> 0.209 | - -> 0.21 | - -> 12.96 | - | - | - | - | - | - | - -> 8 | only in B | +| 7bgu | open | - -> pass | - -> P 1 | - -> 2.30 | - -> 10.0 | - -> 12.4% | - -> 0.937 | - -> 0.17 | - -> 0.286 | - -> 0.304 | - -> 0.11 | - -> 0.26 | - | - | - | - | - | - | - -> 41 | only in B | +| 7q6j | open | - -> pass | - -> P 21 21 21 | - -> 1.98 | - -> 11.0 | - -> 14.6% | - -> 0.996 | - -> 0.25 | - -> 0.224 | - -> 0.236 | - -> 0.14 | - -> 1.62 | - | - | - | - | - | - | - -> 126 | only in B | +| 8c3e | open | - -> fail | - -> P 6 2 2 | - -> 1.79 | - -> 5.8 | - -> 29.4% | - -> 0.980 | - -> 0.24 | - -> 0.359 | - -> 0.400 | - -> 0.29 | - -> 0.96 | - | - | - | - | - | - | - -> 5 | only in B | +| 8v2t | open | - -> pass | - -> P 42 21 2 | - -> 1.17 | - -> 11.9 | - -> 8.8% | - -> 0.999 | - -> 0.24 | - -> 0.166 | - -> 0.181 | - -> 0.21 | - -> 10.86 | - | - | - | - | - | - | - -> 31 | only in B | +| 8v4j | open | - -> pass | - -> P 42 21 2 | - -> 1.10 | - -> 22.1 | - -> 5.4% | - -> 1.000 | - -> 0.04 | - -> 0.170 | - -> 0.177 | - -> 0.11 | - -> 12.35 | - | - | - | - | - | - | - -> 110 | only in B | +| 9jq9 | open | - -> pass | - -> P 21 21 21 | - -> 1.65 | - -> 19.5 | - -> 6.9% | - -> 0.999 | - -> 0.12 | - -> 0.212 | - -> 0.242 | - -> 0.15 | - -> 4.82 | - | - | - | - | - | - | - -> 6 | only in B | +| 9lxl | open | - -> pass | - -> P 41 21 2 | - -> 2.06 | - -> 6.2 | - -> 41.9% | - -> 0.996 | - -> 0.49 | - -> 0.306 | - -> 0.321 | - -> 0.10 | - -> 0.69 | - | - | - | - | - | - | - -> 60 | only in B | +| 9qvv | open | - -> pass | - -> I 2 2 2 | - -> 2.49 | - -> 37.3 | - -> 11.2% | - -> 1.000 | - -> 0.42 | - -> 0.235 | - -> 0.239 | - -> 0.20 | - -> 1.64 | - | - | - | - | - | - | - -> 110 | only in B | +| 9qw2 | open | - -> pass | - -> P 1 21 1 | - -> 1.76 | - -> 7.8 | - -> 18.5% | - -> 0.958 | - -> 0.77 | - -> 0.240 | - -> 0.245 | - -> 0.04 | - -> 0.54 | - | - | - | - | - | - | - -> 98 | only in B | +| 9s02 | open | - -> pass | - -> P 21 21 2 | - -> 1.42 | - -> 26.0 | - -> 10.2% | - -> 0.999 | - -> 0.06 | - -> 0.180 | - -> 0.189 | - -> 0.11 | - -> 2.25 | - | - | - | - | - | - | - -> 127 | only in B | diff --git a/_sources/CBOR.md.txt b/_sources/CBOR.md.txt new file mode 100644 index 000000000..9bcb8c441 --- /dev/null +++ b/_sources/CBOR.md.txt @@ -0,0 +1,385 @@ +# CBOR messages + +To communicate between the FPGA-equipped receiver system and the writers, +Jungfraujoch uses binary CBOR encoding with the tinycbor library (Intel). +The protocol is based on and compatible with [DECTRIS Stream2](https://github.com/dectris/documentation/tree/main/stream_v2). +There are minor differences at the moment: + +* LZ4 alone is not allowed; Bitshuffle+LZ4 and Bitshuffle+Zstandard are allowed +* A few fields are currently absent +* Extra fields are present beyond DECTRIS standard +* There are calibration and metadata messages defined beyond DECTRIS specification + +## Start message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "start" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| detector_distance | float | Detector distance \[m\] | | +| detector_translation | Array(float) | Detector translation vector \[m\] | X | +| beam_center_x | float | Beam center in X direction \[pixels\] | X | +| beam_center_y | float | Beam center in Y direction \[pixels\] | X | +| direct_beam_x | float (optional) | Where the undeflected beam lands on the detector, X \[pixels\]. Not the same point as `beam_center_x`, which is the PONI - the foot of the perpendicular from the sample - and separates from the beam position as soon as the detector is tilted. This is the number a program that asks for "the beam centre" (XDS `ORGX`, for one) wants | | +| direct_beam_y | float (optional) | Where the undeflected beam lands on the detector, Y \[pixels\] (XDS `ORGY`) | | +| countrate_correction_enabled | bool | Countrate correction enabled | X | +| countrate_correction_lookup_table | uint32 array (optional) | Maps a measured count c to its corrected value \[c\], as sent by a DECTRIS detector | X | +| flatfield_enabled | bool | Flatfield enabled | X | +| virtual_pixel_interpolation_enabled | bool (optional) | Virtual pixel interpolation enabled, as reported by a DECTRIS detector | X | +| number_of_images | uint64 | Number of images in the series | X | +| image_size_x | uint64 | Image width \[pixels\] | X | +| image_size_y | uint64 | Image height \[pixels\] | X | +| mirror_y | bool | Whether the assembled image is mirrored in Y relative to the detector's raw readout order. True is the MX convention - row 0 at the top of the detector seen from the sample - and is what absence of the key means | | +| detector_orientation_mirror_y | bool | Whether the assembled image is mirrored in Y relative to the frame the PONI angles are stated in. A different setting from `mirror_y` above, which is about the module layout; this one changes no pixel. Absence means false | | +| detector_orientation_quarter_turns | int | How many multiples of 90 degrees about the beam the assembled image is turned by, relative to the frame the PONI angles are stated in (0-3). Absence means 0 | | +| incident_energy | float | X-ray energy \[eV\] | X | +| incident_wavelength | float | X-ray wavelength \[Angstrom\] | X | +| incident_wavelength_spread | float (optional) | FWHM of the X-ray wavelength distribution \[Angstrom\] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic | | +| beam_size_x | float (optional) | Horizontal size of the X-ray beam at the sample \[m\] (first element of NXmx incident_beam_size) | | +| beam_size_y | float (optional) | Vertical size of the X-ray beam at the sample \[m\] (second element of NXmx incident_beam_size) | | +| frame_time | float | Frame time, if multiple frames per trigger \[s\] | X | +| count_time | float | Exposure time \[s\] | X | +| saturation_value | int64 | Maximum valid sample value | X | +| error_value | int64 (optional) | Value used in images to describe pixels that are in error state or missing | | +| pixel_size_x | float | Pixel width \[m\] | X | +| pixel_size_y | float | Pixel height \[m\] | X | +| sensor_thickness | float | Sensor thickness \[m\] | X | +| sensor_material | string | Sensor material | X | +| arm_date | date | Approximate date of arming | X | +| pixel_mask_enabled | bool | Pixel mask applied on images | X | +| detector_description | string | Name of the detector | X | +| detector_serial_number | string | Detector serial number | X | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| fluorescence | object (optional) | X-ray fluorescence spectrum collected at start | | +| - energy | Array(float) | Energy of measuring point \[eV\] | | +| - data | Array(float) | Fluorescence scan result `data` \[arbitrary units\]; must be strictly the same length as energy | | +| goniometer | Map | Definition of rotation axis (optional) | X | +| - `AXIS` | string | Rotation axis name (e.g. omega) - only one axis is supported in Jungfraujoch | X | +| - - increment | float | Rotation axis increment (per image) in degree \[deg\] | X | +| - - start | float | Rotation axis start angle \[deg\] | X | +| - - axis | Array(float) | Vector for the rotation axis | | +| - - helical_step | Array(float) | Translation for helical scan for 1 image \[m\] | | +| - - screening_wedge | Array(float) | Wedge for screening \[deg\] (increment would correspond to difference between screening points) | | +| grid_scan | object | Grid scan definition (optional). Send `goniometer` with it, `increment` 0, to state the angle the spindle stood at; without one the spindle is recorded at 0, meaning "nobody said" | | +| - n_fast | uint64 | Number of elements along fast axis | | +| - n_slow | uint64 | Number of elements along slow axis | | +| - step_x_axis | float | Step along X axis, can be negative \[m\] | | +| - step_y_axis | float | Step along Y axis, can be negative \[m\] | | +| - snake_scan | bool | Snake scan (rows alternate direction) | | +| - vertical_scan | bool | Vertical scan (enabled: fast direction = Y, disabled: fast direction = X) | | +| jungfrau_conversion_enabled | bool (optional) | Applying JUNGFRAU pixel conversion (to photons or keV) | | +| jungfrau_conversion_factor | float (optional) | Factor used for JUNGFRAU conversion \[eV\] | | +| geometry_transformation_enabled | bool (optional) | Transformation from detector module geometry (512x1024) to full detector geometry | | +| pixel_mask | Map(string -> Image) | Pixel mask - multiple in case of storage cells | X | +| channels | Array(string) | List of image channels | X | +| max_spot_count | uint64 | Maximum number of spots identified in spot finding | | +| max_extra_lattices | uint64 | Maximum number of extra lattices | | +| storage_cell_number | uint64 (optional) | Number of storage cells used by JUNGFRAU | | +| storage_cell_delay | Rational | Delay of storage cells in JUNGFRAU | | +| threshold_energy | Map(string -> float) | Per-channel threshold energy \[eV\] (map of channel name to value) | | +| image_dtype | string | Pixel type of the image data: `uint8`, `uint16`, `uint32` (DECTRIS), plus `int8`, `int16`, `int32` as a Jungfraujoch extension. Sole wire encoding of both the bit depth and the sign, and must agree with the per-image typed-array tag | X | +| unit_cell | object (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| az_int_q_bin_count | uint64 | Number of azimuthal integration bins in the radial direction | | +| az_int_phi_bin_count | uint64 | Number of azimuthal integration bins in the phi angle direction | | +| az_int_bin_to_q | Array(float) | Q value for each azimuthal integration bin \[angstrom^-1\] | | +| az_int_bin_to_two_theta | Array(float) | Two theta angle value for each azimuthal integration bin \[deg\] | | +| az_int_bin_to_phi | Array(float) | Phi value for each azimuthal integration bin \[deg\] | | +| az_int_map | Image | Mapping between pixel and bin number | | +| summation | uint64 | Factor of frame summation | | +| user_data | string | JSON serialized to string that can contain the following fields (all fields are optional): | X | +| - file_prefix | string | File prefix | | +| - images_per_file | uint64 | Number of images written per file | | +| - images_per_trigger | uint64 | Number of images collected per trigger | | +| - source_name | string | Facility name | | +| - source_type | string | Type of X-ray source (use NXsource/type values, for example "Synchrotron X-ray Source" or "Free-Electron Laser") | | +| - instrument_name | string | Instrument name | | +| - sample_name | string | Name of the sample | | +| - user | any valid JSON | Value of header_appendix provided at collection start to Jungfraujoch | | +| - attenuator_transmission | float | Attenuator transmission \[\] | | +| - total_flux | float | Total flux \[ph/s\] | | +| - space_group_number | uint64 | Space group number | | +| - summation_mode | string | Summation mode (internal\|fpga\|cpu) | | +| - overwrite | bool | Overwrite existing HDF5 files | | +| - file_format | int | File writer format: 0 = only data files, 1 = NXmx legacy external links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF (retired; rejected), 5 = TIFF (retired; rejected), 6 = no file written | | +| - roi | Array(object) | ROI configurations; each element is one of: | | +| | | type "box": xmin, xmax, ymin, ymax (numbers) | | +| | | type "circle": r, x, y (numbers) | | +| | | type "azim": qmin, qmax (numbers); optional phi_min, phi_max (numbers, deg) for an angular sector | | +| - gain_file_names | Array(string) | Names of JUNGFRAU gain files used for the current detector | | +| - write_master_file | bool | With multiple sockets, it selects which socket will provide master file | | +| - write_images | bool | Write images in the HDF5 file (if false, will only write metadata) | | +| - data_reduction_factor_serialmx | uint64 | Data reduction factor for serial MX | | +| - experiment_group | string | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number | | +| - jfjoch_release | string | Jungfraujoch release number | | +| - socket_number | uint64 | Number of ZeroMQ socket (on `jfjoch_broker` side) used for transmission | | +| - bit_depth_readout | uint64 | Bit depth of the **stored image** (see note below), copied to NXmx `bit_depth_readout` | | +| - underload_value | int64 | Lowest valid value; copied to NXmx `underload_value`. `0` for an unsigned image, `INTx_MIN + 1` for a signed one | | +| - writer_notification_zmq_addr | string | ZeroMQ address to inform `jfjoch_broker` about writers that finished operation | | +| - xfel_pulse_id | uint64 | Pulse IDs are recorded for images | | +| - ring_current_mA | float | Ring current at the start of the measurement | | +| - sample_temperature_K | float | Sample temperature \[K\] | | +| - detect_ice_rings | bool | Ice ring detection feature is enabled | | +| - indexing_algorithm | string | Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none | | +| - geom_refinement_algorithm | string | Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center | | +| - poni_rot1 | float | Tilt of the detector rot1 according to PyFAI PONI convention \[rad\] | | +| - poni_rot2 | float | Tilt of the detector rot2 according to PyFAI PONI convention \[rad\] | | +| - poni_rot3 | float | Tilt of the detector rot3 according to PyFAI PONI convention \[rad\] | | + +See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/stream_v2) for definition of Image as MultiDimArray with optional compression. + +## Image message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|-----------------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "image" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| real_time | Rational | Exposure time | X | | +| start_time | Rational | Exposure start time (highly approximate) | X | | +| end_time | Rational | Exposure end time (highly approximate) | X | | +| spots | Array(object) | Spots: | | | +| - x | float | observed position in x (pixels) | | | +| - y | float | observed position in y (pixels) | | | +| - I | float | intensity (photons) | | | +| - maxc | int64 | max count (photons) | | | +| - ice_ring | bool | spot in resolution range for ice rings | | | +| - indexed | bool | indexed solution | | | +| - latt | int64 | Lattice to which the peak belongs (negative number = not indexed) | | | +| - image | int64 | image number the spot belongs to | | | +| - h | int64 | Miller index (indexed spots only) | | | +| - k | int64 | Miller index (indexed spots only) | | | +| - l | int64 | Miller index (indexed spots only) | | | +| - dist_ewald | float | distance to Ewald sphere \[Angstrom^-1\] (indexed spots only) | | | +| reflections | Array(object) | Reflections: | | | +| - h | int64 | Miller index | | | +| - k | int64 | Miller index | | | +| - l | int64 | Miller index | | | +| - x | float | predicted position in x (pixels) | | | +| - y | float | predicted position in y (pixels) | | | +| - obs_x | float | observed position in x (pixels) | | | +| - obs_y | float | observed position in y (pixels) | | | +| - d | float | resolution \[Angstrom\] | | | +| - I | float | integrated intensity (photons) | | | +| - bkg | float | mean background value (photons) | | | +| - var_bkg | float | non-signal (background) part of sigma^2, carried to the merge (photons^2) | | | +| - sigma | float | standard deviation, estimated from counting statistics (photons) | | | +| - image | int64 | image number the reflection belongs to | | | +| - rp | float | Distance to Ewald sphere \[Angstrom^-1\] | | | +| - rlp | float | Reciprocal Lorentz-polarization factor: the multiplier taking the raw integrated count toward a quantity proportional to \|F\|^2. Lorentz x polarization only - a still has no Lorentz term, so there it is the polarization alone | | | +| - qe | float | Sensor efficiency at the reflection's angle of incidence, QE(0)/QE(alpha); <= 1, and 1 where the sensor is opaque or unknown. Carried beside `rlp`, not inside it: the total correction is `rlp * qe`. Optional | | | +| - flight | float | Attenuation of the reflection in the flight path between the sample and its pixel, normalised to normal incidence; >= 1, and exactly 1 for a vacuum path. Carried beside `rlp` and `qe`: the total correction is `rlp * qe * flight`. Optional | | | +| - partiality | float | Partiality of the reflection | | | +| - phi | float | phi angle from XDS: difference from middle of current frame, not absolute \[deg\] | | | +| - zeta | float | Lorentz zeta factor (reciprocal-space geometry term) | | | +| - image_scale_corr | float | Per-image scale correction; I_true = image_scale_corr * I | | | +| spot_count | uint64 | Spot count | | | +| spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | +| spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | +| az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| | | NaN is used for empty bins and has to be taken care by the receiver | | | +| az_int_profile_std | Array(float) | Standard deviation for azimuthal integration. (NaN for less than 2 samples) | | | +| az_int_profile_count | Array(uint64) | Number of pixels contributing to azimuthal bin | | | +| indexing_result | bool | Indexing successful | | | +| indexing_lattice_count | int64 | Number of indexing lattices found for this image | | | +| indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| indexing_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | | +| indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| profile_radius | float | Profile radius of the image - describes distance of observed reflections from the Ewald sphere \[Angstrom^-1\] | | | +| integrated_reflections | int64 | Count of integrated reflections | | | +| mosaicity | float | Angular range of spots in image from a rotation scan \[degree\] | | | +| b_factor | float | Estimated B-factor (Angstrom^2) | | | +| compression_time | float | Time spent on compression/decompressing image \[s\] | | | +| preprocessing_time | float | Time spent on preparing the image for analysis \[s\] | | | +| azint_time | float | Time spent on azimuthal integration \[s\] | | | +| spot_finding_time | float | Time spent on spot finding \[s\] | | | +| indexing_time | float | Time spent on indexing \[s\] | | | +| refinement_time | float | Time spent on refinement of indexing solution and experimental geometry \[s\] | | | +| index_analysis_time | float | Time spent on analyzing indexing solution, calculating profile radius and mosaicity \[s\] | | | +| bragg_prediction_time | float | Time spent on predicting Bragg spots \[s\] | | | +| integration_time | float | Time spent on Bragg integration \[s\] | | | +| image_scale_time | float | Time spent on on-the-fly scaling \[s\] | | | +| processing_time | float | Total processing time \[s\] | | | +| xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| lattice_type | object | Bravais lattice classification of the indexing result (present only if available) | | X | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | | +| jf_info | uint64 | Detector info field | | | +| receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| receiver_free_send_buf | uint64 | Receiver internal number of available buffer locations | | | +| receiver_buf_in_sending | uint64 | Receiver internal number of buffer locations currently in sending/writing | | | +| receiver_buf_in_preparation | uint64 | Receiver internal number of buffer locations currently in processing | | | +| storage_cell | uint64 | Storage cell number | | | +| saturated_pixel_count | uint64 | Saturated pixel count | | | +| pixel_sum | uint64 | Sum of all pixels, excl. error and saturation | | | +| error_pixel_count | uint64 | Error pixel count | | | +| strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| min_viable_pixel_value | int64 | Minimal pixel value, excl. error and saturation | | | +| max_viable_pixel_value | int64 | Maximal pixel value, excl. error and saturation | | | +| resolution_estimate | float | Resolution the merged data are predicted to reach, from this image's spots alone \[Angstrom\] | | X | +| data_collection_efficiency | float | Image collection efficiency \[\] | | | +| packets_expected | uint64 | Number of packets expected per image (in units of 2 kB) | | | +| packets_received | uint64 | Number of packets received per image (in units of 2 kB) | | | +| bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] | | | +| ice_ring_score | float | Strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | | | +| spindle_blind_fraction | float | Fraction (0-1) of a rotation sweep's blind cone this orientation makes unrecoverable, as a lone-2-fold worst-case bound; >= 0.5 should engage a recovery protocol, and ABSENT means the frame could not be assessed, which automation must treat the same way | | | +| spot_count_ice_control | float | Spots in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands' own q width (control for spot_count_ice_rings) | | | +| beam_corr_x | float | Beam center correction X applied during processing \[pixel\] | | X | +| beam_corr_y | float | Beam center correction Y applied during processing \[pixel\] | | X | +| image_scale_factor | float | Scaling result: Image scale factor (g) | | X | +| image_scale_mosaicity | float | Scaling result: Image scale mosaicity \[deg\] | | X | +| image_scale_cc | float | Scaling result: Image scale CC | | X | +| adu_histogram | Array(uint64) | ADU histogram | | | +| roi_integrals | object | Results of ROI calculation | | X | +| - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - pixels | uint64 | Valid pixels in ROI area | | | +| - max_count | int64 | Highest count in ROI area \[photons\] | | | +| - x_weighted_sum | int64 | ROI pixel X position multiplied by photon count \[photons * pixels\] | | | +| - y_weighted_sum | int64 | ROI pixel Y position multiplied by photon count \[photons * pixels\] | | | +| user_data | string | Optional user defined text information - this is image_appendix serialized to JSON format | X | | +| data | Map(string -> Image) | Image | X | | + +## Metadata message + +| Field name | Type | Description | Present in DECTRIS format | Optional | +|------------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:|:--------:| +| type | String | value "metadata" | X | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | | +| images | Array(object) | Array of images (order and size of the array are not guaranteed) | X | | +| - image_id | uint64 | Number of image within the series; for MX lossy compression this is sequential excluding removed frames | X | | +| - original_image_id | uint64 | Number of image within the series; for MX lossy compression this includes removed frames in the count | | | +| - real_time | Rational | Exposure time | X | | +| - start_time | Rational | Exposure start time (highly approximate) | X | | +| - end_time | Rational | Exposure end time (highly approximate) | X | | +| - spot_count | uint64 | Spot count | | | +| - spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | +| - az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | +| - indexing_result | bool | Indexing successful | | | +| - indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | +| - indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | +| | | Unit cell is redundant to lattice - yet to simplify downstream programs to analyze results, both are provided | | | +| - xfel_pulse_id | uint64 | Bunch ID (for pulsed source, e.g., SwissFEL) | | X | +| - xfel_event_code | uint64 | Event code (for pulsed source, e.g., SwissFEL) | | X | +| - jf_info | uint64 | Detector info field | | | +| - receiver_aq_dev_delay | uint64 | Receiver internal delay | | | +| - receiver_free_send_buf | uint64 | Receiver internal number of available send buffers | | | +| - storage_cell | uint64 | Storage cell number | | | +| - saturated_pixel_count | uint64 | Saturated pixel count | | | +| - error_pixel_count | uint64 | Error pixel count | | | +| - strong_pixel_count | uint64 | Strong pixel count (first stage of spot finding) | | | +| - data_collection_efficiency | float | Image collection efficiency \[\] | | | +| - bkg_estimate | float | Mean value for pixels in resolution range from 3.0 to 5.0 A \[photons\] (with solid angle/polarization corrections, if applied) | | X | +| - resolution_estimate | float | Predicted merged resolution, from spots alone | | X | +| - adu_histogram | Array(uint64) | ADU histogram | | X | +| - roi_integrals | object | Results of ROI calculation | | X | +| - - sum | int64 | Sum of pixels in ROI area \[photons\] | | | +| - - sum_square | int64 | Sum of squares of pixels in ROI area \[photons\] | | | +| - - pixels | uint64 | Valid pixels in ROI area | | | +| - - max_count | int64 | Highest count in ROI area \[photons\] | | | + +## End message + +| Field name | Type | Description | Present in DECTRIS format | +|----------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "end" | X | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| series_unique_id | string | Unique text ID of the series (run_name parameter) | X | +| series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | +| end_date | string | Approximate end date | | +| max_image_number | uint64 | Number of image with the highest number; counted from 1 to distinguish zero images and one image | | +| transformations | Array(object) (optional) | Sample transformation chain in mounting order, base first. Each element mirrors a NeXus NXtransformations axis: `name`, `transformation_type` (`rotation`/`translation`), `units`, `vector`, `offset`, `depends_on` (the axis this one is mounted on, empty for the base), and `values` - a single number for an axis that does not move, otherwise one per image. An ARRAY because the order matters and a CBOR map has none. Optional: when absent the writer builds the same chain from the start message. It is in the END message because a producer may want to report positions that were **measured** rather than commanded, which are only known once the run is over | | +| images_collected | uint64 | Number of images collected | | +| images_sent_to_write | uint64 | Number of images sent to writer; if writer queues were full, it is possible this is less than images collected | | +| data_collection_efficiency | float | Overall network packets collected / network packets expected | | +| az_int_result | Map(text->Array(float)) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | +| adu_histogram | Map(text->Array(uint64)) | ADU values histogram | | +| adu_histogram_bin_width | uint64 | Width of bins in the above histogram \[ADU\] | | +| max_receiver_delay | uint64 | Internal performance of Jungfraujoch | | +| bkg_estimate | float | Mean background estimate for the whole run | | +| spindle_blind_fraction | float | Run mean of the per-image spindle_blind_fraction, over the frames that had one | | +| spindle_lost_unique_fraction | float | Fraction (0-1) of unique reflections the mounting made unmeasurable, exact under the measured point group; offline (Rugnux) only | | +| indexing_rate | float | Mean indexing rate for the whole run | | +| unit_cell | object (optional) | Unit cell of the system, based on the actual experiment: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | +| rotation_lattice_type | object | Bravais lattice classification of the total rotation solution over the run, if available; same schema as `lattice_type` | | +| - centering | string | One-letter centering code: P, A, B, C, I, F, or R | | +| - niggli_class | int64 | Integer identifier for the Niggli-reduced Bravais class | | +| - system | string | Crystal system: triclinic, monoclinic, orthorhombic, tetragonal, trigonal, hexagonal, cubic | | +| rotation_lattice | Array(9 * float) | Real-space lattice basis, flattened 3x3 in row-major order | | +| rotation_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | +| data_collection_efficiency_image | Array(float) | Per-image data collection efficiency. Missing values are encoded as 0 or 1 depending on producer context | | +| spot_count | Array(int32) | Per-image spot count | | +| spot_count_ice_ring | Array(int32) | Per-image number of spots within identified ice-ring resolution ranges | key is singular here; the per-image message uses `spot_count_ice_rings` | +| spot_count_low_res | Array(int32) | Per-image number of low-resolution spots | | +| spot_count_indexed | Array(int32) | Per-image number of spots fitting indexing solution | | +| image_indexed | Array(uint8) | Per-image indexing result; 0 = not indexed, nonzero = indexed | | +| v_bkg_estimate | Array(float) | Per-image background estimate | | +| v_spindle_blind_fraction | Array(float) | Per-image spindle_blind_fraction; NaN where the frame had no value (which is "cannot say", not zero) | | +| ice_ring_score | Array(float) | Per-image strongest ice-ring intensity over the smooth radial background (1 = no ice) | | +| spot_count_ice_control | Array(float) | Per-image spot count in the ice-free flanks beside the hexagonal rings, rescaled to the ring bands' q width | | +| ice_ring_score_mean | float | Mean ice-ring score for the whole run (1 = no ice) | | +| profile_radius | Array(float) | Per-image profile radius \[Angstrom^-1\] | | +| mosaicity | Array(float) | Per-image mosaicity \[degree\] | | +| bFactor | Array(float) | Per-image estimated B-factor \[Angstrom^2\] | | +| resolution_estimate | Array(float) | Per-image predicted merged resolution, from spots alone \[Angstrom\] | | +| min_viable_pixel_value | Array(int64) | Per-image minimum valid pixel value, excluding error/saturated pixels | | +| max_viable_pixel_value | Array(int64) | Per-image maximum valid pixel value, excluding error/saturated pixels | | +| saturated_pixel_count | Array(int32) | Per-image saturated pixel count | | +| error_pixel_count | Array(int32) | Per-image error pixel count | | +| image_scale_factor | Array(float) | Per-image scale factor, if scaling/merging was performed | | +| integrated_reflections | Array(int32) | Per-image count of integrated reflections | | +| indexed_lattice_count | Array(int32) | Per-image count of indexed lattices | | +| niggli_class | Array(uint8) | Per-image Niggli class identifier for indexed images; 0 if unavailable | | +| pixel_sum | Array(int64) | Per-image sum of all valid pixels, excluding error/saturated pixels | | +| image_scale_mosaicity | Array(float) | Scaling result: Image scale mosaicity \[deg\] | | +| image_scale_cc | Array(float) | Scaling result: Image scale CC | | + +End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted. + +## Calibration message + +| Field name | Type | Description | Present in DECTRIS format | +|--------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------|:-------------------------:| +| type | String | value "calibration" | | +| magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | +| data | Map(string -> Image) | Calibration map (only single pedestal array per message) | | + +## User data +Facilities often need to forward more metadata than Jungfraujoch models explicitly. +For this reason two fields can be provided: `header_appendix` (sent with the start message) and `image_appendix` (sent with the image message). +To increase flexibility, both appendices can contain any valid JSON message. +These appendices are serialized into string and stored in CBOR messages as `user_data`. + +Notably for start message, `user_data` can contain more information (non-DECTRIS compliant metadata). +Therefore `user_data` is serialized by Jungfraujoch as CBOR object. There is member `user` which contains `header_appendix` defined in OpenAPI of Jungfraujoch. + +### Notes on images and compression + +- Images are encoded as DECTRIS MultiDimArray with typed array tags: + - For RGB: shape \[3, height, width\], type: u8 + - For grayscale: shape \[height, width\], type according to bit depth and sign (e.g., uint16 LE) +- Compression: + - Uncompressed: raw CBOR byte string + - Bitshuffle+LZ4: tag with \["bslz4", elem_size, bytes\] + - Bitshuffle+Zstandard: tag with \["bszstd", elem_size, bytes\] + +### Notes on typed arrays + +Jungfraujoch uses RFC 8746-style typed byte-string tags for compact numeric arrays. + +Common tags used in this protocol include: + +- float32 little-endian arrays for `Array(float)` +- uint8 arrays for compact boolean/integer flags such as `image_indexed` +- int32 little-endian arrays for per-image counts +- int64 little-endian arrays for large per-image integer values +- uint64 little-endian arrays for histograms \ No newline at end of file diff --git a/_sources/CHANGELOG.md.txt b/_sources/CHANGELOG.md.txt new file mode 100644 index 000000000..7ae58cb2e --- /dev/null +++ b/_sources/CHANGELOG.md.txt @@ -0,0 +1,1350 @@ +# Changelog +## 1.0.0 + +### 1.0.0-rc.174 + +* Rugnux: Performance improvements on GPU and CPU (more of the pre-scan and of scaling on the GPU, faster CPU spot finding and crystal refinement), with unchanged results. +* Rugnux: More robust processing - patches of persistently hot pixels are masked, an inconsistent merge triggers a retry at the measured beam centre, and builds targeting different CPU levels give the same results. +* Rugnux: Improved scaling and merging - reflections with an overloaded pixel are dropped, as in XDS, sparse rotation sweeps are scaled more reliably, and French-Wilson amplitudes use an anisotropic Wilson prior. +* Rugnux: Improved space-group determination - glide planes in groups without a centre of symmetry, screw axes from short or weak axial rows kept when a higher group is adopted, and more reliable decisions on twinned and pseudo-symmetric crystals. +* Rugnux: Improved small-molecule processing - spots that grow wider than the integration disk and split spots are integrated over their measured footprint, sparse lattices are integrated on every frame, and the `.hkl` file holds unmerged scaled reflections (SHELX HKLF 4). +* Rugnux: Reads Rigaku d*TREK SMV images (Saturn CCD), including detector 2theta and encoded pixel overflows; home-source (rotating-anode) datasets were added to the validation battery. +* jfjoch_viewer: Fixed processing failing at the end with "Wrong JPEG library version" on Linux; the merge window shows the space group with proper subscripts and a checklist of crystal pathologies. + +### 1.0.0-rc.173 + +* jfjoch_broker: Optional per-dataset authentication - statistics, images and plots can require a bearer token, which jfjoch_viewer supports. +* jfjoch_viewer: Dark mode and a theme-matched colour scheme, a magnifier panel, and simpler contrast and background controls. +* Rugnux: Multiple performance improvements on GPU and CPU (CPU-only processing up to 40% faster, faster image decoding on ARM), with unchanged results. +* Rugnux: `--model` rigid-body refinement runs on the GPU, and the model-validation check is faster and more reliable. +* Rugnux: Improved scaling and merging - error model, outlier rejection, absorption correction and French-Wilson amplitudes now agree more closely with XDS and ctruncate. +* Rugnux: Improved integration - radial background on powder and ice rings, crowded rotation data keep their reflections, and CPU-only builds integrate large unit cells as GPU builds do. +* Rugnux: More robust detector geometry - measured beam centre, X-ray bandwidth and goniometer rate, and geometry refinement accepted only on significant evidence. +* Rugnux: Merged files are written in the standard setting, or in the setting of a reference MTZ, structure-factor mmCIF or model, with its free-R flags. +* Rugnux: Richer report - ice and powder rings, further lattices, superstructure candidates and mosaicity, with warnings worded as prompts to check. +* Rugnux: Clear error messages when a data set needs more GPU or host memory than is available. + +### 1.0.0-rc.172 + +* Fixed `jfjoch_broker` cancelling every data collection with a CUDA "out of memory" error after long operation: GPU memory no longer leaks with each collection. +* Rugnux scales a rotation sweep until the per-frame scales settle instead of for a fixed three rounds, and says so when they did not - merged intensities, and the space group, resolution cut and frame rejection read off them, change accordingly; `--scaling-iterations` is now the cap on that loop (default 100). +* Rugnux places every frame of a marCCD, SMV or miniCBF series at the spindle angle its own header states, so a series with missing frames, or with angles written modulo 360, is no longer read at the wrong geometry or refused. +* Every rotation run writes two diagnostic files beside its reflections: `_detector.jpg`, the detector projection with the pixel mask and the detected beam-stop shadow drawn on it, and `_plot.txt`, one row per image. + +### 1.0.0-rc.171 + +* Rugnux: basic support for CCD images (marCCD, SMV) and for gzipped miniCBF. +* `jfjoch_viewer`: opens the CCD formats, and fixes to the dataset plots. +* Documentation updates. + +### 1.0.0-rc.170 + +* Fixed a `jfjoch_broker` crash during indexing: sorting no longer misbehaves on non-finite values, and GPU FFT indexer kernel launches are now error-checked. +* Rugnux needs about 40% less peak memory to scale, merge and post-refine rotation data, with identical results. +* `rugnux --model`: the placed coordinate file carries the space group its own coordinates obey, and says so when that is not the group the reflection files beside it carry. +* `jfjoch_viewer`: fixes in the dataset plots, inspector and layout; spot markers lose their black outline by default (a checkbox under "Image features" restores it) and the highest-pixel markers are white boxes around the pixel. + +### 1.0.0-rc.169 + +* Building Jungfraujoch no longer needs zlib or Eigen installed on the machine, and the dependencies the build fetches are pinned and updated to current releases. +* Rugnux: improvements in indexing, lattice selection and geometry post-refinement, which index crystals that previously returned no lattice and keep the better of the two geometries a run measures. +* Rugnux: improvements in beam-centre measurement, beam-stop detection and space-group determination. +* Rugnux: the unit cell reported with a determined space group now obeys that group - a cell whose symmetry was confirmed from the intensities is re-refined under it, and a cell the group cannot describe is reported with a warning rather than as it stands. +* Rugnux drops the stretches of a rotation sweep whose removal measurably improves the merged intensities and reports what became of every frame, and decides the resolution cut on the crystal's own diffraction rather than on its ice rings. +* The Rugnux results report is machine-readable - every line that is not `KEY= value` data starts with `#` - and states the build it was written by, its authorship and its terms of use (`REPORT_VERSION= 8`). +* `jfjoch_viewer`: improvements in the file manager (CBF frames beside HDF5 datasets, a remembered root), the dataset plots, the inspector and the image statistics, plus a settable font size, a view of the Rugnux results report, usable performance over a remote display (`ssh -X`) and a reset of all settings to defaults; the reciprocal-space window is removed. +* Broker fixes around DECTRIS collections and dark-mask calibration: re-initialising after a run that never started no longer freezes the broker, a cancelled calibration is abandoned instead of reported as done, and a collection whose start message never arrives ends by itself. + +### 1.0.0-rc.168 + +* Rugnux is substantially faster - a corpus of 145 rotation datasets processes in about two thirds of the time - with identical results. +* A crystal whose lattice looks more symmetric than it is because the beam centre is off is no longer processed on the wrong cell. +* Rugnux prints at startup, and writes at the foot of every results report, a short acknowledgement of the X-ray research community whose methods it implements and of the open-source projects it builds on; `ACKNOWLEDGEMENT.md` now ships in every package beside `LICENSE` and `THIRD_PARTY_NOTICES.md`. + +### 1.0.0-rc.167 + +* `rugnux --model` reports CC(model, data) - the correlation of the merged intensities with the placed, scaled model - by resolution shell, on the same shells as CC1/2, with the reflection count and a significance for each. +* `rugnux --model` fits the model's scale, anisotropic B and bulk-solvent parameters on the working reflections only, so the R-free it reports is measured against a model no free reflection helped scale. +* The bulk-solvent parameters of `rugnux --model` are searched over their physically meaningful range instead of being fitted without bounds, so a model is never scaled with a solvent term that has silently switched itself off. +* The rigid-body placement of `rugnux --model` uses the same bounded bulk solvent as the reported fit, so a model is no longer placed against a target carrying a solvent term with no physical meaning. +* `rugnux --model` puts the model into the data's own description of the lattice before placing it, so a model whose cell is written on other axes - I-centred where the run indexed C-centred, a different unique axis, a permuted orthorhombic cell - is placed rather than scored where it was read; `MODEL_CHANGE_OF_BASIS=` and `MODEL_SETTING_AS_READ=` report it when it happens. +* The Rugnux results report opens with a summary - `VERDICT=` (`OK`, `WARNINGS`, `UNUSABLE`, `FAILED`), `VERDICT_TEXT=`, `PATHOLOGY_FLAGS=` with one closed-vocabulary code per condition that warned, and the `WARNING:` lines, which used to close the file - and the sections after it are renumbered 1-5 with no gaps. +* `rugnux --developer` writes the full results report - the pipeline-internal keys and the long explanations the default report now leaves out - and `--finalist-ledger` adds the evidence for every space group the search considered, not only the one it adopted. +* The results report warns when the merged data carry no usable signal and when too little of reciprocal space was measured inside the fitted resolution, and omits `FITTED_RESOLUTION` where the CC1/2 curve it is fitted on never falls off. +* Rugnux detects translational pseudo-symmetry and reports it under the `PSEUDO_TRANSLATION` flag as `TNCS_DETECTED=` and the `TNCS_*` keys - a translation the merged data are exactly invariant under is reported as `UNDECLARED_LATTICE_TRANSLATION=` under `LATTICE_TRANSLATION` instead - and a detected pseudo-translation can no longer buy a false screw axis in the space-group search or hide a twin from the L-test (`L_TEST_VS_TNCS=`). +* The space-group search determines glide planes from zonal systematic absences, so a non-Sohncke space group such as P 2_1/c or Pbca is named where the run previously stopped at its Sohncke subgroup; `SOHNCKE_SPACE_GROUP=` carries the best Sohncke group beside it on every run that searched, and a centre of symmetry is never claimed. +* Where the cell metric carries more rotational symmetry than the Bravais class the indexer named, the extra rotations are put to the intensities and the space-group search is asked again on the metric's own cell - adopted only where the intensities confirm the higher symmetry - so a lattice that is nearly but not exactly hexagonal, or whose reduction landed in a sub-cell, still reaches its true point group. +* Systematic-absence calls rest on the evidence rather than on counts: a screw axis whose absent class the data show extinct is no longer refused because a handful of reflections in it read as present, and `SPACE_GROUP_ALTERNATIVES=` no longer drops a candidate that differs only on a zone the sweep never measured. +* A reference correlation measured on too few reflections is refused instead of scored zero, so a run given a reference MTZ is no longer reindexed on an operator that mapped almost everything outside the reference's coverage. +* A frame counts as indexed from 6 spots on its lattice rather than 9, so a weakly diffracting crystal whose frames cannot carry 9 is no longer refused the lattice it fits; `--min-indexed-spots` overrides it. +* `-C` accepts a known cell in any equivalent description - conventional or primitive, centred or not - instead of only the reduced primitive form, so a centred cell given the way it is published no longer makes the run report that it found no lattice. +* Each reflection is corrected for the sensor's quantum efficiency at the angle it meets the detector (attenuation lengths from the NIST tables, which also fixes the spot-width parallax term on CdTe) and for the attenuation of the flight path between the sample and its pixel; `--flight-path air|helium|vacuum` declares the medium - default air, since no file states it - and the report says what was assumed and what it was worth. The unmerged MTZ records the factors in new `QE` and `FLIGHT` columns beside `LP`, so raw counts are `I / LP * QE * FLIGHT`, and `_process.h5` in new optional `qe` and `flight` datasets. +* Rotation geometry post-refinement fits the crystal and the detector at once, against the observed spot positions and the observed rocking angles together, so the refined distance depends far less on how wrong the file's distance was. +* A coarsely sliced sweep integrates correctly: partials are joined into one rocking event by angle rather than by frame count, so two crossings of the Ewald sphere are no longer summed into one full, and at 0.5 degrees per image or coarser the per-frame geometry refinement accepts a spot whose miss the exposure's own rotation accounts for. +* `rugnux --mode scale` reports the detector tilt and direct beam of the geometry it re-scaled at, instead of zeros that read as a flat detector, and no longer warns that no image was indexed on a run whose lattice came from its input file. +* Every rotation run that determined a space group and merged reports what the mounting cost: `SPINDLE_LOST_UNIQUE_FRACTION=` is the fraction (0-1) of unique reflections the mounting made unmeasurable under the measured point group, also written to the master as `/entry/MX/spindleLostUniqueFraction` and what the mounting warning fires on; `SPINDLE_SYMMETRY_AXIS_ANGLE_DEG=` / `SPINDLE_SYMMETRY_AXIS_ORDER=` describe the mounting in the `--developer` report. +* Stills and grid scans carry a per-image `spindle_blind_fraction` - how much of a rotation sweep's blind cone this orientation would make unrecoverable, 0.5 and above calling for a second orientation - through the CBOR stream, HDF5 (`/entry/MX/spindleBlindFraction`), the plot and scan-result APIs, and the viewer and frontend plots; an absent value means the frame could not be assessed and is not a 0. +* `jfjoch_viewer` gains Help entries for the mouse shortcuts and the acknowledgements, and Inspector toggles that hide non-indexed spots and spots on ice rings. +* The results report's `REPORT_VERSION` is 7. + +### 1.0.0-rc.166 + +* `rugnux --model` treats the model as a hypothesis: it decides the enantiomorph and the indexing only where its R-work beats that of the same model in random orientations, and a model the data reject is still scored, placed and mapped, but leaves the reflection files byte for byte what a run with no model writes. +* `rugnux --model` places the model against the data as a rigid body before scoring it, writes sigma_A-weighted 2mFo-DFc and mFo-DFc maps in place of the unweighted 2Fo-Fc and Fo-Fc, and writes the model as it was placed - `_model.cif`, and `_model.pdb` where the PDB format can express the cell - in the cell and space group of the reflection files beside it. +* `rugnux` and `jfjoch_viewer` read PILATUS miniCBF sweeps natively, and open masters written at other facilities, including Eiger 1.x and third-party NXmx. +* `rugnux` determines the lattice and the space group more reliably - the true cell where the first pass offers a whole-number multiple of it, so a pseudo-translated crystal keeps its full-length axis and a small molecule is indexed on its own cell rather than a protein-sized one, and the point group, the setting and the systematic absences - and `-S` refuses or re-seats a fixed space group whose symmetry axes the indexed cell does not carry. +* `rugnux` measures the beam centre on every run and indexes with it when the file's value indexes nothing, refines only the detector-tilt component the data determine - a beam-centre error is no longer reported as a tilt - and places a detector swung out on a 2theta arm where the file says it stands. +* `rugnux` writes the unmerged MTZ by default, with a P1 merge beside it, a batch header for every image the observations span, and events kept to the same `--min-captured-fraction` as the merge, so a wrong space group can be re-merged in a scaling program without reprocessing. +* `rugnux` writes reflection files in the conventions downstream programs read: `FreeR_flag` is 0 for the test set and 1 for the working set - it was the other way round - the merged and P1 MTZ carry the reserved `HKL_base` dataset so a CCP4 program reads the wavelength instead of falling back to 1.54187 A, and the merged mmCIF marks the free set as `_refln.status` = `f`. +* The Rugnux results report carries the space groups the data cannot separate and the enantiomorph state, the model's verdict and what it was allowed to decide, the detector geometry measured and what a single sweep cannot determine, the resolution the CC1/2 fit reached, which reciprocal axis each anisotropic diffraction limit belongs to, and twinning measured before and after the space group was decided; `REPORT_VERSION` is 6, and `SPACE_GROUP_ENANTIOMORPH= DETERMINED_FROM_MODEL` is now `ASSUMED_FROM_MODEL`. +* `rugnux --mode calibration` writes `.json` beside the `.poni`, holding the geometry as a `jfjoch_broker` `dataset_settings` body, and refuses a fit that is not a measurement - no `.poni`, a non-zero exit, `converged` recorded in the `.json`; `--no-refine-tilt` holds the detector tilt at the file's value instead of zeroing it. +* A snake grid scan with a negative slow step and an even number of rows no longer has its positions mirrored along the fast axis in the HDF5 master and the grid map, so the positions recorded for that configuration change; `jfjoch_viewer` draws grid scan cells in the proportion of the scan steps, labels the merge-statistics plot over the range the axis is drawn on, and builds its powder-calibration ring list from the loaded dataset's space group as well as its cell, so a centred sample cell no longer scales the whole fit. +* The HDF5 master records `direct_beam_x`/`direct_beam_y` - where the undeflected beam lands, sent on the CBOR start message too - the beam size at the sample as `incident_beam_size` from the new `dataset_settings` `beam_size_x_um`/`beam_size_y_um`, and `/entry/MX/peakCountUnfiltered`; `dataset_settings` accepts any `smargon.chi_deg`, which was restricted to 0-90 degrees. +* Reported completeness counts the reflections the beam stop, a detector mask or the low-resolution limit kept out of the merge as missing: the denominator, and the resolution shells it is binned into, now span the run's declared resolution range rather than the range of the reflections that survived, so the innermost shell boundary moves and its numbers are not comparable with those of an earlier release. +* The Rugnux results report carries `CC_ANOM` beside `SIGANO`, overall and per shell: the anomalous difference measured from one half of the observations correlated against the same difference from the other half, which says whether there is an anomalous signal to phase on without depending on the error model. Reported on Friedel-merged runs too; it matches AIMLESS and phenix, and XDS's similarly named `Anomal Corr` is a different quantity. +* A quantity a Rugnux run did not measure is left out of the results report altogether instead of being written as `nan` - `SIGANO=` on a Friedel-merged run, which is the default, is the case a script meets first - and a merging-statistics shell prints `-` in its place. +* `FITTED_RESOLUTION=` in the results report is the resolution fit of the run's own merge rather than of the P1 cross-check, and the unmerged MTZ is written in the space-group setting the run adopted rather than in that setting's reference one. +* `rugnux --mode calibration` refuses to write a `.poni` for a detector whose stored image is mirrored or turned by a multiple of 90 degrees, the format having no field for it. +* CC1/2 is computed on half-sets of equal size, so every reflection measured more than once contributes to it, as in XDS and phenix.merging_statistics, and a CPU-only build now reports the same value as a CUDA one; reported CC1/2 values move slightly, most where multiplicity is low. +* The Rugnux manual is reorganised into task pages with a run overview and worked phenix / REFMAC5 / Phaser / SHELXC/D/E / POINTLESS-AIMLESS / careless examples, and the HDF5 and API documentation say how a grid scan records the angle its spindle stood at: the goniometer axis with a step of 0. + +### 1.0.0-rc.165 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* `rugnux --model` adopts the model's space group as a label where the data were merged in its enantiomorph, instead of reindexing the reflections - which swapped I(+) with I(-). +* `rugnux --model` warns, naming the atom, when the anomalous density at the model's atoms comes out inverted, which means the data and the model are in opposite hands. +* `rugnux --model` writes an anomalous difference map (`_anom.ccp4`) when the merge kept the Bijvoet split, and names the ten model atoms it peaks highest on as `ANOMALOUS_SITE_01`..`_10`. +* `MEAN_ATOM_DENSITY_SIGMA` is read from the map by cubic rather than linear interpolation and comes out around a tenth higher; it is no longer comparable with the figure earlier versions printed. +* `rugnux --model` reads an mmCIF coordinate file as well as a PDB one, gzipped or not, taking the format from the file's content rather than its name. +* A model `rugnux --model` cannot use is reported as a `WARNING:` line in the results report instead of only in the log. +* The Rugnux results report has a `10. MODEL VALIDATION` section when `--model` was given; `REPORT_VERSION` is 4, `WARNINGS` moves to section 11 and no existing key changed. +* The Rugnux results report records how the run was invoked, what it cost and what it ran on: `COMMAND_LINE=`, `WALL_TIME=` and `GPU_COUNT=` / `GPU=`. +* Rugnux says which GPUs it can see before it starts processing. +* `rugnux --export-unmerged` also writes `_unmerged.mtz` on a `--no-merge` run, and is ignored on a run with no output prefix instead of writing a file called `_unmerged.mtz`. +* `/start` asks the writer whether the run can be written before the detector is armed, so a run whose master file already exists, or whose output directory cannot be created, is refused up front with the writer's own message. This needs the TCP image stream or the built-in HDF5 writer; the ZeroMQ stream is unchanged. +* A calibration that fails goes to `Error` carrying the reason instead of `Inactive`, so `/wait_till_done` and `/wait_until_running` report it; a cancelled calibration still goes to `Inactive`. +* `/wait_till_done` answers 500 with the message when a collection ended in an error. A cancelled collection and a collection that only triggered a warning still answer 200. +* A pending start failure is discarded by `/cancel` and `/deactivate`, as it already was by `/start` and `/initialize`. +* `/scan_result` no longer reports the previous run's images after a collection that failed to start, or after `/deactivate`. +* The TCP image stream protocol version is 4. `jfjoch_writer` and `jfjoch_broker` have to be of the same release, as before. + +### 1.0.0-rc.164 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Rugnux now tells you whether a crystal diffracts anisotropically and how far it reaches in each direction, without a second program: a new `9. DIFFRACTION ANISOTROPY` section in `_report.txt` and matching `_reflns.pdbx_aniso_B_tensor_*` / `_reflns.jfjoch_aniso_*` items in the merged mmCIF report the anisotropic deltaB, the diffraction limit along each principal direction, and a `NOT DETECTED` / `DETECTED` / `CANNOT DETERMINE` verdict measured against the data set's own systematic error. It is a description only - no intensity is corrected, no reflection is removed, and the merged data do not depend on direction. +* Rugnux can hand its integrated observations to another scaling program: `--export-unmerged` writes `_unmerged.mtz`, an unmerged MTZ readable by aimless, pointless, careless and `iotbx.merging_statistics`, in `--mode mx` and `--mode scale` alike. Each rotation reflection's partials are summed into one full; `--export-unmerged-partials` writes one row per image instead. Intensities carry the Lorentz-polarization factor and nothing else, since those programs scale the data themselves. Lattice-centring absences are not written; screw and glide absences are. +* Rugnux integrates crystals with broad spots better - where it changes anything, per-shell mean I/sigma improves by up to 31% and R_meas by up to 24% - because on rotation data the integration signal radius is now taken from the crystal's own measured spot width instead of a fixed 4 px. `--adaptive-integration-radius=off` restores the fixed radius and an explicit `--integration-radius` still overrides both. The widened radius applies to the final integration pass only, and a pattern too dense for it is re-integrated at 4 px with a note in the log. +* Rugnux discards fewer stills reflections for want of a background ring, improving per-shell R_meas over most of the signal-bearing range: the stills background ring now runs to 14 px instead of 12. The gain reverses in shells below a mean I/sigma of about 4. +* Rugnux determines the space group with thresholds that mean the same thing on a weak crystal as on a strong one: symmetry operators are scored on resolution-normalised intensities (E squared) instead of raw merged intensities, and a reflection counts as genuinely present on its counting significance instead of on the merged I/sigma, which saturates at the merge's own ISa. The search resolution cut is no longer able to move the answer, and the twin-law H bound moves from 1.70 to 1.85, which stops one class of correct high-symmetry assignment being refused as twinning. +* Rugnux says what the space-group search tested and what it could not: the twin-law disagreement H is printed for every operator together with the adopted point group's H ratio and its bound; alternatives that are not on the reported lattice are named with how their cell differs; and a lattice centring the data could not test - the crystal having been integrated on the primitive sub-cell, so the reflections it extinguishes were never measured - is marked `UNTESTED` and warned about where it is adopted, as coming from the lattice metric rather than from the intensities. +* Rugnux `--mode scale` re-merges a `_process.h5` in the right symmetry without being told it: the file now records the space group on every run - a two-pass rotation run wrote none before, so re-merging defaulted to P1 - together with the change of basis under `/entry/MX/reindexMatrix` where the lattice was re-seated, and `--mode scale` also reports the Wilson B-factor estimate instead of `WILSON_B= nan`. A file written before this stops with a message naming the two cells and the override to use, instead of failing inside the merge. A third-party reader of a `_process.h5` must apply `reindexMatrix` where it is present. +* Rugnux installs on its own, as a package called `rugnux` - `dnf install rugnux` or `apt install rugnux` - instead of arriving inside `jfjoch-viewer`. It pulls in none of the acquisition stack, so a machine that only processes data no longer has to carry the broker, the detector libraries or Qt to get it. Installing it over a `jfjoch-viewer` from rc.163 or earlier, which still owns `/usr/bin/rugnux`, upgrades cleanly rather than failing on the duplicate file. +* Rugnux is also a standalone download, built for arm64 as well as x86_64: `rugnux--linux-{x86_64|aarch64}-cuda.tgz` and `rugnux--win64-cuda.zip` on the release page, for machines that are not managed by a package manager. The aarch64 build targets GH200 and DGX Spark, and is untested on hardware. +* Every portable Linux binary is now a single self-contained file: cuFFT is linked statically instead of being shipped beside the executable and found through an rpath, so `rugnux` and `jfjoch_viewer` need nothing but an NVIDIA driver, and only to use the GPU. The `.rpm`/`.deb` continue to take cuFFT from the distribution. The developer utilities `jfjoch_extract_hkl` and `jfjoch_recompress` are no longer packaged anywhere. +* Jungfraujoch needs six fewer shared libraries on the machine - libopenblas and libmetis, and libgfortran, libquadmath, libgomp and libz behind them - because the Ceres LAPACK, METIS and SuiteSparse back-ends are no longer built. Nothing in the code ever selected them, and results are unchanged. +* The PCIe driver DKMS package builds for the kernel it is being installed for instead of the running one, so a module built while a kernel update is being applied loads after the reboot. +* The PCIe driver builds on RHEL 9.5 and later, and on their CentOS Stream, Rocky and AlmaLinux equivalents, where the `vm_flags` kernel interface was backported into the 5.14 kernel. +* A data collection started with `async_start` that fails to start - a writer refusing to overwrite an existing file, for instance - is reported as an error by `/wait_until_running` and `/wait_till_done` instead of as a timeout and a successful collection respectively. The error message is the one the writer gave. +* A calibration that is cancelled or that fails to collect its pedestals is no longer reported as a successful one. The broker goes to `Inactive` with an error message and has to be initialized again, instead of sitting in `Idle` looking ready to measure while holding partial pedestals - data collected in that state was silently mis-converted. +* A failed `/initialize` is reported to `/wait_until_running` and `/wait_till_done` as soon as it happens, instead of when their timeout expires. +* `space_group_number` accepts space groups up to 230 in the API schema, so cubic space groups can be recorded. The broker always accepted them; the generated clients rejected them before the request was sent. +* The results report's `REPORT_VERSION` is 3, two sections having been added. Existing key names and table columns are unchanged. +* The merged statistics table has **9** resolution shells instead of 10, which is what XDS reports. The bins were already XDS's - equal steps in 1/d^2 between the lowest- and the highest-resolution reflection the merge kept - so at the same resolution limits the two tables now have the same shell boundaries and can be read row for row. `--resolution-shells` sets a different count. +* `rugnux --model` now settles the frame the merged reflections are written in, not only the frame the R-factors and the maps are computed in: the `.mtz`/`.cif`/`.hkl` come out in the model's indexing, and where the data were merged in the model's enantiomorph they take the model's hand and space group - which on anomalous data puts I(+) and I(-) the right way round. The indexing choice is logged with the winning R-free and the runner-up, so a decision made within noise is visible. +* `rugnux --model` can resolve the indexing ambiguity of a **serial stills** run, which a model could not do before: structure factors computed from the model become the per-image reference, the same role a reference MTZ plays. It needs the cell and space group up front (`-C` / `-S`). Without one or the other, a merohedral serial run still merges both hands together and says so. +* The Rugnux documentation opens with a quick start - the default run, and runs with a reference MTZ, with a model, or with the space group and cell pinned - and explains the indexing ambiguity: what it costs on rotation and on serial data, and which of `-z` / `--model` resolves it in each case. The long reference pages now carry a table of contents. + +### 1.0.0-rc.163 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Packaging: each package installs its license notices under a directory of its own - `share/doc/jfjoch_broker`, `share/doc/jfjoch_writer`, `share/doc/jfjoch_viewer`, `share/doc/jfjoch_driver_dkms` - instead of all of them into the shared `share/doc/jfjoch`, so `jfjoch` and `jfjoch-writer` can be upgraded one at a time instead of only together. + +### 1.0.0-rc.162 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +**Files written by Jungfraujoch now import correctly in DIALS, XDS and pyFAI.** A tilted detector, a grid scan, a still recorded at a goniometer position, and saturated or unreadable pixels were each described in a way that a third-party program acted on wrongly. If you process Jungfraujoch data outside Jungfraujoch, prefer this release to any earlier one. + +* HDF5: the detector tilt (`rot1`/`rot2`/`rot3`) is exported correctly in the NXmx transformation chain; untilted geometries are unaffected. +* HDF5: a still recorded at a goniometer position is no longer read back as a single image, and a grid scan records a stationary spindle so a program that requires a rotation axis can open it. +* HDF5: the sample transformation chain is written in mounting order, with a Smargon head position told apart from the spindle, one entry per image, `module_offset` as a float unit vector, and `offset_units` on every offset. +* HDF5: saturated, underloaded and unreadable pixels are described so a downstream program masks them - `saturation_value`, `underload_value`, `error_value` and `bit_depth_readout` are written correctly, and a data file missing next to a VDS master reads as the error marker rather than as zero counts. +* HDF5: the rotation axis is read back under whatever name it carries, and `mirror_y` records whether the assembled image is mirrored in Y relative to the detector's raw readout. +* A grid scan and a goniometer axis can both be set; they are no longer alternatives. +* `images_per_file` is chosen from the acquisition when it is not given: a rotation sweep of at most 20000 images goes into a single data file, a grid scan splits on whole fast-axis rows, and stills and serial keep 1000. +* The writer refuses a stream whose start message declares a different pixel format than its images carry, and a DECTRIS detector sending signed images is no longer declared unsigned. +* The image stream can carry the sample transformation chain (`transformations`, in the END message); a producer that does not send it gets the same chain built by the writer. +* Rugnux: fixing the space group with `-S` no longer prevents the lattice from being found - a lattice indexed in a different setting is reindexed into that group's own setting, and a run whose crystal does not have that group's lattice stops and names the cell it indexed as, rather than reporting statistics that cannot describe it. +* Rugnux: the per-image resolution estimate now predicts the resolution the merged data reach rather than the highest-resolution spot found, and is reported as `SPOT_RESOLUTION_ESTIMATE`. +* Rugnux: two runs of the same command on the same images produce the same merged intensities; the azimuthal profile written alongside them is not yet reproducible in the same way. +* Rugnux: the offline lattice refinement is bounded by iterations rather than by a wall clock, so a loaded machine can no longer refine to a different lattice; a live acquisition keeps its real-time bound. +* Rugnux: the detector-frame modulation correction is fitted on a grid spanning the detector, so whether it is applied no longer depends on how far integration reached. +* Rugnux: the geometry pre-pass no longer writes `_01.mtz`, `_01.cif`, `_01.hkl` and `_01_image.dat`; the refined second pass writes those files under ``, and that is the result to use. +* Rugnux: `_process.h5` describes the pixel format of the images it links to, and is written on a thread of its own. +* Rugnux: the detector geometry is also logged in XDS's convention (`ORGX`/`ORGY`, detector axis vectors, rotation axis), so it can be compared with an XDS refinement. +* Rugnux: an image integrated in pyFAI through the `.poni` file written by `--mode calibration` comes out with the correct azimuth, and the file declares pyFAI's `orientation`, which needs pyFAI 2024.01 or newer. Radial integration is unchanged. +* Rugnux: a rotation run is substantially faster throughout - beam-stop detection, first-pass indexing, geometry refinement, integration, scaling and merging - and observations outside the scaling resolution range are dropped as they are ingested. The refined geometry, the space group chosen and the merged statistics are unchanged. +* Faster spot finding and indexing, on the broker as well as in Rugnux; the spots found and the lattices indexed are unchanged. +* A run reserves substantially less GPU memory: nothing is allocated for buffers that are never read, and a worker builds only the engines it uses. +* Rugnux: with `-N` left at its default the per-image loop of `--mode mx` uses at most 16 workers per GPU, rather than one per hardware thread; an explicit `-N` is obeyed as given. +* CUDA 12 builds now contain device code for Volta, so the RHEL 8 packages and the portable Linux `.tgz` run on a V100; the CUDA 13 artefacts (RHEL 9, Ubuntu, Windows) remain Turing and newer. +* The build resolves a single Eigen for the whole project, and refuses to configure if Ceres picks up a different one; a build that mixed two Eigen versions was undefined behaviour and crashed at -O2. +* Documentation: a security page, and the supported GPU generations and minimum NVIDIA driver version of every released artefact. + +**Breaking change to OpenAPI** - regenerate the client (`jfjoch-client` 1.0.0-rc.162, `frontend/src/client`): +* `dataset_settings.images_per_file` is no longer `default: 1000` and no longer accepts `0`; it is optional, and its minimum is 1. A client sending `0` (previously "one file for the whole run") is now rejected - omit the field instead, which for a rotation sweep gives the same single file. +* `file_writer_format` now defaults to `NXmxVDS`, matching the server's own default and the layout recommended for DIALS, XDS and CrystFEL. A generated client that fills in schema defaults and does not set the format explicitly will write VDS masters where it previously wrote legacy ones; set `NXmxLegacy` explicitly to keep them. + +### 1.0.0-rc.161 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* **Rugnux: significantly better quality of results, and faster.** A large rework of integration, scaling, merging, geometry refinement and space-group determination, together with measurements the program previously made no attempt at - the direct beam before indexing, the beam stop, the goniometer rotation scale, and the stretches of a sweep the crystal did not deliver. A rotation dataset typically gains observations at better and R_meas, and every `mx` and `scale` run writes a `_report.txt` results report modelled on XDS's `CORRECT.LP`. Many defaults moved with it: spot detection is self-calibrating, beam-stop detection and rotation geometry post-refinement are on, resolution limits default to as far as the detector reaches, and ice-ring handling engages only where the crystal is measured to have ice. +* **jfjoch_viewer:** the beam-stop shadow, the detector calibration and the beam-centre measurement are reachable from "Analyze dataset"; the settings panel reports how the sample moved and how polarized the beam was; image rendering and interaction are faster. +* **Performance:** bitshuffle+LZ4 images are decoded on the GPU rather than on the host, with the bitshuffle inverse fused into preprocessing so the decompressed frame is never held in device memory. +* **Broker, writer, packaging and build:** image-slot lifetime and locking fixes, per-image datasets sized by the images actually written, the Debian/Ubuntu broker package renamed to `jfjoch`, and `image_analysis` compiling under MSVC again. + +**Breaking change to the Rugnux command line:** +* `--azint-only` and `--scale` are **removed**, replaced by `--mode azint` and `--mode scale`; the full pipeline is `--mode mx` and remains the default. A script passing the old flags now fails with the list of valid modes rather than silently running the wrong one. +* `-t`/`--stride` is **refused on rotation data**: skipping frames cuts every reflection's rocking curve, so the combined fulls and their partiality would be measured over frames the sweep never recorded. Select a contiguous range with `-s`/`-e` instead. `--mode azint` and `--force-still` still take a stride. + +**Breaking changes to OpenAPI** - regenerate the client (`jfjoch-client` 1.0.0-rc.161, `frontend/src/client`) or read the affected fields as optional: +* `image_scale_b` is removed from the `plot_type` enum, so a client requesting that plot now gets an error rather than a curve. +* `azim_int_settings.high_q_recipA`, `spot_finding_settings.high_resolution_limit` and `spot_finding_settings.low_resolution_limit` are no longer `required`. All three mean "no limit at that end" when unset and are omitted from the response instead of carrying a placeholder value, which raises in a client generated from an rc.160-or-earlier spec. A value of 0 is still accepted and means the same thing. + +**Breaking changes to the stored formats** - a consumer reading these fields must treat them as optional: +* The per-image image-scale B factor is no longer computed, so `/entry/MX/imageScaleBFactor` is absent from newly written HDF5 files and the corresponding key is absent from the CBOR DataMessage and END blocks. Files written by rc.160 and earlier still contain it and still open; nothing in the pipeline reads it any more. +* `_reflns.jfjoch_diffrn_ISa` now carries the whole-range `1/sqrt(a*b)` that XDS's ISa denotes, and the error-model `a` and `b` are reported in XDS's convention; the strong-reflection asymptote moves to `_reflns.jfjoch_diffrn_ISa_asymptotic`. **A file written by an earlier version carries the asymptote under the plain `ISa` name.** + +### 1.0.0-rc.160 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Rugnux: Rotation **geometry post-refinement** is now on by default (`--rotation-no-postrefine` to disable; also a viewer checkbox). A first pass integrates at the header geometry, then the shared detector distance + beam centre and the crystal cell/goniometer-axis are post-refined over all frames (cross-validated, committed only for a small < 1 % move, with the gauge-weak beam centre restrained toward the header); a second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical `_*` output; the header-geometry pass is kept as `_01_*`. +* Rugnux: Optional per-batch **relative-B** correction for rotation (`--relative-b[=deg]`, default 10°-of-rotation batches when bare, off otherwise) - a cross-validated, curvature-smoothed resolution×dose correction beyond the single global decay slope. +* Rugnux: Always-on **radiation-damage report** for rotation - the per-image scale correlation-to-merge and mosaicity versus dose, plus the relative B-factor change over the run (first→last) as a scalar and a per-batch relative-B curve, printed to the log and written to the merged mmCIF. Report-only; it never alters the merge. +* Rugnux: De-novo space-group search ranks candidate lattice **centerings by net absences** (systematically-absent minus violating), not the gross absent count, fixing an over-centering of a genuinely C-centred lattice to F. +* Rugnux: Record the **producing software** (name and version) and the refined **detector distance and beam centre** in the merged mmCIF (and the software in the MTZ history). +* Bragg integration: Carry the box-sum observed centroid through the profile-fit path, so the observed spot centroid is emitted in every integrator mode. +* Rugnux: Report **ISa** as the counting-subtracted strong-reflection asymptote, not `1/b` of the whole-range fit; it also sets the merged-sigma floor. CC1/2, R-meas and per-obs sigmas unchanged. +* Frontend: Azimuthal-integration Q fields (Q spacing / Low Q / High Q) accept 5 decimals (was 3), matching the 1e-5 `q_spacing` minimum; number-field precision is now configurable. +* Rugnux: Add a dataset-wide **Wilson B-factor** estimate to the merged output (mmCIF, stats table, log); the per-image viewer Wilson B emits NaN for implausible fits. +* Rugnux: De-novo space-group search vetoes a merohedral-twin over-promotion whose systematic error-model `b` balloons past a calibrated bound (keeps R3 as R3, not R32). +* Rugnux: De-novo space-group search decides lattice **centering** from the strength (mean I/sigma) of the systematically-absent class, not a per-reflection violation count. +* Rugnux: Recover lattice **centering** on weak / low-energy data via a floor-independent test (rate of significant absent vs present reflections), fixing a missed I-centring at 5/13 keV. +* Rugnux: Report anomalous signal-to-noise **SigAno** = <|I(+)-I(-)|>/ per shell and overall (mmCIF PDBx items + stats-table column); anomalous merges only. +* Rugnux: De-novo space-group search recovers a genuine high-symmetry group on weak data with a broken sigma model by confirming on the systematic-`b` test alone (restores an F432 case). +* Rugnux: Print the adopted **space group and unit cell** as a one-line summary at the end of the run (de-novo or user-fixed `-S`). +* Rugnux: Score the radiation-damage **decay** cross-validation on a sigma-independent (R-meas-like) metric, so a spurious slope can't pass by reshaping sigmas. +* Rugnux: Fix de-novo rotation indexing committing a spurious axis-multiple supercell (collapsing to P1) via a cross-scheme smaller-cell tie-break on near-integer volume ratios. +* Rugnux: Widen refined-cell angle bounds to [30, 150] deg (rotation candidate and per-frame stills refinement); check refined angles against the reference cell. +* Rugnux: `-S`/`--space-group` now accepts a Hermann-Mauguin symbol (e.g. `P43212`) as well as a space-group number. +* Rugnux: Warn when the chosen cell/space group carries an indexing (merohedral) ambiguity needing a reference to resolve. +* Indexing: Requesting the FFTW (CPU) indexer on a GPU node now fails with a clear, actionable message (rotation always uses the GPU FFT indexer there). +* Rugnux: Stills `--refine-geometry[=N|off]` - first-pass bundle-adjust of beam/distance/cell then re-index (default ON with a reference cell); accepts reference `F`/`FP` columns. +* Rugnux: Per-image geometry refinement `-r flex` tries all three algorithms per image and keeps the best (old name `multi` kept as an alias). +* Rugnux: Experimental stills partiality `--still-partiality` (Gaussian excitation-error) and `--partiality-uncertainty ` down-weighting the least-complete partials. +* Rugnux: Default the stills Bragg-integration box to r=6 (integration radii 6, 8, 12). +* Rugnux: Self-referenced stills scale in a single pass (fixing a weak-data collapse); a reference MTZ (`-z`) fixes SG/cell/ambiguity but never anchors the scale (stills and rotation). +* Rugnux: Cap normalised intensity (E^2) on second-lattice overlaps in the de-novo space-group search, so strong overlaps don't skew the symmetry decision. +* Rugnux: Fix `--scale` on a self-contained `_process.h5` (stored reflections and error model reload correctly). +* Rugnux: Add `--spot-low-resolution ` (default 50 A) and `--min-pix-per-spot ` (default 2) to tune spot finding on weak serial data. +* jfjoch_viewer: Expose stills processing settings in the settings dock, rename geometry-refinement `multi` to `flex`, and refit the initial image on resize. +* jfjoch_writer: Remove the CBF and TIFF image writers - only NXmx HDF5 is written (all three layouts remain). +* Reader: Treat a negative `total_flux` in a stored dataset as unknown/absent rather than a valid flux. +* Packaging: Build the self-contained Linux viewer against a static libdbus with glib disabled; add parallel image-build and in-container viewer-verification scripts. +* Rugnux: Write anomalous data as a standard CCP4 anomalous MTZ (one row per reflection: `IMEAN`, `I(+)`/`I(-)`, `F`/`F(+)`/`F(-)`), readable by aimless/mtz2sca/ANODE. +* Rugnux: Always write merged reflections as both `.mtz` and `.cif`; the `--scaling-output` selector and text `.hkl` output are removed. +* Rugnux: Add a detector-plane **modulation** (flat-field) correction surface to rotation scaling (cross-validated, on by default; `--no-scaling-corrections` disables all), dropping R-meas. +* Rugnux: Add optional **stills detector-plane modulation** (`--stills-modulation`, default off) - the same cross-validated surface for the on-the-fly stills path. +* Bragg integration: Local background is now a **symmetric trimmed mean** of the ring (`--background-trim `, default 0.10; monochromatic), improving and resolution-edge CC1/2. + +### 1.0.0-rc.159 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Rugnux: Add `--model model.pdb` - score the merged data against an atomic model and compute initial maps. It reports R-work/R-free (scaling the model to the observed amplitudes with an overall scale, an anisotropic B and a flat bulk solvent - the standard few-parameter model, so a batch of maps stays directly comparable) and writes 2Fo-Fc / Fo-Fc electron-density maps (CCP4) plus a map-coefficient MTZ. The structure itself is not refined; the model is only re-fractionalised into the data cell. +* Rugnux: The merged reflection output now carries French-Wilson amplitudes (|F| and its sigma) next to the intensities - MTZ `F`/`SIGF`, mmCIF `_refln.F_meas_au`, and the text HKL - computed with the correct centric/acentric Wilson prior and epsilon multiplicity, so a downstream program (e.g. phenix.refine) can refine against amplitudes. The intensity columns are unchanged. +* Rugnux: R-free test-set flags are now assigned deterministically and consistently across symmetry - a Bijvoet pair I(+)/I(-) is never split between the work and free sets, and the assignment is a reproducible per-hkl hash that depends only on the reflection index, so every dataset of one crystal form gets the same ~5% free set (what a multi-dataset campaign such as PanDDA needs). On small data the fraction is floored so the test set stays large enough for a stable R-free (~500 reflections, capped at 10%); it stays flat at 5% on ordinary data. When a reference MTZ carries a `FreeR_flag` column its test set is imported instead, letting a whole campaign inherit one shared free set. +* Rugnux: A reference MTZ (`--reference-mtz`) can now fix the space group and cell for rotation data too (previously rejected), without being used to scale - the rotation merge stays self-consistent. When the crystal has an indexing (merohedral) ambiguity - a lattice symmetry higher than its Laue symmetry, e.g. P3/P4/P6/C2 - the reference also resolves it: each candidate reindexing (identity plus the twin-law cosets of the metric symmetry) is scored by its intensity correlation against the reference and the data are re-merged in the best-correlating one. This is a metric-preserving relabelling of hkl (the cell is unchanged) and a no-op for a holohedral crystal (which has no twin laws). +* Rugnux: `--model` validation now aligns the data to the model before scoring - the observed reflections are reindexed into the model's enantiomorph when the two differ only by hand (indistinguishable from merged intensities). A merohedral indexing ambiguity is resolved against the reference MTZ when one is given (so a whole campaign shares one indexing convention); only with a model and no reference does validation fall back to fitting each candidate reindexing and keeping the lowest R-free. +* Rugnux: De-novo symmetry - recover a genuine high-symmetry group whose data are imperfectly scaled. Such a merge's within-orbit chi² lands just past the self-consistency bound (each real symmetry step adds a little systematic scatter), right where a merohedral twin also lands, so the chi² ratio alone cannot separate them. The candidate is now rescued when the extra intensity-proportional systematic error it invokes stays small relative to the confirmed subgroup - a genuine symmetry step gains multiplicity without inflating the merge error model's b, whereas a twin forces non-equivalent reflections together and b balloons. Fixes a cubic case (I23 instead of I222) with no change to any other crystal in the test battery, including the twins that must stay in their lower symmetry. +* Docs: Document the French-Wilson amplitude estimation, R-free flagging, reference-based space-group/ambiguity resolution, and model-based validation/maps in CPU_DATA_ANALYSIS.md. +* Frontend: The status-bar pill now shows a progress bar during detector calibration (previously only during measurement), and the calibration state and its button are labelled "Calibration"/"CALIBRATE" (the internal `Pedestal` state name is unchanged for back-compatibility). + +### 1.0.0-rc.158 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Analysis: The azimuthal-integration solid-angle correction now follows the incidence angle to the detector normal (`cos^3` of that angle) instead of `cos^3(2*theta)`, so it is correct for a tilted detector and matches PyFAI `solidAngleArray` and MAX IV azint (unchanged for an untilted detector). Crystal geometry refinement (`XtalOptimizer`) no longer silently ignores an imported PONI `rot3` (rotation about the beam): it is applied as a fixed rotation in the residual so refinement stays consistent with the rest of the pipeline. Polarization and azimuthal binning already honoured `rot3` through the full PONI rotation. +* jfjoch_viewer: Open datasets on the WSL2/UNC filesystem (paths starting `\\`); write processing outputs next to the input file, with a Browse button and independent `_process.h5` / merged `.mtz`/`.cif` toggles; and show the determined space group in the merge-statistics window. +* jfjoch_viewer: Connect to a broker over `https` (an http/https selector in the connect dialog), and keep the HTTP connection alive across reads for faster live-follow. +* jfjoch_viewer: Time out stalled HTTP requests (5 s) so an unreachable broker cannot hang the reader thread, and drop the cached pixel mask when switching data source. +* Rugnux: Accept an absolute `-o` output prefix in offline processing. +* Rugnux: Faster two-pass rotation indexing - the first pass now runs its FFT indexing and geometry refinement in parallel (results unchanged). +* Rugnux: Rotation indexing now works on standard DECTRIS datasets that store no spots - the first pass finds spots itself instead of failing. +* Rugnux: De-novo symmetry robustness - don't over-promote a merohedral twin to the holohedral group (keep e.g. R3, not R32), make the intensity second-moment twinning statistic robust on weak/mis-integrated data, and don't flag twinning in holohedral Laue classes where no twin law can exist. +* jfjoch_writer: Fold the refined beam centre into the NXmx detector `translation` vector too (not only the informational `beam_center` fields), so a reprocessed `_process.h5` has a self-consistent refined geometry. +* Robustness: Harden size handling of untrusted input in TIFF reading and raw-TCP frames. +* Packaging: The self-contained Linux viewer `.tgz` now bundles cuFFT, so it runs without a system CUDA toolkit (`.deb`/`.rpm` are unchanged, distro-managed). +* Docs: Documentation updated to match the current analysis code and CLI. + +### 1.0.0-rc.157 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* Rugnux: Rebrand the offline data-processing subsystem as `rugnux` and consolidate all offline analysis into the single `rugnux` binary - `jfjoch_process` is now `rugnux`, the former `jfjoch_azint` is now `rugnux --azint-only`, and `jfjoch_scale` is now `rugnux --scale` (see the new docs/NAMING.md and docs/RUGNUX.md). Scaling and merging are on by default for rotation and stills (`--no-merge` disables them), replacing the previous opt-in `-M, --scale-merge`. +* Rugnux: CLI fixes - default `-N` to all hardware threads, parse numeric option arguments strictly (reject non-numeric or trailing input instead of silently yielding 0), require `--wavelength > 0`, and correct the reproduced command line and `--scale` reference-cell handling. +* Rugnux: De-novo space-group improvements - recover genuine high symmetry and centred Bravais lattices from intensities, add an automatic CC1/2 high-resolution cutoff, and report L-test twinning statistics. +* Rugnux: Index weakly-diffracting low-resolution rotation data that previously failed (e.g. F-cubic crystals that diffract only to ~4 A on a detector reaching ~1.5 A). The per-frame indexing gate now measures the indexed fraction only within the resolution range the lattice actually diffracts to, so the many sub-diffraction ice/noise spots no longer make the fraction floor unreachable; the two-pass first pass tries several image-sampling schemes (spread across the whole rotation vs a consecutive wedge whose native stride keeps a reflection's rocking curve continuous, letting the FFT resolve a long axis) and keeps the one that indexes the most frames; and the de-novo space-group search no longer discards all reflections (and crashes) when every resolution shell falls below = 1. +* Rugnux: Lower the low-resolution R-meas for strongly-diffracting rotation data - drop edge-of-sweep truncated fulls whose rocking curve was captured below `--min-captured-fraction` (default 0.7 for rotation), and report R-meas only over the observations kept by outlier rejection (matching XDS). The 0.7 default also strips the partiality-extrapolated fulls that dominate the intensity second moment on weakly-diffracting crystals, so the de-novo space-group search is no longer starved by the error-model I/sigma floor and recovers the correct symmetry (e.g. for F-centred cubic lattices that would otherwise be under-assigned). +* Rugnux: Write the refined geometry (beam, tilt, axis) to _process.h5 and place non-standard mmCIF items under a reserved `jfjoch` prefix. +* jfjoch_broker: Ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) now return to the Idle state with an Error-severity message, so a run can be retried without an expensive re-initialisation; only failures that leave the detector in an undefined state (new JFJochCriticalException, e.g. PCIe/FPGA faults) go to the Error state and force re-initialisation. +* jfjoch_broker: A synchronous /start now reports its failure to the HTTP caller instead of returning HTTP 200, and an incomplete or truncated dataset (missing packets, writer disconnect) is reported as an error rather than a "reduce frame rate" warning. +* jfjoch_broker: Drop uncollected placeholder rows (number = -1) from the scan_result REST endpoint. +* jfjoch_broker: Fix the inverted per-image compression ratio reported by the Lite receiver (was compressed/uncompressed instead of uncompressed/compressed). +* jfjoch_broker: Bragg integration adds a quantization-noise variance floor with a box-sum fallback, and treats the type-maximum marker as an invalid pixel for unsigned image types. +* jfjoch_writer: Detect file-overwrite conflicts at start for back-channel transports, and reset the writer when end-of-collection finalisation fails. +* jfjoch_viewer: Preview overlays follow the geometry (resolution/ROI arcs, true beam centre, predictions, coral secondary-lattice spots, legend), add save-as-JPEG, and fix an HTTP live-follow memory leak. +* Frontend: Improved aesthetics and usability, and added in-browser pixel-mask and JUNGFRAU-pedestal visualisation. +* CI: Name the Windows installer jfjoch-viewer-* instead of jfjoch-*. + +### 1.0.0-rc.156 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output. +* jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed. +* jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections. +* jfjoch_process: Judge systematic absences by resolution-normalised intensity instead of I/sigma alone, so screw axes are no longer missed when the error model under-estimates sigma on weak axial reflections (e.g. the monoclinic 2_1 screw). +* jfjoch_process: GPU-accelerated rotation scaling and merging (RotationScaleMerge), substantially faster than the previous CPU path. +* jfjoch_process: Unify still and rotation processing on a single --force-still flag (replaces the -P partiality-model option); rotation is auto-detected from the goniometer and processed as rot3d two-pass by default, the default reflection output is mmCIF, and the experimental --reciprocal-profile option is removed. +* jfjoch_process: Add EXPERIMENTAL ice-ring detection (--detect-ice-rings) that excludes ice reflections from scaling. +* jfjoch_broker: The Bragg integration model (profile-fit Gaussian, empirical, or box-sum) is now selectable via the REST API (/config/bragg_integration) and the web frontend. +* jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata. +* jfjoch_writer: Don't write empty grid-scan position arrays when the dataset has no images. +* Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don't fit rather than aborting), and add the jfjoch_recompress tool. +* jfjoch_viewer: Report "Multiple lattices detected" and grey out "Analyze dataset" on a live connection. +* jfjoch_viewer: Frontend fixes - detector settings widget, panel/preview overflow, and navigation icons. +* CI: Build Windows (CUDA and non-CUDA) installers. +* CI: Ship jfjoch_viewer to the release as a Linux-agnostic .tgz. + +### 1.0.0-rc.155 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D) +* jfjoch_viewer: Some graphical improvements. +* jfjoch_viewer: Simplify and unify data analysis settings. +* jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker "dies" in the middle of data acquisition. + +### 1.0.0-rc.154 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Fix to TCP file pusher (remove kernel zero copy to improve reliability) + +### 1.0.0-rc.153 +This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. + +* jfjoch_broker: Add EXPERIMENTAL pixelrefine mode for image processing +* jfjoch_broker: Allow to load user mask from 8-bit and 16-bit TIFF files +* jfjoch_broker: Add ROI calculation in non-FPGA workflow +* jfjoch_broker: Fixes to TCP image pusher +* jfjoch_broker: Remove NUMA bindings +* jfjoch_broker: Improvements to indexing +* jfjoch_broker: For PSI EIGER, trimming energies are taken from the detector configuration (now compulsory) instead of hardcoded values +* jfjoch_writer: Save ROI definitions and the per-pixel ROI bitmap in the master file; azimuthal ROIs support phi (angular) sectors +* jfjoch_viewer: Major redesign with dockable panels and saved layouts, plus on-canvas creation/move/resize of box, circle and azimuthal ROIs +* jfjoch_viewer: Run jfjoch_process reprocessing jobs from inside the GUI and overlay per-run results + +### 1.0.0-rc.152 +* jfjoch_broker: Fix bounds for azimuthal integration for Q spacing (allow Q of 1e-5) +* jfjoch_viewer: Adjust Q bounds for azimuthal integration +* jfjoch_azint: Add tool to do quick azimuthal integration + +### 1.0.0-rc.151 +* jfjoch_broker: For PSI EIGER detector allow to disable individual half-modules by putting empty hostname + +### 1.0.0-rc.150 +* jfjoch_broker: When in FPGA workflow (with PSI detectors) azimuthal integration might be forced to CPU - this will require more computational power, but it enables more integration bins and reports standard deviation of each bin. +* jfjoch_broker: Raise error if one is in FPGA flow and there are too many azimuthal integration bins. + +### 1.0.0-rc.149 +* XDS plugin: Fix HDF5 mutex to run on multiple processors + +### 1.0.0-rc.148 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Improve azimuthal integration (add calculation) +* jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated) +* jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin + +### 1.0.0-rc.147 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* CI pipeline builds software with x86_64-v3 architecture, it should be compatible with practically all x86 hardware manufactured after 2015. +* jfjoch_viewer: Add reciprocal space viewer +* jfjoch_process: Two pass algorithm that does spot finding/indexing + integration of full dataset +* jfjoch_process: Improve logic for rotation indexer, to make execution more deterministic (still work in progress) + +### 1.0.0-rc.146 +This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144. + +* jfjoch_broker: Add a lattice-orientation-only refinement option, in addition to full refinement (beam center, lattice orientation, lattice dimension) +* jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset. +* jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling. +* jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file. +* jfjoch_scale: Dedicated tool for rescaling/merging existing data. +* jfjoch_viewer: Fix bugs where pixel labels were displayed on a wrong pixel. + +WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being. + +### 1.0.0-rc.145 +This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144. + +* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future. +* XDS plugin: Improve performance of VDS reading. +* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD. +* jfjoch_writer: Clean-up of HDF5 routines to better handle issues. + +### 1.0.0-rc.144 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve performance of preview JPEG image generator at receiver startup (saving about 150 ms on measurement start for 16M) + +### 1.0.0-rc.143 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Avoid copying gain calibration together with DiffractionExperiment + +### 1.0.0-rc.142 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Support for newer CUDA architectures (notably Blackwell); minimum CUDA version 12.8 +* Minor changes to jfjoch_process, jfjoch_fpga_test and jfjoch_lite_perf_test to make them more consistent + +### 1.0.0-rc.141 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Azimuthal integration mapping is generated with parallel computations, significantly reducing setup times +* frontend: Fix selection of FFTW in indexing settings + +### 1.0.0-rc.140 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: For DECTRIS detectors, ZeroMQ link is persistent, to save time for establishing new connection +* jfjoch_broker: Minor bug fixes for rare conditions +* jfjoch_process: Significantly improve performance + +### 1.0.0-rc.139 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by selectively modifying SIMPLON parameters on `/start` +* jfjoch_broker: Further reduce startup time for DECTRIS detectors by not setting beam center and detector distance via SIMPLON API on '/start' +* jfjoch_broker: Add an extra message to ZeroMQ puller ready to monitor Lite workflow preparation time +* jfjoch_broker: Image buffer configuration is postponed for Lite receiver flow till start message is received +* jfjoch_broker: Use nanoseconds internally for frame/image/readout time +* jfjoch_broker: Extra messages added for receiver operation (to be removed after debugging finished) +* jfjoch_broker: Improve profiling of different data analysis steps +* jfjoch_broker: Record integration reflection count +* jfjoch_broker: Fix bug where ZeroMQ preview frequency was confusing time units (micro vs. milliseconds) +* jfjoch_broker: Fix bug where '/wait_till_done' got deadlocked +* jfjoch_writer: Fix confusion between NaN and zero in floating-point datasets + +**Breaking changes**: detector definition is now using nanoseconds to define minimum frame time, minimum count time and readout time. + +### 1.0.0-rc.138 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Cleanup DECTRIS start-up code to enable a shorter start time +* jfjoch_broker: Allow for asynchronous start to allow overlapping detector configuration with other beamline preparations +* jfjoch_broker: Goniometer axis name is converted to lowercase +* jfjoch_broker: Fix bug, where wrong HTTP error codes were returned +* jfjoch_process: Improve sigma estimation during merging (K. Takaba) +* jfjoch_process: Modify spot finding thresholds + +### 1.0.0-rc.137 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Better track time for each operation in the processing stack +* jfjoch_broker: Rewrite preprocessing of diffraction images in the non-FPGA workflow to better use GPUs (work in progress) +* jfjoch_broker: Remove ROI calculation in the non-FPGA workflow (work in progress) +* jfjoch_viewer: Toolbar displays image number starting from 1 (instead of 0) + +### 1.0.0-rc.136 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Improve logic regarding indexing architecture and thread pools (work in progress). + +### 1.0.0-rc.135 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* Multiple small bug fixes scattered across the whole code base. (detected with GPT-5.4) +* jfjoch_viewer: Improve image render performance + +### 1.0.0-rc.134 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long +* jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata +* XDS plugin: Add generation of Jungfraujoch plugin for XDS +* CI: Add tests with XDS and DIALS (`xia2.ssx`) + +### 1.0.0-rc.133 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. + +* jfjoch_broker: Use httplib for HTTP server instead of Pistache +* jfjoch_broker: Drop OpenSSL support +* jfjoch_broker: Base work for multi-lattice support in the future +* jfjoch_broker: Improve recording time of data analysis steps +* jfjoch_writer: Save per-image information about data analysis timing +* Update dependencies to more recent versions (spdlog, HDF5, Catch2, httplib) + +### 1.0.0-rc.132 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* Documentation: Fix equation rendering + +### 1.0.0-rc.131 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug in saving JUNGFRAU calibration (pedestal/pedestalRMS) +* jfjoch_viewer: Fix calibration (pedestal) images being open flipped +* jfjoch_process: Add space group detection (EXPERIMENTAL) + +### 1.0.0-rc.130 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Rotation indexer has two retries if it fails +* jfjoch_broker: Rotation indexer handles small number of rotation images (like test shot) +* jfjoch_broker: Integration calculates background mask based on R2 radius +* jfjoch_process: HDF5 files are not saved by default + +### 1.0.0-rc.129 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Significant improvements in TCP image socket, as a viable alternative for ZeroMQ sockets (only a single port on broker side, dynamically change number of writers, acknowledgments for written files) +* jfjoch_broker: Delta phi is calculated also for still data in Bragg prediction +* jfjoch_broker: Image pusher statistics are accessible via the REST interface +* jfjoch_writer: Supports TCP image socket and for these auto-forking option + +### 1.0.0-rc.128 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Handle properly reuse of image buffer locations +* jfjoch_broker: Fix bug in counting idle slots +* jfjoch_broker: Force obtuse angle for monoclinic cells +* jfjoch_process: Change scaling refinement tolerance + +### 1.0.0-rc.127 +This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Default EIGER readout time is 20 microseconds +* jfjoch_broker: Multiple improvements regarding performance +* jfjoch_broker: Image buffer allows to track frames in preparation and sending +* jfjoch_broker: Dedicated thread for ZeroMQ transmission to better utilize the image buffer +* jfjoch_broker: Experimental implementation of transmission with raw TCP/IP sockets +* jfjoch_writer: Fixes regarding properly closing files in long data collections +* jfjoch_process: Scale & merge has been significantly improved, but it is not yet integrated into mainstream code + +### 1.0.0-rc.126 +This is an UNSTABLE release. If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Fix bug for monoclinic space groups being wrongly refined when beta is much different from 90 deg. + +### 1.0.0-rc.125 +This is an UNSTABLE release. This version adds scaling and merging. These are experimental at the moment, and should not be used for production analysis. +If things go wrong with analysis, it is better to revert to 1.0.0-rc.124. + +* jfjoch_broker: Improve logic on switching on/off spot finding +* jfjoch_broker: Increase maximum spot count for FFBIDX to 65536 +* jfjoch_broker: Increase default maximum unit cell for FFT to 500 A (could have performance impact, TBD) +* jfjoch_process: Add scaling and merging functionality - program is experimental at the moment and should not be used for production analysis +* jfjoch_viewer: Display partiality and reciprocal Lorentz-polarization correction for each reflection +* jfjoch_writer: Save more information about each reflection + +### 1.0.0-rc.124 +This is an UNSTABLE release. This version significantly rewrites code to predict reflection position and integrate them, +especially in case of rotation crystallography. If things go wrong with analysis, it is better to revert to 1.0.0-rc.123. + +* jfjoch_broker: Improve reflection position prediction and Bragg integration code. +* jfjoch_broker: Align with XDS way of calculating Lorentz correction and general notation. +* jfjoch_writer: Fix saving mosaicity properly in HDF5 file. +* jfjoch_viewer: Introduce high-dynamic range mode for images +* jfjoch_viewer: Ctrl+mouse wheel has exponential change in foreground (+/-15%) +* jfjoch_viewer: Zoom-in numbers have better readability + +### 1.0.0-rc.123 +This is an UNSTABLE release. + +* jfjoch_broker: Use newer version of Google Ceres for (potential) CUDA 13 compatibility +* jfjoch_broker: Improve performance of generating preview images, especially for large detectors (9M-16M) +* jfjoch_viewer: Improve performance of displaying images, especially for large detectors (9M-16M) +* jfjoch_viewer: Add more color schemes for better image readability +* HDF5: Common mutex for reading and writing HDF5 if both operations were to happen in the same executable +* HDF5: suppress warning if path (upstream group) doesn't exist when checking if leaf exists + +### 1.0.0-rc.122 +This is an UNSTABLE release. + +* jfjoch_broker: Add thresholding to prefer shorter vectors after FFT +* jfjoch_broker: Add experimental mosaicity estimation for rotation experiments (this is work in progress) +* jfjoch_broker: Update nlohmann::json to 3.12.0 +* jfjoch_viewer: Display file opening errors +* jfjoch_viewer: When loading files over DBus add retry/back-off till the file is available + +### 1.0.0-rc.121 +This is an UNSTABLE release. + +* jfjoch_broker: Report changes in the image buffer, so viewer doesn't reload constantly +* jfjoch_viewer: Improve performance of loading images +* jfjoch_viewer: Auto-throttle image loading in HTTP-sync / movie modes +* jfjoch_viewer: Auto-foreground calculated with histogram +* jfjoch_viewer: Fix rare segmentation fault + +### 1.0.0-rc.120 +This is an UNSTABLE release. + +* jfjoch_broker: Improve performance of binary plot export + +### 1.0.0-rc.119 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Add binary export of data analysis plots over OpenAPI +* jfjoch_broker: Minor fixes to HTTP error handling +* jfjoch_viewer: Prefer binary plots over JSON plots +* jfjoch_viewer: Change foreground with F button + wheel +* jfjoch_viewer: Change way how angles are displayed +* jfjoch_viewer: Display resolution of the mouse cursor in top left corner + +### 1.0.0-rc.118 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Fix issue when HTTP sync silently disconnected when it was enabled when the broker was starting measurement. +* jfjoch_broker: Add protections on time of geometry optimization and reduce rotation recalculations + +### 1.0.0-rc.117 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add ROI results to the dataset info plots +* jfjoch_writer: Remove HTTP interface, as it is not needed/used at the moment + +### 1.0.0-rc.116 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_viewer: Add binning options in the context menu + +### 1.0.0-rc.115 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Default spot finding settings can be configured via config JSON +* jfjoch_viewer: FFT analysis of data in the dataset plot + +### 1.0.0-rc.114 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Fix generating JPEG images with resolution estimation + +### 1.0.0-rc.113 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Improve handling of rotation indexing +* jfjoch_broker: More information saved in CBOR end message (WIP) +* jfjoch_writer: Save rotation indexing lattice parameters and Niggli class +* jfjoch_viewer: Remove (for now) primitive cell information +* jfjoch_viewer: Use angle for dataset info plot for rotation scans + +### 1.0.0-rc.112 +This is an UNSTABLE release and not recommended for production use (please use rc.111 instead). + +* jfjoch_broker: Experimental rotation (3D) indexing +* jfjoch_broker: Minor fix to error in optimizer potentially returning NaN values + +### 1.0.0-rc.111 +This is an UNSTABLE release. + +* jfjoch_viewer: Remove 3D lattice viewer (not really useful at this moment) +* jfjoch_viewer: Fix auto contrast not refreshing image + +### 1.0.0-rc.110 +This is an UNSTABLE release. + +* jfjoch_broker: Add auto-contrast option for preview images +* Frontend: Add logo image +* jfjoch_viewer: Add logo image +* jfjoch_viewer: For image chart allow to set min value to zero +* jfjoch_viewer: For resolution estimation plots, visualization uses 1/d^2 as measure +* jfjoch_viewer: Add 3D unit cell visualization (experimental/WIP/not really there) +* Documentation: Add logo image + +### 1.0.0-rc.109 +This is an UNSTABLE release. + +* jfjoch_viewer: Add keyboard shortcuts and option to copy image to clipboard +* jfjoch_broker: Fix bit-width and exposure time for PSI EIGER detectors + +### 1.0.0-rc.108 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when resolution estimation/B-Factor/Profile radius were not set (NaN) +* jfjoch_viewer: Show spots is off by default, resolution ring mode is enabled by default +* jfjoch_viewer: Fit to window of image is now default when size of the grid changes + +### 1.0.0-rc.107 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor polishing of new functionality +* jfjoch_broker: Use NaN for empty azimuthal bins + +### 1.0.0-rc.106 +This is an UNSTABLE release. + +* jfjoch_viewer: Allow for multiple dataset info plots +* jfjoch_viewer: Highlight current element in grid + +### 1.0.0-rc.105 +This is an UNSTABLE release. + +* jfjoch_viewer: Clean-up widgets slightly +* jfjoch_viewer: Limit right panel to 600 pixels +* jfjoch_viewer: Parse crystal symmetry type +* jfjoch_viewer: Grid scan view takes color map and can be fit to zoom + +### 1.0.0-rc.104 +This is an UNSTABLE release. + +* jfjoch_writer: Fix and improve the way grid scan geometry is saved (non-NXmx extension makes it way easier) +* jfjoch_viewer: Display grid scan results in 2D (work in progress) +* jfjoch_viewer: Improve auto-scaling on start of images (work in progress) +* jfjoch_viewer: Add B-factor and resolution estimate to the dataset info plots + +### 1.0.0-rc.103 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer +* jfjoch_broker: Change behavior for modular detectors: coordinates of 0-th pixel can be now arbitrary and detector will be cropped to the smallest rectangle limited by module coordinates + +### 1.0.0-rc.102 +This is an UNSTABLE release. + +* jfjoch_viewer: Minor improvements to the viewer + +### 1.0.0-rc.101 +This is an UNSTABLE release. + +* jfjoch_viewer: Auto load is better handling change of states +* jfjoch_viewer: Fix DBus registration +* jfjoch_viewer: Handle charts better with vertical lines on hover and status bar update +* jfjoch_viewer: Calculate ROI in a more efficient way + +### 1.0.0-rc.100 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix dbus registration +* jfjoch_viewer: Remove background slider for diffraction image +* jfjoch_viewer: Adjustments for 2D azimuthal image viewer + +### 1.0.0-rc.99 +This is an UNSTABLE release. + +* jfjoch_broker: Fix output during mask data collection + +### 1.0.0-rc.98 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors fix dark data collection during initialization + +### 1.0.0-rc.97 +This is an UNSTABLE release and not recommended for production use (please use rc.96 instead). + +* jfjoch_broker: For DECTRIS detectors add dark data collection during initialization for bad pixel mask +* jfjoch_broker: Refactor of calibration logic for more clear code (likely to introduce problems) +* jfjoch_viewer: Add option to handle user pixel mask (experimental) +* jfjoch_viewer: More options for ROI +* jfjoch_viewer: Add window to display calibration + +### 1.0.0-rc.96 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* jfjoch_broker: Remove PNG preview, no dependency on libpng +* jfjoch_writer: Fix UTC timestamp being generated wrong (mix between milli- and microseconds) +* jfjoch_viewer: Show data collection time in dataset tooltip +* jfjoch_viewer: Allow to choose the calibrant (presets for LaB6 and silver behenate) +* jfjoch_viewer: Auto foreground value +* Use external libjpeg-turbo and libtiff: simpler build stack, these are built and linked statically in automated Docker builds +* Remove OpenBLAS dependency + +### 1.0.0-rc.95 +This is an UNSTABLE release. + +* Fixes in CI pipeline +* Add git-lfs to Rocky8 docker image + +Previous releases (91-94) had a wrong FPGA image upload to Gitlab release. This is now solved. + +### 1.0.0-rc.94 +This is an UNSTABLE release. + +* FFTIndexer: Add limit on angles to avoid colinear vectors +* Docker images: Add 3D Qt +* Gitea: Fixes to the pipeline + +### 1.0.0-rc.93 +This is an UNSTABLE release. + +* CI: Fixes to Gitlab based pipeline +* PCIe driver: Fix PCIe revision being hex number + +### 1.0.0-rc.92 +This is an UNSTABLE release. + +* jfjoch_broker: Fix code that predicted Bragg reflections scattering back from the sample. + +### 1.0.0-rc.91 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* FPGA: Implement high pixel value threshold - pixels above the given value will be considered saturated +* jfjoch_broker: Spot finding and integration predictions are ported to a GPU +* jfjoch_broker: Estimate resolution +* jfjoch_broker: Lattice search +* jfjoch_broker: Many more improvements in image analysis + +### 1.0.0-rc.90 +This is an UNSTABLE release. + +* jfjoch_broker: for indexing min index spots for a viable cell can be changed via OpenAPI +* jfjoch_viewer: Optional auto-reanalyze images +* jfjoch_writer: Add option where no files at all are saved +* Documentation: improvements + +### 1.0.0-rc.89 +This is an UNSTABLE release. + +* jfjoch_broker: Fix resolution estimation code +* jfjoch_broker: Fix Wilson B-factor calculation code +* jfjoch_viewer: Improve display of plots +* jfjoch_viewer: Fix segmentation fault +* jfjoch_viewer: Display missing metadata when using HTTP +* jfjoch_viewer: Fix bug when opening the same file twice + +### 1.0.0-rc.88 +This is an UNSTABLE release. + +* jfjoch_viewer: Add resolution estimation to the image information +* jfjoch_broker: Minor changes to resolution estimate routine + +### 1.0.0-rc.87 +This is an UNSTABLE release. + +* jfjoch_viewer: Display more image metadata (angle / exposure time) +* jfjoch_viewer: Improve I/sigma and B-factor plots +* jfjoch_broker: Estimate resolution based on visible spots + +### 1.0.0-rc.86 +This is an UNSTABLE release. + +* jfjoch_broker: Update logic when initializing detector to make it a bit more resilient +* Gitea pipelines have nocuda option for all architectures + +### 1.0.0-rc.85 +This is an UNSTABLE release. + +* jfjoch_viewer: When using online view, dataset info plots are not switched back to the first category for each image +* jfjoch_viewer: Handle spot count better in dataset info plots +* jfjoch_viewer: Highlight spots in ice ring resolutions in cyan, when detection is enabled + +### 1.0.0-rc.84 +This is an UNSTABLE release. + +* jfjoch_broker: Write in log which detector is being initialized +* Changes to automated build system + +### 1.0.0-rc.83 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix in generating preview image for signed data (wrong bit-width was assumed before) +* CI: Fix script to generate python client + +### 1.0.0-rc.82 +This is an UNSTABLE release. + +* jfjoch_viewer: Enable FFTW based indexing in viewer (very slow at the moment) +* Frontend: Minor fixes +* Build scripts: Minor fixes to FFTW + +### 1.0.0-rc.81 +This is an UNSTABLE release. This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Add option to detect ice rings, adjust width of ice ring and change of logic to exclude ice rings in indexing +* jfjoch_broker: Add FFTW based indexer for CPU only indexing +* jfjoch_broker: Enable saving X-ray fluorescence spectra +* jfjoch_writer: Write total spot count (before filtering) +* jfjoch_viewer: Add more information on source, sample, and button to show ice rings +* jfjoch_viewer: Enable data processing inside the viewer + +CI: Moving from Gitlab to Gitea at PSI + +### 1.0.0-rc.80 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when wrong value for a plot (NaN or infinity) would lead to a null in a plot, which cannot be parsed by viewer + +### 1.0.0-rc.79 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix bug when loading new dataset was creating a cascade of signals leading to poor performance +* jfjoch_writer: Save nimages_per_trigger in detectorSpecific + +### 1.0.0-rc.78 +This is an UNSTABLE release. + +* jfjoch_viewer: Using a single event loop (reading images is not in dedicated thread anymore) + +### 1.0.0-rc.77 +This is an UNSTABLE release. + +* jfjoch_viewer: Display detector and dataset settings with tooltips +* jfjoch_viewer: Clean excessive HDF5 warnings +* jfjoch_viewer: Display unit cell +* jfjoch_extract_hkl: Write a tool to extract reflection intensity from a dataset + +### 1.0.0-rc.76 +This is an UNSTABLE release. + +* jfjoch_broker: Increase predicted hkl to 100.0, use lighter math to exclude too-high resolution ones +* jfjoch_broker: Use standard deviation formula to find profile radius (not the one using median) +* jfjoch_writer: Save space group number (non-NXmx addition) in addition to name +* jfjoch_viewer: Fix the bug on reading space_group as string +* jfjoch_viewer: Add missing resolution labels on rings +* jfjoch_viewer: Remove Q value from the status bar + +### 1.0.0-rc.75 +This is an UNSTABLE release. + +* jfjoch_broker: EIGER2 missing minimum threshold - hardcoded to 2.7 keV for the time being + +### 1.0.0-rc.74 +This is an UNSTABLE release. + +* jfjoch_broker: Fix for EIGER UDP port settings (vertical half of the module missing) +* jfjoch_broker: Detector settings were not applied for EIGER/DECTRIS detector when changed after initialization + +### 1.0.0-rc.73 +This is an UNSTABLE release. + +* jfjoch_broker: Space group number treatment in OpenAPI was wrong, zero value is no longer allowed and no longer default + +### 1.0.0-rc.72 +This is an UNSTABLE release. +This release introduces new features, which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Refactor of indexing and geometry refinement code +* jfjoch_broker: Handle space group/centering in refinement code +* jfjoch_broker: Replace mosaicity with profile radius: refining the former is difficult with still images +* jfjoch_broker: There is no longer 0.5 pxl offset for spots-to-reciprocal-space conversion +* jfjoch_writer: Experimental saving of reflections +* jfjoch_writer: Save space group name as string +* jfjoch_viewer: Add profile radius and B-factor +* jfjoch_viewer: Show 4 digits for wavelength +* jfjoch_viewer: Match rings between calibrant and observation (will handle missing/wrong rings) +* FPGA: Use UDP destination port to distinguish between detector modules and data streams +* FPGA: Add experimental PTP core (PTP over L2, only Sync/Follow_up) +* FPGA driver: Fix for Linux kernel 6.12+ (thanks to Tim Gruene) + +### 1.0.0-rc.71 +This is an UNSTABLE release. + +* jfjoch_broker: Remove resolution estimation via machine learning +* jfjoch_broker: Harmonize code to analyze spot finding results (indexing/refinement/integration) between CPU and FPGA receivers +* jfjoch_viewer: Fix error when HDF5 files with indexing results couldn't be loaded on a machine without GPU + +### 1.0.0-rc.70 +This is an UNSTABLE release. +This release introduces new features (geometry refinement), which usually means these need more field testing before enough maturity. +For production use we recommend waiting for a future bug-fix release. + +* jfjoch_broker: Fix bug when PSI EIGER frame time was not set properly at the start of the measurement +* jfjoch_broker: Fix PONI rot2 angle rotating detector in a wrong direction (PyFAI convention is for this angle to rotate detector downwards) +* jfjoch_broker: Enable geometry refinement - first try (work in progress) +* jfjoch_viewer: Fix deadlock when opening HTTP connections +* jfjoch_viewer: Display rings as ellipses with detector tilt +* jfjoch_viewer: Add button to calibrate detector geometry based on LaB6 image +* jfjoch_writer: Save detector tilt angles (rot1, rot2, rot3) + +* Add Google Ceres a non-linear least-square optimization library to Jungfraujoch +* Add experimental detector calibration routines (for LaB6) +* Improve documentation on the ZeroMQ writer notification socket and detector geometry + +### 1.0.0-rc.69 +This is an UNSTABLE release. + +* jfjoch_viewer: Metadata can be modified for an open dataset (no option to save) +* jfjoch_viewer: Refactor multiple issues in the viewer regarding image reading code to allow for further developments +* jfjoch_viewer: Resolution rings not enabled by default +* jfjoch_broker: Handle properly PONI rotations in dataset settings though still not updated properly in the HDF5 file + +### 1.0.0-rc.68 +This is an UNSTABLE release. + +* jfjoch_broker: Temperature threshold can be changed for JUNGFRAU detector +* jfjoch_broker: Default detector settings can be configured for each detector separately +* jfjoch_broker: Refactor spot filtering code, max spot count can be modified for dataset settings +* jfjoch_broker: Refactor indexing refinement, make it the same for both FFBIDX and FFT indexing +* jfjoch_broker: Reference unit cell will be taken into account for FFT indexing to filter +* jfjoch_broker: Review PONI rotation angles and azimuthal angle conventions along with PyFAI + +### 1.0.0-rc.67 +This is an UNSTABLE release. + +* jfjoch_broker: Enable SSL +* jfjoch_broker: Wilson B-factor only provided if fit is relatively OK (R^2 > 0.3); this will be refined much more in the future + +### 1.0.0-rc.66 +This is an UNSTABLE release. + +* jfjoch_broker: Indexers operate as a thread pool +* jfjoch_viewer: Increase interval between loading images + fix too many verbose messages + +### 1.0.0-rc.65 +This is an UNSTABLE release. + +* jfjoch_broker: Print information regarding used image pushers +* jfjoch_viewer: Allow syncing with Jungfraujoch server +* OpenAPI: Clarify licensing terms in the file + +### 1.0.0-rc.64 +This is an UNSTABLE release. + +* jfjoch_broker: Fix issue in receiver light with very long preparation time for threads +* jfjoch_broker: Add verbose option +* jfjoch_broker: Don't trigger pedestal if critical settings are not changed when loading detector settings +* jfjoch_broker: Detector left in busy state when detector settings were improper +* jfjoch_viewer: Modify DBus interface to avoid loading same file and image 0 multiple times +* jfjoch_lite_perf_test: Add verbose option + +### 1.0.0-rc.63 +This is an UNSTABLE release. + +* jfjoch_broker: Save NX/NY for grid scan result +* jfjoch_broker: Add processing time to CBOR output and plot +* jfjoch_writer: Add processing time to data file + +### 1.0.0-rc.62 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where low resolution spots were not counted properly +* jfjoch_broker: Spot count is provided prior to filtering of spots to max_spot_count +* jfjoch_broker: Add more spot count information to CBOR +* jfjoch_viewer: Fix issue with ROI drawing resulting in multiple overlapping rectangles + +### 1.0.0-rc.61 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug where FFT indexing could result in a very short or even zero length vector +* jfjoch_broker: Ice ring and indexed spot count enabled as plots and saved in grid scan results +* jfjoch_broker: High resolution limit for low res. spot counting can be adjusted + +### 1.0.0-rc.60 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when the neural network inference client was busy and this status was never released +* jfjoch_broker: Revert the indexing threshold with distance from integer for Miller indices +* jfjoch_broker: Fix bug in scattering vector calculation, resulting in indexing not working outside 1.0 A X-ray wavelength + +### 1.0.0-rc.59 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when broker was waiting for notification message before sending end message, resulting in deadlock. +* jfjoch_writer: Verbose option for debugging. + +### 1.0.0-rc.58 +This is an UNSTABLE release. + +* jfjoch_viewer: Fix memory leak +* jfjoch_writer: Add detector_number/serial_number to master file + +### 1.0.0-rc.57 +This is an UNSTABLE release. + +* jfjoch_broker: Fix bug when enabling ML resolution estimation was not possible +* jfjoch_viewer: "Movie" mode + +### 1.0.0-rc.56 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing more bugs related to neural network inference for ML estimation + +### 1.0.0-rc.55 +This is an UNSTABLE release. + +* jfjoch_broker: Fixing minor bugs related to neural network inference for ML estimation + +### 1.0.0-rc.54 +This is an UNSTABLE release. + +* jfjoch_broker: Indexing with AUTO settings (FFBIDX if unit cell provided; FFT if not) +* jfjoch_broker: Don't remove shared memory area when deactivating detector +* jfjoch_writer: Save writer release +* jfjoch_viewer: Increase time for the messages in the status bar + +### 1.0.0-rc.53 +This is an UNSTABLE release. + +* PCIe driver: Imperfect solution for RHEL 9.5+ changes +* jfjoch_writer: Fix to angle containers for AutoProc compatibility +* jfjoch_fpga_test: Use consecutive number for devices, not interleaved + +### 1.0.0-rc.52 +This is an UNSTABLE release. + +* jfjoch_viewer: Use warmer colors to distinguish from AareGUI +* jfjoch_viewer: Minor adjustments to DBus setting image number +* jfjoch_broker: Fix in low resolution spot count plotting + +### 1.0.0-rc.51 +This is an UNSTABLE release. + +* jfjoch_broker: Send preview in PNG format +* jfjoch_broker: Provide count of spots in 50.0 - 5.0 A range +* jfjoch_broker: Provide ML resolution estimation in scan result +* jfjoch_broker: Allow removing beam center in web preview + +### 1.0.0-rc.50 +This is an UNSTABLE release. + +* The release fixes some of many bugs introduced in recent releases +* jfjoch_viewer: display predictions for indexed cells + +### 1.0.0-rc.49 +This is an UNSTABLE release. + +* jfjoch_broker: Add sample temperature (K) and ring current (mA) to metadata +* jfjoch_writer: For angle containers in NXmx add _end dataset, sample temp. and ring current + +### 1.0.0-rc.48 +This is an UNSTABLE release. + +* jfjoch_broker: fix the bug when a unit cell was not exported for a scan result. + +### 1.0.0-rc.47 +This is an UNSTABLE release. + +* jfjoch_viewer: fix dbus service path +* jfjoch_writer: fix CBF/TIFF writing + +### 1.0.0-rc.46 +This is an UNSTABLE release. + +* jfjoch_viewer: remove dependency on image analysis + +### 1.0.0-rc.45 +This is an UNSTABLE release. + +* jfjoch_broker: Detector list returns pixel size (mm) + +### 1.0.0-rc.44 +This is an UNSTABLE release. + +* jfjoch_broker: more general definition of scan result export + +Breaking changes: +* It removes additions to OpenAPI from 1.0.0-rc.43 +* It makes changes to the "unit_cell" definition in OpenAPI specs. It might be harmless in some languages and may result in errors in other implementations. + +### 1.0.0-rc.43 +This is an UNSTABLE release. + +* jfjoch_broker: Export grid scan results into a single data structure + +### 1.0.0-rc.42 +This is an UNSTABLE release. + +* jfjoch_broker: Add pixel_sum to CBOR output. +* jfjoch_broker: Changes to sigma estimation in QuickIntegrate routine +* jfjoch_writer: Save pixel_sum + +### 1.0.0-rc.41 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Estimate B-factor, mosaicity to evaluate crystal diffraction +* jfjoch_broker: Export GPU count via OpenAPI +* jfjoch_broker: Enable 2D azimuthal integration and PONI rotations for detector + +* FPGA: Increase the number of integration bins to 2048 + +### 1.0.0-rc.40 +This is an UNSTABLE release. This release includes multiple new features, it should not be used in production at the moment. + +* jfjoch_broker: Jungfraujoch supports grid scan metadata, including dedicated plotting schemes and NXmx structures +* jfjoch_broker: Improve metadata for rotation data collection +* jfjoch_broker: Better handling of plotting +* jfjoch_broker: FFT based indexing +* jfjoch_broker: Integration, first try, results not saved at the moment +* jfjoch_broker: Internal improvements in image handling + +* jfjoch_writer: Multiple adjustments adapt to changes in this release for new features +* jfjoch_writer: New state management model to improve clarity of error reporting + +* jfjoch_viewer: Remote control via DBus + +* Frontend: Multiple adjustments for new features +* Frontend: Grid scan plots + +WARNING! OpenAPI contains breaking changes in regard to plotting results, so care has to be taken. + +### 1.0.0-rc.39 +* FPGA: Bugfix for pixel masked for data analysis if summation was on +* jfjoch_viewer: Fix segmentation fault when cursor was outside of image + +### 1.0.0-rc.38 +* jfjoch_broker: Neural net model is not linked with C++ code due to deployment issues, it is rather distributed as python code, connected via REST +* jfjoch_broker: Neural net model can use all 4 quadrants of the detector +* jfjoch_broker: For EIGER image time can be provided through /start +* jfjoch_viewer: Add image list option +* jfjoch_viewer: Drawing circular ROIs with shift +* jfjoch_viewer: Enable image summation +* jfjoch_viewer: Image reader is significantly reworked, hopefully without affecting the viewer + +### 1.0.0-rc.37 +* jfjoch_broker: Make locking rules more flexible +* jfjoch_broker: Load mask via SIMPLON interface for DECTRIS detectors +* jfjoch_viewer: Add status bar + +### 1.0.0-rc.36 +This is an UNSTABLE release. Wait for a new version to use in a production environment. + +* jfjoch_broker: Support for Jungfraujoch Lite is enabled - software-based receiver for DECTRIS detectors (required a lot of refactoring, potentially leading to unstable code) +* jfjoch_broker: Enable Resonet support (ML-based diffraction resolution estimation) +* jfjoch_broker: Fix error in compression, where bitshuffle/LZ4 and bitshuffle/Zstd HDF5 headers were wrongly generated for 8-bit and 32-bit data +* jfjoch_writer: Increase buffering to 1000 images in the receiver +* jfjoch_writer: Images can be written as CBF or TIFF in addition to HDF5 + +### 1.0.0-rc.35 +This is an UNSTABLE release, not properly tested. Wait for a new version before using it in production. + +* jfjoch_broker: If module is delayed by more than 50 frames versus other modules, it will be ignored and receiver is not waiting. +* jfjoch_writer: Save EIGER energy threshold +* jfjoch_writer: Add `/entry/sample/goniometer` for compatibility with `eiger2cbf` program + +### 1.0.0-rc.34 +This is an UNSTABLE release - introducing new features, but not properly tested. Wait for a new version before using it in production. + +* jfjoch_broker: More consistency for file format definition (breaking change in API from 1.0.0-rc.31 for file writer settings) +* jfjoch_broker: For storage cells mask is logical sum of detector bad pixels for all storage cells +* jfjoch_broker: Handle situation when detector doesn't want to gracefully stop (to be tested) +* jfjoch_broker: Center-of-mass position and mean for ROI is added to available plots +* jfjoch_viewer: Can extract data analysis results from "legacy" format +* jfjoch_viewer: Display dataset name +* FPGA: Pixel mask is used for data analysis part even if it is not applied to pixels +* FPGA: Add pixel sum to module statistics +* FPGA: ROI number is reduced to 16, but pixel can belong to every defined ROI +* FPGA: Spot finder is back to full dynamic range (24-bit) +* FPGA: More debug features for internal FIFOs + +Known issues: +* ROI count flag was added to firmware. For the time being the flag will be wrongly set to 10 due to mismatch of FPGA build scripts. +* EIGER data acquisition has an issue that is currently debugged + +### 1.0.0-rc.33 +* jfjoch_broker: Fix issue with EIGER settings being loaded improperly + +### 1.0.0-rc.32 +* jfjoch_broker: Refactor code for azimuthal integration for further improvements +* jfjoch_broker: Minor fix for EIGER (trim energies are manually set for E9M, to be fixed properly later) +* jfjoch_writer: Fix too much verbose information +* FPGA: Minor fixes to spot finder (enable two-pass operation and limit number range to int20) + +### 1.0.0-rc.31 +This is UNSTABLE release - introducing many features, but still needs more testing. +Expecting soon to put bugfix release. + +* jfjoch_writer: Allow to enable overwriting existing files (not enabled by default) +* jfjoch_writer: Add new HDF5 master file format, which uses HDF5 virtual data sets and links processing results to data files (not enabled by default) +* jfjoch_viewer: Image viewer, early test version +* jfjoch_broker: Fixes to counting packets per dataset/image +* jfjoch_broker: Image buffer is accessible for outside to check images +* jfjoch_broker: error/saturated pixels and dedicated ROI "beam" can be tracked online +* jfjoch_broker: Fix bug in handling pedestal G1/G2 count time for JUNGFRAU +* jfjoch_broker: Fix bug in applying pixel mask interfering with pedestal calculation +* jfjoch_broker: Fix bug in EIGER initializing +* jfjoch_broker: Save maximum pixel value to HDF5 file and export as Web plot +* PCIe driver: Add PCIe link speed and width +* FPGA: Improve counting error/saturated/min/max pixels +* FPGA: Spot finder is gradual column-wise (15 columns up/down) and fixed row-wise (32 pixel boxes); previously it was fixed both column- and row-wise with 32x32 pixel areas +* FPGA: Require Vivado 2022.2 + +Warning: +There are breaking changes to HDF5 file format, renaming entries regarding image storage cell number and image collection efficiency. + +### 1.0.0-rc.30 +* jfjoch_writer: replace non-blocking with blocking operation on internal queues - less likely to "lose" images within the writer + +### 1.0.0-rc.29 +* jfjoch_broker: refactor logic regarding frame time and count time for more flexibility for EIGER and JUNGFRAU +* jfjoch_broker: readout time for EIGER is 3 us and JUNGFRAU is 20 us, this can be changed in input file +* jfjoch_broker: OpenAPI interface includes more ways to provide information on the status (error/warning/info) +* jfjoch_broker: ROIs handling via OpenAPI and frontend is more user friendly + +Warning - two breaking changes to OpenAPI: +* Handling of ROIs is through `/config/roi` path only for both circle and box ROIs, paths in `/roi` are no longer accessible +* `broker_status` structure introduced in 1.0.0-rc.28 has member `message` and not `error_message` to allow +handling info/warning messages as well + +### 1.0.0-rc.28 +* jfjoch_broker: save error message for initialization and data collection and provide these with OpenAPI +* jfjoch_broker: fixed issue when in error state, response to /wait_till_done was not compliant with OpenAPI specs +* jfjoch_test: remove header that failed when CUDA is absent during compilation +* frontend: add soft trigger button in data collection tab +* frontend: show error message when in error state +* CMake: add option to force compilation without CUDA (-DJFJOCH_USE_CUDA=OFF) + +### 1.0.0-rc.27 +* jfjoch_broker: add option to select electron source in instrument metadata, adapt wavelength calculation +* jfjoch_broker: update pistache web server version +* jfjoch_writer: minor changes to republish logic +* Improvements to documentation + +### 1.0.0-rc.26 +* jfjoch_broker: implement ZeroMQ stream for image metadata information +* jfjoch_broker: refactor ZeroMQ stream for preview: start/end messages always sent +* jfjoch_broker: add crystal lattice plots +* jfjoch_broker: remove empty bins from the plots +* jfjoch_broker: Fix bugs in ModuleSummation and MXAnalyzer for CPU "long" summation +* jfjoch_broker: Fix bug when mean background estimation / indexing rate were affected by previous experiment +* jfjoch_writer: fix missing "-w" parameter +* jfjoch_writer: temporary files have ".tmp" suffix +* jfjoch_writer: refactor logic for watermarks +* jfjoch_writer: report on internal FIFO utilization +* jfjoch_writer: clean-up naming for azimuthal integration and background estimate +* jfjoch_writer: write final background estimate and indexing rate in the master file +* tools/: remove unnecessary tools, make naming consistent +* CBOR: Add indexing rate and background estimate to end message +* CBOR: Clean-up documentation + +### 1.0.0-rc.25 + +* Updates to documentation +* License set to GPLv3 / OHL-S +* Fix bug in DiffractionExperiment::GetDefaultPlotBinning() - resulting in division by 0 if image time longer than 500ms +* Add information on JUNGFRAU conversion and geometry transformation to CBOR and HDF5 + +### 1.0.0-rc.24 + +New FPGA functionality: +* EIGER supports 8, 16 and 32-bit data input (for 8-bit mode at half performance; for 32-bit "real" depth is 23-bit + 1-bit signed) +* Output possible to 8, 16 and 32-bit data +* Threshold is applied before summation +* Pixel mask can be applied on FPGA +* Mark pixels with ADC content = 0 as bad pixels +* FPGA stores semantic version information (access via /sys/class/misc/jfjoch.../version) + +New software functionality: +* Long summation (above 256 frames) done on CPU +* Mechanism to save arbitrary data to HDF5 file +* ZeroMQ preview has option to send start message +* Rework pixel mask + add statistics displayed in web interface + +Bug fixes: +* Web frontend: Update preview image automatically during data acquisition +* jfjoch_broker: Error handling if CUDA driver is not installed +* jfjoch_broker: Correctly update progress during pedestal +* jfjoch_broker: Provide proper error when uploaded file is not a proper TIFF +* jfjoch_action_test: enable HLS simulation + +Documentation improvement and placement in a dedicated directory diff --git a/_sources/CPU_DATA_ANALYSIS.md.txt b/_sources/CPU_DATA_ANALYSIS.md.txt new file mode 100644 index 000000000..bfb7de4f6 --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS.md.txt @@ -0,0 +1,82 @@ +# CPU-side crystallographic data analysis (Jungfraujoch) + +This document describes the crystallographic algorithms implemented in Jungfraujoch for **CPU**- and **GPU**-side real‑time and near‑real‑time data analysis. + +**Scope.** The pipeline covered here comprises: + +1. geometry mapping and corrections, +2. azimuthal integration (powder/radial profiles), +3. Bragg spot finding (strong pixels → connected components → spot descriptors), +4. indexing (still and rotation modes), +5. Bravais lattice / centering inference, +6. geometry and lattice refinement, +7. reflection prediction (still and rotation), +8. Bragg integration by either 2D box summation or profile fitting (Kabsch, reference-free), +9. scaling and merging, +10. merge-level error modelling, outlier rejection and the resolution cutoff, +11. space-group determination from the merged intensities (Laue group, screw axes, glide planes, centering, the centre of symmetry), the twinning check and the translational pseudo-symmetry check, +12. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref), +13. amplitude estimation (French–Wilson) and R-free test-set flagging, +14. optional model-based validation: rigid-body placement of a supplied model, R-free against it, sigma_A-weighted 2mFo−DFc / mFo−DFc electron-density maps, and an anomalous difference map with the strongest anomalous sites named. + + +The reference is split into four parts, in pipeline order; the section numbers run continuously +across them and are the ones the rest of the documentation cites. + +- [From images to spots (§0–§3)](CPU_DATA_ANALYSIS_IMAGE.md) — device-side decoding, geometry and + reciprocal-space mapping, azimuthal integration, spot finding. +- [Indexing and geometry refinement (§4–§7)](CPU_DATA_ANALYSIS_INDEXING.md) — FFT and fast-feedback + indexing, the lattice search, geometry refinement, post-refinement and powder calibration. +- [Prediction, integration, scaling and merging (§8–§12)](CPU_DATA_ANALYSIS_INTEGRATION.md) — + reflection prediction, profile-fitted integration, scaling, merging, mosaicity and the auxiliary + statistics. +- [Space group and validation (§13–§14)](CPU_DATA_ANALYSIS_DECISIONS.md) — the space-group search, + twinning and translational pseudo-symmetry, the resolution cutoff, diffraction anisotropy, and + model-based validation. + +## References + +The methods draw on, and in places reimplement, solutions from: + +- W. Kabsch, “XDS”, *Acta Cryst.* **D66** (2010), 125–132 and related XDS papers (rotation geometry, partiality, scaling concepts). +- W. Kabsch, “Integration, scaling, space-group assignment and post-refinement”, *Acta Cryst.* **D66** (2010), 133–144 (mosaicity/partiality likelihood treatment; notation such as ζ and rotation factors). +- T. A. White et al., CrystFEL method papers (spot finding, three‑ring integration, serial/still diffraction processing concepts). +- J. Kieffer & J. P. Wright, "PyFAI: a Python library for high performance azimuthal integration on GPU", *Powder Diffraction* **28** (2013), S339-S350 (detector geometry definition, azimuthal integration) +- I. Steller, R. Bolotovsky & M. G. Rossmann, "An algorithm for automatic indexing of oscillation images using Fourier analysis", *J. Appl. Cryst.* **30** (1997), 1036-1040 (the projection/1D-FFT autoindexing algorithm of §5). +- H. Powell, "The Rossmann Fourier autoindexing algorithm in MOSFLM", *Acta Cryst.* **D55** (1999), 1690-1695 (the MOSFLM implementation of it, whose practice is followed) +- P. Gasparotto, L. Barba, H.-C. Stadler et al., "TORO Indexer: a PyTorch-based indexing algorithm for kilohertz serial crystallography", *J. Appl. Cryst.* **57** (2024), 931-944 (the algorithm of the `ffbidx` fast-feedback indexer, §4). +- I. Křivý & B. Gruber, "A unified algorithm for determining the reduced (Niggli) cell", *Acta Cryst.* **A32** (1976), 297-298, and International Tables for Crystallography Vol. A, Table 9.2.5.1 (the Niggli reduction and the lattice-character table of §5.3/§6). +- J. E. Padilla & T. O. Yeates, "A statistic for local intensity differences: robustness to anisotropy and pseudo-centering and utility for detecting twinning", *Acta Cryst.* **D59** (2003), 1124-1130 (the L test, §13.2). +- A. J. C. Wilson, "The probability distribution of X-ray intensities", *Acta Cryst.* **2** (1949), 318-321, and E. R. Howells, D. C. Phillips & D. Rogers, "The probability distribution of X-ray intensities. II. Experimental investigation and the X-ray detection of centres of symmetry", *Acta Cryst.* **3** (1950), 210-214 (the acentric and centric intensity distributions behind the centre-of-symmetry statistics of §13.1 and the Wilson outlier test of §13.3). +- W. H. Baur & D. Kassner, "The perils of Cc: comparing the frequencies of falsely assigned space groups with their general population", *Acta Cryst.* **B48** (1992), 356-369 (writing the centrosymmetric group where the absences cannot decide the centre, §13.1). +- R. J. Read, P. D. Adams & A. J. McCoy, "Intensity statistics in the presence of translational noncrystallographic symmetry", *Acta Cryst.* **D69** (2013), 176-183 (the native-Patterson detection of translational pseudo-symmetry, and the intensity modulation it produces, which the axial-zone screw-absence test scores against). +- A. Barty, R. A. Kirian, F. R. N. C. Maia et al., "Cheetah: software for high-throughput reduction and analysis of serial femtosecond X-ray diffraction data", *J. Appl. Cryst.* **47** (2014), 1118-1131 (peakfinder8: the per-resolution-ring background statistics of §3.2). +- A. Hennequin, B. Couturier, V. V. Gligorov & L. Lacassagne, "SparseCCL: Connected Components Labeling and Analysis for sparse images", DASIP 2019, 65-70 (the connected-component labelling of §3.4, used via ACTS/traccc). +- S. French & K. Wilson, "On the treatment of negative intensity observations", *Acta Cryst.* **A34** (1978), 517-525 (Bayesian amplitude estimation from intensities), and CCP4's ctruncate (C. Ballard & N. Stein), whose anisotropic Wilson prior §10.8 follows, cited through M. D. Winn et al., "Overview of the CCP4 suite and current developments", *Acta Cryst.* **D67** (2011), 235-242. +- A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free cross-validation). +- P. H. C. Eilers, "A perfect smoother", *Anal. Chem.* **75** (2003), 3631-3636, after E. T. Whittaker, "On a new method of graduation", *Proc. Edinburgh Math. Soc.* **41** (1923), 63-75 (the penalised smoother of the fulls' per-frame scale, §10.6). +- R. A. Fisher, "Frequency distribution of the values of the correlation coefficient in samples from an indefinitely large population", *Biometrika* **10** (1915), 507-521 (the z-transformation on which the correction surfaces' held-out half-set CC1/2 is compared). +- M. Wojdyr, "GEMMI: A library for structural biology", *J. Open Source Softw.* **7** (2022), 4200 (model / structure-factor / map machinery used in §14). +- J. P. Wright, "Experiences with GPU decompression for bitshuffle + LZ4 data", HDF5 User Group meeting (2021), and [github.com/jonwright/bslz4decoders](https://github.com/jonwright/bslz4decoders) (device-side decoding of bitshuffle+LZ4 images, §0). +- A. Thorn & G. M. Sheldrick, "ANODE: anomalous and heavy-atom density calculation", *J. Appl. Cryst.* **44** (2011), 1285-1287 (anomalous difference density read at the model's sites). +- R. Kahn, R. Fourme, A. Gadet, J. Janin, C. Dumas & D. Andre, "Macromolecular crystallography with synchrotron radiation: photographic data collection and polarization correction", *J. Appl. Cryst.* **15** (1982), 330-337 (the azimuthal polarization factor of §2.2, applied to the azimuthal profile, the Bragg intensities and the ring background the beam-stop shadow test compares against). +- R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with errors", *Acta Cryst.* **A42** (1986), 140-149 (the sigma_A formalism and the m, D weighting of the map coefficients of §14.4). +- A. Fokine & A. Urzhumtsev, "Flat bulk-solvent model: obtaining optimal parameters", *Acta Cryst.* **D58** (2002), 1387-1392 (the flat bulk-solvent model, its optimal parameters and the range they are physically meaningful over, used when scaling a model to the data in §14). +- P. V. Afonine, R. W. Grosse-Kunstleve & P. D. Adams, "A robust bulk-solvent correction and anisotropic scaling procedure", *Acta Cryst.* **D61** (2005), 850-855 (the grid search over that range that fits k_sol and b_sol, with the overall scale and anisotropic B refitted at each grid point). +- K. Shoemake, "Uniform Random Rotations", in *Graphics Gems III*, ed. D. Kirk, Academic Press (1992), 124-132 (the uniform random rotations the model-fit null of §14.5 is built from). +- G. H. Golub & V. Pereyra, "The differentiation of pseudo-inverses and nonlinear least squares problems whose variables separate", *SIAM J. Numer. Anal.* **10** (1973), 413-432, and L. Kaufman, "A variable projection method for solving separable nonlinear least squares problems", *BIT* **15** (1975), 49-57 (the scale re-fit folded into the rigid-body Jacobian of §14.8). +- Z. Otwinowski & W. Minor, "Processing of X-ray diffraction data collected in oscillation mode", *Methods Enzymol.* **276** (1997), 307-326 (reweighted, de-biased profile-fit variances). +- G. Winter et al., "DIALS: implementation and evaluation of a new integration package", *Acta Cryst.* **D74** (2018), 85-97, and J. Beilsten-Edmands et al., *Acta Cryst.* **D76** (2020), 385-399 (CC1/2 resolution cutoff, merge outlier rejection, scaling error model). +- R. H. Blessing, "An empirical correction for absorption anisotropy", *Acta Cryst.* **A51** (1995), 33-38 (absorption as spherical harmonics of the beam directions). +- P. Evans, "Scaling and assessment of data quality", *Acta Cryst.* **D62** (2006), 72-82, and P. R. Evans, *Acta Cryst.* **D67** (2011), 282-292 (POINTLESS: operator-by-operator point-group scoring, and the axial-zone screw-absence test). +- A. G. W. Leslie & H. R. Powell, "Processing diffraction data with MOSFLM" (2007), NATO Science Series II **245**, 41-51 (post-refinement practice: what is refined per image and what over a wedge). +- D. W. Moreau, H. Atakisi & R. E. Thorne, "Ice in biomolecular cryocrystallography", *Acta Cryst.* **D77** (2021), 540-554 (measured hexagonal-ice ring positions, used by the ice-ring score, the ice flagging and the ice calibrant). +- K. Röttger, A. Endriss, J. Ihringer, S. Doyle & W. F. Kuhs, "Lattice constants and thermal expansion of H2O and D2O ice Ih between 10 and 265 K", *Acta Cryst.* **B50** (1994), 644-648 (the ice Ih cell the ring positions below 1.522 Å are calculated from). +- S. Sheriff & W. A. Hendrickson, "Description of overall anisotropy in diffraction from macromolecular crystals", *Acta Cryst.* **A43** (1987), 118-121 (the overall anisotropic B tensor and its symmetry constraints), and A. N. Popov & G. P. Bourenkov, "Choice of data-collection parameters based on statistic modelling", *Acta Cryst.* **D59** (2003), 1145-1153 (the sigma-aware estimation of the anisotropy of the observed intensity distribution, part of that paper's statistic modelling). +- P. R. Evans & G. N. Murshudov, "How good are my data and what is the resolution?", *Acta Cryst.* **D69** (2013), 1204-1214 (AIMLESS: the anisotropic deltaB as the range of the principal components, and diffraction limits from a cone about each principal direction). +- G. Assmann, W. Brehm & K. Diederichs, "Identification of rogue datasets in serial crystallography", *J. Appl. Cryst.* **49** (2016), 1021-1028, and G. M. Assmann, M. Wang & K. Diederichs, *Acta Cryst.* **D76** (2020), 636-652 (XDSCC12: sigma-tau CC1/2, delta-CC1/2, the Fisher transformation and the rejection discipline the frame disposition follows). +- K. Diederichs & P. A. Karplus, *Nat. Struct. Biol.* **4** (1997), 269-275, and P. A. Karplus & K. Diederichs, *Science* **336** (2012), 1030-1033 (R_meas / R_pim, CC1/2 and CC\*). +- IUCr Commission on Crystallographic Nomenclature, "Statistical descriptors in crystallography", *Acta Cryst.* **A45** (1989), 63-75, and *Acta Cryst.* **A51** (1995), 565-569 (uncertainty conventions). + +(list is not exhaustive; the full citations, with DOIs, are in [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md)) + diff --git a/_sources/CPU_DATA_ANALYSIS_DECISIONS.md.txt b/_sources/CPU_DATA_ANALYSIS_DECISIONS.md.txt new file mode 100644 index 000000000..e803211b1 --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS_DECISIONS.md.txt @@ -0,0 +1,270 @@ +# Data analysis: space group and validation (§13–§14) + +Part of the [CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md); the section numbers are continuous across its four parts. + +```{contents} On this page +:local: +:depth: 2 +``` + + +## 13. Space-group determination and merge-level decisions + +### 13.1 Space-group determination + +When no space group is supplied, a POINTLESS-like search scores Laue-group symmetry (CC of $E^2(h)$ vs $E^2(Rh)$ — the intensities normalised by the mean of their own resolution shell — plus merge self-consistency) and detects screw/centering absences from the $P1$-merged intensities. Three tests weigh a promotion to higher symmetry, all aimed at the merohedral twin, whose twin law forces non-equivalent reflections together and so mimics symmetry: + +1. **Merge self-consistency** ($\chi^2$ under the candidate group, relative to the confirmed subgroup). On its own this is not sufficient: it is a ratio to an error model that moves with the *amount* of data — the parent's systematic term grows as $\sigma$ shrinks with $1/\sqrt{N}$, while a twin's is already saturated — so its verdict depends on how much data the search saw. +2. **Error-model $b$** (the intensity-proportional systematic). A genuine symmetry step gains multiplicity without inflating $b$; merging a twin law's extra operator inflates it. This is a **rescue and never a veto**: a $\chi^2$-borderline promotion whose $b$ stays close to the confirmed subgroup's is confirmed on it, which is what recovers a genuine high-symmetry group on imperfectly-scaled data, but it cannot demote a promotion that passed. The veto it once carried was removed. It was guarded by "$H$ could not be computed", so it could only ever act where the one statistic that discriminates here is blind, and what it compared there was not a measurement but the error-model bisection's own ceiling — which made a space group depend on the compiler flags, the same source refusing a correct $P2_12_12_1$ in one build and not in the other. No bound repairs it: genuine promotions have since been measured up to 5.8 where the bound was calibrated at 1.62 against a twin at 2.1, so the two populations have swapped sides. The case it was meant to catch — a merohedral twin too sparse for $H$ — is answered below it by the added-operator $R$ gate, whose reference is the global best rather than the parent. +3. **Operator disagreement**, a sigma-free statistic $H=\mathrm{median}\,|I_1-I_2|/(I_1+I_2)$, formed as the ratio of the operators a promotion *adds* to the parent's own, measured on the same reflections. Normalising against the parent divides out the systematic floor that symmetry mates carry on real data, which varies by crystal and by operator; a median is used because a twin perturbs every pair whereas a badly-measured minority perturbs only the tail. Where a candidate has several parents of the same order, it is judged against the worst of them, since a rival subgroup can itself contain the twin laws. + +The correlation is on **resolution-normalised** intensity $E^2 = I/\langle I\rangle(\text{shell})$, normalised over exactly the reflections the correlation pairs. Both members of a symmetry pair lie at the same $|s|$, so on raw $I$ the resolution fall-off is variance shared perfectly between the two arms and appears as a positive correlation for *any* pairing at all: a shell-matched random pairing — the exact null for a metrically-allowed false operator — scores a median 0.31 across the rotation battery, and on one crystal 0.53 — above the bound the correlation is tested against. That floor varies more from crystal to crystal (spread 0.46) than the whole true/false gap is wide (0.38), so an absolute bound on the raw statistic is a different test on every crystal; and it moves with the search resolution cut, which is what made that cut a symmetry-deciding parameter. Normalised, the floor has a median of 0.015, never exceeds 0.06, and barely moves with the cut. + +Both the correlation stage and the absence tests need to know whether a reflection is **genuinely present**, and that question is asked of its *counting* significance, not of the merged $I/\sigma$. A merged $\sigma$ carries the error model's intensity-proportional term, $\sigma^2 = a\,\sigma_0^2 + (b\,I)^2$, so merged $I/\sigma$ saturates — at $\mathrm{ISa}\sqrt{n}$ for a reflection observed $n$ times, and at $\mathrm{ISa}$ exactly for one observed once. Above that knee it stops rising with the intensity: on the weakest search merge of the rotation battery the $I/\sigma$ of every decile of $E^2$ reads 1.58–1.65 against an $\mathrm{ISa}$ of 1.70, so reflections an order of magnitude apart in real intensity report the same number. A single constant applied there demands anywhere between 2.2 and 11.9 in counting significance depending on the crystal. + +The nominal cut $T$ (default 3.0) is therefore converted once, using the ISa of the merge being searched. For a reflection observed once $\sigma_\text{counting}^2 = \sigma^2 - (b\,I)^2$, so $I/\sigma_\text{counting} \ge T$ is exactly + +$$\frac{I}{\sigma} \;\ge\; \frac{T}{\sqrt{1 + (T/\mathrm{ISa})^2}}$$ + +The converted cut lies strictly below $\mathrm{ISa}$ for every $\mathrm{ISa}$, so it is always reachable by the reflection the ceiling binds hardest, and it is within 1 % of $T$ on any merge with $\mathrm{ISa} \ge 21$ — a healthy merge is left exactly where it was. Multiplicity is taken as 1 deliberately rather than estimated, for the same reason. Where the merge reports no ISa (or $b = 0$) the cut is used as it stands. + +The candidates are enumerated in **every setting the refined cell can host**, not only in the settings the International Tables call the reference one. A setting is a statement about direction — $P112_1$ puts its $2_1$ on $c$ where $P12_11$ puts it on $b$, and both are space group 4 — so a search restricted to reference settings can only ever put a screw or a centring on the axis the convention chose, whatever the data say. Two things bound the widened set. The cell's own metric is asked once, when a point group's rotation set is chosen — a set whose rotations it does not host ($\alpha=\beta=90^\circ$ for a $2$-fold on $c$) is not offered — and every setting of a chosen set is then offered without asking again, because a cell refined free after integration sits a few tenths of a degree off $90^\circ$ and asking per setting kept only the reference one; and a candidate predicting exactly the absences another candidate already predicts is dropped as the same hypothesis under a second name. A non-reference setting is additionally refused when its centring class holds no reflection in this merge, since there is then nothing to confirm it with. Because the candidates of a point group all share its rotations, this cannot raise the symmetry: it decides *which axes* carry the screws and the centring, never how many operators there are. The setting found this way is the one the *decisions* are made in. The files are then written in the ITA standard setting (as XDS and POINTLESS write it: $P2_12_12$ with the pure axis on $c$ and $a, <|E²-1|> and N(0.1), each calibrated against acentric and centric intensities simulated with the reflections' own sigmas, every reading lies in the range a Wilson population can produce, the reflections centric in both groups read centric, and the lattice excludes twinning (the metric admits no rotation beyond the Laue class and no twin-domain lattice was found), because a twinned centrosymmetric crystal reads exactly like an untwinned acentric one. Between two groups without a centre, the lowest-numbered is written. + +The Lorentz factor $\zeta$ (§8.3) governs how well a reflection can be measured, so when the spindle lies in a plane of the lattice, an operator permuting the two in-plane axes samples a different mixture of measurement qualities than one that only flips signs. The search is therefore run a second time on a merge of only the well-measured observations (`--search-min-zeta`, rotation default 0.85), both answers are reported, and **where they disagree the merge of all the observations decides**. The filter discards 40–80 % of the observations, which can starve an operator correlation the full merge confirms and can equally leave an operator confirmed that the full merge refuses, so the decision — the point group as well as the absences, which live in the weak reflections the filter removes — rests on the arm with every observation behind it. A tie (same order, different symmetry) is reported with both candidates named, for trying in molecular replacement. + +Both search merges are cut in resolution where the data stop carrying signal: the shell means of $\langle I/\sigma\rangle$ are given their best non-increasing fit (pool-adjacent-violators, weighted by shell size) and the cut is where the **fit** drops under 1. Cutting at the first single shell to dip under 1 made the cut a coin toss on a merge whose $\langle I/\sigma\rangle$ is flat near 1: one crystal was cut at 0.85 Å or 1.19 Å depending on the compiler flags, and the coarser cut left its glide zones too few absences to be judged. The fit moves only as much as its input does, and on a profile that already falls monotonically the cut is unchanged. + +Both search merges also drop the **frames whose fitted per-frame scale came out below a tenth of the run median** — the stretches where the crystal was barely in the beam. The scale enters as $1/G$, so such a frame's intensities arrive amplified tenfold or more, with their $\sigma$ amplified by the identical factor and the scale's own error nowhere in it; the production merge survives that because a reflection in the determined symmetry is measured ten or twenty times and `--reject-outliers` removes the amplified observation, but the search merges in $P1$, where a reflection has two or three observations and no majority exists to call any of them an outlier. On a crystal that repeatedly left the beam over a full turn this cost the $422$ point group outright, its operators reading $0.08$–$0.38$ on a merge that gives $\mathrm{CC}_{1/2} = 96\,\%$ in that same point group once it is assumed. Like the $\zeta$ filter, this is a filter of the search passes alone — the production merge keeps every frame. + +**Screw axes** are scored **per axial zone**, not pooled over them. The conditions on $h00$, $0k0$ and $00l$ are independent, so their log-likelihoods add, and a zone that was never measured is allowed to **abstain** rather than to argue: pooling let one unmeasured row veto a confirmed one, and it read three genuine screws as weaker than two whenever the third row was shallow. A screw is claimed when its absence evidence reaches **8 nats**. The bound is set on the zones themselves, every single-axis candidate of a battery of rotation datasets labelled against the deposited group: a false zone with no violation reads at most +4 nats, a true one from +10, and the true zones below the former bound of 20 were all short rows — a $2_1$ along a monoclinic axis of 25–30 Å reaches four to six odd reflections inside the search's resolution range, which on weak data sit at a few per cent of their row rather than at zero. Two absences clear the bound if both read below about 2 % of their row. + +**A pseudo-translation is divided out before an axial absence is judged.** A translational pseudo-symmetry (§13.2) splits the reflections into two classes by the parity of the index along the translation, one systematically strong and the other systematically weak. Where the translation is half-integer along an axis, those two classes are exactly the absent class and the control class of a screw on that same axis, so the absence test would divide one by the other and pay the suppression twice — reading a class that is present but suppressed as extinct, and buying a screw the crystal does not have. The modulation is therefore measured along each axis from the intensities themselves — per axis, not pooled, since one row can be strongly modulated while its neighbours are not — and divided out of both the absence evidence and the violation count before the screw is scored. Only order-two screws are treated this way; a three-fold with a one-third translation is left unanswered rather than answered no. + +**The violation count can be deferred to the absences, one zone at a time.** A screw candidate is refused when too many of its predicted-absent reflections read as present — but on a strong axial row near the spindle those readings need not be structure factors at all: the same reflection can read far positive at one Ewald crossing and negative at the other, the directional smear tail of the strong reflections beside it. A zone whose per-reflection absence evidence clears the claim bar may therefore stand despite its count, since the likelihood has already priced those reflections in and still reads the class as extinct. The deferral is licensed **zone by zone** — the evidence is a group-level number while the count indicts particular zones, so an overwhelming genuine zone must not pay another zone's debts — and it is not available on a zone corrected for a measured pseudo-translation, where the corrected count is the one instrument the modulation does not reach. + +**Centering** is accepted when the systematically-absent class is weak relative to the present one by *either* of two floor-independent tests: its mean signed $I/\sigma$ well below the present mean, *or* its rate of individually-significant reflections well below the present class's own significant rate. The second test covers weak and low-energy data, where a positive intensity floor (background and profile leakage) lifts the absent class's mean $I/\sigma$ well above zero and, when the present class is itself weak, carries the plain mean ratio past its bound; a false centering fails both tests, its absent class being as strong as the present one. When several centerings pass they are ranked by a **Beta-tail likelihood** rather than by a count of net absences: a count lets a centering that extinguishes many more reflections out-rank one that violates far fewer, and the likelihood weighs the violations against the class each belongs to instead. The acceptance bounds above are unchanged by that ranking. + +**A confirmed promotion that is refused is decided by the twin-immune zone of its added operators, and only where that is undecided by merging under it.** The zone (§13.2) is the one reading that does not depend on the per-frame scales or on a reference operator: read on the all-observation P1 merge where the two search arms disagree, and on the adopted group's own merge at the remerge, an index-2 promotion whose calibrated zone evidence is acentric by 20 nats stays refused — on all observations, and the first added operator is recorded as the twin law — and one whose zone is centric by 20 nats is taken. Where the zone cannot decide (a higher index, no zone reflections in the shells with signal, within the margin), the merge is asked. Every gate that refuses a *confirmed* higher point group is a **ratio** — the added operators' disagreement against the parent's, or their $R$ against the best-agreeing operator anywhere — and both references are properties of how the crystal was mounted rather than of its symmetry. A 2-fold within a degree of the spindle records its mates on the same detector pixel half a turn later, so it carries no geometry-dependent systematic at all, and by being that clean it makes every other operator look bad against it; the best-agreeing operator can also be one of the operators under test, which collapses the ratio on worse data and raises it on better. So where a promotion is confirmed and then refused, the **merge** is asked instead of the ratio. Both groups are scaled and merged over one pinned resolution range — the adopted group's own cut — and the promotion is taken only if **neither $R_\mathrm{meas}$ nor ISa gets worse**. $R_\mathrm{meas}$ is multiplicity-corrected, so folding non-equivalent reflections together has to inflate it, and the error model is refitted per merge, so ISa says whether the extra multiplicity was bought with systematic disagreement. $CC_{1/2}$ and $\langle I/\sigma\rangle$ cannot arbitrate this: both *rise* on a false promotion too. Two details the comparison turns on: the refused point group's representative is primitive and symmorphic, so the arm merged under it takes the **adopted group's centring** — merged in the bare representative, a centred crystal is handed its centring-absent class as data, which inverts the decision — and the absence stage is asked of the adopted merge, so both arms carry the same absence classes and differ by the added rotations alone. That merge no longer holds the adopted group's screw-absent axial reflections, so the higher group also **keeps the screws the adopted group decided** — among the higher candidates the merge cannot separate, the one that predicts every such axial absence is taken — rather than the lowest-numbered one, which claims none. The cost is two extra merges, and only on a run that records a refusal, which is a few in a hundred. + +**On a merge that reads twinned, the search asks the zone itself.** A twin of an index-2 subgroup by exactly the operators a promotion adds defeats every agreement gate once its fraction is high — the added operators then agree like real ones — so where the P1 merge's $\langle|L|\rangle$ lies in the partial-twin band (0.375 to 0.44) and the candidate does not hold every rotation of its lattice (so a twin law outside it is possible), the candidate is refused when the twin-immune zone of one of its index-2 subgroups reads acentric by 20 nats. Below 0.375 something other than a twin compresses the intensities, the zones with them, and they are not read. A $P3_1$ crystal twinned at $\alpha \ge 0.4$ by its 321 law, whose 32 promotion every agreement gate passed, reads −230 nats there. + +**A point group's own operators must agree with each other.** A point group is the claim that all of its operators are symmetries of one crystal, and a mixture of real and false operators has a specific shape: the real ones form a subgroup — the true group's intersection with the candidate — and everything outside it is false. The gate therefore reads the operators sorted by their $R$ against the merge, finds every prefix that closes into a subgroup, and takes the largest factor by which every operator outside such a subgroup reads worse than every operator inside it; above 3.2 the candidate is refused. A genuine group whose operators merely spread over a continuum — anisotropic systematic error spreads a cubic crystal's operators twofold — has no such gap. The earlier form, worst-agreeing over best-agreeing operator, divided by an extreme order statistic and refused a genuine $432$ as soon as one of its operators happened to agree unusually well. + +**A group that holds every rotation of its lattice is refused when merging under it narrows the L-test like a twin.** Every gate above compares the added operators with a reference, and a pseudo-symmetric structure or a twin defeats them all. For a candidate that holds every rotation its lattice admits no twin law exists, so intensities merged under it that read like a twin's are not a twin. The Padilla–Yeates $L$-test is read twice on the same pairs of the $P1$ merge — as measured, and with every intensity replaced by the mean of its orbit under the candidate — and the candidate is refused where the merged $\langle|L| angle$ reads twin-like ($0.375 \le \langle|L| angle < 0.44$) and the averaging closed a third or more of the gap from the unmerged value to 0.375. Genuine lattice-holohedral groups close 0.01–0.25 of it. A merge reading below 0.375 is not read: nothing a twin or a false operator does reaches that, so something else compresses the intensities. + +**The lattice class is re-asked where the metric knows more.** The Bravais class comes from Niggli reduction and a lattice-character lookup (§6), and that lookup can land short of the truth or beside it: near the Niggli type-I/type-II boundary it is decided by the last digits of the refined cell, and a lattice that is nearly but not exactly hexagonal matches the hexagonal character even when no point group that class can host carries the two-folds the data actually have. Two recoveries run, both settled by the intensities. Every rotation the cell metric can host beyond the named class — read off Le Page's two-fold search on the lattice itself, which measures each rotation's obliquity in a primitive basis and owes nothing to the character table — is put to the intensities as a single operator, scored exactly as the search scores its own operators, on the same reflection population and the same $E^2$ normalisation. And where the metric group is larger than the adopted class's holohedry, the merge is reindexed into the metric group's conventional cell and the space-group search is run again there, with every gate live; the reindex is committed only where the search in the new setting confirms a strictly higher point group *and* the centring the new cell describes, so a pseudo-symmetric metric leaves the answer already in hand standing. + +### 13.2 Twinning check, and translational pseudo-symmetry + +A Padilla–Yeates $L$-test ($\langle|L|\rangle$, $\langle L^2\rangle$ — 0.500 and 0.333 untwinned, 0.375 and 0.200 for a perfect twin) and the second moment $\langle I^2\rangle/\langle I\rangle^2$ (2.0 for untwinned acentric data, 1.5 for a perfect twin) are written to the merged mmCIF as a twinning diagnostic. Both are taken on intensities divided by their resolution-shell mean, over the shells whose $\langle I/\sigma\rangle$ reaches 1, with Wilson outliers rejected; the $L$-test pairs are therefore compared on one scale even where two index steps span a steep fall-off, as in phenix.xtriage and ctruncate, and the selection is by shell, never by the individual reflection's $I/\sigma$, which would cut the weak tail and bias $\langle|L|\rangle$ down. The twin fraction is quoted from the statistic that carries the verdict — the $L$-test unless the call rests on the second moment alone — and a second moment is not turned into a fraction under a detected pseudo-translation, which inflates it. A merohedral twin law exists only where the Laue class is a proper subgroup of the lattice holohedry, so in the high-symmetry holohedral classes ($4/mmm$, $6/mmm$, $m\bar{3}m$, and $\bar{3}m$ on a rhombohedral lattice) no twin is called. The $L$-test is still read there, for a different question: merging $I(h)$ with $I(Th)$ under a false operator $T$ gives $(I(h)+I(Th))/2$ whatever the twin fraction, which has the perfect-twin distribution, so $\langle|L|\rangle$ below 0.42 in a holohedral class is reported as **an adopted operator averaging unequal intensities** — the space group is too high, or a twin law was absorbed into the point group — a warning, never a change to the space group. A genuine operator leaves the untwinned 0.5. Reflections that overlap along a very long axis narrow the distribution the same way, which is one reason it stays a warning. The same numbers measured on the P1 merge the space-group search was given, before any point group was adopted, are reported beside them (`_BEFORE_SEARCH`), with that merge's own pseudo-translation declared and the reflections its lattice centring extinguishes left out. The low-symmetry holohedral classes ($\bar{1}$, $2/m$, $mmm$) also admit no strictly merohedral law, but they stay eligible for the twin call on purpose: *pseudo*-merohedral twinning through an accidentally special metric cannot be ruled out from the symmetry alone, and those are the classes it happens in. + +Beside them, on rotation data, the run reports **twin-immune zone evidence** for the operators the adopted point group adds over each of its index-2 subgroups, read on the P1 cross-check merge. Under a twin law $T$, $I_\mathrm{obs}(h)=(1-\alpha)I(h)+\alpha I(Th)$; a reflection whose twin mate is itself up to the subgroup and Friedel is untouched at every $\alpha$, and those are exactly the reflections centric in the group but acentric in the subgroup. They read centric ($\langle|E^2-1|\rangle = 0.968$) if the added operators are real and acentric (0.736) if they are a twin law or a pseudo-symmetry — the one intensity statistic that still separates the two at $\alpha = 0.5$, where every operator statistic reads "real". Each zone is normalised against its own mean in resolution bins (and within the two phase classes of a detected pseudo-translation), read only in shells with $\langle I/\sigma\rangle \ge 5$ because noise inflates every class towards centric, and reported with $n$, a standard error and the centric-over-acentric Wilson log-likelihood ratio in nats (both densities convolved with each reflection's measurement error), beside the acentric control. It is read absolutely, never as the difference to the control: a perfect twin's control (0.541) plus noise inflation would otherwise look like true symmetry. An absolute reading is only as good as the normalisation, though, and the acentric control certifies it: an acentric population reads $-0.130$ nats per reflection when the normalisation is right, and whatever the control reads above that is the normalisation's — anisotropy, a pseudo-translation, a pseudo-centring, noise all inflate every class towards centric alike — so the zone, normalised the same way, carries the same per reflection and the **calibrated** evidence has it taken off (a twinned control reads *below* the expectation, so nothing is taken off a twin). The centric side is read **twinned at the fraction the lattice's other operations show** by their own correlation — the strongest CC of a lattice rotation outside the group, relative to the mean CC of the group's own operators, inverted through $\rho = 2\alpha(1-\alpha)/((1-\alpha)^2+\alpha^2)$ — because a twin by another law reaches a genuine zone exactly as it reaches the control: a genuine 321 crystal twinned by a 622 law read its 2-folds at −206 nats against the untwinned centric density and +209 at that law's fraction of 0.07. The operators' own twin law cannot reach their zone, so the acentric side stays untwinned, and the control's expectation is taken at the same fraction. Measured, a 6/m crystal with a 67 Ų anisotropy read its control at $+0.10$ nats per reflection and the zone of a refused 622 the same, so the zone's $+124$ nats were the normalisation's; calibrated it reads $-115$, and genuine promotions keep $+200$ and above. The calibrated zone is what a refused promotion is decided on (above). It is an in-house method: published practice (Yeates' $H$-test, xtriage) excludes these reflections as uninformative about the twin fraction, and no published test was found that uses them positively; the one precedent here is a trigonal crystal whose twin-immune zone read centric and whose higher group was then confirmed by refinement. + +**Translational pseudo-symmetry** — two copies of the contents of the asymmetric unit related by a pure translation that is not a lattice vector — is looked for on every merging run, because it is the classic predictor of a failed molecular replacement, and because it raises the second moment where twinning lowers it, so each can mask the other's test. The detection is the native-Patterson route of Read, Adams & McCoy (see the [references](CPU_DATA_ANALYSIS.md#references)): the largest off-origin peak of a Patterson computed from the merged intensities, as a fraction of the origin peak, with the peak vector then refined against the intensity modulation it should produce — the ratio of the strongest to the weakest bin mean of $\langle E^2\rangle$ over the phase $\mathrm{frac}(\mathbf{h}\cdot\mathbf{u})$. Both halves are scored against a null computed for the crystal at hand rather than against a fixed bound — both against the same intensities **permuted within resolution shells** — the peak against that map, the modulation against the same measurement made on the shuffled intensities and started where the measurement starts — because the noise floor of the peak statistic spans an order of magnitude across data sets, so no fixed percentage means the same thing twice. The shuffle leaves the reflection count, the $E^2$ distribution and the phase-bin populations untouched and removes only the correlation between a reflection's intensity and $\mathbf{h}\cdot\mathbf{u}$, which is the thing the modulation claims to see. Re-running the greedy search from random starting vectors over the *same* intensities does not work as a control: a greedy search started anywhere walks into a real modulation's basin, so it measures the search **and** whatever modulation the crystal carries — measured over 110 merged datasets, single draws of that control reached 199× and 6162× on crystals whose own modulation reads 4.0× and 67×, and on 28 of the 110 it came out at or above the signal it is subtracted from. Requiring both halves is what keeps the false-positive rate down; either alone over-calls by about a factor of two. A translation the merged data are **exactly** invariant under is reported as an undeclared lattice translation instead — a translation the data are exactly invariant under is a lattice vector by definition, so the centring or the cell is wrong, not the packing — and the pseudo-symmetry search continues underneath it, so a real pseudo-translation sitting under an undeclared centring is still found. The finding itself is report-only (the `TNCS_*` keys of `_report.txt`): it gates nothing and changes no reflection, no scale and no group — but the axial-absence test (§13.1) and the $L$-test here both correct themselves against the modulation it measures. + +**The $L$-test partners are chosen so a pseudo-translation cannot silence it.** The test compares each reflection with a partner a fixed index step away, and a pseudo-translation biases $\langle|L|\rangle$ upwards unless the partner shares its modulation class. The ordinary step of 2 preserves the class of a half-integer translation — the pseudo-centering the statistic's authors call it robust to — but not of one of a third, which inflates $\langle|L|\rangle$ past the bound that is read as evidence *against* twinning and silently loses the twin call on a crystal that has one. Where a pseudo-translation is detected, the partner steps are restricted to those that preserve its class; where no step does, the statistic is dropped from the twin verdict **in both directions** — it can no longer indicate a twin and can no longer be read as proof that there is none — and the second moment decides alone. `L_TEST_VS_TNCS=` in the report says which of the three happened. + +### 13.3 Outlier rejection + +Merging applies an optional per-observation median-based $N\sigma$ cut (`--reject-outliers`, default 6σ for `rot3d`, off otherwise). On the rotation path the median is weighted by each full's counting variance taken at the reflection's **expected** intensity, as the merge weights are (§10.4): an observation's own Poisson variance falls with its own intensity, so own-variance weights hand the median to whichever equivalents came out low — on a strongly absorbing crystal, where some frames read equivalents ten to a hundred times down, the median sat on the absorbed ones and the test removed correct measurements far above it. The band about the median keeps the counting part linear, $\pm N\sigma_\text{counting}$, but takes the systematic part as a **factor**, $e^{\pm N s}$ about the expected intensity: the error model's systematic term is multiplicative — absorption, illuminated volume, a scale that is off — and as likely to be a factor $1/f$ as $f$, while a band symmetric in $I$ reaches far below the median and only a little above it once that term is large. $s$ is the spread of $\ln(I/\text{median})$ measured on the merge's own fulls (their weighted median of $|\ln(I/\text{median})|$, weighted by $(\langle I\rangle/\sigma_\text{counting})^2$ so the fulls whose ratio counting noise does not blur carry it), with the error model's $b$ as the fallback where it cannot be measured; to first order in $Ns$ the band is the linear one. The same $N\sigma$ cut is fed back into the error model: after an initial $a,b$ fit the parameters are re-fit once on the reflections that survive rejection (dropping any whose squared deviation exceeds $N^2\,[a\,\sigma^2 + (b\,\langle I\rangle)^2]$), so the calibrated errors describe the reflections that actually enter the merge rather than the pre-rejection pool. + +The median needs three observations, so a reflection measured once or twice — typically one good observation and one artefact (a hot pixel, a zinger) after Friedel merging — is never tested by it, and an artefact that looks precise (thousands of counts, a small relative $\sigma$) can outweigh mates from weak frames and become the median itself. Every observation of the written merge is therefore also judged by **Wilson statistics**. Beyond 4 Å its $E^2 = I/(\varepsilon\,\langle I/\varepsilon\rangle_\mathrm{shell})$ is compared with a bound set by a budget of $\alpha = 0.01$ expected false rejections per dataset: with $N$ observations tested, $\ln(2N/\alpha)$ for acentrics and twice that for centrics, and the observation's lower confidence limit $I - z\sigma$ (with its own $\sigma$, and $z$ from the same budget) must exceed it, so noise in weak shells does not trigger it; a shell whose $\langle I\rangle$ is not itself established at that significance is not judged. The bound is widened by the tail scale of the dataset's own intensity distribution, measured on its well-measured observations (1 for a Wilson crystal; larger under a pseudo-translation or anisotropy). An improbable singleton is dropped; an improbable observation with company only when most of the reflection's other observations are probable and it disagrees with their mean beyond the errors — several large observations confirm each other. An observation whose signal disk lost pixels to the mask or to saturation is never counted as such a witness, since its profile estimate may be low: a strong reflection is not replaced by its clipped mate. The count is `OBSERVATIONS_REJECTED_WILSON=` in the report (included in `OBSERVATIONS_REJECTED=`), and the developer report lists each observation with its image and detector position. + +### 13.4 Automatic resolution cutoff + +By default the reported/written high-resolution limit is trimmed where $\mathrm{CC}_{1/2}$ falls off: a logistic is fitted to $\mathrm{CC}_{1/2}(s)$, and the limit is set **one reported-shell width past** the point where the fit crosses 0.30 — deliberately "one shell too far", so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. `--scaling-high-resolution` overrides the limit and `--resolution-cutoff off` disables it. + +**Ice sits out of this fit, and only this fit.** A powder ring is reproducible: past the crystal's own limit the ice is still there and still the same in both half-sets, so the two halves agree *about the ring*, and a Pearson $\mathrm{CC}_{1/2}$ cannot tell that agreement from diffraction. Shells have been measured at $\mathrm{CC}_{1/2}=0.765$ where $\langle I/\sigma\rangle$ is $-0.1$ and $R_\mathrm{meas}$ is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, still written and still counted in the shell table (§10.10), and what changes is only that the resolution **decision** now reads the same curve that scaling, the error model and the space-group search already read. + +**A cut the shell table refutes is not quoted.** Two things keep the number inside what the bins show. The fitted crossing has to lie **inside the fitted bins**: on a real fall-off it sits between the last bin at the target and the first bin below it, with the extension bins to spare, and a crossing past the last fitted bin is an extrapolation of a fall-off these data never showed — the crossing is then read off the bins themselves, between the two that straddle the target. And no single resolution is quoted at all where the curve is not a fall-off: where $\mathrm{CC}_{1/2}$ never reaches the target in any shell, where the fit reads finer than the finest shell whose $\mathrm{CC}_{1/2}$ still reaches it, or where $\mathrm{CC}_{1/2}$ **climbs back over the target** after falling below it — a noise shell that climbs back is not the edge of the data. The report then says which of the three happened and points at the shell table instead. + +### 13.5 Diffraction anisotropy + +How fast the intensity falls off with resolution can depend on direction. Rugnux measures that and reports it. No intensity is corrected and no reflection is removed on a directional criterion; the one use of the tensor is the Wilson prior of the French–Wilson amplitudes (§10.8), so `F`/`SIGF` follow the fall-off along each reflection's direction while the intensities do not depend on direction at all. + +**The tensor.** A deviatoric anisotropic displacement tensor is fitted to the merged intensities as + +$$\ln \langle I(\mathbf{s})\rangle = c(\text{shell}) - \tfrac{1}{2}\,\mathbf{s}^\mathsf{T} B\, \mathbf{s},\qquad \mathbf{s} = \text{reciprocal-space vector},\ |\mathbf{s}| = 1/d$$ + +(§10.6 and §14.2 write $s$ for $\sin\theta/\lambda = 1/2d$, so their $s^2 = 1/4d^2$; the two conventions give the same exponent $-(B/2)(1/d^2)$, and $B$ is the same $B$.) + +with one free constant per resolution shell, so every isotropic feature — the Wilson curve, an ice ring, a noise floor, a scaling error — is absorbed exactly and only the $\ell = 2$ angular part drives the tensor. For an isotropic $B$ this reduces to the ordinary Wilson plot, so $B$ here is the ordinary crystallographic ($B = 8\pi^2 U$) $B$, directly comparable with phenix.xtriage's `B_cart`, ctruncate's anisotropic $B$ eigenvalues and AIMLESS's anisotropic $\Delta B$. Only the deviatoric part is fitted: the isotropic part is degenerate with the overall scale. The tensor is constrained to the directions the Laue class allows — five free deviatoric parameters in triclinic, three in monoclinic, two in orthorhombic, one in tetragonal, trigonal and hexagonal, and **none at all in cubic**, where symmetry forces $\Delta B$ to be exactly zero. + +The fit is on **intensities, with no positivity cut**. Fitting amplitudes, or dropping non-positive intensities as an amplitude-based tool must, loses roughly 40% of the signal: in a direction that has died half the merged intensities are negative, so a positivity cut keeps only the positive noise excursions and flattens the fall-off exactly where the anisotropy is largest. + +**Two different quantities are reported, and they are not interchangeable.** $\Delta B$ (the range of the principal components) is a *rate*; the diffraction limit along each principal direction — where $\langle I/\sigma(I)\rangle$ in a 20° cone about that direction falls through 2, read by interpolation in $s^2$ over equal-count shells — is where the signal actually runs out. A crystal can have a large $\Delta B$ and almost no spread in directional limit, or the reverse. Where $\langle I/\sigma(I)\rangle$ never falls through 2 in a direction, the limit returned is the **edge of the measured data** rather than the crystal's own; such a direction is marked — with a `<` in the report, a 1 in `ANISOTROPY_D_MIN_CENSORED`, and a note in the mmCIF — so the spread is not read as a measurement when it is a lower bound. Both constants are AIMLESS's defaults (cone half-angle 20°, $\langle I/\sigma\rangle = 2$). The 2 is a per-direction diagnostic level only: the dataset-wide resolution cut (§13.4) is CC$_{1/2}$-based, and the two criteria are not interchangeable. + +**The resolution signature.** A genuine Debye–Waller $B$ makes the directional deficit a straight line through the origin in $s^2$. The per-shell $\ell = 2$ amplitude is therefore fitted against $s^2$ and the curve is classified: *linear* (a real $B$), *flat* (a deficit that does not follow $\exp(-\tfrac12 \mathbf{s}^\mathsf{T} B \mathbf{s})$ at all, so the fitted $\Delta B$ describes the data with the wrong functional form and may be an **under**-estimate), or *convex* (a deficit that grows faster than $s^2$, which a $B$ cannot do). The verdict is re-derived at 8 and at 16 shells, and reported as undetermined if it moves. + +**The verdict, and what it is measured against.** Whether an anisotropy is real is not decided against a counting-statistics error bar. Real data carry systematic error far larger than counting error, and gating on the latter reports anisotropy on datasets that have none. Instead the data set measures its own systematic error: in the tensor directions the Laue class *forbids*, the true tensor is exactly zero whatever the crystal is, so whatever is measured there is systematic. That measurement needs the unmerged observations — a merge has exact Laue symmetry by construction, and the forbidden directions are identically zero in it — so it is made on the scaled, rocking-curve-assembled observations. The counting part is subtracted, the counting error of the directions actually being tested is added back, and the result is the **floor**. What is tested against it is $\Delta B_\text{linear}$ — the part of the fall-off that actually follows $\exp(-\tfrac12\mathbf{s}^\mathsf{T}B\mathbf{s})$, clamped at zero — and **not** the headline $\Delta B$; the report names which of the two it is quoting. The ratio is banded: below 2 not established, 2–3.5 marginal, above 3.5 established, above 5 strong. The bands are calibrated against known ground truth — merging cubic crystals in proper subgroups of their own Laue class, where the true anisotropy is exactly zero — which puts the false-positive rate at 24 % at 2.0, 10 % at 3.5 and 5 % at 5.0. + +The report says **NOT DETECTED**, **DETECTED**, or **CANNOT DETERMINE**, and the third is a real answer rather than an evasion. It is returned when the Laue class is triclinic (no forbidden direction exists, so there is no internal measurement of the systematic error and no substitute for it), when the observed rotation range is under about 90° (a lab-fixed systematic then reaches several tensor directions instead of one), when the merged data are at the noise floor, when the scale model carried no dose term (an uncorrected dose ramp manufactures anisotropy that no significance test can see through), or when no unmerged observations were available. The smallest $\Delta B$ that could have been established on the data set is reported with the verdict; it is set by the systematic error rather than by counting, so it does **not** improve with more reflections or a longer exposure. + +A too-high space-group assignment is the one failure mode that is silent: real anisotropy is then pushed into the directions used to measure the systematic error, which inflates the floor and biases the answer towards reporting none. A caution says so, but only where that is actually indicated — a single free direction, no detection, **and** a forbidden-direction measurement far above its own counting noise — rather than on every tetragonal, trigonal and hexagonal data set. + +Everything lands in `_report.txt` section 4 (`ANISOTROPY_*` keys), in the printed statistics, and in the merged mmCIF: the eigen-decomposition of the tensor as the standard `_reflns.pdbx_aniso_B_tensor_*` items (relative to the weakest direction, since only the deviatoric part is determined), and the directional limits, the shape and the verdict under the `_reflns.jfjoch_aniso_*` local prefix. + +### 13.6 Practical notes and limitations + +- **Bragg integration is profile-fitted by default** (per-shell Gaussian profile, Kabsch extraction; §9.3), with plain box summation available as a fallback (`--integrator boxsum`). The profiles are built per frame from that frame's strong spots, which suits fast-feedback and serial/streaming use; a profile shared across many frames (as in full offline workflows) is not currently formed. +- **Space-group symmetry** beyond centering absences is not enforced during prediction/integration unless the space group is supplied and used downstream. +- **Resolution masking** is controllable, and so is every stage of ice-ring handling (§3.3, §10.10). None of it runs unless the crystal is measured to have ice, because the fixed bands are a fixed cost in unique reflections whether it does or not. +- **Rotation vs still modes** differ substantially in prediction and scaling: partiality is angle-driven in rotation data, while stills are predicted within an excitation-error window and get their partiality from the default-on per-crystal tilt post-refinement (§10.2) — or unit partiality with `--simple-stills`. +- **Amplitudes and intensities.** The merged output carries both intensities (mmCIF `intensity_meas`, MTZ `IMEAN`/`SIGIMEAN`) and French–Wilson amplitudes (mmCIF `F_meas_au`, MTZ `F`/`SIGF`; §10.8), so a downstream program can refine against either. + +--- + +## 14. Model-based validation: R-free against a model and electron-density maps + +Offline (`rugnux --model model.pdb`) the merged data can be scored against a supplied atomic model and electron-density maps computed — enough to confirm that a model fits the data and to inspect the density, not a substitute for refinement. **The structure itself is not refined**; the model is re-fractionalized into the data unit cell and then **placed as one rigid body** (§14.8), and the observed amplitudes are the French–Wilson $|F|$ from §10.8, so the R-free and the maps use exactly the same amplitudes as the written reflection file. The model, structure-factor, bulk-solvent and FFT machinery is provided by GEMMI. + +### 14.1 Model structure factors + +The model electron density is sampled on a grid (IT92 X-ray form factors, with a Refmac-compatible Gaussian blur chosen for the grid spacing) and Fourier-transformed to structure factors $F_\mathrm{calc}(hkl)$ up to the data resolution. + +### 14.2 Bulk solvent and scaling + +A flat bulk-solvent mask around the model is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$: + +$ +F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2. +$ + +This is the standard, few-parameter scaling model used by refinement programs. No free-form per-resolution-shell rescale is applied: such a rescale is dataset-specific and reshapes each map's radial amplitude profile differently, which would make maps from a multi-dataset campaign no longer directly comparable. + +That decision has a cost, and it is paid by the R-factors rather than by the maps. $k\exp(-\mathbf{s}^\mathsf{T} B \mathbf{s})$ can only bend one way with resolution, so whatever a dataset's radial amplitude profile does that this shape cannot follow is reported as R — which makes R a reading of *this* dataset and not a quantity comparable with another reduction of the same crystal. Both uses are wanted, so both are served, separately: the maps and `R_WORK` / `R_FREE` keep the scale above, and a second reading, `R_MODEL_SHELL_SCALED`, applies one free scale per resolution shell to the *same* fit and is reported beside them, with `MODEL_RADIAL_MISFIT` saying how much that rescale had to do. Nothing is written from the second reading; it never reaches a map, a map coefficient or a decision (§14.3, and RUGNUX_REPORT). + +The fit sees the **working reflections only**; the parameters it returns are then applied to every reflection, free ones included, so that R-free (§14.3) is computed against them. The parameters are few, but they are fitted by minimising the very sum R is made of, and a free reflection that helped choose them is no longer held out. + +### 14.3 R-work and R-free + +Crystallographic R-factors are reported over the work and free sets (the §10.7 flags): + +$ +R = \frac{\sum \big|\,|F_o| - |F_\mathrm{model}|\,\big|}{\sum |F_o|}, +$ + +with R-free the same sum restricted to the free set. Every parameter $F_\mathrm{model}$ carries — the scaling of §14.2 and the rigid-body placement of §14.8 alike — is fitted on the working set alone, so the free reflections are held out of the fit as well as out of the sum. Two choices are still *selected* with the free set rather than fitted to it, and both are single discrete decisions rather than continuous parameters: the rigid-body step is committed only if it lowers R-free (§14.8), and the alternative indexing of §14.6 is the candidate with the lowest R-free. Both are the conventional use of a test set to accept or reject a step; both leave R-free very slightly optimistic where they fire. + +### 14.4 Electron-density maps + +Two maps are formed with the model phases $\varphi_\mathrm{model}$, both $\sigma_A$ weighted: a $2mF_o-DF_c$ map and an $mF_o-DF_c$ difference map, each inverse-Fourier-transformed to a real-space CCP4 map (`_2fofc.ccp4`, `_fofc.ccp4`). A map-coefficient MTZ (`_maps.mtz`: `FP`, `FC`, `PHIC`, `FWT`/`PHWT`, `DELFWT`/`PHDELWT`, `FOM`, `FREE`) is written alongside so the maps can be reopened or rebuilt in Coot / PyMOL. + +$\sigma_A$ is estimated by maximum likelihood **per resolution shell**, on the **free reflections only** — on the working set the model has been fitted to the data, so $\sigma_A$ would come out too high and the weighting would understate exactly the model error the map is meant to reveal. The shells are cut by equal reflection count, and it is the *number of shells* that is chosen from the size of the free set (about 50 free reflections to a shell, at most 20 shells), so no shell is thin by construction; a shell that still ends up with fewer than 10 free reflections takes the estimate made over the whole free set instead. Acentric and centric reflections enter with their own likelihoods (Rice and Woolfson respectively), and the epsilon factor is divided out before normalising both amplitudes to $\langle|E|^2\rangle = 1$ within the shell. The figure of merit is then $m = I_1(X)/I_0(X)$ with $X = 2\sigma_A|E_o||E_c|/(1-\sigma_A^2)$ for an acentric reflection and $m = \tanh(X/2)$ for a centric one, and $D = \sigma_A\sqrt{\Sigma_o/\Sigma_c}$ carries $F_c$ onto the observed amplitudes' scale. The $2F_o-F_c$ combination becomes $2mF_o - DF_c$ for acentric reflections and $mF_o$ for centric ones — a centric reflection's phase is either exactly right or 180° wrong, never in between — while the difference coefficient is $mF_o - DF_c$ throughout. + +$m$ and $D$ are estimated on each dataset separately, so two datasets of one crystal form get slightly different weights and their maps are to that extent no longer scaled identically — the same property §14.2 deliberately protects by refusing a free-form per-shell rescale. The two are not the same thing: the weighting never rescales $F_o$ and never touches the R-factors of §14.3, and the difference between two datasets' $\sigma_A$ curves is the difference in how well the model explains each of them, which is what a screening campaign is looking for. A PanDDA-style analysis consumes $2mF_o-DF_c$ maps and normalises each dataset's map against the ensemble before comparing them. The per-reflection `FOM` is written to the MTZ so the weighting can be read off and undone. + +### 14.5 Does the model fit? The null it is scored against + +A model supplied with `--model` is a **hypothesis about the crystal**, not an instruction. It is +always scaled, always placed (§14.8) and always scored, and its R-factors and maps are always +reported — but before it is allowed to change anything about the reflections that are written, the +data have to accept it. There are only two such things (§14.6), and where the model claims neither the +question is never put: see the end of this section. + +**No fixed threshold on R can decide that**, and this was measured three ways. The classical acentric +random-structure value is $R = 0.586$ at unit scale, but the scale here is fitted to minimise the very +sum the R is made of, which pulls it to about 0.550; observed nulls on real data land at 0.599–0.615; +and the value moves with the *model* — its atom count and its B-factors — as much as with the data, so +a cut calibrated on one pair misjudges the next. In one measured arm an unrelated protein reached a +*lower* raw R-free than the correct model did on other data. + +The only null that fits both the model and the data is therefore **made out of them**: the same model +is re-oriented at random about its own centroid and run through the identical path — the same scaling, +the same rigid-body placement, the same R — nine times, and the real fit is asked how far above the +resulting distribution it sits (`MODEL_FIT_SIGMA`, accepted at 3σ). The replicates are placed as well +as fitted, or the comparison would be between a placed model and unplaced nulls and the margin would +be inflated by the placement rather than by the model. Measured on one rotation data set: the crystal's +own model +15.0σ, an unrelated protein +1.8σ, and the correct model rigidly rotated 90° −1.0σ. + +A replicate is a sample of the null only while it stays away from the model's own solution. A draw +within the rigid body's reach of an orientation equivalent to the model's — under the space group's +rotations or a twin law of the lattice — is drawn again before it is placed, and a replicate the +placement nevertheless carries to within that reach is drawn and placed again afterwards, each +replicate from a generator of its own so the result does not depend on the order the concurrent +replicates finish in. + +**R-work carries the decision, not R-free.** Nothing is refined against the working set here — the +scale has a handful of parameters (a scale, an anisotropic $B$ and two solvent terms) and the placement six — so R-work carries no optimism, and it has an order +of magnitude more reflections than R-free, and so that much more power to separate the two arms. R-free +is still reported, and is still what the placement of §14.8 is committed on, since that *is* a fit. + +What the verdict gates is exactly the two decisions of §14.6 that **rewrite the data**: the +space-group label and the indexing. It does not gate the R-factors, the maps or the rigid-body +placement, which are statements about the model and cannot corrupt a reflection. A rejected model +therefore leaves the written files byte for byte what a run with no model would have produced, and the +report says so beside the R it was rejected on: a negative result, not a failed run. + +**The null is only built where a decision is actually pending.** A model can change exactly two things, +and it does not always claim either: one already in the space group the data were merged in, on a +crystal with no merohedral ambiguity, asserts no enantiomorph and prefers the data's own indexing, so +there is nothing to arbitrate and nothing for a null to gate. That case reports `MODEL_FIT= NOT_TESTED` +and skips the replicates. It is not an edge case — it is the isomorphous fragment-screening run, the +one that has to deliver a map seconds after the last image — and it is why the ordering matters: the +indexing probe is a handful of scaling fits and runs first anyway, so whether the expensive part is +worth paying for is known before it starts. + +The count is nine, and it is the *spread* that sets it rather than the mean: the verdict is +(mean − real)/sd of the sample, the relative error on an sd from $n$ draws is $1/\sqrt{2(n-1)}$, and a +sample that happens to come out narrow is what turns a model that does not fit into one that appears +to. Measured on the case nearest the gate — an unrelated protein at 1.83σ against a threshold of 3 — +the chance of reading over the gate on a different seed is 31 % at $n=3$, 17 % at $n=5$ and 6 % at +$n=9$. Nine is affordable only because the replicates run at once: the null costs the slowest of them +rather than their sum, and the slowest of nine is barely above the slowest of five. + +The null costs one full fit and one rigid-body placement per replicate. The replicates share nothing +— each is the same model under a different rotation, scored the same way — so each takes a copy of +the model and of its structure factors and they run **concurrently**, on the run's own thread budget +(`-N`); the real model is not touched by any of them. Measured on an idle machine, five replicates +cost 2.5 s together where running them one after another costs 10 s, on a `--mode scale` run that +merges in 2 s. What remains is the cost of the *slowest* replicate: how many placement evaluations an +orientation needs varies by nearly half between them, so the concurrency saturates around 4×, and +more threads than replicates buy nothing. The orientations are drawn up front, in order, from the +fixed seed, so replicate $i$ gets the same orientation whatever order the threads run in — a σ that +depended on the scheduling would not be a measurement. Where the null is skipped, `--model` costs +what it did before the null existed (measured: indistinguishable, ~4 s against ~4 s, on a loaded +machine where a single run scatters by a second). Where it runs, that is the price of the answer +being a measurement rather than a threshold. + +### 14.6 Aligning the data to the model: enantiomorph and indexing ambiguity + +The model fixes a definite hand and indexing, but the merged data need not share them, so before comparison the observed reflections are brought into the model's frame. + +- **Enantiomorph / screw.** When the data space group is the enantiomorph of the model's (e.g. data $P4_12_12$, model $P4_32_12$; or $P3_1/P3_2$), the two are **indistinguishable from merged intensities** — $|F_\mathrm{calc}|$ is invariant under the change of hand, so R-free cannot choose between them and probing would be meaningless. Where the model was accepted (§14.5), its group is therefore adopted as the **label** the reflections are written under, and the reflections themselves are left untouched. Where it was not, the label is left alone: adopting it is arithmetic on two group numbers, which a model that does not belong to this crystal can do exactly as readily as one that does, and the anomalous map of §14.7 vetoes it outright where it says the two are in opposite hands. The two groups of an enantiomorphic pair differ only in the translations of their operations: their rotations are identical, so they transform $hkl$ identically, share a reciprocal ASU, and assign the Bijvoet hands identically. The label carries no handedness, and there is nothing about it to undo. Reindexing by the change-of-hand operator — which is the inversion — would instead **swap $I(+)$ with $I(-)$**, flipping every anomalous difference on the strength of a label the space-group search itself reports as undetermined; where the model is genuinely the wrong enantiomorph for the crystal, it would manufacture agreement rather than reveal the mismatch. What does carry the hand is the indexing the data already have, from the diffraction geometry, and the anomalous differences that come with it. §14.7 is what tests them against the model. +- **Indexing (merohedral) ambiguity.** When the crystal has a merohedral ambiguity (§10.9), the observed intensities *do* differ between indexings, and the right one is chosen against the best available reference. **If a reference MTZ was supplied, the data were already reindexed to agree with it** (§10.9 — by the reference-intensity correlation, at the merge stage for rotation data or per image in stills scaling), and model validation keeps that authoritative choice. **Only with a model and no reference** does validation resolve the ambiguity itself, as a fallback: the scaled model is fit to each reindexing of the data (identity plus the twin-law cosets) and the one giving the **lowest R-free** is kept — but only where its lead over the runner-up beats the lead the *same model in a random orientation* takes, since a random model also picks a winner, and, as measured, by a comparable margin. Where it does not, the data keep the indexing they were merged in. The candidate operators are enumerated from the **data's** space group, not the model's: it is the observed intensities that are being relabelled, and asking the model's group enumerates nothing at all wherever the two differ. This matters for a multi-dataset campaign — a single shared reference fixes one indexing convention for every dataset, whereas an independent per-dataset lowest-R-free choice could send borderline datasets to different conventions. A no-op either way for a holohedral crystal (no twin laws). The two decisions then reach the written output differently, because only one of them moves reflections. The **ambiguity choice is applied to the merged reflections themselves** — and to the integrated observations behind the unmerged export — which are written after this step, so the reflection file, the R-factors and the maps describe one indexing. The **change of hand changes only the space group the files are written under** (the model's enantiomorph): no reflection moves, exactly as the first bullet says, so $I(+)$ and $I(-)$ stay as measured and §14.7's hand check remains a genuine test rather than an agreement manufactured by reindexing. The ambiguity choice is reported with that margin and with the null it was judged against, since the margin alone is what cannot say whether the data decided or the two came out within noise of each other. + +### 14.7 Anomalous difference map and the sites it names + +Where the merge kept the Bijvoet split (§10.5) — which a rotation merge does by default, whether or not the mates were averaged — an **anomalous difference map** is computed as well, with coefficients + +$ +\big(|F(+)| - |F(-)|\big)\, e^{i(\varphi_\mathrm{model} - \pi/2)}, +$ + +over the acentric reflections that have both hands (a centric reflection has no anomalous difference, only noise). Turning the model phase back by 90° is what makes the anomalous scattering, which is 90° out of phase with the normal scattering, add up in the real part: the map's peaks then sit on the anomalous scatterers. It is written as `_anom.ccp4`. Its hand is the one §14.6 settled: with the mates the wrong way round every peak becomes a trough, so a map of clean peaks is itself a check that the frame is right. + +Rather than searching the map for blobs and leaving a list of coordinates, the map is read **at the model's own atom centres** (hydrogens excluded — they scatter no anomalous signal), and the ten highest, in units of the map's r.m.s., are reported in the log and as `ANOMALOUS_SITE_01`…`ANOMALOUS_SITE_10` in the results report. Each site is therefore named — the atom, residue and chain it belongs to — which is what says *what* carries the signal, not just where it is. The reading is cubic, not linear: the map is sampled every $d_\mathrm{min}/3$, and a peak that sharp read by trilinear interpolation comes out up to a quarter low — unevenly enough to reorder the sites. This reading is the one ANODE reports. The same reading averaged over the model's **anomalous scatterers** — every atom from phosphorus ($Z = 15$) up: the S of Met and Cys, metals, Cl, I — is reported as `ANOMALOUS_SCATTERER_MEAN_SIGMA` (with their count, `ANOMALOUS_SCATTERERS`): one number for how much anomalous signal the merge carries, which does not depend on which ten atoms happen to come out on top. Below phosphorus (C, N, O, Na, Mg) $f''$ is a fraction of sulfur's at any wavelength these data are taken at, and counting those atoms would only dilute the mean with noise. + +Because the map is built on the model's phases and the data's own indexing, it is also the only test of whether the two agree about the **hand** (§14.6). A model and a dataset in opposite hands turn every anomalous peak into a trough, so a map whose deepest hole at an atom is both deeper than $5\sigma$ and deeper than its highest peak says so, and the run reports it as a warning naming that atom. It is not repaired by reindexing: that would make the two agree by construction and destroy the evidence for which of the model and the data is in the wrong hand. Note that R-free cannot see this at all — a mirrored model gives R-free to four decimal places unchanged, and an exactly inverted anomalous map. + +The list is always ten entries long, so it is their height that carries the information: on a sulfur-SAD dataset the sulfurs fill the top of the list and are followed by a clear drop to the couple of sigma that is the map's noise, while a dataset with no anomalous signal has no such separation and lists ten unrelated atoms at noise level. A scatterer the **model does not contain** — a bound ion, a soaked heavy atom — is by construction invisible in the list, and is what the map file is for. + +### 14.8 Rigid-body placement of the model + +Re-fractionalizing a model into the data cell puts it in the right box but not necessarily in the right place: a non-isomorphous cell squeezes the box without moving the body inside it, and the body's own position in the cell differs from crystal to crystal. Six parameters recover that — an angle-axis rotation about the model's own centroid, then a translation. Rotating about the centroid rather than the cell origin is what keeps the rotation from moving the body bodily, so the two triplets are close to independent. Parameters are carried as six lengths in ångström (the rotation vector multiplied by the model's r.m.s. radius), so a unit of each moves a typical atom by the same amount. + +It is **one** rigid body. A fragment-screening model arrives already solved and isomorphous, and what is being recovered is the crystal's movement, not the molecule's; splitting it into domains, or giving a bound ligand six parameters of its own, would refine against evidence these data do not separately carry — and the ligand is what the difference map is there to show, not to model away. + +The refinement walks a coarse-to-fine ladder, 6 Å → 4.5 Å → 3.5 Å, each zone starting from the previous one's answer. It stops at 3.5 Å because that is where rigid-body refinement is conventionally run and because the movement being recovered is a few tenths of an ångström, a tenth of that resolution; a finer zone costs $(1/d)^3$ in grid points and reflections for a placement it cannot meaningfully sharpen. Each evaluation recomputes $F_\mathrm{calc}$ and the solvent mask for the moved model **and re-fits the scale of §14.2** — otherwise the target would measure the scale as much as the placement, and the body would translate to repair a scale error instead of moving to where the density is. The minimiser is Levenberg–Marquardt on the amplitude residuals. $F_\mathrm{calc}$ is computed from **one copy** of the model, gridded without symmetrization and transformed once, and composed over the space group's operators, $F(\mathbf h) = n_\mathrm{cen}\sum_{(R,\mathbf t)} e^{2\pi i\,\mathbf h\cdot\mathbf t}\,F_1(\mathbf hR)$ — the transform of the symmetrized map, rearranged. That makes the derivative with respect to the translation exact and free ($F_1(\mathbf k)$ only picks up the phase $e^{2\pi i\,\mathbf s_\mathbf k\cdot\Delta}$). The rotation's three columns are forward differences, the step a fixed fraction of the zone's resolution, each needing only one copy gridded and transformed. The Jacobian holds the scale and the bulk-solvent mask at the evaluation's; the residuals keep both exact, and the scale re-fit is folded into the Jacobian by variable projection (Golub & Pereyra 1973, in Kaufman's 1975 form), so a step is judged on the target the evaluations actually compute. + +The refinement sees **only the working reflections**. The step is then committed only if it lowers **R-free**, computed at full resolution on the free set it never saw; otherwise the model is put back exactly where it was read and the maps are the ones it would have given. The whole addition costs about two seconds. Note that the origin is a gauge in some space groups — free in all three directions in $P1$, and along the unique axis in a polar group — so those components of the translation are undetermined; nothing is done about that beyond the Levenberg–Marquardt damping and the R-free gate, which between them make an undetermined direction harmless rather than unstable. + +Where a CUDA GPU is present, the refinement's target is evaluated on it: the same density, the same composition of $F_\mathrm{calc}$ from one copy of the model, the same bulk-solvent mask with its islands removed, the same scale fit and the same Jacobian, computed on the device, where a fit takes a fraction of a second rather than several seconds. The device is chosen once per validation - the real fit and every replicate of the null of §14.5 on the same one - and the CPU is used where there is no GPU, where the card has too little free memory for the fit's buffers, or after a CUDA failure, in which case the whole validation is run again on the CPU. The two agree to rounding and not bit for bit: the device measures distances in single precision and transforms with cuFFT instead of FFTW, and GEMMI's scale fit, whose stopping rule leaves it unconverged at about $10^{-4}$ of $|F|$, can respond to that rounding with a step of its own. A rigid-body step accepted or rejected on an R-free difference below about $5\cdot10^{-4}$ can therefore go either way between the two; every kernel is deterministic, so either one gives the same answer on every run. + +The rotation and translation actually taken are reported (`RIGID_BODY_ROTATION_DEG`, `RIGID_BODY_SHIFT_A`), together with the R-free before it (`R_FREE_BEFORE_RIGID_BODY`), so what the placement bought is visible. + +The coordinates **as placed** are written as `_model.cif`, and as `_model.pdb` where the PDB format can hold the cell (the fragment-screening tools the file feeds — PanDDA, and the pair dimple produces — take a PDB beside the MTZ), so there is a coordinate file that describes the maps: the input's chains, residues, ligands, waters, B-factors, occupancies and anisotropic $U$s, moved, in the same cell and space group as `.mtz`. Taking the frame from the written reflections rather than from the input model matters — §14.6 may have relabelled them to the model's enantiomorph, which is neither the data's original group nor, necessarily, the model's. It is written **whenever the maps are**, not only where the rigid-body step was committed: the model is re-fractionalized into the data cell and may be relabelled whatever the placement decided, so an unmoved model is still not the input file; and a model the null of §14.5 rejected is scored, placed and mapped like any other, which is precisely the case where the density is worth looking at. diff --git a/_sources/CPU_DATA_ANALYSIS_IMAGE.md.txt b/_sources/CPU_DATA_ANALYSIS_IMAGE.md.txt new file mode 100644 index 000000000..63acebc02 --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS_IMAGE.md.txt @@ -0,0 +1,654 @@ +# Data analysis: from images to spots (§0–§3) + +Part of the [CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md); the section numbers are continuous across its four parts. + +```{contents} On this page +:local: +:depth: 2 +``` + + +## 0. Getting the image onto the GPU: device-side bitshuffle+LZ4 decoding + +Images arrive bitshuffle+LZ4 compressed (HDF5 filter 32008), and everything from §1 onwards runs on +the GPU when one is present. Instead of decompressing on the host and uploading the image, the +compressed chunk is uploaded — a few MB rather than tens of MB — and decoded on the device. The +approach follows Jon Wright (ESRF); the kernels are Jungfraujoch's own. + +One kernel does the work: one CUDA block owns one bitshuffle block, from the compressed payload +through to finished pixels. + +1. **LZ4 into shared memory, one warp per bitshuffle block.** Blocks are independent, so the + parallelism is across them; within the warp every lane runs the same sequence parser over the + same bytes, and the literal and match copies are split across the 32 lanes so the stores + coalesce. An overlapping match is treated as a pattern of period `offset` sourced from bytes + that already precede the write position, which keeps it parallel rather than a serial byte loop; + `offset == 1` (a run of one repeated byte, the common case in sparse detector data) and + power-of-two offsets avoid the modulo altogether. Because the lanes cooperate on the copies, + each one is followed by `__syncwarp()` — a later match can read bytes another lane wrote, and + since Volta that ordering is not implicit. +2. **The bitshuffle inverse fused with preprocessing.** The whole CUDA block then reads that shared + buffer back: one thread owns one group of 8 elements across every byte-plane, so once it has + transposed its 8 bytes out of each plane it holds 8 complete elements — and it applies the pixel + mask, the error marker and the saturation cap and emits 8 finished `int32` pixels directly. + Nothing of the block reaches device memory but the pixels — neither the bitshuffled bytes nor the + decompressed image is ever materialised. For 8-bit images there is a single plane and the + assembly degenerates to a copy. + +Decoding into shared memory is worth more than the bandwidth it saves: an LZ4 match reads back bytes +written a few sequences earlier, so every copy step is a dependent round trip — tens of cycles in +shared memory against hundreds in device memory. It is paid for in residency, because the buffer +holds a whole bitshuffle block, and a block larger than 16 kB (larger than either writer this +pipeline reads produces) falls back to a pair of kernels instead, the first writing the shuffled +image to device memory and the second un-transposing and preprocessing out of it. + +The block offsets inside the container can only be discovered by reading the block lengths in +order, so that scan stays on the host. + +Only `BSHUF_LZ4` is decoded on the device. For the zstd variants (`BSHUF_ZSTD`, `BSHUF_ZSTD_RLE`, +`BSHUF_ZSTD_RLE_HUFF`), and for uncompressed or float images, `BSLZ4DecoderGPU::Supports()` returns +false and the pipeline decompresses on the host and uploads as before. + +The container arrives off the network or off disk and is not trusted. Everything checkable on the +host — declared sizes, the block scan, a block size that is not a multiple of 8 elements, a block +count the chunk could not hold, trailing bytes — is rejected before any work is queued; the kernel +additionally flags a block that did not decode to exactly its declared length, which becomes an +exception once the caller has synchronised. That last check matters because the decode buffers are +reused frame to frame: a block that stopped early would leave the *previous* image's most significant +byte-plane in place, which reads not as a missing corner but as real pixels several powers of two too +bright. + +## 1. Geometry, reciprocal-space mapping, and basic quantities + +### 1.1 Coordinate conventions + +For a pixel coordinate $(x,y)$ (in pixels), Jungfraujoch converts to a laboratory direction vector via: + +1. shift by the beam-centre pixel $(x_\mathrm{beam}, y_\mathrm{beam})$ — the **PONI**, pyFAI's point of normal incidence, which coincides with the direct-beam impact point only for an untilted detector (see [Detector geometry](DETECTOR_GEOMETRY.md)), +2. scale by pixel size $p$ (mm), +3. set detector distance $D$ (mm), +4. apply detector orientation rotation $R_\mathrm{det}$ (PyFAI-like parameterization). + +The unnormalized detector coordinate (mm) is: +$ +\mathbf{r}_\mathrm{det}(x,y) = +\begin{pmatrix} +(x-x_\mathrm{beam})p\\ +(y-y_\mathrm{beam})p\\ +D +\end{pmatrix}. +$ + +The lab-frame vector is: +$ +\mathbf{r}_\mathrm{lab} = R_\mathrm{det}\,\mathbf{r}_\mathrm{det}. +$ + +By this construction $(x_\mathrm{beam}, y_\mathrm{beam})$ maps to $(0,0,D)$ *before* the rotation — the point where the detector normal through the sample meets the detector — which is what makes it the PONI rather than the direct beam; the two differ by $D\tan(\mathrm{tilt})$ on a tilted detector. The laboratory frame is fixed the same way everywhere in the system: $+z$ along the beam propagation, $+x$ along increasing pixel **column** (the fast axis) and $+y$ along increasing pixel **row** (the slow axis) — a right-handed triple that coincides with XDS's laboratory frame, which is what makes the geometry echo of [Rugnux](RUGNUX_INTEGRATION.md#comparing-the-geometry-with-xds) directly comparable. The absolute hand of an indexing — and with it the Bijvoet hands of §14.6–§14.7 — follows from this convention. + +Let the incident wavevector magnitude be $k = 1/\lambda$ in Å$^{-1}$, and define: +$ +\mathbf{S}_0 = (0,0,k). +$ + +The **reciprocal-space scattering vector** associated with pixel $(x,y)$ is: +$ +\mathbf{s}(x,y) = k\,\frac{\mathbf{r}_\mathrm{lab}}{\lVert \mathbf{r}_\mathrm{lab}\rVert} - \mathbf{S}_0. +$ + +This $\mathbf{s}$ is the fundamental quantity used for spot finding (resolution filters), indexing, and refinement. + +### 1.2 Two-theta, azimuth, resolution and $q$ + +The scattering angle $2\theta$ is computed from $\mathbf{r}_\mathrm{lab}$ via: +$ +2\theta = \mathrm{atan2}\!\left(\sqrt{x_\mathrm{lab}^2 + y_\mathrm{lab}^2},\; z_\mathrm{lab}\right), +$ + +evaluated as a two-argument arctangent, so the mapping stays correct where a strongly tilted detector's far corner reaches past $2\theta = 90°$. + +Resolution (Å) at a pixel is: +$ +d = \frac{\lambda}{2\sin\theta}. +$ + +The magnitude $q = 2\pi/d$ is used for radial binning and ice-ring handling. + +### 1.3 Distance from the Ewald sphere + +For a reciprocal lattice point $\mathbf{p}$ (Å$^{-1}$), define: +$ +\Delta_\mathrm{Ewald}(\mathbf{p}) = \lVert \mathbf{p} + \mathbf{S}_0\rVert - k. +$ +Jungfraujoch uses $|\Delta_\mathrm{Ewald}|$ as an operational proxy for excitation error. This appears in: +- still prediction (accept if $|\Delta_\mathrm{Ewald}|\le \Delta_\mathrm{cut}$), +- profile radius estimation (see §11.1), +- still partiality option in scaling/merging (§10.2). + +### 1.4 The beam centre + +A run starts from the beam centre in the file, or from `--beam-x`/`--beam-y` where they are given. +Header centres are often typed rather than measured, and on rotation data a wrong one is not repaired +by anything downstream: the error is fixed in the laboratory frame, so accumulating a sweep smears +every reciprocal-lattice point around a circle, and a displacement $\delta p$ on the detector multiplies the FFT +amplitude at an axis of length $a$ by $J_0(2\pi\,\delta p\,a/(D\lambda))$. Past the first zero the +true axis is gone and its harmonic wins. Rugnux therefore measures the centre itself on every run and +treats the measurement as a second hypothesis to be tested against the file's, not as a replacement +for it. The steps, in the order they run: + +| Step | Runs | Effect on the centre | +| --- | --- | --- | +| Header check | every run, before a frame is read | none; reported | +| Background measurement (`--beam-center-check`) | every run with the beam-stop pre-scan | none at this point; reported | +| `--estimate-beam-center` | only when asked | replaces the file's before indexing, where measured precisely enough | +| Second first pass at the measured centre | rotation | adopts the measured centre in the cases listed below | +| Both centres judged on the merge | rotation, two-pass | adopts the measured centre where its first pass merges better | +| `--beam-center-search` | rotation, after a first pass that indexes under half the validation frames | steps the centre a pixel at a time | +| Post-refinement (§7.5) | rotation, two-pass | refines the centre from the integrated reflections, within 15 px | + +On most runs the measured centre is reported, the second first pass finds the same lattice at both +centres, and the run continues at the file's centre until post-refinement moves it. + +**What the file says.** Before any frame is read, a centre within one pixel of the geometric centre of +the detector, or a whole number of pixels in both coordinates, is reported as *a value written rather +than measured*, and a centre that lands on a masked pixel is a warning — a real beam does not sit on a +dead pixel or in a module gap. These lines change nothing. + +**Measured on every run (`--beam-center-check`, on by default).** The isotropy of the scattered +background places the point the beam lands on; the leverage is the curvature of the solvent ring. The +fit runs over the mean image the beam-stop pre-scan already builds (§1.5), with the opaque part of the +shadow masked, so it costs no frames of its own. For the same reason `--detect-beam-stop=off` leaves +nothing to measure it on, and the check then does nothing. + +The fit starts with a **whole-detector capture**. The centrosymmetry score of the projection about a +candidate centre $\mathbf{c}$, $\sum_\mathbf{x} I(\mathbf{x})\,I(2\mathbf{c}-\mathbf{x})$, is the +self-convolution $(I*I)(2\mathbf{c})$, so **one FFT pair scores every candidate centre on the +detector** at half-pixel spacing: the cost is $O(N\log N)$ and independent of how far the header is +from the truth. Three surfaces are scored — the 2D point inversion, which measures the same isotropic +background the walk below fits, and a 1D line mirror per detector axis, which along the spindle is +exact Friedel physics (a reflection's mate half a turn later lands mirrored in the line through the +beam across the spindle). The beam-stop shadow is blanked out of the scored image, a one-sided opaque +obstruction being a centrosymmetry defect in its own right. What comes back is a shortlist and a +margin per surface, not a centre: the surface can be locally flat over tens of pixels, and the +peak-to-runner-up margin says so. A **local walk** is then seeded at the capture and refines it; where +it declines there it is started again from the file's centre, and only where neither start gives it +something to fit does the capture stand alone. The transforms run on the GPU where one is present and +on FFTW otherwise, with everything that decides anything shared between the two. + +The run log then carries: + +- `Beam centre capture:` the strongest point-symmetry centre, the two line-mirror coordinates and the + three margins (under about 3 % the surface is flat). +- `Beam centre check: the file puts the direct beam at (x,y), the scattered background puts it at + (x,y) +- σ px - a difference of d px, against the t px this geometry asks the centre to be right to.` + The comparison is against the **direct beam**, not the PONI: `beam_x_pxl` is the point of normal + incidence, and the two part by $D\tan(\mathrm{rot})/p$ as soon as the detector is tilted, while the + background is centrosymmetric about where the beam lands. $t = 0.183\,D\lambda/(p\,a_\mathrm{max})$ + is the displacement at which the $J_0$ factor above has fallen to 0.70 for the longest axis of the + cell given with `-C` (200 Å where none is given); it runs from well under a pixel to several. It is + reported, not used as a gate: a centre change far smaller than $t$ can still decide between a cell + and its harmonic. +- One of three follow-ups: the difference is under three times the fit's σ (the file's centre is as + good as this measurement can tell); more than $t$ of it lies **across** the spindle, where it can cost + peaks or an axis; or it is a real difference, mostly **along** the spindle, where a wrong centre does + not announce itself. + +None of these lines moves the centre. What consumes the measurement is the second first pass below +(rotation runs), `--estimate-beam-center`'s fall-through, and the bound on the post-refinement. + +**The second first pass (rotation).** After the first pass — and after the rotation-axis sign rescue +below — the first pass is indexed **again at the measured centre**, however small the difference. A +trial centre gets its own azimuthal mapping, spot engines and spot cache: raw centroids do not move +with the centre, but *which spots are in the list* does (the resolution limit, the ice-ring flag, the +beam-stop mask and the strongest-$N$ ranking are all radial), and the starting centre's spots are +parked and restored so a trial that adopts nothing leaves the run as it was. + +The indexed frame count is **not** used to choose between two centres that both index: acceptance is a +fractional-Miller test, so a cell twice as long must place every spot twice as accurately to score the +same, and a halved axis can index *more* frames than the true cell. "Indexes" below means more than +half of the validation frames; "same lattice" means the same Bravais class with primitive volumes +within 2 %. The outcomes: + +1. **The file's centre indexes and the measured one does not.** The file's centre stands. +2. **Both index the same lattice.** The file's centre is kept, and the log says the difference does not + decide this crystal's cell. A centre off by about one reflection spacing still indexes the same cell, + with part of the sweep given indices one out, so where the two centres are further apart than both + $t$ and three σ of the fit, the measured centre is kept as a hypothesis for the merge (next + paragraph). +3. **Both index the same cell in a different Bravais class.** Whether the centre could have decided the + class is arithmetic: a shift of $n$ pixels of size $p$ moves an axis of length $a$ by $n\,p\,a/(D\lambda)$ + relative, and the lattice search holds an axis equality to 3 %. Where the shift is worth that much on + the pair of reduced-cell axes the equality compares, both centres are carried to the merge; otherwise + the file's centre and its class are kept. +4. **Both index, on different lattices.** Both cells are reported in a warning. Where the primitive + volumes differ by an integer factor of 2 to 4 — an axis harmonic, which a centre error along the + spindle produces — the choice is made on the **pooled validation spots**: the measured centre is + adopted where its lattice, larger or smaller, beats chance and puts a share of the spots on itself + over its own wrong-spindle null that exceeds the file's by more than the binomial noise of the two + (z = 3.29). Otherwise the file's centre is kept and the warning suggests `--estimate-beam-center`. +5. **The file's centre does not index.** Nothing defends it, so: + - where the measured centre indexes, it is **adopted**; + - where it does not, the rotation-axis sign is flipped **at the measured centre**, since each of the + two errors can hide the other; where that indexes, both are adopted; + - where it still does not, the first-pass depth ladder of §6 (how much of each frame the pass reads) + is run at **both** centres, so the centre moves only when it is the centre that pays. The measured + centre is adopted where its lean rung indexes and beats the file's; where both index the same + lattice once less of each frame is read, both centres are carried to the merge; where the file's + centre wins or ties, the depth was the problem and the file's centre stays; + - where nothing indexes at either centre, the pooled spots decide as in case 4; failing that, the + file's centre is put back, so a run that fails for some other reason fails at the geometry it was + given. + +**Both centres judged on the merge (rotation, two-pass).** Where case 3 or 5 carried both centres, the +whole first pass is run again at the measured centre — with its own short-axis pass and +post-refinement — and the two arms are compared on their search merges ($P1$, whole range, before the +correction surfaces). Where case 2 held a hypothesis, this is done only if the file's arm merges +inconsistently: $\mathrm{CC}_{1/2}$ below 0.9 in the lowest-resolution shell of its search merge, where +every reflection is strong and a consistent merge reads close to 1. The measured centre wins where: + +- both arms ended on the same lattice, and it merges **more reflections at $I/\sigma \ge 2$**, or its + lowest-shell $\mathrm{CC}_{1/2}$ is higher by more than 0.05; +- the arms ended on different lattices, and its search-merge $\mathrm{CC}_{1/2}$ is higher by more than + 0.05. + +Otherwise, or where the measured arm does not complete, the run keeps the file's centre. The log lines +are `Beam centre check: running the first pass again at the measured centre …` followed by +`… the run adopts the measured centre and the lattice it finds` or `the measured centre does not win`. +Over the validation battery about one rotation run in ten runs this second arm, and about one in ten +ends on the measured centre by one of the routes above. + +**After a first pass that failed.** A rotation sweep leaves two further things the input often cannot +settle, both decided on the same count the rest of the first pass uses — the right answer indexes and +the wrong one does not. + +* **The rotation-axis sign.** A miniCBF header names the axis but gives no direction, and an NXmx + vector is only meaningful together with the detector mounting. So after a poor first pass the + opposite sign is tried and whichever indexes more validation frames is kept, the flipped one only + where its lattice also beats chance on the pooled spots (§4.1), so that one frame against none on a + sparse pattern cannot flip it. The decision holds for the rest of the run, and it flips the + **axis**, not the angles, because prediction reads the axis too. It runs before the second first + pass above and before the long-axis rescue, since with the sign wrong every candidate lattice is + wrong. +* **The beam-centre search (`--beam-center-search[=N|off]`, on by default, N = 12 px).** Where the + first pass, after the second first pass above, still indexes fewer than half the validation frames, + the centre is stepped a pixel at a time out to N px along **both** detector axes, each rung with its + own spots, and the first rung that indexes a majority and beats the starting count is adopted + (`Beam centre from indexing: …`). Both directions are searched deliberately: an error *across* the + spindle collapses the indexed fraction and announces itself, while an error *along* it leaves the + transform's peaks sharp, holds nearly every frame indexed and lets the lattice fit commit to an axis + harmonic. A rung whose primitive volume is an integer or $\sqrt{3}$ multiple of the starting cell's is + refused for that reason. The search is skipped where the background measurement places the centre + further away than both N px and three σ, since no rung could reach it, and it stops after two rings + where no rung has reached a third of the majority. The step is a flat pixel: derived from the $J_0$ + law it would have to use the cell the *failed* pass returned, which can be a small spurious sub-cell + and asks for a step that jumps the lobe being looked for. + +**Post-refinement.** On a two-pass rotation run, whatever centre the first pass ends on is refined +together with the distance, cell, orientation and axis by the post-refinement of §7.5, and the second +pass is integrated there. The background measurement does not move that fit; it only widens where the +fit may go: the refined beam is committed only within 15 px of the **nearer** of the pass-1 centre and +the measured one (the measured one counting where its σ is within $\max(1\ \mathrm{px}, t)$), so a +header that is far out can still be corrected. Where the refined pass is judged worse, the run goes +back to the pass-1 geometry. + +**Committed before indexing (`--estimate-beam-center`, off by default).** Here the centre is measured +before anything is indexed and used in place of the file's. On a sweep that reaches half a turn it +comes from the spot positions alone; two exact facts about a rotation sweep supply the two +coordinates. + +**Friedel mates half a turn apart.** The Laue condition fixes the component of $\mathbf{q}$ along the +beam, $q_\parallel = -\lVert\mathbf{q}\rVert^2\lambda/2$. Rotating 180° about the spindle $\mathbf{m}$ +negates the two components perpendicular to $\mathbf{m}$ and taking $-h$ negates all three, so +together they negate **only** the component along $\mathbf{m}$ and leave $q_\parallel$ untouched. With +the spindle perpendicular to the beam, $-h$ therefore diffracts at $\varphi+180°$ exactly where $h$ +diffracts at $\varphi$, and its spot sits at the mirror image of $h$'s along the spindle. This gives +the beam coordinate **along** the spindle. Only the reciprocal lattice's centrosymmetry is needed for +the geometry; Friedel's law $|F(h)|=|F(-h)|$ is used separately, to tell a true pairing from an +accidental one. + +**The second crossing.** The same reflection meets the Ewald sphere twice, at two angles that are +generally *not* 180° apart, differing only in the sign of the lab component perpendicular to both +$\mathbf{m}$ and the beam. This gives the remaining coordinate. The two crossings are separated by a +sweep angle fixed by the reflection's own position, which is what identifies genuine pairs. + +Neither observable requires a cell or an orientation matrix: each candidate pairing votes for a beam +coordinate, and the true value accumulates while wrong pairings scatter. A Friedel pair needs both +$\varphi$ and $\varphi+180°$ recorded, so a sweep of $S°$ yields only $S-180$ degrees' worth of pairs. + +The mirror is exact in the **laboratory** frame, so it is sensitive to the spindle direction. A skew +of the spindle about the beam *spreads* the vote rather than shifting it, and is fitted alongside the +centre (`--no-fit-spindle` keeps the axis from the file); a tilt of the spindle towards the beam is +measured and reported but not applied, being confounded with the detector tilt. Nothing inside the fit +can tell that the vote settled on the wrong periodic maximum — every frame pair agrees with every +other — so its σ is how far the answer moves when the search is started from a different position. +The frames are read in pairs half a turn apart, twice as many as the beam-stop projection uses and +away from both ends of the sweep, where shutter synchronisation can spoil an image; where the answer +does not come out on them, up to 400 more are read. + +Where the spot symmetry does not come out — in practice on sweeps shorter than about 220° — the centre +is taken from the background measurement above. The estimate replaces the file's centre only where its +σ is within $\max(1\ \mathrm{px}, t)$ and it moves the centre by more than three σ; otherwise, or where +neither method measures anything, the file's centre is kept. The log line is +`Beam centre from spot symmetry|background: (x,y) -> (x,y), moved d px, sigma σ px against a c px ceiling => COMMIT` +(or `reject`, with the reason). `--estimate-beam-center` is ignored where `--beam-x`/`--beam-y` are +given, and on stills where `--refine-geometry` has already placed the centre from indexed spots. + +**Stills.** There is no second first pass, no search and no post-refinement on stills; the background +measurement is reported only. The centre moves through `--estimate-beam-center` (background estimator) +or through the stills geometry refinement, `--refine-geometry`, which bundle-adjusts the beam centre, +distance and cell over strongly indexed frames and is on by default where a reference cell is given. + +**What ends up in the output.** The report's `BEAM_CENTRE` (the PONI) and `DIRECT_BEAM` are the +geometry the result was integrated at, and so is `JFJOCH_DATASET_SETTINGS`; `POSTREFINE_BEAM_CENTRE` +gives the post-refinement's move as `before -> after`. Which of the steps above changed the centre, and +why, is in the run log only. + +**Options.** + +- `--beam-x`/`--beam-y` set the starting centre and switch `--estimate-beam-center` off. The background + check still runs and can still adopt the measured centre in the cases above; add + `--beam-center-check=off` to hold a typed centre until post-refinement. +- `--beam-center-check=off` skips the background measurement and the second first pass, and removes the + measured centre from the post-refinement bound. +- `--beam-center-search=off` or `=N` turns off or resizes the search after a failed pass. +- `--estimate-beam-center` measures and commits the centre before indexing; `--no-fit-spindle` keeps the + spindle direction from the file in that estimate. +- `--detect-beam-stop=off` removes the projection the background measurement is read from. +- `--rotation-no-postrefine` removes the post-refinement, and with it the merge-judged arms. + +### 1.5 Finding the beam stop + +The beam stop and its holder arm shadow part of the detector. A reflection behind them is attenuated +but otherwise ordinary — it integrates low, with a plausible $\sigma$, and no outlier test catches +it — so the shadow is found and masked instead. `--detect-beam-stop[=N|off]` (on by default, $N=60$ +frames) projects those frames to a per-pixel mean and maximum, and writes the result into +the pixel mask as bit 9, from where it excludes those pixels from every later stage. + +The shadow is a place where the background is *missing*, so it is found by comparing each pixel's +background against the background at the same radius. The mean is pooled over a small box first — +a single pixel of a sparse background carries too few counts to tell a shadow from a Poisson hole, +and the stop is much wider than the box — and the comparison is against the median of the pixel's own +radius ring, taken over the pixels not already excluded and iterated a few times so the shadow stays +out of the baseline it is measured against. A pixel is shadow when the ratio falls below 0.50 **and** +the deficit is significant against its own Poisson scatter, $\sqrt{2\,(E-N+N\ln(N/E))}$ over the pooled +counts. A ratio says nothing when the background behind it is a handful of photons, and it is that +significance that makes the comparison scale-free rather than tuned to one exposure: rebuilt from six +frames of a low-background sweep, the same ratio without it masks three quarters of the detector. The +index of dispersion does *not* do this job — it is 1.0 inside the shadow and 1.0 outside it, a +shadowed pixel being Poisson at a low rate and a lit one Poisson at a high rate — only the rate +relative to the ring separates them. + +Nothing assumes the stop and the beam are concentric, because only the per-ring comparison is used; a +ring lying wholly inside the stop has no unshadowed pixel for its median to find, which is exactly the +case where the comparison must fail. Such a ring is judged against what this detector's background +typically is — the median over the rings the walk is willing to judge — and is shadow in its entirety +when it falls far below that. Comparing it instead against the *largest* background further out reads +an ordinary background inside a strong ring as blocked, and the walk then runs out to that ring and +returns a filled disk of good detector with diffraction rings plainly visible inside it. + +**The rings are drawn about the centre the data measure, not the one the file claims.** Displacing the +centre costs nothing for tens of pixels and a great deal beyond: at 150 px out the comparison describes +the background's own radial fall-off rather than the hardware, and returns a tenth of the detector as +shadow with nothing blocking it — and header centres are wrong by that much. The centre is therefore +fitted from the same per-pixel mean the mask is computed from, before the mask is read, and named to +the finder; it costs no frame of its own. That first fit is itself biased by the shadow it has not yet +masked (on a sweep with a third of the detector behind a pin it landed 5 px out and called itself +0.40 px), and it does not have to be unbiased: the ring comparison does not notice tens of pixels, and +the centre the run *reports and consumes* is the second fit of §1.4, the one that runs after the mask +is loaded with the shadow out of the way. The order is fit, mask, fit, and only the second answer +leaves. Two rounds are enough, measured rather than assumed — on clean sweeps the two fits agree to +0.03 px, so a third would draw the same rings. + +**A ring is only flat once the polarization is divided out.** The comparison assumes the background is +flat around a ring with nothing in the beam, and it is not: a polarized source suppresses the +background in its own plane by the azimuthal factor of §2.2, a factor of three at $2\theta=55°$ and +four at $70°$ — several times the dip the test is looking for, so on a short-distance geometry the two +in-plane lobes of every outer ring read as shadow (measured: 5 % and 11 % of two detectors, with +nothing visible under either mask). The mean projection is therefore divided by that factor before the +ring comparison, and the Poisson deficit multiplies it back in so the significance is still the +significance of the counts that were recorded. The factor is the geometry's own, evaluated about the +centre just fitted rather than the one in the file: polarization is the only correction a ring carries +that varies *along* it — solid angle, detector response and air absorption are functions of $2\theta$ +and the ring median absorbs them — and the geometry is what knows where the polarization plane lies +once the detector is tilted, the stored image quarter-turned or the detector rotated in its own plane. + +What survives is then shaped into a region, and what makes it specific is **size**, not connection to +the beam. A shadow is cast by something physical and is correspondingly large, while the background +wanders a pixel or two at a time, so the connected components holding at least 2000 core pixels are +kept and the rest dropped; measured on clean sweeps spanning 0.05 to 9.5 counts/px/frame of +background, every one returns exactly one such component — the beam stop — and the largest spurious +candidate anywhere is 74 pixels. Requiring the region to touch the direct beam instead is written for +the stop and its holder arm; hardware standing in the beam further out — a pin, a loop — casts a +shadow that begins some way out in radius, with lit detector between it and the stop and no bridge +across the gap, and on one sweep that discarded 950 k correctly-found pixels and sent them into +integration as measured-and-near-zero. The kept components are bridged across the module gaps the +holder arm crosses, grown outward through the partially shadowed penumbra — much wider for a pin than +for a stop edge — closed, and with the interior of the disk filled; the rings found lying wholly +inside the stop join the region after the size filter rather than being asked to be large themselves. + +Last, any pixel that ever recorded a real +reflection — a maximum over the frames well above background, in a cluster, so that a single-frame +zinger does not count — is given back, because a beam stop cannot have blocked a reflection that was +measured. + +### 1.6 Defective pixels + +A pixel that reads high frame after frame, whatever the crystal does, is integrated into whichever +reflection's box it falls in; under rotation that is one pixel collecting a different reflection on +every frame that reaches it, and a merged intensity hundreds or thousands of times its shell mean, +carried by one or two observations the merge's outlier test cannot judge. The file's mask misses +such pixels on many detectors, so on rotation data from a counting sensor Rugnux measures them on +the pre-scan's own sample of frames (`HotPixelFinder`, `rugnux/HotPixels.h`) and masks them as bit 10. +The frames are read a second time for it, once the beam-stop projection has measured where the +scattered background puts the beam: the rings have to be drawn about the true beam, and a file's +centre can be far enough out that a ring crosses the background's radial fall-off. + +On each frame a pixel is **lit** when it exceeds $b + 3.3\,\max(\sqrt{b}, 1.4826\,\mathrm{MAD}) + 2$, +with $b$ the larger of the median of its 2 px iso-$2\theta$ ring and of that ring's sixteenth in +azimuth, so ice and powder rings, the polarization dip and partial shadows set their own level. It +is **persistent** when it is lit on at least $\max(k_1, k_B)$ of the sampled frames: one reflection +stays on a pixel for $(\Delta\phi + w)/|\zeta|$ of rotation ($w = 5°$, a generous rocking width), +which covers fewer than $k_1 = 1 + \lceil (\Delta\phi + w)/(|\zeta|\,\delta) \rceil$ frames sampled +$\delta$ apart; and $k_B$ is the binomial bound, from the lit rate of the ring's other pixels, that +fewer than 0.01 pixels of the whole detector reach by chance. A persistent pixel is masked if it +reads on average at least ten times its ring and its mean excess is above the Poisson bound; a +weaker one cannot make an outlier. This holds for a patch of such pixels as for a lone one: a patch +lit through the whole sweep is stationary in the lab, so it is no reflection of the rotating +crystal, and a reflection crossing it would read the patch. Pixels holding the detector's error +value on most frames (already invalid on every frame, but not in the static mask) are masked with +them. A CCD (no sensor depth: read with an offset, not Poisson) is left alone. One log line says how +many pixels were masked and why. + +--- + +## 2. Azimuthal integration (radial profiles) + +Azimuthal integration produces a radial profile $I(q)$ or $I(d)$ by histogramming pixels into radial bins. Pixels are **not split** across bins; each pixel contributes wholly to a single bin. By default the profile is purely radial (a single azimuthal bin), but the azimuth can optionally be split into up to 512 $\phi$ sectors (`azim_bins`, `--azim-phi-bins`), giving a **2D $q\times\phi$ profile** that exposes azimuthal anisotropy such as detector shadowing or sample texture. + +### 2.1 Histogram estimator + +Let bin index $b(x,y)$ be precomputed from $q(x,y)$ (or equivalently from $d(x,y)$) and, when $\phi$ sectors are enabled, the azimuth $\phi(x,y)$ — so $b = b_q + b_\phi B_q$. For each bin $b$: + +- accumulate corrected intensity and its square: + $ + S_b = \sum_{(x,y):\,b(x,y)=b} I(x,y)\,C(x,y),\qquad + S^{(2)}_b = \sum I(x,y)^2\,C(x,y)^2, + $ +- and count: + $ + N_b = \#\{(x,y):\,b(x,y)=b \text{ and pixel is valid}\}. + $ + +The profile reports both the mean $\bar{I}_b = S_b / N_b$ (when $N_b>0$) and a per-bin sample standard deviation $\sigma_b = \sqrt{(S^{(2)}_b - S_b^2/N_b)/(N_b-1)}$ (a spread/error estimate for each radial point). Invalid pixels (masked, saturated, detector error codes) are excluded. + +### 2.2 Corrections applied + +Two standard corrections are available: + +**(i) Solid angle / geometric correction.** A flat pixel's solid angle falls off with the **incidence angle $\alpha$ between the scattered ray and the detector normal**. With the in-plane detector offsets $u=(x-x_\mathrm{beam})p$ and $v=(y-y_\mathrm{beam})p$ — measured from the PONI (§1.1), which is what the tilt-invariance below rests on — and detector distance $D$, +$ +\cos\alpha = \frac{D}{\sqrt{u^2+v^2+D^2}},\qquad +C_\Omega = \cos^3\alpha, +$ +applied — like the polarization term below — as a **divisor** (intensities are scaled by $1/\cos^3\alpha$), so pixels at oblique incidence, which subtend a smaller solid angle, are boosted. Because $\alpha$ is evaluated in the detector's own frame it is **invariant under detector tilt** ($\mathrm{rot1}/\mathrm{rot2}/\mathrm{rot3}$), matching PyFAI's `solidAngleArray` and MAX IV azint. It reduces to the commonly quoted $\cos^3(2\theta)$ form only for an untilted detector, where the incidence angle coincides with the scattering angle. + +**(ii) Polarization correction.** With polarization coefficient $P$ (beamline dependent) and azimuth $\phi$: +$ +C_\mathrm{pol}(2\theta,\phi) = +\frac{1}{2}\left(1+\cos^2(2\theta) - P\cos(2\phi)\left(1-\cos^2(2\theta)\right)\right), +$ +applied as a divisor to intensities (i.e. scale by $1/C_\mathrm{pol}$) when enabled. This is the +factor of Kahn *et al.* (1982); $\phi$ is the azimuth in the **lab** frame, so the correction follows +detector tilt and a swung-out $2\theta$ arm without further work. + +The **polarization plane** is taken to contain the lab $x$ axis — a horizontally polarized source. +A vertically polarized one is expressed by a **negative** $P$: flipping the sign of the $\cos 2\phi$ +term is exactly a $90^\circ$ rotation of the plane. The plane is **not** autodetected and is not read +from the file — no format Rugnux reads declares one — so a vertical beamline must say so with +`--polarization`. $P$ is the polarization *degree* (XDS's `FRACTION_OF_POLARIZATION` is $(1+P)/2$). +The default $P = 0.99$, an undulator value, is applied to every Rugnux run; it is not fitted, because +a single dataset does not determine it. Bending-magnet and wiggler beamlines are typically lower +(0.8–0.95), and a known value for one should be passed with `--polarization`. + +### 2.3 Background estimate for profiles + +A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid $Q$ window (default $2\pi/5$ to $2\pi/3$ Å$^{-1}$). This background is used for monitoring and diagnostics; it is **not** the same as the local Bragg-spot background used in summation integration (§9.2). + +--- + +## 3. Spot finding (strong pixels → Bragg spots) + +Spot finding is a two-stage process: + +1. **Strong-pixel selection** using intensity and/or local signal-to-noise criteria. +2. **Connected-component labeling (CCL)** to group strong pixels into candidate spots, followed by spot-level filtering and feature extraction. + +### 3.1 Strong-pixel detection by local statistics + +For each pixel $i$ with value $v_i$, consider a square window (nominally $31\times 31$ pixels) around it. Let the window contain $n$ valid pixels (excluding masked/bad/saturated), and define: +$ +\Sigma = \sum v,\qquad \Sigma_2 = \sum v^2. +$ + +To avoid biasing the local statistics by the test pixel itself, Jungfraujoch evaluates the pixel against the window with the pixel removed: +$ +\Sigma' = \Sigma - v_i,\quad \Sigma_2' = \Sigma_2 - v_i^2,\quad n' = n-1. +$ + +A variance-like quantity proportional to $n'^2$ is formed: +$ +V = n'\Sigma_2' - (\Sigma')^2, +$ +and the deviation-from-mean quantity: +$ +\Delta = v_i n' - \Sigma'. +$ + +A pixel is considered strong if: +- it is above a photon/count threshold, and +- its window contains enough valid neighbours (more than 100), so the local statistics are meaningful, and +- $\Delta>0$, and +- the squared deviation exceeds a scaled variance: + $ + \Delta^2 > V\cdot T^2, + $ + where $T$ is the configured signal-to-noise threshold. + +This is equivalent to a local z-score criterion but implemented in integer arithmetic to be robust and fast. + +The test is applied in **two passes** over the image. The first is as described above. The second repeats it with every pixel found strong by the first excluded from the local background — it is treated exactly like a saturated pixel, so it contributes to no window it falls into and stays strong itself. This matters for any spot wide enough to reach into its own background box: on a single pass such a spot inflates the mean and variance it is then tested against, and its outer pixels fail the criterion. Excluding the core recovers them, so the spot is reported with its true extent rather than its brightest few pixels. Both the CPU and GPU implementations run these two passes and return the same spot list for the same frame. + +Special cases: +- saturated pixels can be forced to “strong” (useful for detecting overloaded Bragg spots), +- invalid pixels are never strong. + +### 3.2 Adaptive (self-calibrating) detection + +The local-statistics test above needs a fixed photon/count threshold whose correct value depends on the background level, which varies between datasets. The **adaptive** mode (`--adaptive-spots`; the default in `rugnux` and in the viewer for both stills and rotation data, `--no-adaptive-spots` reverts) derives that threshold from each image's own noise, per resolution ring, so no per-dataset value is needed. It admits more spots than the fixed threshold, including genuine reflections that belong to no indexed lattice; these are down-weighted rather than filtered in the per-image geometry fit (§7.4). + +Pixels are binned into the same resolution rings as the azimuthal integrator (§2). For each ring a robust background is estimated in three passes: one plain pass over all valid pixels, then two $\sigma$-clipping passes that keep only pixels within $\pm 3\sigma$ of the current ring mean (removing the Bragg peaks from the background estimate). This yields a per-ring background mean $\mu_b$ and scatter $\sigma_b$. + +The ring's detection threshold is the larger of two arms, +$ +t_b = \max\!\big(\;\mu_b + z\,\sqrt{\sigma_b^2 + \sigma_\mathrm{read}^2}\;,\;\; k_\mathrm{Poisson}(\mu_b, p)\;\big), +$ +where $k_\mathrm{Poisson}(\mu_b,p)$ is the smallest count whose Poisson$(\mu_b)$ upper tail is $\le p$. The Poisson arm is correct where the background is countable (a bright low-resolution ring gets a high threshold); the Gaussian arm — floored by a detector-level excess-noise constant $\sigma_\mathrm{read}$ — takes over on near-empty high-resolution rings, where the Poisson arm degenerates to "one photon is significant" and would flood. The operating point $p = E/N$ is set from a single portable knob $E$, the expected number of false pixels tolerated per frame (`--spot-false-pixels`, default 100), with $N$ the number of valid pixels. Because $p$ and every $\mu_b,\sigma_b$ come from the image itself, the same $E$ lands a sensible photon threshold on strong and weak datasets alike, with no per-dataset tuning. Rings too sparse to characterise (fewer than ~40 pixels) fall back to a whole-frame background. A pixel is strong when $v_i \ge t_b$ for its ring (saturated pixels are still forced strong); the strong pixels then feed the same CCL stage (§3.4). The signal-to-noise and photon-count criteria of §3.1 are not used in this mode. + +Because detection reads the pixel's ring, a pixel that falls outside the azimuthal-integration $q$ range has no ring and can never be strong: the integration range bounds what adaptive detection can see. Both upper limits are therefore optional and default to the detector itself — the azimuthal integration runs to the highest $q$ any pixel of the detector reaches (`--azim-max-q` unset), and spot finding is not clipped in resolution (`--spot-high-resolution` unset), for rotation data as well as stills. Setting either one narrows detection accordingly — appropriate for weak, high-background data, where the spots admitted at the detector edge are dominated by noise. + +**Fused GPU engine.** The per-ring reduction the adaptive threshold needs is the *same* reduction the azimuthal integrator performs. On the GPU path the two are fused into a single image pass (`AdaptiveSpotFinderGPU`): one reduction accumulates the corrected per-ring sums for the azimuthal profile (§2) *and* the raw per-ring statistics for the threshold, after which a light kernel flags the strong pixels. One GPU pass therefore replaces both the separate azimuthal-integration pass and the host-side adaptive spot-finding pass, at a small fraction of the CPU finder's cost per frame and producing the same spot list and azimuthal profile. It is enabled by default in the offline `rugnux` path, the interactive viewer and the online receiver. + +**Online.** `spot_finding_settings` in the REST API carries `adaptive_threshold` and `false_pixels_per_frame`, so the mode is reachable from the broker and from the web frontend as well as from `rugnux` and the viewer. It defaults to *off* online, unlike `rugnux` and the viewer, because the broker serves both workflows and only one of them can run it: spots are found in software only on the DECTRIS/SIMPLON path, while the JUNGFRAU and EIGER workflows find them on the FPGA at its own fixed threshold. Setting `adaptive_threshold` on those is refused with an error rather than accepted and ignored, so a detection setting that had no effect cannot be mistaken for one that did. + +### 3.3 Resolution and ice-ring handling + +Spot finding can be restricted to a resolution range $[d_\mathrm{high}, d_\mathrm{low}]$ by masking pixels outside the range. Optionally, spots in identified ice-ring regions can be tagged so that subsequent indexing/refinement may include or exclude them (see §4 and §6). + +A single per-image **ice-ring score** is derived from a radial profile: for each hexagonal-ice powder ring (see *Where the ring positions come from* below), the profile intensity at the ring is divided by a smooth background estimated from the *whole* profile — a running median of the non-ice bins, interpolated under each ring — and the strongest ring's ratio is reported (1 = no ice, $>1$ = ice above background). A whole-profile background is used rather than a couple of adjacent shoulder bins so the estimate is robust to the radial binning: at a coarse Q-spacing a local shoulder can be only ~1 bin and would double-count the ring's own edge (offline processing defaults to a fine 0.01 Å⁻¹ spacing — in $q = 2\pi/d$, like every $q$ in this document (§1.2) — `--azim-q-spacing`, so the rings are well resolved). The reported quantity is the ice *magnitude* rather than a significance: with many photons any real ice ring is statistically significant, so significance does not discriminate. + +The profile the score is read off is the **peak-excluded** one, not the plain azimuthal integration: where adaptive spot finding runs (§3.2 — the offline and viewer default), the score uses the sigma-clipped per-resolution-ring background that finder already computes for its threshold. This matters more than it sounds. A plain azimuthal profile is a per-ring *mean*, so a few strong low-resolution reflections landing in a ring's bin raise it exactly as ice would; measured over a rotation battery, that alone ranked ice-free crystals above crystals that really are iced. An ice ring is azimuthally smooth and survives the sigma clip, while Bragg peaks do not, so on the clipped profile ice-free crystals sit near 1 and crystals with confirmed ice above 2. Only where no adaptive finder ran (the FPGA workflow) does the score fall back to the plain profile. + +The radial profile sees ice only as a **smooth powder ring**. Ice in large crystallites diffracts as discrete spots, leaves the profile flat, and is invisible to the score above, so a second channel is read from the spot list itself: the spots found on the ice bands are counted against the spots found in the ice-free flanks $[w,2w)$ either side of each band, rescaled to the bands' own $q$ width (a flank landing on another ring is dropped with its width). The indicator is the ratio pooled over the run — per image the control is a handful of spots and the ratio means nothing — and it is taken before the spot-count filter, which orders ice spots last and would discard them first. The two channels barely overlap: smooth ice reads high on the profile and ~1 on the spots, textured ice the reverse, and a clean crystal ~1 on both. Both counts are stored per image (`spot_count_ice_rings`, `spot_count_ice_control`; HDF5 `/entry/MX/peakCountIceRingRes`, `/entry/MX/peakCountIceRingControl`). + +Both channels are used offline as a **gate** on ice handling: unless the run reaches `--ice-min-score` (default 1.5) on the profile or `--ice-min-spot-ratio` (default 2.0) on the spots — 0 disables a channel — ice-ring flagging and the exclusion from the scale fit (§10.10) are skipped. The fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, and more than that on a detector reaching past 1.5 Å, so handling ice on a clean crystal is a pure loss. + +**Where the ring positions come from.** The eleven bands from 3.895 to 1.522 Å are the *measured* positions of Moreau, Atakisi & Thorne (Acta Cryst D77, 2021, 540–554). That list ends at 1.522 Å by its own scope — the paper states that hexagonal ice "has 11 diffraction rings between 4 and 1.5 Å resolution", and its subject was detecting ice in deposited data rather than masking it — not because ice stops there. On a detector that reaches further, the rings it does not list are the ones left in the data. + +The eight bands below it are **calculated**, since past that paper there is nothing measured to copy. Enumerating $hkl$ from the ice Ih cell is not enough: ice Ih is $P6_3/mmc$ with oxygen on $4f$, and most of what enumeration emits is extinguished by the *oxygen sublattice* rather than by the space group — which is why (004) at 1.830 Å and (104) at 1.657 Å are missing from the measured list even though they lie inside its range and its reflection conditions allow them (for even $l$ the $4f$ structure factor carries a factor $\cos 2\pi l z$ at every $hk$, and $z \approx 1/16$ kills $l=4$ — $(104)$ along with $(004)$). Structure factors are computed instead — oxygen only, the hydrogens being half-occupancy disordered and weak to X-rays — and the lines kept are those reaching 3 % of the strongest. That rule **reproduces the measured eleven exactly**, and every line it drops inside their range computes to zero, which is what makes it trustworthy below 1.522 Å. The cell is that of Röttger *et al.* (Acta Cryst B50, 1994, 644–648). The list stops at 1.170 Å because below it the calculated real lines fall to 2–3 % while the extinct ones rise to about 1 %, and an oxygen-only calculation cannot separate them any further. + +One consequence is worth stating: the profile score is the **strongest** ring's ratio, a maximum over the bands, so a longer list can only raise it. The gate at 1.5 is therefore read against a list of this length, and lengthening it again would need the gate re-checked. + +**Rings this sample actually shows.** Hexagonal ice is the only phase whose rings can be named in +advance, and it is not the only thing that powders: a shower of microcrystals around the crystal, or a +salt out of the cryoprotectant, leaves the same textured rings at $d$-spacings no fixed list carries, +and their spots are otherwise handed to the indexer as if they were this crystal's. The rings are +therefore also **measured from the run's own pre-scan spots** — read off as what stands above the +smooth fall-off of spot density with $q$ — on every run, and reported whether or not anything acted on +them (`POWDER_*` in the results report: the ring count, the fraction of the pre-scan's spots they +hold, and their $d$-spacings). The two lists are not the same list: on one such crystal 16 of the 24 +measured rings were ice and the rest were not. A measured ring is set aside from indexing only where a +first pass that found no lattice needs it set aside (§6); the fixed ice bands above are what the ice +score, the scale fit and the resolution cut read. + +A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in $1/d$ (or $q$) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low. + +### 3.4 Connected-component labeling (CCL) + +Strong pixels are grouped into connected components (adjacent strong pixels) using a CCL algorithm. Each component yields a candidate spot with: + +- centroid $(x,y)$ (often intensity-weighted), +- pixel count (spot size), +- integrated spot intensity proxy (sum of pixel values), +- resolution $d$ at the centroid (or mean over pixels), +- and quality flags (e.g. ice-ring classification). + +Spot-level filters include minimum/maximum pixel count and resolution limits. + +**The upper bound is on how large a *round* spot may be, not on how bright one may be.** Under the self-calibrating threshold (§3.2) a component's area above the contour grows as $\sigma^2\ln(A/T)$ — without bound in the peak amplitude $A$ — so an upper bound on pixel count alone becomes an *intensity ceiling*: measured on a strongly diffracting crystal, footprints run 3 px at 30–100 counts to 50 px above 10 000, four times the slope the fixed local-box test gives, and the old bound of 50 discarded the brightest reflections on every image. The bound is therefore 200 (the same as CrystFEL `peakfinder8`'s `--max-pix-count`; XDS has no such parameter at all and guards on shape instead, with `SPOT_MAXIMUM-CENTROID`), and a component above 50 pixels must in addition **fill at least a fifth of the square its bounding box fits inside**. A Bragg reflection is round and fills about half of that square however bright it is; an ice arc, a cosmic-ray track or a lit detector row fills a fifth or less, which is what an upper bound was ever protecting against. Below 50 pixels no shape is asked for, so everything accepted before still is. The shape test is inert on every dataset it has been measured on — it exists to bound the *shape* of what the larger size bound now admits, on data carrying arcs or tracks, not because the crystals measured here needed it. The test is integer arithmetic, so the host and the GPU extractor agree by construction. + +The host implementation (`StrongPixelSet::sparseccl`) is the SparseCCL of the ACTS/traccc project: it +runs over the strong pixels sorted row-major, uses a sliding window over the previous line and a +union-find whose root is each component's lowest index. On the GPU the same labelling runs **on the +device** (`SpotExtractorGPU`): the packed strong-pixel bitmask is compacted into that same sorted +list without leaving the card, each pixel finds its at most four earlier 8-neighbours by binary +search, and a lock-free union-find with path halving labels them. Only the finished spot list — a few +hundred entries — comes back to the host, instead of the whole bitmask (2.26 MB per frame at 18 MP). +The two implementations produce the same components, in the same order, with the same pixel counts +and intensities; `tests/SpotExtractorGPUParityTest.cpp` holds them to it. The device version is also +insensitive to frame content: the host sliding window becomes quadratic when many pixels light up in +one detector line — a hot module, or a diffraction ring where it runs tangent to a row — which costs +tens to hundreds of milliseconds on such a frame, while the device version stays under a millisecond. + +### 3.5 Adaptive per-image minimum spot size + +The minimum-pixels-per-spot filter (§3.4) trades sensitivity against noise: a small value keeps faint one- or two-pixel spots — real signal on strong data, but detector noise on high-background frames — while a larger value keeps only well-formed spots. The best value is dataset-dependent, so for serial-stills indexing it can be chosen **per image** rather than fixed. The frame is indexed three times, at min-pix 3, 2 and 1, and the setting that maximises + +$$ \frac{n_\mathrm{indexed}^2}{n_\mathrm{total}} \quad\text{(indexed-spot count weighted by indexed fraction)} $$ + +is kept; the frame is then integrated once at that min-pix. The fraction factor discounts the extra spots a smaller min-pix admits *unless the lattice actually explains them*, so strong frames keep their real weak spots (extending resolution) while noise-flooded frames stay strict. Because min-pix filters the connected components *after* detection, strong-pixel detection AND the connected-component labelling both run **once** per frame, and the three attempts only repeat the spot-level filter; the azimuthal profile is the one that single detection pass computed. The winning attempt's spot list is kept rather than re-extracted, so the frame that is integrated is exactly the frame that was scored. This is a **stills-only, indexing-path** option — rotation indexing builds one global lattice from all frames and keeps a fixed min-pix. In `rugnux` it is the default; giving an explicit `--min-pix-per-spot` pins a fixed value instead. + +### 3.6 Predicting the resolution the merged data will reach + +A per-image **resolution estimate** is read off the finished spot list. It predicts how far the *merged* data will reach, not how far the furthest spot on this image lies. Each non-ice spot is weighted by $\sqrt{I}$ — the intensity is a summed photon count, so $\sqrt{I}$ is its Poisson significance — the $1/d^2$ is found beyond which a fraction $f=0.30$ of that weight lies, and the estimate is that resolution taken $2.25\times$ further in $1/d$. It is deliberately **not** limited to what the detector records: the quantile sits in the middle of the fall-off, well inside the recorded range, so it goes on measuring the crystal where the detector stops before the diffraction does, and on such a run it reads finer than the detector corner. The dataset value is the median over images. + +Both constants carry a mechanism. A quantile from the middle of the distribution measures the *shape* of the fall-off, which is the crystal's own $\exp(-B/2d^{2})$, where the extreme end of it measures where detection stops — a threshold that moves with the exposure and with how many reflections the unit cell puts on a frame. And merging averages many observations of each reflection, so intensities go on being measurable a fixed factor in $1/d$ past the point at which one image's spot finder still detects them; that factor is the $2.25$. Both are calibrated on rotation data against the resolution at which per-shell CC1/2 falls through 0.30, and the estimate is good to about 0.2 Å there. It is a prediction and not a measurement of what a run achieved: nothing downstream is cut on it, and it is reported alone (Rugnux `SPOT_RESOLUTION_ESTIMATE`, and per image in the stream, the plots and HDF5). diff --git a/_sources/CPU_DATA_ANALYSIS_INDEXING.md.txt b/_sources/CPU_DATA_ANALYSIS_INDEXING.md.txt new file mode 100644 index 000000000..689902f0b --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS_INDEXING.md.txt @@ -0,0 +1,358 @@ +# Data analysis: indexing and geometry (§4–§7) + +Part of the [CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md); the section numbers are continuous across its four parts. + +```{contents} On this page +:local: +:depth: 2 +``` + + +## 4. Indexing overview + +Indexing maps observed reciprocal-space vectors $\mathbf{s}_i$ to a lattice such that: +$ +\mathbf{s}_i \approx h_i\mathbf{a}^* + k_i\mathbf{b}^* + l_i\mathbf{c}^*, +$ +with integer $(h_i,k_i,l_i)$. + +Jungfraujoch supports two complementary indexing strategies: + +1. **FFT-based indexing** (Rossmann-type): does not require an a priori unit cell; suitable for unknown samples. +2. **Fast-feedback indexing** (TORO-like): requires an approximate unit cell; optimized for speed and feedback. + +Both feed into a common robust refinement/selection stage which maximizes the number of inliers under an indexing tolerance, and which can return **more than one lattice** per image (multi-lattice indexing; see §5.4). + +### 4.1 Indexed-spot decision (inlier test) + +Given a trial lattice with direct basis vectors $\mathbf{a},\mathbf{b},\mathbf{c}$ (used here as reciprocal-space dot-test vectors), fractional indices are estimated by: +$ +h_f = \mathbf{s}\cdot\mathbf{a},\quad +k_f = \mathbf{s}\cdot\mathbf{b},\quad +l_f = \mathbf{s}\cdot\mathbf{c}. +$ +Let $(h,k,l)=(\mathrm{round}(h_f),\mathrm{round}(k_f),\mathrm{round}(l_f))$ and define the fractional residual: +$ +\delta^2 = (h_f-h)^2 + (k_f-k)^2 + (l_f-l)^2. +$ +A spot is indexed if $\delta^2 < \tau^2$, where $\tau$ is the configured tolerance. + +For indexed spots, the reciprocal lattice point $\mathbf{p} = h\mathbf{a}^*+k\mathbf{b}^*+l\mathbf{c}^*$ is used to compute $\Delta_\mathrm{Ewald}(\mathbf{p})$ (stored as a diagnostic and later used in profile-radius estimation). + +A frame is taken to be this crystal's when at least a fraction $g = 0.20$ of its in-resolution, non-ice spots index. On rotation data that decision is what admits the frame to integration, so its denominator matters: every spot handed to it that is not a reflection of this crystal argues against the frame. Where it cannot do that job — a lattice whose pooled spots fall below $g$ and which fewer than half the validation frames clear — every frame is integrated instead: the frames that clear $g$ are then only the upper tail of the same sparse population, not the frames the crystal was in, and admitting them alone dropped most of a small-molecule sweep and its completeness with it. + +That test decides a **frame**. Whether a rotation run has a lattice **at all** is decided on the spots instead. A frame count comes from serial crystallography, where each image is its own experiment; a rotation sweep is one crystal and one orientation matrix, its frames are not independent of each other, and what such a count mostly measures is how many spots happen to land on a frame — a sweep carrying four spots an image cannot reach a six-spot bar on three frames in four however right the lattice is. The refusal therefore compares the fraction of *all* spots in the sampled frames that the lattice explains against what the same lattice explains when each frame's spots are put at **another frame's angle**: same lattice, same spots, same detector, same refinement, with only the claim that these spots were seen at *these* angles removed. That difference is the evidence, and it carries no spots-per-frame number anywhere, so nothing has to be chosen for a crystal that diffracts weakly. Measured over a hundred datasets the permuted level never exceeds 2.6 % and the smallest true margin is seventeen points. `--min-indexed-spots` (default 6, floor 4 — four is where a lattice stops being fitted by any three spots) still sets the reported indexing rate and the count the first pass *ranks* candidate lattices by; every rescue and every arbiter still counts frames. + +### 4.2 The spot budget + +Only the strongest `--max-spots` spots of an image are kept (`FilterSpotsByCount`), and that budget therefore sets the denominator above. Detections are not all reflections — background structure, unlisted ice and detector artefacts are found too — so a budget deeper than an image's reflections makes the test above a measurement of the background rather than of the crystal, and a *larger* budget can integrate *fewer* images. + +rugnux measures the budget instead of fixing it. With the sweep's lattice in hand, the first pass tallies the spots of a sample of frames by their rank in the intensity-ordered list: how many images carried a spot at that rank, and on how many of them it indexed. Weighting each indexed spot by $1-g$ and each unindexed one by $-g$ — the same weighting the frame test applies to the list as a whole — the running sum over ranks + +$$ E(N) = n_\mathrm{indexed}(N) - g\, n_\mathrm{counted}(N) $$ + +rises exactly while the spots at that depth lie on the lattice more often than $g$, and falls after. The budget is $\arg\max_N E(N)$. Its meaning is "as deep into the list as the image is still showing reflections of this crystal": deeper spots cannot help the frame test and can only push a frame towards rejection. On crystals whose spot lists are reflections all the way down the maximum is at the end of the list and the budget is unchanged. + +The peak has to be one. Under the null — the spots lie on the lattice at the same rate at every depth — $E$ is a driftless random walk in the counted spots, with per-spot variance $g(1-g)$, and the maximum of such a walk is positive whatever the data; an $\arg\max$ taken on its own would shorten every dataset, including one with nothing to shorten. What the budget acts on is the fall from the peak to the end of the list, $E(N^*) - E(L)$, which is the maximum of the same walk read backwards; by the reflection principle its null law is $P(\mathrm{fall} > z\sqrt{g(1-g)T}) = 2(1-\Phi(z))$ over $T$ counted spots in all, so the search over ranks is already accounted for and no further multiple-comparison correction applies. The budget is taken only where the fall clears that bar at $z = 3.29$, one false shortening in a thousand measurements; otherwise the whole list is kept. + +--- + +## 5. FFT indexing (unknown unit cell) + +FFT indexing follows a classical approach: detect dominant periodicities by projecting reciprocal-space points onto many directions and Fourier transforming the resulting 1D histograms. + +### 5.1 Directional projections and histograms + +Choose a set of unit vectors $\{\mathbf{u}_d\}$ on a half-sphere (a near-uniform distribution generated via a golden-angle construction). For each direction $d$, form a histogram in the scalar projection: +$ +t_{id} = \left|\mathbf{u}_d\cdot \mathbf{s}_i\right|. +$ + +Bin width is chosen approximately as: +$ +\Delta t \approx \frac{1}{2 L_\mathrm{max}}, +$ +where $L_\mathrm{max}$ is the maximum expected real-space unit-cell edge (Å). The histogram extent is tied to the maximum $q$ used (set by a high-resolution cutoff for indexing). + +### 5.2 FFT peak picking and candidate vectors + +For each direction, the FFT magnitude spectrum is computed; peaks correspond to periodicities along $\mathbf{u}_d$. Each direction yields a candidate real-space length $L$ chosen **not** by raw magnitude but by **maximum prominence above a running-mean local background** (subtracting the broad low-frequency envelope that otherwise dominates on weak or pink-beam frames), subject to $L\ge L_\mathrm{min}$. + +The running-mean background window keeps a **constant width** and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel's running sum is still valid. + +The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. + +Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. + +A collinearity filter removes nearly parallel vectors (e.g. within 5°) and attempts to resolve harmonic ambiguity: shorter “fundamental” vectors may be preferred over longer harmonics if their peak magnitude is sufficiently strong relative to the dominant peak. + +### 5.3 Lattice reduction and cell candidates + +Triples of candidate vectors are combined to form candidate bases $(\mathbf{A},\mathbf{B},\mathbf{C})$, each reduced to its **Niggli-reduced cell** (Gruber-vector reduction) before comparison, and filtered by allowed length and angle ranges. Two passes are run: a standard pass forms shortest-vector triples from the ~30 strongest filtered directions; if the best cell then indexes fewer than half the spots, a **widened fallback** anchors the two shortest axes and lets the third range over up to ~60 candidate vectors (deduplicated by Niggli cell), catching large, elongated or superstructure cells the first pass misses. + +A triple whose three vectors are **coplanar** is rejected before refinement. The length and angle filters cannot see it — any flat combination satisfies them — and a cell that flat has a metric determinant small enough for `float` to get its *sign* wrong, after which the guard against a negative argument to the square root places $\mathbf{c}$ in the $\mathbf{a}$-$\mathbf{b}$ plane, the reciprocal volume diverges and the solver reports a not-a-number Jacobian. The test is the volume fraction $|V|/(|\mathbf{a}||\mathbf{b}||\mathbf{c}|)$, which must reach 0.02 — about 1.1° off flat, well below the flattest genuine candidate observed and far above where `float` loses the sign — and it is applied both where triples are produced and at the optimizer's entry points. + +A shortlist **confined to one plane** cannot close a cell at all, and the row it is missing is the plane normal. That is detected from the eigenvalue ratio of the shortlisted directions' scatter matrix, and one further transform is then spent with the same direction count inside a narrow cap about the normal. More directions do not substitute for it: at the exact true direction a very long axis can still rank far below the shortlist cut, so for **this** rescue the obstacle is the ranking rather than the sampling, and a denser grid costs several times the device memory for the same answer. + +Sampling has a limit of its own, and it binds well before the 1200 Å the accepted range for `fft_max_unit_cell_A` admits (§5.2). A direction off a real-space axis of length $a$ by an angle $\theta$ smears each projected lattice plane by about $\theta/d_\mathrm{min}$ in the projection, so the planes (spacing $1/a$) stay resolved only while $\theta \lesssim d_\mathrm{min}/(2a)$. The shipped grid of 16384 directions puts the nearest one within about 0.6° of any axis, which satisfies that bound only up to roughly 120–150 Å at typical indexing resolutions; a longer axis is not refused but returned as a plausible sub-cell or harmonic. Raising the maximum cell alone therefore does not extend the reach — the direction grid has to resolve the axis before the histogram can represent it. + +### 5.4 Robust refinement and best-cell selection + +Candidate bases are refined against observed spots using an iterative inlier‑focused least‑squares procedure (trimmed/contracting threshold). Candidates are then ranked: +1. more indexed spots wins — **unless** two candidates index within ~10 % of each other, in which case +2. the **smaller-volume** cell is preferred (when the volumes differ by more than ~5 %), avoiding a doubled supercell, then +3. the smaller refinement score, then the spot count again. + +Selection is **not limited to a single lattice**: after the best cell is accepted, further lattices are added as separate crystals provided fewer than ~40 % of their indexed spots overlap an already-accepted lattice (up to two extra by default), so split or multi-lattice crystals are indexed rather than discarded. + +An optional reference unit cell (if supplied) restricts acceptance to cells within a relative distance tolerance in edge lengths (permutation-invariant). + +### 5.5 Spindle alignment: the part of the blind cone symmetry cannot repair + +A sweep about the spindle $\hat{\mathbf{e}}$ never brings a reciprocal point closer than +$\theta_\mathrm{max}=\arcsin(\lambda/2d)$ to the axis onto the Ewald sphere, so a double cone of +half-angle $\theta_\mathrm{max}$ is missing from every resolution shell — each shell losing its own +$1-\cos\theta(d)$ — however long the sweep runs. Crystal symmetry normally repairs that loss by +mapping the cone onto measured territory. It fails to when a symmetry axis lies inside the cone (the +cone maps onto itself) — and, for a **2-fold**, equally when the axis is perpendicular to the +spindle, because the diad carries the cone onto its opposite lobe, which the sweep leaves just as +unmeasured. Friedel never helps: the cone is double-sided. The loss is a coherent cap rather than a +scatter of absences, so it costs a map more than the same percentage lost at random. + +The per-image score asks how much of that cone the frame's own orientation makes unrecoverable. +The crystal's short lattice rows are read off the FFT row shortlist of §5.2 (a symmetry axis is +always a lattice row, and usually among the short ones), and each plausible direction is scored as +if it carried a lone 2-fold: + +$$ \text{spindle blind fraction} = \frac{2}{\pi}\left(\arccos x - x\sqrt{1-x^{2}}\right), +\qquad x = \min(\beta,\,90^\circ-\beta)\,/\,\theta_\mathrm{max}, $$ + +where $\beta$ is the direction's miss-angle from the spindle. The **fold** of $\beta$ about +$45^\circ$ is the diad geometry above: both ends of the range are the bad case, and the closed form +reproduces a Monte Carlo of the true double-cone self-overlap to 0.004 at +$\theta_\mathrm{max}=15^\circ$ and 0.008 at $25^\circ$ (past $45^\circ$ it under-reports, by 0.05 +at $50^\circ$). $\theta_\mathrm{max}$ is taken from the **geometric** resolution of the setup — the +detector corner at the recorded distance and wavelength — an upper bound on any sweep collected +without moving the detector; the still's own spot resolution would understate the cone on exactly the +weak frames that mislead. The worst direction wins, and the directions scored are the strong +in-window rows **and the normals of their pairs** — the normal to two lattice rows is itself a +reciprocal-lattice row and a symmetry axis is parallel in both bases, so a lone 2-fold on an axis far +beyond the length window (a long monoclinic unique axis) is still seen by direction: measured on a +synthetic lone-diad crystal with a 300 Å unique axis, the fraction of severe mounts reported severe +rises from 0.60 to 1.00 with the pair normals, at no extra engagement on that class's harmless +mounts. Nothing about the goniometer enters: the number describes the problem and leaves the remedy — +a second sweep, a reorientation — to the beamline. + +**This is a worst-case bound under an assumption of no symmetry, not an estimate.** A still cannot +know the point group, so the nearest plausible row is scored as a lone 2-fold. An axis of order +$\geq 3$ perpendicular to the spindle in fact **fully repairs** the cone (measured unrepaired +fraction 0.000 for orders 3, 4 and 6, against 1.000 for a diad), which a still cannot see, so the +bound is deliberately pessimistic on higher-symmetry crystals — that is the intended trade, because +the number exists as a **trigger** for beamline automation, not as a physical quantity a user +interprets. + +**Trigger states.** The stored quantity is the continuous score; automation reads it through three +fixed states with nothing to tune (`SpindleTrigger` in `SpindleBlindFraction.h`): **engage** at +score ≥ 0.5, **don't engage** below, and **cannot say** when there is no value at all — too few +spots, no shortlist, the consistency guard refused, the path never computed one. **Automation must +treat CANNOT SAY as ENGAGE**: the error costs are asymmetric — a false negative is unrecoverable +(one sweep is collected and the data stay short forever) while a false positive costs minutes of +beamtime. Every transport keeps absence distinguishable from a measured zero (an absent CBOR key, a +NaN in the HDF5 arrays, an absent optional after read-back). The 0.5 threshold is geometry, not +tuning: the score is monotone in the folded miss-angle, so a threshold is a fold-angle gate, and 0.5 +gates at $\min(\beta,90^\circ-\beta) \le 0.404\,\theta_\mathrm{max}$; engaging on any overlap at +all would gate at the cone edge, whose perpendicular band alone spans $\sin\theta_\mathrm{max}$ of +orientation space per row (26 % at $15^\circ$) and unions over a frame's rows to well over half of +all mountings — a trigger that always fires decides nothing. + +**Reach and honest rates.** The score needs 60 spots (calibrated per crystal — 22 independent +mounts — misses triple below it); below that, a frame that still indexed answers from the winning +lattice's shortest rows, and otherwise the state is *cannot say*. Because the bound is pessimistic +by design, it engages on a substantial share of harmless mountings: a single strong row's +perpendicular band alone covers ~11 % of orientation space at the severe level +($\theta_\mathrm{max}=15^\circ$), and the union over a frame's rows and pair normals reaches +roughly a quarter to three quarters of random mountings depending on cone width and row count +(measured 0.74 on a generic triclinic cell at $15^\circ$ via the lattice path). That is accepted: +the cheap error is the extra wedge. An earlier figure of ~1 % false alarms (AUC 0.948) came from a +null of five *decoy directions per frame* — it shows the estimator does not hallucinate rows near +arbitrary directions, which is worth knowing, but it is **not** a false-alarm rate over harmless +mountings, which geometry forbids to be that low. + +**Offline, the guessing stops.** Once `rugnux` has merged a rotation run it holds the measured +point group and the exact indexed orientation, and the run-level number is computed exactly instead: +the group's proper rotations are applied to the blind double cone in the crystal's actual +orientation, and the fraction of unique reflections no operator can recover is reported as +`SPINDLE_LOST_UNIQUE_FRACTION` in the processing report and `/entry/MX/spindleLostUniqueFraction` in +the master file ([the results report](RUGNUX_REPORT.md)). That number clears or convicts a mounting the per-image +bound can only be pessimistic about: a dihedral crystal with an in-plane diad on the spindle, or any +cubic crystal in any orientation, loses nothing at all. + +--- + +## 6. Bravais lattice / centering inference (“lattice search”) + +If the space group is supplied by the user, its lattice constraints are assumed for refinement and subsequent processing. + +If not, Jungfraujoch attempts to infer the most plausible Bravais lattice type from the metric tensor after Niggli reduction: + +1. **Niggli reduction** is performed to obtain a reduced cell in $G^6$ representation (Gruber vector). +2. The reduced cell is compared against a list of Niggli classes corresponding to Bravais lattices and centerings. +3. The highest-symmetry class that matches within tolerances is selected (relative metric tolerance and angular tolerance). The list is walked in order of decreasing symmetry and the first class that fits is taken, so a class that only just fits can pre-empt a lower-symmetry one that fits exactly. + +The output includes: +- a conventional cell, +- crystal system (triclinic, monoclinic, …), +- centering symbol (one of $P, C, I, F, R$; the $A/B$ variants are not emitted here — they are handled only later as prediction absences, §8.4). + +This stage provides centering information used for systematic absences in prediction (§8.4) and for reporting. + +**A metric symmetry has to earn itself.** The class is chosen from the *unrefined* candidate against a fixed angular tolerance (3°), so a lattice that is pseudo-symmetric to a few tenths of a degree is promoted a class too far — and the constraint then snaps a real angle to the ideal one, which throws nearly every reflection of every frame outside tolerance. Measured on a monoclinic crystal pseudo-C-orthorhombic to 0.42°, the promoted cell indexed 2 of 60 frames where its own primitive cell indexed 39: the same lattice, $\mathbf{b}_{oC}=-(\mathbf{a}+2\mathbf{c})$, at exactly twice the volume. A more accurate candidate is more likely to be promoted, not less: a run escapes the promotion only when the raw candidate misses the tolerance window. + +The lookup can also land *short*: near the Niggli type-I/type-II boundary the character is decided by the last digits of the refined cell, and the class it names caps which point groups the space-group search enumerates. The metric-symmetry re-ask of §13.1 covers this — rotations the named class has no room for are put to the intensities directly, and where the metric group exceeds the class's holohedry the merge is reindexed into the metric cell and the search re-run there. + +The rotation first pass refines the **twelve** best candidate lattices rather than four. The pre-refinement indexed fraction is an unreliable ranking, so a correct cell can sit below several degenerate ones and never be refined at all. + +For rotation data the first pass therefore refines the constrained cell *and* an unconstrained (triclinic) one from the same spots — which it finds itself, over a sample spread across the sweep, rather than reading what the acquisition wrote — and settles the two on how many of a fixed set of validation frames each actually indexes. The bar is a clear majority, not a margin. An unconstrained refinement holds no cell parameter fixed, so it can only index at least as many frames as the constrained one, and on genuine symmetry it indexes a few more — a percentage margin therefore demotes real lattices (measured: a genuine $I$-centred orthorhombic to $P1$). Only a constrained cell that fails outright while its unconstrained cell works distinguishes a false promotion. The intensities settle the space group later regardless (§13). + +Two further hypotheses are weighed at the same point, both by default. Where two first-pass answers +have primitive volumes in a small integer ratio (2–4×) and tie on the validation frames, the frame +count has saturated — a spurious axis multiple indexes every frame its true sub-cell does — so the +tie is settled at the granularity that does not saturate: **which cell accounts for more of the +validation frames' spots**. That comparison leans toward the smaller cell by construction (indexing +is a fractional-Miller test, so multiplying an axis by $n$ multiplies that axis's residual by $n$), +and only a reflection class the larger cell genuinely adds — a real superstructure's satellite rows +— can pay for the loss; the occupancy of that added class is computed and logged beside the decision, +deliberately without a threshold, but the spot count is what decides. Separately, the 10 Å floor of +§5.2 is applied to the FFT's per-direction peak search, so a lattice row whose true repeat is below +it is reported at its first harmonic and the cell assembled from those harmonics is an exact integer +supercell of the true one; a **second first-pass hypothesis with the floor lowered to 5 Å** therefore +also runs (rotation only, and not when a cell was given — `-C` already lowers the floor to cover it), +and its answer is adopted only when it is an integer sub-cell of the standing one and ties or beats +it on the validation frames. Everywhere else the standard pass's answer stands. + +**A first pass that ends with no usable lattice tries a leaner, shallower one.** A crystal sitting in +a crystalline powder floods the pass with spots that are not its own — measured, 2112 spots a frame +against the 80 a clean crystal on the same beamline gives — and the FFT then takes its cell out of the +powder shells. So the pass is retried over **how much of each frame it reads**: the strongest 30, 80, +200 or all spots an image, each at the file's resolution and at the resolutions a quarter and a half +of this sample's own spots lie coarser than, with the rings measured in §3.3 set aside where there are +any. The rungs are decided late, on the validation-frame count — the way the rotation-axis sign +already is — and one is adopted only on a win of a sixth of the frames. The same ladder is asked +inside the beam-centre check of §1.4, where a centre that is wrong and a spot list that is too deep +otherwise hide each other. + +The frame count cannot arbitrate an axis harmonic, though: a cell twice as long must place every spot +twice as accurately to score the same, and across rungs the bias compounds, since the leanest and +shallowest rung is both the one a halved axis scores best on and the one a smaller cell is easiest on. +The winning rung is therefore put through the same harmonic arbiter as the hypotheses above — which of +two cells accounts for more of the validation frames' spots — against every rung that cleared the bar +and whose primitive volume is a near-integer multiple of the winner's. A winner that loses that +question is a sub-multiple, and the ladder then adopts nothing rather than promoting the rival, which +on the crystal this was measured on is a multiple of the true axis in its own right. + +**Note.** In ambiguous or special cases, forcing space group to $P1$ (no symmetry assumptions) is recommended. + +--- + +## 7. Geometry and lattice refinement + +Refinement adjusts experimental geometry and crystal parameters to minimize discrepancies between observed spot reciprocal vectors and those predicted by a lattice model with integer indices. + +### 7.1 Parameterization + +The refinement jointly optimizes, depending on mode and constraints: + +- beam center $(x_\mathrm{beam}, y_\mathrm{beam})$, +- detector distance $D$, +- detector tilt angles (two-angle model; third rotation often held at 0), +- rotation axis direction (for rotation datasets), +- crystal orientation (a global rotation), +- unit-cell parameters, with constraints determined by inferred crystal system. + +The detector distance is not refined against one crystal's spots at all: the positional residual leaves it degenerate with the cell scale, so it is fitted elsewhere - by the rotation post-refinement, which frees it alongside the whole crystal and adds the distance-independent rocking-angle residual that breaks the degeneracy, and by the stills `--refine-geometry` bundle. Per image, the beam centre and the crystal orientation are refined, and the unit cell as well for stills. The first-pass rotation indexing refines the detector tilt and the rotation-axis direction too, against the spots accumulated across the sweep; everywhere else both are held fixed, because on a single crystal a tilt is absorbed almost exactly by the crystal orientation. A lighter **orientation-only** mode refines just the crystal orientation, for stills whose geometry is already trusted. It carries a weak small-rotation prior penalising the whole angle-axis vector (all three components, at a low weight); what it is there for is the poorly-determined out-of-plane component, which is the one the data barely constrain. + +For higher symmetries, constraints are enforced, e.g. +- cubic: $a=b=c,\ \alpha=\beta=\gamma=90^\circ$, +- tetragonal: $a=b$, +- hexagonal: $a=b,\ \gamma=120^\circ$, +- monoclinic (unique axis $b$): $\alpha=\gamma=90^\circ$, $\beta$ refined. + +### 7.2 Residuals and objective + +For each indexed spot assigned integer $(h,k,l)$, compute: + +- observed reciprocal vector $\mathbf{s}_\mathrm{obs}$ from its detector position and current geometry, +- predicted reciprocal vector $\mathbf{s}_\mathrm{pred}(h,k,l;\ \text{lattice params})$. + +Residual is: +$ +\mathbf{r} = \mathbf{s}_\mathrm{obs} - \mathbf{s}_\mathrm{pred}. +$ + +A non-linear least squares solver minimizes $\sum \|\mathbf{r}\|^2$ over all selected inlier spots. + +### 7.3 Rotation datasets: bringing observations to a common reference frame + +For oscillation/rotation data, each image corresponds to a rotation angle $\phi$ about an axis $\mathbf{m}_2$. Observed reciprocal vectors are rotated “back to start” so that all images are refined in a single reference crystal frame: +$ +\mathbf{s}_\mathrm{obs,ref} = R(\phi)\,\mathbf{s}_\mathrm{obs}, +$ +where $R(\phi)$ is the rotation by $+\phi$ about the goniometer axis **as stored in the file**. The sign is a convention and it is load-bearing: rotating the observations forward by $+\phi$ means the crystal itself turns by $-\phi$ about that stored axis, i.e. $R(\phi)$ is the *inverse* of the crystal's own rotation from the reference orientation to frame $\phi$. The same convention is why the unmerged-MTZ batch headers and the XDS geometry echo carry the axis **negated** relative to the input file ([Rugnux ▸ the unmerged export](RUGNUX_INTEGRATION.md#the-unmerged-export)) — a reimplementation that takes $R(\phi)$ as the crystal rotation must use $R(-\phi)$ here instead. The angle $\phi$ is taken at the centre of each frame's oscillation (the frame angle plus half the oscillation width). + +### 7.4 Multi-stage tightening of inlier tolerance + +Refinement is performed in stages with decreasing acceptance tolerance for including reflections (three stages, indexing tolerance $0.3\to0.2\to0.1$), which stabilizes convergence when starting from imperfect indexing and approximate geometry. + +The loose first stage necessarily admits some spots that are not reflections of this lattice — the fraction of *randomly* placed spots inside a fractional-Miller tolerance $t$ is $\tfrac{4}{3}\pi t^3$, i.e. 11 % at $t=0.3$ — and an unweighted fit lets them pull the orientation. Each residual is therefore weighted by how strong its spot is **for its resolution**: the frame's spots are cut into equal-count resolution shells and each intensity is divided by its shell median, mapped to $w^2=r/(1+r)$. The shell normalisation is what makes this safe — genuine high-resolution spots are legitimately weaker and carry the cell and distance information, so an un-normalised intensity weight would suppress exactly the spots the fit needs. The weight is a property of the spot and never of the current residual, so it does not depend on how far the geometry is from convergence. + +**The rotation chain commits its best round, not the round it stops on.** After the winning candidate is selected, the refinement is run again — solve, re-accumulate the reciprocal-space cloud under the refined geometry, solve again — up to twenty times, and the loop stops on a test of the detector-tilt step. The chain is a trajectory and its last point is not always its best one: every solve ends by fitting only the spots inside its **tightest** gate, so a cell with a direction the data barely constrain (which is what a free cell whose metric is near a Bravais class has) can slide along it, pulling a core of spots tighter while the periphery falls out of the fit altogether. Measured on such a chain, the spots inside the tight gate rise over seventeen rounds while the spots inside the widest gate peak at round three and fall away — and round three is the round that merges at ISa 11.0 against 6.3 and $R_\mathrm{meas}$ 0.148 against 0.213. Nothing the run consulted could see it: indexed fraction, validation frames, validation spots and the tight-gate count all prefer the overfitted end. + +So every round is scored on the **widest** gate — the population the first pass selects on and the last pass does not fit, which makes it the one a converged solve is not optimising — and the best-scoring round is committed. Two conditions keep that from acting on noise. The score is a *count* of spots, so a lead of fewer than $\sqrt{\text{count}}$ of them leaves the last round standing. And the round taken has to be the less distorted lattice as well as the better-fitting one: the lattice search (§6) is re-asked every round to **measure** how far the cell sits from the ideal metric of the class it matches (imposing that class is measured fatal — the snap puts almost everything outside the refinement's own gate), and an earlier round is taken only when it matched the *same* class and sits closer to it. Same class is a precondition and not a precaution: the deviation is a fraction of whichever class's tolerance admitted it, so two classes' deviations are not the same quantity, and a round that matched no class reports zero, which means "nothing was asserted" rather than "undistorted". A chain that has settled scores its rounds within a spot or two of each other and a symmetry-constrained solve holds its distortion at zero throughout, so the rule fires on neither: measured over 914 chains, an earlier round scores higher on 44 % of them and the committed round changes on 1 dataset in 54. + +**A tilt no mounting can have has to prove itself.** The detector tilt is refined freely because on a sweep whose spots reach far enough in $2\theta$ it is a measurement, and restraining it costs those crystals resolution (measured: 0.13–0.16 Å and up to a quarter of ISa on the crystals whose fitted tilt is largest). What makes it a measurement is the *keystone* — a tilted plane puts one side of the detector nearer and the other further, so the spots move by an amount that grows with their distance from the beam — and a first pass made of spots reaching a few degrees of $2\theta$ sees a keystone of a pixel or two at most. To such a fit a tilt is a whole-pattern shift the beam centre imitates exactly, its size is whatever the centroids' own systematics happen to prefer, and the value it commits then mispredicts the far corner of the detector by tens of pixels against an integration disc of a few: a first pass seeded to $2\theta = 5°$ committed 2.5° and the run collapsed from 2.8 Å to 6.4 Å. For a tilt of a few tenths of a degree nothing in the spots says whether it is real — the held-out positional residual, the rocking angles and a re-fit at the header tilt were all measured unable to, at the same insignificance on crystals whose tilt is real and on the one whose tilt was the artefact — so below what a mounting can be off square by the fit is trusted as before: a detector is mounted square to the beam to a fraction of a degree, and over 211 datasets the fitted tilt left the file's by more than 0.56° on one. A chain that has walked more than 1° from the tilt it started at has either measured nothing or found a detector the file misdescribes (that one: a $2\theta$ arm swung out 12.8° that the file records as square), and at that size the spots *do* tell the two apart, because a real tilt of degrees has a keystone of tens of pixels over the spots the fit is made of and an artefact has none. So the candidate is refined again from where it started with the tilt held there — the beam centre takes the shift the tilt is equivalent to — and the two are judged as the rounds of one chain are, on the spots each indexes inside the wide gate: the walked tilt stands only when it leads by more than the count's own noise. Held, not bounded — a box the fit lands on is the same wrong answer at a smaller size. The log says what the walked tilt would have moved the far corner by and how the two counts came out; where it was refused, the report's `REFINED_DETECTOR_TILT` is the tilt the pass started at and `REFUSED_DETECTOR_TILT` the tilt the fit had walked to. + +### 7.5 Rotation geometry post-refinement (two-pass) + +The refinement above (§7.2) runs per image against that image's spots. For rotation data an additional **post-refinement** (on by default; `--rotation-no-postrefine` disables it) improves the detector distance, beam centre and crystal cell/axis using **all** frames at once, then re-integrates: + +1. **Pass 1** integrates, scales and merges at the header geometry. +2. From pass-1's integrated reflections, the crystal and the detector are refined together over all frames (Ceres, robust loss) in **one joint fit**, against both residuals at once: + - the **positional** detector↔reciprocal residual at each partial's observed spot, and + - a distance-independent **Ewald excitation** residual at each reflection's observed rocking centroid $\phi_\mathrm{obs}$. + + Free: the crystal orientation, the unit cell (every parameter the crystal system leaves free, not one overall scale), the goniometer-axis direction, the detector distance and the beam centre. The positional residual on its own *is* degenerate with the cell scale — that is why this used to be split into a cell-scale step and a distance step — but the excitation residual does not involve the detector at all, so it fixes the absolute size of the reciprocal lattice and breaks the degeneracy inside the same problem. Splitting it instead cost accuracy twice over: pass 1 frees the whole lattice against a frozen distance, so the distortion it absorbs is *anisotropic* and no single scale can undo it; and whatever bias is left in that scale goes straight into the distance, which is only ever determined relative to the cell. + + The fit is **cross-validated** on a deterministic split of the *reflections* (an avalanche-mixed $hkl$ hash, not a frame split and not an $h+k+l$ parity, which would collide with a centering condition and leave the held-out half empty): fitted on one half, committed only if it lowers the held-out residual by more than that residual's own noise (the standard errors of the two held-out means combined, the bar a round of the geometry walk has to clear) and its two families agree: the positional residual has to fall by more than its own noise, because the positions are the only evidence of the detector geometry a commit hands the next pass, and the excitation residual must not rise by more than its own noise, because it is the only evidence of the cell scale and the positional values outnumber it about three to one. That noise is the *paired* standard error — both geometries are read on the same held-out reflections, so what a change has to beat is the scatter of the change each reflection sees; bare signs stood here, and were a coin toss wherever a family did not move — and the move stays inside its bounds: every free cell angle within 1° and the beam centre within 15 px of the nearest centre anything already believes. + + The **distance and the cell lengths are bounded one step at a time, not as a whole**. One per cent was once a cap on the entire move, and as a cap it was the opposite of its job — a header is most worth correcting when it is most wrong, and a geometry genuinely several per cent out could never be reached (measured: a refused fit of 310.000 → 305.692 mm whose cell landed within 0.06 % of the deposited one). It is a **trust region** instead. The first solve is asked in the wide box around nominal exactly as before, so a fit that settles within one step commits unchanged; a fit that wants more is re-fitted as a *walk* of one-per-cent steps, each seeded where the last arrived and each required to lower the held-out residual, stopping where a step stops paying. A walk that uses every step it is allowed has not settled — it stopped because it ran out of steps, not because it arrived — and is refused, which is the runaway the cap stood in for, tested where it can be seen. A move of more than one step is additionally **ratified by re-indexing at where it arrived**: that is what separates the failure the cap was really aimed at (a second lattice, whose spots bias every cross-validation fold identically) from a wrong header, since a second lattice does not index better at the new geometry and a real distance error does. + + The geometry the run commits is re-fitted on all the reflections once the held-out half has approved it — a walk from where it arrived, one step wide; everything else from nominal in the wide box — and the bounds are asked again of that fit rather than only of the half that earned it. A move outside them leaves the geometry at nominal, as every other refusal does, and the refusal says so in the report (`POSTREFINE_REFUSED`, with what the fit wanted) rather than passing silently. Detector tilt is held fixed, being gauge-coupled to the crystal orientation on a single crystal. + + **Whether the data determine the distance at all is asked, not assumed.** At a detector far enough away that no reflection reaches more than a few degrees of $2\theta$, a longer distance and a larger cell move every spot the same way to first order — the difference is of order $\sin^2\theta$ of the spot's own position, about 0.4 px rms per per cent of distance over a 2M detector at 820 mm against 2–3 px at the distances a crystal is usually collected at. The joint fit then finds a distance/cell pair that fits its own spot positions a little better than the header, commits it, and the pass re-integrated there finds the next pair: a walk along the degenerate direction that the realised residual never ratifies (measured on such a sweep: a header at 820 mm walked to 846 mm with the cell 3.3 % too large, the realised held-out residual flat at every round). So the same fit is asked once more with the distance **held at the header**, every other block as free as before — the nested hypothesis "the header distance is right" — and the two are compared on the one residual family that can tell them apart: the **excitation** residual. It never involves the detector, so it is blind to the distance itself; what it sees is the cell scale, and a held fit at a wrong header distance is forced into a wrong cell scale by the spot positions, which the rocking angles then refuse (measured: a header 1.4 % long leaves the held fit's excitation residual seventeen times the free fit's). Where freeing the distance lowers the held-out excitation residual below the held fit's by more than that residual's own standard error, the free fit is committed exactly as before; where it does not, the held fit is — header distance, refined beam, cell, orientation and axis — and the report says so (`POSTREFINE_DISTANCE_HELD`). The positional residual is deliberately not consulted for this: it is the family whose in-fit gain along the degenerate direction re-integration erases, and pooled with the excitation family it either drowns a decisive excitation gain in its own noise (a 54 % excitation gain read as 9 % pooled against a 9 % noise) or lends the degenerate direction a gain that is not there. Nothing is tuned here: the only input is the standard error of the residual itself, the same noise the geometry walk's rounds have to beat. The wavelength is never refined on a single crystal for the same reason in its exact form: it scales the spot positions and the rocking angles identically to the cell, so no sweep can tell the two apart at any $2\theta$. +3. **Pass 2** re-indexes de novo and re-integrates at the committed geometry. Only the **detector distance and beam centre** carry over: the refined cell, orientation and axis are what make the distance identifiable, but pass 2 re-indexes from scratch, so they are not propagated. Where the re-index finds a **different** lattice — the two compared on their Niggli-reduced primitive edges, within 2 %, so a symmetric setting is never told apart from its own primitive cell — pass 1's lattice is scored at the refined geometry as a hypothesis of its own, and integrated when it indexes more validation frames; the same lattice found again is kept as the re-index refined it. The run likewise falls back to pass 1's lattice where the re-index indexes too few frames, and integrates it at the refined geometry — and the cell is then **scaled to the distance it will be used at**, since a real-space cell is measured against the distance its spots were seen at, and carrying it across a distance change otherwise scales the whole cell by the ratio of the two. The orientation is untouched. + + Pass 2 measures the post-refinement again, and where it still moves the geometry the run **walks**: it re-indexes and re-integrates at what the fit asks for, and repeats. A round is kept only for what it *realises*, not for what the fit predicts, and it can realise a gain in two ways, either of which has to beat its own noise: a lower held-out residual (the standard errors of the two means combined), or a larger share of the validation spots on the lattice beyond chance (the binomial noise of the two shares, z = 3.29). The residual alone misses exactly the errors that cost resolution: it is dominated by the low-resolution reflections, where a distance and the cell scale that compensates it move every spot alike, and its centroids are taken inside a disc centred on the prediction, so they follow the prediction part of the way; the high-resolution validation spots are the first to leave the lattice (measured: 0.6 % of distance read 0.74 of the residual's noise and 70.4 % against 58.1 % of the validation spots, and cost 0.07 Å of resolution). A move of a trust-region step or more starts the walk outright. A smaller one is first tried as two indexing probes on the validation frames — at the fit's geometry and at the one in hand, each stopping once the lattice is scored — and pays for a re-integrated round only where the fit's geometry scores higher. The run keeps the best round it reached. + +The space group is determined **after** pass 2, on the geometry the run refined, and pass 1 does not search at all: a decision taken on the worse of the two passes and then carried forward is a constraint on the better one, and would have to be reconciled with what pass 2 later found. The guard that chooses which pass is written compares each pass's **first** merge — $P1$ on both sides, full resolution range, before the correction surfaces — which both passes produce anyway, so it never compares statistics computed in two different space groups. What it compares there is the **signal each pass measured**: the count of unique reflections merged at $I/\sigma \ge 2$, less the count at $I/\sigma \le -2$. An empty reflection is as likely to land in either tail, so the second count is the merge's own measure of how much of the first is noise — which is what makes two passes on different lattices comparable: a pass on an $n$-fold supercell merges $n$ times the reflections, most of them empty, and their noise alone once out-counted the crystal's own lattice. Self-consistency cannot do this job — against an external arbiter the signal count named the more accurate geometry on 23 of 27 arm-dataset pairs where $R_\mathrm{meas}$, $CC_{1/2}$ and ISa managed 13, a coin flip — and that merge's own $CC_{1/2}$ least of all, being pooled over the whole range, uncut and uncorrected, so the shells with no signal in them dominate it and they are exactly the shells a geometry move disturbs (it reads 0.13 on a crystal whose data merge at 0.995). The refined pass is sent back only where it merges more unique reflections than its cell can hold, where it measured decisively less signal (10 %, and only where it covers no more of reciprocal space either — a wider integration disk pulls weak reflections in and dilutes the strong fraction without measuring less; between two lattices the reflection count of the smaller is scaled by the integer index of one in the other), or where it lost the axial rows the systematic absences are read off. One index-time veto remains and is keyed to pass 1's **lattice** rather than its group: a centred pass-1 lattice against a primitive pass-2 one. A pass sent back is pass 1 as it was judged — its whole indexing result is reused, not indexed again de novo at the header geometry, which is a hypothesis nobody had judged. + +Only pass 2 is written, as the canonical `_*` output. Pass 1's merge exists to give the guard something to judge pass 2 against, so it stops short of the parts of the merge that only fill in a file — the correction surfaces, the twinning and radiation-damage analyses, the R-free flags and the amplitudes — and writes no merged files of its own. + +**Goniometer rotation scale.** A stage that turns further than it was commanded to leaves no trace in the file, because the stored $\omega$ values *are* the commanded ones; the excess then presents as the crystal drifting, in this program and in others. The excitation residual already measures it without a new degree of freedom: it rotates by $-\phi\,\mathbf{u}$ with $\mathbf{u}$ an **unnormalised** 3-vector, so $|\mathbf{u}|$ is the factor $k$ by which the stage actually turned, and normalising the axis throws it away. Pass 1 fits $k$ as a single parameter on its rocking events, with the crystal and the axis direction held at their committed values and the angle measured from the centre of the sweep. That fit **under-reads** a real error: it only sees the frames the stored angles still track, and a rate error is exactly what stops them tracking the rest. So it is not acted on directly. Between the passes, at the detector geometry pass 2 runs at, the lattice is indexed under the stored angles and under the fitted $k$, and each is scored on the validation frames spread over the whole sweep, as the share of their spots it puts on the lattice beyond what it puts there at a wrong spindle angle. The fitted $k$ is adopted only where it scores higher by more than the binomial noise of the two scores (z = 3.29); the run then integrates and post-refines at it, fits $k$ again on top of it, and repeats until the next $k$ no longer scores better - the fixed point of the fit. Otherwise the stored angles stand. The adopted $k$ drives every later pass (prediction, integration, scaling and the reported oscillation) and is reported as `GONIOMETER_ROTATION_SCALE`, with `GONIOMETER_ROTATION_SCALE_SUSPECT= TRUE`. `--rotation-scale ` asserts a calibration and skips all of this. + +### 7.6 Detector geometry from powder rings + +Everything above fits the geometry to *Bragg* data, where the beam centre is the weakest parameter: it is gauge-coupled to the crystal orientation, which is why §7.5 bounds it to within 15 px of a centre something already believes rather than letting the spots place it freely. A **powder ring has no orientation to be coupled to**. Where it falls on the detector depends on the geometry and on nothing else, which makes a calibrant — LaB₆, silver behenate, CeO₂, silicon — or even ice an independent constraint on exactly the quantity Bragg data cannot pin. + +The ring positions are matched to the observed rings and the geometry is refined (Ceres, five parameters: beam centre, distance, and the two detector tilts) so that the $|s|$ predicted at each observed ring point matches the ring it belongs to. The two tilts can be held fixed (`rugnux --no-refine-tilt`, the viewer's *Refine detector tilt* tick box), leaving a three-parameter fit: a tilt a downstream program cannot express is better left out of the fit than refined and then dropped, since the centre and the distance of a tilted fit have already absorbed it. + +**Calibrants.** LaB₆, silver behenate, CeO₂ and silicon are held as unit cells and their rings enumerated from them. Ice is held as the hexagonal-ice ring positions of §3.3 instead — measured to 1.522 Å, calculated below it — because hexagonal ice is $P6_3/mmc$ with oxygen on $4f$ and enumerating $hkl$ from its cell would emit rings the oxygen sublattice extinguishes. A calibrant is therefore a list of ring $q$ values throughout, not a cell. + +**What a ring can and cannot determine.** A ring is a conic centred on the beam, so a wrong centre makes its apparent radius oscillate once per turn, $r(\phi)=R+\delta_x\cos\phi+\delta_y\sin\phi$, with the **same amplitude on every ring**. A detector tilt $\beta$ produces a $\cos\phi$ term too — not the $\cos2\phi$ one might expect — but one that grows as the ring's radius *squared*, $r(\phi)=R+(R^2/F)(\beta_x\cos\phi+\beta_y\sin\phi)$; the true $\cos2\phi$ term is $O(R^3\beta^2/F^2)$, hundredths of a pixel. The two are therefore separated by how the amplitude scales with radius, which needs **at least two rings** — on a single ring they are exactly degenerate. None of this uses the calibrant's $d$-spacings, so the centre is determined without assuming anything about the standard. + +The **distance** is different: it follows from $r=F\tan2\theta$ with $\sin\theta=\lambda/2d$, so a fractional error in the lattice constant passes straight into it, and the $\lambda$–$F$ pair is separated only by the curvature of $\tan(2\arcsin(\lambda/2d))$ across the rings — $\partial\ln r/\partial\ln F=1$ at every ring against $\partial\ln r/\partial\ln\lambda=4\tan\theta/\sin4\theta$, which runs from about 1.05 at low angle to 1.43 at high. That lever collapses as the detector moves back and the rings crowd into small $2\theta$, so distance is a short-distance measurement and the wavelength is better calibrated by other means. + +**Reading the rings.** The ring points come from one of two measurements, both accumulated over **every processed image** rather than one. The default reads the **azimuthally-binned profile (§2) summed over the run**: for each ring and each azimuthal sector, the radial peak is fitted against a locally interpolated background and the measured $(q,\phi)$ mapped back through the current geometry to the pixel it came from. The alternative pools the **spot lists**, which samples each arc wherever the spot finder's threshold happens to bite. The accumulated profile is the same size however many images went into it; the pooled spot list is capped, each image contributing an equal share. + +A plain radial profile — one azimuthal sector — has averaged the ring over every direction and carries no centre at all, so the profile route requires at least four sectors and uses 32 by default. Sixteen to thirty-two are enough; beyond that the limit is the ring's own texture, not counting statistics. + +The extraction window around a ring is capped at half the gap to its neighbour, because the background under a peak is taken from the ends of that window: hexagonal ice has a triplet of rings (1.947, 1.916 and 1.882 Å) whose neighbours sit only 0.05–0.06 Å⁻¹ apart in $q = 2\pi/d$, which a fixed window merges into a single peak. Where only one ring is in reach the two tilts are held at their input values rather than fitted, since on a single ring they are degenerate with the centre (above) and the fit would otherwise trade the centre away for them. diff --git a/_sources/CPU_DATA_ANALYSIS_INTEGRATION.md.txt b/_sources/CPU_DATA_ANALYSIS_INTEGRATION.md.txt new file mode 100644 index 000000000..4fbda7ff7 --- /dev/null +++ b/_sources/CPU_DATA_ANALYSIS_INTEGRATION.md.txt @@ -0,0 +1,501 @@ +# Data analysis: integration, scaling and merging (§8–§12) + +Part of the [CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md); the section numbers are continuous across its four parts. + +```{contents} On this page +:local: +:depth: 2 +``` + + +## 8. Reflection prediction + +Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model. + +### 8.1 Enumerating reciprocal lattice points + +For a maximum resolution $d_\mathrm{min}$, accept $(h,k,l)$ such that: +$ +\lVert \mathbf{p}(h,k,l)\rVert^2 = \lVert h\mathbf{a}^* + k\mathbf{b}^* + l\mathbf{c}^*\rVert^2 \le \left(\frac{1}{d_\mathrm{min}}\right)^2. +$ + +### 8.2 Still prediction (excitation-error cutoff) + +For still images, the diffracting condition is approximated by an excitation-error cutoff: +$ +\left|\Delta_\mathrm{Ewald}(\mathbf{p})\right| \le \Delta_\mathrm{cut}. +$ +Accepted reflections are projected to the detector by intersecting the diffracted direction $\mathbf{S}=\mathbf{S}_0+\mathbf{p}$ with the detector plane, using the current geometry. + +When the beam has a finite energy bandwidth, this window is **broadened radially per reflection**: the cutoff is combined in quadrature with a bandwidth smear, $\sqrt{\Delta_\mathrm{cut}^2 + (3\,\sigma_\mathrm{bw})^2}$, where $\sigma_\mathrm{bw}\propto|p_z|$ (the reciprocal-space depth along the beam, growing as $\sim 1/d^2$). This keeps high-resolution reflections — smeared by the bandwidth into radial streaks — from being clipped. The same $\sigma_\mathrm{bw}$ is deconvolved from the measured profile radius (§11.1), so it is not double-counted. + +### 8.3 Rotation prediction (Laue equation + partiality model) + +For rotation/oscillation datasets, Jungfraujoch solves for rotation angles $\phi$ where the rotated reciprocal lattice point satisfies the Ewald-sphere condition. In an XDS-like notation, define: + +- rotation axis unit vector $\mathbf{m}_2$, +- $\mathbf{S}_0$ incident vector, +- $\mathbf{S}(\phi)=\mathbf{S}_0+\mathbf{p}(\phi)$. + +A key quantity is: +$ +\zeta = \left|\mathbf{m}_2\cdot \mathbf{e}_1\right|,\quad +\mathbf{e}_1 = \frac{\mathbf{S}\times \mathbf{S}_0}{\lVert \mathbf{S}\times \mathbf{S}_0\rVert}, +$ +which also appears in XDS as the Lorentz component linked to the rotation axis. + +A Gaussian mosaicity model yields a partiality fraction over an oscillation width $\Delta\phi$: + +$ P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], $ + +with mosaicity $\sigma_M$ in radians. + +Reflections are predicted if they meet minimum $\zeta$ and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area. + +### 8.4 Systematic absences (centering) + +Systematic absences are applied at the centering level (prior to full space-group symmetry) **when the space group is supplied by the user**. With no user-fixed space group, prediction runs in $P$ regardless of the centering the lattice search inferred: the centering-absent reflections are integrated so that the space-group search (§13) can confirm or disprove the centering from the measured intensities, and so that a missed superstructure shows up. For each centering symbol: + +- $I$: absent if $h+k+l$ odd, +- $A$: absent if $k+l$ odd, +- $B$: absent if $h+l$ odd, +- $C$: absent if $h+k$ odd, +- $F$: absent if any of $h+k, h+l, k+l$ is odd, +- $R$: absent if $(-h+k+l)\bmod 3 \ne 0$, +- $P$: no centering absences. + +--- + +## 9. 2D Bragg integration (profile fitting over a three-ring ROI) + +Jungfraujoch integrates each predicted reflection in the detector plane over a CrystFEL-inspired “three-ring” region of interest (§9.1). The **default** extraction is **profile fitting** (Kabsch; §9.3), which weights each pixel by a fitted spot profile and so recovers weak reflections far better than plain summation; plain box summation (§9.2) is retained as the seed for the profile and as a fallback. Both methods share the same ROI and background model, and emit the same per-reflection $(I,\sigma,\text{partiality},d)$, so scaling, the rotation combine (§10.6) and merging consume either unchanged. + +### 9.1 Regions of interest + +For each predicted reflection at $(x_p,y_p)$, define three radii: + +- $r_1$: inner signal radius, +- $r_2$: inner background radius, +- $r_3$: outer background radius. + +The defaults are $4,6,13$ px for rotation data and $6,8,14$ px for stills, which have a sparser +pattern and can afford the wider ring. `--integration-radius` sets them by hand; on rotation data +$r_1$ is otherwise measured from the crystal's own spots (§9.5). + +Pixels are classified by their squared distance $r^2=(x-x_p)^2+(y-y_p)^2$: + +- **signal region:** $r^2 < r_1^2$, +- **background annulus:** $r_2^2 \le r^2 < r_3^2$. + +Invalid pixels (masked/bad/saturated) are excluded from both sums. In addition, pixels lying inside the signal disk ($r`, default 0).** The three radii above are one triple for the whole run, identical for every reflection at every resolution. A reflection is not round, though: a finite bandwidth streaks it radially by $\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}$, with $R_\mathrm{px}$ the distance from the beam centre — the same physical smear as §8.2's and §11.1's $\sigma_\mathrm{bw}$, expressed here in detector pixels where those sections use reciprocal units; the two forms are never mixed in one formula. Throughout, $\text{bandwidth}$ is the **rms** relative energy spread: the user-facing `--bandwidth` takes a FWHM (a DMM's usual specification) and it is divided by 2.355 on input. On a radially smeared spot the fixed $6\ldots13$ px ring therefore sits only $\approx1.3$–$2.2$ radial $\sigma$ from the centre — on the reflection's own tails, which it then measures as background. + +With $k>0$ the **background ring becomes an ellipse**, elongated along the beam→reflection direction by $k\sigma_\mathrm{bw}$. The **radial** semi-axes become $r_2+k\sigma_\mathrm{bw}$ and $r_3+k\sigma_\mathrm{bw}$; the **tangential** half-widths stay $r_2$ and $r_3$; and the growth is capped at $2r_3$, which bounds what a mis-declared bandwidth can do to the bounding box. Pixels are then classified as + +- **signal region:** $r^2 < r_1^2$ — a circle, unchanged, +- **background ring:** $r^2-q_\mathrm{in}\rho^2 \ge r_2^2$ **and** $r^2-q_\mathrm{out}\rho^2 < r_3^2$, + +where $\rho$ is the pixel's radial offset (its projection on the beam→reflection direction), $g=\min(k\sigma_\mathrm{bw},\,2r_3)$ is the capped growth, and $q=1-\big(r/(r+g)\big)^2$ for the boundary concerned. Written this way $k=0$ gives $q=0$ and both tests collapse onto $r^2$ **exactly in floating point**, so the default classifies every pixel exactly as the circular stencil did. The neighbour exclusion above follows: each neighbour's **inner ellipse**, taken in that neighbour's own radial frame, is what is masked out of this reflection's ring. + +The width is the bandwidth streak alone, and deliberately **not** the profile's full radial variance of §9.3, which also carries the sensor parallax and weak-spot capture terms. Those two are the only terms there are on a monochromatic beam, and widening the ring by them was measured on the rotation battery: it neither helped the crystals with clean high-resolution shells nor left the weak ones alone. The bandwidth streak, by contrast, is a measured elongation of the recorded spot — principal axis along the radius to within a couple of degrees, and azimuth-independent. Keeping only it also makes the option exactly inert on a monochromatic beam, where $\sigma_\mathrm{bw}$ is zero. + +Growing the ring also grows the neighbour exclusion, so on a crowded pattern fewer background pixels survive; a reflection left with too few is rejected outright. On the data this was measured on the loss is under 0.1% of reflections, but it is not structurally zero. + +Only the ring moves. The signal disk $r_1$ stays circular, deliberately: it sets $n_S$, it sets $\mathrm{var}(\hat b)$, it is the domain the profile *width* is learned over (§9.3), and with `--integrator boxsum` it drives the all-or-nothing “every signal pixel valid” acceptance gate (§9.2), so growing it would reject any box sum carrying a single bad pixel anywhere along a long streak. In the default `gaussian` mode $r_1$ does not set the intensity at all — the fit grid, $\lceil r_2\rceil$, does. + +What a circular $r_1$ loses is flux, and that loss is **not** a function of resolution alone: measured per reflection, it carries a directional component worth several Ų with a definite principal axis, on top of the isotropic part. Nor is there anything in the merge to absorb it. There is **no per-shell scale**, and there cannot usefully be one: every scale in §10 is fitted against a reference built from a reflection's own symmetry equivalents, and equivalents share $s^2$ exactly, so any function of $s^2$ lies in the exact null space of the whole scaling model — a per-shell parameter would have zero residual to fit against. (XDS and DIALS have the same null space, for the same reason.) The isotropic part of the loss is instead degenerate with the overall Wilson $B$ and is silently reported as part of it, so **the reported `WILSON_B` / `_reflns.B_iso_Wilson_estimate` carries an $r_1$-dependent contribution**: measured across a constant-ring-area radius sweep it falls monotonically as the disk grows, by 0.5 Ų on sharp strong data and by up to ~10 Ų on weak wide-spot data. What this costs the *data* is much less than what it costs the flux, because most of the loss is matched by a proportional $\sigma$: it moves no CC$_{1/2}$ and no $R_\text{meas}$, and — to within a few hundredths of an ångström — no resolution cut. + +**Measured spot footprint (automatic).** The radii above are chosen from spots at 5 Å, which at high X-ray energy sit close to the beam. Away from it a spot can grow several times wider — radially from the sensor's parallax and the obliquity of the incidence, tangentially from the crystal's azimuthal spread, which rotates the diffracted beam about the incident one and smears the spot along its ring. On small-molecule data at 20–25 keV the standard deviation grows from ~1 px near the beam to ~5 px at the detector edge: the $r_1 = 4$ disk holds a quarter of the flux there, the $6\ldots13$ px ring a third of it, and the profile widths learned inside $r_1$ (§9.3) saturate near $r_1^2/4$. So the pre-scan measures every spot it finds with a window that follows the spot — three of its own standard deviations, iterated and re-centred — separately along and across the radius, and tabulates the median widths $\sigma_\rho,\sigma_\tau$ against the distance from the beam. Wherever $3\max(\sigma_\rho,\sigma_\tau)>r_1$ the integrator then (i) starts the background ring at $4\sigma$ along each axis, (ii) sums the reflection over the $r_1$ disk **and** the $4\sigma$ footprint ellipse, so the summation — the profile fit's seed and its fallback — holds the spot rather than its core, and (iii) builds the per-reflection Gaussian at the measured widths on a grid grown to hold them. Where every spot fits the disk nothing is installed and the integration is unchanged bit for bit, which is the case for compact protein spots; like the measured radius, the footprint applies to the canonical pass and not to the geometry pre-pass, and a canonical pass whose wider rings the neighbours starve falls back to the settings without it. The reach is $4\sigma$ rather than the $3\sigma$ that decides whether a spot outgrew the disk because wide spots are not Gaussian: mosaic streaks and diffuse halos carry flux past $3\sigma$, which a ring starting there reads as background. Judged by refining the published structures with SHELXL, it removes the intensity loss that grew with resolution on the small-molecule sets (rugnux/model intensity in the outermost shell 0.81–0.91 → 0.98–1.02). + +**Split reflections (automatic).** A crystal made of slightly misaligned domains — the ferroelastic domains a crystal forms below a phase transition, or a cracked or split crystal — records each reflection as two or more compact spots on either side of the position the averaged lattice predicts, moving apart with resolution. The widths above are measured about each spot and so see compact spots; the $r_1$ disk then holds the gap between them and the background ring lands on them, and the loss grows to almost everything at the detector edge. Once the geometry pre-pass has a lattice, every spot it indexes is compared with the predicted position of its own reflection on the same frame, and the mean square of that offset, along and across the radius and in the same distance bins, is added to the pre-scan widths. The table then describes the *reflection* rather than the spot, and the canonical pass integrates with it exactly as above. Where spots sit on their predictions it moves the widths by the prediction error alone (0.2–1 px on protein data, where no reflection then outgrows $r_1$); on an inorganic crystal measured below its ferroelectric transition, where every reflection off one zone is a doublet, the offsets reach 8–16 px at the edge, and SHELXL refinement of the published structure goes from $R_1 = 0.27$ with a spurious extinction parameter to $R_1 = 0.05$. + +### 9.2 Box summation (seed and fallback) + +Let: +- $S = \sum I(x,y)$ over signal pixels, +- $n_S$ = number of valid signal pixels, +- $B = \sum I(x,y)$ over background pixels, +- $n_B$ = number of valid background pixels. + +Background per pixel and integrated intensity: +$ +\hat{b} = \frac{B}{n_B},\qquad +\hat{I} = S - n_S \hat{b}, +$ +with a Poisson-like uncertainty $\sigma(\hat{I})=\max\!\big(1,\ r_\sigma\hat{I},\ \sqrt{S + n_S^2\,\mathrm{var}(\hat{b})}\big)$, i.e. $\sqrt{S}$ floored both at 1 count (pixel values are photon counts) and at a small fraction $r_\sigma$ of the intensity. The second term under the root is the **uncertainty of the background estimate itself**: $\hat b$ is measured from a finite number of ring pixels, $\mathrm{var}(\hat b)=\hat b/n_B$, and it is subtracted $n_S$ times over, so it enters squared. Omitting it understates the **variance** by $1+n_S/n_B$ — 1.11 with the shipped circular stencil ($n_S = 45$, $n_B = 408$) — and so understates $\sigma$ by up to $\sqrt{1+n_S/n_B} \approx 1.05$, a bound attained on background-limited (weak) reflections and falling towards 1 on strong ones, where $S$ dominates; with an elongated ring $n_B$ grows with resolution, so the factor is no longer one number for a run. The same term is carried into the profile fit (§9.3), where it adds $\big(\sum P/v \,\big/ \sum P^2/v\big)^2\,\mathrm{var}(\hat b)$ — the square of $\partial I/\partial\hat b$ for that fit; $n_B$ is the count of pixels behind the *final* background value, so a clip or trim that discards ring pixels raises it. A box sum is accepted as “observed” only if all signal pixels were valid and $n_B$ exceeds a minimum — it measures what is in the disk with no model of what should be there, so it cannot renormalise a disk it has lost pixels out of. The profile modes can, and do (§9.3). This box sum is the classical estimator; it is used directly with `--integrator boxsum`, and otherwise seeds the profile fit below, where $S$ and $n_S$ then count only the pixels that were actually read. + +**High-side clipped background (default on).** Because $\hat{I}=S-n_S\hat{b}$ is a small difference of large numbers for weak reflections, a per-pixel background bias $\delta\hat{b}$ becomes a *fractional* intensity bias $\approx n_S\,\delta\hat{b}/\hat{I}$ that grows as $\hat{I}$ shrinks — worst at the resolution edge. A plain ring mean reads high there, because neighbour-spot wings that survive the signal-disk mask, tails and zingers are one-sided (positive) contaminants. The ring mean is therefore made robust: pixels above $\hat{b}+n\sqrt{\hat{b}}$ are rejected and the mean recomputed, with $n=4$ (`--background-clip`; $n=0$ disables), whatever the bandwidth. A clean Poisson ring is essentially unchanged by the cut (measured false-rejection rate 0.04–0.39 % at $4\sigma$), while a 40-pixel neighbour core at $+100$ counts shifts the estimate by $+0.009$ ct/px. + +The clip cuts only the high tail, which matters: the **symmetric** trimmed mean it replaced (drop the lowest and highest fraction $f$ of ring pixels, $f=0.10$; still reachable with `--background-trim`, which switches the clip off) is *not* a consistent estimator of the mean of a right-skewed Poisson sample. It sits $\approx0.1$ ct/px **below** the true mean at every level, and with $n_S = 45$ signal pixels in the $r_1$ disk that under-estimate adds $\approx4.5$ counts to **every** partial — negligible at low resolution, but a large fraction of a partial in the outermost shell. The trim also collapses once contamination exceeds $\approx10\,\%$ of the ring, where the clip does not. Note that removing a positive background bias *lowers* $\langle I/\sigma\rangle$ and *raises* edge $R_\text{meas}$, because both are inflated by information-free counts — so neither may be read as evidence against the change. + +Both estimators are computed in the shared background pass, but only the trim reaches plain box summation: the high-side clip is skipped for `--integrator boxsum`, which therefore uses the plain ring mean unless `--background-trim` is given. + +**Radial background correction (opt-in).** A ring mean estimates the background *under* the signal disk correctly only if the background is flat there. The signal disk and the ring are concentric, so for a background that is **linear** in position $\langle B\rangle_\mathrm{ring}=\langle B\rangle_\mathrm{disk}$ identically — a plane or gradient fit buys exactly nothing. The leading error is the **curvature** of the radial background, which is negligible on a smooth background but reaches tens of counts on a single reflection sitting on a sharp powder ring. That error is a kernel over radial offset, +$ +\delta \hat b \;=\; \textstyle\sum_k \kappa_k\, \bar B(r_0+k), +$ +with $\kappa$ the annulus-minus-disk histogram of the stencil over radial offset, averaged over azimuth, and $\bar B(r)$ the image's own radial background curve. With the fixed circular stencil ($k=0$, §9.1) that single kernel serves every reflection. An elongated ring does not: its radial-offset histogram depends on how far that particular reflection's ring was grown, so $\kappa$ becomes a small table of kernels, indexed by the growth rounded to whole pixels. The azimuthal average survives the change unaltered, because the stencil is rebuilt in the reflection's own frame at each azimuth and so stays radially aligned: what is averaged over is the sub-pixel phase of the detector grid against the radius, which is what genuinely differs between reflections. Applying it costs one short dot product per reflection and no extra pixel reads; correcting the background *scalar* means the box sum, the profile fit and the variance all pick it up. The curve is accumulated from the same annulus pixels the background pass already reads (a pixel's radius is the reflection's radius plus the pixel's projection on the beam→reflection direction, so no per-pixel square root is needed) and specifically from the **clipped** pixels, or it would carry neighbour tails and zingers — which is why the correction is inert under `--integrator boxsum`, that path having no clip pass. + +The model is a function of **radius alone**, so it is applied only where that is true of the background. `--background-radial` takes `on`, `off` or `auto`. In Rugnux it is **`auto` by default** (the broker keeps it off); under `auto` each image's peak-excluded ice score (§3.3) is taken after spot detection and before integration, and the correction is applied to that image when the score reaches the same `--ice-min-score` gate. Smooth powder ice *is* a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes **43 % of the bands' excess amplitude**, and the improvement is **7× larger inside the bands than outside**, which is its stated mechanism; on a crystal whose ice is textured the same correction *increased* the excess amplitude by half; on a clean crystal it is inert to four decimal places. Auto engages only where a peak-excluded score exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off. + +### 9.3 Profile-fitted extraction (default) + +A fixed signal disk captures a *width-dependent* fraction of each spot, which puts a multiplicative floor on the per-observation precision of strong reflections and weights weak reflections poorly. Profile fitting removes this by extracting each intensity against a fitted spot shape, without needing reference intensities. Per frame: + +1. **Seed.** Box-sum every reflection (§9.2) to get a rough intensity and observed centroid, and select strong spots (significance $\ge 5$). +2. **Build the profile.** For `gaussian` (the default) the width is taken **per resolution shell** from the measured second moments of the strong spots (shell-dependent because spot size grows with resolution). The moments are **anisotropic**: each strong spot's pixels are rotated into its *own* radial/tangential frame before being accumulated, giving $\sigma^2_r$ and $\sigma^2_t$ separately. Stacking the spots in the detector frame instead — they sit at every azimuth — averages the two directions away, leaving only $\sigma_r^2+\sigma_t^2$, so radial smearing is read back as a wider *tangential* spot. For `empirical` the profile is instead the averaged, background-subtracted pixel grid of the shell's strong spots, accumulated in the detector frame on their **rounded predicted** positions. For `gaussian` only, the profile is then **rebuilt for each reflection**, centred on its **sub-pixel predicted position** (the noise-free geometric centre, not the observed centroid) and, where needed, **elongated only along the radial direction** (away from the beam centre) — because two effects stretch a spot radially but not tangentially: + - a finite energy **bandwidth** smears each spot by $\sigma_\mathrm{bw}=\text{bandwidth}\cdot R_\mathrm{px}$ ($R_\mathrm{px}$ = distance from the beam centre, large at high resolution), and + - sensor **parallax** — the depth over which a photon converts in a thick Si/CdTe sensor — adds a term $\propto\tan^2(2\theta)$ (material- and energy-dependent), plus a small fixed weak-spot capture term. + + The two enter as a floor on the measured radial excess: $\sigma^2_\mathrm{radial}=\sigma^2_t+\max\!\left(\sigma^2_r-\sigma^2_t,\ \sigma_\mathrm{bw}^2+c_\mathrm{par}\tan^2(2\theta)\right)$, tangential unchanged at $\sigma^2_t$. The measured excess is what the signal disk can resolve; the analytic term takes over for a streak too long to be measured there. The Gaussian is built on a grid grown to hold the streak — capturing it without the tangential background an isotropic widening would add. The `empirical` profile keeps the fixed per-shell grid and gets none of this. +3. **Fit (Kabsch).** With profile $P$, background $B$ and the shell variance model, the intensity and its uncertainty are +$ +I = \frac{\sum P\,(c-B)/v}{\sum P^2/v},\qquad +\sigma = \sqrt{\frac{1}{\sum P^2/v}},\qquad +v = \max\!\left(B + I\,P,\ \tfrac{1}{2}B\right), +$ +where $c$ is the pixel value and the de-biased variance $v$ (background plus model signal, rather than the down-fluctuating observed count) is iterated (a few passes). The plug-in $I$ enters **as it is**: half-wave rectifying it, $v=B+\max(I,0)P$, lets $v$ — and with it the reported $1/\sum P^2/v$ — respond only to *upward* fluctuations of a noisy estimate, which adds $\approx0.4\,\sigma\sum P^3/(\sum P^2)^2$ to every $\sigma$ whatever the count rate. That offset is invisible on strong reflections and a large fractional inflation on weak ones; the $\tfrac12 B$ clamp keeps $v$ positive without reintroducing it. As a guard, if the profile intensity runs away from the box-sum seed (by more than ~10 box-sum $\sigma$) it falls back to the seed, and the background term is floored at $0.01$ ct/px — enough to keep $P^2/v$ finite when the ring mean reads exactly zero, which a ring of $n_B$ pixels cannot distinguish from any background below $\approx1/n_B$. The rotation/excitation partiality is carried exactly as in the box-sum path. + +**Pixels the fit cannot use (MINPK).** A profile fit is the amplitude of a *normalised* profile, so a pixel left out of the sum renormalises the estimator by construction: it costs information — $\sum P^2/v$ shrinks and $\sigma$ grows — but biases nothing. That is what keeps a reflection whose signal disk is cut by a mask, an untrusted region, a detector gap or an overload: those pixels are simply not read, and the fit is taken over the rest, exactly as the shared pixels of a crowded reflection are (`--overlap exclude`). The reflection is kept only while enough of the expected profile survives — at least `--overlap-minpk` of the profile mass that falls on the detector at all, default 0.75, which is XDS's `MINPK` and dials' `valid_foreground_threshold`. The complete reflections alone teach the profile, its resolution shells and their widths. `--integrator boxsum` has no profile to renormalise with and keeps the all-or-nothing rule of §9.2. + +"Biases nothing" holds only while the profile *model* is exact. Lose the peak and the amplitude is set by the wings alone, so the result stops being a measurement of the reflection and becomes a measurement of how well the fitted shape describes it. The worst case is a pixel invalidated *by the flux it saw* — a detector's per-frame overload marker: that pixel goes missing **because** the reflection was bright, so the loss concentrates on the strong low-resolution reflections that are the largest terms of $R_\mathrm{meas}$, where the fit reads $-50\%$ against the symmetry mates. MINPK cannot catch it, because it cuts on profile *mass* and the peak of a broad spot is a few percent of the mass. So a second condition applies alongside it, on any unreadable pixel whatever made it unreadable: **no unreadable pixel may carry more than 0.9 of the profile's own peak value**. As a fraction of the peak rather than a radius in pixels, that scales with the spot — for a Gaussian it is a cut at $\sqrt{-2\ln f}\,\sigma = 0.46\sigma$, the peak pixel alone where $\sigma$ is 0.8 px and the crest of the ridge where the profile is a bandwidth streak — and it costs well under 0.1 % of the recovered observations. + +The integrator is selected by `--integrator boxsum|gaussian|empirical` (default `gaussian`). + +### 9.4 The prescaling correction + +The deterministic per-reflection corrections are carried as three multiplicative factors. `prescaling_corr` holds the reciprocal Lorentz factor (rotation only — a still's Lorentz factor is one) times the reciprocal polarization factor from the geometry-based term (§2.2), and nothing else: it is Lorentz x polarization, which is what `LP` means in every format the field reads. Beside it sit the two terms that describe what happened to the photon between leaving the sample and being counted, and that are kept apart from `LP` because detector response and beam/crystal geometry are different things: `qe_corr`, the sensor's angle-dependent efficiency, and `flight_corr`, the medium in the flight path (§9.7). The total deterministic correction on a reflection is the product `prescaling_corr * qe_corr * flight_corr`, and every site that corrects an intensity — the integrator, the scaling fits, the merge ingest, the anisotropy analysis and the unmerged export — multiplies all three. None of them is a scale: the fitted per-image scale and the partiality are separate. The three reach the unmerged MTZ as its `LP`, `QE` and `FLIGHT` columns unchanged (`QE` and `FLIGHT` as divisors normalised to 1 at normal incidence), so raw counts are `I / LP * QE * FLIGHT`. + +One term is deliberately **not** in it. The **per-pixel solid angle**, which the azimuthal profile +does divide out (§2.2), is correctly absent here: a Bragg integration sums all the photons in a +reflection, and how many pixels the detector happens to tile that footprint with does not change the +count. Detector obliquity does stretch the footprint, and that enters as the parallax term of the +spot-width variance rather than as an intensity scale. + +### 9.5 Choosing the signal radius from the crystal's own spots (rotation) + +The three radii are one triple for the whole run, but on rotation data they are no longer a fixed constant: $r_1$ is measured from how wide *this* crystal's spots actually are (`--adaptive-integration-radius`, on by default for rotation, off for stills, ignored when `--integration-radius` is given). + +**Why $r_1$ matters even though it does not set the intensity.** In the default `gaussian` mode the intensity is a profile-fit amplitude over the grid $\lceil r_2\rceil$ (§9.3), so $r_1$ is not the integration domain. It *is* the aperture the profile **width** is learned over, and a second moment taken over a disk of radius $a$ saturates at $a^2/4$. At $r_1 = 4$ the learned $\sigma$ can therefore never exceed 2 px, and a crystal whose spots are broader than that is fitted with a profile the model cannot represent. + +**The measurement is independent of the integrator.** It is made in the pre-scan, on the frames the beam-stop projection already reads, so it costs no extra frame reads and there is no feedback loop. On the spots the spot finder has already found, a spot is used only if it is clear of the detector edge and of the direct beam, has no neighbouring spot within 28 px, is one of the 40 strongest in its resolution band, sits on a fully readable disk, and reaches a signal-to-noise of 15 with its centroid within 2 px of the found position. For each surviving spot the background-subtracted **encircled-flux curve** is accumulated in 1-px annuli out to a fixed 14 px aperture and normalised at 8 px — an aperture that owes nothing to $r_1$, $r_2$ or $r_3$. + +**Pooling.** Spots are stratified into five resolution bands (2–3, 3–4.5, 4.5–7, 7–12, 12–30 Å), because a weak crystal's strongest spots sit at high angle and a strong one's at low angle. Each band with enough members contributes the radius at which its **median** curve reaches 0.80 of its normalised flux — $r_{80}$ — at the band's median $d$. Those points are fitted by weighted least squares against $1/d$ (the mosaic contribution to the detector footprint grows as $1/d$) and evaluated at a common 5 Å, then clamped to the range the bands actually measured so the fit never extrapolates. + +**The radius.** + +$$r_1 = \mathrm{clamp}\!\left(\mathrm{round}(2\,r_{80}),\ 4,\ 6\right),\qquad r_2 = r_1 + 2,\qquad r_3 = \sqrt{r_2^2 + 133}$$ + +The factor 2 is not fitted: for a Gaussian $r_{80} = 1.794\,\sigma$, so $r_1 = 2r_{80} = 3.59\,\sigma$, where the truncated second moment recovers 0.990 of $\sigma^2$. The expression for $r_3$ holds the **background-ring area constant** at its value for the shipped $4,6,13$ ($13^2 - 6^2 = 133$) — a ring that shrank with the disk is what makes a bare `--integration-radius` worse than the default it replaces. The floor of 4 is that shipped default; the ceiling of 6 is pattern density, since $r_2$ also drives the neighbour-ownership radius and the ring's inner edge. At $r_1 = 4$ the triple is bit-for-bit the shipped default, so a crystal with ordinary spots is left exactly where it was. + +**The sample grows until the answer settles.** The frames are measured in tiers of stride 8, 4, 2, 1, each tier's sample strictly containing the previous one, and the pooling is redone after each. Measuring stops when the new $r_{80}$ is within 0.40 px of what the smaller sample said **and** is at least 0.25 px clear of both radii at which the rounding in $r_1$ changes answer. Both conditions are load-bearing: clearance alone lets a small sample settle across a switch, and the step test alone lets it settle *on* one. Every frame of the sample is still read — the beam-stop mask and the beam centre are unchanged; what the tiers save is the decompression, preprocessing and spot finding the width measurement adds on top of the read. + +**It applies to the final pass only.** A rotation run integrates twice (§7.5), and the widened radius is handed to the canonical second pass, not to the geometry pre-pass. The reason is that post-refinement takes its observed positions from the integrator, and an observed position is a first moment over the signal disk with the background still in it: a flat background adds nothing to the numerator but adds $n\,b$ to the denominator, so every measured offset is pulled toward its prediction by $I/(I + n b)$, and $n$ nearly doubles between $r_1 = 4$ and $r_1 = 6$. A wider disk therefore *under*-corrects the geometry — enough, on a crystal whose metric is half a degree off orthorhombic, to flip the second pass's de-novo Bravais choice. + +**The density guard.** Widening $r_1$ pushes $r_2$, the ring's inner edge, into the neighbours; a reflection whose ring is left with five or fewer clean pixels has no background and is dropped whole. The integrator counts the rings the **neighbouring reflections** would starve — every predicted neighbour, the rocking-curve tails the background itself does not exclude included, so the count measures the pattern's density — apart from the ones the detector itself starves (module gaps, the beam stop, the resolution mask), a floor that reaches a couple of percent on some geometries and does not move with $r_1$. Where the neighbour-driven count exceeds 1.13 % of the predicted reflections, the pattern is too dense for the widened radius and the final pass is integrated again at the fixed $4,6,13$, reported as pass 3 of 3 with the reason in `PASS_DECISION`. + +Every integration pass, adaptive or not, now logs the radii it used together with the fraction of predicted reflections that lost their background ring, the fraction of rings the neighbouring predictions crowd, and the profile-fit fallback rate. + +--- + +### 9.6 Measuring the bandwidth + +A finite energy spread $\sigma$ ($\Delta\lambda/\lambda$, rms) smears a reflection along its own radius by $2\tan\theta\,\sigma$ radians of $2\theta$ and not at all across it. Most files do not state it — a multilayer monochromator is a beamline option, not a header field — so Rugnux reads it off the spots, on the same isolated strong spots the width of §9.5 is measured on (`spot_width::EstimateBandwidth`). The width settles on fewer spots than this slope needs, so the pre-scan keeps measuring spot shapes for the bandwidth alone until it holds 1000 of them or its sample runs out. For each spot, with $u$ along the radius and $v$ across it, the second moments about its centroid are + +$$m_u = \tfrac1{12} + p_u + j_r^2\big(s^2 + 4\tan^2\theta\,\sigma^2\big),\qquad m_v = \tfrac1{12} + p_v + j_t^2 s^2,$$ + +with $j_r$, $j_t$ the exact pixels per radian of $2\theta$ and of the angle across the scattering plane at that spot, $s$ everything isotropic in angle (divergence, crystal size, mosaic spread), and $p_u$, $p_v$ the sensor parallax, fixed from the sensor's physics: a photon converting at depth $z$ (exponential with length $L\cos\psi$ at angle $\psi$ to the normal, truncated at the thickness) lands $z\tan\psi$ along the ray's in-plane direction. Then + +$$y = (m_u - \tfrac1{12} - p_u) - (j_r/j_t)^2\,(m_v - \tfrac1{12} - p_v) = a + \sigma^2\,(2 j_r\tan\theta)^2$$ + +is a straight line whose slope is the bandwidth. It is fitted over eight equal-count bins of the abscissa, each a 20 %-trimmed mean weighted by its own scatter; the slope's error is the spread of 200 bootstrap re-draws of the spots, inflated by the reduced $\chi^2$ of the binned fit where the line fits worse than the scatter says, and the log calls the estimate significant at $z>3$. It is a **lower bound** — mosaic spread seen along the radius subtracts — and a spread of cell edges is exactly degenerate with it, so what it measures is the effective radial broadening — which is what its consumers need. + +A significant estimate is the run's bandwidth, unless the file states one or `--bandwidth` is given (`--bandwidth 0` forces a monochromatic beam); anything short of $z>3$ leaves the beam monochromatic. From there it acts continuously, with no broadband mode: prediction and partiality (§8), the profile's radial width (§9.3) and the background ring's elongation (`--integration-stencil`, §9.1) all scale with it and are exactly what they were at zero bandwidth. + +### 9.7 The flight path + +A reflection leaving the sample at incidence angle $\alpha$ to the detector normal reaches its pixel +after $D/\cos\alpha$ of flight rather than $D$, so it crosses more of whatever fills the flight path +than one arriving head-on and arrives attenuated. This is the **same $\cos\alpha$ geometry as the +sensor crossing of §9.4, with the opposite sign**: the sensor makes an oblique reflection read high, +the medium makes it read low. + +$$T(\alpha) = \exp\!\left(-\frac{D}{L\cos\alpha}\right), \qquad +\text{correction to } I = \frac{T(0)}{T(\alpha)} = \exp\!\left[\frac{D}{L}\left(\frac{1}{\cos\alpha} - 1\right)\right]$$ + +with $L = 1/\mu$ the attenuation length of the medium at the photon energy, from the same NIST +tabulation the sensor uses. Nothing here is fitted: $\mu$ is tabulated, and $D$ and $\lambda$ are +stated by the file. Normalising at $\alpha = 0$ divides out $\exp(-D/L)$, a constant for the dataset +that the fitted per-image scale absorbs; what is left is the only part of the flight path that is not +degenerate with that scale. + +For **air**, $L$ falls steeply toward low energy — **8.2 m at 18 keV, 3.0 m at 12.4 keV, 0.089 m at +3.8 keV** — and the size of the correction follows it: + +| photon energy | $L$ (air) | $D/L$ at 160 mm | correction at $\alpha = 30°$ | at $\alpha = 55°$ | +| --- | --- | --- | --- | --- | +| 18 keV | 8.17 m | 0.0196 | +0.30 % | +1.5 % | +| 12.4 keV | 2.99 m | 0.0535 | +0.83 % | +4.1 % | +| 8 keV | 0.84 m | 0.191 | +3.0 % | +15 % | +| 3.8 keV | 0.089 m | 1.80 | ×1.32 | ×3.8 | + +**Helium** attenuates about 1/600 of air at 3.8 keV — two electrons an atom against nitrogen and +oxygen, and a seventh of the density — which is exactly why long-wavelength stations use it. It is +not vacuum, and is modelled rather than treated as one, though at these distances it is worth well +under a per cent. **Vacuum** leaves every intensity untouched. + +#### What it does to merged data + +On an **untilted** detector $\alpha$ is the scattering angle, so the correction is a pure function of +resolution. It therefore cancels within a resolution shell and cannot move $R_\mathrm{meas}$ or +CC$_{1/2}$ there; **its whole effect on merged data is a shift in the Wilson $B$**. Friedel mates +share $2\theta$ and so receive an identical factor, which also means it cannot act on an anomalous +difference at all. Both statements are quantitative predictions with no free parameter, and both are +confirmed: the Wilson-$B$ shift is reproduced to within 8 % on three datasets spanning an order of +magnitude in $D/L$, and where a pooled statistic does move, every resolution shell is unchanged and +the pooled shift is reproduced by re-weighting alone. + +That is what `FLIGHT_PATH_WILSON_B` in the results report quotes: the correction's worth, in the one +number it can move. + +#### Why the medium is declared and not detected + +**No field of the NXmx application definition, and no field of any master file `rugnux` reads, +describes the medium in the flight path** — there is no air, helium, vacuum, flight-tube or pressure +entry anywhere to detect it from. + +Inferring it from the physics was considered and **refuted**. The natural idea is that air becomes +unusable at low energy, so an implausibly low implied air transmission would mean helium; but in this +corpus a **confirmed helium** station sits at 51 % implied transmission and a **confirmed air** +station at 63 %. No criterion separates those two without being a threshold fitted between two +points, so there is none. + +`rugnux` therefore **assumes air** — which is what a beamline has unless it was built not to have one +— and `--flight-path helium|vacuum` declares otherwise. Because that is an assumption made on the +user's behalf, the report states it (`FLIGHT_PATH`) together with what it was worth +(`FLIGHT_PATH_WILSON_B`), and warns where it is worth enough to matter. + +--- + +## 10. Scaling and merging + +After per-image integration, Jungfraujoch scales observations and merges them into unique reflections. The design is intentionally compatible with XDS/XSCALE concepts, and handles both still and rotation data. + +### 10.1 Observation model + +For an observation $j$ of a unique reflection $h$ on image (or image group) $i$, the predicted measured intensity is modeled as: +$ +I_{ij} \approx G_i \, L_{ij}\, P_{ij}\, I_h, +$ +where: + +- $G_i$ is the image scale factor, +- $L_{ij}$ is the whole deterministic per-reflection correction of §9.4 - Lorentz x polarization, the sensor's efficiency and the flight path together, not the `LP` term alone. Predictions carry its **reciprocal** split across three fields, so $L = 1/(\texttt{prescaling\_corr} \cdot \texttt{qe\_corr} \cdot \texttt{flight\_corr})$ and the correction below is applied as a multiplication by their product; the scaling fits, the merge ingest and the anisotropy analysis all use the same three, +- $P_{ij}$ is a partiality term (model-dependent), +- $I_h$ is the merged (true) intensity parameter for that unique reflection. + +A least-squares objective is minimized: +$ +\sum_{ij} \left(\frac{I_{ij}^{\mathrm{pred}} - I_{ij}^{\mathrm{obs}}}{\sigma_{ij}}\right)^2 +$ +solved by robust (Cauchy) weighted least squares, with optional post-fit smoothing of the per-frame scales for rotation series (§10.3). + +### 10.2 Partiality models + +The partiality applied is fixed by the data type and scaling stage, not chosen from a user menu: + +1. **Rotation partiality** (XDS-like; see §8.3), used for the per-frame scaling of rotation partials: + $ + P_{ij} = \frac{1}{2}\left[ + \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - + \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) + \right]. + $ + Here $\Delta\phi_{ij}$ is observation $j$'s rocking offset from its exact Bragg angle on image $i$, and the unsubscripted $\Delta\phi$ is the oscillation width per frame — two different quantities that share a letter. The mosaicity $\sigma_{M,i}$ is **measured once per image at indexing** (MLE, §11.2) and held fixed during scaling — only smoothed in frame order (§10.3), never re-refined (it is degenerate with the scale $G$; §11.2). + +2. **Unity** ($P_{ij}=1$): used for the scale-on-fulls refit (§10.6), where each observation is already a complete reflection. + +3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius (resolution-independent), the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature. The fit typically drives $\gamma_e\to0$, leaving the resolution-independent $\gamma_0$ as the effective width. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times. + +Reflections below a minimum partiality can be rejected from merging to avoid unstable corrections. + +### 10.3 Smoothing of per-frame scales + +The per-frame scales $G_i$ are fit by robust (Cauchy) inverse-variance-weighted ratios; there is no explicit $G\approx1$ prior. For rotation datasets, optional smoothing enforces the expectation that scale and mosaicity vary slowly across a sweep: **after** the per-frame fit, $\log G_i$ (and the mosaicity) are replaced by a centred **moving average** over a window spanning a configurable rotation range (XDS DELPHI-like; `--smooth-g`, default 5° for rot3d, off otherwise). It is a post-fit smoothing pass, not a curvature penalty inside the least-squares objective. (The per-frame scale refitted on the combined fulls, §10.6, is smoothed differently: by a penalised smoother whose smoothness is chosen by cross-validation.) + +The **crystal orientation** is smoothed the same way, and for the same reason. Geometry is re-refined independently on every frame against that frame's spots alone — as few as a dozen on a sparse crystal — so the per-frame orientation carries a real slow drift (crystal slippage, up to ~1.3° across a sweep) on top of fit noise that scales with spots per frame. Before scaling, the per-frame lattices are de-rotated to a common reference, averaged in frame order, rotated back, and every partial's $\Delta\phi$ — hence its partiality — is recomputed from the smoothed lattice. The window is chosen per dataset by leave-one-out cross-validation (does a frame's neighbours predict its orientation?) rather than fixed, because drift and noise both vary by two orders of magnitude between crystals; it is capped, because the per-frame fit also absorbs a real per-frame systematic that smoothing too wide destroys. Only frames that actually indexed take part: a frame that did not carries an all-zero lattice, which is *finite* and so passes a validity check written as a finite test, and would otherwise be both averaged into its neighbours' orientation and scored in the cross-validation that picks the window. Refining *less* is not an alternative: with per-image refinement off the space group is lost on several crystals. + +A per-frame scale enters every intensity as $1/G$, so a frame whose fit is not determined by its data can amplify it without bound — and $\sigma$ is amplified by the same factor, which makes it invisible to any $\sigma$-based outlier test. A fitted $G$ far below the run's median is therefore treated as *undetermined* rather than as a successful fit, both here and in the separate refit on the combined fulls (§10.6). The bound is a ratio to the run's own median because $G$ is not gauge-fixed: it and the merged means have an exact global multiplicative degeneracy, so no absolute value is meaningful. + +### 10.4 Merging estimator + +After refinement, corrected observations are formed: +$ +I^{\mathrm{corr}}_{ij} = \frac{I^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}},\qquad +\sigma^{\mathrm{corr}}_{ij} = \frac{\sigma^{\mathrm{obs}}_{ij}}{G_i L_{ij} P_{ij}}. +$ + +Unique intensities are merged by inverse-variance weighted mean: +$ +I_h = \frac{\sum_j w_j I^{\mathrm{corr}}_{ij}}{\sum_j w_j},\qquad +w_j = \frac{1}{(\sigma^{\mathrm{corr}}_{ij})^2}. +$ + +The weights use an **expected** variance: the Poisson signal part of each $\sigma^{\mathrm{corr}}_{ij}$ is rebuilt at the reflection's merged $\langle I\rangle$ rather than at that observation's own intensity. Weighting by an observation's own $\sigma^2$ biases the inverse-variance mean low below about one photon, because an up-fluctuated observation gets a larger sigma and is then down-weighted too hard. The rotation combine already does this; for stills it is on by default, and `--no-expected-variance-merge` restores the observed-sigma weighting. + +An internal-consistency term can inflate uncertainties when multiple observations are present, in the spirit of XSCALE. + +### 10.5 Merging statistics + +The shells are **nine bins of equal width in $1/d^2$**, laid between the **declared** low-resolution +limit (`--scaling-low-resolution`, or the whole sphere where it is switched off) and the +highest-resolution reflection the merge actually kept — XDS's rule and XDS's count, so at the same +resolution limits the two programs' tables have the same shell boundaries and can be read row for +row. `--resolution-shells` changes the count; the binning rule does not change with it. + +Per-shell and overall merging statistics are computed on corrected intensities, including: +- number of observations and of unique reflections, and multiplicity, +- mean $I/\sigma(I)$, +- $R_\mathrm{meas}$ (the redundancy-independent Diederichs–Karplus form) from within‑HKL deviations, +- $\mathrm{CC}_{1/2}$, correlating two half-sets of **equal size**: an observation's half is the parity of its rank among its own reflection's observations, ordered by a key built from the raw Miller index and the peak frame. Every reflection measured more than once therefore contributes (a hash of the image alone leaves $2^{1-n}$ of the multiplicity-$n$ reflections entirely in one half, with no second mean to correlate), and because a rank is a property of the set rather than of the order it is walked in, a CUDA build and a `JFJOCH_USE_CUDA=OFF` build report the same $\mathrm{CC}_{1/2}$ and the same $\mathrm{CC}_\mathrm{anom}$. It is the split cctbx's `compute_cc_one_half` — and `phenix.merging_statistics` through it — uses. The stills path balances its halves sequentially instead, in image order. When a reference dataset is supplied, $\mathrm{CC}_\mathrm{ref}$ is reported beside it, +- completeness against the reflections the cell and symmetry can give over that same declared range, + so low-resolution terms lost to the beam stop, to a detector mask or to the low-resolution limit + itself count as missing instead of leaving the denominator along with the data, +- the anomalous signal-to-noise $\mathrm{SigAno}$ and the half-set anomalous correlation $\mathrm{CC}_\mathrm{anom}$ (below). + +The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly. + +$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. + +**Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ($\mathrm{CC}_\mathrm{anom}$) is reported beside it: $\Delta I$ is formed once per half-set and the two are correlated over the acentric pairs where **both** hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic $\sigma$ cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS's `CCanom` and `phenix.merging_statistics`' `cc_anom`; XDS's `Anomal Corr` is a **different quantity and is not comparable with it** — on the same observations it reads two to three times higher in the low shells. $\mathrm{CC}_\mathrm{anom}$ is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the `CCanom` column of the printed merge-statistics table and as `CC_ANOM=` in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints `-` and the results report writes no key, which is not the same statement as a value measured to be zero. + +### 10.6 Rotation datasets: combining partials into fulls (3D integration) + +In a rotation scan a reflection is recorded as a series of *partials* spread across the frames its rocking curve crosses. Merging those partials directly would force the merge error model to absorb the rocking-curve slicing as if it were measurement noise, capping the achievable $I/\sigma$. For rotation data Jungfraujoch instead **combines** each reflection's partials into a single *full* intensity first, then scales and merges the fulls — a 3D integration over the rocking curve. + +The combine groups each reflection's partials into rocking events (contiguous runs of frames) and reduces each event to one full: + +- **De-biased weighted sum.** Partials are combined by inverse-variance weighting, where each partial's variance is its background-noise component plus the *model* signal shared across the event (Kabsch profile-fit form). Using the shared model signal rather than the individual down-fluctuating intensity stops weak partials from being over-weighted, which would otherwise inflate the merged error model. The weights depend on the full, so the estimate is iterated. +- **Captured fraction.** The partiality summed over the event, $f=\min(1,\sum_j p_j)$, measures how completely the rocking curve was sampled. A full whose curve was captured below a threshold (`--min-captured-fraction`, default 0.7 for rotation) is dropped — an event seen over only a small fraction of its curve is unreliable however many frames it spans. (The per-partial minimum-partiality cut of §10.2 still applies upstream, in the per-frame scaling.) +- **Per-image rejection (opt-in).** A frame whose observations correlate poorly with the merged reference is not measuring the crystal being merged — it may be off-crystal, or on a *different* crystal where two lattices occupy separate regions of the sample. `--min-image-cc` drops such frames. It has no default: the per-frame correlation measures data quality as much as frame validity, and its typical level varies widely between datasets, so no single absolute bound is generally valid. +- **Capture-aware uncertainty.** A full captured incompletely ($f<1$) is extrapolated and biased high. The unobserved fraction is charged as an extra systematic uncertainty, $\sigma^2 \leftarrow \sigma^2 + \big(c\,(1-f)\,I\big)^2$, so the merge down-weights these extrapolated fulls and the error model treats their scatter as expected. The merge rebuilds every full's variance at the reflection's mean intensity (§10.4), and the capture term is rebuilt there too, as $\big(c\,(1-f)\,\langle I\rangle\big)^2$. It is enabled by default for the rotation path. +- **Overloaded events.** An event in which any partial had a saturated pixel in its signal disk — or a pixel unreadable on that frame alone, beyond the run's pixel mask, which is how a detector that writes its error value for a pixel it could not count reports an overload — is dropped whole, as XDS drops an overloaded reflection. The brightest part of such a rocking curve is exactly what is missing, so neither the sum of the remaining partials nor their extrapolation by the partiality model measures the reflection: on a strongly diffracting small-molecule crystal these were the strongest low-order reflections, and they read 2–3× low. The integration keeps an overloaded partial, unfitted and flagged, only so that the event can be recognised; nothing else reads it. The count is `OBSERVATIONS_REJECTED_OVERLOAD=` in the report. + +The fulls are then re-scaled in the XDS sense — a per-image scale refit directly on the complete reflections under the unity partiality model — and merged (§10.4). + +Before the combine a per-frame scale can also be fitted on the partials themselves. That needs a frame to hold many rocking events caught at different points of their curves: within one rocking curve a change of scale and an error of the partiality model are the same thing, and on a finely sliced sparse sweep the fit takes one for the other. The rocking events per frame are the prior (the partials are scaled from 50 events per frame), but the counts of small-molecule sweeps (3–43) and proteins (5–900) overlap. So the first merge of a run that is not a space-group search can be made **both ways**, with the partials scaled and with the scale taken from the fulls alone, and the prior stands unless its merge has no resolved error model ($b$ not resolved from zero: the strong equivalents do not agree to within a measurable systematic error) while the other merge has one. Where the prior is to scale the partials and that merge resolves its error model, the other cannot change the choice and is not made. The two ISa values are deliberately not compared beyond that: the partiality-model error a partial scale takes up is shared by symmetry mates measured at the same rocking geometry, so their agreement cannot see it — a small-molecule sweep with six events per frame read ISa 10.5 with its partials scaled against 8.7 from the fulls alone, and refined to $R_1$ 0.105 against 0.062. The log states the choice and the ISa of every arm that was made. Because every merged observation is now a counting-statistics-limited full rather than a partiality-divided slice, the error model reaches a far higher asymptotic $I/\sigma$. + +How smooth that scale is over the rotation is left to the data rather than to a fixed window. Each round fits every frame on its own fulls against the current reference, giving a scale $G_f$ and its information $D_f=\sum w^2c^2$; the scale is then the curve $x=\log G$ minimising $\sum_f J_f\,(x_f-y_f)^2+\lambda\sum_f(\Delta^2 x)_f^2$, with $y_f$ the frame's own fit in log scale and $J_f$ its information carried there — a penalised (Whittaker–Eilers) smoother, solved as a five-band linear system. $\lambda$ is chosen by cross-validation: blocks of frames one rocking curve wide are left out in turn and predicted from the curve through the rest (neighbours closer than a rocking curve share their measurement, since a full sums those frames). A frame of hundreds of fulls is then followed frame by frame, a frame of two or three is carried by its neighbours, a stretch with none is bridged by a straight line, and a scale that falls a hundredfold over a few degrees — an absorbing crystal turning edge-on — is followed where a window would average across it. Once the curve settles, one free fit is shrunk toward it frame by frame by how much of each frame's deviation its neighbour shares (the lag-1 covariance), which hands back a real per-frame systematic and discards fit noise. + +After scale-fulls, five **correction surfaces** are fitted on the combined fulls (rotation path, **on by default**; disable all with `--no-scaling-corrections`), each an alternating multiplicative refinement of the per-full scale against the merged reference: + +- **Decay.** Radiation damage weakens later frames more at higher resolution — a resolution×time (Debye–Waller) systematic the resolution-flat per-image scale cannot capture. A single global relative-$B$ rate is fitted, $\ln(I_\mathrm{ref}/I_\mathrm{obs}) = 2\,(\mathrm{d}B/\mathrm{d}n)\,(n-\bar n)\,s^2$ (frame $n$, $s^2 = 1/4d^2$), and folded into the scale. It engages only when the total relative-$B$ over the run exceeds a physical floor (2 Ų); below that the decay is negligible and "correcting" it only spreads symmetry equivalents (same $s^2$, different frames). An optional **per-batch relative-$B$** (`--relative-b[=deg]`, off unless requested; 10°-of-rotation batches by default) extends the single global rate to a smooth $B(n)$ curve — the same $s^2$-weighted decay fit solved independently over short frame batches, curvature-penalized so it cannot over-fit and cross-validated like the surfaces below — for crystals whose decay is non-linear in dose. Its cross-validation splits on **ASU-group parity**, not the frame parity the surfaces below use: a per-batch parameter owns whole frames and so cannot be scored on a held-out frame, whereas splitting the symmetry equivalents tests whether a batch's $B$ generalises to reflections it was not fitted on. +- **Absorption.** A smooth multiplicative factor over the diffracted-beam direction expressed in the goniometer (crystal) frame: each full's predicted detector position gives the lab diffracted direction, de-rotated by the spindle so a fixed crystal-frame direction is sampled at many rotation angles and its grid cell is well-determined. Negligible at hard X-rays / thin crystals; it matters at low photon energy. +- **Modulation** (detector-plane flat-field). A smooth multiplicative factor over where each reflection lands on the detector (predicted $x,y$): symmetry-equivalents land at different positions as the crystal rotates, over-determining the surface. It absorbs detector-response and geometric systematics that inflate $R_\mathrm{meas}$. +- **Absorption as spherical harmonics.** The same crystal-frame absorption as a smooth function instead of a grid: the logarithm of the factor is a sum of real spherical harmonics of the de-rotated diffracted-beam direction, degrees 1 to 6 (48 terms; the incident-beam path depends on the rotation angle alone and is part of the per-image scale). It is fitted through 32 × 64 equal-solid-angle direction cells, one ridge-regularised least-squares step on the coefficients per round, with a prior of width 0.1/l on each degree-l coefficient. It is a candidate like the others and passes the same held-out test; the grid is then tested on what it left. +- **Time-dependent absorption.** The same surface as *Absorption*, but over (rotation angle × detector position) instead of the crystal-frame direction alone. The two agree while the illuminated volume stays put — the incident path then depends only on the spindle angle, which the per-image scale already takes, and the exit path is fixed in the crystal frame. Once the diffracting volume drifts through the beam the exit path becomes a function of the spindle angle as well, and nothing time-independent describes it. Fitted on 12 rotation bins × a 10×10 equal-occupancy detector grid. + +The surfaces overlap, so the order decides what is adopted: modulation first (every frame measures the static detector pattern, so its fine grid is the best determined), then time-dependent absorption, then the spherical-harmonic absorption, then the goniometer-frame grid, whose cells collect directions from the whole sweep and the whole resolution range and which, fitted first, takes up part of what the other two describe. + +Each cell's factor is fitted under a **prior pull to 1** (a Gaussian prior of width 0.1 on its logarithm): a cell moves off 1 in proportion to the information its observations carry, so a cell with little signal stays near 1 instead of being fitted to its noise. Negative observations enter the fit as measured. On the detector-plane surfaces (modulation, time-dependent absorption) the component that is a function of resolution alone is projected out within resolution shells, since the symmetry equivalents of a reflection share one resolution and such a factor cannot be determined from them. + +Each surface is **cross-validated** on the half-set agreement it is meant to improve: fitted on even-numbered frames and used to merge the odd ones, fitted on the odd frames and used to merge the even ones, and kept only if the correlation between the two half-set means, taken within resolution shells, rises above the same two halves merged with no surface. Each half is corrected by a surface it did not help to fit, so a surface fitted to noise lowers the correlation, and a correlation within a shell is blind to the resolution-dependent scale the surface cannot determine. The change is averaged over the shells on Fisher's $z = \operatorname{atanh}(CC)$, not on $CC$: the strong shells, where a multiplicative error matters, sit at $CC_{1/2} \approx 0.999$, where even a large reduction of the error moves $CC$ in the fourth decimal, and averaged on $CC$ itself the shells of pure noise beyond the reach of the data decide the sign. + +**Radiation-damage report (rotation, report-only).** Independently of whether any decay correction is applied, Rugnux measures and reports the relative Debye–Waller $B$ across the sweep: the per-image scale's correlation to the merge and the per-image mosaicity versus frame (dose), together with a per-batch relative-$B$ curve whose first→last change is a single headline number (measured before any decay correction, against the least-damaged early wedge). It is written to the log and to the merged mmCIF as a data-quality-vs-dose diagnostic and **never** alters the merged intensities — distinct from the decay correction above, which does fold into the scale. + +Each batch's $B$ is fitted on **resolution-shell means**, not on single observations: $\ln(I_\mathrm{ref}/I_\mathrm{obs})$ of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs $I_\mathrm{obs} > 0$, which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely *dimmer* than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as **absent** rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report ([the results report](RUGNUX_REPORT.md)) to name. + +**Frame disposition (rotation).** After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures **ΔCC1/2** — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run) with each reflection's error variance taken from the **observed** scatter of its own observations rather than from the error model's σ's: a bad stretch claims the same σ's as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group's own $(n, \sum I, \sum I^2)$ from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic's own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, $(1-CC^2)/\sqrt{n_\mathrm{refl}-3}$, and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is **both** several standard errors below zero (Fisher-transformed) **and** well below the run's own per-batch distribution (median − 3 robust σ, measured once before anything is removed), **and** where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame `merged` / `downgraded` / `rejected` ledger is described in [the results report](RUGNUX_REPORT.md). + +### 10.7 R-free test-set flags + +A fraction of the unique reflections (`rfree_fraction`, default 0.05) is flagged as a **free (test) set**, written to the output (MTZ `FreeR_flag`, mmCIF `_refln.status` = `f`, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection's orbit under the **lattice holohedry** — the point group of the cell's metric, found as twin laws are (Le Page two-folds within 3° obliquity, taking the lattice of the cell's own basis vectors), which contains the merging group — together with its Friedel mate. Where the cell does not carry the merging group (a space group forced on a metric without it), the Friedel-merged (Laue) ASU index of the merging group is used instead. That gives four properties: + +- all symmetry- and Friedel-equivalent reflections share one flag — in particular a Bijvoet pair $I(+)/I(-)$, kept as two separate merged rows in anomalous mode, is **never split** across the work and free sets (which would bias R-free); +- **twin-law mates share one flag too**, since a twin law is a lattice symmetry the crystal lacks. Keyed on the merging group instead, nearly every free reflection's twin mate lands in the working set, and in twin refinement the free reflection's calculated intensity then carries the working-set fit. The same set is what phenix.refine generates by default (`use_lattice_symmetry`). It also makes the free set independent of the space group a file is merged in: the merged file, the P1 cross-check and a re-merge in any subgroup carry one free set (nested, where the small-data floor below lifts the fraction of one of them more than another's); +- the free/work decision is a deterministic hash of that key, so the same reflection always lands in the same set — reproducible run-to-run and independent of the order in which observations were merged; +- the hash depends only on the reflection index, **not** on this dataset's resolution range or which reflections it happens to contain, so a uniform draw takes ~`rfree_fraction` of the distinct reflections free and — crucially — **every dataset of one crystal form gets the same free set**. That cross-dataset consistency is what a multi-dataset campaign (ensemble refinement, PanDDA) requires; a per-shell stratification tied to each dataset's own $d_\mathrm{min}$ would break it. + +On small data, where `rfree_fraction` (default 0.05) would give too few test reflections for a statistically stable R-free (Brünger's ~500–2000 rule), the fraction is **floored** so at least ~500 distinct reflections are free — capped at 10 % so a large test set never steals working data. For ordinary data this floor is inactive and the fraction stays flat at `rfree_fraction`, preserving the cross-dataset-identical property above; it only lifts the fraction on genuinely small datasets, where per-dataset R-free stability outweighs cross-dataset identity (and a shared reference set is the way to keep exact identity there). + +When a reference (`--reference`: an MTZ, or a PDB structure-factor mmCIF, which is converted to one with `_refln.status` `f`/`o` becoming flag 0/1) carries a `FreeR_flag` column, its test set is **imported** instead: every merged reflection whose Laue-ASU index matches the reference takes the reference's flag (reflections absent from the reference keep the hash flag). The match is made in the reference's frame — on rotation data the merge is first put onto the reference's axes, choosing among every description of the lattice on them (and every alternative indexing) by the intensity correlation with the reference — and only where the merge then matches it: an intensity CC of at least 0.5 over at least half of the merged reflections in the reference's resolution range. Below that the reference is not the same crystal form on the same axes, its flags would land on unrelated reflections, and they are not imported (`REFERENCE_MISMATCH`). This lets a whole fragment-screening campaign inherit one shared free set from the apo/reference dataset. The CCP4/refmac convention (test set = flag 0, including the historical 0–19 form) is assumed, with the complement taken automatically if flag 0 would be the majority (a phenix-style file where 1 marks free). + +### 10.8 French–Wilson amplitudes + +The last step of the merge estimates a Bayesian structure-factor amplitude $|F|$ for each unique reflection from its intensity $I$ and error $\sigma$, so the output carries amplitudes alongside intensities (a naïve $\sqrt{\max(I,0)}$ turns every weak or negative measurement into a biased — or zero — amplitude). With the Wilson prior for the true intensity $J\ge 0$ at that resolution, + +$ +P_\mathrm{acentric}(J) \propto e^{-J/\Sigma},\qquad +P_\mathrm{centric}(J) \propto J^{-1/2}\,e^{-J/2\Sigma}, +$ + +and a Gaussian likelihood $\mathcal{N}(I;J,\sigma^2)$, the posterior mean amplitude and its uncertainty are + +$ +\langle |F|\rangle = \frac{\int_0^\infty \sqrt{J}\,\mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J}{\int_0^\infty \mathcal{N}(I;J,\sigma^2)\,P(J)\,\mathrm{d}J},\qquad +\sigma_F = \sqrt{\langle J\rangle - \langle|F|\rangle^2}. +$ + +The prior mean is $\Sigma = \varepsilon\,K_\mathrm{shell}\,a(\mathbf{h})$, where $\varepsilon$ is the reflection's epsilon (symmetry-enhancement) multiplicity, $a(\mathbf{h}) = \exp(-\tfrac12\mathbf{s}^\mathsf{T}B\,\mathbf{s})$ carries the deviatoric anisotropy tensor $B$ of §13.5 along the reflection's own direction, and $K_\mathrm{shell} = \sum I/\varepsilon \,/ \sum a$ over its resolution shell, so the priors of a shell still average to its measured Wilson mean. The amplitudes are first made with $a = 1$ at the end of the merge and made again once the tensor has been fitted; only $F$/$\sigma_F$ change, never an intensity. With an isotropic prior the weak direction of an anisotropic crystal gets a prior set mostly by the strong direction, which turns its noise into amplitude; for isotropic data the tensor is near zero and the prior reduces to the shell mean, so it is used whenever a tensor was fitted. A shell whose $K$ is not positive takes that of the nearest lower-resolution shell. As in `ctruncate`, an intensity more than 3.7σ below zero gets no amplitude (the intensity is kept) and does not enter the shell mean. Strong reflections ($I>20\sigma$) short-circuit to $|F|=\sqrt{I}$, where the French–Wilson bias is below 0.3%; a reflection with an unusable $I/\sigma$ falls back to $\sqrt{\max(I,0)}$. The integral is evaluated numerically with a log-shift for stability. + +Amplitudes are written as MTZ `F`/`SIGF` (and `F(+)`/`F(-)`) and mmCIF `_refln.F_meas_au`/`F_meas_sigma_au`, alongside the intensity columns; the SHELX `.hkl` holds intensities only. The **same** $|F|$ feed the model-validation step (§14), so the reflection file and the maps use one consistent set of amplitudes. + +### 10.9 Reference data: fixing the space group and resolving the indexing ambiguity + +A reference dataset (`--reference`, MTZ or SF-mmCIF) supplies known intensities for the same crystal form, and is used in two ways. + +**Fix the space group and cell.** Unless overridden on the command line (`-S` for the space group, `-C` for the cell), the reference's space group is adopted and its cell is used as the soft reference cell — indexing may still drift the cell within tolerance, so a small mismatch between reference and data is absorbed rather than rejected. This applies to both stills and rotation data. + +**Resolve the indexing (merohedral) ambiguity.** When the lattice symmetry is higher than the crystal's Laue symmetry (e.g. $P3$, $P4$, $P6$, $C2$), more than one indexing of the same lattice is geometrically valid, and the two solutions produce *different* merged intensities that a self-consistent scale cannot tell apart — only an external reference can. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation $\mathrm{CC}_\mathrm{ref}$ of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is **metric-preserving** — only the $hkl$ labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined, and the whole merge is then repeated in the chosen indexing. For stills it has to be done **per image**, at integration time: each crystal is indexed independently, so a run resolves the ambiguity image by image (the image's partiality/Lorentz-corrected intensities are correlated with the reference under each candidate operator, which is scale-invariant, and the best-correlating one is adopted once and for good) — otherwise the merge would average reflections that are not symmetry mates. In neither workflow is the reference a **scale** target: both scale against their own data (§10.2), so $\mathrm{ISa}$ and the merging statistics come from the data alone and no cross-dataset systematic is imported. Because the stills choice is made at integration time, a later re-merge of stored reflections cannot repair a dataset integrated without a reference. Where there is no reference dataset but there is a **model** (`--model`), the reference intensities are computed from it instead - $|F_\mathrm{model}|^2$ from the atomic structure factors with a flat bulk-solvent contribution at the standard constants ($k_\mathrm{sol}=0.35$, $B_\mathrm{sol}=46$ Å$^2$), which are not fitted because there are no observations yet. Nothing is scaled against them; they serve only to rank the candidate indexings, and the correlation that does the ranking is scale-invariant. This needs the cell and space group up front (`-C` / `-S`, as serial indexing wants anyway); on rotation data the same job is done after the merge, in §14.5, where a merge exists to fit the model to properly. + +### 10.10 Ice rings at the scale and merge stages + +Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q = 2\pi/d$ (§1.2) of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are also left out of the **resolution-cut fit** (§13.4), which is the one consumer that reads the *merged* reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge's own accumulator (host and device alike) as well as on the stills one. They are otherwise **kept in the final merge**, which is also what the established scaling programs do by default, so the affected shells keep their completeness. + +Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge's own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged $\mathrm{CC}_{1/2}$ and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell. + +--- + +## 11. Mosaicity and “profile radius” monitoring + +### 11.1 Profile radius (intrinsic excitation-error width) + +The “profile radius” is the intrinsic angular width of a reflection — crystal mosaicity plus beam divergence — estimated from the spread of $\Delta_\mathrm{Ewald}$ over indexed spots, +$ +R \approx \sqrt{\tfrac{1}{N}\sum_i \Delta_{\mathrm{Ewald},i}^2}. +$ +When the beam has a finite energy bandwidth, that bandwidth smears each reflection radially by $\sigma_\mathrm{bw}\approx \mathrm{bandwidth}\cdot\lambda/2d^2$ (largest at high resolution), which also broadens the measured $\Delta_\mathrm{Ewald}$ spread. Since prediction re-applies the bandwidth term per reflection (§8.2), this contribution is deconvolved from the estimate — $R^2 = \langle\Delta_\mathrm{Ewald}^2\rangle - \langle\sigma_\mathrm{bw}^2\rangle$ — so that $R$ is the intrinsic width and bandwidth is not double-counted. Still predictions use an excitation-error cutoff proportional to $R$. + +### 11.2 Mosaicity from rotation data + +For rotation data the mosaicity $\sigma_M$ is estimated by maximum likelihood from the rocking offsets $\tau$ of indexed spots, using the XDS reflection-fraction model $R(\tau;\sigma_M/\zeta)$ (Kabsch 2010): each spot's exact Bragg angle is located near its frame, $\zeta$ (the rotation-axis Lorentz component) is computed, and $\sigma_M$ is chosen to maximize $\sum_i \log R(\tau_i;\sigma_M/\zeta_i)$. + +The $\phi$ search window for the Bragg angle is set **wider than the oscillation**, so that reflections recorded at large rocking offset are included. These tail reflections carry most of the information about the mosaic width; a window limited to the oscillation range would truncate the $\tau$ distribution and bias $\sigma_M$ low. + +The fit uses only the **strongest 250 spots** of an image, whatever the indexing spot budget (`--max-spots`) is. A spot is detected when $I_\mathrm{full}R(\tau)$ clears the finder threshold, so selecting spots by intensity censors on $R(\tau)$: a deeper list holds proportionally more large-$\tau$ partially recorded spots and the fit widens with it. Left uncapped, $\sigma_M$ therefore tracks the spot budget rather than the crystal — and since an over-wide mosaicity mis-states every partiality, the merge degrades sharply with it. + +The estimated mosaicity feeds the rotation prediction (how many frames each reflection spans, §8.3) and the rotation partiality (§10.2). It is **held fixed during scaling**: in the per-image scale fit the mosaicity is degenerate with the scale $G$ (both rescale the predicted intensity), so refining it there is unstable. A correct mosaicity matters because it controls both how much of each rocking curve is captured and the partiality used to form fulls (§10.6); too small a value truncates the captured curve and over-peaks the partiality, degrading the combined fulls. + +--- + +## 12. Auxiliary statistics: ⟨I/σ(I)⟩ and Wilson plot + +### 12.1 Per-shell ⟨I/σ(I)⟩ + +For monitoring integration quality, Jungfraujoch reports mean $\langle I/\sigma(I)\rangle$ in a fixed number of resolution shells. Shelling is performed in $1/d^2$ space (typical of crystallographic practice). + +### 12.2 Wilson plot (B-factor proxy) + +A Wilson-type analysis is computed by binning intensities by resolution and fitting: +$ +\langle I\rangle \propto \exp\!\left(-\frac{B}{2}\frac{1}{d^2}\right), +$ +i.e. +$ +\log \langle I\rangle = \mathrm{const} - \frac{B}{2}\left(\frac{1}{d^2}\right). +$ +A linear regression of $\log\langle I\rangle$ vs $1/d^2$ provides an estimate of $B$, subject to basic quality checks (e.g. $R^2$ threshold). + +A **dataset-wide** Wilson $B$ is also estimated over the merged reflections — restricted to the meaningful resolution range (skipping the low-resolution non-linear region below ~4 Å and shells past the signal limit $\langle I/\sigma\rangle < 1$, so it is insensitive to how far the merged data extend) — and written to the merged mmCIF as `_reflns.B_iso_Wilson_estimate` (and reported as `WILSON_B=`), the analogue of XDS's Wilson-line $B$. It is diagnostic only and is not fed back into scaling. +**It is the XDS convention, not the CCP4/Phenix one, and the two are not comparable.** The slope here is fitted to $\log\langle I\rangle$ directly; TRUNCATE, `ctruncate` and `phenix.xtriage` fit $\log(\langle I\rangle/\Sigma)$, dividing out $\Sigma=\sum_j f_j^2(s)$, the falloff of the atomic form factors for an assumed composition. Leaving $\Sigma$ in the slope inflates $B$ by roughly 2 to 8 Ų (measured across in-house merges; the arithmetic gives +6.9 Ų over 4.0-1.5 Å for a generic protein), and the fitted range accounts for more still: against `ctruncate` and `phenix.xtriage` on the same merged files this number runs 10 to 36 Ų high, in the same direction every time - though those two disagree with each other by 7 to 16 Ų, so there is a band rather than a right answer. Read it as a relative quantity, comparable between Rugnux runs and against XDS, and do not compare it with a value quoted from a CCP4 or Phenix log. The same caveat applies to `_reflns.B_iso_Wilson_estimate` in the merged mmCIF, whose deposited values are conventionally the $\Sigma$-normalised kind. It is also where the flux the fixed integration disk clips lands: that loss is degenerate with an overall $B$, so the reported number carries an $r_1$-dependent contribution and is not a property of the crystal alone (§9.1). The **per-image** estimate (used for the live radiation-damage plot) is accepted only when the fit is well-correlated and physically plausible ($0 < B < 200$ Ų); on a bad frame (an indexing glitch, too few reflections) the Wilson line runs wildly steep, so an implausible $B$ is reported as NaN rather than a spurious hundreds-of-Ų value. diff --git a/_sources/DEPLOYMENT.md.txt b/_sources/DEPLOYMENT.md.txt new file mode 100644 index 000000000..b9f1872be --- /dev/null +++ b/_sources/DEPLOYMENT.md.txt @@ -0,0 +1,203 @@ +# Deployment + +To deploy Jungfraujoch, one needs to follow these steps: + +1. Install main Jungfraujoch code and frontend web interface +2. Flash the U55C FPGA card with a proper image and install Linux kernel driver +3. Install Jungfraujoch writer +4. Install Jungfraujoch image viewer (optional) +5. Install Python OpenAPI client + +[`rugnux`](RUGNUX.md), the offline analysis tool, is installed separately and independently of +all of them — see [Install Rugnux](#install-rugnux-offline-analysis) at the end of this page. + +The installation procedure depends a lot on the operating system. For Red Hat Enterprise Linux 8/9, Rocky 8/9, +Ubuntu 22.04/24.04 or compatible, installation can be done with prebuilt packages from the +[package repositories](REPOSITORIES.md) and is relatively straightforward. For other systems one needs +to build software from source. Both ways will be presented. What each released package contains, and +what it needs on the machine, is described in [Release contents](RELEASE_CONTENTS.md). + + +## Install main Jungfraujoch code and frontend web interface + +On RHEL 8 systems there is a `jfjoch--1.el8.x86_64.rpm` that needs to be installed and contains all the necessary software and web interface. + +On other OSes one needs to compile Jungfraujoch from source (from the repo directory): +``` +$ mkdir build +$ cd build +$ cmake .. -DCMAKE_INSTALL_PREFIX= +$ make +$ sudo make install +``` +For manual installation, we recommend using a non-standard directory (like `/opt/jfjoch`), to facilitate upgrades and removal. +For DKMS to manage kernel module sources it is necessary to copy driver sources to `/usr/src/jfjoch-` directory. This requires an extra CMake flag `-DJFJOCH_INSTALL_DRIVER_SOURCE=ON`. + +Frontend web user interface has to be built separately with: +``` +$ cd build +$ make frontend +``` +Frontend files (.html and .js) will be placed in `frontend/dist` (outside of `build/` directory!) and have to be copied to a general location, e.g. `/usr/local/jfjoch/frontend` or `/opt/jfjoch/frontend`. + +## Flash the U55C FPGA card with a proper image and install Linux kernel driver + +### Firmware flashing +1. Check that the card is detected by OS with "lspci |grep Xilinx" and check the PCIe bus/device/function (BDF) number, `23:00.0` in this case: +``` +$ lspci |grep Xilinx +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 (rev 2) +``` +Note the device number `3450` that identifies Jungfraujoch device (Jungfraujoch pass is 3450 m above sea level) and `rev 2` identifying release of the firmware. + +2. Check the speed of the card, that it is detected as PCIe Gen4x8 device (needs to be done as root, otherwise configuration details are not given): +``` +$ sudo lspci -vv -s +23:00.0 Processing accelerators: Xilinx Corporation Device 3450 +(...) +LnkSta: Speed 16GT/s (ok), Width x8 (ok) +(...) +``` + +3. Download the MCS image from release files or build it using Vivado (WARNING! building time can be about 8 hours and doesn't always reach correct timing). +4. Flash the card with `xbflash.qspi` tool (part of Jungfraujoch). For fresh card use: +``` +sudo xbflash.qspi --primary --card --bar-offset 0x1f06000 +``` +For card that was already flashed with Jungfraujoch images: + +``` +sudo xbflash.qspi --primary --card +``` +It is necessary to confirm the operation by pressing `Y` key or one can add `--force` option to avoid confirmation. +It is safe to run multiple flashing processes in parallel for different cards, for example in separate screen sessions. + +5. Cold reboot: +``` +sudo ipmitool chassis power cycle +``` + +### Install PCIe driver + +For the first run it is recommended to try the driver without installing it into the kernel directory: +``` +$ cd fpga/pcie_driver +$ make +$ sudo insmod jfjoch.ko +``` + +Check with `dmesg` that the device was properly found: +``` +$ dmesg |grep jfjoch +[ 431.624933] jfjoch 0000:23:00.0: enabling device (0140 -> 0142) +[ 431.919147] misc jfjoch0: Jungfraujoch FPGA loaded with FW build: 5610030a +``` + +If things work, it is recommended to install the driver with DKMS, so it is rebuilt for kernel updates. +Install the prebuilt `jfjoch-driver-dkms` package from the +[Gitea package registry](REPOSITORIES.md); on other systems follow the procedure in +[PCIe driver](FPGA_PCIE_DRIVER.md). + +DKMS builds the module for the kernel it is being installed for rather than the running one, so a +module built during a kernel update loads correctly after the reboot. RHEL 9.5 and later — and their +CentOS Stream, Rocky and AlmaLinux equivalents — build unaided; the `HAVE_VM_FLAGS_SET` workaround +earlier releases needed is obsolete. + +NOTE: In case the driver is included in the init RAM-disk image, it is necessary to rebuild the RAM-disk when the driver is updated: +``` +$ sudo dracut -f +``` +### Configure network +Configure switch according to [FPGA network guide](FPGA_NETWORK.md) - specifically set manual speed and turn off auto-negotiation +for the port used to connect U55C card and connect card to switch. + +## Running Jungfraujoch software +Main Jungfraujoch service is called `jfjoch_broker`. It is responsible for handling data from FPGAs, doing processing, analysis, compression and sending images on ZeroMQ output. +It is recommended to run the service as `systemd` service. + +`jfjoch_broker` takes two parameters: JSON configuration file and HTTP port (default is 5232). +Example JSON files are placed in `etc/` folder. JSON file format is also explained in the OpenAPI definition, as `jfjoch_settings` data structure. + +When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring. + +Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one `jfjoch_broker` on a single machine, each confined to a disjoint subset of GPUs, set `CUDA_VISIBLE_DEVICES`; setting `CUDA_DEVICE_ORDER=PCI_BUS_ID` keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host: +``` +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232 +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233 +``` + +To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. +These need to be obtained from the PSI Detector Group. + +## Card verification + +To test that the FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_fpga_test` tool. +For example, to simulate a 10M pixel system with 4 FPGA cards and 200k images: +``` +jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 +``` +Or a 1M pixel system with one FPGA card: +``` +jfjoch_fpga_test ~/nextgendcu/ -m2 -s1 -i 200000 +``` + +## Install Jungfraujoch writer +Jungfraujoch writer is an additional service that connects to the `jfjoch_broker` ZeroMQ interface and writes files according to NeXus/NXmx HDF5 standard. + +At the moment it is better to have a separate machine, with access to a distributed file system, for writing images. + +Writer can be installed with a dedicated RPM file or compiled from source. For compilation, you can use the following commands: +``` +mkdir build +cd build +cmake -DJFJOCH_WRITER_ONLY=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch_writer +``` + +## Install Jungfraujoch image viewer +The Jungfraujoch viewer is an X-ray diffraction image viewer optimized to open Jungfraujoch HDF5 files. + +The viewer is a Qt application and it requires a recent version of the library, therefore it is an optional dependency. + +To include it in the building of Jungfraujoch use `-DJFJOCH_VIEWER_BUILD=ON` directive for CMake: +``` +mkdir build +cd build +cmake -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_INSTALL_PREFIX= .. +make jfjoch_viewer +``` + +Pre-built viewers for Windows and macOS, and a portable Linux archive, are on the Gitea release +page — see [Release contents](RELEASE_CONTENTS.md) and [jfjoch_viewer](JFJOCH_VIEWER.md). + + +## Install Jungfraujoch Python client +Use pip: +```shell +pip install jfjoch-client +``` + +## Install Rugnux (offline analysis) + +`rugnux` is not part of the server stack and is installed independently of all of the above. It +needs neither the broker, the writer, Qt nor a CUDA toolkit — only an NVIDIA driver if you want to +use the GPU — and it does not have to run on the acquisition machine at all. + +From the [package repositories](REPOSITORIES.md): + +``` +sudo dnf install rugnux # RHEL / Rocky +sudo apt install rugnux # Ubuntu +``` + +Or, on a machine no repository covers, from the standalone archive: + +``` +mkdir -p /opt/rugnux- +tar xzf rugnux--linux-x86_64-cuda12.tgz -C /opt/rugnux- +/opt/rugnux-/bin/rugnux +``` + +The archive has no top-level directory, so the `-C` is required. See +[Installing Rugnux](RUGNUX_INSTALL.md) for the Arm, Windows and macOS archives, the driver +versions and building from source. \ No newline at end of file diff --git a/_sources/DETECTORS.md.txt b/_sources/DETECTORS.md.txt new file mode 100644 index 000000000..bed139b03 --- /dev/null +++ b/_sources/DETECTORS.md.txt @@ -0,0 +1,15 @@ +# Supported detectors + +## PSI detectors +Jungfraujoch supports PSI JUNGFRAU and PSI EIGER detectors. Jungfraujoch controls the detector via `slsDetectorPackage`, which is statically compiled into its source code. +The detector firmware must match the `slsDetectorPackage` version used in Jungfraujoch. +The default is 8.0.2; 9.2.0 is built with the `SLS9=ON` CMake option and published in the `slsdet9` +[package repositories](REPOSITORIES.md). +See [PSI Detector group website](https://www.psi.ch/en/lxn/software-releases) for details. + +## DECTRIS detectors + +Jungfraujoch can be used with DECTRIS detectors, as a data analysis tool. +In this solution Jungfraujoch controls the Detector Control Unit (DCU) of the detector, and handles the output data stream of the DCU. +This mode, called "lite" mode, doesn't use FPGA boards, but mostly CPUs and GPUs for indexing. +The mode is currently experimental and intended for low data rates (100 Hz). diff --git a/_sources/DETECTOR_GEOMETRY.md.txt b/_sources/DETECTOR_GEOMETRY.md.txt new file mode 100644 index 000000000..f51440b90 --- /dev/null +++ b/_sources/DETECTOR_GEOMETRY.md.txt @@ -0,0 +1,133 @@ +# Detector geometry + +At the moment Jungfraujoch supports solely flat detectors. The default option is to place modules in their actual location +relative to the detector frame. It is not recommended to place detector modules stacked. + +The simplest case is a detector perpendicular to the beam. In this case it is enough to provide beam center, detector distance +and wavelength. + +For a more complex case, one can provide the detector tilt in the PyFAI convention. +This convention uses Point Of Nominal Interaction (PONI) definition. Beam X and Y would correspond to the location on the detector, +where beam from the sample is perpendicular to the detector surface and not to the actual direct beam location. Then tilt of the detector +is defined with three rotation angles: `rot1` (rotating detector right), `rot2` (rotating detector downwards), `rot3` (rotating detector clockwise). +See [PyFAI documentation](https://pyfai.readthedocs.io/en/stable/) for more details. + +## What a pixel coordinate means: (0, 0) is the centre of the first pixel + +Pixel coordinates in Jungfraujoch and Rugnux are **0-based and pixel-centred**: an integer coordinate +is the *centre* of that pixel, so pixel *i* covers [*i* − 0.5, *i* + 0.5) and the sensor spans +−0.5 … width − 0.5. A beam centre of 948.0 × 546.0 sits in the middle of pixel [546][948], not on any +of its corners; 948.5 is the boundary between pixel 948 and 949. + +This holds throughout the code: spot and reflection centroids are intensity-weighted sums of the +integer pixel indices, the resolution and azimuthal-bin maps evaluate pixel (col, row) at exactly +(col, row), and a fractional coordinate is turned back into a pixel index by rounding, not by +truncation. The same convention applies to every coordinate the system exposes — the beam centre +(`beam_x_pxl`/`beam_y_pxl` in the API and broker configuration, `--beam-x`/`--beam-y` in Rugnux, +`beam_center_x`/`beam_center_y` in NXmx and in the CBOR stream), the spot and predicted-reflection +positions written to HDF5, and the PONI reported by `--mode calibration`. + +Other programs place the origin differently, and the difference is worth half a pixel — enough to +matter when a geometry is copied between programs and then refined: + +| Convention | Beam centre equivalent to our *x* = 948.0 | +|---|---| +| Jungfraujoch, Rugnux | 948.0 | +| XDS (`ORGX`/`ORGY`) | 949.0 — also pixel-centred, but pixels are numbered from 1 | +| Measured from the edge of the sensor, in length units — pyFAI (`Poni2`, fast axis), DIALS/dxtbx | (948.0 + 0.5) × pixel size, because the centre of pixel *i* is at (*i* + 0.5) × pixel size from the edge | +| pyFAI `Poni1` (slow axis) | (height − 1 − *y* + 0.5) × pixel size — pyFAI measures the slow axis from the opposite edge, and the `.poni` declares `orientation: 2` to say so | + +The `.poni` file written by `rugnux --mode calibration` is in pyFAI's frame and so already carries +that half pixel; the pixel values the same run reports are ours. `Rot3` in that file is our rot3 +negated and turned by 180°: the half turn sets the azimuthal reference, because pyFAI's in-plane axes +are the negatives of ours. It leaves 2θ untouched, so it moves only the azimuth. + +## Inside: two axis vectors; outside: rot1/rot2/rot3 + +Internally the detector plane is one orthogonal matrix whose columns are the **fast axis** (the +laboratory direction of a +1 column step), the **slow axis** (+1 row step) and the **normal** (the +sample→PONI direction). Every geometry calculation — resolution, azimuth, polarization, prediction, +refinement — is that matrix applied to the offset of a pixel from the PONI. + +`rot1`/`rot2`/`rot3` remain the way the tilt is stated from outside, and the two views convert both +ways: `R = Rz(-rot3)·Rx(-rot2)·Ry(+rot1)` in the internal frame, and back from the columns as + +``` +rot2 = asin(-slow.z) rot1 = atan2(-fast.z, normal.z) rot3 = atan2(slow.x, slow.y) +``` + +with `rot2` in [-90°, 90°]. The angles are what is stored and what is written out, so a geometry +given as angles comes back exactly as it was given. + +## What a miniCBF header states about the mounting + +A PILATUS miniCBF gives the geometry twice. The `# ` lines every writer produces carry the distance, the +beam centre and the angles; some beamlines then append a CBF template block holding a full **imgCIF axis +table**, which states the laboratory direction of the image's fast and slow pixel directions, of the base +goniometer axis, and of a 2theta arm where there is one. Where that table is present it is read, in +preference to any assumption - it is the same information NXmx puts in `fast_pixel_direction` / +`slow_pixel_direction` and the goniometer `vector`, in the form this format states it. + +imgCIF's laboratory frame has Z from the sample towards the source and Y opposite gravity, so it differs +from the internal frame by a half turn about x - a rotation, not a mirror, so an axis carried through it +turns the same way by the same angle. + +There are two things a header can state that an assumption gets wrong by 90 degrees — an error no +refinement recovers, and one the run's axis-sign rescue cannot reach either, a quarter turn not +being a sign: + +* the image mounted a quarter turn round, so its columns run vertically; +* a spindle that turns about the **vertical** rather than the horizontal. + +Where a header carries no axis table, a `+SLOW` on its `# Oscillation_axis` line still says the spindle +runs along the image's slow direction rather than its fast one. The axis *name* on that line is not +usable - one header says `X.CW +SLOW` where its own table says the axis is Y - but the direction token is, +and on the header that states both they agree. + +## A detector swung out on a 2theta arm + +Chemical crystallography reaches high angle by swinging the detector out on a 2theta arm rather than by +moving it closer. The arm turns the detector about the sample, so it changes nothing else: the distance +is still measured along the detector normal, and the beam centre is still the point of normal incidence, +which is where the arm's own axis meets the detector and does not move. The swing is therefore exactly a +PONI rotation, and the direct beam is what moves - by `distance * tan(2theta)`, off the beam centre and +often off the detector altogether. + +Nothing has to be given for this: Rugnux takes it from the file. An NXmx master states the detector's +position as a `depends_on` chain of transformations, and the arm is one rotation in that chain - so the +chain is followed, rather than a field of one particular name being looked for. A PILATUS miniCBF states +it as `# Detector_2theta`, which turns about the same axis as the base spindle, the two being one axis on +the four-circle geometry those headers describe. + +## Mirrored and quarter-turned detectors + +On top of the continuous tilt the detector setup carries a **discrete image orientation**: whether the +stored image is mirrored in Y, and how many multiples of 90° about the beam it is turned by. It is +applied to the offset from the PONI before the tilt. + +The distinction matters because these two operations are exact pixel remappings — an image can be +shown the right way up without resampling anything — while an arbitrary in-plane rotation cannot. +`rot3` is therefore reserved for the genuinely arbitrary part: an in-plane angle is **never** +decomposed into a quarter turn plus a residual, and the discrete part is set only where something +states it (the detector configuration, `--detector-mirror-y` / `--detector-quarter-turns`, or the +value a Jungfraujoch-written file records). + +Both operations leave the distance from the PONI unchanged, so resolution, the solid-angle correction +and anything else that needs only a radius are unaffected by them. Polarization *is* affected, and +correctly so: it is computed from the azimuth in the **laboratory**, and what these operations change +is which pixel index lands at which laboratory azimuth. + +This is a different setting from `mirror_y` in the JSON configuration file (described below), which flips the +**module layout** while the image is being assembled and so decides what the stored pixels are. The +discrete image orientation changes no pixel at all. + +## Macromolecular crystallography convention for the vertical direction +One place of confusion is the convention to have point (0,0) of the detector in the top left corner of the detector, +with Y values increasing downwards. This is also consistent with computer image formats. + +However, other techniques (as well as internal operation of PSI X-ray detectors) might follow a convention where point (0,0) +is in the bottom left corner and Y values increase upwards. Such a convention is used, for example, by PyFAI. + +In general, the convention is controlled in Jungfraujoch with a setting in the JSON configuration file, which allows the detector to be mirrored in Y. + +The convention in use is worth checking whenever a geometry is carried between programs. \ No newline at end of file diff --git a/_sources/EXTERNAL_TEST_DATA.md.txt b/_sources/EXTERNAL_TEST_DATA.md.txt new file mode 100644 index 000000000..8eed9c53f --- /dev/null +++ b/_sources/EXTERNAL_TEST_DATA.md.txt @@ -0,0 +1,554 @@ +# External test data + +Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeline that only +ever sees its own detectors is not tested. The datasets below were collected by other people, +on detectors and in file formats we do not produce ourselves, and are used here to check that +`rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected +at other facilities, ten on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone +else's detector and someone else's acquisition system. Their authors published all of these for +exactly this kind of reuse, and this page is where we credit them. + +**None of these data were collected by us.** If you use any of them, cite the dataset DOI in +the table below; the repositories themselves are cited in +[ACKNOWLEDGEMENT](ACKNOWLEDGEMENT.md). + +## Where the values come from + +- **Source** is the repository we downloaded from and that repository's own citable DOI for + the archive we took. Every DOI on this page was resolved against DataCite - or, for 6NEN, + whose DOI is registered with Crossref, against Crossref - before it was written down, and the identity of each dataset was taken from the repository's record for + the archive - not from our directory names. +- **Beamline, resolution, space group and cell are the values deposited with the PDB entry**, + read from the RCSB data API. They describe the published experiment. They are *not* our + reprocessing results; no quantity measured by Jungfraujoch appears on this page. +- **Detector is read out of the image files themselves** - the NXmx + `/entry/instrument/detector/description`, the miniCBF `# Detector:` header, the marCCD + instrument header or the SMV key block - which is authoritative where the PDB entry names a + different detector. +- Anything that could not be established from one of those sources is left blank. + +## Datasets + +| PDB | Source | Facility / beamline | dmin (Å) | Space group | Unit cell a b c α β γ (Å, °) | Detector (from file) | Title | +|---|---|---|---|---|---|---|---| +| [11IF](https://www.rcsb.org/structure/11IF) | IRRMC [10.18430/M311IF](https://doi.org/10.18430/M311IF) | NSLS-II 19-ID | 1.51 | P 43 | 51.1 51.1 71.9 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of an exported phospholipid binding protein from Bordetella pertussis in complex with Di-palmitoyl-3-sn-phosphatidylethanolamine (DPPE), P43 form 2 | +| [36GK](https://www.rcsb.org/structure/36GK) | IRRMC [10.18430/M336GK](https://doi.org/10.18430/M336GK) | CLSI 08ID-1 | 2.28 | I 2 2 2 | 120.6 189.5 199.7 90.0 90.0 90.0 | Dectris Eiger 9M | D-GlcNAc-bound structure of Vibrio vulnificus putative carbohydrate binding module and split domain | +| [3INP](https://www.rcsb.org/structure/3INP) | IRRMC [10.18430/m33inp](https://doi.org/10.18430/m33inp) | APS 21-ID-F | 2.05 | F 41 3 2 | 224.1 224.1 224.1 90.0 90.0 90.0 | marCCD, 225 mm plate | 2.05 Angstrom Resolution Crystal Structure of D-ribulose-phosphate 3-epimerase from Francisella tularensis. | +| [3KY7](https://www.rcsb.org/structure/3KY7) | IRRMC [10.18430/m33ky7](https://doi.org/10.18430/m33ky7) | APS 21-ID-G | 2.35 | P 43 3 2 | 125.2 125.2 125.2 90.0 90.0 90.0 | marCCD, 300 mm plate | 2.35 Angstrom resolution crystal structure of a putative tRNA (guanine-7-)-methyltransferase (trmD) from Staphylococcus aureus subsp. aureus MRSA252 | +| [3MC4](https://www.rcsb.org/structure/3MC4) | IRRMC [10.18430/M33MC4](https://doi.org/10.18430/M33MC4) | Home source, Rigaku MicroMax-007 HF | 1.95 | H 3 | 104.0 104.0 105.5 90.0 90.0 120.0 | Rigaku Saturn 944+ | Crystal structure of WW/RSP5/WWP domain: bacterial transferase hexapeptide repeat: serine O-Acetyltransferase from Brucella Melitensis | +| [3MEB](https://www.rcsb.org/structure/3MEB) | IRRMC [10.18430/M33MEB](https://doi.org/10.18430/M33MEB) | Home source, Rigaku MicroMax-007 HF | 1.90 | P 1 21 1 | 58.6 101.2 81.5 90.0 90.6 90.0 | Rigaku Saturn 944 | Structure of cytoplasmic aspartate aminotransferase from giardia lamblia | +| [3P85](https://www.rcsb.org/structure/3P85) | IRRMC [10.18430/M33P85](https://doi.org/10.18430/M33P85) | Home source, Rigaku FR-E+ SuperBright | 1.90 | P 63 2 2 | 127.3 127.3 72.9 90.0 90.0 120.0 | Rigaku Saturn 944+ | Crystal structure enoyl-coa hydratase from mycobacterium avium | +| [3R6O](https://www.rcsb.org/structure/3R6O) | IRRMC [10.18430/M33R6O](https://doi.org/10.18430/M33R6O) | Home source, Rigaku FR-E+ SuperBright | 1.95 | I 41 | 90.7 90.7 76.1 90.0 90.0 90.0 | Rigaku Saturn 944+ | Crystal structure of a probable 2-hydroxyhepta-2,4-diene-1, 7-dioateisomerase from Mycobacterium abscessus | +| [5CC8](https://www.rcsb.org/structure/5CC8) | IRRMC [10.18430/M35CC8](https://doi.org/10.18430/M35CC8) | Home source, Rigaku MicroMax-007 HF | 1.75 | P 21 21 2 | 87.1 93.8 72.5 90.0 90.0 90.0 | Rigaku Saturn 944+ | Structure of thiamine-monophosphate kinase from Acinetobacter baumannii in complex with AMPPNP | +| [5EBI](https://www.rcsb.org/structure/5EBI) | MXRDR [10.18150/9887707](https://doi.org/10.18150/9887707) | BESSY 14.2 | 1.09 | P 1 21 1 | 35.7 44.1 35.7 90.0 120.0 90.0 | marCCD, 225 mm plate | Crystal structure of a DNA-RNA chimera in complex with Ba2+ ions: a case of unusual multi-domain twinning | +| [5EPE](https://www.rcsb.org/structure/5EPE) | IRRMC [10.18430/m3159c](https://doi.org/10.18430/m3159c) | APS 21-ID-G | 1.90 | F 2 3 | 157.5 157.5 157.5 90.0 90.0 90.0 | Rayonix MX-300 | Crystal structure of SAM-dependent methyltransferase from Thiobacillus denitrificans in complex with S-Adenosyl-L-homocysteine | +| [5F6M](https://www.rcsb.org/structure/5F6M) | SBGrid [10.15785/sbgrid/201](https://doi.org/10.15785/sbgrid/201) | SSRL BL11-1 | 1.10 | P 21 21 21 | 54.8 58.5 67.4 90.0 90.0 90.0 | PILATUS 6M | Isotropic Trypsin Model for Comparison of Diffuse Scattering | +| [5J23](https://www.rcsb.org/structure/5J23) | IRRMC [10.18430/M35J23](https://doi.org/10.18430/M35J23) | APS 21-ID-G | 2.30 | H 3 | 175.8 175.8 136.8 90.0 90.0 120.0 | Rayonix MX-300 | Crystal structure of NADPH-dependent glyoxylate/hydroxypyruvate reductase SMc04462 (SmGhrB) from Sinorhizobium meliloti in complex with 2'-phospho-ADP-ribose | +| [5JK4](https://www.rcsb.org/structure/5JK4) | Zenodo [10.5281/zenodo.49859](https://doi.org/10.5281/zenodo.49859) | ESRF ID14-2 | 1.10 | P 1 21 1 | 37.7 77.9 56.3 90.0 102.1 90.0 | ADSC Quantum 4 | Phosphate-Binding Protein from Stenotrophomonas maltophilia. | +| [5JVN](https://www.rcsb.org/structure/5JVN) | IRRMC [10.18430/m35jvn](https://doi.org/10.18430/m35jvn) | ESRF ID29 | 2.90 | P 6 2 2 | 249.4 249.4 84.1 90.0 90.0 120.0 | PILATUS3 6M | C3-type pyruvate phosphate dikinase: intermediate state of the swiveling-domain mechanism | +| [5KY6](https://www.rcsb.org/structure/5KY6) | MXRDR [10.18150/repod.1494374](https://doi.org/10.18150/repod.1494374) | BESSY 14.2 | 1.94 | P 1 21 1 | 84.5 57.3 164.0 90.0 102.6 90.0 | marCCD, 225 mm plate | Human muscle fructose-1,6-bisphosphate aldolase | +| [5LZL](https://www.rcsb.org/structure/5LZL) | Zenodo [10.5281/zenodo.54757](https://doi.org/10.5281/zenodo.54757) | Diamond I02 | 3.47 | P 31 2 1 | 205.6 205.6 199.2 90.0 90.0 120.0 | PILATUS 6M-F | Pyrobaculum calidifontis 5-aminolaevulinic acid dehydratase | +| [5M17](https://www.rcsb.org/structure/5M17) | Zenodo [10.5281/zenodo.4300323](https://doi.org/10.5281/zenodo.4300323) | Diamond I02 | 1.03 | I 4 | 108.6 108.6 67.7 90.0 90.0 90.0 | PILATUS 6M-F | Structure of the GH99 endo-alpha-mannanase from Bacteroides xylanisolvens | +| [5MLN](https://www.rcsb.org/structure/5MLN) | IRRMC [10.18430/m35mln](https://doi.org/10.18430/m35mln) | ESRF ID23-2 | 1.60 | P 21 2 21 | 74.2 80.4 80.5 90.0 90.0 90.0 | PILATUS3 2M | The crystal structure of alcohol dehydrogenase 10 from Candida magnoliae | +| [5NW5](https://www.rcsb.org/structure/5NW5) | SBGrid [10.15785/sbgrid/446](https://doi.org/10.15785/sbgrid/446) | SLS X06DA | 6.50 | P 21 21 21 | 92.1 169.8 390.2 90.0 90.0 90.0 | PILATUS 2MF | Crystal structure of the Rif1 N-terminal domain (RIF1-NTD) from Saccharomyces cerevisiae in complex with DNA | +| [5REO](https://www.rcsb.org/structure/5REO) | Zenodo [10.5281/zenodo.3730956](https://doi.org/10.5281/zenodo.3730956) | Diamond I04-1 | 1.88 | C 1 2 1 | 112.4 52.6 44.4 90.0 103.0 90.0 | PILATUS 6M-F | PanDDA analysis group deposition -- Crystal Structure of SARS-CoV-2 main protease in complex with PCM-0102578 | +| [5SRC](https://www.rcsb.org/structure/5SRC) | IRRMC [10.18430/M35SRC](https://doi.org/10.18430/M35SRC) | ALS 8.3.1 | 1.05 | P 43 | 88.7 88.7 39.2 90.0 90.0 90.0 | PILATUS3 6M | PanDDA analysis group deposition -- Crystal structure of SARS-CoV-2 NSP3 macrodomain in complex with Z5198562500 - (R,R) and (R,S) isomers | +| [5T39](https://www.rcsb.org/structure/5T39) | SBGrid [10.15785/sbgrid/356](https://doi.org/10.15785/sbgrid/356) | APS 21-ID-F | 1.10 | P 1 21 1 | 50.2 41.3 58.5 90.0 98.6 90.0 | Rayonix MX-300 | Crystal Structure of the N-terminal domain of EvdMO1 in the presence of SAH and D-fucose | +| [5UTH](https://www.rcsb.org/structure/5UTH) | IRRMC [10.18430/M35UTH](https://doi.org/10.18430/M35UTH) | Home source, Rigaku FR-E+ SuperBright | 1.95 | P 31 2 1 | 69.3 69.3 153.8 90.0 90.0 120.0 | Rigaku Saturn 944+ | Crystal structure of thioredoxin reductase from Mycobacterium smegmatis in complex with FAD | +| [5VML](https://www.rcsb.org/structure/5VML) | IRRMC [10.18430/M35VML](https://doi.org/10.18430/M35VML) | Home source, Rigaku FR-E+ SuperBright | 1.70 | P 42 21 2 | 66.3 66.3 115.3 90.0 90.0 90.0 | Rigaku Saturn 944+ | Crystal Structure of Acetoacetyl-CoA Reductase from Burkholderia Pseudomallei 1710b with bound NADP | +| [6CDL](https://www.rcsb.org/structure/6CDL) | IRRMC [10.18430/m36cdl](https://doi.org/10.18430/m36cdl) | APS 22-ID | 1.25 | P 21 21 2 | 58.3 85.9 46.1 90.0 90.0 90.0 | marCCD, 300 mm plate | HIV-1 wild type protease with GRL-03214A, 6-5-5-ring fused umbrella-like tetrahydropyranofuran as the P2-ligand, a cyclopropylaminobenzothiazole as the P2'-ligand and 3,5-difluorophenylmethyl as the P1-ligand | +| [6CEE](https://www.rcsb.org/structure/6CEE) | IRRMC [10.18430/M36CEE](https://doi.org/10.18430/M36CEE) | Home source, Rigaku FR-E SuperBright | 1.55 | P 21 21 21 | 40.7 44.1 55.9 90.0 90.0 90.0 | Rigaku Saturn A200 | Crystal structure of fragment 3-(1-Methyl-2-oxo-1,2-dihydroquinoxalin-3-yl)propionic acid bound in the ubiquitin binding pocket of the HDAC6 zinc-finger domain | +| [6CS9](https://www.rcsb.org/structure/6CS9) | SBGrid [10.15785/SBGRID/568](https://doi.org/10.15785/SBGRID/568) | Australian Synchrotron MX2 | 1.85 | P 1 21 1 | 32.9 25.5 40.2 90.0 98.6 90.0 | ADSC Quantum 210r | Crystal structure of human beta-defensin 2 in complex with PIP2 | +| [6F3P](https://www.rcsb.org/structure/6F3P) | IRRMC [10.18430/M36F3P](https://doi.org/10.18430/M36F3P) | APS 22-ID | 1.35 | C 1 2 1 | 142.9 85.7 112.0 90.0 122.2 90.0 | marCCD, 300 mm plate | Crystal structure of S-adenosyl-L-homocysteine hydrolase from Pseudomonas aeruginosa in complex with 3'-deoxyadenosine and K+ cation | +| [6FID](https://www.rcsb.org/structure/6FID) | SBGrid [10.15785/sbgrid/541](https://doi.org/10.15785/sbgrid/541) | ESRF ID30B | 2.20 | P 21 21 21 | 59.9 64.1 69.7 90.0 90.0 90.0 | PILATUS3 6M | Bovine trypsin solved by S-SAD on ID30B | +| [6FVZ](https://www.rcsb.org/structure/6FVZ) | IRRMC [10.18430/m36fvz](https://doi.org/10.18430/m36fvz) | ESRF ID23-2 | 1.80 | C 2 2 2 | 131.2 222.8 86.5 90.0 90.0 90.0 | PILATUS3 X 2M | Crystal structure of human monoamine oxidase B (MAO B) in complex with an inhibitor | +| [6FWC](https://www.rcsb.org/structure/6FWC) | IRRMC [10.18430/m36fwc](https://doi.org/10.18430/m36fwc) | ESRF MASSIF-3 | 1.70 | C 2 2 2 | 131.7 222.1 86.3 90.0 90.0 90.0 | PILATUS 2MF | Crystal structure of human monoamine oxidase B (MAO B) in complex with fluorophenyl-chromone-carboxamide | +| [6G1F](https://www.rcsb.org/structure/6G1F) | Zenodo [10.5281/zenodo.1059413](https://doi.org/10.5281/zenodo.1059413) | Diamond I03 | 2.25 | C 1 2 1 | 329.3 83.9 133.4 90.0 111.6 90.0 | PILATUS3 6M | Crystal structure of D-phenylglycine aninotransferase (D-PhgAT) from Pseudomonas stutzeri with PLP internal aldimine | +| [6GVK](https://www.rcsb.org/structure/6GVK) | Zenodo [10.5281/zenodo.1286854](https://doi.org/10.5281/zenodo.1286854) | ALBA XALOC | 1.55 | C 1 2 1 | 105.6 59.5 42.4 90.0 113.5 90.0 | PILATUS 6M | Second pair of Fibronectin type III domains of integrin beta4 (T1663R mutant) bound to the bullous pemphigoid antigen BP230 (BPAG1e) | +| [6H2P](https://www.rcsb.org/structure/6H2P) | IRRMC [10.18430/m36h2p](https://doi.org/10.18430/m36h2p) | BESSY 14.1 | 1.48 | C 2 2 21 | 103.5 107.1 216.5 90.0 90.0 90.0 | PILATUS 6M | Crystal Structure of Arg184Gln mutant of Human Prolidase with Mn ions and Cacodylate ligand | +| [6H5T](https://www.rcsb.org/structure/6H5T) | IRRMC [10.18430/m36h5t](https://doi.org/10.18430/m36h5t) | BESSY 14.3 | 1.69 | I 4 2 2 | 86.8 86.8 141.8 90.0 90.0 90.0 | marCCD, 225 mm plate | Intersectin SH3A short isoform | +| [6HV2](https://www.rcsb.org/structure/6HV2) | IRRMC [10.18430/m36hv2](https://doi.org/10.18430/m36hv2) | SLS X06SA | 1.71 | P 61 2 2 | 68.9 68.9 133.6 90.0 90.0 120.0 | Dectris Eiger 16M | MMP-13 in complex with the peptide IMISF | +| [6HWJ](https://www.rcsb.org/structure/6HWJ) | SBGrid [10.15785/sbgrid/614](https://doi.org/10.15785/sbgrid/614) | ALBA XALOC | 1.98 | P 1 21 1 | 59.8 96.1 80.3 90.0 106.7 90.0 | PILATUS 6M | Glucosamine kinase (crystal form A) | +| [6I3J](https://www.rcsb.org/structure/6I3J) | IRRMC [10.18430/m36i3j](https://doi.org/10.18430/m36i3j) | BESSY 14.1 | 2.59 | F 2 2 2 | 134.4 203.8 226.7 90.0 90.0 90.0 | marCCD, 225 mm plate | Bilirubin oxidase from Myrothecium verrucaria in complex with ferricyanide | +| [6IU5](https://www.rcsb.org/structure/6IU5) | Zenodo [10.5281/zenodo.2532134](https://doi.org/10.5281/zenodo.2532134) | SPring-8 BL41XU | 2.25 | P 31 | 84.9 84.9 98.2 90.0 90.0 120.0 | PILATUS3 6M | Crystal structure of cytoplasmic metal binding domain with zinc ions | +| [6IU6](https://www.rcsb.org/structure/6IU6) | Zenodo [10.5281/zenodo.2532134](https://doi.org/10.5281/zenodo.2532134) | SPring-8 BL41XU | 2.90 | P 31 | 84.7 84.7 97.4 90.0 90.0 120.0 | PILATUS3 6M | Crystal structure of cytoplasmic metal binding domain with nickel ions | +| [6IU8](https://www.rcsb.org/structure/6IU8) | Zenodo [10.5281/zenodo.2532134](https://doi.org/10.5281/zenodo.2532134) | SPring-8 BL41XU | 2.70 | P 31 | 85.5 85.5 98.4 90.0 90.0 120.0 | PILATUS3 6M | Crystal structure of cytoplasmic metal binding domain with cobalt | +| [6IU9](https://www.rcsb.org/structure/6IU9) | Zenodo [10.5281/zenodo.2532134](https://doi.org/10.5281/zenodo.2532134) | SPring-8 BL41XU | 3.00 | P 31 | 85.3 85.3 97.6 90.0 90.0 120.0 | PILATUS3 6M | Crystal structure of cytoplasmic metal binding domain with iron ions | +| [6JGH](https://www.rcsb.org/structure/6JGH) | IRRMC [10.18430/m36jgh](https://doi.org/10.18430/m36jgh) | SPring-8 BL44XU | 0.94 | P 21 21 21 | 50.6 62.5 68.2 90.0 90.0 90.0 | marCCD, 300 mm plate | Crystal structure of the F99S/M153T/V163A/T203I variant of GFP at 0.94 A | +| [6JGI](https://www.rcsb.org/structure/6JGI) | IRRMC [10.18430/m36jgi](https://doi.org/10.18430/m36jgi) | SPring-8 BL44XU | 0.85 | P 21 21 21 | 50.9 62.4 69.2 90.0 90.0 90.0 | marCCD, 300 mm plate | Crystal structure of the S65T/F99S/M153T/V163A variant of GFP at 0.85 A | +| [6JGJ](https://www.rcsb.org/structure/6JGJ) | IRRMC [10.18430/m36jgj](https://doi.org/10.18430/m36jgj) | SPring-8 BL41XU | 0.77 | P 21 21 21 | 50.9 62.3 68.8 90.0 90.0 90.0 | PILATUS3 300K | Crystal structure of the F99S/M153T/V163A/E222Q variant of GFP at 0.78 A | +| [6MOJ](https://www.rcsb.org/structure/6MOJ) | SBGrid [10.15785/sbgrid/620](https://doi.org/10.15785/sbgrid/620) | ALS 5.0.1 | 2.43 | I 41 2 2 | 130.4 130.4 293.5 90.0 90.0 90.0 | PILATUS3 6M | Dimeric DARPin A_angle_R5 complex with EpoR | +| [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC | +| [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset | +| [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor | +| [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 | +| [6PB3](https://www.rcsb.org/structure/6PB3) | SBGrid [10.15785/sbgrid/681](https://doi.org/10.15785/sbgrid/681) | APS 24-ID-E | 2.05 | P 6 | 100.4 100.4 48.9 90.0 90.0 120.0 | Dectris Eiger 16M | Structure of Rhizobiales Trip13 | +| [6PXB](https://www.rcsb.org/structure/6PXB) | SBGrid [10.15785/sbgrid/698](https://doi.org/10.15785/sbgrid/698) | APS 24-ID-E | 1.75 | P 32 | 64.0 64.0 119.4 90.0 90.0 120.0 | PILATUS 6M-F | N-Terminal SH2 domain of the p120RasGAP | +| [6PXC](https://www.rcsb.org/structure/6PXC) | SBGrid [10.15785/sbgrid/699](https://doi.org/10.15785/sbgrid/699) | APS 24-ID-E | 1.60 | I 2 2 2 | 44.2 64.8 87.2 90.0 90.0 90.0 | PILATUS 6M-F | N-Terminal SH2 domain of the p120RasGAP bound to a p190RhoGAP phosphotyrosine peptide | +| [6QAJ](https://www.rcsb.org/structure/6QAJ) | SBGrid [10.15785/sbgrid/637](https://doi.org/10.15785/sbgrid/637) | Diamond I03 | 2.90 | C 2 2 21 | 59.8 169.3 374.5 90.0 90.0 90.0 | PILATUS3 6M | Structure of the tripartite motif of KAP1/TRIM28 | +| [6R72](https://www.rcsb.org/structure/6R72) | Zenodo [10.5281/zenodo.14894181](https://doi.org/10.5281/zenodo.14894181) | SOLEIL PROXIMA 2 | 3.95 | P 1 21 1 | 117.8 110.8 155.6 90.0 93.2 90.0 | Dectris Eiger 9M | Crystal structure of BmrA-E504A in an outward-facing conformation | +| [6RLR](https://www.rcsb.org/structure/6RLR) | Zenodo [10.5281/zenodo.5886687](https://doi.org/10.5281/zenodo.5886687) | Diamond I04 | 2.00 | P 1 | 40.0 40.0 63.6 80.4 76.3 68.2 | Eiger 16M | Crystal structure of CD9 large extracellular loop | +| [6RYM](https://www.rcsb.org/structure/6RYM) | Keele University [10.21252/xbsq-d621](https://doi.org/10.21252/xbsq-d621) | SRS PX10.1 (Daresbury) | 1.46 | P 43 | 50.2 50.2 51.9 90.0 90.0 90.0 | marCCD 165 mm | Structure of carbohydrate recognition domain with GlcNAc bound | +| [6S1U](https://www.rcsb.org/structure/6S1U) | MXRDR [10.18150/repod.0005795](https://doi.org/10.18150/repod.0005795) | BESSY 14.2 | 1.90 | P 1 21 1 | 51.6 29.4 85.5 90.0 103.8 90.0 | marCCD, 225 mm plate | Crystal structure of dimeric M-PMV protease C7A/D26N/C106A mutant in complex with inhibitor | +| [6TOC](https://www.rcsb.org/structure/6TOC) | Zenodo [10.5281/zenodo.3571040](https://doi.org/10.5281/zenodo.3571040) | SLS X06DA | 1.85 | P 42 | 31.5 31.5 81.6 90.0 90.0 90.0 | PILATUS 2MF | Crystal structure of the oligomerisation domain of the transcription factor PHOSPHATE STARVATION RESPONSE 1 from Arabidopsis (crystal form 3). | +| [6TTN](https://www.rcsb.org/structure/6TTN) | IRRMC [10.18430/m36ttn](https://doi.org/10.18430/m36ttn) | BESSY 14.1 | 1.12 | P 21 21 21 | 39.9 79.8 104.7 90.0 90.0 90.0 | PILATUS 6M | N-terminally truncated hyoscyamine 6-hydroxylase (tH6H) in complex with N-oxalylglycine and hyoscyamine | +| [6U7G](https://www.rcsb.org/structure/6U7G) | IRRMC [10.18430/m36u7g](https://doi.org/10.18430/m36u7g) | APS 23-ID-B | 2.35 | P 1 21 1 | 99.6 98.7 147.5 90.0 104.6 90.0 | Dectris Eiger 16M | HCoV-229E RBD Class V in complex with human APN | +| [6UKF](https://www.rcsb.org/structure/6UKF) | IRRMC [10.18430/m36ukf](https://doi.org/10.18430/m36ukf) | APS 22-ID | 1.00 | P 1 21 1 | 61.0 37.3 69.0 90.0 109.8 90.0 | Dectris Eiger 16M | HhaI endonuclease in Complex with DNA at 1 Angstrom Resolution | +| [6V2R](https://www.rcsb.org/structure/6V2R) | IRRMC [10.18430/m36v2r](https://doi.org/10.18430/m36v2r) | Home source, Rigaku FR-E | 1.60 | P 41 21 2 | 40.2 40.2 83.1 90.0 90.0 90.0 | Rigaku Saturn A200 | Crystal Structure of chromodomain of CBX7 mutant V13A in complex with inhibitor UNC3866 | +| [6VWW](https://www.rcsb.org/structure/6VWW) | IRRMC [10.18430/m36vww](https://doi.org/10.18430/m36vww) | APS 19-ID | 2.20 | P 63 | 150.5 150.5 111.3 90.0 90.0 120.0 | PILATUS3 6M | Crystal Structure of NSP15 Endoribonuclease from SARS CoV-2. | +| [6W4H](https://www.rcsb.org/structure/6W4H) | IRRMC [10.18430/m36w4h](https://doi.org/10.18430/m36w4h) | APS 21-ID-F | 1.80 | P 31 2 1 | 167.7 167.7 51.9 90.0 90.0 120.0 | Rayonix MX-300 | 1.80 Angstrom Resolution Crystal Structure of NSP16 - NSP10 Complex from SARS-CoV-2 | +| [6W75](https://www.rcsb.org/structure/6W75) | IRRMC [10.18430/m36w75](https://doi.org/10.18430/m36w75) | APS 21-ID-F | 1.95 | P 32 2 1 | 166.2 166.2 98.3 90.0 90.0 120.0 | Rayonix MX-300 | 1.95 Angstrom Resolution Crystal Structure of NSP10 - NSP16 Complex from SARS-CoV-2 | +| [6WZO](https://www.rcsb.org/structure/6WZO) | SBGrid [10.15785/sbgrid/785](https://doi.org/10.15785/sbgrid/785) | APS 24-ID-E | 1.42 | P 1 | 43.7 50.1 69.3 106.5 90.1 97.1 | Dectris Eiger 16M | Structure of SARS-CoV-2 Nucleocapsid dimerization domain, P1 form | +| [6YQF](https://www.rcsb.org/structure/6YQF) | IRRMC [10.18430/m36yqf](https://doi.org/10.18430/m36yqf) | Diamond I24 | 3.33 | P 21 21 2 | 42.7 59.7 156.5 90.0 90.0 90.0 | PILATUS3 6M | Crystal structure of the SYCE2-TEX12 delta-Ctip complex in a 4:4 assembly | +| [6Z8O](https://www.rcsb.org/structure/6Z8O) | Zenodo [10.5281/zenodo.3873216](https://doi.org/10.5281/zenodo.3873216) | ESRF ID30B | 2.20 | P 1 21 1 | 63.7 97.0 121.3 90.0 104.7 90.0 | Dectris Eiger 4M | Structure of [NiFeSe] hydrogenase G491A variant from Desulfovibrio vulgaris Hildenborough pressurized with Krypton gas - structure G491A-Kr | +| [6Z9G](https://www.rcsb.org/structure/6Z9G) | Zenodo [10.5281/zenodo.3874714](https://doi.org/10.5281/zenodo.3874714) | ESRF ID30B | 1.76 | P 1 21 1 | 120.3 93.8 127.0 90.0 105.2 90.0 | Dectris Eiger 4M | Structure of [NiFeSe] hydrogenase G491A variant from Desulfovibrio vulgaris Hildenborough pressurized with Oxygen gas - structure G491A-O2 | +| [6ZE4](https://www.rcsb.org/structure/6ZE4) | SBGrid [10.15785/sbgrid/806](https://doi.org/10.15785/sbgrid/806) | BESSY 14.1 | 1.60 | P 21 21 21 | 93.6 109.9 116.1 90.0 90.0 90.0 | PILATUS 6M | FAD-dependent oxidoreductase from Chaetomium thermophilum in complex with fragment 4-oxo-N-[(1S)-1-(pyridin-3-yl)ethyl]-4-(thiophen-2-yl)butanamide | +| [6ZQR](https://www.rcsb.org/structure/6ZQR) | Keele University [10.21252/r2nx-0425](https://doi.org/10.21252/r2nx-0425) | Diamond I02 | 1.93 | P 4 | 113.6 113.6 44.1 90.0 90.0 90.0 | SMV, S/N 922 | Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with GlcNAc ligand bound | +| [6ZQY](https://www.rcsb.org/structure/6ZQY) | Keele University [10.21252/hx7e-rd04](https://doi.org/10.21252/hx7e-rd04) | Diamond I04 | 1.85 | P 4 | 119.3 119.3 44.2 90.0 90.0 90.0 | SMV, S/N 921 | Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with Neu5Ac ligand bound | +| [6ZR0](https://www.rcsb.org/structure/6ZR0) | Keele University [10.21252/zcfy-cw20](https://doi.org/10.21252/zcfy-cw20) | Diamond I04 | 1.94 | P 4 | 119.2 119.2 44.2 90.0 90.0 90.0 | PILATUS 6M Prosport+ | Crystal structure of tetrameric fibrinogen-like recognition domain of FIBCD1 with N-acetylalanine ligand bound | +| [7ARR](https://www.rcsb.org/structure/7ARR) | MXRDR [10.18150/EM87YL](https://doi.org/10.18150/EM87YL) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.10 | P 1 | 30.9 32.1 43.1 114.2 91.9 109.9 | PILATUS 6M-F | The de novo designed hybrid alpha/beta-miniprotein | +| [7ATG](https://www.rcsb.org/structure/7ATG) | IRRMC [10.18430/m37atg](https://doi.org/10.18430/m37atg) | PETRA III, EMBL c/o DESY P13 (MX1) | 0.60 | P 21 21 21 | 18.0 31.0 43.9 90.0 90.0 90.0 | PILATUS 6M-F | Crystal structure of Z-DNA in complex with putrescinium and potassium cations at ultrahigh-resolution | +| [7BGT](https://www.rcsb.org/structure/7BGT) | MXRDR [10.18150/1HQGWO](https://doi.org/10.18150/1HQGWO) | BESSY 14.2 | 1.93 | P 1 | 29.3 67.6 69.7 76.8 83.9 83.6 | marCCD, 225 mm plate | Mason-Pfizer Monkey Virus Protease mutant C7A/D26N/C106A in complex with peptidomimetic inhibitor | +| [7BGU](https://www.rcsb.org/structure/7BGU) | MXRDR [10.18150/C9DYSH](https://doi.org/10.18150/C9DYSH) | EMBL/DESY Hamburg (DORIS) X13 | 2.43 | P 1 | 29.1 67.9 69.7 77.1 83.3 83.2 | marCCD 165 mm | Mason-Pfizer Monkey Virus Protease mutant C7A/D26N/C106A in complex with peptidomimetic inhibitor | +| [7D1M](https://www.rcsb.org/structure/7D1M) | IRRMC [10.18430/m37brr](https://doi.org/10.18430/m37brr) | SSRF BL17U1 | 1.35 | P 1 21 1 | 55.5 99.0 59.6 90.0 108.5 90.0 | Dectris Eiger 16M | CRYSTAL STRUCTURE OF THE SARS-CoV-2 MAIN PROTEASE COMPLEXED WITH GC376 | +| [7DKP](https://www.rcsb.org/structure/7DKP) | IRRMC [10.18430/M37DKP](https://doi.org/10.18430/M37DKP) | ESRF MASSIF-3 | 1.45 | P 1 21 1 | 49.8 169.5 49.8 90.0 93.5 90.0 | Dectris Eiger 4M | Crystal structure of E. coli Grx2 in complex with GSH at 1.45 A resolution | +| [7K1L](https://www.rcsb.org/structure/7K1L) | IRRMC [10.18430/m37k1l](https://doi.org/10.18430/m37k1l) | APS 19-ID | 2.25 | P 63 | 150.8 150.8 110.7 90.0 90.0 120.0 | PILATUS3 6M | Crystal Structure of NSP15 Endoribonuclease from SARS CoV-2 in the Complex with Uridine-2',3'-Vanadate | +| [7KCN](https://www.rcsb.org/structure/7KCN) | IRRMC [10.18430/m37kcn](https://doi.org/10.18430/m37kcn) | LNLS W01B-MX2 | 1.46 | P 41 2 2 | 67.0 67.0 116.9 90.0 90.0 90.0 | PILATUS 2M | Reconstructed ancestor of HIUases and Transthyretins | +| [7L6J](https://www.rcsb.org/structure/7L6J) | IRRMC [10.18430/m37l6j](https://doi.org/10.18430/m37l6j) | APS 21-ID-F | 1.78 | I 41 3 2 | 171.7 171.7 171.7 90.0 90.0 90.0 | Rayonix MX-300 | Crystal Structure of the Putative Hydrolase from Stenotrophomonas maltophilia | +| [7L84](https://www.rcsb.org/structure/7L84) | SBGrid [10.15785/sbgrid/816](https://doi.org/10.15785/sbgrid/816) | APS 24-ID-C | 1.60 | P 43 21 2 | 79.3 79.3 37.8 90.0 90.0 90.0 | PILATUS 6M-F | Hen Egg White Lysozyme by Native S-SAD at Room Temperature | +| [7MZT](https://www.rcsb.org/structure/7MZT) | IRRMC [10.18430/m37mzt](https://doi.org/10.18430/m37mzt) | APS 22-ID | 4.07 | P 21 21 2 | 113.6 97.0 108.3 90.0 90.0 90.0 | Dectris Eiger 16M | Borrelia burgdorferi BBK32-C in complex with an autolytic fragment of human C1r at 4.1A | +| [7N0I](https://www.rcsb.org/structure/7N0I) | SBGrid [10.15785/sbgrid/835](https://doi.org/10.15785/sbgrid/835) | ALS 5.0.2 | 2.20 | P 21 21 21 | 75.8 131.6 140.0 90.0 90.0 90.0 | PILATUS3 6M | Structure of the SARS-CoV-2 N protein C-terminal domain bound to single-domain antibody E2 | +| [7N2S](https://www.rcsb.org/structure/7N2S) | SBGrid [10.15785/sbgrid/916](https://doi.org/10.15785/sbgrid/916) | SSRL BL12-1 | 2.37 | P 1 21 1 | 83.2 52.8 106.3 90.0 98.3 90.0 | PILATUS 6M | AS3.1-PRPF3-HLA*B27 | +| [7ORR](https://www.rcsb.org/structure/7ORR) | IRRMC [10.18430/M37ORR](https://doi.org/10.18430/M37ORR) | MAX IV BioMAX | 1.79 | I 21 3 | 105.9 105.9 105.9 90.0 90.0 90.0 | Dectris Eiger 16M | Non-structural protein 10 (nsp10) from SARS CoV-2 in complex with fragment VT00022 | +| [7OS3](https://www.rcsb.org/structure/7OS3) | MXRDR [10.18150/74YTYQ](https://doi.org/10.18150/74YTYQ) | PETRA III, EMBL c/o DESY P13 (MX1) | 2.18 | P 21 21 21 | 78.2 91.0 105.8 90.0 90.0 90.0 | PILATUS 6M-F | Crystal structure of Rhizobium etli inducible L-asparaginase | +| [7OU1](https://www.rcsb.org/structure/7OU1) | MXRDR [10.18150/VQQIHQ](https://doi.org/10.18150/VQQIHQ) | BESSY 14.3 | 1.65 | P 1 21 1 | 77.9 91.3 114.2 90.0 97.1 90.0 | marCCD, 225 mm plate | Crystal structure of Rhizobium etli inducible L-asparaginase ReAV (monoclinic form MP2) | +| [7PH1](https://www.rcsb.org/structure/7PH1) | IRRMC [10.18430/M37PH1](https://doi.org/10.18430/M37PH1) | BESSY 14.2 | 1.18 | I 2 2 2 | 75.0 81.3 124.2 90.0 90.0 90.0 | PILATUS3 2M | Trypsin in complex with BPTI mutant (2S)-2-amino-4-monofluorobutanoic acid | +| [7PQ7](https://www.rcsb.org/structure/7PQ7) | IRRMC [10.18430/M3.IRRMC.6072](https://doi.org/10.18430/M3.IRRMC.6072) | ELETTRA 11.2C | 1.55 | C 1 2 1 | 120.9 51.7 75.5 90.0 125.1 90.0 | PILATUS 6M | Crystal structure of Campylobacter jejuni DsbA1 | +| [7QIJ](https://www.rcsb.org/structure/7QIJ) | SBGrid [10.15785/sbgrid/907](https://doi.org/10.15785/sbgrid/907) | PETRA III, EMBL c/o DESY P13 (MX1) | 4.10 | P 21 21 21 | 143.5 324.9 369.4 90.0 90.0 90.0 | PILATUS 6M-F | Complex of the Yersinia enterocolitica Type III secretion export gate YscV with substrate:chaperone complex YscX:YscY | +| [7QIS](https://www.rcsb.org/structure/7QIS) | IRRMC [10.18430/M37QIS](https://doi.org/10.18430/M37QIS) | BESSY 14.2 | 1.83 | P 61 | 100.3 100.3 206.2 90.0 90.0 120.0 | PILATUS3 2M | CRYSTAL STRUCTURE OF THE P1 difluoroethylglycine (DfeGly) BPTI MUTANT- BOVINE CHYMOTRYPSIN COMPLEX | +| [7RAA](https://www.rcsb.org/structure/7RAA) | SBGrid [10.15785/sbgrid/881](https://doi.org/10.15785/sbgrid/881) | SSRL BL12-2 | 2.69 | P 43 21 2 | 66.4 66.4 298.3 90.0 90.0 90.0 | PILATUS 6M | Designed StabIL-2 seq15 | +| [7RIS](https://www.rcsb.org/structure/7RIS) | IRRMC [10.18430/M37RIS](https://doi.org/10.18430/M37RIS) | APS 21-ID-D | 1.72 | P 32 2 1 | 44.5 44.5 189.9 90.0 90.0 120.0 | Dectris Eiger 9M | Crystal structure of RPA3624, a beta-propeller lactonase from Rhodopseudomonas palustris, with active-site bound phosphate | +| [7RJI](https://www.rcsb.org/structure/7RJI) | IRRMC [10.18430/M37RJI](https://doi.org/10.18430/M37RJI) | LNLS W01B-MX2 | 1.71 | H 3 2 | 83.0 83.0 124.8 90.0 90.0 120.0 | PILATUS 2M | BthTX-II variant b, from Bothrops jararacussu venom, complexed with stearic acid | +| [7T5T](https://www.rcsb.org/structure/7T5T) | SBGrid [10.15785/sbgrid/864](https://doi.org/10.15785/sbgrid/864) | SSRL BL9-2 | 1.35 | P 42 21 2 | 95.3 95.3 104.9 90.0 90.0 90.0 | PILATUS 6M | Structure of Thauera sp. K11 CapP | +| [7TCD](https://www.rcsb.org/structure/7TCD) | IRRMC [10.18430/m37tcd](https://doi.org/10.18430/m37tcd) | SLS X06SA | 1.70 | C 1 2 1 | 138.5 47.9 78.1 90.0 107.6 90.0 | Dectris Eiger 16M | LOV2-DARPIN fusion: D13 | +| [7YZX](https://www.rcsb.org/structure/7YZX) | IRRMC [10.18430/M37YZX](https://doi.org/10.18430/M37YZX) | Diamond I24 | 1.90 | P 63 2 2 | 169.4 169.4 141.8 90.0 90.0 120.0 | PILATUS3 6M | ScpA from Streptococcus pyogenes, D783A mutant. | +| [8A1A](https://www.rcsb.org/structure/8A1A) | IRRMC [10.18430/M38A1A](https://doi.org/10.18430/M38A1A) | SLS X06SA | 2.05 | P 65 | 191.9 191.9 122.4 90.0 90.0 120.0 | Dectris Eiger 16M | Structure of a leucinostatin derivative determined by host lattice display : L1F11V1 construct | +| [8AGQ](https://www.rcsb.org/structure/8AGQ) | IRRMC [10.18430/M38AGQ](https://doi.org/10.18430/M38AGQ) | SLS X06DA | 1.09 | C 1 2 1 | 89.9 55.4 54.8 90.0 113.5 90.0 | PILATUS 2MF | Crystal structure of anthocyanin-related GSTF8 from Populus trichocarpa in complex with (-)-catechin and glutathione | +| [8DQB](https://www.rcsb.org/structure/8DQB) | IRRMC [10.18430/m38dqb](https://doi.org/10.18430/m38dqb) | NSLS-II 19-ID | 2.50 | I 2 3 | 164.1 164.1 164.1 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal structure of 3-dehydroquinate dehydratase I from Klebsiella oxytoca (I23 Form) | +| [8DYZ](https://www.rcsb.org/structure/8DYZ) | SBGrid [10.15785/sbgrid/957](https://doi.org/10.15785/sbgrid/957) | CHESS F1 | 1.27 | P 43 21 2 | 79.6 79.6 38.3 90.0 90.0 90.0 | PILATUS3 6M | Hen lysozyme in tetragonal space group at ambient temperature - diffuse scattering dataset | +| [8DZ7](https://www.rcsb.org/structure/8DZ7) | SBGrid [10.15785/sbgrid/958](https://doi.org/10.15785/sbgrid/958) | CHESS F1 | 1.34 | P 21 21 21 | 30.5 56.4 73.9 90.0 90.0 90.0 | PILATUS3 6M | Hen lysozyme in orthorhombic space group at ambient temperature - diffuse scattering dataset | +| [8EGN](https://www.rcsb.org/structure/8EGN) | IRRMC [10.18430/M38EGN](https://doi.org/10.18430/M38EGN) | CLSI 08B1-1 | 1.95 | P 21 21 21 | 71.7 75.2 109.8 90.0 90.0 90.0 | PILATUS3 6M | Crystal Structure of UDP-N-acetylmuramate-L-alanine ligase (UDP-N-acetylmuramoyl-L-alanine synthetase, MurC) Pseudomonas aeruginosa in complex with ligand AZ-13643701 | +| [8IYA](https://www.rcsb.org/structure/8IYA) | IRRMC [10.18430/m38iya](https://doi.org/10.18430/m38iya) | SSRF BL02U1 | 2.43 | C 1 2 1 | 102.7 50.1 109.2 90.0 91.8 90.0 | Dectris EIGER2 Si 9M | Complex of SETDB1-derived peptide bound to UBE2E1 | +| [8K1G](https://www.rcsb.org/structure/8K1G) | IRRMC [10.18430/M38K1G](https://doi.org/10.18430/M38K1G) | PAL/PLS 11C | 2.09 | I 4 2 2 | 182.0 182.0 80.7 90.0 90.0 90.0 | PILATUS3 6M | Crystal structure of ethylene glycol-bound glycerol dehydrogenase from Klebsiella pneumoniae | +| [8OIC](https://www.rcsb.org/structure/8OIC) | IRRMC [10.18430/m38oic](https://doi.org/10.18430/m38oic) | Diamond I04 | 2.80 | P 1 | 73.1 94.7 120.6 105.1 90.0 93.8 | Eiger 16M | Trichomonas vaginalis riboside hydrolase (His-tagged) | +| [8OWM](https://www.rcsb.org/structure/8OWM) | MXRDR [10.18150/II5MT4](https://doi.org/10.18150/II5MT4) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.70 | P 1 | 95.5 95.6 95.8 90.4 93.6 117.8 | Dectris Eiger 16M | Crystal structure of glutamate dehydrogenase 2 from Arabidopsis thaliana binding Ca, NAD and 2,2-dihydroxyglutarate | +| [8PQD](https://www.rcsb.org/structure/8PQD) | IRRMC [10.18430/m38pqd](https://doi.org/10.18430/m38pqd) | ESRF MASSIF-3 | 1.50 | P 21 21 21 | 59.4 59.4 192.9 90.0 90.0 90.0 | Dectris Eiger 4M | c-KIT kinase domain in complex with avapritinib derivative 10 | +| [8QAW](https://www.rcsb.org/structure/8QAW) | MXRDR [10.18150/INUP4Q](https://doi.org/10.18150/INUP4Q) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.55 | H 3 | 137.7 137.7 265.9 90.0 90.0 120.0 | Dectris Eiger 16M | Medicago truncatula HISN5 (IGPD) in complex with MN, IMD, EDO, FMT, GOL and TRS | +| [8QJ5](https://www.rcsb.org/structure/8QJ5) | IRRMC [10.18430/m38qj5](https://doi.org/10.18430/m38qj5) | ELETTRA 11.2C | 1.63 | P 1 21 1 | 57.6 100.6 77.9 90.0 96.1 90.0 | PILATUS 6M | Crystal structure of the Levansucrase beta from Pseudomonas syringae pv. actinidiae | +| [8QQ7](https://www.rcsb.org/structure/8QQ7) | Zenodo [10.5281/zenodo.14901515](https://doi.org/10.5281/zenodo.14901515) | ESRF MASSIF-1 | 3.62 | P 64 2 2 | 146.0 146.0 153.6 90.0 90.0 120.0 | PILATUS3 2M | Structure of SpNOX: a Bacterial NADPH oxidase | +| [8R5R](https://www.rcsb.org/structure/8R5R) | IRRMC [10.18430/m38r5r](https://doi.org/10.18430/m38r5r) | ESRF ID23-1 | 3.08 | P 21 21 21 | 91.7 132.9 137.5 90.0 90.0 90.0 | Dectris EIGER2 CdTe 16M | Structure of apo TDO with a bound inhibitor | +| [8RUD](https://www.rcsb.org/structure/8RUD) | MXRDR [10.18150/RBG2F9](https://doi.org/10.18150/RBG2F9) | PETRA III, EMBL c/o DESY P13 (MX1) | 2.10 | P 1 21 1 | 78.1 91.4 114.5 90.0 96.9 90.0 | Dectris Eiger 16M | Crystal structure of Rhizobium etli L-asparaginase ReAV K138A mutant | +| [8S38](https://www.rcsb.org/structure/8S38) | MXRDR [10.18150/CGLBVH](https://doi.org/10.18150/CGLBVH) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.89 | I 21 21 21 | 95.4 163.1 219.0 90.0 90.0 90.0 | PILATUS 6M-F | Crystal structure of Medicago truncatula glutamate dehydrogenase 2 in complex with citrate and NAD | +| [8SA8](https://www.rcsb.org/structure/8SA8) | IRRMC [10.18430/M38SA8](https://doi.org/10.18430/M38SA8) | NSLS-II 19-ID | 1.30 | I 1 2 1 | 87.9 131.5 165.4 90.0 104.5 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of Cystathionine beta lyase from Klebsiella aerogenes, Covalently bound and free PLP (I2 form) | +| [8SQO](https://www.rcsb.org/structure/8SQO) | IRRMC [10.18430/m38sqo](https://doi.org/10.18430/m38sqo) | NSLS-II 19-ID | 1.55 | P 4 3 2 | 112.9 112.9 112.9 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (magnesium bound, F16L mutant) | +| [8SQQ](https://www.rcsb.org/structure/8SQQ) | IRRMC [10.18430/M38SQQ](https://doi.org/10.18430/M38SQQ) | NSLS-II 19-ID | 2.25 | F 4 3 2 | 171.5 171.5 171.5 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (Apo Cubic Form 2, F16L mutant) | +| [8SQT](https://www.rcsb.org/structure/8SQT) | IRRMC [10.18430/M38SQT](https://doi.org/10.18430/M38SQT) | NSLS-II 19-ID | 2.20 | F 4 3 2 | 170.7 170.7 170.7 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of Bacterioferritin (Bfr) from Brucella abortus (iron bound, cubic form 2, F16L mutant) | +| [8T7R](https://www.rcsb.org/structure/8T7R) | IRRMC [10.18430/M38T7R](https://doi.org/10.18430/M38T7R) | APS 22-ID | 3.84 | C 1 2 1 | 357.1 259.6 255.4 90.0 133.1 90.0 | Dectris Eiger 16M | Crystal structure of human leukocyte antigen A*0101 in complex with the Fab of alloreactive antibody E07 | +| [8THA](https://www.rcsb.org/structure/8THA) | IRRMC [10.18430/m38tha](https://doi.org/10.18430/m38tha) | SSRL BL9-2 | 1.68 | P 64 | 69.2 69.2 29.1 90.0 90.0 120.0 | PILATUS 6M | 1TEL, non-compressed, double-helical crystal form | +| [8TYY](https://www.rcsb.org/structure/8TYY) | SBGrid [10.15785/sbgrid/1040](https://doi.org/10.15785/sbgrid/1040) | APS 24-ID-E | 1.68 | F 4 3 2 | 214.9 214.9 214.9 90.0 90.0 90.0 | Dectris Eiger 16M | Structure of a bacterial Ubl-deubiquitinase complex (form 2) | +| [8U0I](https://www.rcsb.org/structure/8U0I) | IRRMC [10.18430/m38u0i](https://doi.org/10.18430/m38u0i) | ALS 8.2.1 | 1.54 | P 43 21 2 | 50.3 50.3 90.6 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal structure of PA0012 complexed with cyclic-di-GMP from Pseudomonas aeruginosa | +| [8V2T](https://www.rcsb.org/structure/8V2T) | Zenodo [10.5281/zenodo.10201899](https://doi.org/10.5281/zenodo.10201899) | NSLS X25 | 1.40 | P 42 21 2 | 60.9 60.9 92.7 90.0 90.0 90.0 | PILATUS 6M | Phosphoheptose isomerase GMHA from Burkholderia pseudomallei bound to inhibitor Mut148591 | +| [8V4J](https://www.rcsb.org/structure/8V4J) | Zenodo [10.5281/zenodo.10222807](https://doi.org/10.5281/zenodo.10222807) | NSLS X29A | 1.31 | P 42 21 2 | 61.0 61.0 92.4 90.0 90.0 90.0 | ADSC Quantum 315 | Phosphoheptose isomerase GMHA from Burkholderia pseudomallei bound to inhibitor Mut148233 | +| [8V4O](https://www.rcsb.org/structure/8V4O) | IRRMC [10.18430/m38v4o](https://doi.org/10.18430/m38v4o) | NSLS-II 19-ID | 2.70 | P 61 2 2 | 139.5 139.5 545.0 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of Acetyl-CoA synthetase 2 in complex with AMP from Candida albicans | +| [8XBP](https://www.rcsb.org/structure/8XBP) | IRRMC [10.18430/M38XBP](https://doi.org/10.18430/M38XBP) | SOLEIL PROXIMA 1 | 1.99 | C 1 2 1 | 148.3 50.8 60.2 90.0 92.3 90.0 | Dectris Eiger 16M | Crystal structure of AtNATA1 bound to Acetyl CoA | +| [8XTE](https://www.rcsb.org/structure/8XTE) | SBGrid [10.15785/sbgrid/1101](https://doi.org/10.15785/sbgrid/1101) | SSRF BL19U1 | 1.99 | P 32 | 208.8 208.8 67.2 90.0 90.0 120.0 | PILATUS3 6M | Crystal structure of methyltransferase MpaG' in complex with SAH and FDHMP | +| [8XTF](https://www.rcsb.org/structure/8XTF) | SBGrid [10.15785/sbgrid/1102](https://doi.org/10.15785/sbgrid/1102) | SSRF BL02U1 | 2.13 | H 3 2 | 211.8 211.8 67.4 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of methyltransferase MpaG' in complex with SAH and FDHMP-3C | +| [8XTG](https://www.rcsb.org/structure/8XTG) | SBGrid [10.15785/sbgrid/1100](https://doi.org/10.15785/sbgrid/1100) | SSRF BL19U1 | 2.00 | P 32 | 199.5 199.5 67.2 90.0 90.0 120.0 | | Crystal structure of methyltransferase MpaG' in complex with SAH and DMMPA | +| [8Y74](https://www.rcsb.org/structure/8Y74) | XRDa [10.51093/xrd-00227](https://doi.org/10.51093/xrd-00227) | SSRF BL02U1 | 1.90 | C 1 2 1 | 125.8 76.6 87.1 90.0 92.4 90.0 | Dectris EIGER2 Si 9M | Crystal structure of 9-mer peptide from H9N2 avian influenza virus in complex with BF2*0201 | +| [8YS9](https://www.rcsb.org/structure/8YS9) | IRRMC [10.18430/M38YS9](https://doi.org/10.18430/M38YS9) | PAL/PLS 5C (4A) | 1.46 | P 21 21 21 | 71.0 77.7 83.2 90.0 90.0 90.0 | Dectris Eiger 9M | Crystal structure of Phosphatidylethanolamine N-methyltransferase from R. thermophilum complexed with DMPE and SAH | +| [9B22](https://www.rcsb.org/structure/9B22) | IRRMC [10.18430/m39b22](https://doi.org/10.18430/m39b22) | NSLS-II 19-ID | 1.30 | P 1 21 1 | 39.8 92.7 57.7 90.0 91.7 90.0 | Dectris EIGER2 Si 9M | Crystal structure of ADP-ribose diphosphatase from Klebsiella pneumoniae (ADP Ribose and AMP bound) | +| [9BN8](https://www.rcsb.org/structure/9BN8) | IRRMC [10.18430/m39bn8](https://doi.org/10.18430/m39bn8) | NSLS-II 19-ID | 1.35 | P 41 | 65.5 65.5 134.8 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of UDP-N-acetylmuramoylalanine--D-glutamate ligase (MurD) from E. coli in complex with UMA and inhibitor A19 | +| [9C18](https://www.rcsb.org/structure/9C18) | Zenodo [10.5281/zenodo.11405662](https://doi.org/10.5281/zenodo.11405662) | NSLS-II 17-ID-1 | 1.90 | P 1 | 41.9 42.0 60.2 84.1 87.2 63.7 | Dectris EIGER1 Si 9M | Human biliverdin IX beta reductase in complex with NADP | +| [9CHW](https://www.rcsb.org/structure/9CHW) | SBGrid [10.15785/sbgrid/1124](https://doi.org/10.15785/sbgrid/1124) | APS 21-ID-F | 2.16 | P 61 | 98.7 98.7 82.1 90.0 90.0 120.0 | Rayonix MX-300 | Crystal structure of human polymerase eta with incoming dAMPnPP nucleotide opposite threofuranosyl thymidine in DNA template | +| [9CRW](https://www.rcsb.org/structure/9CRW) | IRRMC [10.18430/m39crw](https://doi.org/10.18430/m39crw) | CLSI 08ID-1 | 2.49 | P 1 21 1 | 84.0 104.6 118.8 90.0 93.4 90.0 | Dectris Eiger 9M | Crystal structure of the Candida albicans kinesin-8 proximal tail domain | +| [9E2T](https://www.rcsb.org/structure/9E2T) | SBGrid [10.15785/sbgrid/1148](https://doi.org/10.15785/sbgrid/1148) | SSRL BL12-1 | 2.28 | P 1 | 75.5 78.1 101.2 94.6 103.4 114.5 | Dectris EIGER2 Si 16M | Structure of a de novo designed interleukin-21 mimetic complex | +| [9EA5](https://www.rcsb.org/structure/9EA5) | SBGrid [10.15785/sbgrid/1142](https://doi.org/10.15785/sbgrid/1142) | SSRL BL9-2 | 2.00 | P 1 21 1 | 65.9 73.1 98.4 90.0 108.7 90.0 | PILATUS 6M | Structure of Citrobacter BubCD D104A mutant | +| [9FCF](https://www.rcsb.org/structure/9FCF) | MXRDR [10.18150/DGZKW3](https://doi.org/10.18150/DGZKW3) | PETRA III, EMBL c/o DESY P13 (MX1) | 2.36 | P 4 | 91.3 91.3 35.8 90.0 90.0 90.0 | Dectris EIGER1 Si 16M | Medicago truncatula 5'-ProFAR isomerase (HISN3) D57N mutant in complex with ProFAR | +| [9FCG](https://www.rcsb.org/structure/9FCG) | MXRDR [10.18150/LDLSBT](https://doi.org/10.18150/LDLSBT) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.54 | P 4 | 87.8 87.8 35.6 90.0 90.0 90.0 | Dectris EIGER1 Si 16M | Medicago truncatula 5'-ProFAR isomerase (HISN3) D57N mutant in complex with PrFAR | +| [9FHC](https://www.rcsb.org/structure/9FHC) | Zenodo [10.5281/zenodo.11472085](https://doi.org/10.5281/zenodo.11472085) | SLS X06SA | 2.20 | I 2 3 | 227.5 227.5 227.5 90.0 90.0 90.0 | marCCD, 225 mm plate | Crystallographic structure of AcrB V612F with bound minocycline | +| [9GDJ](https://www.rcsb.org/structure/9GDJ) | ESRF [10.15151/ESRF-DC-1848199439](https://doi.org/10.15151/ESRF-DC-1848199439) | ESRF ID23-1 | 1.47 | P 41 21 2 | 123.9 123.9 126.4 90.0 90.0 90.0 | Dectris EIGER2 CdTe 16M | C-Methyltransferase SgMT from Streptomyces griseoviridis | +| [9GJX](https://www.rcsb.org/structure/9GJX) | IRRMC [10.18430/M39GJX](https://doi.org/10.18430/M39GJX) | Diamond I04 | 2.40 | P 1 21 1 | 76.8 115.8 103.8 90.0 110.3 90.0 | Eiger 16M | Bacillus licheniformis nitroreductase | +| [9GQG](https://www.rcsb.org/structure/9GQG) | ESRF [10.15151/ESRF-DC-1900353437](https://doi.org/10.15151/ESRF-DC-1900353437) | ESRF ID30B | 2.00 | P 32 2 1 | 48.2 48.2 188.0 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | The FK1 domain of FKBP51 in complex with the macrocyclic SAFit analog m5(10,7)-(E)-OH | +| [9H0Q](https://www.rcsb.org/structure/9H0Q) | Zenodo [10.5281/zenodo.13912326](https://doi.org/10.5281/zenodo.13912326) | SOLEIL PROXIMA 2 | 2.55 | H 3 2 | 169.5 169.5 344.0 90.0 90.0 120.0 | Dectris EIGER1 Si 9M | N terminal domain of BC2L-C lectin in complex with N-(beta-L-Fucopyranosyl)-biphenyl-3-carboxamide | +| [9HNC](https://www.rcsb.org/structure/9HNC) | MXRDR [10.60884/0K7B68](https://doi.org/10.60884/0K7B68) | PETRA III, EMBL c/o DESY P13 (MX1) | 1.88 | P 1 2 1 | 123.8 123.6 187.7 90.0 90.1 90.0 | PILATUS 6M-F | Crystal structure of potassium-independent L-asparaginase | +| [9HS7](https://www.rcsb.org/structure/9HS7) | IRRMC [10.18430/M39HS7](https://doi.org/10.18430/M39HS7) | ALBA XALOC | 1.70 | P 65 | 65.4 65.4 88.8 90.0 90.0 120.0 | PILATUS3 X 6M | Anti-HIV-1 chimeric miniprotein mimicking the N-terminal half of gp41 NHR with an extended region targeting the MPER | +| [9I0A](https://www.rcsb.org/structure/9I0A) | IRRMC [10.18430/M39I0A](https://doi.org/10.18430/M39I0A) | SOLEIL PROXIMA 1 | 2.22 | P 21 21 2 | 75.2 98.7 208.6 90.0 90.0 90.0 | Dectris Eiger 16M | CARM1 in complex with arg-aDMA analog | +| [9I80](https://www.rcsb.org/structure/9I80) | Zenodo [10.5281/zenodo.14844040](https://doi.org/10.5281/zenodo.14844040) | SOLEIL PROXIMA 1 | 1.95 | P 41 | 81.2 81.2 165.0 90.0 90.0 90.0 | Dectris Eiger 16M | LecA in complex with a tolcapone derivative glycomimetic | +| [9IG7](https://www.rcsb.org/structure/9IG7) | IRRMC [10.18430/M39IG7](https://doi.org/10.18430/M39IG7) | PETRA III, EMBL c/o DESY P13 (MX1) | 2.60 | P 21 21 2 | 111.5 153.5 69.0 90.0 90.0 90.0 | Dectris EIGER1 Si 16M | KOD-H4 DNA polymerase mutant in a binary complex with DNA:DNA containing two AtNA nucleotides | +| [9IH9](https://www.rcsb.org/structure/9IH9) | IRRMC [10.18430/M39IH9](https://doi.org/10.18430/M39IH9) | ESRF MASSIF-3 | 1.70 | C 1 2 1 | 78.8 133.9 82.3 90.0 101.4 90.0 | Dectris EIGER1 Si 4M | KEAP1 complexed to linear peptide 6 | +| [9JQ9](https://www.rcsb.org/structure/9JQ9) | IRRMC [10.18430/M39JQ9](https://doi.org/10.18430/M39JQ9) | Home source, Excillum MetalJet D2+ | 1.90 | P 21 21 21 | 48.6 50.5 78.6 90.0 90.0 90.0 | PILATUS3 1M | Crystal structure of Plasmoredoxin from Plasmodium falciparum a disulfide oxidoreductase protein unique to Plasmodium species | +| [9JZO](https://www.rcsb.org/structure/9JZO) | IRRMC [10.18430/m39jzo](https://doi.org/10.18430/m39jzo) | PAL/PLS 11C | 1.40 | P 1 | 41.6 43.1 54.2 113.0 90.1 118.2 | PILATUS3 6M | Crystal structure of PHICD111_20024_EAD. | +| [9KHR](https://www.rcsb.org/structure/9KHR) | Zenodo [10.5281/zenodo.14070468](https://doi.org/10.5281/zenodo.14070468) | RRCAT INDUS-2 PX-BL21 | 2.00 | P 21 21 21 | 48.7 50.3 78.0 90.0 90.0 90.0 | marCCD, 225 mm plate | Crystal structure of Plasmoredoxin, a disulfide oxidoreductase from Plasmodium falciparum crystallized in the presence of Dithiothreitol (DTT) | +| [9LXL](https://www.rcsb.org/structure/9LXL) | Zenodo [10.5281/zenodo.15005358](https://doi.org/10.5281/zenodo.15005358) | SSRF BL17UM | 2.19 | P 41 21 2 | 76.8 76.8 225.3 90.0 90.0 90.0 | EIGER2 S 16M | Crystal structure of GH29 family alpha-L-fucosidase from Fusarium proliferatum LE1 | +| [9MH4](https://www.rcsb.org/structure/9MH4) | IRRMC [10.18430/M39MH4](https://doi.org/10.18430/M39MH4) | NSLS-II 19-ID | 3.05 | P 21 3 | 138.7 138.7 138.7 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of Bifunctional protein GlmU from Klebsiella aerogenes | +| [9MIN](https://www.rcsb.org/structure/9MIN) | SBGrid [10.15785/sbgrid/1151](https://doi.org/10.15785/sbgrid/1151) | ALS 8.2.1 | 2.05 | P 21 21 21 | 95.5 98.5 155.7 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Structure of a designed minibinder to NYESO1-A*02:01 | +| [9O0H](https://www.rcsb.org/structure/9O0H) | IRRMC [10.18430/M39O0H](https://doi.org/10.18430/M39O0H) | SSRL BL12-2 | 2.24 | P 21 21 21 | 55.2 65.5 112.9 90.0 90.0 90.0 | Dectris EIGER2 Si 16M | The ubiquitin-associated domain of human thirty-eight negative kinase 1, fused to the 3TEL crystallization chaperone via a 2-glycine linker | +| [9P7Q](https://www.rcsb.org/structure/9P7Q) | IRRMC [10.18430/M39P7Q](https://doi.org/10.18430/M39P7Q) | SSRL BL12-1 | 2.21 | C 1 2 1 | 97.0 45.0 72.1 90.0 105.1 90.0 | Dectris EIGER2 Si 16M | 273K human S-adenosylmethionine decarboxylase | +| [9PBB](https://www.rcsb.org/structure/9PBB) | IRRMC [10.18430/M39PBB](https://doi.org/10.18430/M39PBB) | SSRL BL12-1 | 2.17 | C 1 2 1 | 97.4 45.9 72.2 90.0 105.0 90.0 | Dectris EIGER2 Si 16M | 293K human S-adenosylmethionine decarboxylase | +| [9Q41](https://www.rcsb.org/structure/9Q41) | SBGrid [10.15785/sbgrid/1194](https://doi.org/10.15785/sbgrid/1194) | CHESS 7B2 | 1.95 | C 2 2 21 | 118.6 133.7 82.4 90.0 90.0 90.0 | Dectris EIGER2 Si 16M | Crystal Structure of Human Apo Spermidine Synthase | +| [9Q66](https://www.rcsb.org/structure/9Q66) | SBGrid [10.15785/sbgrid/1208](https://doi.org/10.15785/sbgrid/1208) | NSLS-II 17-ID-1 | 2.01 | P 1 21 1 | 105.9 67.3 158.0 90.0 99.1 90.0 | Dectris EIGER1 Si 9M | Human prolyl endopeptidase (PREP) - complex with JP-4-1-7 | +| [9QW8](https://www.rcsb.org/structure/9QW8) | ESRF [10.15151/ESRF-DC-2127908021](https://doi.org/10.15151/ESRF-DC-2127908021) | ESRF ID23-1 | 1.80 | P 1 | 35.6 35.6 100.9 86.5 84.2 72.5 | Dectris EIGER2 CdTe 16M | FKBP12 in complex with bifunctional ligand 1ad | +| [9RCI](https://www.rcsb.org/structure/9RCI) | Zenodo [10.5281/zenodo.15615368](https://doi.org/10.5281/zenodo.15615368) | SOLEIL PROXIMA 2 | 1.66 | P 1 | 35.9 39.3 100.9 98.3 90.3 90.1 | Dectris Eiger 9M | Crystal Structure of Flap Endonuclease FEN1 with Compound 28 | +| [9RCS](https://www.rcsb.org/structure/9RCS) | XRDa [10.51093/xrd-00383](https://doi.org/10.51093/xrd-00383) | Diamond I24 | 3.01 | P 1 21 1 | 70.0 78.8 82.3 90.0 88.6 90.0 | Eiger 9M | Cardioderma bat coronavirus KY43 receptor binding domain in complex with human CEACAM6 | +| [9RP9](https://www.rcsb.org/structure/9RP9) | IRRMC [10.18430/M39RP9](https://doi.org/10.18430/M39RP9) | SOLEIL PROXIMA 1 | 2.10 | C 1 2 1 | 73.5 59.8 91.7 90.0 100.8 90.0 | Dectris Eiger 16M | Crystal structure of mouse pVHL-ElonginB-ElonginC complex | +| [9S02](https://www.rcsb.org/structure/9S02) | MXRDR [10.60884/NRNGS4](https://doi.org/10.60884/NRNGS4) | MAX IV BioMAX | 1.65 | P 21 21 2 | 163.7 88.0 116.7 90.0 90.0 90.0 | EIGER2 X 16M | PYCR1 in complex with 3-(2-thiazolyl)propionic acid | +| [9SL0](https://www.rcsb.org/structure/9SL0) | IRRMC [10.18430/M39SL0](https://doi.org/10.18430/M39SL0) | ESRF MASSIF-1 | 1.60 | P 21 21 21 | 60.2 80.2 111.6 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal structure of HLA-A0201 in complex with peptide LLWNGPMAV | +| [9T6S](https://www.rcsb.org/structure/9T6S) | SBGrid [10.15785/sbgrid/1260](https://doi.org/10.15785/sbgrid/1260) | ESRF ID30B | 2.00 | P 21 21 21 | 63.0 64.6 102.7 90.0 90.0 90.0 | Dectris EIGER2 Si 9M | Crystal Structure of the Listeria monocytogenes CadC with Cadmium | +| [9UPT](https://www.rcsb.org/structure/9UPT) | XRDa [10.51093/xrd-00191](https://doi.org/10.51093/xrd-00191) | NSRRC TPS 05A | 2.37 | P 6 | 158.3 158.3 54.0 90.0 90.0 120.0 | SMV, S/N 930 | Structure of AtBgl1A, a GH1 beta-Glucosidase from Acetivibrio thermocellus | +| [9VX7](https://www.rcsb.org/structure/9VX7) | IRRMC [10.18430/M39VX7](https://doi.org/10.18430/M39VX7) | PAL/PLS 5C (4A) | 4.85 | P 64 | 122.5 122.5 118.9 90.0 90.0 120.0 | PILATUS3 6M | Transcription factor | +| [9VYB](https://www.rcsb.org/structure/9VYB) | IRRMC [10.18430/M39VYB](https://doi.org/10.18430/M39VYB) | PAL/PLS 5C (4A) | 2.12 | P 21 21 21 | 44.4 47.8 48.4 90.0 90.0 90.0 | Dectris Eiger 9M | Antitoxin Phd | +| [9W3Y](https://www.rcsb.org/structure/9W3Y) | IRRMC [10.18430/M39W3Y](https://doi.org/10.18430/M39W3Y) | Photon Factory BL-1A | 1.50 | P 21 21 21 | 60.7 70.0 94.2 90.0 90.0 90.0 | Dectris EIGER1 Si 4M | X-ray Crystal Structure of Pseudoazurin Met16Gly variant (Tris-HCl pH 7.6) | +| [9YL4](https://www.rcsb.org/structure/9YL4) | Zenodo [10.5281/zenodo.17298261](https://doi.org/10.5281/zenodo.17298261) | APS 17-ID | 3.70 | P 21 21 21 | 95.8 111.3 403.0 90.0 90.0 90.0 | PILATUS 6M | Crystal structure of PprA S-F filament from Deinococcus radiodurans | +| [9YZK](https://www.rcsb.org/structure/9YZK) | IRRMC [10.18430/M39YZK](https://doi.org/10.18430/M39YZK) | ALS 8.2.2 | 4.44 | I 1 2 1 | 75.8 163.0 192.3 90.0 98.6 90.0 | PILATUS3 S 2M | Isoreticular co-crystal 1 with symmetrical expanded duplex (42mer) containing insert sequence ACCCTTCTATGACCTACTCCA | +| [9Z44](https://www.rcsb.org/structure/9Z44) | IRRMC [10.18430/M39Z44](https://doi.org/10.18430/M39Z44) | ALS 8.2.1 | 7.20 | I 1 2 1 | 73.5 127.7 141.2 90.0 92.0 90.0 | Dectris EIGER2 Si 9M | Isoreticular co-crystal 1 with symmetrical expanded duplex (31mer) containing insert sequence CCCGGCCGGA and loaded with C-clamp domain | +| [9Z72](https://www.rcsb.org/structure/9Z72) | SBGrid [10.15785/sbgrid/1239](https://doi.org/10.15785/sbgrid/1239) | SSRL BL9-2 | 2.38 | P 31 2 1 | 59.2 59.2 426.2 90.0 90.0 120.0 | Dectris EIGER2 Si 16M | Structure of V. cholerae CapS (form 1) | +| [9ZLO](https://www.rcsb.org/structure/9ZLO) | Zenodo [10.5281/zenodo.18652652](https://doi.org/10.5281/zenodo.18652652) | Australian Synchrotron MX2 | 2.00 | P 21 21 21 | 38.4 90.0 107.0 90.0 90.0 90.0 | Dectris EIGER1 Si 16M | Crystal structure of Proteus mirabilis UreE | +| [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 | +| [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) | + +Seven rows have no PDB code. Six are small-molecule / chemical-crystallography datasets, kept +because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero detector +2θ; the seventh is the second collection in the 6R72 Zenodo record, described below. They have +no deposited macromolecular values, so those columns are blank, and their titles are the +repository record titles verbatim. + +Five datasets are in primitive space groups with no screw axis - 6ZQR, 6ZQY, +6ZR0 and 9FCF in P 4, and 6NEN in P 3 1 2. They are in the battery as negative controls for +screw-axis detection: the correct answer for each has no systematic absences. + +6Z9G is the collection's only index-4 superstructure: its deposited cell is four times the +sublattice a/2, b, c/2, and the four copies of each of its two entities are related by the +XOR-closed trio of near-pure translations (1/2,0,0), (0,0,1/2) and (1/2,0,1/2). Every other +pseudo-translation in the collection is index 2 or a setting artefact, so it is the one set that +exercises a supercell of index greater than two. + +## Archives that are not a single sweep + +Most rows above are a single continuous rotation. The archives described in this section are +not, or needed special handling to obtain the images; their layout is read from the image files +themselves, from the repository file listings and from the depositors' own description of the +record. Not every archive in the table has had its layout audited to this depth. Where an archive +held more than one collection, only one is kept - +the repository's project page is not a reliable guide to this, because it describes the project +rather than the tarball (7TCD's page lists a 900-frame miniCBF sweep the archive does not +contain). + +**6R72 - two collections on one crystal.** The Zenodo record holds two complete 360° sweeps of +3600 × 0.1° frames taken from the same crystal: a helical collection, which produced the +deposited structure, and a low-dose collection from a single position, which was not used for a +deposition and therefore has no PDB entry. Both are in the table, sharing one DOI; the deposited +values belong to the helical collection only. The record also ships the authors' `XDS.INP`. + +**The three CHESS depositions - wedges plus a measured background.** Each crystal was rotated in +50° wedges of 500 × 0.1° frames and translated between wedges to spread the dose. Each crystal +also has a rotation at 1° per frame taken with the crystal translated out of the beam, which the +depositors include as a measured background and say can be matched to the diffraction frames by +the `phi` value in the image header. + +| PDB | Crystals | Wedges per crystal | Background rotation | +|---|---|---|---| +| 8DYZ | 1 | 8 | 360 frames | +| 8DZ7 | 2 | 4 | 200 frames per crystal | +| 6O2H | 4 | 1, 3, 2, 5 - 11 in all | 50, 145, 95, 235 frames, one per crystal | + +**Four archives added for facility coverage hold more than one sweep.** One sweep is kept, and +the row is pinned to it. 7BGU's MXRDR record is one directory of 900 marCCD frames that are two +sweeps with different oscillation widths: frames 1001-1674 (0.4°) are the row, frames 1675-1900 +were moved to `sweep2/` so the reader sees one sweep. 5JK4's archive holds a high-resolution +sweep of 185 frames (80 mm, 1°) and a low-resolution one of 93 frames (250 mm, 2°); the row is +the high-resolution sweep. 6RYM's zip holds two sweeps (`jmp47a2_1`, 70 frames; `jmp47a2_2`, 60 +frames); the row is the first. 6GVK's Zenodo record has three tarballs (`set1`-`set3`); only +`set1` (1800 miniCBF frames) was downloaded and is the row. + +**Seven IRRMC archives hold more than one collection.** In six of them one sweep is kept and +the rest were deleted, so a run over the data directory sees a single collection per dataset. +7RIS is the exception: its two sweeps are at different wavelengths and both are kept. + +| PDB | What the archive holds | Kept | +|---|---|---| +| 6UKF | two sweeps on one crystal - 960 x 0.25° (240°) and 1440 x 0.25° (360°) | the 360° sweep | +| 7DKP | two complete 360° sweeps on one crystal, 3° apart in ω | the first | +| 9PBB | two overlapping 135° wedges of one crystal, 90 x 1.5° each | the first | +| 8U0I | a 69-frame screening wedge and three 180° sweeps on three crystals | the first 180° sweep | +| 36GK | two 360° sweeps of 1800 x 0.2° at the same geometry | the one the archive and DOI are named for | +| 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å | +| 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** | + +**Ten further archives hold more than one collection.** Their layout was read from the +image files and repository listings; one sweep is kept for a run over the data directory unless +noted. + +| PDB / dataset | What the archive holds | Kept | +|---|---|---| +| 5JVN | two 360° sweeps of one crystal, 3600 × 0.1° each (`w1_3`, `w1_4`) | the `w1_3` sweep | +| 6FID | two 360° sweeps of one crystal, 3600 × 0.1° each | the first | +| 6IU8 | a two-wavelength MAD pair, 720 × 0.5° each at 1.605 Å (low remote) and 1.740 Å (peak) | **both** - the pair is the point | +| 7OS3 | four 360° sweeps at λ 2.066 Å, 3600 × 0.1° each, from two crystal positions (`pos2_1/2`, `pos3_1/2`) | all four are kept as separate sweep directories `pos*/` | +| 7L84 | two ~720° helical sweeps, 1439 × 0.5° each at λ 1.892 Å, room temperature | the `301_helical_1` sweep | +| 5M17 | seven crystals in one tar (5M03/5M17/5MEL/5MC8/5M5D/5M3W/5LYR), one 1800-frame sweep each | only the 5M17 tar was downloaded | +| cytidine | six scans, three ω and three φ, at 2θ = 30° (I19-1 commissioning) | the 1800-frame φ scan | +| lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run | +| 9E2T | one continuous sweep plus screening images | the 2700-frame sweep | +| 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) | + +**Three archives needed special handling to obtain the images.** + +- **5KY6** is served by MXRDR as 11 separate RAR archives, one folder of frames per archive, 50 + frames per archive except the last, 564 frames in all. Reading them needs a RAR reader with + RAR3 filter support: the official 7-Zip `7zz` reads them, while the unrar-free and p7zip builds + of Enterprise Linux 8 cannot. +- **6ZR0**'s zip, as the Keele University repository serves it, is damaged: it has no central + directory. Frames 1-1059 of the 1060 were recovered from the zip's local file headers; the last + frame is lost. +- **6NEN**'s University of Queensland eSpace record blocks scripted download, so its archive was + downloaded by hand in a browser. + +## Datasets published as Raw Data Letters + +Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a +format whose purpose is to make raw images citable and re-processable in their own right. The +letters describe the collections and the difficulties in them, and are the reference for what the +data are: + +- V. Zampieri, A. Vermot, M. Thepaut, I. Petit-Hartlein, F. Fieschi, P. Falson and V. Chaptal, + "X-ray diffraction images for two membrane protein crystals presenting high anisotropy; the + *B. subtilis* ABC transporter BmrA and the *S. pneumoniae* NADPH oxidase" (2025), IUCrData 10, + x250591 [doi:10.1107/S2414314625005917](https://doi.org/10.1107/S2414314625005917) - covers + 6R72 and 8QQ7. +- V. Neviani, M. Lutz, W. Oosterheert, P. Gros and L. Kroon-Batenburg, "Crystal structure of the + second extracellular domain of human tetraspanin CD9: twinning and diffuse scattering" (2022), + IUCrData 7, x220852 + [doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR. + +The authors of the second letter also published their own reciprocal-space reconstruction of the +6RLR data as a separate Zenodo record, +[10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763). + +## Detector column for marCCD and SMV files + +marCCD and SMV files name the detector differently - or not at all. A marCCD file names no model: +its instrument header states the image dimensions and the pixel size, from which the plate size +follows (3072 x 73.242 um = 225 mm, 4096 x 73.242 um = 300 mm), and its comment block a serial +number; the LS-CAT beamlines additionally write `detector='Rayonix MX-300 s/n 023'` into the +dataset comment. An ADSC-style SMV header names only a serial (`DETECTOR_SN=930`); a Rigaku d*TREK one names the +model (`CCD_DETECTOR_DESCRIPTION=Saturn944+`), and that is what its row carries. For those rows the +Detector column carries what the file itself establishes: the plate size (`marCCD, 225 mm +plate`), the comment's name where one is present (`Rayonix MX-300`), or the serial (`SMV, S/N +930`). + +## Deposited models and structure factors + +184 of the 191 datasets have a released PDB entry, and RCSB +reports released structure factors (`status_code_sf = REL`) for every one of them. A merged +result from this pipeline can therefore be checked against the deposited model or against the +deposited intensities. + +## Rows where our reduction and the deposition disagree + +Six of the 181 rows are ones where `rugnux` does not reproduce the deposited space group or +cell, and where we have looked at the disagreement closely enough to change how the row is +scored. They are collected here because a scoring row that silently disagrees with a published +entry is not something a reader should have to discover from the code. + +**These are open questions, not errors we are attributing to the PDB.** A deposited entry was +arrived at by someone who had something we do not: a model that had to refine, and usually more +knowledge of the crystal than the images carry. Where we describe evidence below, it is evidence +about *what these images support*, which is a narrower thing than what the crystal is. In every +one of the symmetry rows the possibility that the crystal really has the lower symmetry, with a +pseudo-symmetry too exact for any test available to us to see, remains live - see the limit at +the end of this section. + +How the manifest records it, in `tools/battery/open.json`: + +- `ref` always keeps the deposited values verbatim, so the deposition is never lost. +- `ref_alternatives` lists the *other* answers the row accepts. Each one replaces the reference + fields it names - a space group, a cell, or both - and the row passes if our answer matches any + of the references, the deposited one included. Each must carry `why`; an alternative with no + stated reason is a schema error, not a silent pass, so the mechanism cannot become a way to + turn a failure into a pass quietly. This is how the five knife-edge rows below are recorded: + we are not asserting that our answer is right, only that both descriptions are defensible and + that picking either one is acceptable. The report counts these rows separately from ordinary + passes and prints the reason, so a reader can see how many there are and judge each. +- `ref_override` replaces the fields the battery scores against, with `ref_override_why`. It + *asserts* a corrected reference, so it is for a reference we can show to be wrong about these + images - 8XBP below - and not for a disagreement that is open. +- `unscored` drops the row from scoring entirely. It is a last resort: it also loses a test that + still works, which is why an open question is now recorded as accepted alternatives instead. + +### 8XBP is a question about provenance, not about symmetry + +8XBP is different in kind from the other five and should not be read alongside them. Nothing +here concerns the deposited model or its space group. The question is whether the raw images +uploaded with the entry are the same crystal the deposited cell describes: the master file +records `data_collection_date` 2023-06-21 where the entry records a collection date of +2023-06-23, and the deposited *b* = 50.78 A is 2.0% away from the *b* these images give. Two +independent signals, one of them nothing to do with our processing. The override replaces the +cell with the one DIALS 3.29 indexes de novo on this master and keeps the deposited space group +and resolution. + +### Four trigonal and tetragonal rows where we read a higher point group + +| PDB | Deposited | `rugnux` reads | Where it stands | +|---|---|---|---| +| 6TOC | P 42 | P 42 2 2 | both acceptable; the refinement test is not unanimous | +| 8XTE | P 32 | P 31 2 1 / P 32 2 1 | both acceptable; ours is the better supported | +| 8XTG | P 32 | P 31 2 1 / P 32 2 1 | both acceptable; the **deposition** is the better supported | +| 6PXB | P 32 | P 31 1 2 / P 32 1 2 | both acceptable; unresolved in either direction | + +All four accept either answer: the deposited group and the one we read both pass. None of them +is a claim that the deposited assignment is wrong - each is a question we cannot close, and 8XTE +and 8XTG do not lean the same way, so they should not be read in one voice. The two 8XT* rows +were for a time scored against our own answer by editing the reference itself, with no reason +recorded; the deposition is back in `ref` verbatim and the disagreement is stated here. + +**6TOC.** The deposited asymmetric unit holds two chains, and they are related by the very +two-fold the higher group adds, to 0.16 A C-alpha RMSD over 43 residues - coordinate error at +the deposited 1.85 A. Merging in P 42 2 2 costs 0.0006 in Rmeas for 1.75 +times the multiplicity, and correlates better with the deposited model than the P 42 +merge does. POINTLESS, run independently on our own P1 merge, reads the same point group. The +refinement test - refine in each candidate group and compare R-free, which is the one comparison +not biased toward the group the deposited model was refined in - does **not** come out +unanimous: ZANUDA 1.097 makes P 42 2 2 the better group at half the parameters and +reports the deposited assignment incorrect, while an independent Refmac 5.8.0431 comparison on a +symmetry-consistent free set makes P 42 the better one, by less than the spread +between refinement protocols - the spread of the test exceeds the effect it is being asked to +measure. Both answers are therefore accepted, with the refinement evidence recorded as split. + +**8XTE.** The distinguishing test is the twin-immune centric zone: reflections that the higher +group makes centric but the subgroup does not are their own twin mates, so a merohedral twin law +cannot make them read centric. They read `<|E^2-1|>` = 0.946 +/- 0.012 against a centric +expectation of 0.968 and an acentric one of 0.736. Re-refinement on a shared free set, with the +twin law removed from both sides, favours the higher group. The deposited entry's published R +values are themselves reproducible only with a twin law the entry does not declare, at a twin +fraction of 0.50 - and a 0.50-twinned target already has the symmetry in question. Of the four +rows this is the one where the evidence most clearly favours what we read; it still cannot be +closed, because the centric zone is the only test that speaks to it (see the limit below), so +both answers are accepted. + +**8XTG.** This row is genuinely open and is flagged as such in the manifest. Every +correlation-based instrument we have - our own operator correlations, and POINTLESS on our P1 +merge - reads the higher point group, but the centric-zone test, the only one of them that can +separate real symmetry from pseudo-symmetry, reads `<|E^2-1|>` = 0.869 at -44.9 nats: between +the two expectations, and on the wrong side. The L-test indicates a twin fraction near 0.20-0.26. +Whether this crystal is partially twinned or purely pseudo-symmetric has not been established. +Here the better-supported answer is the deposited one, which is the opposite of 8XTE: the two +rows look alike in the table and are not alike in the evidence. Both answers are accepted. + +**6PXB.** Unscored rather than overridden, because the evidence does not settle either way. Our +merge and POINTLESS both read a 312 point group, the added two-folds correlate at or above the +level of the three-folds nobody disputes, and merging in the higher group *lowers* +Rmeas at twice the multiplicity. Against that, the deposited asymmetric unit's six +chains pair under the added two-fold at 0.3-0.7 A, which is more than coordinate error at 1.75 A, +and ZANUDA settles on a different trigonal supergroup - 321 rather than 312 - whose operators +these data do not support. Neither answer is established in either direction, so both are +accepted and the row still tests everything else about the set. + +### 9RCI: two defensible descriptions of one lattice + +The sixth row is not about symmetry but about which cell describes the crystal. The Patterson +has an off-origin peak at 62.5% of the origin, so a genuine translational NCS relates the two +halves of the cell `rugnux` reports, and the deposited cell is that supercell's +(0, 1/2, 1/2)-centred sublattice to 0.17%. Both are correct descriptions of the same diffraction: +one leaves the near-translation in the contents of a doubled cell, the other absorbs it into the +lattice and indexes only the strong sublattice. Which one a program should prefer is a choice, +not a measurement, so the row accepts either. The alternative cell recorded in the manifest is +computed from the deposited cell alone (c' = b + 2c, centring removed), not copied from our +output, so it stays a statement about the deposition's lattice. + +### The limit that applies to all four symmetry rows + +A merohedral twin at a twin fraction of exactly 0.5 and a crystal that genuinely has the higher +symmetry predict **identical** intensities. No amount of data and no refinement R separates them, +and the same holds, approximately, for a pseudo-symmetry that is merely very exact. Every test +described above measures how nearly a symmetry operator holds on these images; none of them can +show that it holds exactly. Where the higher symmetry is right, merging in it gains multiplicity +and completeness; where it is a pseudo-symmetry that close, merging in it costs nothing +measurable either. That is why these rows are described as open questions, and why none of them +should be read as a statement that a deposited model is wrong. + +## Dataset directories whose name is not the PDB code + +| Directory | PDB code in the table | Why | +|---|---|---| +| `7brr` | 7D1M | The IRRMC archive and its DOI are published under 7BRR, which the PDB obsoleted on 2020-10-28 and replaced with 7D1M. The directory and the DOI keep the archive's own name; the deposited values are 7D1M's. | + +## An archive that ships placeholder images + +8AGQ's `data/` directory contains 30 files named `ForBackgroundOnly_000NN.img` alongside the +1800-frame sweep. They are not images: each is a 64-byte text file holding a path string. A +reader that globs `*.img` will pick them up, so they are named here rather than silently left. + +## Datasets with no PDB entry + +| Dataset | Repository record | Why there is no PDB code | +|---|---|---| +| `6r72/ld` | Zenodo record 10.5281/zenodo.14894181, file prefix `V-CK63-8-ld_1_` | a second collection in the 6R72 record - a low-dose sweep on the same crystal, not the one the deposited structure was built from | +| `cuhf2` | Zenodo record 10.5281/zenodo.6347466 | a small-molecule dataset, not a PDB deposition | +| `dnba` | Zenodo record 10.5281/zenodo.1036416 | a small-molecule dataset, not a PDB deposition | +| `metformin` | Zenodo record 10.5281/zenodo.20135265 | a small-molecule dataset, not a PDB deposition | +| `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition | +| `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition | +| `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition | + +Five of the six small-molecule sets have a published structure to check a run against. These are +reference values from the literature, not results obtained here. + +| Dataset | Space group | Cell (A, deg) | T | Reference | +|---|---|---|---|---| +| `dnba` | `C 1 2/c 1` (15) | 20.2635 8.7575 9.6697 / 90 109.941 90 | 30 K | the Zenodo record's own title and the `xia2.html` the depositors ship inside it, corroborated by COD 4510614/4510615 - Cryst. Growth Des. **13** (2013) 1861-1871 [doi:10.1021/cg300906j](https://doi.org/10.1021/cg300906j) | +| `metformin` | `P 1 21/c 1` (14) | 7.9104 13.8794 7.9310 / 90 114.606 90 | 100 K | the hydrochloride, form I; COD 2108029 - Acta Cryst. B**73** (2017) 10-22 [doi:10.1107/S2052520616017844](https://doi.org/10.1107/S2052520616017844) | +| `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) | +| `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) | +| `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) | + +`cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**, +064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no +numeric cell was located, so a run on it can be scored on the space group and not on the cell. + +## The collection in numbers + +The collection was chosen to widen the spread of file formats, detectors, facilities and +symmetries rather than to be easy to process. The counts below describe where it comes from; +like everything else on this page, they are metadata about the depositions and their files, not +measurements. + +- **Repository:** IRRMC 94, SBGrid 36, Zenodo 34, MXRDR 16, Keele University 4, ESRF 3, XRDa 3, + UQ eSpace 1. +- **Facility** - counted from the facility part of the Facility / beamline column, the beamline + ignored so that entries deposited with and without one count the same, over the 180 rows that + name one: APS 26, Diamond 20, ESRF 17, NSLS-II 14, BESSY 12, PETRA III 12, SSRL 11, ALS 8, + SLS 7, SOLEIL 7, SPring-8 7, SSRF 7, PAL/PLS 5, CHESS 4, CLSI 4, ALBA 3, Australian + Synchrotron 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, and one each from NSRRC, Photon Factory, + RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other ten rows were collected on laboratory sources: nine on + rotating anodes and one on a liquid-metal jet. +- **Crystal system, from the deposited space group of the 184 PDB-coded rows:** orthorhombic 46, + monoclinic 44, tetragonal 30, trigonal 21, hexagonal 17, cubic 13, triclinic 13. +- **Pink beam:** none of these datasets was collected with pink beam. All 184 PDB-coded rows + are deposited as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all + but 5REO, which leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` + `M`); 9Q41 is the one row recorded with a multilayer rather than a crystal monochromator + (CHESS Rh/B4C). The battery's pink-beam data are in-house SLS measurements (tag `pink-beam` + in `tools/battery/inhouse.json`), not on this page. +- **Long cell axes:** eleven PDB-coded rows have a deposited cell axis longer than 320 Å - 8V4O, + 9ZMU, 9Z72, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL. + +The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats +natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with. + +## Licences + +Each dataset carries the licence of its own deposition, stated on the record page linked +above. IRRMC and the SBGrid Data Bank both release under CC0 and both ask that the dataset's +own citation - its DOI - be used; the Zenodo records here are CC0 or CC BY 4.0, as each +record states. None of these data are redistributed with Jungfraujoch; this page only records +where they came from. diff --git a/_sources/FPGA.md.txt b/_sources/FPGA.md.txt new file mode 100644 index 000000000..1a4a0c9b4 --- /dev/null +++ b/_sources/FPGA.md.txt @@ -0,0 +1,83 @@ +# FPGA smartNIC + +See separate document for [installation instructions](DEPLOYMENT.md). + +## Hardware +Currently supported FPGA is only **Xilinx Alveo U55C**. + +See AMD/Xilinx webpage for [card user guide (UG1469)](https://docs.xilinx.com/r/en-US/ug1469-alveo-u55c). +According to the user guide: +``` +Alveo data center accelerator cards are designed to be installed into a data center server, where controlled air flow provides direct cooling. +``` + +The card needs to be placed in a PCI Express (PCIe) Gen4 x8 slot, though mechanically slot has to accommodate x16 card. +There is no need to connect additional power cable, as power of the card is not exceeding 75 W load available from PCIe edge connector. +Current power estimation is about 30 W when idle and 45 W in operation. The card has built-in protection, which will cut power to the card if HBM temperature is above 120°C. + +Two variants of the card are available: +* `100g` - this variant operates one port in 100 Gbit/s mode and should be used when connecting detector via a switch. +* `8x10g` - this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables +are necessary. It is designed for detector directly connected to the Jungfraujoch server, without switch. + +See [network documentation](FPGA_NETWORK.md) for details of network. + +## Building firmware +The firmware build targets are generated by CMake only when `vivado` and `vitis_hls` are detected in +the path, and the Vivado version has to match the one below precisely. + +### Xilinx Vivado +The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolsets version **2022.2** installed on the machine. +Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - +specifically newer versions of Vivado will not work. + +In addition to the Intellectual Property (IP) cores included in Vivado, two additional licenses are necessary: +* Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html), to build `100g` design. +* A paid license for the 10G/25G Ethernet Subsystem for Ultrascale+ is necessary to build the `8x10g` design. +PSI received non-cost licenses from Xilinx University Program for the latter cores. Therefore, usage of bitstreams +generated by PSI continuous integration pipeline for `8x10g` is only allowed for non-commercial use. + +### HLS compilation +Make HLS routines: +``` +mkdir build +cd build +cmake .. +make hls +``` + +### Synthesis +Create PCIe `100g` bitstream with the following command: +``` +mkdir build +cd build +cmake .. +make pcie_100g +``` +and `8x10g`: +``` +mkdir build +cd build +cmake .. +make pcie_8x10g +``` +### When Vivado is not present + +During CMake execution, the following executables: `vivado` and `vitis_hls` must be present in the path. +If not, build targets will not be generated, and such or similar error message will show up: +``` +$ make pcie_100g +make: *** No rule to make target 'pcie_100g'. Stop. +``` + +### Firmware releases +The firmware is stable and is carried from version to version: the MCS files attached to a release +are normally the ones from the release before it (see [Release contents](RELEASE_CONTENTS.md)). When +it does need to change, it is rebuilt with the targets above on a machine with Vivado. + +## Frame generator + +The Jungfraujoch card is equipped with a frame generator. It allows simulating a JUNGFRAU detector without having access to such a system. +It sits in parallel with the Ethernet MAC, so it is placed before the network stack and before any processing happening on the card. +In the future a redirection will be possible to send the simulated stream through the 100G TX network link. +Frame generator is written in HLS and controlled with AXI-Lite. \ No newline at end of file diff --git a/_sources/FPGA_DATA_ANALYSIS.md.txt b/_sources/FPGA_DATA_ANALYSIS.md.txt new file mode 100644 index 000000000..2afd2c48c --- /dev/null +++ b/_sources/FPGA_DATA_ANALYSIS.md.txt @@ -0,0 +1,84 @@ +# FPGA data analysis + +Jungfraujoch FPGA design has incorporated X-ray diffraction image analysis capabilities. + +## Pixel mask +Pixels can be masked. For each module a 32-bit map of pixels is loaded to FPGA, with non-zero value meaning masked pixels. +According to this map, pixels will be assigned a special value (minimum number for signed types and maximum number for non-signed types) +and will be excluded from subsequent analysis. + +## ADU histogram +Before conversion to photons/energy, an ADU histogram can be calculated for a module. This allows to preserve some signature +of unconverted values. This is done on a module-basis and works with bins with 32 ADU width. + +For EIGER this can be used as just a histogram procedure. + +## JUNGFRAU conversion +For JUNGFRAU, module images are converted from ADUs to an energy value and divided by a given number to give units of keV. +Result of the operation is rounded to integers. + +## Pixel thresholding +Pixel range can be specified. +Pixels below a minimum threshold will be assigned zero. +Pixels above a maximum threshold will be assigned saturated pixel value (the largest number for a given bit-width and sign type). +This is specifically designed to operate on unsummed frames, so frame-specific parameters (overload/noise) can be handled. + +## Frame summation +Frames can be summed together (on a per-module basis) in Jungfraujoch, with a limit of 256 frames added together. + +## Azimuthal integration +To implement azimuthal integration, FPGA is able to sum pixels based on a provided integration map and per-pixel corrections. +This way Jungfraujoch implements azimuthal integration with solid angle and polarization corrections. +Corrections were implemented according to formulas developed by [Jensen et al. (J. Synchrotron Rad., 29, 1420-1428, 2022)](https://journals.iucr.org/s/issues/2022/06/00/fv5148/). + +Given FPGA limitations, split-pixels cannot be implemented and number of bins is limited to 2048 per detector module. +This way 2D azimuthal integration, as needed for example by SAS-TT, cannot be currently implemented with the FPGA card and needs to be done on a CPU. +One needs to be careful with per-pixel corrections - their acceptable range is constrained by 16-bit fixed point integer implementation +and is tuned for standard SAXS/WAXS range. + +As with ROIs, azimuthal integration is also available on CPU through the shared analysis library, +so it applies to both the FPGA-accelerated (JUNGFRAU/PSI) and the DECTRIS-driven (EIGER) workflows. + +## Spot finding +Jungfraujoch FPGA implements a built-in spot finder. Spot finder allows to apply the following criteria for finding strong pixels: +1. Resolution criterion - pixels only within a provided resolution range can be considered as strong pixels (calculating resolution map needs to happen on CPU before data collection run). +2. Bad pixels - pixels marked as bad, as well as chip edges and module edges are excluded from spot finding, +3. Overloads - pixels marked as overloads on JUNGFRAU are always included in the strong pixel output, but are excluded for signal-to-noise ratio calculation, +4. Pixel value - pixels above certain threshold value can be marked as strong, +5. Signal-to-noise (SNR) ratio - pixels with SNR above a threshold can be marked as strong, +6. Connected pixels - strong pixels can be discarded if they are "alone", so their 8 directly neighboring pixels are not counted as strong pixels. + +All the above criteria except the bad-pixel criterion are optional (can be turned off); only pixels that fulfill all enabled criteria are selected as strong pixels. + +### SNR ratio calculation +Signal-to-noise ratio is calculated for a rectangular area. +In horizontal direction the area is fixed - line of 1024 pixels is divided into 32 areas each of 32 pixels. +This is dictated by the data flow within the FPGA. +In vertical direction the area is flexible - it is 15 lines above and below of the given pixel. +Given the very large box size, approximations are made, for example that `N ≈ N-1` in calculating standard deviation. + +## Region-of-interest (ROI) integration +There are 16 ROIs, and the ROI map holds a 16-bit mask per pixel, so a pixel can belong to any subset of them (including none). For each ROI, sum, sum of squares, max count, and number of valid pixels will be calculated. +Jungfraujoch also calculates X and Y values weighted by pixel values, though this feature is not properly tested at the moment and not integrated in downstream analysis. + +ROIs are not specific to the FPGA path. The same ROI definitions — box, circle, and azimuthal +(Q-range with an optional φ-sector) — are also evaluated on CPU by the shared `image_analysis/roi/` +engine, so ROI statistics are produced both for the FPGA-accelerated JUNGFRAU/PSI workflow and for +detectors driven through DECTRIS SIMPLON (e.g. EIGER), which have no FPGA acquisition path. + +## Pixel statistics +The following statistics are collected for each module: +* Number of masked pixels +* Number of saturated pixels (excl. masked) +* Number of error pixels (excl. masked) +* Sum of valid pixels in the module +* Minimum value of valid pixels in the module +* Maximum value of valid pixels in the module + +Valid pixels are not masked, not saturated, not error pixels. + +## Square root compression +Jungfraujoch FPGA includes lossy compression preserving counting statistic properties of X-ray image, while reducing bit width of an image. +Scheme was described in [Wakonig et al., J. Appl. Cryst., 53, 574-586, 2020](https://doi.org/10.1107/S1600576720001776). +Pixel value `X` is replaced with `round(sqrt(N*N*X))`, i.e. `round(N*sqrt(X))`, where `N` is integer constant in range 1 to 16. +`N` is what the host writes to the `sqrtmult` register; the FPGA squares it before multiplying the pixel value. diff --git a/_sources/FPGA_DESIGN.md.txt b/_sources/FPGA_DESIGN.md.txt new file mode 100644 index 000000000..be7e1285c --- /dev/null +++ b/_sources/FPGA_DESIGN.md.txt @@ -0,0 +1,22 @@ +# FPGA data flow + +The following steps are performed on FPGA (in the order of operation): + +1. UDP header decoding +2. SLS detector header decoding +3. State machine that controls data acquisition (start/stop/cancel) +4. High-bandwidth memory cache to buffer network packets and reorder them to form full modules +5. ADU histogram for JUNGFRAU +6. Mask pixels from missing packets with special value +7. Reorder lines for EIGER to form a proper module +8. Mask pixels based on provided pixel mask +9. JUNGFRAU conversion with gain and pedestal corrections +10. Threshold: pixel values below a set minimum are zeroed, values above a set maximum saturated +11. Frame summation (up to 256 frames) +12. Integration according to predefined map (e.g., 1D azimuthal integration) +13. Spot finding +14. ROI calculation +15. Image lossy compression using N*sqrt(pixel) values +16. Send images, analysis results and metadata to host memory via PCI Express + +Each step has a dedicated core, written in high-level synthesis. Exact operation of cores for data analysis is explained in dedicated [document](FPGA_DATA_ANALYSIS.md). \ No newline at end of file diff --git a/_sources/FPGA_LICENSE.md.txt b/_sources/FPGA_LICENSE.md.txt new file mode 100644 index 000000000..540a96050 --- /dev/null +++ b/_sources/FPGA_LICENSE.md.txt @@ -0,0 +1,295 @@ +# FPGA license + +FPGA components of Jungfraujoch are licensed using OHL-S license. See full text below. +The license is equivalent of GNU Public License with adaptations for hardware. +See [OHL webpage](https://ohwr.org/project/cernohl/-/wikis/Documents/CERN-OHL-version-2) for details and FAQs. + +## CERN Open Hardware Licence Version 2 - Strongly Reciprocal + + +Preamble + +CERN has developed this licence to promote collaboration among +hardware designers and to provide a legal tool which supports the +freedom to use, study, modify, share and distribute hardware designs +and products based on those designs. Version 2 of the CERN Open +Hardware Licence comes in three variants: CERN-OHL-P (permissive); and +two reciprocal licences: CERN-OHL-W (weakly reciprocal) and this +licence, CERN-OHL-S (strongly reciprocal). + +The CERN-OHL-S is copyright CERN 2020. Anyone is welcome to use it, in +unmodified form only. + +Use of this Licence does not imply any endorsement by CERN of any +Licensor or their designs nor does it imply any involvement by CERN in +their development. + + +1 Definitions + +1.1 'Licence' means this CERN-OHL-S. + +1.2 'Compatible Licence' means + +a) any earlier version of the CERN Open Hardware licence, or + +b) any version of the CERN-OHL-S, or + +c) any licence which permits You to treat the Source to which + it applies as licensed under CERN-OHL-S provided that on + Conveyance of any such Source, or any associated Product You + treat the Source in question as being licensed under + CERN-OHL-S. + +1.3 'Source' means information such as design materials or digital +code which can be applied to Make or test a Product or to +prepare a Product for use, Conveyance or sale, regardless of its +medium or how it is expressed. It may include Notices. + +1.4 'Covered Source' means Source that is explicitly made available +under this Licence. + +1.5 'Product' means any device, component, work or physical object, +whether in finished or intermediate form, arising from the use, +application or processing of Covered Source. + +1.6 'Make' means to create or configure something, whether by +manufacture, assembly, compiling, loading or applying Covered +Source or another Product or otherwise. + +1.7 'Available Component' means any part, sub-assembly, library or +code which: + +a) is licensed to You as Complete Source under a Compatible + Licence; or + +b) is available, at the time a Product or the Source containing + it is first Conveyed, to You and any other prospective + licensees + +i) as a physical part with sufficient rights and + information (including any configuration and + programming files and information about its + characteristics and interfaces) to enable it either to + be Made itself, or to be sourced and used to Make the + Product; or +ii) as part of the normal distribution of a tool used to + design or Make the Product. + +1.8 'Complete Source' means the set of all Source necessary to Make +a Product, in the preferred form for making modifications, +including necessary installation and interfacing information +both for the Product, and for any included Available Components. +If the format is proprietary, it must also be made available in +a format (if the proprietary tool can create it) which is +viewable with a tool available to potential licensees and +licensed under a licence approved by the Free Software +Foundation or the Open Source Initiative. Complete Source need +not include the Source of any Available Component, provided that +You include in the Complete Source sufficient information to +enable a recipient to Make or source and use the Available +Component to Make the Product. + +1.9 'Source Location' means a location where a Licensor has placed +Covered Source, and which that Licensor reasonably believes will +remain easily accessible for at least three years for anyone to +obtain a digital copy. + +1.10 'Notice' means copyright, acknowledgement and trademark notices, +Source Location references, modification notices (subsection +3.3(b)) and all notices that refer to this Licence and to the +disclaimer of warranties that are included in the Covered +Source. + +1.11 'Licensee' or 'You' means any person exercising rights under +this Licence. + +1.12 'Licensor' means a natural or legal person who creates or +modifies Covered Source. A person may be a Licensee and a +Licensor at the same time. + +1.13 'Convey' means to communicate to the public or distribute. + + +2 Applicability + +2.1 This Licence governs the use, copying, modification, Conveying +of Covered Source and Products, and the Making of Products. By +exercising any right granted under this Licence, You irrevocably +accept these terms and conditions. + +2.2 This Licence is granted by the Licensor directly to You, and +shall apply worldwide and without limitation in time. + +2.3 You shall not attempt to restrict by contract or otherwise the +rights granted under this Licence to other Licensees. + +2.4 This Licence is not intended to restrict fair use, fair dealing, +or any other similar right. + + +3 Copying, Modifying and Conveying Covered Source + +3.1 You may copy and Convey verbatim copies of Covered Source, in +any medium, provided You retain all Notices. + +3.2 You may modify Covered Source, other than Notices, provided that +You irrevocably undertake to make that modified Covered Source +available from a Source Location should You Convey a Product in +circumstances where the recipient does not otherwise receive a +copy of the modified Covered Source. In each case subsection 3.3 +shall apply. + + You may only delete Notices if they are no longer applicable to + the corresponding Covered Source as modified by You and You may + add additional Notices applicable to Your modifications. + Including Covered Source in a larger work is modifying the + Covered Source, and the larger work becomes modified Covered + Source. + +3.3 You may Convey modified Covered Source (with the effect that You +shall also become a Licensor) provided that You: + +a) retain Notices as required in subsection 3.2; + +b) add a Notice to the modified Covered Source stating that You + have modified it, with the date and brief description of how + You have modified it; + +c) add a Source Location Notice for the modified Covered Source + if You Convey in circumstances where the recipient does not + otherwise receive a copy of the modified Covered Source; and + +d) license the modified Covered Source under the terms and + conditions of this Licence (or, as set out in subsection + 8.3, a later version, if permitted by the licence of the + original Covered Source). Such modified Covered Source must + be licensed as a whole, but excluding Available Components + contained in it, which remain licensed under their own + applicable licences. + + +4 Making and Conveying Products + +You may Make Products, and/or Convey them, provided that You either +provide each recipient with a copy of the Complete Source or ensure +that each recipient is notified of the Source Location of the Complete +Source. That Complete Source is Covered Source, and You must +accordingly satisfy Your obligations set out in subsection 3.3. If +specified in a Notice, the Product must visibly and securely display +the Source Location on it or its packaging or documentation in the +manner specified in that Notice. + + +5 Research and Development + +You may Convey Covered Source, modified Covered Source or Products to +a legal entity carrying out development, testing or quality assurance +work on Your behalf provided that the work is performed on terms which +prevent the entity from both using the Source or Products for its own +internal purposes and Conveying the Source or Products or any +modifications to them to any person other than You. Any modifications +made by the entity shall be deemed to be made by You pursuant to +subsection 3.2. + + +6 DISCLAIMER AND LIABILITY + +6.1 DISCLAIMER OF WARRANTY -- The Covered Source and any Products +are provided 'as is' and any express or implied warranties, +including, but not limited to, implied warranties of +merchantability, of satisfactory quality, non-infringement of +third party rights, and fitness for a particular purpose or use +are disclaimed in respect of any Source or Product to the +maximum extent permitted by law. The Licensor makes no +representation that any Source or Product does not or will not +infringe any patent, copyright, trade secret or other +proprietary right. The entire risk as to the use, quality, and +performance of any Source or Product shall be with You and not +the Licensor. This disclaimer of warranty is an essential part +of this Licence and a condition for the grant of any rights +granted under this Licence. + +6.2 EXCLUSION AND LIMITATION OF LIABILITY -- The Licensor shall, to +the maximum extent permitted by law, have no liability for +direct, indirect, special, incidental, consequential, exemplary, +punitive or other damages of any character including, without +limitation, procurement of substitute goods or services, loss of +use, data or profits, or business interruption, however caused +and on any theory of contract, warranty, tort (including +negligence), product liability or otherwise, arising in any way +in relation to the Covered Source, modified Covered Source +and/or the Making or Conveyance of a Product, even if advised of +the possibility of such damages, and You shall hold the +Licensor(s) free and harmless from any liability, costs, +damages, fees and expenses, including claims by third parties, +in relation to such use. + + +7 Patents + +7.1 Subject to the terms and conditions of this Licence, each +Licensor hereby grants to You a perpetual, worldwide, +non-exclusive, no-charge, royalty-free, irrevocable (except as +stated in subsections 7.2 and 8.4) patent licence to Make, have +Made, use, offer to sell, sell, import, and otherwise transfer +the Covered Source and Products, where such licence applies only +to those patent claims licensable by such Licensor that are +necessarily infringed by exercising rights under the Covered +Source as Conveyed by that Licensor. + +7.2 If You institute patent litigation against any entity (including +a cross-claim or counterclaim in a lawsuit) alleging that the +Covered Source or a Product constitutes direct or contributory +patent infringement, or You seek any declaration that a patent +licensed to You under this Licence is invalid or unenforceable +then any rights granted to You under this Licence shall +terminate as of the date such process is initiated. + + +8 General + +8.1 If any provisions of this Licence are or subsequently become +invalid or unenforceable for any reason, the remaining +provisions shall remain effective. + +8.2 You shall not use any of the name (including acronyms and +abbreviations), image, or logo by which the Licensor or CERN is +known, except where needed to comply with section 3, or where +the use is otherwise allowed by law. Any such permitted use +shall be factual and shall not be made so as to suggest any kind +of endorsement or implication of involvement by the Licensor or +its personnel. + +8.3 CERN may publish updated versions and variants of this Licence +which it considers to be in the spirit of this version, but may +differ in detail to address new problems or concerns. New +versions will be published with a unique version number and a +variant identifier specifying the variant. If the Licensor has +specified that a given variant applies to the Covered Source +without specifying a version, You may treat that Covered Source +as being released under any version of the CERN-OHL with that +variant. If no variant is specified, the Covered Source shall be +treated as being released under CERN-OHL-S. The Licensor may +also specify that the Covered Source is subject to a specific +version of the CERN-OHL or any later version in which case You +may apply this or any later version of CERN-OHL with the same +variant identifier published by CERN. + +8.4 This Licence shall terminate with immediate effect if You fail +to comply with any of its terms and conditions. + +8.5 However, if You cease all breaches of this Licence, then Your +Licence from any Licensor is reinstated unless such Licensor has +terminated this Licence by giving You, while You remain in +breach, a notice specifying the breach and requiring You to cure +it within 30 days, and You have failed to come into compliance +in all material respects by the end of the 30 day period. Should +You repeat the breach after receipt of a cure notice and +subsequent reinstatement, this Licence will terminate +immediately and permanently. Section 6 shall continue to apply +after any termination. + +8.6 This Licence shall not be enforceable except by a Licensor +acting as such, and third party beneficiary rights are +specifically excluded. \ No newline at end of file diff --git a/_sources/FPGA_NETWORK.md.txt b/_sources/FPGA_NETWORK.md.txt new file mode 100644 index 000000000..c26c4c1a7 --- /dev/null +++ b/_sources/FPGA_NETWORK.md.txt @@ -0,0 +1,39 @@ +# FPGA network + +The U55C card is equipped with two network connectors - QSFP0 is the upper port and QSFP1 the lower port (when PCIe connector is on the bottom). +The card FPGA design is offered in two variants `100g` and `8x10g`. These have different behavior regarding the network: + +`100g` — this variant operates the QSFP0 port in 100 Gbit/s mode and should be used when connecting detector via a **switch**. +QSFP28 transceivers are necessary. + +`8x10g` — this variant operates both QSFP ports at 4x10 Gbit/s. QSFP+ (40 Gbit/s) transceivers and MTO/MTP harness cables +are necessary. It is designed for **detector directly connected** to the Jungfraujoch server, without switch. + +## Transceivers +AMD doesn't provide transceiver compatibility matrix for Alveo U55C. +In our experience operating the card we haven't seen issues with transceivers from various providers (FS.com, Mellanox, Finisar). +We have also successfully operated card with correct direct attach cables instead of fiber optics. Given the card doesn't +support link training functionality of 100 Gbit/s ethernet, it could result in performance problems with copper cables, though we haven't +encountered such a situation. + +## Switch configuration +Special care has to be taken for switch operation, given the FPGA core doesn't support auto-negotiation. It is necessary to configure switch port +to fixed speed (100 Gbit/s or 10 Gbit/s) and to disable auto-negotiation. It is also necessary to enable jumbo frames (MTU of 9000). + +## Network LEDs +Each QSFP connector is equipped with green and orange LEDs. These LEDs are connected to Ethernet physical layer status port (rx_status). +LED on corresponds to having a physical connection to a switch/computer/detector on the other side of the network. +For 100 Gbit/s only green is used, for 8x10 Gbit/s green LEDs mean all ports connected, orange LEDs at least one of the ports connected. + +## Network stack +Each Ethernet link has its own basic network stack. Functionality for Ethernet/ARP/IPv4/ICMP is therefore separately handled for each port. +Each link will get dedicated MAC address, and IPv4 addresses can be also assigned independently if needed. + +The card will send gratuitous ARP messages every 5 seconds to keep its entry in switch MAC table. +The card will also reply to ARP requests for its IP and to ICMP ping requests sent with the card IPv4 address. +The card won't respond to broadcast ICMP pings. + +Each link can be put in `direct` mode. In this case destination Ethernet MAC and IPv4 addresses are not enforced for incoming UDP packets. +This setting should be used for connecting detector modules directly to the FPGA card, so any detector module can be connected to any +10 Gbit/s link on the same card. Currently `direct` mode is turned OFF for `100g` design and ON for `8x10g` design. +This can be manually adjusted for each link. \ No newline at end of file diff --git a/_sources/FPGA_PCIE_DRIVER.md.txt b/_sources/FPGA_PCIE_DRIVER.md.txt new file mode 100644 index 000000000..c5f56f230 --- /dev/null +++ b/_sources/FPGA_PCIE_DRIVER.md.txt @@ -0,0 +1,103 @@ +# FPGA PCIe driver + +## Compilation +To compile the kernel module, type: +``` +make +``` + +## Installation +To install kernel module, you need to have root permissions and run: +``` +sudo make install +``` + +## Loading driver into kernel +After installing the kernel driver, it should be possible to insert it into the kernel via: +``` +modprobe jfjoch +``` + +## Ownership of the character devices +By default, character devices `/dev/jfjoch` are owned by root (user/group) and are not accessible by others. +This means that `jfjoch_broker` must be running as superuser, which might not be optimal for security reasons in most cases. +The behavior can be changed by creating `udev` rules. Create a file called `/etc/udev/rules.d/99-jfjoch.rules` +with the following content: +``` +KERNEL=="jfjoch*", OWNER="", GROUP="" +``` +It is OK to provide only group, for example to make the devices accessible by group `jungfrau`: +``` +KERNEL=="jfjoch*", GROUP="jungfrau" +``` + +## DKMS +To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel +module, when new kernel is installed. For RHEL 8 it is well tested to use the RPM module built automatically from Jungfraujoch source. +For other systems, it is necessary to follow the procedure below, though it is not well tested. + +This first requires installing DKMS - for RHEL it is available via EPEL repository: +``` +sudo dnf install dkms +``` +Then use the script provided in the driver directory to copy driver code to DKMS directory: +``` +./install_dkms.sh +``` +If upgrading the driver, please first remove the current driver from DKMS system: +``` +dkms remove jfjoch -v --all +``` + +## Driver parameters +Currently, there is one driver parameter `nbuffers`, that defines count of exchange buffers (see below). +This can be adjusted in the modprobe operation, for example: +``` +modprobe jfjoch nbuffers=1024 +``` + +## Exchange buffers +The parameter defines number of buffers used to exchange data between card and host application. +Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. +These buffers are used by both card-to-host and host-to-card operations. + +Buffers use special allocation, as they are contiguous in physical address space, which helps the FPGA card to transfer all +data associated with detector module in two DMA transfers (one data, one metadata). +Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in powers of two, **4 MiB** is a safe number for one buffer size. +A buffer can be mapped into user space by performing the `mmap` system call on the `/dev/jfjoch` character device. + +Buffer count can be adjusted by setting `nbuffers` parameter. There are two considerations for setting optimal value: +1. For card-to-host transfers, minimal value is roughly +` * `, +this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64. +2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. +So minimal count is ` * (3 + 3 * )`. + +Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration. + +## Known problems +To avoid inconsistent behavior, this driver won't load if release number differs between the kernel driver and FPGA card. + +## CMake file +While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. +It is not made for actual compilation of the kernel driver and should not be used for that purpose. + +## Character device access +For each FPGA device a character device is created called `/dev/jfjoch`. +When the device is opened, two operations are possible: +* `mmap()` to map exchange buffers +* `ioctl()` to communicate with the card +Interfacing should be done through the JungfraujochDevice class in `fpga/host_library` directory. + +## Sysfs access +Certain performance counters can be read through sysfs mechanism in the kernel. +One needs to `cat` files in `/sys/class/misc/jfjoch/` directory. + +## RHEL 9.5+ virtual memory flags +RedHat Enterprise Linux 9.5 backported the `vm_flags_set` interface from Linux kernel 6.3 while still reporting kernel version 5.14, so a plain kernel-version test picks the wrong branch and the build fails. +This is now detected automatically from `RHEL_RELEASE_CODE`, so the module builds unaided on RHEL 9.5 and later and on the CentOS Stream, Rocky and AlmaLinux equivalents, as well as on distributions that have not backported it. +**No user action is needed.** The `HAVE_VM_FLAGS_SET` environment variable that earlier releases required is obsolete; it is still honoured if set, but setting it is no longer necessary and the DKMS packaging never passed it anyway. + +## Which kernel DKMS builds for +The DKMS package builds the module for the kernel it is being **installed for**, not the one currently running, so a module built while a kernel update is being applied loads correctly after the reboot. +Building by hand in `fpga/pcie_driver/` still defaults to the running kernel; pass `KDIR=/lib/modules//build` (or `KVER=`) to target another one. diff --git a/_sources/FPGA_SETTINGS.md.txt b/_sources/FPGA_SETTINGS.md.txt new file mode 100644 index 000000000..079650630 --- /dev/null +++ b/_sources/FPGA_SETTINGS.md.txt @@ -0,0 +1,122 @@ +# FPGA advanced reference +## Register map +FPGA setup can be done via registers: + +| Address | Bits | Meaning | Mode | Notes | +|---------------------|------|------------------------------------------------------------------------------------------------|:-----|----------------------------------------------| +| 0x000000 - 0x00FFFF | | Reserved (in case using MicroBlaze in the future, this has to be reserved for internal memory) | | | +| 0x010000 | 32 | Action Control Register | | | +| | | Bit 0 - Action start | R/W | | +| | | Bit 1 - Action idle | R | | +| | | Bit 2 - Action cancel | R/W | cleared on reset or action start | +| | | Bit 3 - Clear network counters | R/W | cleared on reset | +| | | Bit 12:4 - Debug signals (see action_config.v for details) | R | | +| | | Bit 16 - AXI Mailbox interrupt 0 | R | | +| 0x010004 | 32 | Reserved | - | | +| 0x010008 | 32 | Reserved | - | | +| 0x01000C | 32 | GIT SHA1 | R | | +| 0x010010 | 32 | Reserved | R | | +| 0x010014 | 32 | Reserved | R | | +| 0x010018 | 32 | Jungfraujoch FPGA variant | R | | +| 0x01001C | 32 | Reserved | R | | +| 0x010020 | 32 | Max. number of supported detector modules | R | constant | +| 0x010024 | 32 | Reserved | R | constant | +| 0x010028 | 64 | Pipeline stalls before writing to host memory | R | reset on action start | +| 0x010030 | 64 | Pipeline stalls before accessing HBM | R | reset on action start | +| 0x010038 | 32 | FIFO status (see action_config.v for details) | R | | +| 0x01003C | 32 | Size of single HBM channel in bytes (default value for the particular card) | R/W | should not be altered for standard operation | +| 0x010040 | 64 | Packets processed by the action | R | cleared on reset or action start | +| 0x010048 | 64 | Valid ethernet packets | R | cleared on reset | +| 0x010050 | 64 | Valid ICMP packets | R | cleared on reset | +| 0x010058 | 64 | Valid UDP packets | R | cleared on reset | +| 0x010060 | 64 | Valid detector packets processed by the card | R | cleared on reset | +| 0x010068 | 64 | Packets flagged as errors by CMAC | R | cleared on reset | +| 0x010070 | 64 | Pipeline stalls before data processing | R | reset on action start | +| 0x010078 | 64 | AXI-beats before accessing HBM | R | reset on action start | +| 0x010080 | 64 | AXI-beats before data processing | R | reset on action start | +| 0x010088 | 64 | AXI-beats before host writer | R | reset on action start | +| 0x010090 | 64 | Last encountered SwissFEL pulse ID | R | cleared on reset | +| 0x010100 | 32 | Spot finder photon count threshold | R/W | | +| 0x010104 | 32 | Spot finder signal-to-noise ratio threshold (single-precision float) | R/W | | +| 0x010200 | 64 | MAC address source for internal frame generator | R/W | network byte order | +| 0x010208 | 32 | IPv4 address source for internal frame generator | R/W | network byte order | +| 0x01020C | 32 | Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.) | R/W | | +| 0x010210 | 32 | Data collection mode | R/W | | +| | | Bit 0 - Conversion to photons | | | +| | | Bit 1 - Output extend to 32-bit | | | +| | | Bit 2 - Output is unsigned integer | | | +| | | Bit 3 - Use sq. root lossy compression | | | +| | | Bit 7 - JUNGFRAU fixed G1 mode | | | +| | | Bit 8 - Set to zero values below threshold | | | +| | | Bit 31:16 - Data collection ID (carried with completions) | | | +| 0x010214 | 32 | Photon energy in keV (single-precision float) | R/W | | +| 0x010218 | 32 | Number of frames expected in the data collection (defines termination condition) | R/W | | +| 0x01021C | 32 | Number of storage cells | R/W | | +| 0x010220 | 32 | Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.) | R/W | | +| 0x010224 | 32 | Coefficient for sq. root compression (need to set bit in data collection mode to apply) | R/W | | +| 0x010228 | 32 | Threshold minimum; values below are set to zero (need to set bit in data collection mode) | R/W | | +| 0x01022C | 32 | Threshold maximum; values above are set to the saturated value | R/W | | +| 0x030000 - 0x03FFFF | | AXI Mailbox for Work Request / Work Completion | | See Xilinx PG114 for register map | +| 0x040000 - 0x04FFFF | | QuadSPI flash | | See Xilinx PG153 for register map | +| 0x050000 - 0x05FFFF | | Interrupt controller | | See Xilinx PG099 for register map | +| 0x060000 - 0x06FFFF | | Load calibration (HLS) | | | +| 0x070000 - 0x07FFFF | | AXI Firewall | | See Xilinx PG293 for register map | +| 0x080000 - 0x08FFFF | | Frame generator (HLS) | | | +| 0x090000 - 0x09FFFF | | PCIe DMA control | | See Xilinx PG195 for register map | +| 0x0A0000 - 0x0AFFFF | | I2C clock generator | | See Xilinx PG090 for register map | +| 0x0C0000 - 0x0FFFFF | | Xilinx Card Management Solution Subsystem | | See Xilinx PG348 for register map | +| 0x100000 - 0x10FFFF | | MAC 10G / CMAC 100G | | See Xilinx PG210/PG203 for register map | +| 0x110000 - 0x11FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x120000 - 0x12FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x130000 - 0x13FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x140000 - 0x14FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x150000 - 0x15FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x160000 - 0x16FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x170000 - 0x17FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x200000 - 0x20FFFF | | Eth/IPv4 network stack for interface #0 | | | +| 0x210000 - 0x21FFFF | | Eth/IPv4 network stack for interface #1 | | | +| 0x220000 - 0x22FFFF | | Eth/IPv4 network stack for interface #2 | | | +| 0x230000 - 0x23FFFF | | Eth/IPv4 network stack for interface #3 | | | +| 0x240000 - 0x24FFFF | | Eth/IPv4 network stack for interface #4 | | | +| 0x250000 - 0x25FFFF | | Eth/IPv4 network stack for interface #5 | | | +| 0x260000 - 0x26FFFF | | Eth/IPv4 network stack for interface #6 | | | +| 0x270000 - 0x27FFFF | | Eth/IPv4 network stack for interface #7 | | | +| 0x400000 - 0x47FFFF | 64 | Address table: decodes handles used by load_calibration and host_writer to DMA addresses | | | + +## AXI Mailbox + +AXI mailbox is used to send work request from host to action, and receive work completions. +Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114). + +Work request has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------------------------| +| 0 | 15 | Work request ID (handle) | + +Work completion has the following structure: + +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------| +| 0 | 15 | Work request ID (handle) | +| | | Special values: | +| | | 65534 - start of data collection | +| | | 65535 - end of data collection | +| 16 | 31 | Data collection ID | + +## HBM memory + +| Interface number | Core | Meaning | +|------------------|------------------|---------------------------------------------| +| 0-1 | jf_conversion | Gain factor G0 | +| 2-3 | jf_conversion | Gain factor G1 | +| 4-5 | jf_conversion | Gain factor G2 | +| 6-7 | jf_conversion | Pedestal G0 | +| 8-9 | jf_conversion | Pedestal G1 | +| 10-11 | jf_conversion | Pedestal G2 | +| 12-13 | integration | Integration map | +| 14-15 | integration | Integration weights | +| 16-17 | spot_finder_mask | Spot finder resolution | +| 18-19 | roi_calc | ROI calculation | +| 20-21 | frame_generator | Frame generator | +| 22-27 | load_from_hbm | Frame summation | diff --git a/_sources/HARDWARE.md.txt b/_sources/HARDWARE.md.txt new file mode 100644 index 000000000..454066233 --- /dev/null +++ b/_sources/HARDWARE.md.txt @@ -0,0 +1,60 @@ +# Hardware requirements +Operating Jungfraujoch requires the following: + +1. High performance server +2. FPGA board(s) installed in the server +3. (optionally) GPU boards +4. (optionally) 100G switch to connect FPGA and the detector + +Unfortunately, at the moment it is not possible to purchase server configuration from a major vendor that would include +AMD FPGA boards. Therefore, the two have to be purchased separately. This might have impact on the warranty for the hardware +and has to be clarified with the vendor. PSI only supports the system on the best effort basis and doesn't take any responsibility +for warranty limitations for operating FPGA boards in the server. Having said this - we didn't encounter any hardware issues so far. + +## High performance server +PSI is using HPE DL380 Gen11 servers at the moment to operate Jungfraujoch systems. However, this is because of general +preference for this vendor, there is no Jungfraujoch-specific reason to buy from this vendor. We do expect that system +from any other vendor with similar specification should work as well. + +At PSI, the configuration of HPE DL380 Gen11 used to operate 9M pixel detectors at 2 kHz is as follows: +* 2 x Intel Xeon 8558P +* 512 GB RAM +* 2 x Nvidia L4 GPU (for indexing) +* 1 x Nvidia ConnectX-6 200G ethernet/IB network (for outgoing traffic; this can be substituted according to facility needs) +* Copper 1G/10G network + +### PCI slots +When ordering the system, check that it can accommodate enough PCIe cards. +In case of our system we need to put at least seven PCIe cards: 4 x FPGA, 2 x GPU, 1 x network. + +Note - for FPGA x8 lane electrically/x16 lane mechanically PCIe slots are OK. + +## FPGA +Jungfraujoch is built for [AMD/Xilinx U55C](https://www.amd.com/en/products/accelerators/alveo/u55c/a-u55c-p00g-pq-g.html) +(A-U55C-P00G-PQ-G) card. Other FPGA cards are currently not supported. + +Single U55C card supports roughly 5 detector modules (2.5M pixels) at 2 kHz and 10 detector modules (5M pixels) at 1 kHz. +For detectors operating at lower frame rates (e.g., 100 Hz) larger detectors can be supported by a single U55C card, though it requires +using TX delay functionality in the detector. + +## GPUs +Operating fast-feedback indexer code requires a graphics processing unit from Nvidia. +For practical reasons, i.e. power consumption and cost, we chose the inference-grade Nvidia L4 card. +In the past we have also used T4 cards. So, in principle any recent CUDA compatible GPU should work. + +Offline processing with [`rugnux`](RUGNUX.md) has a requirement of its own, unrelated to the frame +rate the acquisition side is sized for: one run wants 3-7 GB on the card and up to 14 GB of host +RAM, so a server that also reprocesses data needs headroom beyond the online pipeline's. The +sizing table is in [Installing Rugnux ▸ Memory](RUGNUX_INSTALL.md#memory). + +## Network switch +Small detectors (up to 4M pixel) can be in principle operated without switch. In this case one needs `8x10g` variant +of the Jungfraujoch FPGA image, which allows 4 JUNGFRAU modules (two 10 Gbit/s links each) to be connected directly to one U55C card. + +Such configuration is however +impractical for larger systems or more complex deployments, like multiple detectors operated from one Jungfraujoch server. +In this case one needs a network switch. + +We currently use Nvidia/Mellanox SN2100 switch, though there is no reason not to use other models/other vendors. +A switch with only 100G ports must support splitting them into 4x10G ports to connect the detector. + diff --git a/_sources/HDF5.md.txt b/_sources/HDF5.md.txt new file mode 100644 index 000000000..2cdc4c73b --- /dev/null +++ b/_sources/HDF5.md.txt @@ -0,0 +1,631 @@ +# HDF5 / NeXus data format + +Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html)-compliant. On top of the +NXmx application definition, Jungfraujoch records a substantial amount of *derived* metadata +(spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These +extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and +reusable. + +This page documents the **file layout and the data fields**. The operational behaviour of the +writer (running, republishing, file finalisation) is described in +[jfjoch_writer](JFJOCH_WRITER.md). The wire format that feeds the writer is described in +[CBOR messages](CBOR.md); fields below frequently correspond one-to-one to CBOR message fields, and +that document is a useful companion for their meaning. + +```{contents} On this page +:local: +:depth: 2 +``` + +## 1. Motivation: derived metadata and FAIR data + +The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep +them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both +**rotation** macromolecular crystallography (single- and multi-crystal, including fine-sliced and +helical scans) and **serial** crystallography (stills, grid scans); the same concerns apply to both: + +* **Findability.** Raw diffraction images carry almost no descriptive metadata about *content*. + Quantities such as background level, number of diffraction spots, or indexing outcome let a user + judge the quality and relevance of a dataset *before* inspecting the raw images. +* **Accessibility at scale.** A single experiment can span tens to hundreds of terabytes. Standard + retrieval (e.g. HTTP) makes a dataset *available* but not *inspectable* — users would otherwise + have to download a large fraction of the data just to decide whether it is useful. Compact + derived representations make discovery, assessment and reuse feasible. + +Because Jungfraujoch couples acquisition with real-time analysis used to *steer* experiments, +transparency and reproducibility of that analysis matter. As a minimum the writer therefore +preserves spot-finding and indexing results together with the filters that were applied, and it can +retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse. + +### Two complementary layouts: per-image spots vs. a reflection table + +Jungfraujoch stores analysis products in two shapes, matching how each is accessed. + +**Per-image spot finding / indexing.** Spot finding and indexing are inherently *image-centric* — +the natural query is "give me the spots for image *n*" — and this holds for serial stills and for +rotation frames alike. For these products Jungfraujoch adopts a layout similar to the +[Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the convention +understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties (position, +intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, +with each image allocated room for up to a predefined maximum number of spots. These dense arrays +are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved +without traversing variable-length structures. The cost is some storage overhead for unused slots +(padded with sentinels), which is acceptable for the access pattern. + +**Integrated reflections.** Integrated intensities are naturally a *dataset-wide* table, which is +exactly the model of the NeXus +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class. +This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration +results (see §4.2 below). We deliberately do *not* force spot finding/indexing into a single +experiment-wide table: across the hundreds of thousands of patterns typical of serial — or +fine-sliced rotation — experiments, that would require aggregating the whole experiment before the +spots of one image can be read. We encourage the community to develop standardised NeXus application +definitions for image-centric crystallography products that combine NeXus interoperability with the +access patterns and scale of modern high-throughput experiments. + +## 2. File layout + +A run is written as one **master file** plus, depending on the format, one or more **data files**: + +``` +_master.h5 # NXmx master file (metadata + links / virtual datasets) +_data_000001.h5 # data file: images + per-image analysis +_data_000002.h5 +... +``` + +The master file is produced by `writer/HDF5NXmx.cpp`; data files by `writer/HDF5DataFile.cpp` and +its plugins (`writer/HDF5DataFilePlugin*.cpp`). Files are written to a temporary `*..tmp` +name and renamed on successful close. + +Three master-file variants exist (set via `file_format`): + +| Format | Value | Master ↔ data linking | +|--------|:-----:|------------------------| +| **NXmxLegacy** | 1 | One external link in `/entry/data` per data file (`data_000001`, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0. | +| **NXmxVDS** (default) | 2 | A single virtual dataset `/entry/data/data` spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+. | +| **NXmxIntegrated** | 3 | No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format. | + +In legacy/VDS mode, image-indexed analysis arrays live in the **data files** and are exposed in the +master file through external links or virtual datasets; in integrated mode they are written +directly into the single file. + +Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or +bitshuffle + Zstd. Signed integer image datasets carry `INTx_MIN` as the HDF5 fill value (the +"masked / no-data" sentinel); unsigned ones are left at HDF5's own fill of 0, because every unsigned +code is a legitimate count. In the master's virtual dataset the fill is the error marker for both, +so a data file missing beside a VDS master reads as masked rather than as zero counts. + +> **Signed images and the Neggia XDS plugin.** Neggia dispatches on the size of the pixel in bytes +> and always casts to an unsigned type, consulting the signedness of the data only for the pixel +> mask. A **signed 16-bit** image is therefore read wrongly: a count of `-2` reaches XDS as `65534`, +> and the `-32768` error marker as `32768`. Signed 32-bit happens to degrade safely, because every +> negative value ends up above `INT32_MAX` and is mapped to `-1`. Use the Jungfraujoch XDS plugin, +> or the Global Phasing build of Durin, for signed data — see +> [Integration with MX data processing software](SOFTWARE_INTEGRATION.md). + +### Reprocessing output: `_process.h5` + +The offline reprocessing tool [`rugnux`](TOOLS.md) (`rugnux/rugnux_cli.cpp`) re-runs the +full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing +dataset and writes its results to a master file named **`_process.h5`**. This file uses the +**integrated** format, but instead of copying the images its `/entry/data/data` is a *virtual +dataset that links back to the original image files* (`hdf5_source_data` → +`NXmx::LinkToData_ProcessingVDS`). The result is a compact, self-describing companion file that +holds *all* the derived analysis (everything in §4) plus a virtual view +of the raw images — without duplicating terabytes of data. + +This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead +of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data` +VDS still resolves to the original images when they are available. `rugnux` can also process +an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. + +## 3. NXmx-standard content + +The entries below are part of, or valid base classes for, the +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html) application definition. +"NXmx" = listed in the application definition; "base" = a valid field of the relevant NeXus base +class (`NXdetector`, `NXsample`, `NXsource`) but not in the NXmx required/recommended subset. + +### `/entry` (NXentry) + +| Field | Std | Notes | +|-------|:---:|-------| +| `definition` | NXmx | value `"NXmx"` | +| `start_time` | NXmx | arming time | +| `end_time`, `end_time_estimated` | NXmx | approximate end time | + +File-level HDF5 attributes `file_name`, `file_time`, `HDF5_Version` are also set. + +### `/entry/source` (NXsource), `/entry/instrument` (NXinstrument) + +| Field | Std | Units | +|-------|:---:|-------| +| `source/name`, `source/type` | NXmx / base | | +| `source/current` | base | A | +| `instrument/name` | NXmx | | + +### `/entry/instrument/beam` (NXbeam) + +| Field | Std | Units | +|-------|:---:|-------| +| `incident_wavelength` | NXmx | angstrom | +| `incident_wavelength_spread` | NXmx | angstrom (only if polychromatic) | +| `total_flux` | NXmx | Hz | +| `incident_beam_size` | NXmx | m (two elements, x then y; written only when both beam sizes are given) | + +### `/entry/instrument/attenuator` (NXattenuator) + +| Field | Std | +|-------|:---:| +| `attenuator_transmission` | NXmx | + +### `/entry/instrument/detector` (NXdetector) + +| Field | Std | Units | +|-------|:---:|-------| +| `depends_on` | NXmx | → `transformations/rot3` | +| `beam_center_x`, `beam_center_y` | NXmx | pixel (0.0 = centre of the first pixel, see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)). The **PONI**, not the direct beam - see below | +| `distance` | NXmx | m | +| `count_time`, `frame_time` | NXmx | s | +| `sensor_thickness` | NXmx | m | +| `sensor_material` | NXmx | | +| `description` | NXmx | | +| `threshold_energy` | NXmx | eV (EIGER; written only for a single channel) | +| `x_pixel_size`, `y_pixel_size` | base | m | +| `serial_number` | base | | +| `bit_depth_readout` | NXmx | bit depth of the **stored image**, not of the detector electronics - see below | +| `saturation_value` | NXmx | highest valid value. Read inclusively by NXmx, by the DIALS `trusted_range` and by the XDS `OVERLOAD` parameter; a saturated pixel carries the value one above it | +| `underload_value` | NXmx | lowest valid value: `0` for an unsigned image, `INTx_MIN + 1` for a signed one | +| `flatfield_applied` | NXmx | | +| `pixel_mask`, `pixel_mask_applied` | NXmx | `pixel_mask` is `[y, x]`, hard-linked from `detectorSpecific/pixel_mask` | +| `countrate_correction_applied` | NXmx | | +| `countrate_correction_lookup_table` | NXmx | only when the detector sent one (DECTRIS) | +| `virtual_pixel_interpolation_applied` | NXmx | only when the detector reported it (DECTRIS) | +| `number_of_cycles` | base | frame-summation factor | + +#### Why `bit_depth_readout` is the image depth + +NXmx defines only `bit_depth_readout`, "how many bits the electronics record per pixel", and has no +field for the depth of the image actually stored. The two differ whenever summation is used: the +readout stays at the detector's native width while the summed image must be wider to hold the sum. + +Jungfraujoch writes the **stored image depth** into `bit_depth_readout` (and the identical value +into the non-standard `bit_depth_image`). The electronic value is a constant of the detector and +tells a data consumer nothing, whereas readers do use `bit_depth_readout` as the width of the +stored pixel — DIALS, for instance, derives its masking markers from it and cannot read a 32-bit +image without it. Writing the electronic value there would therefore mislead exactly in the case +where the two differ. + +Note that `bit_depth_readout` gives the width only. The **sign** is carried solely by the HDF5 +element type of `/entry/data/data` (and, on the wire, by `image_dtype`); there is no NXmx field for +it. + +### `/entry/instrument/detector/transformations` (NXtransformations) + +The NXtransformations *mechanism* (the `depends_on` chain, `transformation_type`, `vector`, +`offset` attributes) is standard. The axis **names** follow the PyFAI PONI convention chosen by +Jungfraujoch (see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)): + +| Axis | Type | Units | Vector | Depends on | +|------|------|-------|--------|-----------| +| `rot3` | rotation | rad | `(0, 0, -1)` | `.` | +| `rot2` | rotation | rad | `(1, 0, 0)` | `rot3` | +| `rot1` | rotation | rad | `(0, -1, 0)` | `rot2` | +| `translation` | translation | m | unit vector along the sample→PONI direction | `rot1` | + +`/entry/instrument/detector/depends_on` is `translation`, and the module's `fast_pixel_direction`, +`slow_pixel_direction` and `module_offset` depend on it in turn. A chain is applied innermost-first, +so reading it outwards the detector is placed at its distance and beam centre and *then* tilted about +the sample — which is what makes a tilt pivot about the crystal rather than about the panel corner. +The `vector` values are in NXmx's **McStas** frame, which is Jungfraujoch's internal frame with x and +y negated. + +The beam centre is encoded in `translation` (its offset from the sample), not only in the +informational `beam_center_x`/`beam_center_y` fields. In a `_process.h5` written by Rugnux these axes +carry the **refined** detector geometry — the refined beam centre folds into `translation` and the +refined tilt into `rot1`/`rot2`/`rot3`; the broker writes the user-provided geometry unchanged. + +#### `beam_center_x`/`beam_center_y` is the PONI; `direct_beam_x`/`direct_beam_y` is the beam + +`beam_center_x`/`beam_center_y` is the **PONI** - the foot of the perpendicular dropped from the +sample onto the detector plane. On a tilted detector that is *not* where the undeflected beam lands: +the two points are `distance * tan(tilt) / pixel_size` apart, which is around 8 px on a real in-house +setup and grows with the distance. + +Most programs that ask for "the beam centre" mean the point the beam lands on - XDS's `ORGX`/`ORGY` +among them - so Jungfraujoch writes that point out as well: + +| Dataset | Units | +|---------|-------| +| `/entry/instrument/detector/detectorSpecific/direct_beam_x` | pixel | +| `/entry/instrument/detector/detectorSpecific/direct_beam_y` | pixel | + +They are computed from the same beam centre, distance and `rot1`/`rot2`/`rot3` written beside them, +so the file cannot disagree with itself; on an untilted detector they equal `beam_center_x`/`_y`. + +The *provenance* differs by which program wrote the file, though the field does not. In a master +written by `jfjoch_broker` the geometry is the one the user stated, so `direct_beam_x`/`_y` is a +statement about the user's geometry; in a `_process.h5` written by Rugnux it is the refined geometry, +so it is a measurement. A user handing either to XDS should know which of the two they have. + +### `/entry/instrument/detector/module` (NXdetector_module) + +`data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all +NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). The two +pixel-direction vectors carry the discrete image orientation (mirror in Y, multiples of 90° about the +beam); for a detector that this system assembled itself, they are the McStas form of the internal +x and ++y, i.e. `(-1, 0, 0)` and `(0, -1, 0)`. + +### `/entry/sample` (NXsample) + +| Field | Std | Units / notes | +|-------|:---:|-------| +| `name` | NXmx | | +| `depends_on` | NXmx | points at the innermost axis of the sample chain, or `.` for stills | +| `temperature` | NXmx | K | +| `transformations/` (NXtransformations) | NXmx | the sample chain, written in mounting order; hard-linked as `/entry/sample/goniometer` | +| `unit_cell` | base | `[a, b, c, α, β, γ]` | +| `space_group_number` | base | International Tables number | +| `space_group` | base | extended Hermann-Mauguin name, e.g. `P 43 21 2`, `R 3:H` — this is the field that carries the **setting**, and the one the reader takes the group from; the number alone always reads back as the reference setting | +| `ub_matrix` | base | `[1, 3, 3]`, Angstrom⁻¹ | + +The chain is written from the base outwards, so the innermost axis — the one `depends_on` names — is +the one nearest the sample. It may hold, in that order: the grid-scan translations `grid_scan_x` and +`grid_scan_y`, the spindle, and a Smargon head's `chi` and `phi`. A grid scan and a goniometer axis +are **not** alternatives; both can be present. + +A **grid scan is collected at a stationary spindle, and the angle it stood at is stated by sending +the goniometer axis with a step of 0** — in `dataset_settings.goniometer`, or as the `goniometer` +map of the CBOR start message. The angle is then written per image as `omega` (or whatever the axis +is named) in the chain above, and read back by `reader/`. Send no axis and the spindle is still +recorded, at 0: NXmx would allow a sample with no goniometer at all (`depends_on = "."`), but a +sample chain of translations alone is not something readers accept, so the placeholder is written +whether or not the angle is known. **A 0 there means "nobody said", not "the spindle was at 0".** + +A **Smargon head position is told apart from the spindle** by the `equipment_component` attribute, +which is `"smargon"` on `chi` and `phi` and absent on the spindle. This is load-bearing: a spindle can +itself be named `phi`, and without the attribute a reader would take a head position for the scan +axis. `chi` and `phi` are written with one value per image even though neither turns, because a +reader takes the image count from the innermost axis of the chain: written as scalars, a still +recorded at a head position would read back as a single image however many were collected. + +For a rotation scan the goniometer axis carries, beyond the per-image angle array ``, the +Jungfraujoch conveniences `_end`, scalar `_range_average` and `_range_total`, and +for helical scans `_helical_x/_y/_z`. + +### `/entry/data` (NXdata) + +`data` (3-D image stack, `[n_images, y, x]`) with `image_nr_low` / `image_nr_high` attributes. +In legacy mode this group instead contains one external link `data_000001`, … per data file. + +## 4. Extensions beyond NXmx + +Everything in this section is **outside the NXmx standard**. Each group is declared with +`NX_class = NXcollection` (the NeXus-sanctioned container for non-standardised content) unless noted. +The per-image arrays are indexed by image number, padded to the run length and filled with a +sentinel (`NaN` for floats, `-1`/`0` for integer indices) where a quantity is absent. + +### 4.1 `/entry/MX` — spot finding and indexing (CXI-style) + +The flagship extension. Spot ("peak") properties are stored as fixed-size `[n_images, max_spots]` +arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as `[n_images]` vectors. +In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master. + +**Per-spot arrays `[n_images, max_spots]`:** + +| Dataset | Units | Meaning | Indexing only | +|---------|-------|---------|:---:| +| `peakXPosRaw`, `peakYPosRaw` | pixel | spot position (raw detector frame) | | +| `peakTotalIntensity` | photons | spot intensity | | +| `peakIceRingRes` | | spot lies in an ice-ring resolution band | | +| `peakH`, `peakK`, `peakL` | | Miller indices of the (indexed) spot | ✓ | +| `peakDistEwaldSphere` | Å⁻¹ | distance of the spot from the Ewald sphere | ✓ | +| `peakIndexed` | | spot fits the indexing solution | ✓ | +| `peakLattice` | | lattice the spot belongs to (`-1` = unindexed) | ✓ | + +**Per-image vectors `[n_images]`:** + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `nPeaks` | | number of spots stored for the image (CXI) | +| `strongPixels` | | strong-pixel count (first spot-finding stage) | +| `peakCountUnfiltered` | | spots found before filtering | +| `peakCountLowRes` | | low-resolution spots | +| `peakCountIceRingRes` | | spots inside ice-ring bands | +| `peakCountIceRingControl` | | spots in the ice-free flanks beside those bands, rescaled to their q width - the control for the count above (their ratio, pooled over the run, is the spot-based ice indicator) | +| `peakCountIndexed` | | spots fitting the indexing solution | +| `imageIndexed` | | image was indexed (0/1) | +| `indexingLatticeCount` | | number of lattices found for the image | +| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A* (2016), Vol. A, [Table 3.1.3.1](https://onlinelibrary.wiley.com/iucr/itc/Ac/ch3o1v0001/table3o1o3o1.pdf)) | +| `bravaisLattice` | | Bravais lattice short code, e.g. `aP`, `mC`, `oF`, `tI`, `hP`, `hR`, `cF` | +| `profileRadius` | Å⁻¹ | crystal profile radius | +| `mosaicity` | deg | mosaicity estimate | +| `bFactor` | Ų | per-image B-factor estimate | +| `resolutionEstimate` | Å | resolution the merged data are predicted to reach, from this image's spots alone | +| `integratedReflections` | | number of integrated reflections | +| `bkgEstimate` | photons | mean background in the 3–5 Å resolution band | +| `iceRingScore` | ratio | strongest hexagonal-ice ring intensity over the smooth radial background (1 = no ice) | +| `spindleBlindFraction` | fraction (0-1) | how much of a rotation sweep's blind cone this orientation makes unrecoverable, as a lone-2-fold worst-case bound; NaN = the frame could not be assessed, which automation must treat like a value at or above the 0.5 trigger, never as 0 | +| `beam_corr_x`, `beam_corr_y` | pixel | beam-center correction applied during processing | +| `imageScaleFactor` | | on-the-fly per-image scale factor *g* | +| `imageScaleCC` | | on-the-fly scaling correlation coefficient | +| `imageScaleMosaicity` | deg | scaling-model mosaicity | +| `sweepQuality` | | why this image's stretch of the sweep was flagged — see below | +| `frameDisposition` | | what became of this image's observations in the merged data — see below | + +**Per-image lattices:** `latticeIndexed` `[n_images, 9]` (Å) — the real-space lattice (flattened +3×3); `latticeIndexedExtra` `[n_images, max_extra_lattices, 9]` (Å) — additional orientation +variants. + +**Run-level summaries** (written into the master `/entry/MX` at finalisation): + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `indexing_algorithm` | | `FFBIDX` / `FFT (CUDA)` / `FFT (FFTW)` | +| `geom_refinement_algorithm` | | e.g. `beam_center` | +| `rotationLatticeIndexed` | Å | whole-run rotation-indexing lattice (`[9]`) | +| `rotationLatticeIndexedExtra` | Å | additional whole-run lattices (`[m, 9]`) | +| `rotationLatticeNiggliClass` | | Niggli class of the run lattice | +| `imageIndexedMean` | | mean indexing rate over the run | +| `bkgEstimateMean` | photons | mean background over the run | +| `spindleBlindFractionMean` | fraction (0-1) | mean `spindleBlindFraction` over the frames that had one | +| `spindleLostUniqueFraction` | fraction (0-1) | unique reflections (to the run's resolution limit) the mounting made unmeasurable, exact under the measured point group and indexed orientation; offline (Rugnux) only | +| `iceRingScoreMean` | ratio | mean `iceRingScore` over the run — the single "how icy was this dataset" number (1 = no ice) | +| `indexedLatticeCount` | | per-image lattice count summary (master). *Note: data files use `indexingLatticeCount`; readers accept either.* | +| `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below | + +**Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is +processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by +the merge, and settling it can re-seat the lattice into the group's conventional setting — so +`/entry/sample/unit_cell`, `/entry/sample/space_group_number` and `rotationLatticeIndexed` can be in a +different setting from the per-image data beside them. `reindexMatrix` **M** is the integral change of +basis between the two: `hkl_cell = M · hkl_written`, and the same **M** takes each per-image lattice +across (`latticeIndexedExtra` is not re-seated and stays as indexed). It is **absent** when the two +settings are the same one, which is the identity — as it is on every file written before Rugnux +recorded it. The Jungfraujoch reader applies it, so everything it hands out is already in the cell's +setting; a third-party reader that ignores it will index the reflections in the wrong frame whenever +the dataset is present. Written by the offline `rugnux` path only — the broker never re-seats a +lattice — and not carried on the CBOR stream, in the same way as the other offline-only fields. + +A `--model` run can leave the merged reflection files (`.mtz`/`.cif`/`.hkl`) in a *different* frame +from the `_process.h5` beside them: the model settles the alternative indexing where nothing else +did, and that choice is applied to the merged reflections as they are written. It also names the +enantiomorph, but that is a change of space-group label only and moves no reflection. +The process file is not rewritten — its per-image reflections went to disk as they were integrated — +so it keeps the space group the run itself determined and stays self-consistent with its own data. +The two frames describe the same measurements; an enantiomorphic pair merges identically, having the +same Laue class and the same absences. + +**Sweep quality.** `sweepQuality` `[n_images]` (`uint8`) says why the stretch of the sweep this +image belongs to was flagged as delivering much less than the rest of the run: **0** means it was +not, and any other value is a **1-based index into `sweepQualityReasons`**, a string vector written +beside it that carries the whole vocabulary, so the codes can be read without this source. The +vocabulary is closed and stable — a code is never renamed and never reused — and currently reads +`no_diffraction`, `crystal_out_of_beam`, `weak_diffraction`, `loss_of_centring`, `radiation_damage`, +`inconsistent_with_merge`; +[the Rugnux results report](RUGNUX_REPORT.md#sweep-quality-the-disposition-and-their-vocabularies) +defines what each one means. Both datasets are **absent** unless the sweep-quality diagnostic ran, +which needs scaling and merging; their absence therefore means "not looked for", *not* "every image +clean". Written by the offline `rugnux` path only — the broker does not merge — and not carried on the +CBOR stream, in the same way as the other offline-only fields (`space_group_number`, the refined +geometry). The condensed, dataset-wide form of the same +finding is in `_report.txt`. + +**Frame disposition.** `frameDisposition` `[n_images]` (`uint8`) says what became of the image's +observations: a **0-based index into `frameDispositionCodes`**, written beside it, which reads +`merged`, `downgraded`, `rejected`. Where `sweepQuality` says what was *seen* over a stretch, this +says what was *done* about it — a `rejected` image contributed nothing to the merged intensities. +Same availability rule as `sweepQuality`: absent means the diagnostic never ran. + +CrystFEL can read the spots directly with: + +``` +peak_list = /entry/MX +peak_list_type = cxi +``` + +### 4.2 `/entry/reflections` — integrated reflections (NXreflections) + +Integrated reflections are stored **per image** as +`/entry/reflections/image_NNNNNN` groups, each declared `NX_class = NXreflections`. The columns map +mostly onto the standard +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class: + +| Dataset | Units | NXreflections | Meaning | +|---------|-------|:-------------:|---------| +| `h`, `k`, `l` | | standard | Miller indices | +| `d` | Å | standard | resolution | +| `int_sum` | photons | standard | integrated intensity (summation) | +| `int_err` | photons | non-standard name | σ of the intensity (standard equivalent: `int_sum_errors`) | +| `background_mean` | photons | standard | mean background under the peak | +| `background_variance` | photons² | non-standard | non-signal part of σ², carried to the merge. Absent in files written before it existed; the reader then recovers it from σ² − I | +| `predicted_x`, `predicted_y` | pixel | name standard, units differ | predicted position. NXreflections `predicted_x/_y` are *physical* lengths; the pixel datasets are `predicted_px_x/_y` | +| `observed_x`, `observed_y` | pixel | name standard, units differ | observed centroid (pixels; standard pixel form is `observed_px_x/_y`) | +| `observed_frame` | | standard | image number of the reflection | +| `lp` | | standard | the Lorentz-polarization factor, stored as the reciprocal of the multiplier that is applied. Lorentz x polarization only, which is what the NXreflections name means; the sensor efficiency is `qe`, beside it | +| `qe` | | **extension** | the sensor efficiency, in the same reciprocal convention as `lp`, so the whole correction applied to a raw count is `1/lp * 1/qe`. Absent in files written before it existed; the reader then takes the whole of `lp` for Lorentz–polarization, which is what those files mean | +| `flight` | | **extension** | the flight path between sample and pixel, in the same reciprocal convention as `lp` and `qe`, so the whole correction applied to a raw count is `1/lp * 1/qe * 1/flight`. It runs the opposite way to `qe` — at most 1 where `qe` is at least 1 — because the medium attenuates an oblique reflection where the sensor favours it. Exactly 1 under `--flight-path vacuum`, and absent in files written before it existed; the reader then takes it as 1 | +| `partiality` | | standard | recorded fraction of the reflection | +| `delta_phi` | deg | **extension** | XDS Δφ: offset from the centre of the current frame | +| `zeta` | | **extension** | Lorentz ζ factor (reciprocal-space geometry term) | +| `image_scale_corr` | | **extension** | per-image scale correction; `I_true = image_scale_corr · int_sum` | + +In the master file these per-image groups are exposed through `/entry/reflections` external links +(VDS/integrated formats). + +### 4.3 `/entry/azint` — azimuthal integration + +| Dataset | Shape | Units | Meaning | +|---------|-------|-------|---------| +| `bin_to_q` | `[φ_bins, q_bins]` | Å⁻¹ | q value of each bin | +| `bin_to_two_theta` | `[φ_bins, q_bins]` | deg | 2θ of each bin | +| `bin_to_phi` | `[φ_bins, q_bins]` | deg | azimuthal angle of each bin | +| `image` | `[n_images, φ_bins, q_bins]` | | per-image integrated profile (NaN for empty bins) | +| `image_std` | `[n_images, φ_bins, q_bins]` | | per-bin standard deviation | +| `image_count` | `[n_images, φ_bins, q_bins]` | | pixels contributing per bin | +| `map` | `[y, x]` | | pixel→bin mapping (master file only) | + +### 4.4 `/entry/roi` — regions of interest (per-image results) + +`/entry/roi/` has one sub-group per configured ROI, holding the **per-image result +vectors** `[n_images]`. These are written into the data files; in VDS mode they are exposed from +the master file through virtual datasets, and in integrated mode they are in the single file. +(In legacy mode they remain only in the data files.) + +| Dataset | Meaning | +|---------|---------| +| `max` | maximum pixel value in the ROI | +| `sum` | sum of pixel values | +| `sum_sq` | sum of squared pixel values | +| `npixel` | number of valid pixels | +| `x`, `y` | intensity-weighted centroid | + +#### 4.4.1 `/entry/roi_defs` — ROI definitions (master file) + +The **dataset-wide ROI definitions** (geometry, fixed for the whole acquisition) live in the +master file under a *separate* `/entry/roi_defs` group — kept apart from `/entry/roi` above so +that older readers, which iterate `/entry/roi`, are unaffected by these entries. One sub-group +`/entry/roi_defs/` per ROI: + +| Dataset | Meaning | +|---------|---------| +| `bit_index` | which bit of `roi_map` (below) marks this ROI | +| `type` | `box`, `circle` or `azim` | +| `min_x_pxl`, `max_x_pxl`, `min_y_pxl`, `max_y_pxl` | box bounds (type `box`) | +| `center_x_pxl`, `center_y_pxl`, `radius_pxl` | circle (type `circle`) | +| `q_min_recipA`, `q_max_recipA` | Q range (type `azim`) | +| `phi_min_deg`, `phi_max_deg` | azimuthal-angle sector (type `azim`, omitted for a full ring) | + +`/entry/roi_defs/roi_map` `[y, x]` is a `uint16` per-pixel bitmask: bit `bit_index` is set for +every pixel belonging to that ROI, so an ROI's footprint can be recovered exactly. + +### 4.5 `/entry/image` — per-image pixel statistics + +`[n_images]` vectors: `max_value`, `min_value` (viable min/max, excluding error/saturated pixels), +`error_pixels`, `saturated_pixels`, `pixel_sum`. Surfaced in the master file under `/entry/image`. + +### 4.6 `/entry/profiling` — per-image timing + +`[n_images]` vectors in seconds: `spotFindingTime`, `indexingTime`, `integrationTime`, +`refinementTime`, `processingTime`, `braggPredictionTime`, `preprocessingTime`, `compressionTime`, +`azIntTime`, `indexAnalysisTime`, `imageScaleTime`. + +### 4.7 `/entry/detector` — acquisition diagnostics (data file) + +A convenience NXcollection in the data file (note: distinct from the standard +`/entry/instrument/detector`). In **integrated** format these datasets are written under +`/entry/instrument/detector/detectorSpecific` instead. + +| Dataset | Meaning | +|---------|---------| +| `timestamp`, `exptime` | per-image timestamp and exposure time | +| `number` | image number (original number if image rejection was used) | +| `det_info` | JUNGFRAU debug field | +| `storage_cell_image` | storage-cell number | +| `rcv_delay`, `rcv_free_send_buffers` | receiver internal diagnostics | +| `packets_expected`, `packets_received` | UDP packets per image | +| `data_collection_efficiency_image` | received / expected packet ratio | + +### 4.8 `/entry/xfel` — pulsed-source metadata + +`[n_images]` vectors `pulseID` and `eventCode`, written for pulsed sources (e.g. SwissFEL). + +### 4.9 Other collections + +| Path | Class | Content | +|------|-------|---------| +| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode`, `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/`, `data_collection_efficiency_image`, `direct_beam_x`, `direct_beam_y` | +| `/entry/instrument/detector/calibration` | NXcollection | per-channel pedestal / calibration images (bitshuffle-compressed) | +| `/entry/instrument/fluorescence` | NXcollection | XRF spectrum: `energy` [eV], `data` | +| `/entry/user` | NXcollection | scalar values supplied under `header_appendix.hdf5` | + +### 4.10 Non-standard fields inside the NXmx detector group + +A few extension scalars are written *inside* the otherwise-standard `/entry/instrument/detector` +group for compatibility with existing tooling: + +| Field | Units | Meaning | +|-------|-------|---------| +| `detector_distance` | m | duplicate of `distance` (Dectris/Neggia compatibility) | +| `detector_number` | | detector identifier (Dectris convention) | +| `mirror_y` (in `detectorSpecific`) | | whether the stored image is mirrored in Y relative to the raw readout; true is the MX convention (row 0 at the top) | +| `detector_orientation_mirror_y` (in `detectorSpecific`) | | whether the stored image is mirrored in Y relative to the frame `rot1`/`rot2`/`rot3` are stated in — a different setting from `mirror_y`, and one that changes no pixel | +| `detector_orientation_quarter_turns` (in `detectorSpecific`) | | multiples of 90° about the beam the stored image is turned by, relative to that same frame (0-3) | +| `direct_beam_x`, `direct_beam_y` (in `detectorSpecific`) | pixel | where the undeflected beam lands, as opposed to the PONI in `beam_center_x`/`beam_center_y` - see above | +| `error_value` | | masked/error pixel sentinel: `UINTx_MAX` unsigned, `INTx_MIN` signed (NXmx has no equivalent). NXmx `underload_value` is written too: `INTx_MIN + 1` for signed, `0` for unsigned | +| `bit_depth_image` | | stored image bit depth (DECTRIS convention, not NXmx). Equal to `bit_depth_readout` where that is written, i.e. for unsigned images | +| `acquisition_type` | | always `triggered` (Dectris convention) | +| `jungfrau_conversion_applied` | | JUNGFRAU photon/keV conversion applied | +| `jungfrau_conversion_factor` | eV | conversion factor | +| `geometry_transformation_applied` | | module→full-detector geometry applied | + +NeXus has no concept of a fill or no-data value — it expects bad pixels to be flagged in `pixel_mask`, +which Jungfraujoch also writes. The in-band sentinel above is a DECTRIS compatibility convention: +SIMPLON specifies that masked pixels are flagged with `2^bit_depth_image - 1`. + +For an **unsigned** image the sentinel and the saturation code are the same value, so a saturated +pixel and a masked one cannot be told apart — the FPGA collapses both onto `UINTx_MAX`. Signed +images keep them separate: `INTx_MIN` is the marker, `INTx_MAX` is saturation. + +`bit_depth_readout` is written for **unsigned** images only. DIALS remaps the top two codes of +`2^bit_depth_readout` to `-1` and `-2` whenever the field is present, regardless of the pixel type: +for an unsigned image those land below `underload_value` and are correctly masked, but for a signed +one they land inside the trusted range and a saturated pixel would be integrated as a count of `-2`. +Signed images are read correctly without the field; unsigned 32-bit cannot be read at all without it. + +### 4.11 User-supplied metadata: `header_appendix` and `image_appendix` + +Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two +free-form JSON fields in the `/start` request (`broker/jfjoch_api.yaml`) provide this without any +schema change; both accept *any valid JSON*: + +| Field | Carried in | Persisted to HDF5? | +|-------|-----------|--------------------| +| `header_appendix` | the **start** message, under `user_data.user` (see [CBOR](CBOR.md)) | no — except the `hdf5` sub-object (below) | +| `image_appendix` | **every image** message, as `user_data` | no | + +Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, +republished analysis, viewers), so they are the recommended channel for facility- or +beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has +no dedicated API field. + +**Persisting selected values to HDF5.** `header_appendix` is normally *not* written to the master +file. As an exception, if it contains a key `hdf5` whose value is a JSON object of scalars (strings +and numbers — no arrays or nested objects), the writer stores each entry under `/entry/user/`. + +For example, a `/start` request containing: + +```json +{ + "header_appendix": { + "proposal": "p20001", + "operator": "jdoe", + "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 } + }, + "image_appendix": { "trigger_source": "external" } +} +``` + +forwards the whole `header_appendix` as `user_data.user` on the start message and +`{"trigger_source": "external"}` as `user_data` on every image message, and writes three scalars +into the master file: + +``` +/entry/user/beamline = "X06SA" +/entry/user/ring_mode = "top-up" +/entry/user/attenuator_foils = 2 +``` + +## 5. Notes + +* **Units** are written as the HDF5 `units` attribute on the dataset (e.g. `m`, `eV`, `deg`, + `Angstrom`, `Angstrom^-1`, `Angstrom^2`, `pixel`, `s`). +* **Sentinels.** Missing per-image values are `NaN` (floats) or `-1`/`0` (integer indices); image + pixels use `INTx_MIN` / `UINTx_MAX`. +* **Master vs data file.** In legacy/VDS formats the analysis arrays physically live in the data + files; the master file links to them (external links in legacy, virtual datasets in VDS). In the + integrated format there are no data files and everything is in one place. +* **CXI / CrystFEL.** `/entry/MX` follows the CXI peak-list convention; see + [CXI file format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf). diff --git a/_sources/IMAGE_STREAM.md.txt b/_sources/IMAGE_STREAM.md.txt new file mode 100644 index 000000000..c9c9e53c8 --- /dev/null +++ b/_sources/IMAGE_STREAM.md.txt @@ -0,0 +1,247 @@ + +# Data streams + +The Jungfraujoch process (`jfjoch_broker`) operates three outputs. +All three can be operated/enabled independently. +These are: +* **Image** - all the images including metadata (ZeroMQ PUSH socket or custom TCP/IP socket) +* **Preview** - images with metadata at a reduced frame rate (PUB socket) +* **Metadata** - only metadata for all the images, bundled into packages (PUB socket) + +## Image stream +Images (with metadata) are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message), [calibration messages](CBOR.md#calibration-message) and [end message](CBOR.md#end-message) with run metadata. + +If `file_prefix` is not provided for a data collection, images won't be sent to image stream (or its HDF5/CBOR replacements). + +### Splitting image stream +Image stream can be split into multiple sockets to increase performance, in this case images will be split according to file number to which the image belongs. +All sockets will forward start and end messages. Only the first socket will forward calibration messages and will be marked to write master file. + +### ZeroMQ image stream +This is using PUSH ZeroMQ socket(s). +Multiple receivers must never be connected to one PUSH ZeroMQ socket. +ZeroMQ will send the images in a round-robin basis to the receivers. +In this case start and end messages will end up only with one receiver. +Instead, Jungfraujoch feature of multiple sockets should be used. +For ZeroMQ image stream, each writer connects to a different port. + +Behavior is as following: +* Start message is sent with timeout of 1s per socket. If within the time the message cannot be put in the outgoing queue or there is no connected puller, an exception is thrown — data collection is stopped with an error due to absence of a writer. +* Calibration message is sent to the first socket only, with timeout of 1s. +* Images are sent via a per-socket writer thread. If a send times out, the pusher switches to non-blocking mode for the remainder of the collection (images may be dropped). +* End message is sent with timeout of 1s per socket. No exception is thrown on timeout, but a transmission error is recorded. + +The format is generally interchangeable with DECTRIS Stream2 format. + +#### ZeroMQ configuration + +ZeroMQ image stream is configured in the broker JSON configuration file under the `zeromq` section (schema `zeromq_settings`): +```json +{ + "image_socket": ["tcp://192.168.0.1:9000", "tcp://192.168.0.1:9001"], + "send_watermark": 100, + "send_buffer_size": 67108864, + "writer_notification_socket": "tcp://192.168.0.1:*" +} +``` + +- `image_socket`: one or more PUSH socket addresses. Multiple entries split the image stream across sockets. Addresses follow ZeroMQ conventions (`tcp://`, `ipc://`). `0.0.0.0` binds on all network interfaces. +- `send_watermark` (optional): ZeroMQ send high-water mark (number of outstanding messages per socket). +- `send_buffer_size` (optional): OS-level send buffer size for the ZeroMQ socket. +- `writer_notification_socket` (optional): see [Writer notification socket](#writer-notification-socket) below. + +### TCP/IP image stream +This is using TCP/IP socket(s) with a fixed binary frame header followed by payload bytes. +This format was introduced to Jungfraujoch as an alternative to ZeroMQ image stream. It allows two-way communication +between the data collection and the writer, and is therefore more robust than ZeroMQ. + +For TCP/IP image stream, Jungfraujoch **listens** on a single TCP port and all writers **connect** to it. Connections are persistent — writers connect once and stay connected across multiple data collections. Jungfraujoch sends periodic `KEEPALIVE` frames when no data collection is active to detect dead connections; writers are expected to respond with a `KEEPALIVE` pong. + +Using `*` as port number (e.g. `tcp://127.0.0.1:*`) is supported — the OS assigns a free port. + +Payloads for `PREFLIGHT`, `START`, `DATA`, `CALIBRATION` and `END` frames are CBOR messages, equivalent in content to the ZeroMQ image stream messages. +`ACK`, `CANCEL`, `KEEPALIVE` and `BUSY` are control frames (no CBOR payload). + +The data collection lifecycle on each connection follows: +`PREFLIGHT` → `START` → `CALIBRATION` (socket 0 only) → `DATA` (repeated) → `END` + +If a `START` ACK fails on any connection, Jungfraujoch sends `CANCEL` to all already-started connections and rolls back. + +For each frame: +1. Read one `TcpFrameHeader` (fixed size, 64-byte aligned). +2. Validate `magic` (`0x4A464A54` / `"JFJT"`) and `version` (`4`). Both ends reject a frame of any other version, so writer and broker must be of the same release. +3. Read `payload_size` bytes (if non-zero). + +#### Pre-flight + +Before a data collection is started - before the detector is armed - Jungfraujoch sends a `PREFLIGHT` frame on every connection and waits for its ACK. It carries the same CBOR start message a `START` would, and asks the writer one question: could this run be written? The writer checks the output path, creates the output directory, and checks that no output file is in the way; it opens nothing, writes nothing, and does not change its state. A run that would fail on the first file it wrote is therefore refused while refusing it is free, with the writer's own message reported to the client by `/start` and `/wait_until_running`. + +`write_master_file` is assigned exactly as it is for `START` (connection index 0), and the writer that owns the master file is the one that answers for the output files - the master and every data file the run will write, its siblings' included. + +The `PREFLIGHT` payload describes the run but carries none of the per-pixel arrays a `START` does - no pixel mask, no azimuthal-integration map, no ROI map - so it stays small: about 1.4 kB on a JUNGFRAU 9M, against about 540 kB for the `START` that follows. + +A rejected `PREFLIGHT` is **not** fatal: nothing was started, so the connection stays usable and the next attempt (a different file prefix, or `overwrite` set) proceeds on it. The check cannot be exhaustive - a file created in the moment between the pre-flight and the start still fails at the start, and free space and quota are not inspected - so it lowers how often a run fails on its output, it does not remove the case. + +When image stream is split into multiple connections: +- `START` and `END` are sent on all connections, +- `CALIBRATION` is sent only on connection 0, +- `DATA` frames are distributed by file grouping: connection index = `(image_number / images_per_file) % num_connections`. + +#### TCP/IP configuration + +TCP/IP image stream is configured in the broker JSON configuration file under the `tcp` section (schema `tcp_settings`): +```json +{ + "image_socket": "tcp://192.168.0.1:9100", + "nwriters": 2, + "send_buffer_size": 67108864 +} +``` + +- `image_socket`: listen address in `tcp://:` format. `0.0.0.0` binds on all interfaces. `*` as port selects a random free port. +- `nwriters` (optional): maximum number of simultaneous writer connections accepted. +- `send_buffer_size` (optional): OS-level `SO_SNDBUF` size for accepted connections. + +#### ACK handling + +ACK handling is mandatory for correct operation: +- `PREFLIGHT` **must** be acknowledged (`ack_for=PREFLIGHT`) on each connection within 5 seconds, otherwise the collection is not started. A rejected pre-flight (`OK` clear) carries the reason as error text and does not break the connection. +- `START` **must** be acknowledged (`ACK` with `ack_for=START`) on each connection within 5 seconds, otherwise collection start fails and a rollback is triggered. +- `END` **must** be acknowledged (`ack_for=END`) on each connection within 10 seconds for successful completion. +- `CANCEL` should be acknowledged during rollback paths (500ms timeout). +- `DATA` should be acknowledged for every frame. A `DATA` ACK with `FATAL` flag set reports a downstream error (e.g. disk full) which is propagated to `jfjoch_broker` via `Finalize()`. A failed `DATA` ACK does **not** break the TCP connection on its own — data continues to flow. +- `CALIBRATION` is not acknowledged at this time. +- `KEEPALIVE` frames are not acknowledged via ACK; the writer responds with a `KEEPALIVE` pong frame instead. + +#### Keepalive + +When no data collection is active, Jungfraujoch sends `KEEPALIVE` frames approximately every 5 seconds on each persistent connection. Writers should respond with a `KEEPALIVE` frame (pong). OS-level TCP keepalive is also enabled (`TCP_KEEPIDLE=30s`, `TCP_KEEPINTVL=10s`, `TCP_KEEPCNT=3`) as a secondary safety net. Dead connections are automatically removed from the pool. + +#### Zero-copy transmission + +On Linux, large payload transmission (`DATA` and `CALIBRATION` frames) can use kernel TCP zero-copy (`SO_ZEROCOPY`/`MSG_ZEROCOPY`) when available. If the kernel does not support it or the socket option fails, transmission transparently falls back to normal `send()` behavior. Zero-copy completion notifications are processed by a dedicated per-connection thread. + +#### Frame types + +| Value | Name | Purpose | +|---:|---|---| +| 1 | `START` | Start-of-run metadata | +| 2 | `DATA` | One image payload | +| 3 | `CALIBRATION` | Calibration payload | +| 4 | `END` | End-of-run metadata | +| 5 | `ACK` | Acknowledgement / error reporting | +| 6 | `CANCEL` | Cancel run initialization/stream | +| 7 | `KEEPALIVE` | Connection liveness probe/pong | +| 8 | `BUSY` | Writer alive but stalled; carries its FIFO occupancy | +| 9 | `PREFLIGHT` | Dry run before a collection starts: can this run be written? | + +#### TCP frame header (`TcpFrameHeader`) + +| Field | Type | Description | +|--------------------------|---|----------------------------------------------------------| +| `magic` | `uint32_t` | Protocol magic (`0x4A464A54`, `"JFJT"`) | +| `version` | `uint16_t` | Protocol version (`4`) | +| `type` | `uint16_t` | Frame type (see table above) | +| `image_number` | `uint64_t` | Image index for `DATA` frames | +| `payload_size` | `uint64_t` | Number of payload bytes after header | +| `socket_number` | `uint32_t` | Connection index in split-stream mode | +| `flags` | `uint32_t` | ACK flags (`OK`, `FATAL`, `HAS_ERROR_TEXT`) | +| `run_number` | `uint64_t` | Run identifier | +| `ack_processed_images` | `uint32_t` | In `ACK`: number of images processed by receiver | +| `ack_code` | `uint16_t` | In `ACK`: error/status code | +| `ack_for` | `uint16_t` | In `ACK`: frame type being acknowledged | +| `ack_fifo_occupancy` | `uint16_t` | In `ACK`: occupancy of input FIFO in the `jfjoch_writer` | +| `ack_fifo_max_occupancy` | `uint64_t` | In `ACK`: max occupancy of input FIFO | + +The header is 64-byte aligned (`alignas(64)`). + +#### ACK semantics + +- `ACK` frames use `ack_for` to indicate which frame type is acknowledged. +- `flags`: + - `OK` (bit 0): operation accepted/successful, + - `FATAL` (bit 1): receiver reports unrecoverable error (primarily for `DATA`), + - `HAS_ERROR_TEXT` (bit 2): ACK payload contains UTF-8 error text. +- `ack_code` can be used to categorize errors: + +| Code | Name | Meaning | +|---:|---|---| +| 0 | `None` | No error | +| 1 | `StartFailed` | START processing failed | +| 2 | `DataWriteFailed` | Image write failed | +| 3 | `EndFailed` | END processing failed | +| 4 | `DiskQuotaExceeded` | Disk quota exceeded | +| 5 | `NoSpaceLeft` | No space left on device | +| 6 | `PermissionDenied` | Permission denied | +| 7 | `IoError` | General I/O error | +| 8 | `ProtocolError` | Protocol-level error | + +### Image stream replacement +Image stream can be replaced with direct HDF5 writer and CBOR dump image pushers, or it can be disabled by selecting "None" image pusher for all the measurements. + +## Writer notification socket +The writer notification socket is used **only with ZeroMQ image stream**. Since ZeroMQ is asynchronous, `jfjoch_broker` does not know whether messages were properly handled downstream (e.g. written to disk). The writer notification socket allows downstream code to report back. + +For TCP/IP image stream, this mechanism is not needed — ACK frames provide synchronous feedback for each control and data frame. + +To use writer notification socket, it has to be first enabled in the JSON configuration file of broker with `writer_notification_socket` entry: +```json +{ + "writer_notification_socket":"tcp://192.168.0.1:*" +} +``` +Such entry will create PULL socket on `192.168.0.1` network interface listening on one, random TCP port. When data processing is started, the +image stream will send CBOR [start message](CBOR.md#start-message). This message will include information on `writer_notification_zmq_addr`, +which needs to be used by downstream code. Since the start message must reference the address of `jfjoch_broker` host, notification +socket should always listen on a particular network interface, and should not be configured with placeholder address `0.0.0.0`. It is, however, OK +to use placeholder `:*` for network port, as it will be substituted for the one chosen by ZeroMQ. + +For every image stream socket, downstream code must send the following message to the PULL socket: +```json +{ + "run_number":135, + "run_name": "sample_1", + "socket_number": 1, + "processed_images":250, + "ok": true +} +``` +Here `run_number`, `run_name` and `socket_number` must match information from the start message. +`ok` is boolean confirming if the writing process was OK. +`processed_images` is number of images that were written/processed, this is to track how many images were ignored by non-blocking ZeroMQ procedures. +If the writing failed, an error message can be included: +```json +{ + "run_number":135, + "run_name": "sample_1", + "socket_number": 1, + "processed_images": 0, + "ok": false, + "error": "Permission error" +} +``` +This way errors from the downstream code are propagated to `jfjoch_broker`. + +If writer notification socket is configured, but downstream code doesn't send proper notification, `jfjoch_broker` will time out after 60 seconds producing an error message. + +## Preview stream +Jungfraujoch can also send images (with metadata) at a reduced frame rate for preview purpose. +Images are serialized as CBOR [image message](CBOR.md#image-message). +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. + +This is using PUB socket with conflate option. I.e., only the last message is kept by ZeroMQ, so if receiver cannot cope +with the messages, it will always receive the last generated message (no backlog). +For this reason it is also recommended to use the same option on receiver side. + +Given PUB socket properties, it is possible to connect multiple viewers to a single socket --- all the viewers should receive all the images sent. + +## Metadata stream +Jungfraujoch can also send pure metadata for the purpose of archiving such information. +Metadata are serialized as CBOR [metadata message](CBOR.md#metadata-message). +This is very similar to the image message, but excludes the actual image array and spot positions. +As metadata are relatively small, to avoid large number of messages, Jungfraujoch bundles metadata of many images in one message. +Order of images within bundle, as well as the size of the bundle, are not guaranteed. +The stream will also include CBOR [start message](CBOR.md#start-message) and [end message](CBOR.md#end-message) with run metadata. + +This is using PUB socket with watermark, so there is some queuing of messages with ZeroMQ. Multiple receivers can be connected. \ No newline at end of file diff --git a/_sources/JFJOCH_BROKER.md.txt b/_sources/JFJOCH_BROKER.md.txt new file mode 100644 index 000000000..a1dd16293 --- /dev/null +++ b/_sources/JFJOCH_BROKER.md.txt @@ -0,0 +1,194 @@ +# jfjoch_broker + +`jfjoch_broker` is the main service for the Jungfraujoch application. It is responsible for: + +* Providing user interface via HTTP and OpenAPI +* Configuring FPGA firmware +* Building images from FPGA output and forwarding the results over ZeroMQ + +## External interfaces +Broker operates four external interfaces. + +**Image stream** ZeroMQ PUSH socket with CBOR serialization is used to send images, metadata and processing results for writing or downstream +processing. See details [here](IMAGE_STREAM.md#image-stream). + +**Preview stream** ZeroMQ PUB socket, as above but limited to subset of frames (1 image/s by default). See details [here](IMAGE_STREAM.md#preview-stream). + +**Metadata stream** ZeroMQ PUB socket, contains metadata for all the images, with bundling. See details [here](IMAGE_STREAM.md#metadata-stream). + +**Configuration, status and results interface** HTTP/REST interface described in the OpenAPI format. +Description of the API is presented in the [OpenAPI specification](OPENAPI_SPECS.rst). + +A dataset can be protected: `/start` takes an optional `tokens` list, and while the current dataset +has any, its statistics (`/statistics/data_collection`, `/result/scan`), buffered images +(`/image_buffer/*.cbor|jpeg|tiff`) and plots (`/preview/plot*`) need `Authorization: Bearer ` +and answer `401` otherwise; `/statistics` omits its `measurement` block instead. See +[Security](SECURITY.md#3-authenticated-read-access--per-dataset-bearer-tokens). + +## Broker configuration +`jfjoch_broker` requires JSON configuration files. The file is described by OpenAPI structure `jfjoch_settings` defined in `jfjoch_api.yaml` file. +It is recommended to go through example files in the `etc/`. + +Example configuration (not every section is shown): + +```json +{ + "pcie": [ + { + "blk": "/dev/jfjoch0", + "ipv4": "10.1.1.7" + }, + { + "blk": "/dev/jfjoch1", + "ipv4": "10.1.1.8" + } + ], + "zeromq": { + "send_watermark": 100, + "send_buffer_size": 1024, + "image_socket": [ + "tcp://1.2.3.4:5000", + "tcp://1.2.3.4:5001" + ], + "writer_notification_socket": "tcp://1.3.4.6:7000" + }, + "instrument": { + "source_name": "Swiss Light Source", + "source_type": "Synchrotron X-ray Source", + "instrument_name": "X06SA", + "pulsed_source": false, + "electron_source": false + }, + "detector": [ + { + "description": "EIGER 1M", + "serial_number": "E1M-01", + "type": "EIGER", + "high_voltage_V": 150, + "udp_interface_count": 1, + "module_sync": true, + "sensor_thickness_um": 320, + "calibration_file": [ + "gainMaps.bin" + ], + "hostname": [ + "e1m-01", + "e1m-02" + ], + "readout_time_us": 3, + "sensor_material": "Si", + "tx_delay": [ + 0,1 + ], + "base_data_ipv4_address": "10.10.10.50", + "standard_geometry": { + "nmodules": 1, + "gap_x": 8, + "gap_y": 36, + "modules_in_row": 1 + }, + "custom_geometry": [ + { + "x0": 0, + "y0": 0, + "fast_axis": "Xp", + "slow_axis": "Yp" + } + ], + "mirror_y": true + } + ], + "detector_settings": { + "frame_time_us": 450, + "count_time_us": 0, + "internal_frame_generator": false, + "internal_frame_generator_images": 1, + "detector_trigger_delay_ns": 0, + "timing": "auto", + "eiger_threshold_keV": 6.0, + "jungfrau_pedestal_g0_frames": 2000, + "jungfrau_pedestal_g1_frames": 300, + "jungfrau_pedestal_g2_frames": 300, + "jungfrau_pedestal_g0_rms_limit": 100, + "jungfrau_pedestal_min_image_count": 128, + "jungfrau_storage_cell_count": 1, + "jungfrau_storage_cell_delay_ns": 5000, + "jungfrau_fixed_gain_g1": false, + "jungfrau_use_gain_hg0": false + }, + "azim_int": { + "polarization_factor": -1, + "solid_angle_corr": true, + "high_q_recipA": 0, + "low_q_recipA": 0, + "q_spacing": 0 + }, + "image_format": { + "summation": true, + "geometry_transform": true, + "jungfrau_conversion": true, + "jungfrau_conversion_factor_keV": 0.001, + "bit_depth_image": 16, + "signed_output": true, + "mask_module_edges": true, + "mask_chip_edges": true + }, + "image_buffer_MiB": 2048, + "receiver_threads": 64, + "frontend_directory": "/usr/share/jfjoch/frontend", + "image_pusher": "ZeroMQ", + "zeromq_metadata": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4357" + }, + "zeromq_preview": { + "enabled": true, + "period_ms": 1000, + "socket_address": "tcp://0.0.0.0:4356" + } +} +``` + +## Setting up a local test for Jungfraujoch +For development, it is possible to set up a local installation of Jungfraujoch. +This will work without FPGA installed in the computer and allows testing the Jungfraujoch software layer, including +ZeroMQ streaming and file writing. + +The workflow simulates FPGA behavior, by running high-level synthesis code on the CPU - the performance is therefore +very low, as fixed-point calculations have a large performance penalty on the CPU. In the CPU simulation mode, one can simulate +using only a single FPGA device. + +To run the test: + +### Compile Jungfraujoch with frontend +``` +mkdir build +cd build +cmake .. +make jfjoch_broker +make frontend +``` +Alternatively, on a RHEL8 system, you can use the RPMs generated by the automated pipeline. +The `jfjoch` package alone is enough. +In this case - it is necessary to update `etc/broker_local.json` file with frontend path in `/usr/share/jfjoch/frontend`. + +### Start service +Start broker: +``` +cd build/broker +./jfjoch_broker ../../etc/broker_local.json 5232 +``` + +### Run tests +To run the test, a Python script is provided: +``` +cd tests/test_data +python jfjoch_broker_test.py +``` +The script will initialize Jungfraujoch, import test image and start data collection. + +### Expected result +You can observe online data analysis by opening the following web page: [http://localhost:5232](http://localhost:5232). +Also, a dataset with images should be written in the `build/broker` directory. + diff --git a/_sources/JFJOCH_VIEWER.md.txt b/_sources/JFJOCH_VIEWER.md.txt new file mode 100644 index 000000000..9a8f56721 --- /dev/null +++ b/_sources/JFJOCH_VIEWER.md.txt @@ -0,0 +1,334 @@ +# jfjoch_viewer + +`jfjoch_viewer` is the **interactive** desktop application of Jungfraujoch. It opens diffraction +datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal +integration, per-image statistics), and can follow a live data collection by syncing with a +running [`jfjoch_broker`](JFJOCH_BROKER.md) over its HTTP interface. + +It is a standalone Qt 6 application, distributed pre-built for **Linux, Windows and macOS** on the +Gitea release page, and for Linux also in the Jungfraujoch RPM/APT repositories — see +[Release contents](RELEASE_CONTENTS.md) for what each package contains and what it requires, and +[Deployment](DEPLOYMENT.md) for how to install it. The macOS build needs **macOS 13 (Ventura) or +newer on an Apple Silicon Mac** (M1 and newer; Intel Macs are not supported) and is CPU-only — see +[Release contents ▸ macOS](RELEASE_CONTENTS.md#macos), which also covers opening it for the first +time, as the release is not yet notarized by Apple. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| **`jfjoch_viewer`** | **Interactive, on-screen exploration** | **Qt desktop application** | **On screen; a processing job can write the same files as `rugnux`** | +| [`rugnux`](RUGNUX.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | + +## Functionality + +- Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the + `*_process.h5` files produced by [`rugnux`](RUGNUX.md). It also opens NXmx files + written by DECTRIS detectors, though that path has had only limited testing. +- Opens PILATUS miniCBF, marCCD and SMV rotation sweeps (the formats listed under + [What Rugnux reads](RUGNUX_FORMATS.md)). These store one frame per file, so naming any frame opens + the whole sweep it belongs to. A raw frame carries the images and the geometry but no analysis + results, so the spot, reflection and per-image plot panels stay empty until something is computed. +- Runs an **embedded data-processing pipeline** — the same analysis code as the rest of + Jungfraujoch — performing spot finding, indexing and integration on the displayed image, with the + result drawn over it. This interactive analysis is not written anywhere. +- Runs **full processing jobs** on the open dataset with *Analyze dataset*, on the same + [`rugnux`](RUGNUX.md) engine and off the GUI thread. The settings panel's **MX / AzInt / Calib** + toggle decides what a run does — full analysis, azimuthal integration only, or a detector + calibration — over a chosen image range, optionally writing `_process.h5` and the merged + `.mtz`/`.cif`. A finished run becomes a selectable view of the dataset, so several processing runs + can be compared against each other, and its merging statistics (or, for a calibration, its fitted + geometry) open in their own window; the *Processing* panel lists the runs and reopens those + results. The equivalent `rugnux` command line can also be copied out to run the same job on a + cluster instead. +- **Detector calibration** against a powder standard, on the *Calib* page: pick the calibrant + (`LaB6`, `AgBh`, `CeO2`, `Si`, `ice`, or the open dataset's own unit cell) and fit either the image + on screen (*Guess* / *Refine detector calibration*) or the whole dataset (*Analyze dataset*, which + writes a pyFAI `.poni`). The whole-dataset fit measures the rings either from the + azimuthally-binned profile summed over the run (*Rings*, the default) or from the pooled spot lists + (*Spots*), and reports PONI x/y, the two tilts and the distance against the header values. Judge it + by the **radial rms**, not the beam-centre sigma: the sigma shrinks with the number of ring points, + so a fit that sits a couple of pixels off every ring can still report a small one. *Rings* needs + the run to be integrated in azimuthal sectors — with the AzInt page's *Azimuthal bins* below 4 the + calibration run raises it to 32, as `rugnux --mode calibration` does, and says so. + *Refine detector tilt* is ticked by default and fits the two tilts along with the centre and the + distance; unticking it holds them where they are, for a calibration meant for a program that + cannot express a tilted detector (`rugnux --no-refine-tilt`). It applies to both buttons and to + *Analyze dataset*. +- **Settings** panel for the geometry, unit cell, spot finding, indexing, azimuthal integration, + Bragg integration, scaling, powder calibration and a reference dataset — the same settings the + CLI takes. +- Auxiliary windows: image list, dataset metadata, spot list, reflection list, + 2D azimuthal-integration image and calibration-image viewer; plus the + *Inspector* (per-image statistics, image features, resolution rings, ROI statistics), the + *Magnifier* below it (three zoom levels: ×64 and ×32 with the pixel values written on the pixels, + ×10 without; *Pop out* moves it to a window of its own) and dataset-info charts. +- The *Inspector*'s **Image features** section decides what the overlay draws — spots, predictions, + saturated and highest pixels, the beam stop — including whether the non-indexed spots and the + spots that fall on an ice ring are drawn at all. +- User-mask editing: build a user mask interactively, load one from TIFF (replacing or adding to the + current one), save it as TIFF, clear it, or upload it to a connected server. +- Mouse-driven navigation of the image, the grid scan and the plots — see + [Mouse shortcuts](#mouse-shortcuts) below, which the viewer also shows under *Help ▸ Mouse + Shortcuts*. +- *Help* shows the mouse shortcuts, the [acknowledgements](ACKNOWLEDGEMENT.md) and the third-party + licenses. +- Layout presets (*View ▸ Image layout / Processing layout / Reset layout*) rearrange the docks for + looking at images or at processing results. +- *View ▸ Theme* picks the light or the dark colour scheme, or *Follow system*. Following the + system needs a desktop that tells Qt its scheme: macOS and Windows do, and so do GNOME and KDE + sessions on Linux (elsewhere, `QT_QPA_PLATFORMTHEME=xdgdesktopportal` reaches the portal + setting); a session Qt cannot read - a bare X server, `ssh -X` - counts as light. The choice is + remembered across restarts, and the half-sun toolbar button toggles light and dark directly. +- *View ▸ Font size* (or `Ctrl`+`+` / `Ctrl`+`-`) enlarges the text to 125 % or 150 %, on top of + whatever scaling the desktop already applies, and the choice is remembered across restarts. The + viewer also follows the desktop's own text scaling or display scaling on its own; over `ssh -X`, + where no settings daemon delivers it, launch as `QT_SCALE_FACTOR=1.5 jfjoch_viewer` instead. + +## A guided tour + +Four views cover most of what the viewer is used for. The screenshots show the lysozyme reference +sweep of the in-house test set and a raster scan. + +### Looking at an image + +`jfjoch_viewer ` opens the file and shows its first image; **File ▸ Open** and the toolbar's +open button do the same, and **File ▸ Open HTTP** connects to a running `jfjoch_broker` instead. + +![The general view: diffraction image, inspector and dataset-info plot](images/viewer_general.png) + +The top toolbars step through the images (slider, first/previous/next/last, `Jump` and `Sum` for +summing consecutive frames) and set the display (foreground limit, `Auto` contrast, HDR, colour +map, font size, theme). The background limit has a slider of its own, shown from *View ▸ +Background slider* or as soon as `B` + wheel raises it; `Auto` sets it back to zero. The image in the middle zooms with the wheel and +pans by dragging; hovering shows the pixel position, its value and the resolution on the status +bar. The **Inspector** on the right lists the dataset's metadata and, once an image has been +analysed, its spot count, background, indexing result and resolution estimate; the **Magnifier** +under it follows the cursor while `Shift` is held. On a window too narrow for both, the inspector (with the magnifier) folds +away and comes back when the window is widened again. The **Dataset info** dock at the bottom plots a per-image quantity +over the whole sweep - background here; the combo offers spot counts, indexing results, scale +factors and more once they exist - and `Shift`-hover on it loads the hovered image. + +### A grid scan + +A raster scan opens on its two-dimensional map: each cell is one image, coloured by the metric +chosen in the combo (spot count by default for a grid scan), so the crystal shows up as the bright +region. `Shift`-hover or double-click a cell to load that image. + +![A grid scan on its map, with the spot count per position](images/viewer_grid_scan.png) + +The **Grid** button switches between the map and the per-image line plot; a grid scan remembers +its own preferred plot, independently of the one used for rotation data. + +### Processing settings + +The **Processing** dock on the left (the tab next to **Files**, or **View ▸ Processing layout**) +holds every setting the analysis takes: geometry and unit cell, the goniometer, spot finding, +indexing, Bragg integration, scaling, and the reference dataset (MTZ or structure-factor mmCIF) +with an optional atomic model to validate the merged data against. The MX / AzInt / Calib switch +selects the kind of analysis and its page. + +![The Processing dock with the spot finding, indexing and reference sections open](images/viewer_processing_settings.png) + +**Analyze image** re-analyses the current image with these settings, now and on every change +while it stays pressed; the inspector and the image overlays update at once. + +### Processing results + +**Analyze dataset** runs the whole sweep through the same pipeline as `rugnux` (a job dialog +takes the image range, the thread count and which files to write, and **Copy command** gives the +equivalent command line for a cluster). The **Jobs** dock follows the run. + +![A finished job: the jobs table and the merge statistics window](images/viewer_processing_results.png) + +When it finishes, the job's graph button opens the merge statistics (completeness, CC1/2, I/sigma +per resolution shell, and the text report). The window names the space group with its screw axes +subscripted and lists the crystal pathologies the report checks, one light each: green where the +check was made and did not fire, red where it fired (its warning in the tooltip), grey where the data +could not answer it. The dataset-info combo gains the per-image +indexing result, mosaicity, integrated reflections and scale factors of that run, plotted next to +the original file's values. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. On systems with a +GPU, use the CUDA build (a separate package variant everywhere: RPM/APT repository, `.tgz` and +Windows installer) for the embedded indexing and integration; the non-CUDA build runs the same +pipeline on the CPU at much lower throughput. The CUDA build also runs on a machine without a GPU — +see [Release contents ▸ CUDA and non-CUDA builds](RELEASE_CONTENTS.md#cuda-and-non-cuda-builds). + +The CUDA build needs an NVIDIA **driver** on the host but no CUDA toolkit — 525.60.13 or newer for +the CUDA 12 artefacts (RHEL 8 packages, portable Linux `.tgz`), 580.65.06 or newer on Linux and an +R580 driver on Windows for the CUDA 13 ones (RHEL 9, Ubuntu, Windows installer). The Windows +installer and the `.tgz` are CUDA 13 and CUDA 12 respectively, which also decides the oldest GPU +they run on — a V100 needs the CUDA 12 `.tgz`. See +[Release contents ▸ GPU generations and the NVIDIA driver](RELEASE_CONTENTS.md#gpu-generations-and-the-nvidia-driver). + +On a Mac there is no CUDA: the viewer always runs the same pipeline on the CPU, with the FFTW +indexer, like the non-CUDA build elsewhere. + +### Remote displays + +The viewer detects a remote display session (`ssh -X` and the like) and limits how often panning, +zooming and live playback repaint, since on such a link every repaint is shipped as pixels. The +detection can be overridden in *View ▸ Remote display mode* or with `JFJOCH_VIEWER_REMOTE=0`/`1`. +A VNC- or xpra-based remote desktop still transports the viewer far more efficiently than plain +X11 forwarding. + +## Mouse shortcuts + +The same list is available in the application under **Help ▸ Mouse Shortcuts**. + +On macOS, `Ctrl` below means the Command key (`⌘`); right click is also `Ctrl`-click, and on a +keyboard without them `Home` / `End` are `Fn`+`←` / `Fn`+`→` and `Page Up` / `Page Down` are +`Fn`+`↑` / `Fn`+`↓`. The Mouse Shortcuts window shows the Mac keys directly. + +### Diffraction image + +| Action | Effect | +| --- | --- | +| Wheel | Zoom in / out, centred on the cursor | +| `Shift` + wheel | Move the foreground (upper contrast limit) in linear steps | +| `Ctrl` + wheel | Move the foreground in multiplicative steps (×1.15 per notch) | +| `F` held + wheel | Same as `Shift` + wheel, for as long as `F` is held | +| `B` held + wheel | Move the background (lower contrast limit) in linear steps, for as long as `B` is held; shows the background slider | +| `A` | Apply auto-contrast once (background back to zero); press it again to switch on continuous Auto | +| `Home` / `End` | Jump to the first / last image in the dataset | +| `Page Up` / `Page Down` | Step one image forward / back | +| Hover | Status bar shows the pixel position, its value and the resolution | +| Drag | Pan the image | +| `Shift` + move | Move the magnifier panel to the cursor; a frame shows the area it covers | +| `Shift` + drag | Draw a rectangular ROI | +| `Shift` + `Ctrl` + drag | Draw a circular ROI | +| Drag an ROI or its handle | Move or resize the selected ROI | +| Right click | Copy / save the image, fit to view, clear the ROI | + +### Grid scan + +| Action | Effect | +| --- | --- | +| Hover | Status bar shows the image number, the grid position and its value | +| `Shift` + hover | Load the image under the cursor while moving over the grid | +| Double click | Load the image under the cursor | + +### Other views + +| Action | Effect | +| --- | --- | +| 2D azimuthal image: double click | Zoom the diffraction image on the corresponding detector position | +| Dataset-info plot: hover | Status bar shows the image number and the plotted value | +| Dataset-info plot: `Shift` + hover | Load the hovered image | +| Spot / reflection list: double click | Zoom the diffraction image on that spot or prediction | +| Image list: double click | Load that image | + +## Opening data + +- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file, or any frame of a miniCBF, marCCD or SMV sweep. +- **File ▸ Open HTTP** (`Ctrl+H`) — connect to a `jfjoch_broker` HTTP endpoint to follow a live + collection. The dialog defaults to host `localhost` and port `8080`; these defaults can be + overridden with the environment variables `JUNGFRAUJOCH_HTTP_HOST` and `JUNGFRAUJOCH_HTTP_PORT`. + The scheme box selects `http://` or `https://` (the latter for a broker behind a TLS proxy), and + the **Token** field takes the dataset's bearer token when the collection was started with one. + The token can also come from `JUNGFRAUJOCH_HTTP_TOKEN` or from D-Bus (below); what is typed in the + dialog overrides both, and nothing is stored between sessions. Without a valid token for a + protected dataset the viewer shows nothing and says so on the status bar — no dialog, since a + dataset changing hands is the normal reason. +- **Command line** — `jfjoch_viewer ` opens a file (or an `http://host:port` URL) on + start-up. `--dbus ` (`-d`) enables or disables the D-Bus interface (default: enabled); + `--help` and `--version` behave as usual. + +## D-Bus interface + +When enabled, the viewer registers the D-Bus interface `ch.psi.jfjoch_viewer`, so other processes +can drive it. D-Bus is **Linux only**: the Windows and macOS builds have no D-Bus interface, and +`--dbus` has no effect there. + +- `LoadFile(filename, image_number=0, summation=1, token="")` — open a file (or an + `http://host:port` URL) and display the given image; a non-empty `token` is the bearer token of a + protected broker dataset. +- `LoadImage(image_number, summation=1)` — navigate to an image in the already-open dataset. +- `SetHttpToken(token)` — set the dataset token without reloading; the next request carries it. + +`summation` sums that many consecutive images before display. A repeated `LoadFile` call naming the +file that is already open is cheap (it just navigates, like `LoadImage`) rather than reopening it, +but a client stepping through images of a dataset it opened itself should still prefer `LoadImage` — +it needs no filename and avoids the file-identity comparison. + +## Building from source on Windows + +`jfjoch_viewer` is cross-platform: it builds on Windows 11 with MSVC and the full CUDA GPU path, and +on macOS (see [below](#building-from-source-on-macos)). (The rest of Jungfraujoch — broker, receiver, FPGA host — is +Linux-only.) A pre-built installer is published with every release, so building from source is only +needed to develop or to change the build options. On Windows the build is automatically restricted +to the viewer and the libraries it needs (`JFJOCH_VIEWER_ONLY` is forced on), and the remaining +dependencies are fetched and built automatically (the first configure needs network access). + +Verified toolchain — the same one the released installer is built with: + +- Windows 11 +- Visual Studio 2026 with the C++ (MSVC) toolset — required; CUDA on Windows builds through MSVC +- CUDA Toolkit 13.3 (12.8 or newer is required) — for the GPU indexing/integration path +- Qt 6.11 for MSVC (`msvc2022_64`), including the **Qt Charts** module — e.g. `C:\Qt\6.11.1\msvc2022_64` +- CMake plus Ninja. The CMake that ships with Visual Studio is the simplest choice and works out of + the box — it comes with the C++ workload, so there is nothing extra to install. Any recent + standalone CMake (from cmake.org, or the one bundled with Qt in `C:\Qt\Tools\CMake_64`) works too. +- Optional: [NSIS](https://nsis.sourceforge.io) to build the `.exe` installer. + +Configure and build from an **x64 Native Tools Command Prompt for VS 2026** (so `cl`, `nvcc` and +`ninja` are on `PATH`): + +``` +cmake -G Ninja -B build-win -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_PREFIX_PATH="C:/Qt/6.11.1/msvc2022_64" +cmake --build build-win --target jfjoch_viewer +``` + +Notes: + +- `CMAKE_PREFIX_PATH` (Qt) is the only required flag. Every other dependency, zlib and Eigen + included, is downloaded and built by the configure itself, so nothing else has to be installed. +- The CUDA toolchain is located automatically from the `CUDA_PATH` environment variable that the + CUDA installer sets (or from `nvcc` on `PATH`). Pass `-DCMAKE_CUDA_COMPILER=".../bin/nvcc.exe"` + only if `nvcc` is installed in a nonstandard location and is not found. +- For a machine without an NVIDIA GPU, add `-DJFJOCH_USE_CUDA=OFF`: the viewer then runs the same + pipeline on the CPU (FFTW indexer) at lower throughput. + +To produce a self-contained installer (bundles the Qt runtime via `windeployqt` and — on the CUDA +build — the cuFFT runtime DLL, so the target host needs neither Qt nor a CUDA toolkit), with NSIS +installed: + +``` +cd build-win +cpack +``` + +The NSIS generator is selected automatically on Windows (no `-G` needed). What comes out, and how +the CUDA and CPU variants are named and told apart, is described in +[Release contents ▸ Windows installer](RELEASE_CONTENTS.md#windows-installer). + +## Building from source on macOS + +The viewer also builds on macOS, **Apple Silicon only** and without CUDA; as on Windows, the build +is automatically restricted to the viewer and the libraries it needs (`JFJOCH_VIEWER_ONLY` is forced +on) and fetches every other dependency itself. A pre-built disk image is published with every +release, so building from source is only needed to develop or to change the build options. + +Verified toolchain — the same one the released `.dmg` is built with: + +- An Apple Silicon Mac; the resulting app needs macOS 13 or newer, whatever the build machine runs +- Xcode (Apple Clang) +- Qt 6.11 for macOS, including the **Qt Charts** module — e.g. `~/Qt/6.11.2/macos` from the Qt + online installer +- CMake (e.g. `CMake.app` from cmake.org, with `/Applications/CMake.app/Contents/bin` on `PATH`) + +``` +cmake -S . -B build-mac -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=$HOME/Qt/6.11.2/macos +cmake --build build-mac -j$(sysctl -n hw.ncpu) --target jfjoch_viewer +open build-mac/viewer/jfjoch_viewer.app +``` + +As on Windows, `CMAKE_PREFIX_PATH` (Qt) is the only required flag. To produce the disk image — the +Qt frameworks and the license notices copied into the app with `macdeployqt`, packed into +`jfjoch-viewer--macos-arm64.dmg` — run `cpack` in the build directory. What comes out is +described in [Release contents ▸ macOS](RELEASE_CONTENTS.md#macos). diff --git a/_sources/JFJOCH_WRITER.md.txt b/_sources/JFJOCH_WRITER.md.txt new file mode 100644 index 000000000..8fbe49441 --- /dev/null +++ b/_sources/JFJOCH_WRITER.md.txt @@ -0,0 +1,173 @@ +# jfjoch_writer + +`jfjoch_writer` is a NeXus-compliant HDF5 file writer. + +## Acknowledgements +Thanks to Zdenek Matej (MAX IV) and Felix Engelmann (MAX IV) for testing and multiple improvement +suggestions. + +## Running directory +The writer needs to run in the base directory for writing files - `file_prefix` is always relative to the writer's running directory. +The writer detects and protects against basic security issues, like `file_prefix` starting with a slash, or starting with `../`, or containing `/../`. + +## Usage +Writer needs to be started as a background service, with the following command: +``` +jfjoch_writer {options}
+ +Options: +-T | --tcp Use raw TCP/IP instead of ZeroMQ +-j | --nproc= Number of forks (only with -T) +-d | --root_dir= Root directory for file writing (-R is a deprecated alias) +-r | --zmq_repub_port= ZeroMQ port for PUSH socket to republish images +-f | --zmq_file_port= ZeroMQ port for PUB socket for notifications on finalized files +-w | --rcv_watermark= Receiving ZeroMQ socket watermark (default = 100) +-W | --repub_watermark= Republish ZeroMQ socket watermark (default = 1000) +-v | --verbose Verbose output +-h This message +``` +for example: +``` +jfjoch_writer -d /data tcp://dcu-address:5400 +``` + +## Status and cancellation +When a data collection is finalized, each writer reports its outcome back to `jfjoch_broker` over +the writer notification socket — a ZeroMQ address the broker passes in the START message +(`writer_notification_socket` in the broker configuration) — as a JSON message with the socket +number, run name and number, processed image count, throughput, and on failure an error string. +That is how the broker learns that a writer could not write. On the TCP/IP image stream, failures +additionally come back in-band as negative acknowledgements +(see [Data streams](IMAGE_STREAM.md#tcpip-image-stream)). + +To stop a writer, send it `SIGINT`, `SIGQUIT`, `SIGTERM` or `SIGHUP`: it closes the HDF5 files it is +writing and exits. This is only for the case where the broker was terminated or disconnected — it is +not the normal way to end a data collection, which the broker finishes on its own. + +## Republish +Republish creates a PUSH socket on the writer, where all the messages are republished for further use by a data analysis pipeline. +Republish is non-blocking, so if there is no receiver on other end or the sending queue is full - images won't be republished. +In case of START/END messages republishing will attempt sending for 100 ms, but if send times out it won't be retried. + +Republish functionality is optional, if republish port number is omitted this functionality is not enabled. + +## Overwriting files +When `jfjoch_writer` creates a HDF5 file, it first adds suffix `..tmp`. +Random value depends on current time-stamp and likely will be different from each file of the particular series. +After file is all saved and closed, it is renamed to remove the suffix. +By default, renaming won't happen if this would overwrite existing file. +However, this behavior can be changed by setting `overwrite` parameter to true in the file writer configuration. + +### When the overwrite conflict is reported +An existing output file is a fatal condition (unless `overwrite` is true). *When* it is detected +depends on whether the transport between the broker and the writer has a back-channel to report the +failure before acquisition starts: + +* **Direct HDF5 pusher and TCP writer (back-channel available).** The conflict is detected at + **start**: the writer that owns the master file checks whether it already exists and refuses to + start. The direct pusher raises the error in-process; the TCP writer returns a START-failure + acknowledgement. Either way the broker learns immediately and aborts the data collection *before* + the detector is armed — no images are taken and nothing is written. Only the master file is + checked up front: in a multi-writer setup the per-image data files are staggered across writers, + and checking them at start would make each writer inspect files it never writes (and race the + writers that do). Data-file conflicts are instead caught by their owning writer at the final + rename, which for the TCP path surfaces as a write-failure acknowledgement to the broker. +* **ZeroMQ writer (no back-channel).** The ZeroMQ image stream is fire-and-forget: the writer has no + way to tell the broker to stop, and the broker would keep streaming images regardless. The writer + therefore does **not** fail at start. It writes the whole series to the `..tmp` files as + usual and only fails at the final rename, leaving the `.tmp` files on disk. This is deliberate: the + acquired images are preserved (in `.tmp` form) rather than being dropped by a writer that aborted + mid-stream. Rename the `.tmp` files by hand, or re-run with `overwrite` set, to recover them. + +## Finalized files information +Creates PUB socket to inform about finalized data files. For each closed file, the socket will send a JSON message, with the following structure: + +``` +{ + "filename": : HDF5 data file name (relative to writer root directory), + "nimages": number of images in the file (counting from 1!), + "file_number": number of file within the acquisition, + "sample_name": name of sample, + "run_name": name of run, + "run_number": number of run, + "experiment_group": number of p-group / proposal (optional), + "user_data": user_data, + "beam_x_pxl": beam center (X) in pixels, + "beam_y_pxl": beam center (Y) in pixels, + "detector_distance_m": detector distance in m, + "detector_height_pxl": detector size (Y) in pixels, + "detector_width_pxl": detector size (X) in pixels, + "incident_energy_eV": photon energy of the X-ray beam, + "pixel_size_m": pixel size in meter (assuming pixel X == Y), + "saturation": this count and higher mean saturation, + "space_group_number": space group number (optional), + "underload": lowest valid count; anything below it is invalid, + "unit_cell": unit cell dimensions in Angstrom/degree { + "a": , "b": , "c": , + "alpha": , "beta": , "gamma": + }, +} +``` +`user_data` is defined as `header_appendix` in the `/start` operation in the `jfjoch_broker`. +Other metadata are also carried over from `/start` operation. + +If the `header_appendix` is a string with valid JSON meaning, it will be embedded as JSON, otherwise it will be escaped as string. +For example `header_appendix` of `{"param1": "test1", "param2": ["test1", "test2"]}`, then the example message will look as follows: +```json +{ + "filename": "dataset_name_data_000001.h5", + "nimages": 1000, + "file_number": 0, + "sample_name": "my_sample", + "run_name": "my_run", + "run_number": 25, + "experiment_group": "p00001", + "beam_x_pxl": 1200, + "beam_y_pxl": 1500, + "detector_distance_m": 0.155, + "detector_height_pxl": 2164, + "detector_width_pxl": 2068, + "incident_energy_eV": 12400.0, + "pixel_size_m": 7.5e-05, + "saturation": 32766, + "space_group_number": 96, + "underload": -32767, + "unit_cell": { + "a": 78.0, + "alpha": 90.0, + "b": 78.0, + "beta": 90.0, + "c": 39.0, + "gamma": 90.0 + }, + "user_data": { + "param1": "test1", + "param2": ["test1", "test2"] + } +} +``` + +Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled. + +## HDF5 file structure + +Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, +integration, azimuthal integration, per-image statistics and timing) stored *beyond* the NXmx +standard. The complete file layout — master vs data files, the three format variants +(`NXmxLegacy`, `NXmxVDS`, `NXmxIntegrated`), every NXmx field that is populated and every +Jungfraujoch extension — is documented in [HDF5 / NeXus data format](HDF5.md). + +If data collection was configured with a `header_appendix` containing a key `hdf5` whose value is a +JSON object of numbers and strings, those entries are written to `/entry/user`. + +## Other formats (CBF and TIFF) +Earlier versions could also write Crystallographic Binary File (CBF, miniCBF) and TIFF images. These +writers have been removed: Jungfraujoch now writes only NXmx HDF5. The `CBF` and `TIFF` values are +retained in the file-format enum for wire back-compatibility, but a request to write either format +is rejected. + +## No file option(s) +There are two options to disable writing of files by the writer: +* Setting `file_prefix` to empty string - this will disable sending files on ZeroMQ image socket. +* Setting file format to `NoFile` - files are streamed over ZeroMQ socket, but `jfjoch_writer` will not write anything. +This can be useful for debugging purposes, or if you only rely on republishing functionality of the `jfjoch_writer` \ No newline at end of file diff --git a/_sources/LICENSE.md.txt b/_sources/LICENSE.md.txt new file mode 100644 index 000000000..222b84bab --- /dev/null +++ b/_sources/LICENSE.md.txt @@ -0,0 +1,690 @@ +# License + +Jungfraujoch software is licensed with GPLv3 license. +Jungfraujoch FPGA is licensed with CERN OHL-S license (see [FPGA license](FPGA_LICENSE.md)). + +## GNU GENERAL PUBLIC LICENSE +Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +### Preamble + +The GNU General Public License is a free, copyleft license for +software and other kinds of works. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + +Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + +Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and +modification follow. + +### TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +1. Source Code. + +The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + +The Corresponding Source for a work in source code form is that +same work. + +2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + +4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + +a) The work must carry prominent notices stating that you modified +it, and giving a relevant date. + +b) The work must carry prominent notices stating that it is +released under this License and any conditions added under section +7. This requirement modifies the requirement in section 4 to +"keep intact all notices". + +c) You must license the entire work, as a whole, under this +License to anyone who comes into possession of a copy. This +License will therefore apply, along with any applicable section 7 +additional terms, to the whole of the work, and all its parts, +regardless of how they are packaged. This License gives no +permission to license the work in any other way, but it does not +invalidate such permission if you have separately received it. + +d) If the work has interactive user interfaces, each must display +Appropriate Legal Notices; however, if the Program has interactive +interfaces that do not display Appropriate Legal Notices, your +work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + +a) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by the +Corresponding Source fixed on a durable physical medium +customarily used for software interchange. + +b) Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by a +written offer, valid for at least three years and valid for as +long as you offer spare parts or customer support for that product +model, to give anyone who possesses the object code either (1) a +copy of the Corresponding Source for all the software in the +product that is covered by this License, on a durable physical +medium customarily used for software interchange, for a price no +more than your reasonable cost of physically performing this +conveying of source, or (2) access to copy the +Corresponding Source from a network server at no charge. + +c) Convey individual copies of the object code with a copy of the +written offer to provide the Corresponding Source. This +alternative is allowed only occasionally and noncommercially, and +only if you received the object code with such an offer, in accord +with subsection 6b. + +d) Convey the object code by offering access from a designated +place (gratis or for a charge), and offer equivalent access to the +Corresponding Source in the same way through the same place at no +further charge. You need not require recipients to copy the +Corresponding Source along with the object code. If the place to +copy the object code is a network server, the Corresponding Source +may be on a different server (operated by you or a third party) +that supports equivalent copying facilities, provided you maintain +clear directions next to the object code saying where to find the +Corresponding Source. Regardless of what server hosts the +Corresponding Source, you remain obligated to ensure that it is +available for as long as needed to satisfy these requirements. + +e) Convey the object code using peer-to-peer transmission, provided +you inform other peers where the object code and Corresponding +Source of the work are being offered to the general public at no +charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + +a) Disclaiming warranty or limiting liability differently from the +terms of sections 15 and 16 of this License; or + +b) Requiring preservation of specified reasonable legal notices or +author attributions in that material or in the Appropriate Legal +Notices displayed by works containing it; or + +c) Prohibiting misrepresentation of the origin of that material, or +requiring that modified versions of such material be marked in +reasonable ways as different from the original version; or + +d) Limiting the use for publicity purposes of names of licensors or +authors of the material; or + +e) Declining to grant rights under trademark law for use of some +trade names, trademarks, or service marks; or + +f) Requiring indemnification of licensors and authors of that +material by anyone who conveys the material (or modified versions of +it) with contractual assumptions of liability to the recipient, for +any liability that these contractual assumptions directly impose on +those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + +13. Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +### How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + +You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + +The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + + +## Jungfraujoch exceptions to GPL + +As a special exception, we specifically permit linking Jungfraujoch code with Nvidia CUDA libraries and Intel MKL. + +We also permit to link Jungfraujoch software (GPLv3) with Jungfraujoch high-level synthesis code (CERN OHL 2.0) for the purpose +of simulating FPGA design on CPU. + +If OpenAPI definition file (jfjoch_api.yaml) is solely used to generate client code or to interact with the Jungfraujoch +API it may be distributed under terms of your choosing without being subject to GPL requirements. diff --git a/_sources/NAMING.md.txt b/_sources/NAMING.md.txt new file mode 100644 index 000000000..fa6a7a83a --- /dev/null +++ b/_sources/NAMING.md.txt @@ -0,0 +1,60 @@ +# Naming + +The software is Swiss, and so are its names: both halves of the system are named after +places in the Alps that are, in one way or another, about moving a *lot* of something up a +steep mountain as efficiently as possible — usually by train. Throughput, in other words. + +| Part | Name | What it does | +| --- | --- | --- | +| Streaming / acquisition | **Jungfraujoch** | Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing. | +| Data processing | **Rugnux** | Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the [`rugnux`](RUGNUX.md) tool). | + +## Jungfraujoch + +The **Jungfraujoch** is a high mountain col in the Bernese Alps, the saddle (*Joch* is German +for "yoke" or "col") between the peaks **Jungfrau** and **Mönch**, at 3,466 m. It is the site of +the [High Altitude Research Station Jungfraujoch](https://www.hfsjg.ch/), whose long-running +atmospheric measurements are **co-operated by the Paul Scherrer Institute** — the same institute +that develops this software and the JUNGFRAU detector. + +The name is also a small piece of word-play. PSI's **JUNGFRAU** detector and DECTRIS's **EIGER** +detector are both named after Bernese Alps peaks (the famous trio is *Eiger*, *Mönch*, *Jungfrau*). +The Jungfraujoch — the pass *between* Jungfrau and Mönch — is where those two detector worlds meet. + +And it fits the theme of the whole project: the Jungfraujoch is reached by the **Jungfraubahn**, +whose terminus is the **highest railway station in Europe** (3,454 m, the "Top of Europe"). It is +the closest you can get to that summit in a genuinely *high-throughput* way — by train, moving +crowds up the mountain — which is exactly what the streaming side of this software does with +detector frames. + +**Pronunciation (German):** *Jungfraujoch* ≈ **YUNG-frow-yokh**. +"Jung" as in *young*, "frau" rhymes with *cow*, and the final "joch" ends in the guttural *ch* of +Scottish *loch* or German *Bach* — not a hard *k*. + +## Rugnux + +**Piz Rugnux** is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern +Switzerland. (*Piz* is the Romansh word for "peak".) It rises above the **Albula line** of the +**Rhaetian Railway** (*Rhätische Bahn*), part of the "Rhaetian Railway in the Albula / Bernina +Landscapes" — a **UNESCO World Heritage Site** (*Welterbe*). + +That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude +in very little horizontal distance, it corkscrews through a series of **helical (spiral) tunnels** +looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a +steep mountain efficiently — the same idea the data-processing side of this software is built +around: pushing a large volume of diffraction data through the analysis pipeline. + +So the theme is consistent — **Swiss mountains, trains, and throughput** — while keeping the two +subsystems clearly distinct: *Jungfraujoch* streams, *Rugnux* processes. + +**Pronunciation (Romansh):** *Piz Rugnux* ≈ **peets roo-NYOOKS**. +The "gn" is a soft palatal *ñ*, as in *canyon* or Italian *gnocchi*, not two separate sounds. + +## What is Romansh? + +**Romansh** (*Rumantsch*) is the **fourth national language of Switzerland**, alongside German, +French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind +in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the +canton of Graubünden. It survives in several regional idioms, brought together in a standard form +called *Rumantsch Grischun*. Naming the processing engine with a Romansh mountain is a small nod to +the least-spoken but no-less-Swiss corner of the country. diff --git a/_sources/OPENAPI.md.txt b/_sources/OPENAPI.md.txt new file mode 100644 index 000000000..1eaa1fba3 --- /dev/null +++ b/_sources/OPENAPI.md.txt @@ -0,0 +1,15 @@ +# OpenAPI +## OpenAPI specs + +See document with detailed [OpenAPI specs](OPENAPI_SPECS.rst). The spec declares one security +scheme, `bearerAuth`, on the endpoints that expose the current dataset; it applies only while the +dataset was started with `tokens` (see [Security](SECURITY.md)). + +## Python client +Jungfraujoch is controlled with HTTP/REST interface defined with an OpenAPI specification. +For convenience, we provide a Python client as the [jfjoch-client](https://pypi.org/project/jfjoch-client/) PyPI package. +To install the client you can use `pip` tool: +``` +pip install jfjoch-client +``` +See [API reference from the OpenAPI generator](python_client/README.md). \ No newline at end of file diff --git a/_sources/OPENAPI_SPECS.rst.txt b/_sources/OPENAPI_SPECS.rst.txt new file mode 100644 index 000000000..15bf4e7f6 --- /dev/null +++ b/_sources/OPENAPI_SPECS.rst.txt @@ -0,0 +1,4 @@ +OpenAPI specification +===================== + +See document with detailed `OpenAPI specs <_static/redoc-static.html>`_ generated with Redocly. \ No newline at end of file diff --git a/_sources/PIXEL_MASK.md.txt b/_sources/PIXEL_MASK.md.txt new file mode 100644 index 000000000..22f5200a4 --- /dev/null +++ b/_sources/PIXEL_MASK.md.txt @@ -0,0 +1,55 @@ +# Pixel mask + +## Mask format + +Jungfraujoch generally follows the [NXmx format](https://manual.nexusformat.org/classes/applications/NXmx.html) for the pixel mask. +The pixel mask is a 32-bit unsigned integer array of the same size as the image. +The conditions for masking a pixel are encoded by setting a particular bit to one. This makes it possible to record the reason why a pixel is included in the mask, and several reasons can be recorded for one pixel at the same time. + +Bit values are set as follows: + +Bit 0 - gap (pixel with no sensor) + +Bit 1 - error pixel (for PSI JUNGFRAU: pixel doesn't set proper gain during pedestal, for DECTRIS: pixel is part of detector pixel mask) + +Bit 4 - noisy pixel (for PSI JUNGFRAU: pixel pedestal G0 RMS is over threshold, for DECTRIS: pixel was flagged with signal during dark data collection at initialization) + +Bit 8 - user defined mask + +Bit 9 - beam stop shadow (found by `rugnux --detect-beam-stop`, on by default; see [Rugnux](RUGNUX.md)). +Unlike the other bits this one belongs to the run that found it, not to the dataset: Rugnux clears it +at the start of every run, so a mask read back from a file that carries one starts clear. The user +mask (bit 8) is left alone. + +Bit 10 - defective pixel found on the run's own frames (rotation data from a counting sensor; see +[CPU data analysis §1.6](CPU_DATA_ANALYSIS_IMAGE.md)): lit above its resolution ring on more frames +than one reflection or chance explains, or holding the detector's error value on most frames. + +Bit 30 - module edge (only for PSI systems) + +Bit 31 - chip edge interpolated pixel (multipixel) + +## Custom user mask + +Jungfraujoch allows a custom user mask to be uploaded. This happens in two steps. First create the mask in TIFF format: + +```python +import numpy as np +import tifffile as tiff + +# Create an array matching a 2068 x 2164 (width x height) image: 2164 rows, 2068 columns +array = np.zeros((2164, 2068), dtype=np.uint32) + +# Mark the pixel at column 400, row 300 with the value 1 +array[300, 400] = 1 + +# Save the array as a TIFF file +tiff.imwrite('mask.tiff', array) +``` + +Pixels with non-zero value in the TIFF file will be marked as belonging to the user mask (bit 8). + +Then upload the mask to Jungfraujoch server: +```shell +curl -v http:///config/user_mask.tiff -XPUT --data-binary @mask.tiff +``` diff --git a/_sources/PYTHON_CLIENT.md.txt b/_sources/PYTHON_CLIENT.md.txt new file mode 100644 index 000000000..9f0b74b21 --- /dev/null +++ b/_sources/PYTHON_CLIENT.md.txt @@ -0,0 +1,34 @@ +# OpenAPI Python client + +The broker's REST API has a generated Python client, published on PyPI as +[`jfjoch-client`](https://pypi.org/project/jfjoch-client/) and regenerated from +`broker/jfjoch_api.yaml` by `update_version.sh` — the YAML is the single source of truth +(see [OpenAPI](OPENAPI.md)). + +- [Client README](python_client/README.md) — installation, quick start, and the index of every + endpoint and model. +- [DefaultApi](python_client/docs/DefaultApi.md) — the full endpoint reference, with a generated + example per call. + +A protected dataset (one started with `tokens`) is read with the token as the client's bearer +credential: + +```python +import jfjoch_client +api = jfjoch_client.DefaultApi(jfjoch_client.ApiClient( + jfjoch_client.Configuration(host="http://localhost:5232", access_token=""))) +api.post_start(jfjoch_client.DatasetSettings(..., tokens=[""])) +api.get_statistics_data_collection() # sends Authorization: Bearer +``` + +The per-model pages are generated as well and are linked from the two pages above. They are built +with the site but kept out of the navigation sidebar on purpose — sixty generated reference pages +would otherwise be most of it. + +```{toctree} +:hidden: +:glob: + +python_client/README +python_client/docs/* +``` diff --git a/_sources/RELEASE_CONTENTS.md.txt b/_sources/RELEASE_CONTENTS.md.txt new file mode 100644 index 000000000..528d90f65 --- /dev/null +++ b/_sources/RELEASE_CONTENTS.md.txt @@ -0,0 +1,199 @@ +# Release contents + +This page describes **what a Jungfraujoch release ships and what each artefact needs on the target +machine** — which CPU instruction set the binaries were compiled for, which CUDA toolkit they were +built against, and which runtime libraries are bundled rather than expected from the host. + +The artefacts in the table below are built and published by the continuous-integration pipeline +(`.gitea/workflows/build_and_test.yml`) when a tag is pushed. For *how* to install and configure the +result see [Deployment](DEPLOYMENT.md); for the package-repository URLs see +[Linux package repositories](REPOSITORIES.md). + +## Artefacts + +| Artefact | Distributed via | Contains | +| --- | --- | --- | +| `.rpm` / `.deb` packages | [package repositories](REPOSITORIES.md) | The full server stack: `jfjoch` (broker, frontend, FPGA and detector tools), `jfjoch-writer`, `jfjoch-viewer` (incl. the XDS plugin), `jfjoch-driver-dkms`, and `rugnux` (offline analysis, independent of the rest) | +| `jfjoch_viewer--linux-cuda.tgz`, `...-linux-cpu.tgz` | Gitea release page | Portable Linux viewer: `jfjoch_viewer`, its desktop entry, icon and D-Bus service, and the license notices | +| `jfjoch-viewer--win64-cuda.exe`, `...-win64-cpu.exe` | Gitea release page | Windows installer for `jfjoch_viewer`, plus the Qt runtime | +| `jfjoch-viewer--macos-arm64.dmg` | Gitea release page | macOS disk image with `jfjoch_viewer.app` (Apple Silicon), the Qt runtime and the license notices inside the app | +| `rugnux--linux-x86_64-cuda.tgz` | Gitea release page | Portable Linux [`rugnux`](RUGNUX.md), the offline analysis CLI, and the license notices. One executable | +| `rugnux--linux-aarch64-cuda.tgz` | Gitea release page | The same, cross-built for 64-bit Arm — NVIDIA GH200 and DGX Spark | +| `rugnux--win64-cuda.zip` | Gitea release page | The same for Windows, plus the cuFFT DLL | +| `rugnux--macos-arm64-cpu.tgz` | Gitea release page | The same for macOS on Apple Silicon, CPU only | +| `jfjoch-writer` `.rpm` / `.deb` | Gitea release page | The writer alone, for a file-writing machine without the rest of the stack | +| `libjfjoch_xds_plugin.so.` | Gitea release page | XDS HDF5 read plugin (built on RHEL 8); see [Integration with MX software](SOFTWARE_INTEGRATION.md) | +| `jfjoch-client` | [PyPI](https://pypi.org/project/jfjoch-client/) and the Gitea PyPI index | Generated Python OpenAPI client | +| Documentation | [Read the Docs](https://jungfraujoch.readthedocs.io) and the `gitea-pages` branch | This documentation set | + +The FPGA firmware (`.mcs`) images are attached to the release as well. The firmware is stable and is +carried from version to version, and is rebuilt with Vivado (see [FPGA smartNIC](FPGA.md)) when it +needs to change — so a card keeps its image across a software upgrade unless the release notes say +otherwise. + +## CPU instruction set + +The architecture flags live in the CI configuration rather than in `CMakeLists.txt`, so a site +building from source picks its own (`x86-64-v4` on an AVX-512 cluster, `-march=native`, or the plain +baseline the compiler defaults to). The released binaries are compiled to a fixed floor: + +| Release | Flags | Minimum CPU | +| --- | --- | --- | +| Linux (all packages, and the portable `.tgz`) | `-march=x86-64-v3 -flto=auto` | AVX2 + FMA + BMI2 — Intel Haswell (2013) / AMD Zen (2017) and newer | +| Windows installer | `/arch:AVX` | AVX — Intel Sandy Bridge (2011) / AMD Bulldozer and newer | +| macOS `.dmg` and `rugnux` `.tgz` | none (the compiler's default Apple Silicon target) | Any Apple Silicon Mac — M1 and newer | + +The Windows floor is lower because MSVC has no spelling for the `x86-64-v2` level; `/arch:AVX` is +the nearest one and implies SSE4.1/4.2, which is what actually matters — without it Eigen has no +vectorised `round` and falls back to a libm call per element. Link-time optimisation is applied on +Linux only. The macOS artefacts need no flag: the Apple compiler's default target is already the +M1, the oldest Apple Silicon chip. + +A binary will fault with an illegal instruction on a CPU below its floor. If you must run on older +hardware, build from source without the flags. + +## Operating-system floor + +The `.rpm` / `.deb` packages are built per distribution (RHEL/Rocky 8 and 9, Ubuntu 22.04 and 24.04) +and are tied to it. The portable viewer `.tgz` and the **x86_64** `rugnux` `.tgz` are built on +RHEL 8, the oldest supported distribution, so their glibc floor is low enough to run on any newer +Linux — that is what they are for, and why they replace the per-distro packaging of those programs +on the release page. The **aarch64** `rugnux` `.tgz` is the exception: it is cross-built against +Ubuntu 24.04, so it needs glibc 2.39 or newer (which DGX OS 7 and any current Arm server distribution +have). The Windows installer is built and verified on Windows 11. The macOS artefacts need +**macOS 13 (Ventura) or newer** — the floor of the Qt 6.11 they bundle — and an **Apple Silicon** +Mac; see [macOS](#macos) below. + +**The portable archives have no top-level directory.** They unpack straight into `bin/` and +`share/`, so always extract them into a directory of their own (`tar xzf … -C /opt/rugnux-`) +rather than into a working directory. + +## CUDA and non-CUDA builds + +Every Linux and Windows binary artefact is released in **two variants**, `cuda` and `cpu` +(the macOS ones exist only as `cpu`: there is no CUDA on macOS). The CUDA variant adds +the GPU fast-feedback indexer (`ffbidx`), the GPU FFT indexer and GPU image processing; the CPU-only +variant runs the same pipeline on the CPU with the FFTW indexer, at much lower throughput. + +The CUDA toolkit used is the one on the corresponding build machine: **CUDA 12** for the RHEL 8 +packages, **CUDA 13** for RHEL 9, Ubuntu and Windows. The major version is part of the artefact and +repository name, so a download is self-identifying. Building from source needs CUDA 12.8 or newer. + +**A CUDA build does not require a CUDA machine.** Jungfraujoch asks how many CUDA devices are +present at start-up and treats "none" (including "no driver installed") as zero GPUs, falling back +to the CPU path. So a CUDA build starts and runs correctly on a machine with no NVIDIA GPU at all. +What each artefact has to find at run time differs: + +- **Portable Linux `.tgz`** — nothing. The CUDA runtime, the fast-feedback indexer **and cuFFT** are + all linked statically, so each archive is a single executable that depends on nothing but the C + and C++ runtimes. On a GPU machine the NVIDIA driver is the only NVIDIA component needed. +- **Windows installer and `.zip`** — the CUDA toolkit ships no static cuFFT for Windows, so the + cuFFT DLL is **part of the distribution**, next to the executable. No CUDA toolkit is needed. +- **`.rpm` / `.deb`** — these deliberately keep cuFFT dynamic, so that one dependency is managed + centrally with the rest of CUDA. Install the cuFFT package alongside, or use the `nocuda` + repositories on a machine where CUDA is not wanted. + +Static CUDA linkage makes the Linux CUDA artefacts substantially bigger than the CPU ones, and the +Windows cuFFT DLL is ~256 MB — which is the other reason for shipping both variants. + +On a machine with an NVIDIA GPU, take the CUDA variant: only that one uses the GPU. + +## GPU generations and the NVIDIA driver + +A CUDA variant carries compiled device code for a fixed set of GPU generations, and which +generations those are follows from the CUDA toolkit it was built with. The CUDA runtime is linked +statically, so the only NVIDIA component the target machine has to supply is the **driver** — there +is no CUDA-toolkit version requirement on the host. + +| Artefact | CUDA toolkit | GPU generations | Minimum driver | +| --- | --- | --- | --- | +| RHEL 8 packages, portable viewer `.tgz`, x86_64 `rugnux` `.tgz` | 12.9 | Volta (V100) through Blackwell: `sm_70`, `75`, `80`, `86`, `89`, `90`, `100`, `120`, `121` | 525.60.13 | +| RHEL 9 and Ubuntu packages, Windows installer, Windows `rugnux` `.zip` | 13.x | Turing (T4) through Blackwell: the same list **without** `sm_70` | 580.65.06 (Linux), R580 (Windows) | +| aarch64 `rugnux` `.tgz` | 13.x | `sm_90` (GH200) and `sm_121` (DGX Spark) only | 580.65.06 | +| any `cpu` / `nocuda` variant, and everything for macOS | — | — | none | + +The aarch64 build is cross-compiled and verified in CI to be Arm, self-contained and to carry both +GPU targets, but it is **not exercised on hardware** — CI has no GH200 or Spark runner. + +**A V100 needs the CUDA 12 build.** CUDA 13 dropped offline compilation for Volta, and the PTX that a +fatbin also carries only ever JIT-compiles *forwards*, so a CUDA 13 artefact contains nothing a V100 +can execute: every kernel launch fails with *no kernel image is available for execution on the +device*. On a V100 host take the RHEL 8 packages or the portable Linux `.tgz`. Nothing older than +Volta is supported. + +Newer GPUs never need a newer build — the highest generation in the list ships PTX as well as SASS, +which the driver JIT-compiles for a GPU that came out after the release. + +The minimum driver above is the floor for the whole CUDA *major* version, which is what applies here +because the CUDA runtime is statically linked +([CUDA minor version compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html)). +Newer drivers are always fine; they are backward compatible. A driver from the same release as the +build toolkit (575.57.08 for the CUDA 12.9 build, 610.43.02 for a CUDA 13.3 one) additionally rules +out the single caveat of minor version compatibility — a call into a driver API newer than the +installed driver, which fails with `cudaErrorCallRequiresNewerDriver`. + +## Windows installer + +The Windows artefacts are the `jfjoch_viewer` installer and the separate `rugnux` `.zip`; the rest +of Jungfraujoch (broker, receiver, FPGA host, detector control) is Linux-only. + +The toolchain bounds of the released installer are: + +- **Visual Studio 2026** with the C++ (MSVC) toolset. MSVC is not optional — CUDA on Windows builds + through it — and it is what the release is compiled with. +- **CUDA Toolkit 13.3** for the `cuda13` variant. +- **Qt 6.11** for MSVC (`msvc2022_64`), including Qt Charts. +- Ninja as the generator; every third-party library is fetched and built by the configure. + +The installer is generated with NSIS and **bundles the Qt runtime** (via `windeployqt`) and, on the +CUDA variant, the cuFFT DLL — so the end user installs neither Qt nor a CUDA toolkit. The two +variants share an install directory and Start Menu group and replace each other (CUDA is a strict +superset); they are told apart by the installer filename and the Add/Remove Programs entry: + +| Build | Installer file | Add/Remove Programs | +| --- | --- | --- | +| CUDA (default) | `jfjoch-viewer--win64-cuda.exe` | `Jungfraujoch (CUDA)` | +| CPU-only | `jfjoch-viewer--win64-cpu.exe` | `Jungfraujoch (CPU)` | + +To build the viewer yourself on Windows, see +[jfjoch_viewer ▸ Building from source on Windows](JFJOCH_VIEWER.md#building-from-source-on-windows). + +## macOS + +The macOS artefacts are the `jfjoch_viewer` disk image and the `rugnux` `.tgz`; as on Windows, the +rest of Jungfraujoch is Linux-only. Both are **CPU-only** — macOS has no CUDA, so indexing uses the +FFTW indexer and the whole pipeline runs on the CPU — and both are built for **Apple Silicon only**. +They need **macOS 13 (Ventura) or newer**. + +**Intel Macs are not supported.** No Intel (`x86_64`) code is built for macOS, and Rosetta cannot +help here: it runs Intel programs on Apple Silicon, not the other way round. + +`jfjoch-viewer--macos-arm64.dmg` holds `jfjoch_viewer.app`; open the image and drag the +app onto the *Applications* shortcut next to it. The app is self-contained: the Qt frameworks are +inside it, and so are the license notices (`Contents/Resources`), which is where a macOS app keeps +them. + +**The release is not notarized yet** (that needs an Apple Developer account), so macOS refuses to +open a downloaded copy the first time. On macOS 15 and newer: try to open it once, then allow it in +*System Settings ▸ Privacy & Security* with *Open Anyway*. On macOS 13 and 14, Control-click the app +and choose *Open*. Alternatively, clear the download flag in Terminal: + +``` +xattr -dr com.apple.quarantine /Applications/jfjoch_viewer.app +``` + +The same applies to the `rugnux` binary from the `.tgz` — see +[Installing Rugnux](RUGNUX_INSTALL.md#from-the-release-archive). + +The toolchain of the released macOS artefacts is Xcode (Apple Clang) and Qt 6.11 for macOS, +including Qt Charts; to build the viewer yourself see +[jfjoch_viewer ▸ Building from source on macOS](JFJOCH_VIEWER.md#building-from-source-on-macos). + +## Licenses + +Every package variant carries the project license, the third-party manifest and the verbatim +license texts of the bundled dependencies, each package under a directory of its own — +`share/doc/jfjoch_broker`, `jfjoch_writer`, `jfjoch_viewer`, `jfjoch_rugnux`, `jfjoch_driver_dkms` — +so that no two packages claim the same path and they can be upgraded independently. The macOS +viewer is the exception: its notices are inside the app, in `jfjoch_viewer.app/Contents/Resources`. See +[Third-party software notices](THIRD_PARTY_NOTICES.md). diff --git a/_sources/REPOSITORIES.md.txt b/_sources/REPOSITORIES.md.txt new file mode 100644 index 000000000..b6e88729d --- /dev/null +++ b/_sources/REPOSITORIES.md.txt @@ -0,0 +1,58 @@ +# Linux package repositories +For convenience, we are providing package repositories, in versions including and excluding CUDA linking. +We recommend installing the Jungfraujoch viewer from a `nocuda` repository (published in the `slsdet8` flavour only) and the remaining packages from a `cuda12`/`cuda13` repository. + +The repository name encodes two choices: the [slsDetectorPackage](DETECTORS.md) version the packages +were built against (`slsdet8` = 8.0.2, `slsdet9` = 9.2.0 — it must match the detector firmware) and +whether CUDA is linked in. What ends up inside each package, and what it needs on the target +machine, is described in [Release contents](RELEASE_CONTENTS.md). + +## RHEL based systems + +For RHEL systems we provide the following repositories: + +| RHEL version | slsDetectorPackage | CUDA | Repository file | +|--------------|--------------------|------|------------------------------------------------------------------------| +| 8.x | 8.0.2 | 12.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo | +| 8.x | 9.2.0 | 12.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12.repo | +| 8.x | 8.0.2 | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda.repo | +| 9.x | 8.0.2 | 13.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13.repo | +| 9.x | 9.2.0 | 13.x | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13.repo | +| 9.x | 8.0.2 | - | https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda.repo | + +To install the repository, run: + +```bash +dnf config-manager --add-repo https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12.repo +``` +RPMs are signed by the Gitea package registry as they are uploaded. If your system cannot verify the +signature, set `gpgcheck=0` in the repository file or install with `--nogpgcheck`. + +We provide the following packages in the repository: +* jfjoch — broker, web frontend, FPGA and detector command-line tools +* jfjoch-driver-dkms — PCIe kernel-module source, built by DKMS +* jfjoch-writer — HDF5 writer service +* jfjoch-viewer — desktop viewer and the XDS plugin +* Rugnux — [offline analysis CLI](RUGNUX.md), independent of the acquisition stack + +Note that `rugnux` is named without the `jfjoch-` prefix, matching the program and the release +archive. Up to 1.0.0-rc.163 it was part of `jfjoch-viewer`; the package declares that move, so +installing it upgrades an older viewer rather than colliding with it. + +## Ubuntu based systems + +For Ubuntu systems, we also provide the following repositories: +``` +sudo curl https://gitea.psi.ch/api/packages/mx/debian/repository.key -o /etc/apt/keyrings/gitea-mx.asc +echo "deb [signed-by=/etc/apt/keyrings/gitea-mx.asc] https://gitea.psi.ch/api/packages/mx/debian $distribution $component" | sudo tee -a /etc/apt/sources.list.d/gitea.list +sudo apt update +``` + +`$distribution` uses Ubuntu names `jammy` (22.04) and `noble` (24.04). `$component` can be set to either `cuda13` or `nocuda`. +Only slsDetectorPackage 8.0.2 is built for Ubuntu. + +The same five packages as above are provided: `jfjoch`, `jfjoch-driver-dkms`, `jfjoch-writer`, +`jfjoch-viewer` and `rugnux`. Up to 1.0.0-rc.160 the first of them was misnamed `jfjoch-jfjoch`; the current +package replaces it, so `apt upgrade` handles the rename. + +Ubuntu packages currently undergo only very limited testing. diff --git a/_sources/RUGNUX.md.txt b/_sources/RUGNUX.md.txt new file mode 100644 index 000000000..ef8681a76 --- /dev/null +++ b/_sources/RUGNUX.md.txt @@ -0,0 +1,156 @@ +# Rugnux + +`rugnux` is the **offline** crystallographic data-analysis tool of Jungfraujoch — the +data-processing half of the system (see [Naming](NAMING.md) for where the name comes from). +It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, +geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the +results to a `_process.h5` file, plus reflection files (`.mtz`/`.cif`/`.hkl`) when merging is +requested. + +It runs the *same* analysis code as the online and interactive tools, just driven from the +command line over a file rather than a live detector stream. + +``` +rugnux {} +``` + +Run it with no arguments to print the usage. + +> **Note.** `rugnux` is under very active development. This page describes the tool and +> its options at a high level; the authoritative, always-current list of options is the program's +> own usage message — run `rugnux` with no arguments. + +```{contents} On this page +:local: +:depth: 2 +``` + +## Quick start + +Four commands cover most of what people ask of `rugnux`. Each takes the **master** file of the +dataset — one written by Jungfraujoch, a DECTRIS EIGER master, or an NXmx master written by another +facility's toolchain; a PILATUS miniCBF, marCCD or SMV sweep works too (see [What Rugnux reads](RUGNUX_FORMATS.md)) +— and names its output files from `-o`: + +``` +# 1. everything from the data - index, integrate, scale and merge with the defaults +rugnux -o myrun dataset_master.h5 + +# 2. with a reference dataset of the same crystal form: it fixes the space group and the cell, +# resolves the indexing ambiguity, and hands over its R-free set +rugnux -o myrun -z reference.mtz dataset_master.h5 + +# 3. with a known structure: R-work / R-free and sigma_A-weighted 2mFo-DFc / mFo-DFc maps +rugnux -o myrun --model model.pdb dataset_master.h5 + +# 4. with the space group and the cell pinned (-S takes either spelling: P43212 or 96) +rugnux -o myrun -S P43212 -C 79,79,38,90,90,90 dataset_master.h5 +``` + +Parallelism needs no asking for: a run already uses the machine's threads. `-N` is there to *limit* +that, or to lift the per-image loop's default ceiling of 16 workers per GPU. + +Nothing more is needed to pick the workflow: a dataset carrying a **goniometer axis** is processed +as a rotation sweep, one without as **independent stills**, and scaling and merging run by default +in both. A rotation run that merges — the default — leaves eight files next to each other: + +``` +myrun.mtz merged intensities + French-Wilson amplitudes, for CCP4 / phenix +myrun.cif the same, as mmCIF - the self-describing format, and what to deposit +myrun.hkl the scaled observations unmerged, as SHELX HKLF 4 - for SHELXL, SHELXC / ANODE +myrun_unmerged.mtz every observation before scaling, for pointless / aimless / careless - + the largest file of the run (--no-export-unmerged skips it) +myrun_P1.mtz the same observations merged in P1, so a wrong space-group call can be + re-merged or re-refined without reprocessing (--no-p1-crosscheck skips it) +myrun_report.txt what the run determined: cell, space group, statistics, warnings +myrun_plot.txt one row per image, for plotting how the crystal behaved over the sweep +myrun_detector.jpg the detector with the pixel mask and the beam-stop shadow drawn on it +``` + +The two MTZ extras are most of the bytes a run writes — worth knowing when sizing a scratch +directory for a campaign, and both have off switches. + +Read `myrun_report.txt` first: it says which space group was chosen and on what evidence, how far +the data go, and anything that needs attention. + +A few things worth knowing before reaching for more flags: + +- **The written reflections stop where CC1/2 falls through 0.30.** Every reflection file is + resolution-trimmed automatically (`--resolution-cutoff cc-logistic`, one shell past the crossing); + `--resolution-cutoff off` keeps the full measured range, `--scaling-high-resolution` fixes the + limit by hand. A Rugnux file reaching less far than another program's on the same data is usually + this default at work, not lost data. +- **Rotation data are best left de novo.** Pinning the cell and space group (recipe 4) is the + normal thing to do for **serial stills**, where the `ffbidx` indexer needs a cell; on a rotation + sweep it tends to *degrade* low-symmetry cases, so prefer recipe 1 and let the run determine both + (see [Rotation data](RUGNUX_TUTORIAL.md#rotation-data)). `-S` takes a Hermann-Mauguin symbol (`P43212`) or a + space-group number (`96`), whichever is to hand. +- **Anomalous data are there without `-A`.** A rotation merge always keeps the Bijvoet split: a + default run's `.mtz` carries `I(+)`/`I(-)` beside `IMEAN`, and its `.hkl` every observation at the index it was measured at — + `FRIEDELS_LAW= TRUE` in the report says how the *statistics* were counted, not that the signal was + averaged away. What `-A` changes is the counting basis and the error model: each hand becomes a + merged observation of its own, so multiplicity, completeness and ⟨I/σ⟩ are counted anomalously and + the sigmas are refitted on the Bijvoet-separated merge. Reach for it for anomalous statistics; the + signal itself is in the file either way. (Stills merges carry no split by default — there `-A` is + what creates one.) +- **A model names the enantiomorph.** Where the data accept the model — it is tested against a + null of the same model in random orientations, and `MODEL_FIT=` in the report says the verdict — + `--model` settles which of P41212 and P43212 the merged + reflections are *labelled* with — a choice no merged intensity can make. It is a label and nothing + more: the two groups have the same rotation operations, so no reflection moves, and in particular + I(+) and I(-) are left exactly as measured. Whether the model agrees with the data about the hand + is then a real question, and the anomalous difference map answers it — a run says so when the + density at the model's atoms comes out inverted. +- **`-z` and `--model` overlap but are not the same.** A reference MTZ steers the processing from + the start; a model scores the merge and settles the frame it is written in — where the data + accept it; a model they reject changes nothing. Either resolves an + [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity), which on serial data decides whether the merged + intensities are usable at all. +- **Small molecules need no flag either.** Short axes, spots wider than the integration disk and + glide-plane space groups are handled by the default run, and `myrun.hkl` goes straight to SHELXT / + SHELXL ([Small-molecule data](RUGNUX_TUTORIAL.md#small-molecule-data)). +- **`--scaling-high-resolution `**, where the resolution is already known, sharpens both the + space-group search and the error model. +- **A run wants memory in proportion to what it integrates**, not to the detector: 2.5-14 GB of + host RAM and 3-7 GB on the card over the datasets measured, both peaking in scaling and + merging. [Installing Rugnux ▸ Memory](RUGNUX_INSTALL.md#memory) has the table and the two + flags that lower it. A very large cell needs far more, and a CPU-only build most of all — tens of + GB of host RAM ([Very large unit cells](RUGNUX_INSTALL.md#very-large-unit-cells)). +- Everything else is in [Running Rugnux](RUGNUX_TUTORIAL.md#running-rugnux) and the full + [Command-line options](RUGNUX_ADVANCED.md#command-line-options). + + +## The rest of the manual + +One page per job, so the answer needed is near the top of a short page: + +- [What Rugnux does](RUGNUX_OVERVIEW.md) — the pipeline from images to merged reflections, + in order. Read this one first. +- [Installing Rugnux](RUGNUX_INSTALL.md) — packages, the release archive, GPU drivers, building + from source, hardware. +- [What Rugnux reads](RUGNUX_FORMATS.md) — will it open your data: NXmx / EIGER masters, PILATUS + miniCBF, marCCD and SMV (ADSC, Rigaku d\*TREK) sweeps, one sweep per input. +- [Running Rugnux](RUGNUX_TUTORIAL.md) — a first run in detail, rotation, serial and small-molecule + data, and every file a run writes. +- [Rugnux with other programs](RUGNUX_INTEGRATION.md) — the reflection-file conventions, the + unmerged export, and worked command lines for phenix, REFMAC, POINTLESS / AIMLESS, careless, + Phaser, SHELXC/D/E and, for small molecules, SHELXT / SHELXL. +- [The results report](RUGNUX_REPORT.md) — the `KEY= value` interface, sweep quality and the + anisotropy section. +- [Advanced usage](RUGNUX_ADVANCED.md) — reference data and the indexing ambiguity, model + validation, re-merging, and the full command-line option tables. +- [Detector calibration](RUGNUX_CALIBRATION.md) — the geometry from a calibrant's powder rings + (`--mode calibration`). +- [CPU/GPU data analysis](CPU_DATA_ANALYSIS.md) — the algorithms behind all of it. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| [`jfjoch_viewer`](JFJOCH_VIEWER.md) | Interactive, on-screen exploration | Qt desktop application | On screen; a processing job can write the same files as `rugnux` | +| **`rugnux`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | + +Use `rugnux` to re-analyse data after acquisition, to experiment with processing +parameters, or to produce merged intensities for downstream structure solution. + diff --git a/_sources/RUGNUX_ADVANCED.md.txt b/_sources/RUGNUX_ADVANCED.md.txt new file mode 100644 index 000000000..9571ec741 --- /dev/null +++ b/_sources/RUGNUX_ADVANCED.md.txt @@ -0,0 +1,427 @@ +# Advanced Rugnux + +The machinery behind a run that needs more than the defaults: external reference data, model +validation, re-merging without re-integration, and the full option tables. + +```{contents} On this page +:local: +:depth: 2 +``` + +## Reference data and the indexing ambiguity + +### What a reference does (`-z`) + +`-z reference.mtz` supplies **known intensities of the same crystal form** — a previously merged +dataset, or `F-model` amplitudes computed from a structure. The reference can equally be a PDB +structure-factor file (`-z 1abc-sf.cif`, SF-mmCIF), gzipped or not; the format is taken from the +file's content. From an mmCIF the first merged reflection block with a usable column is read (the log +names it), its columns under the MTZ labels `gemmi cif2mtz` gives them (`IMEAN`, `FP`, `FC`, …), and +its R-free set from `_refln.status` (`f` free, `o` working), or from `_refln.pdbx_r_free_flag` where +the status marks no test set. It is read once, before processing starts, and used for four things: + +- **It fixes the space group and the unit cell** the run works in, unless `-S` / `-C` override them. + The cell is a *soft* reference: indexing may still drift within tolerance, so a small mismatch + between reference and data is absorbed rather than rejected. +- **It fixes the axes the files are written on.** The group is taken in the setting the reference + is written in (P 2 21 21 stays P 2 21 21, not only "number 18"), and once the run has merged, every + description of its lattice on the reference's axes - axis permutations, sign flips, I- against + C-centred - and every alternative indexing on top is correlated with the reference; the best one is + re-merged and written. `REFERENCE_OPERATOR=` names the operator (`h,k,l` where the data were + already on the reference's axes), `REFERENCE_CC=` and `REFERENCE_MATCHED_FRACTION=` how well the + result matches. +- **It resolves the indexing ambiguity** (below) — the one thing the data cannot settle for + themselves. +- **It hands over its R-free test set**, where the file carries one, so every dataset of a campaign + is scored on the same free reflections - in the reference's frame, and only where the merge + matches the reference (CC at least 0.5 over at least half of the merged reflections in its + resolution range). A reference that does not is reported (`REFERENCE_MISMATCH`), and the merge keeps + its own test set; `REFERENCE_FREE_FLAGS_INHERITED=` says which happened. +- **It reports CCref**, the correlation of the merged intensities against the reference, + in the statistics table. Stills only — the rotation merge never scores itself against the + reference, and its table shows `-` in that column. + +A reference is **not** a scale anchor. Both workflows scale against their own data — scaling images +against a foreign dataset injects that dataset's systematics — so `-z` never puts the reference's +errors into the intensities. `--reference-column` picks the column to read where the automatic +choice (`F-model`, else `IMEAN`/`I`, else `FP`/`FOBS`/`F`) is not the right one. + +For the second of those four jobs — and only that one — an atomic model does as well: `--model` +computes the intensities it needs from the structure. Where a reference dataset exists, prefer it; +where only a model does, it resolves the ambiguity just the same. + +### The indexing ambiguity + +Some crystals can be indexed in **more than one way, each equally valid geometrically, and each +giving different merged intensities**. This happens whenever the lattice is more symmetric than the +crystal: in P3, P4, P6, P31, C2 and their relatives (*merohedral*), and also where the +cell is metrically more symmetric than the Laue class by accident (*pseudo-merohedral*, up to 2° of +obliquity). The alternatives are related by the crystal's **twin laws** — reindexing operators such +as `k,h,-l`. + +Nothing in the data breaks the tie: the merge is equally self-consistent either way, so which +solution comes out is arbitrary. What that costs depends on the workflow: + +- **Rotation.** The whole sweep is one lattice, so the whole dataset lands in one indexing, picked + at random. The merge itself is sound; it may simply be the *other* solution from an earlier + dataset of the same crystal form, and the two cannot be combined, compared or phased against the + same model. +- **Serial stills.** Every crystal is indexed independently, so a run mixes both indexings into one + merge. That is not a labelling matter — reflections that are not symmetry mates get averaged + together, and CC1/2, Rmeas and the anomalous signal all degrade. + +Every run that merges tests for the ambiguity, and where it exists and nothing resolves it, says so +in the log and in the report's warnings: + +``` +Indexing ambiguity: this cell / space group admits alternative indexing (reindex operator(s): +-h,-k,l). Serial-stills crystals are indexed in one hand at random, and rugnux can only break this +against an external reference. WITHOUT one the merge mixes the hands and CC1/2 is degraded - supply +a reference MTZ (-z) or a model (--model, which needs -C and -S here) to resolve it. +``` + +Where something does resolve it, the run says so instead — `Indexing ambiguity present (reindex +operator(s): -h,-k,l); resolved against the supplied model`. + +How to resolve it: + +| Situation | What to do | +| --- | --- | +| **Rotation**, a reference dataset exists | `-z reference.mtz`. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the *hkl* labels change; the cell does not | +| **Serial stills**, a reference dataset exists | `-z reference.mtz`. Resolved **per image**, at integration time, by correlating each crystal's intensities with the reference — so the merge never mixes hands in the first place | +| **Rotation**, only a model | `--model model.pdb`. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (`MODEL_FIT= ACCEPTED`) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (`MODEL_INDEXING_MARGIN_SIGMA`). It is the **data** that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model's indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand | +| **Serial stills**, only a model | `--model model.pdb`, with the cell and space group given (`-C` / `-S`). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be — the ambiguity has to be broken at integration time, and a model can supply the intensities to break it with | +| Neither | The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame, for stills a degraded one | + +Two things to know about where the choice lands: + +- **`--mode scale` cannot repair a stills run after the fact.** The per-image test happens at + integration time, so a `_process.h5` whose images were integrated without a reference — an MTZ or a + model — has already lost the distinction, and no re-merge brings it back. On rotation data, where + the ambiguity is one choice for the whole dataset, `--mode scale --model model.pdb` does resolve it. +- **The choice reaches the written reflections**, not only the R-factors and the maps: the merged + `.mtz` / `.cif` / `.hkl` (and `_unmerged.mtz`, where it is asked for) are written in the indexing + the model or the reference settled on, so the file can be refined against that model as it stands. + +Two things the indexing ambiguity is **not**: + +- **Not the enantiomorph.** P41212 versus P43212 (or + P31 versus P32) leaves the merged intensities *unchanged*, so no amount of + data can choose between them and Rugnux never tries — the run reports the pair it cannot separate. + A model the data accept names it (`MODEL_FIT= ACCEPTED` in the report), and the naming is a + label only: the written reflections take + the model's space group, no reflection moves, and I(+)/I(-) stay exactly as measured — reindexing + by the change of hand would flip every anomalous difference, so it is never done. Whether the + model and the data really agree about the hand is answered afterwards by the anomalous difference + map, which warns when the density at the model's atoms comes out inverted. The + `_process.h5`, whose per-image reflections were written as they were integrated, keeps the group + the run determined and is left alone. +- **Not twinning.** The twin laws are the same operators, but twinning is a property of the crystal + — two orientations diffracting at once — and is reported separately in the report's twinning + section. A crystal can have an indexing ambiguity without being twinned, and usually is not. + +The algorithms behind both are in +[CPU/GPU data analysis ▸ Reference data](CPU_DATA_ANALYSIS_INTEGRATION.md#109-reference-data-fixing-the-space-group-and-resolving-the-indexing-ambiguity). + +## The setting the files are written in + +The space-group search decides which axes carry the screws and the centring on the cell as it was +indexed, whose axes are ordered by length. The files are then written in the **standard setting**, +as XDS and POINTLESS write it (`SETTING_SOURCE=STANDARD`): P2221 and P21212 with the unique axis on +c and a= 90; triclinic keeps the reduced cell. So a crystal +indexed as 52.51 87.87 137.72 in P 2 21 21 is written as 87.87 137.72 52.51 in P 21 21 2. + +What the user gives takes precedence, in this order: a reference MTZ (`-z`, its own setting), a +model that fits (`--model`, its setting), the axis order of a cell given with `-C` +(`SETTING_SOURCE=CELL`), and the setting of a group given with `-S` by a non-standard symbol +(`-S "P 2 21 21"`, `SETTING_SOURCE=SPACE_GROUP`). A group given with `-S` is also placed on the axes +its absences name before anything is merged, so `-S 18` on a cell whose pure two-fold is its shortest +axis puts the screws on the right rows. + +The change is a relabelling made after every decision - nothing measured, and no decision, depends +on it - and it is applied to everything the run writes: the merged `.mtz`/`.cif`/`.hkl`, +`_unmerged.mtz`, `_P1.mtz` and the `_process.h5` (whose per-image reflections carry the reindex +matrix). `SETTING_OPERATOR=` gives it as a reindexing operator on the indices the space group was +determined on, CCP4 style (`k,l,h` is h'=k, k'=l, l'=h); `h,k,l` where nothing moved. + +## Validating against a model (`rugnux --model`) + +Given an atomic model of the same structure, `--model model.pdb` scales the model structure +factors to the merged amplitudes — fitting a flat bulk-solvent contribution and an overall +anisotropic *B* — and reports **R-work / R-free** and the mean 2mFo−DFc density at the atom centres. +It also writes the σA-weighted maps `_2fofc.ccp4` (2mFo−DFc) and +`_fofc.ccp4` (mFo−DFc), and the map-coefficient MTZ `_maps.mtz`, next to the +merged reflections — and, where the merge kept the Bijvoet split (a rotation merge always does), +`_anom.ccp4`, the anomalous difference map whose strongest sites the report names +(`ANOMALOUS_SITE_01`…`10`), and the map's mean height at the model's anomalous scatterers - every +atom from phosphorus up - as one number for the anomalous signal (`ANOMALOUS_SCATTERER_MEAN_SIGMA`). +The structure itself is not refined; the model is re-fractionalized into +the data cell and then placed as **one rigid body**, so a deposited model from a crystal that is not +quite isomorphous still sits where the density is. The placement runs on the GPU where there is one +and on the CPU otherwise, with nothing to set; the two agree to rounding, not bit for bit. + +Because the model moves, the input file no longer describes these maps, so the model **as placed** is +written as `_model.cif` — the input's chains, residues, ligands, waters, B-factors, +occupancies and anisotropic *U*s, at the coordinates the maps were computed from, in the same unit +cell and space group as `.mtz` beside it. The same coordinates are written as +`_model.pdb` as well, because the fragment-screening tools this file exists to feed take a +PDB: PanDDA's per-dataset input is `.pdb` beside `.mtz`, and dimple produces that same +pair. (The PDB is skipped, with a log line, for a cell its fixed-width format cannot hold; the mmCIF +is unconditional.) That is the file to open with the maps, and the one +to hand to REFMAC5 or `phenix.refine`; its starting R-free is the `R_FREE=` the report quotes. +(`.cif` is the merged **reflections** — hence the separate name.) Both are written whenever the +maps are, including for a model the data rejected: the rejection is a result, and it is exactly the +case where someone wants to look at the model in the density. + +The model may be **PDB or mmCIF**, gzipped or not, and the format is taken from the file's own +content rather than from its name — a model downloaded as `.cif`, `.pdb`, `.ent` or with no useful +extension at all is read the same way. A model that cannot be read, or that has no atoms, no unit +cell or no usable space group, does not fail the run: it is logged, and the results report carries a +`WARNING: Model validation did not run: …` line, so a run that silently produced no R-free and no +maps cannot be mistaken for one that was never given `--model`. + +Either way the results report carries a **`5. MODEL VALIDATION`** section: `R_WORK=` / `R_FREE=` +with their reflection counts, `R_MODEL_SHELL_SCALED=` and `MODEL_RADIAL_MISFIT=` beside them (the R +that compares between two runs, and why the other two do not — see +[RUGNUX_REPORT](RUGNUX_REPORT.md)), the bulk-solvent and overall scale parameters, the mean 2mFo−DFc density +at the atom centres, the reindexing operators the written reflections were brought into the model's +frame with, and `MAPS_PREFIX=`; or `MODEL_VALIDATION= NOT_PERFORMED` with +`MODEL_VALIDATION_REASON=` when the model could not be used. A run given no `--model` has no such +section at all. + +It is a *data-quality lens*, independent of the internal statistics: R-free measures the merged +intensities against external truth, where CC1/2 and Rmeas only measure them against +themselves. It also settles the two things merged intensities alone cannot, in two different ways — +though only where the data **accept the model first**: the same model is refitted, and re-placed, from +random orientations about its own centroid, and the real fit has to beat that null (`MODEL_FIT=` in +the report; [§14.5](CPU_DATA_ANALYSIS_DECISIONS.md#145-does-the-model-fit-the-null-it-is-scored-against)). +A model that fits no better than its own random placements decides nothing, and the reflection files +are byte for byte what a run with no model would have written. The +enantiomorph is a **relabelling**: data merged in P41212 against a +P43212 model take the model's space group as the label they are written +under, with no reflection moved and I(+)/I(-) exactly as measured. A merohedral +[indexing ambiguity](#the-indexing-ambiguity) — when no reference MTZ has already fixed it — is a +**reindexing**: the candidate with the lowest R-free is kept — where its lead over the runner-up +beats the lead a random placement of the same model takes — and applied to the **written +reflections** as well as to the R-factors and the maps; otherwise the data keep the indexing they +were merged in. This is distinct from a model written in another *description* of the lattice — other +axes, a different unique axis, I-centred where the run indexed C-centred — or in another point group: +there the model is first put into the data's description (`MODEL_CHANGE_OF_BASIS=`) to be scored. +A model that asserts such a setting is then tested against its random placements like any other +claim, and where it **fits**, the data move instead: the files are written in the model's setting +(`SETTING_SOURCE=MODEL`), and the validation is made once more on those axes so the maps and +`_model.pdb` match them. Where it does not fit, the files stay in the standard setting. +Validation runs before the reflection +files, so the `.mtz` / `.cif` / `.hkl` come out in the model's frame either way — reindexed where +the ambiguity decided, and carrying the model's space group where the hand was adopted. The log names the operator in each case, and for the indexing choice gives the winning +R-free together with the runner-up. + +## Re-scaling and re-merging (`rugnux --mode scale`) + +The `scale` mode re-scales and merges the *already-integrated* reflections stored in a +`_process.h5` file, without re-running spot finding or integration. Use it to re-merge quickly with a +different space group, resolution limit, anomalous setting or outlier rejection. It reuses the same +`-o/-N/-s/-e/-S/-A/-z/--scaling-*` options as the full run, and (unlike the full pipeline) does +not run a space-group search: it merges in the space group and unit cell the file records, and `-S` +/ `-C` override them. A `_process.h5` written before the group was stored carries none, and merges +in P1 unless `-S` says otherwise. + +A reference MTZ (`-z`) is accepted here on **stills** data, where it fixes the space group and cell, +reports CCref and hands over its R-free flags; on rotation data the rotation scaler +declines it and the run stops with a message saying so. `--model` works here exactly as in a full +run: on rotation data, where the [indexing ambiguity](#the-indexing-ambiguity) is one choice for +the whole dataset, it can settle the enantiomorph label and the indexing — under the same fit test +as always — and the re-merged reflections are written in what it settled. What no re-merge can +repair is a **stills** `_process.h5` integrated without a reference: there the ambiguity was +decided per image, at integration time, and the distinction is gone from the stored reflections. + +Where the full run re-seated the lattice — the space group it settled on is in a different setting +from the one each image was indexed in — the file records the change of basis as +`/entry/MX/reindexMatrix`, and `rugnux` applies it on read, so the reflections and the stored cell +describe the same frame. An older file that was affected by this cannot be repaired (the matrix is +not recoverable after the fact); such a file now stops with a message naming both cells and the +exact `-S`/`-C` override to merge it in its own setting, instead of failing inside the merge. + +## Command-line options + +General: + +| Option | Description | +| --- | --- | +| `-o, --output-prefix ` | Output file prefix (default: `output`) | +| `-N, --threads ` | Number of worker threads (default, and for any value ≤ 0: all hardware threads). Some stages take fewer, because past a point more workers make them slower: the per-image loop of `--mode mx` uses at most 16 per GPU unless `-N` was given a positive value, and first-pass spot finding and the beam-stop pre-scan have ceilings of their own that `-N` does not lift. Scaling, merging and the space-group search use the full count | +| `-s, --start-image ` | First image to process (default: 0) | +| `-e, --end-image ` | Last image to process (default: all) | +| `-t, --stride ` | Process every *n*-th image (default: 1) | +| `-v, --verbose` | Verbose output | + +Mode — `--mode ` (default `mx`): + +| Value | Description | +| --- | --- | +| `mx` | Full analysis — spot finding, indexing, integration and merging | +| `azint` | Only azimuthal integration (no spot finding/indexing); writes `_process.h5` | +| `scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) | +| `calibration` | Determine the detector geometry from powder rings; writes `.poni` and `.json` | + +Calibration (`--mode calibration`): + +| Option | Description | +| --- | --- | +| `--calibrant ` | Powder standard: `lab6` \| `agbh` \| `ceo2` \| `si` \| `ice` (default `lab6`, case-insensitive) | +| `--calibration ` | How the rings are measured: `rings` \| `spots` (default `rings`; see above). `rings` defaults `--azim-phi-bins` to 32 | +| `--no-refine-tilt` | Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance, for a calibration handed to a program that cannot express a tilted detector (XDS) | + +Detector mask: + +| Option | Description | +| --- | --- | +| `--detect-beam-stop[=N\|off]` | Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. **On by default** (60 images); `=off` disables. Reflections behind the stop are attenuated but not flagged, so they integrate low with a plausible sigma and no existing rejection catches them | + +Geometry: + +| Option | Description | +| --- | --- | +| `--beam-center-check[=off]` | Measure the beam centre from the isotropy of the scattered background on **every** run, report how far the file's value is from it (against how right this geometry needs it to be), and on rotation data index a **second first pass** at the measured centre. The fit reads the projection `--detect-beam-stop` already builds, so it costs no extra frames. **On by default**; `=off` disables it, and with it the measured centre's part in the post-refinement bound. The measured centre is adopted where the file's centre indexes nothing and the measured one indexes a majority; where the two return cells related by an integer volume factor (2 to 4) and the measured centre's cell, larger or smaller, carries materially more of the pooled validation spots against its own chance level; and, on a two-pass run, where the first pass is run at both centres and the measured one merges better — done where the two give the same cell in different metric symmetries, where they agree only once less of each frame is read, and where they agree but lie further apart than the geometry absorbs and the file's centre merges inconsistently in the lowest-resolution shell. Otherwise the file's centre is kept. Still runs when `--beam-x`/`--beam-y` are given. See [§1.4](CPU_DATA_ANALYSIS_IMAGE.md) | +| `--beam-center-search[=N\|off]` | After a rotation first pass that indexes fewer than half the validation frames, step the centre a pixel at a time out to N px along **each** detector axis, re-finding the spots at every rung, and keep the first rung that indexes a majority. **On by default** (12 px); `=off` disables. It runs only after a pass that has already failed, so a run that indexes never pays for it. Both detector directions are searched: a centre error *across* the spindle collapses the indexed fraction and announces itself, while one *along* it holds the frame count up and quietly returns an axis harmonic, so a rung whose cell is an integer or √3 volume multiple of the starting one is refused. Skipped where the background places the centre further away than N px | +| `--estimate-beam-center` | Measure the beam centre before indexing and use it in place of the file's: from the symmetry of the spot positions where the sweep reaches half a turn, from the scattered background otherwise. Adopted only where its sigma is within the larger of 1 px and what this geometry needs, and the move is over three sigma; otherwise the file's value is kept. Ignored with `--beam-x`/`--beam-y`. Off by default | +| `--no-fit-spindle` | With `--estimate-beam-center`, keep the rotation axis given in the file instead of fitting its skew about the beam | + +Spot finding: + +| Option | Description | +| --- | --- | +| `--spot-sigma ` | Noise sigma level for spot finding (default: 4.0) | +| `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | +| `--adaptive-spots` | Self-calibrating detection (**default**, stills and rotation alike): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed `--spot-threshold`, so one setting adapts across datasets (no per-dataset `--spot-threshold`/`--spot-sigma` tuning) | +| `--no-adaptive-spots` | Turn adaptive detection off and use the fixed `--spot-threshold` / `--spot-sigma` finder | +| `--spot-false-pixels ` | Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies `--adaptive-spots`) | +| `--spot-high-resolution ` | High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills | +| `--spot-low-resolution ` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit) | +| `--min-pix-per-spot ` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. | +| `--max-spots ` | Maximum spots kept per image (the strongest ones) and handed to indexing. **If omitted, the budget is measured** on rotation data: the first pass reads how deep into an image's spot list its spots still lie on the lattice it found, and the run keeps that many (never more than 1000). Give a value to pin it. Stills always use the fixed 1000. | +| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** | + +Azimuthal integration (the radial profile behind the per-image ice-ring score). Every *q* here is +*q* = 2π/*d*, in Å⁻¹: + +| Option | Description | +| --- | --- | +| `-q, --azim-q-spacing ` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) | +| `--azim-min-q ` | Minimum Q, 1/Å | +| `--azim-max-q ` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see | +| `--azim-phi-bins ` | Number of azimuthal (phi) bins (default: 1) | +| `--polarization-correction ` | Enable/disable the azimuthal polarization correction | +| `--solid-angle-correction ` | Enable/disable the azimuthal solid-angle correction | + +Indexing: + +A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation +indexing) by default; a dataset without one is processed as independent stills. `--force-still` +overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request +rotation explicitly and pick the pass or lattice. + +The FFT search looks for cell axes between 10 Å (`--fft-min-unit-cell`) and a default **longest +axis of 500 Å**, which has no flag of its own — a reference cell (`-C`) moves both bounds to cover +the cell it names. A de-novo rotation run also tries a second first-pass hypothesis with the short +end lowered to 5 Å, so a small-molecule cell below the 10 Å floor is indexed on its true axes +rather than as a supercell of them; the standard pass's answer stands unless that evidence takes it. An axis beyond the search's reach is **not refused**: the run returns a +plausible shorter sub-cell or an axis harmonic and processes it happily, so a cell that comes out +at a half or a third of the expected long axis should be read as this limit, not as the crystal. +For very long axes the direction grid's angular resolution binds well below 500 Å — see the +[analysis reference](CPU_DATA_ANALYSIS_INDEXING.md#5-fft-indexing-unknown-unit-cell) on FFT indexing. + +| Option | Description | +| --- | --- | +| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | +| `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it | +| `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered | +| `--min-indexed-spots ` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it | +| `-S, --space-group ` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling | +| `-r, --refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) | +| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) | +| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) | +| `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass | +| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see the rotation section) | +| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), `=off` disables | +| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) | + +Indexer choice in brief: `ffbidx` (GPU) refines toward a **known cell** and is best for sparse +serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the +[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS_INDEXING.md) for the algorithms. + +Scaling and merging: + +| Option | Description | +| --- | --- | +| `--no-merge` | Skip scaling and merging, which are on by default; the per-image `_process.h5` is then written instead of the merged files (`_unmerged.mtz` and the results report are still written) | +| `-A, --anomalous` | Anomalous mode: merge each Bijvoet hand as its own unique reflection, so multiplicity, completeness, ⟨I/σ⟩ and the error model are counted anomalously. A default rotation merge already writes `I(+)`/`I(-)` (see [Reflection-file conventions](RUGNUX_INTEGRATION.md#reflection-file-conventions)); `-A` changes the counting basis, not whether the anomalous signal is in the file | +| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | +| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) | +| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below) | +| `--relative-b[=deg]` | rot3d: fit a per-batch relative-*B* beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise) | +| `--simple-stills` | Stills: treat every reflection as a full (*p* = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement | +| `--no-expected-variance-merge` | Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting | +| `--capture-uncertainty ` | rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise) | +| `--min-captured-fraction ` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) | +| `--scaling-high-resolution ` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) | +| `--scaling-low-resolution ` | Low-resolution limit for scaling and merging, Å (default: 50, which is also XDS's own default; 0 removes the limit). The beam stop suppresses air scatter well beyond the shadow it casts, so the background stays depressed across pixels the beam-stop mask leaves open — measured at 30–44% of the field value, recovering only around 50 Å. A background ring in that zone over-estimates the background, so reflections coarser than the limit mostly integrate negative. Beam-stop masking therefore does not make this redundant. A large cell (a few hundred Å) does have real reflections coarser than 50 Å, but on the geometries measured here they fall in that zone and are not usable as integrated; raise the limit only if the low-resolution shell statistics justify it | +| `--resolution-cutoff ` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) | +| `--resolution-cc-target ` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) | +| `--resolution-shells ` | Number of resolution shells in the reported statistics table (default: 9). The bins are equal steps in 1/d² between the lowest- and highest-resolution reflection merged, which is XDS's rule, and 9 is XDS's count — so at the same resolution limits the two tables have the same shells and can be read row for row | +| `--report-resolution [,]` | Also report the merging statistics over this resolution range, Å (either order; `dmax` defaults to the run's own low-resolution limit), as a second table beside the run's own — the `REFRES_*` keys of the [results report](RUGNUX_REPORT.md#the-reference-range-table) — binned from the same merged reflections, so a run can be read against another program's table at that program's range. Report-only: the cut, the scaling, the error model and every decision stay the run's own, and shells finer than the run's own limit are reported as not merged rather than filled from data the run did not keep | +| `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02) | +| `--ice-min-score ` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness | +| `--ice-min-spot-ratio ` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) | +| `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for `rot3d`, off otherwise) | +| `--min-image-cc ` | Per-image CC limit, percent (default: no limit) | +| `--search-min-zeta ` | De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry \|ζ\| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences | +| `--mosaicity ` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed | +| `--scaling-iterations ` | Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle, and a merge that hits the cap is flagged `SCALING_NOT_CONVERGED` (default: 100) | +| `-z, --reference ` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](#the-indexing-ambiguity), hands over the R-free set and reports CCref. Not a scale anchor | +| `--reference-column