From ce87b98280bab5bcef1c376c3f01721a8de18702 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 21:11:14 +0200 Subject: [PATCH 01/12] ModelValidation: run the null's replicates on threads of their own, so their inner passes go parallel The nine null replicates ran under ParallelFor, i.e. on pool workers, and a parallel pass reached from a pool worker runs inline. So the scale fit (FitModelScale's grid) and the rigid-body Jacobian of every replicate but the one on the calling thread ran serially, and the null lasted as long as its slowest serial replicate. Sampled from /proc every 0.5 s through the null of a 1.63 A P2 set (36793 atoms, 4 indexing candidates): CPU use sat at exactly 8.0 cores of 32 for most of its 220 s, falling to 1-4 at the tail; on every set measured the null averaged 6-8 busy cores. The replicates now run on std::async threads outside the pool, so their passes queue on the pool as the real fit's do. Exact: every pass splits its work the same way wherever it runs (ParallelChunks is split on the thread count, not on the caller), so p.hkl, p.mtz, p.cif, p_maps.mtz, p_model.cif are byte-identical and the model-validation section of p_report.txt unchanged on 9hnc, 8sa8, 8xtg, 9fhc, 5epe and 6oel (battery commands, --model). (6oel's report also differs in SUPERCELL_DOUBLED_CELL, 106.695/119.998 vs the equivalent 73.305/60.002 - that line flips between runs of the same code already, and 6oel never reaches the null.) Null wall clock / mean busy cores (shared 16-core/32-thread box with other jobs, so indicative): 9hnc 220 s / 7.8 -> 152 s / 15.9 8sa8 167 s / 8.2 -> 60 s / 23.7 8xtg 88 s / 7.3 -> 46 s / 20.3 9fhc 77 s / 6.7 -> 57 s / 17.7 5epe 45 s / 6.1 -> 36 s / 17.0 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- rugnux/ModelValidation.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/rugnux/ModelValidation.cpp b/rugnux/ModelValidation.cpp index 0138440fb..e90287ff6 100644 --- a/rugnux/ModelValidation.cpp +++ b/rugnux/ModelValidation.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,7 +31,6 @@ #include "../common/CorrelationCoefficient.h" #include "../common/JFJochMath.h" // PI (M_PI is not standard, and MSVC does not define it) #include "../common/Logger.h" -#include "../common/ParallelFor.h" // ParallelFor #include "../image_analysis/scale_merge/ReindexAmbiguity.h" // ReindexReflections #include "../image_analysis/scale_merge/CrystalSetting.h" // CellMappingOperators #include "ModelFFT.h" @@ -749,7 +750,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & // replicates share nothing else, so they run concurrently and the real model - which the // maps and the atom-density readout below are still taken from - is never moved. std::vector null_r_work(NULL_REPLICATES), null_margin(NULL_REPLICATES); - ParallelFor(NULL_REPLICATES, nthreads, [&](int i) { + auto run_replicate = [&](int i) { ModelState rep; rep.st = mdl.st; SetModelPositions(rep.st.models[0], as_read); @@ -762,7 +763,22 @@ ModelValidationResult ValidateAgainstModel(const std::vector & identity_fit = std::move(probe.identity_fit); // replicates are placed against obs } null_r_work[i] = place_and_fit(rep, obs, std::move(identity_fit)).fit.r_work; - }); + }; + // On threads of their own, not on ParallelFor's pool. A parallel pass reached from a pool + // worker runs inline (ParallelFor.h), so there the scale fits and the rigid-body Jacobian of + // every replicate but the one on the calling thread ran serially, and the null lasted as long + // as its slowest serial replicate. From threads outside the pool those passes spread over the + // pool as the real fit's do. Each pass splits its work the same way wherever it runs, so the + // numbers are the same. + std::atomic next_replicate{0}; + std::vector> replicate_threads; + for (size_t t = 0; t < std::min(std::max(nthreads, 1), NULL_REPLICATES); t++) + replicate_threads.push_back(std::async(std::launch::async, [&] { + for (int i = next_replicate++; i < NULL_REPLICATES; i = next_replicate++) + run_replicate(i); + })); + for (std::future &f : replicate_threads) + f.get(); const auto [null_mean, null_sd] = mean_sd(null_r_work); result.fit_tested = true; result.null_replicates = NULL_REPLICATES; From dff1543bd758520f3cb854a39d82f0d896bf2cc9 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 21:11:14 +0200 Subject: [PATCH 02/12] RigidBodyRefine: reuse the residuals of an accepted step when Ceres asks for its Jacobian Ceres evaluates some points twice - the residuals alone to decide whether to take a step, then residuals and Jacobian together at the same point - and every evaluation is a full Fcalc, a solvent mask and a scale fit. The residuals are a function of the placement alone (the zone's bulk solvent is fixed by its first evaluation), so the second computation is replaced by a copy of the first. On the null of a 1.63 A P2 set the replicates' evaluations drop 46/64/72/89/136 -> 44/59/68/82/123, and each saved evaluation is a serial one on the replicate's critical path. With the previous commit the null there goes 152 s -> 87 s (27 busy cores); 8sa8 60 -> 53 s. Exact: p.hkl, p.mtz, p.cif, p_maps.mtz, p_model.cif byte-identical to rc173 and the model-validation report unchanged on 9hnc, 8sa8, 8xtg, 9fhc, 5epe, 6oel (battery commands, --model); rigid-body rotations and translations in the log identical. [ModelValidation] tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- rugnux/RigidBodyRefine.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rugnux/RigidBodyRefine.cpp b/rugnux/RigidBodyRefine.cpp index 8b36f44d5..e60339c57 100644 --- a/rugnux/RigidBodyRefine.cpp +++ b/rugnux/RigidBodyRefine.cpp @@ -214,8 +214,17 @@ public: bool Evaluate(double const *const *parameters, double *residuals, double **jacobians) const override { const int n = num_residuals(); - if (!ev_.Residuals(parameters[0], residuals)) - return false; + // Ceres evaluates every step it accepts twice: the residuals alone to decide whether to take + // it, then residuals and Jacobian together at the same point. The residuals are a function of + // the placement alone, so the second time they are the first time's, copied. + if (!last_residuals_.empty() && std::equal(parameters[0], parameters[0] + 6, last_q_.begin())) { + std::copy(last_residuals_.begin(), last_residuals_.end(), residuals); + } else { + if (!ev_.Residuals(parameters[0], residuals)) + return false; + std::copy(parameters[0], parameters[0] + 6, last_q_.begin()); + last_residuals_.assign(residuals, residuals + n); + } if (jacobians != nullptr && jacobians[0] != nullptr) { const double step = ev_.JacobianStep(); std::vector> shifted(6, std::vector(n)); @@ -244,6 +253,8 @@ private: Evaluator &ev_; std::vector &columns_; size_t nthreads_; + mutable std::array last_q_{}; + mutable std::vector last_residuals_; // at last_q_; empty until an evaluation succeeds }; // The directions in which this space group's origin is free. Translating the whole cell content along From 7eaf0f21b0dc047542e1c80092759133f20847dd Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 21:49:20 +0200 Subject: [PATCH 03/12] ModelValidation: floor the null's spread instead of zeroing the sigma below it [changes verdicts] record_fit set r_work_sigma to 0 whenever the null's sd fell under NULL_SD_FLOOR = 1e-3, and the indexing margin had the same guard at 1e-4. The floor was there so that sd == 0 and sd == 1e-7 do not land on opposite verdicts, but a zero sigma is itself a verdict: DOES NOT FIT. A large, well-measured crystal has a null that narrow for real - the replicates of a random placement agree to 0.0006-0.0010 in R-work over hundreds of thousands of reflections - so the crystal's own model, at R-work 0.17-0.29 against a null of 0.50-0.55, read "+0.00 sigma, the model DOES NOT FIT", and the decisions it gates (setting, indexing, enantiomorph label) were refused. The spread is now floored in the denominator, (mean - R) / max(sd, floor): continuous at sd -> 0, and a null with no spread because the model is too small or symmetric for a rotation to matter still reads near zero, since the real fit is then no better than its own random placements. Changed, measured on the battery commands with --model (every set in the last battery whose sigma was 0.00 for this reason; all three were REJECTED): 8sa8 +0.00 -> +328 sigma, FITS; the model's setting is adopted, R-free 0.2273 -> 0.1747 (deposited 0.1525) 9bn8 +0.00 -> +378 sigma, FITS; reindexed to the model's indexing (+137 sigma), R-free 0.5462 -> 0.1721 (deposited 0.1575) 8xtg +0.00 -> +215 sigma, FITS; the indexing it gates was already the model's ("kept"), so the reflection files are byte-identical and only the verdict changes Unchanged (files byte-identical, sigmas identical): 9hnc, 9fhc, 5epe; no margin null in the battery was under its own 1e-4 floor. [ModelValidation] tests pass. 5epe stays DOES NOT FIT for a different reason, not touched here: in F23 one of the nine random placements lies within the rigid body's reach of the twin-related orientation, walks there (held-out R-free 0.51 -> 0.18), and the null's sd becomes 0.13. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- rugnux/ModelValidation.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rugnux/ModelValidation.cpp b/rugnux/ModelValidation.cpp index e90287ff6..84bbb1d45 100644 --- a/rugnux/ModelValidation.cpp +++ b/rugnux/ModelValidation.cpp @@ -786,13 +786,14 @@ ModelValidationResult ValidateAgainstModel(const std::vector & result.null_r_work_sd = null_sd; auto record_fit = [&](const Placement &p) { - // Guarded at a floor, not at zero: sd == 0 and sd == 1e-7 are one ulp apart and land on - // opposite verdicts, and a null with no spread - a model too small or too symmetric for a - // rotation about its own centroid to move |Fcalc| - is the case that produces it. Measured - // nulls sit near 0.60 with sd ~0.009, so this is nowhere near anything genuine. + // The spread is floored, not trusted down to zero: sd == 0 and sd == 1e-7 are one ulp apart + // and would give sigmas of nothing and of millions. A null with no spread - a model too + // small or too symmetric for a rotation about its own centroid to move |Fcalc| - fits no + // better than its own random placements, so it still reads near zero here. The floor is + // not a gate: a large, well-measured crystal has a null as narrow as 0.0006-0.0010 + // (measured), and a real fit 0.3 below it is hundreds of sigma, not none. constexpr double NULL_SD_FLOOR = 1e-3; - result.r_work_sigma = - null_sd >= NULL_SD_FLOOR ? (null_mean - p.fit.r_work) / null_sd : 0.0; + result.r_work_sigma = (null_mean - p.fit.r_work) / std::max(null_sd, NULL_SD_FLOOR); result.model_fits = result.r_work_sigma >= MODEL_FIT_SIGMA; }; record_fit(real); @@ -810,8 +811,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & result.indexing_margin_null_mean = margin_mean; result.indexing_margin_null_sd = margin_sd; constexpr double MARGIN_SD_FLOOR = 1e-4; - result.indexing_margin_sigma = - margin_sd >= MARGIN_SD_FLOOR ? (indexing.margin - margin_mean) / margin_sd : 0.0; + result.indexing_margin_sigma = (indexing.margin - margin_mean) / std::max(margin_sd, MARGIN_SD_FLOOR); // The margin is judged against the margin a random placement produces, not against a value. // A random model also picks a winner, and on these data it picks one by a comparable lead // (measured), so the raw margin says nothing on its own. From a8c559bf6bcfc3ac68cfe9d11f32cc442f22bfda Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 23:44:53 +0200 Subject: [PATCH 04/12] tests: jfjoch_portable_test, quick [portable] checks for the macOS and Windows jobs A Catch2 executable that builds in the portable configurations (JFJOCH_VIEWER_ONLY / JFJOCH_RUGNUX_ONLY) and links only what those build - JFJochRugnux, JFJochReader, JFJochImageAnalysis, JFJochWriter, JFJochCommon - so it can run on the macOS arm64 and Windows x64 jobs, where the receiver/broker/FPGA/HLS sources are not built and there is no GPU. EXCLUDE_FROM_ALL, so a product build does not pay for it; catch2 is now made available in the portable configure as well (it is EXCLUDE_FROM_ALL too). The cases tagged [portable] cover what depends on the architecture, the compiler or the standard library: bitshuffle/LZ4/zstd, HDF5 read-back (legacy/VDS/integrated, the direct-chunk path), miniCBF/marCCD/SMV header parsing, CBOR, CPU spot finding, azimuthal mapping, Bragg prediction/integration, gemmi MTZ/mmCIF. New in tests/PortableTest.cpp: - a golden FNV-1a hash of two frames of compression_benchmark.h5, decoded from the raw chunk by the hperf and the classic bitshuffle and through the HDF5 filter (x86 hashes affea29c511b6ec2 / e46913009c95a1f1); - a golden hash of a bitshuffle/LZ4 encode (the writer must produce the same bytes everywhere); - the shipped bitshuffle block selector against the classic reference over elem 1/2/4/8 and block tails; - the FFTW indexer, named explicitly, on a synthetic orthorhombic lattice (the existing FFT indexer lattice tests run only under CUDA); - a 4-frame end-to-end rugnux run on the git-LFS rotation dataset (HDF5 via external links, CPU spot finding, indexing, integration); SKIPs when LFS was not pulled. All 45 take ~4 s on Linux (~2 s without the LFS case), CPU-only build. M_PI replaced by PI (common/JFJochMath.h) in the two tagged files that used it, as MSVC does not define M_PI. The CBF gzip test shells out to gzip and is left untagged. CI: build and run jfjoch_portable_test "[portable]" in build-windows (both variants), build-rugnux-windows, build-macos-viewer and build-rugnux-macos, after the build and before packaging. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .gitea/workflows/build_and_test.yml | 40 ++++ CMakeLists.txt | 10 +- tests/AdaptiveSpotFinderCPUTest.cpp | 2 +- tests/AzimuthalIntegrationTest.cpp | 7 +- ...ggIntegrationEngineCompressedImageTest.cpp | 2 +- tests/BraggStencilTest.cpp | 2 +- tests/CBORTest.cpp | 8 +- tests/CMakeLists.txt | 37 ++++ tests/CalcBraggPredictionTest.cpp | 15 +- tests/ImageSpotFinderCPUTest.cpp | 4 +- tests/JFJochReaderTest.cpp | 32 +-- tests/LoadReferenceMtzFreeFlagsTest.cpp | 2 +- tests/LoadReferenceSfCifTest.cpp | 2 +- tests/PortableTest.cpp | 184 ++++++++++++++++++ tests/SweepLayoutTest.cpp | 4 +- tests/WriteReflectionsTest.cpp | 4 +- tests/ZSTDCompressorTest.cpp | 6 +- tests/data/README.md | 3 +- 18 files changed, 315 insertions(+), 49 deletions(-) create mode 100644 tests/PortableTest.cpp diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index ee6cd62e2..58893e991 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -126,6 +126,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build\tests + jfjoch_portable_test.exe "[portable]" - name: Build installer (NSIS) shell: cmd run: | @@ -168,6 +181,13 @@ jobs: # JFJOCH_VIEWER_ONLY is forced on macOS. No -mcpu: the arm64 baseline is what every Apple # Silicon Mac runs, and NEON is part of it. run: cmake -S . -B build-macos -DCMAKE_BUILD_TYPE=Release -DJFJOCH_USE_CUDA=OFF -DCMAKE_PREFIX_PATH="$QT_PREFIX" + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build viewer and DMG shell: bash run: | @@ -380,6 +400,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build-rugnux + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build-rugnux --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build-rugnux\tests + jfjoch_portable_test.exe "[portable]" - name: Build rugnux zip shell: cmd run: | @@ -415,6 +448,13 @@ jobs: - name: Configure rugnux build shell: bash run: cmake -S . -B build-rugnux-macos -DJFJOCH_RUGNUX_ONLY=ON -DJFJOCH_USE_CUDA=OFF -DCMAKE_BUILD_TYPE=Release + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-rugnux-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-rugnux-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build rugnux tgz shell: bash run: cmake --build build-rugnux-macos --target package -j "$(sysctl -n hw.ncpu)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 33fd421d5..27c88cba7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -365,10 +365,10 @@ SET(CMAKE_POLICY_VERSION_MINIMUM 3.5) # libzmq must be made available BEFORE sls_detector_package for the override above to take effect. FetchContent_MakeAvailable(libzmq) IF (JFJOCH_PORTABLE_ONLY) - # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib). - # Only sls_detector_package (detector) and catch2 (tests) are not built here -- and sls - # in particular does not configure under MSVC -- so skip just those two. - FetchContent_MakeAvailable(zstd hdf5 spdlog httplib) + # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2 + # for jfjoch_portable_test. Only sls_detector_package (detector) is not built here -- it does + # not configure under MSVC. + FetchContent_MakeAvailable(zstd catch2 hdf5 spdlog httplib) ELSE() FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib) ENDIF() @@ -555,6 +555,7 @@ IF (JFJOCH_RUGNUX_ONLY) ADD_SUBDIRECTORY(broker) ADD_SUBDIRECTORY(reader) ADD_SUBDIRECTORY(rugnux) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSEIF (JFJOCH_VIEWER_ONLY) # Minimal subtree: jfjoch_viewer and only the libraries it transitively links. # (broker here provides JFJochAPI only; its service targets are gated out.) @@ -571,6 +572,7 @@ ELSEIF (JFJOCH_VIEWER_ONLY) ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(viewer) ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/extract_hkl) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSE() ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) diff --git a/tests/AdaptiveSpotFinderCPUTest.cpp b/tests/AdaptiveSpotFinderCPUTest.cpp index 0db0270a6..14e2cbc61 100644 --- a/tests/AdaptiveSpotFinderCPUTest.cpp +++ b/tests/AdaptiveSpotFinderCPUTest.cpp @@ -28,7 +28,7 @@ SpotFindingSettings AdaptiveSettings() { // Raw (untransformed) geometry: the mapping is built over the raw module layout, which is smaller // than the converted image. The finder has to walk the raw image, so a spot planted at a raw pixel // comes back at that pixel. -TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder]") { +TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(80).BeamX_pxl(1030).BeamY_pxl(1080); x.QSpacingForAzimInt_recipA(0.05).QRangeForAzimInt_recipA(0.05, 5.0); diff --git a/tests/AzimuthalIntegrationTest.cpp b/tests/AzimuthalIntegrationTest.cpp index d5adf25fc..5151c4faf 100644 --- a/tests/AzimuthalIntegrationTest.cpp +++ b/tests/AzimuthalIntegrationTest.cpp @@ -5,6 +5,7 @@ #include "../common/AzimuthalIntegrationProfile.h" #include "../common/AzimuthalIntegrationMapping.h" +#include "../common/JFJochMath.h" TEST_CASE("AzimuthalIntegrationMapping_Constructor","[AzimuthalIntegration]") { DiffractionExperiment x(DetJF4M()); @@ -39,7 +40,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetBinNumber_mask","[AzimuthalIntegration for (int row = 0; row < x.GetYPixelsNum(); row++) { for (int col = 0; col < x.GetXPixelsNum(); col++) { float d = geom.PxlToRes(col, row); - float q = 2 * M_PI / d; + float q = 2 * PI / d; if (q >= 3.1) pixel_mask[row * x.GetXPixelsNum() + col] = 1; } @@ -147,7 +148,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetMapping","[AzimuthalIntegration]") { CHECK(map[x.GetXPixelsNum() * 1500 + 1500] / 40 == 0); } -TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4); @@ -400,7 +401,7 @@ TEST_CASE("AzimuthalIntegrationProfile_GetMeanValueOfBins","[AzimuthalIntegratio REQUIRE(profile.GetBkgEstimate(x.GetAzimuthalIntegrationSettings()) == Catch::Approx((sum[0] + sum[1] + sum[2]) / double(count[0] + count[1] + count[2]))); } -TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); diff --git a/tests/BraggIntegrationEngineCompressedImageTest.cpp b/tests/BraggIntegrationEngineCompressedImageTest.cpp index 46344bb22..881c4b4b8 100644 --- a/tests/BraggIntegrationEngineCompressedImageTest.cpp +++ b/tests/BraggIntegrationEngineCompressedImageTest.cpp @@ -93,7 +93,7 @@ void RequireIdentical(const std::vector &a, const std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -491,7 +491,7 @@ TEST_CASE("CBORSerialize_Calibration", "[CBOR]") { CHECK(memcmp(output_message.GetCompressed(), calib1.data(), 256 * sizeof(float)) == 0); } -TEST_CASE("CBORSerialize_End", "[CBOR]") { +TEST_CASE("CBORSerialize_End", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -897,7 +897,7 @@ TEST_CASE("CBORSerialize_Image_Rgb", "[CBOR]") { REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(rgb)) == 0); } -TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -1026,7 +1026,7 @@ TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") { } -TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Spots", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b386a5a4..81ab48540 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,41 @@ +# Quick cross-platform checks: the results that depend on the architecture, the compiler or the +# standard library (bitshuffle/LZ4 SIMD paths, HDF5 read-back, CBF parsing, CPU spot finding, FFTW +# indexing, gemmi MTZ/CIF). Built in every configuration, the portable (viewer/rugnux-only) ones +# included, and linked only against what those build - so it runs on the macOS and Windows jobs. +# Only the cases tagged [portable] are meant to run there (a few seconds in all); the rest of these +# files is compiled in because the tagged cases live next to them. +# EXCLUDE_FROM_ALL: a product build does not pay for it, CI builds the target by name. +ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL + PortableTest.cpp + TestData.h + ZSTDCompressorTest.cpp + JFJochReaderTest.cpp + SweepLayoutTest.cpp + CBORTest.cpp + ImageSpotFinderCPUTest.cpp + AdaptiveSpotFinderCPUTest.cpp + AzimuthalIntegrationTest.cpp + CalcBraggPredictionTest.cpp + BraggStencilTest.cpp + BraggIntegrationEngineCompressedImageTest.cpp + WriteReflectionsTest.cpp + LoadReferenceMtzFreeFlagsTest.cpp + LoadReferenceSfCifTest.cpp +) +target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis + JFJochWriter JFJochCommon) +target_include_directories(jfjoch_portable_test PRIVATE .) +# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically. +IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) + SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) +ENDIF() + +IF (JFJOCH_PORTABLE_ONLY) + RETURN() +ENDIF() + ADD_EXECUTABLE(jfjoch_test + PortableTest.cpp DiffractionExperimentTest.cpp RawToConvertedGeometryTest.cpp ../common/RawToConvertedGeometry.h diff --git a/tests/CalcBraggPredictionTest.cpp b/tests/CalcBraggPredictionTest.cpp index 9d94c9a90..48f0c1661 100644 --- a/tests/CalcBraggPredictionTest.cpp +++ b/tests/CalcBraggPredictionTest.cpp @@ -4,6 +4,7 @@ #include #include "../image_analysis/bragg_prediction/BraggPrediction.h" +#include "../common/JFJochMath.h" #include #include "../image_analysis/SensorAbsorption.h" #include "../image_analysis/bragg_prediction/BraggPredictionRot.h" @@ -40,11 +41,11 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { // crossed strictly more of the medium. float prev = 1.0f; for (double alpha_deg : {10.0, 20.0, 30.0, 40.0, 50.0, 55.0}) { - const float f = a18.Factor(static_cast(std::cos(alpha_deg * M_PI / 180.0))); + const float f = a18.Factor(static_cast(std::cos(alpha_deg * PI / 180.0))); CHECK(f > prev); prev = f; } - const float cos55 = static_cast(std::cos(55.0 * M_PI / 180.0)); + const float cos55 = static_cast(std::cos(55.0 * PI / 180.0)); // 18 keV over 160 mm at 55 degrees: +1.5%. CHECK(a18.Factor(cos55) == Catch::Approx(1.0147).epsilon(0.002)); @@ -88,7 +89,7 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { CHECK(vac.WilsonBShift_A2(3.02, 50.0, lambda_of(3.76)) == 0.0); } -TEST_CASE("BraggPrediction_11keV") { +TEST_CASE("BraggPrediction_11keV", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(11.0); @@ -160,10 +161,10 @@ TEST_CASE("BraggPrediction_15keV") { } } -TEST_CASE("BraggPrediction_Rot1_Rot2") { +TEST_CASE("BraggPrediction_Rot1_Rot2", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); @@ -224,7 +225,7 @@ TEST_CASE("BraggPrediction_backscattering") { } } -TEST_CASE("BraggPrediction_systematic_absences") { +TEST_CASE("BraggPrediction_systematic_absences", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(120.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(12.0); @@ -404,7 +405,7 @@ TEST_CASE("BraggPredictionGPU") { TEST_CASE("BraggPredictionGPU_Rot1_Rot2") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); diff --git a/tests/ImageSpotFinderCPUTest.cpp b/tests/ImageSpotFinderCPUTest.cpp index fe680a6cb..35d06773e 100644 --- a/tests/ImageSpotFinderCPUTest.cpp +++ b/tests/ImageSpotFinderCPUTest.cpp @@ -4,7 +4,7 @@ #include #include "../image_analysis/spot_finding/ImageSpotFinderCPU.h" -TEST_CASE("ImageSpotFinderCPU_SignalToNoise") { +TEST_CASE("ImageSpotFinderCPU_SignalToNoise", "[portable]") { size_t width = 100; size_t height = 100; @@ -108,7 +108,7 @@ TEST_CASE("ImageSpotFinderCPU_CountThreshold_Resolution") { REQUIRE(spots[1].RawCoord().y == 75); } -TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask") { +TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask", "[portable]") { size_t width = 100; size_t height = 100; diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index ba8d01780..9ade44cb7 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -608,7 +608,7 @@ TEST_CASE("JFJochReader_GridScan", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test09").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -774,7 +774,7 @@ TEST_CASE("JFJochReader_DataI16_OldMasterFormat", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test10").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -846,7 +846,7 @@ TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test11").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -913,7 +913,7 @@ TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test12").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1786,7 +1786,7 @@ TEST_CASE("JFJochReader_NXmxIntegrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1906,7 +1906,7 @@ TEST_CASE("JFJochReader_GetRawImage_Concurrent", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1965,7 +1965,7 @@ TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -2022,7 +2022,7 @@ TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_integrated").ImagesPerTrigger(5).OverwriteExistingFiles(true); @@ -2070,7 +2070,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_vds_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -2131,7 +2131,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_legacy_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -3434,7 +3434,7 @@ TEST_CASE("JFJochReader_AxisRecovery", "[HDF5][Full]") { // saturation_value is written inclusive and used exclusive, so a read has to add the count back. It // did not, and the value fell by one on every write-read-write cycle - unbounded, and compounding // whenever a _process.h5 was reprocessed. Nothing caught it: no test asserted the read-back limit. -TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full]") { +TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.ImagesPerTrigger(2).OverwriteExistingFiles(true).FilePrefix("test_satrt"); x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) @@ -3958,7 +3958,7 @@ namespace { } } -TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a pixel at Count_cutoff is saturated") { @@ -4081,7 +4081,7 @@ namespace { // Three ways a CBF that is not a detector image, or is one with a hole in its head, used to be // opened anyway - each of them silently, which is the failure this project cares most about. -TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a byte-offset CBF with no PILATUS header is not ours to read") { @@ -4128,7 +4128,7 @@ TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Fu } } -TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full]") { +TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full][portable]") { // The header is read with a small first probe, and again with the large one when it does not end // inside it; what stands past the first 16 kB must come out exactly as before. std::string filler; @@ -4261,7 +4261,7 @@ namespace { } } -TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("the header's geometry reaches the experiment in the units the rest of the code uses") { @@ -4425,7 +4425,7 @@ namespace { } } -TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full][portable]") { SECTION("a pixel at CCD_IMAGE_SATURATION is saturated, stated or not") { // The pixels are handed out as 32-bit, so without an explicit limit the fallback would be // INT32_MAX and no 16-bit image could ever reach it. diff --git a/tests/LoadReferenceMtzFreeFlagsTest.cpp b/tests/LoadReferenceMtzFreeFlagsTest.cpp index c2c5b811f..12bc73e43 100644 --- a/tests/LoadReferenceMtzFreeFlagsTest.cpp +++ b/tests/LoadReferenceMtzFreeFlagsTest.cpp @@ -42,7 +42,7 @@ namespace { bool ExpectedFree(int h, int k, int l) { return (h * 7 + k * 13 + l * 17) % 10 == 0; } } -TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz][portable]") { const auto path = (std::filesystem::temp_directory_path() / "rugnux_freer_ccp4.mtz").string(); WriteMtzWithFreeR(path, /*free_value=*/0); const auto ref = LoadReferenceMtz(path); diff --git a/tests/LoadReferenceSfCifTest.cpp b/tests/LoadReferenceSfCifTest.cpp index b56a2dff6..c8027fb15 100644 --- a/tests/LoadReferenceSfCifTest.cpp +++ b/tests/LoadReferenceSfCifTest.cpp @@ -100,7 +100,7 @@ _refln.intensity_sigma } } -TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz][portable]") { // Named without a .cif extension: the format is recognised by content. const auto path = WriteFile("rugnux_ref_sf.dat", SF_CIF); const auto ref = LoadReferenceMtz(path); diff --git a/tests/PortableTest.cpp b/tests/PortableTest.cpp new file mode 100644 index 000000000..3c67eed15 --- /dev/null +++ b/tests/PortableTest.cpp @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Checks that the results which depend on the architecture, the compiler or the standard library are +// the same everywhere: the bitshuffle/LZ4 path (hperf SSE2/AVX2 on x86, NEON on aarch64, SSE2 under +// MSVC) and an end-to-end run of the analysis. Golden values were taken on x86-64 Linux; an arm64 +// or MSVC build that disagrees has a portability defect, not a new reference. + +#include + +#include +#include + +#include "TestData.h" +#include "../compression/JFJochCompressor.h" +#include "../compression/JFJochDecompress.h" +#include "../common/JFJochMath.h" +#include "../writer/HDF5Objects.h" +#include "../image_analysis/indexing/IndexerFactory.h" +#include "../reader/JFJochHDF5Reader.h" +#include "../rugnux/Rugnux.h" + +namespace { + // FNV-1a, spelled out: std::hash is not the same function on every standard library. + uint64_t Fnv1a(const void *data, size_t size) { + auto p = static_cast(data); + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < size; i++) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; + } + + uint64_t SplitMix64(uint64_t &state) { + uint64_t z = (state += 0x9e3779b97f4a7c15ULL); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; + z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; + return z ^ (z >> 31); + } +} + +// compression_benchmark.h5 holds 25 int16 frames, bitshuffle/LZ4 (HDF5 filter 32008). One frame is +// read three ways - the raw chunk decoded by the hperf path and by the classic bitshuffle, and +// through the HDF5 filter - and all three must give the frame whose hash was recorded on x86. +TEST_CASE("Portable_BSLZ4_BenchmarkFrameGoldenHash", "[portable][compression]") { + RegisterHDF5Filter(); + HDF5ReadOnlyFile file("../../tests/test_data/compression_benchmark.h5"); + HDF5DataSet dataset(file, "/entry/data/data"); + HDF5DataSpace file_space(dataset); + const auto dims = file_space.GetDimensions(); + REQUIRE(dims.size() == 3); + const size_t npixel = dims[1] * dims[2]; + + const std::pair golden[] = { + {0, 0xaffea29c511b6ec2ULL}, + {24, 0xe46913009c95a1f1ULL} + }; + + for (const auto &[frame, hash]: golden) { + std::vector chunk; + dataset.ReadDirectChunk(chunk, {frame, 0, 0}); + + std::vector hperf, classic, filtered(npixel); + JFJochDecompress(hperf, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, true); + JFJochDecompress(classic, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, false); + dataset.ReadVector(filtered, {frame, 0, 0}, {1, dims[1], dims[2]}); + + CHECK(Fnv1a(hperf.data(), hperf.size() * sizeof(int16_t)) == hash); + CHECK(hperf == classic); + CHECK(hperf == filtered); + } +} + +// The encoder must write the same bytes everywhere: a file is decoded on a different machine from +// the one that wrote it, and the header of every block is big-endian regardless of the host. +TEST_CASE("Portable_BSLZ4_EncodeGoldenHash", "[portable][compression]") { + uint64_t state = 42; + std::vector image(1024 * 1024 + 13); // a tail that is not a whole block + for (auto &v: image) + v = (SplitMix64(state) % 16 == 0) ? SplitMix64(state) % 200 : 0; + + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + auto compressed = compressor.Compress(image); + CHECK(compressed.size() == 562349); + CHECK(Fnv1a(compressed.data(), compressed.size()) == 0x874e45eb17a51ed2ULL); + + std::vector decoded; + JFJochDecompress(decoded, CompressionAlgorithm::BSHUF_LZ4, compressed, image.size()); + CHECK(decoded == image); +} + +// One bitshuffle block through the shipped selector (hperf on x86, classic NEON on aarch64) against +// the classic reference transform, over the element sizes and block tails the file format allows. +TEST_CASE("Portable_BitShuffleBlock_MatchesReference", "[portable][compression]") { + uint64_t state = 7; + for (size_t elem: {1, 2, 4, 8}) { + for (size_t size: {8, 16, 24, 136, 1032, 2040, 2048, 8192}) { + for (int pattern = 0; pattern < 3; pattern++) { + std::vector orig(size * elem), ref(size * elem), enc(size * elem), + scratch(size * elem), out(size * elem); + for (size_t i = 0; i < orig.size(); i++) { + if (pattern == 0) orig[i] = static_cast(SplitMix64(state)); // noise + else if (pattern == 1) orig[i] = (i % elem == 0) ? static_cast(i % 7) : 0; // counts + else orig[i] = static_cast((i % elem == elem - 1) ? 0x80 : 0); // sign bit + } + INFO("elem " << elem << " size " << size << " pattern " << pattern); + REQUIRE(bshuf_trans_bit_elem(orig.data(), ref.data(), size, elem) >= 0); + REQUIRE(JFJochBitShuffleBlock(enc.data(), orig.data(), scratch.data(), size, elem) >= 0); + CHECK(enc == ref); + REQUIRE(JFJochBitUnshuffleBlock(out.data(), ref.data(), scratch.data(), size, elem) >= 0); + CHECK(out == orig); + } + } + } +} + +// The CPU indexer (FFTW, NEON codelets on arm64) on a synthetic orthorhombic lattice. Named +// explicitly rather than through Auto, so a CUDA build on a runner without a GPU tests it too. +TEST_CASE("Portable_FFTWIndexer_SyntheticLattice", "[portable][Indexing]") { + const UnitCell uc(39, 45, 78, 90, 90, 90); + const CrystalLattice cl(uc); + + DiffractionExperiment experiment; + IndexingSettings settings; + settings.Algorithm(IndexingAlgorithmEnum::FFTW).FFT_MaxUnitCell_A(250.0).FFT_HighResolution_A(2 * PI / 3.0); + experiment.ImportIndexingSettings(settings).SetUnitCell(uc); + auto indexer = CreateIndexer(experiment); + REQUIRE(indexer); + + std::vector spots; + for (int h = -2; h < 10; h++) + for (int k = -5; k < 10; k++) + for (int l = -3; l < 10; l++) + spots.push_back(h * cl.Astar() + k * cl.Bstar() + l * cl.Cstar()); + + const auto result = indexer->Run(spots); + REQUIRE(result.lattice.size() == 1); + const auto out = result.lattice[0].GetUnitCell(); + std::array lengths = {out.a, out.b, out.c}; + std::sort(lengths.begin(), lengths.end()); + CHECK(lengths[0] == Catch::Approx(39.0)); + CHECK(lengths[1] == Catch::Approx(45.0)); + CHECK(lengths[2] == Catch::Approx(78.0)); + CHECK(out.alpha == Catch::Approx(90.0)); + CHECK(out.beta == Catch::Approx(90.0)); + CHECK(out.gamma == Catch::Approx(90.0)); +} + +// End to end on the git-LFS rotation dataset (tests/data, see README.md there): HDF5 read through +// external links, CPU spot finding, the default indexer (FFTW on a CPU build), refinement and +// integration, on a short wedge. SKIPs when LFS was not pulled. +TEST_CASE("Portable_Rugnux_RotationWedge", "[portable][lfs]") { + const auto master = jfjoch_test::LargeDataFile("rotation_master.h5"); + if (!master) + SKIP("rotation_master.h5 not available (git-lfs data not pulled)"); + + RegisterHDF5Filter(); + JFJochHDF5Reader reader; + REQUIRE_NOTHROW(reader.ReadFile(*master)); + auto dataset = reader.GetDataset(); + REQUIRE(dataset); + + DiffractionExperiment experiment(dataset->experiment); + IndexingSettings indexing; + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + experiment.ImportIndexingSettings(indexing); + + ProcessConfig config; + config.mode = ProcessMode::FullAnalysis; + config.nthreads = std::max(1u, std::thread::hardware_concurrency()); + config.end_image = 4; + config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); + config.spot_finding.indexing = true; + + Rugnux process(reader, experiment, *dataset->pixel_mask, config); + ProcessResult result; + REQUIRE_NOTHROW(result = process.Run()); + + CHECK(result.images_processed == 4); + REQUIRE(result.indexing_rate.has_value()); + CHECK(result.indexing_rate.value() > 0.5f); + reader.Close(); +} diff --git a/tests/SweepLayoutTest.cpp b/tests/SweepLayoutTest.cpp index a3ef530d3..88663dde5 100644 --- a/tests/SweepLayoutTest.cpp +++ b/tests/SweepLayoutTest.cpp @@ -21,7 +21,7 @@ std::vector Series(const std::vector &angles, double incre } // namespace -TEST_CASE("SweepLayout_Contiguous") { +TEST_CASE("SweepLayout_Contiguous", "[portable]") { std::vector angles; for (int i = 0; i < 100; i++) angles.push_back(20.0 + 0.1 * i); @@ -34,7 +34,7 @@ TEST_CASE("SweepLayout_Contiguous") { CHECK(l.files[i] == "f" + std::to_string(i) + ".cbf"); } -TEST_CASE("SweepLayout_Gapped") { +TEST_CASE("SweepLayout_Gapped", "[portable]") { // Frames 3, 4 and 7 of a ten-frame sweep never made it into the archive. The sweep is still ten // steps wide and every frame keeps its own angle. const auto l = sweep::Place(Series({0.0, 0.1, 0.2, 0.5, 0.6, 0.8, 0.9}, 0.1), "test"); diff --git a/tests/WriteReflectionsTest.cpp b/tests/WriteReflectionsTest.cpp index 8609e410e..b8e6d06c4 100644 --- a/tests/WriteReflectionsTest.cpp +++ b/tests/WriteReflectionsTest.cpp @@ -31,7 +31,7 @@ namespace { } } -TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections]") { +TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections][portable]") { jfjoch_test::SyntheticMergeParams params; params.true_space_group = "P 43 21 2"; params.twin_supergroup = "P 43 21 2"; @@ -74,7 +74,7 @@ TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[w } TEST_CASE("Unmerged MTZ: LP is Lorentz-polarization, QE the sensor efficiency, FLIGHT the flight path", - "[write_reflections]") { + "[write_reflections][portable]") { // The whole point of the split: LP must mean what XDS and DIALS mean by it, and the raw count // sum must still be recoverable from the file alone, as I / LP * QE * FLIGHT. auto experiment = TestExperiment(); diff --git a/tests/ZSTDCompressorTest.cpp b/tests/ZSTDCompressorTest.cpp index 5a497601e..f6f3e3575 100644 --- a/tests/ZSTDCompressorTest.cpp +++ b/tests/ZSTDCompressorTest.cpp @@ -229,7 +229,7 @@ TEST_CASE("JFjochZstdCompressor_Frame_ones","[ZSTD]") { REQUIRE(memcmp(input_shuffled.data(), output.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_ZSTD).BitDepthImage(32).PixelSigned(true); @@ -269,7 +269,7 @@ TEST_CASE("JFJochCompressor_DestTooSmall_Throws","[ZSTD]") { REQUIRE_NOTHROW(compressor.Compress(big.data(), big.size(), image)); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(32).PixelSigned(true); @@ -396,7 +396,7 @@ TEST_CASE("ZstdHuff_MaskLike", "[ZSTD]") { RequireHuffRoundTrip(image); } -TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD]") { +TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD][portable]") { RegisterHDF5Filter(); // bitshuffle filter, needed to read the compressed benchmark dataset HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); HDF5DataSet dataset(data, "/entry/data/data"); diff --git a/tests/data/README.md b/tests/data/README.md index 3227cd93b..b1353efcf 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -23,7 +23,8 @@ git lfs pull # or: git lfs pull --include "tests/data/*.h5" | `rotation_master.h5` | protein rotation series (~1800 images) | yes (LFS) | `rotation_master.h5` (plus its `_data_NNNNNN.h5` files) is fetched by `git lfs pull` and -drives `Rugnux_Rotation`. A separate serial dataset is intentionally **not** shipped +drives `Rugnux_Rotation`, and its first four frames `Portable_Rugnux_RotationWedge` (the +`[portable]` end-to-end check, which skips on the macOS/Windows jobs because they do not pull LFS). A separate serial dataset is intentionally **not** shipped to keep the repository small — the rotation series can be run in serial mode (full analysis without rotation indexing) to exercise that path. To add your own dataset, drop the master + its data files here as real files (not symlinks, if you intend to commit them via LFS); the master From 301964bb0627c6ae548980a36afd7607fccfc425 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 00:16:27 +0200 Subject: [PATCH 05/12] tests: a rotation-indexer case in [portable], on the Auto indexer choice "RotationIndexer judges a tilt no mounting can have on the spots" (the cheapest of the four, ~3.5 s on FFTW) joins the portable set, and RotationIndexerTest.cpp is compiled into jfjoch_portable_test. The three rotation-indexer setups now ask for Auto instead of naming FFT under CUDA and FFTW otherwise: for rotation indexing DiffractionExperiment::GetIndexingAlgorithm resolves FFT/FFTW on get_gpu_count() whatever was requested, so this changes nothing on either build, and states what a run without a GPU relies on. Checked on a CUDA build with CUDA_VISIBLE_DEVICES="": all 45 [portable] cases pass (Auto falls back to FFTW - the rotation case takes 3.8 s against 1.4 s with the GPU visible - and the 4-frame rugnux wedge runs on the CPU path). No change to the indexer selection was needed. The FFTIndexer* unit tests are not added: none goes through Auto - each names FFT (and FFTW) explicitly, and the explicit FFT request fails with "cudaHostRegister failed" when no GPU is visible, as it should. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- tests/CMakeLists.txt | 1 + tests/RotationIndexerTest.cpp | 26 ++++---------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 81ab48540..7ce420525 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL WriteReflectionsTest.cpp LoadReferenceMtzFreeFlagsTest.cpp LoadReferenceSfCifTest.cpp + RotationIndexerTest.cpp ) target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter JFJochCommon) diff --git a/tests/RotationIndexerTest.cpp b/tests/RotationIndexerTest.cpp index 73cfa44e4..a98f74fe6 100644 --- a/tests/RotationIndexerTest.cpp +++ b/tests/RotationIndexerTest.cpp @@ -18,13 +18,7 @@ TEST_CASE("RotationIndexer") { .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_i.ImportIndexingSettings(settings); @@ -113,13 +107,7 @@ TEST_CASE("RotationIndexer::RefineConstrained puts a free metric back on its cla .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_i.ImportIndexingSettings(settings); @@ -195,13 +183,7 @@ static std::optional IndexOnTiltedDetector(double true_ti .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return {}; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_header.ImportIndexingSettings(settings); @@ -278,7 +260,7 @@ TEST_CASE("RotationIndexer keeps a tilt a mounting can have") { // index; the result records the walk, both counts, the verdict, and a geometry that matches it. The // lattice is not checked: a 1.5 deg detector error on 6 A data already puts the FFT on a different // cell before any fit. -TEST_CASE("RotationIndexer judges a tilt no mounting can have on the spots") { +TEST_CASE("RotationIndexer judges a tilt no mounting can have on the spots", "[portable]") { const auto ret = IndexOnTiltedDetector(1.5, 6.0f); REQUIRE(ret.has_value()); REQUIRE(ret->tilt_walk.has_value()); From 9c5141c7c0f47015f4d827551955fc3b86cef684 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:00:50 +0200 Subject: [PATCH 06/12] CPU pixel pipeline: decode, preprocess and ring pass per bitshuffle block; flag rings in the local sweep The CPU-only image loop is DRAM-bound on 16M frames: decode, preprocess, the adaptive finder's plain ring pass and FlagRings each streamed the whole frame through memory. - JFJochDecompressHperfBlocks hands each decoded bitshuffle block to a callback; with no output buffer the block is unshuffled into a reused block-sized scratch (JFJochDecompressBlocks). - MXAnalysisWithoutFPGA::PreprocessCPU preprocesses each block into the int32 buffer (ImagePreprocessorCPU::AnalyzeBlock) and, when the fused CPU finder runs, puts it through the plain ring pass + fused azint (AdaptiveSpotFinderCPU::AccumulateRingsBlock) while it is in cache. Detect() then starts from those sums. The per-worker decompression buffer is no longer allocated for bitshuffled data. - FlagRings becomes FlagRow, called by DetectAt's first pass for row y+NBX just before that row enters the vertical sums; first_pass_needed is marked from each row's candidates at the same point. Exact: blocks arrive in pixel order, so the float azint sums see the same pixels in the same order; the per-pixel expressions are unchanged; everything else is integer. p.hkl, p.mtz, p_P1.mtz and p_unmerged.mtz byte-identical to rc173 on myob, cytc, lyso, sparse (CPU-only build), GPU myob identical (the GPU path does not take this route). CPU-only, 32 workers, under the gpulock on a shared (loaded) machine, base -> fused, two rounds (second in reversed order): myob 155.9 -> 97.7 s, 139.9 -> 88.1 s (loop 46.1 -> 26.8 s/pass; user 3690 -> 2221 s) cytc 220.8 -> 156.6 s, 216.6 -> 155.1 s (user 5160 -> 4128 s) lyso 134.5 -> 123.3 s, 69.5 -> 64.8 s peak RSS myob 15.2 -> 10.8 GB, cytc 12.6 -> 10.4 GB, lyso 7.0 -> 6.7 GB Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- compression/JFJochDecompress.h | 86 +++++++-- image_analysis/MXAnalysisWithoutFPGA.cpp | 48 +++-- image_analysis/MXAnalysisWithoutFPGA.h | 10 ++ .../ImagePreprocessorCPU.cpp | 45 ++--- .../ImagePreprocessorCPU.h | 7 +- .../spot_finding/AdaptiveSpotFinderCPU.cpp | 165 ++++++++++-------- .../spot_finding/AdaptiveSpotFinderCPU.h | 24 ++- .../spot_finding/ImageSpotFinderCPU.cpp | 57 ++++-- .../spot_finding/ImageSpotFinderCPU.h | 12 +- tests/ZSTDCompressorTest.cpp | 28 +++ 10 files changed, 332 insertions(+), 150 deletions(-) diff --git a/compression/JFJochDecompress.h b/compression/JFJochDecompress.h index 73c0518f4..c5a90df63 100644 --- a/compression/JFJochDecompress.h +++ b/compression/JFJochDecompress.h @@ -21,13 +21,19 @@ extern "C" { uint64_t bshuf_read_uint64_BE(const void* buf); }; -inline size_t JFJochDecompressHperfPtr(uint8_t *output, - CompressionAlgorithm algorithm, - const uint8_t *source, - size_t source_size, - size_t nelements, - size_t elem_size, - size_t block_size) { +// Decode the bitshuffle blocks one after another and hand each to block_done(first_element, data, +// nelements), in element order. With output set, each block is decoded in place into output; with +// output = nullptr, into a block-sized buffer that is reused for the next block, so a caller that +// consumes the block straight away reads it from cache and the whole image is never written out. +template +size_t JFJochDecompressHperfBlocks(uint8_t *output, + CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + size_t block_size, + F &&block_done) { if ((algorithm != CompressionAlgorithm::BSHUF_LZ4) && (algorithm != CompressionAlgorithm::BSHUF_ZSTD) && (algorithm != CompressionAlgorithm::BSHUF_ZSTD_RLE) && @@ -39,10 +45,11 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, std::vector decompressed_block(block_size * elem_size); std::vector scratch(block_size * elem_size); + std::vector unshuffled(output ? 0 : block_size * elem_size); const uint8_t *src_ptr = source; const uint8_t *const source_end = source + source_size; - uint8_t *dst_ptr = output; + size_t first_element = 0; const size_t num_full_blocks = nelements / block_size; const size_t reminder_size = nelements - num_full_blocks * block_size; @@ -90,15 +97,17 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, throw JFJochException(JFJochExceptionCategory::Compression, "Algorithm not supported"); } + uint8_t *dst_ptr = output ? output + first_element * elem_size : unshuffled.data(); if (JFJochBitUnshuffleBlock(reinterpret_cast(dst_ptr), decompressed_block.data(), scratch.data(), current_nelements, elem_size) < 0) throw JFJochException(JFJochExceptionCategory::Compression, "bitshuffle block decode error"); + block_done(first_element, dst_ptr, current_nelements); src_ptr += compressed_size; - dst_ptr += decompressed_size; + first_element += decompressed_size / elem_size; }; for (size_t i = 0; i < num_full_blocks; ++i) @@ -111,13 +120,26 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, if (leftover_bytes > 0) { if (static_cast(source_end - src_ptr) < leftover_bytes) throw JFJochException(JFJochExceptionCategory::Compression, "Truncated trailing bytes"); + uint8_t *dst_ptr = output ? output + first_element * elem_size : unshuffled.data(); memcpy(dst_ptr, src_ptr, leftover_bytes); + block_done(first_element, dst_ptr, leftover_bytes / elem_size); src_ptr += leftover_bytes; } return static_cast(src_ptr - source); } +inline size_t JFJochDecompressHperfPtr(uint8_t *output, + CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + size_t block_size) { + return JFJochDecompressHperfBlocks(output, algorithm, source, source_size, nelements, elem_size, block_size, + [](size_t, const uint8_t *, size_t) {}); +} + // Plain LZ4, HDF5 filter 32004, as DECTRIS Eiger firmware 1.x wrote it. The framing is the same as // bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size IN BYTES, then each // block prefixed by its 32-bit big-endian compressed size - and the only difference is that no bit @@ -166,6 +188,40 @@ inline void JFJochDecompressLZ4Ptr(uint8_t *output, } } +// Check the 12-byte bitshuffle header and return the block size (in elements) it gives. +inline size_t JFJochBitshuffleHeaderBlockSize(const uint8_t *source, size_t source_size, + size_t nelements, size_t elem_size) { + // The header must be there before it can be read, and before source_size - 12 is handed to the + // decompressors. + if (source_size < 12) + throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the bitshuffle header"); + if (bshuf_read_uint64_BE(const_cast(source)) != nelements * elem_size) + throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size"); + return bshuf_read_uint32_BE(source + 8) / elem_size; +} + +inline bool JFJochIsBitshuffle(CompressionAlgorithm algorithm) { + return (algorithm == CompressionAlgorithm::BSHUF_LZ4) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD_RLE) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF); +} + +// A bitshuffle-compressed image, handed block by block to block_done(first_element, data, nelements) +// in element order (see JFJochDecompressHperfBlocks), without the image ever being written out whole. +template +void JFJochDecompressBlocks(CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + F &&block_done) { + const size_t block_size = JFJochBitshuffleHeaderBlockSize(source, source_size, nelements, elem_size); + if (JFJochDecompressHperfBlocks(nullptr, algorithm, source + 12, source_size - 12, + nelements, elem_size, block_size, block_done) != source_size - 12) + throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error"); +} + inline void JFJochDecompressPtr(uint8_t *output, CompressionAlgorithm algorithm, const uint8_t *source, @@ -179,16 +235,8 @@ inline void JFJochDecompressPtr(uint8_t *output, } size_t block_size = 0; - if (algorithm != CompressionAlgorithm::NO_COMPRESSION) { - // The 12-byte bitshuffle header must be there before it can be read, and before - // source_size - 12 is handed to the decompressors below. - if (source_size < 12) - throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the bitshuffle header"); - if (bshuf_read_uint64_BE(const_cast(source)) != nelements * elem_size) - throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size"); - auto tmp = bshuf_read_uint32_BE(source + 8); - block_size = tmp / elem_size; - } + if (algorithm != CompressionAlgorithm::NO_COMPRESSION) + block_size = JFJochBitshuffleHeaderBlockSize(source, source_size, nelements, elem_size); switch (algorithm) { case CompressionAlgorithm::NO_COMPRESSION: diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index a861f50bf..ce3d2b173 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -50,7 +50,9 @@ MXAnalysisWithoutFPGA::MXAnalysisWithoutFPGA(const DiffractionExperiment &in_exp if (get_gpu_count() == 0) { #endif preprocessor_buffer = std::make_unique(experiment.GetPixelsNum()); - preprocessor = std::make_unique(in_experiment, in_mask); + auto cpu_preprocessor = std::make_unique(in_experiment, in_mask); + preprocessor_cpu = cpu_preprocessor.get(); + preprocessor = std::move(cpu_preprocessor); bragg_engine = std::make_unique(in_experiment); if (experiment.ROI().size() >= 1) roi = std::make_unique(experiment); @@ -128,7 +130,23 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, } const auto compression_end_time = std::chrono::steady_clock::now(); - if (!decoded_on_device) { + // The fused GPU engine (rugnux offline, GPU, adaptive detection) produces the azimuthal profile as + // a byproduct of spot finding, so the separate azint pass is skipped in that case and the profile is + // lifted out of the finder below. + const bool fused = enable_fused_adaptive_gpu && spot_finding_settings.enable + && spot_finding_settings.adaptive_threshold && fused_adaptive != nullptr; + const bool fused_cpu = spot_finding_settings.enable && spot_finding_settings.adaptive_threshold + && fused_adaptive_cpu != nullptr; + + if (preprocessor_cpu) { + const auto preprocessing_start_time = std::chrono::steady_clock::now(); + ret = PreprocessCPU(output.image, fused_cpu); + const auto preprocessing_end_time = std::chrono::steady_clock::now(); + output.preprocessing_time_s = std::chrono::duration(preprocessing_end_time - preprocessing_start_time).count(); + // Decoded inside the preprocessing pass, whose time includes it. + if (output.image.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION) + output.compression_time_s = 0.0f; + } else if (!decoded_on_device) { const uint8_t *image_ptr = Decompress(output.image); const auto decompressed_time = std::chrono::steady_clock::now(); if (output.image.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION) @@ -149,14 +167,6 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, output.preprocessing_time_s = total_s - decompress_s; } - // The fused GPU engine (rugnux offline, GPU, adaptive detection) produces the azimuthal profile as - // a byproduct of spot finding, so the separate azint pass is skipped in that case and the profile is - // lifted out of the finder below. - const bool fused = enable_fused_adaptive_gpu && spot_finding_settings.enable - && spot_finding_settings.adaptive_threshold && fused_adaptive != nullptr; - const bool fused_cpu = spot_finding_settings.enable && spot_finding_settings.adaptive_threshold - && fused_adaptive_cpu != nullptr; - if (!fused && !fused_cpu) { const auto azint_start_time = std::chrono::steady_clock::now(); AzInt().Run(*preprocessor_buffer, profile); @@ -397,6 +407,24 @@ const uint8_t *MXAnalysisWithoutFPGA::Decompress(const CompressedImage &image) { return image.GetUncompressedPtr(decompression_buffer); } +ImageStatistics MXAnalysisWithoutFPGA::PreprocessCPU(const CompressedImage &image, bool ring_pass) { + ImageStatistics ret{}; + if (ring_pass) + fused_adaptive_cpu->BeginRings(); + const auto process_block = [&](size_t first, const uint8_t *data, size_t n) { + preprocessor_cpu->AnalyzeBlock(*preprocessor_buffer, first, data, n, image.GetMode(), ret); + if (ring_pass) + fused_adaptive_cpu->AccumulateRingsBlock(*preprocessor_buffer, first, n); + }; + if (JFJochIsBitshuffle(image.GetCompressionAlgorithm())) + JFJochDecompressBlocks(image.GetCompressionAlgorithm(), image.GetCompressed(), image.GetCompressedSize(), + image.GetWidth() * image.GetHeight() * image.GetNumChannels(), image.GetByteDepth(), + process_block); + else + process_block(0, Decompress(image), npixels); + return ret; +} + void MXAnalysisWithoutFPGA::RunROIOnly(DataMessage &output) { output.roi.clear(); if (roi) diff --git a/image_analysis/MXAnalysisWithoutFPGA.h b/image_analysis/MXAnalysisWithoutFPGA.h index 20d4d5cb8..670a8b1e5 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.h +++ b/image_analysis/MXAnalysisWithoutFPGA.h @@ -20,6 +20,7 @@ #include "IndexAndRefine.h" #include "image_preprocessing/ImagePreprocessor.h" #include "image_preprocessing/ImagePreprocessorBuffer.h" +#include "image_preprocessing/ImagePreprocessorCPU.h" class CudaStream; class AdaptiveSpotFinderGPU; @@ -32,6 +33,8 @@ class MXAnalysisWithoutFPGA { std::vector decompression_buffer; std::unique_ptr preprocessor; + // The preprocessor, where it is the CPU one. + ImagePreprocessorCPU *preprocessor_cpu = nullptr; size_t npixels; size_t xpixels; @@ -68,6 +71,13 @@ class MXAnalysisWithoutFPGA { // not compressed) and return where it landed. const uint8_t *Decompress(const CompressedImage &image); + // The CPU preprocessing. A bitshuffled image is decoded a block (~32 KB) at a time and each block + // is preprocessed while it is in cache, then - with ring_pass - put through the adaptive finder's + // plain ring pass as well, so the image is never written out whole before it is preprocessed and + // the preprocessed pixels are read back from cache, not from memory. The blocks come in pixel + // order, so every sum is taken in the same order as by separate passes. + ImageStatistics PreprocessCPU(const CompressedImage &image, bool ring_pass); + // Pixels outside the resolution limits, bit-packed. Built by the integration mapping, which is // shared by every worker's engine and hands out the same mask to all of them. std::shared_ptr> mask_resolution; diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp index 8803a73db..cc4c9e263 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp @@ -11,57 +11,62 @@ ImagePreprocessorCPU::ImagePreprocessorCPU(const DiffractionExperiment &experime } ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *image_ptr, CompressedImageMode image_mode) { + if (processed_image.size() != npixels) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Processed image size mismatch"); + + ImageStatistics ret{}; + AnalyzeBlock(processed_image, 0, image_ptr, npixels, image_mode, ret); + return ret; +} + +void ImagePreprocessorCPU::AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, CompressedImageMode image_mode, ImageStatistics &stats) { switch (image_mode) { case CompressedImageMode::Int8: - return Analyze(processed_image, image_ptr, INT8_MIN, INT8_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT8_MIN, INT8_MAX, stats); case CompressedImageMode::Int16: - return Analyze(processed_image, image_ptr, INT16_MIN, INT16_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT16_MIN, INT16_MAX, stats); case CompressedImageMode::Int32: - return Analyze(processed_image, image_ptr, INT32_MIN, INT32_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT32_MIN, INT32_MAX, stats); case CompressedImageMode::Uint8: - return Analyze(processed_image, image_ptr, UINT8_MAX, UINT8_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT8_MAX, UINT8_MAX, stats); case CompressedImageMode::Uint16: - return Analyze(processed_image, image_ptr, UINT16_MAX, UINT16_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT16_MAX, UINT16_MAX, stats); case CompressedImageMode::Uint32: - return Analyze(processed_image, image_ptr, UINT32_MAX, UINT32_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT32_MAX, UINT32_MAX, stats); default: throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "RGB/float mode not supported"); } } template -ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *input, T err_pixel_val, T sat_pixel_val) { - - if (processed_image.size() != npixels) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Processed image size mismatch"); - +void ImagePreprocessorCPU::AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, T err_pixel_val, T sat_pixel_val, ImageStatistics &ret) { auto image = reinterpret_cast(input); - - ImageStatistics ret{}; + int32_t *out = processed_image.data() + first; if (sat_pixel_val > saturation_limit) sat_pixel_val = static_cast(saturation_limit); - for (int i = 0; i < npixels; i++) { - if (mask_1bit[i] != 0) { - processed_image[i] = INT32_MIN; + for (size_t i = 0; i < n; i++) { + if (mask_1bit[first + i] != 0) { + out[i] = INT32_MIN; ++ret.masked_pixel_count; } else if (image[i] == err_pixel_val) { // Error/invalid marker = the pixel type's extreme value (0xFFFFFFFF for EIGER uint32). // Tested before saturation, since for unsigned types the marker also exceeds sat_pixel_val // (which is clipped above to the HDF5 saturation_value). - processed_image[i] = INT32_MIN; + out[i] = INT32_MIN; ++ret.error_pixel_count; } else if (image[i] >= sat_pixel_val) { - processed_image[i] = INT32_MAX; + out[i] = INT32_MAX; ++ret.saturated_pixel_count; } else { - processed_image[i] = static_cast(image[i]); + out[i] = static_cast(image[i]); if (image[i] > ret.max_value) ret.max_value = image[i]; if (image[i] < ret.min_value) ret.min_value = image[i]; } } - return ret; } diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h index aa326d77b..f14706d64 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h @@ -9,8 +9,13 @@ class ImagePreprocessorCPU : public ImagePreprocessor { std::vector mask_1bit; - template ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *input, T err_value, T sat_value); + template void AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, T err_value, T sat_value, ImageStatistics &stats); public: ImagePreprocessorCPU(const DiffractionExperiment &experiment, const PixelMask &mask); ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *decompressed_image, CompressedImageMode image_mode) override; + // Analyze for the n pixels from pixel `first` on, whose values `input` holds, adding to stats. The + // blocks of an image can come in any order and together give Analyze()'s result. + void AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, size_t n, + CompressedImageMode image_mode, ImageStatistics &stats); }; diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 93b7b9597..a23b12d99 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only #include -#include #include #include "AdaptiveSpotFinderCPU.h" @@ -34,71 +33,87 @@ void AdaptiveSpotFinderCPU::GetProfile(AzimuthalIntegrationProfile &profile) con // Per-ring background statistics with iterated peak exclusion following peakfinder8: // Barty et al. (2014) J. Appl. Cryst. 47, 1118-1131 -// Accumulate per-ring mean/variance from the raw (photon) image. clip_k <= 0 -> use every valid -// pixel (first pass); clip_k > 0 -> keep only pixels within clip_k sigma of the current ring mean, -// which removes the Bragg peaks from the background estimate. -void AdaptiveSpotFinderCPU::AccumulateRings(const ImagePreprocessorBuffer &image, float clip_k) { +// Per-ring mean/variance from the raw (photon) image: a plain pass over every valid pixel, then +// sigma-clip passes keeping only pixels within clip_k sigma of the current ring mean, which removes +// the Bragg peaks from the background estimate. +void AdaptiveSpotFinderCPU::ResetRings() { + std::fill(ring_sum.begin(), ring_sum.end(), 0); + std::fill(ring_sum2.begin(), ring_sum2.end(), 0); + std::fill(ring_cnt.begin(), ring_cnt.end(), 0); + std::fill(ring_hist.begin(), ring_hist.end(), 0); + ring_overflow.clear(); + if (fuse_azint) { + std::fill(azint_sum.begin(), azint_sum.end(), 0.0f); + std::fill(azint_sum2.begin(), azint_sum2.end(), 0.0f); + std::fill(azint_count.begin(), azint_count.end(), 0); + } +} + +void AdaptiveSpotFinderCPU::BeginRings() { + ResetRings(); + rings_from_blocks = true; +} + +// The plain pass over pixels [first, first + n). +void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n) { const auto &pixel_to_bin = mapping.GetPixelToBin(); const size_t nbins = ring_sum.size(); - const size_t npix = static_cast(width) * height; + const float *corrections = mapping.Corrections().data(); + + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (fuse_azint) { + const float val = static_cast(v) * corrections[pxl]; + const float val_sq = val * val; + azint_sum[b] += val; + azint_sum2[b] += val_sq; + ++azint_count[b]; + } + ring_sum[b] += v; + ring_sum2[b] += static_cast(static_cast(v) * v); + ring_cnt[b] += 1; + if (v >= 0 && v < HIST_VALUES) + ring_hist[b * HIST_VALUES + v] += 1; + else + ring_overflow.emplace_back(b, v); + } +} + +// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the +// pixels holding it would, and its pixels are added as a count. +void AdaptiveSpotFinderCPU::ClipRings(float clip_k) { + const size_t nbins = ring_sum.size(); std::fill(ring_sum.begin(), ring_sum.end(), 0); std::fill(ring_sum2.begin(), ring_sum2.end(), 0); std::fill(ring_cnt.begin(), ring_cnt.end(), 0); - if (clip_k <= 0.0f) { - std::fill(ring_hist.begin(), ring_hist.end(), 0); - ring_overflow.clear(); - const float *corrections = mapping.Corrections().data(); - if (fuse_azint) { - std::fill(azint_sum.begin(), azint_sum.end(), 0.0f); - std::fill(azint_sum2.begin(), azint_sum2.end(), 0.0f); - std::fill(azint_count.begin(), azint_count.end(), 0); - } - for (size_t pxl = 0; pxl < npix; ++pxl) { - const int32_t v = image[pxl]; - if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated - const uint16_t b = pixel_to_bin[pxl]; - if (b >= nbins) continue; // masked / out of range (UINT16_MAX) - if (fuse_azint) { - const float val = static_cast(v) * corrections[pxl]; - const float val_sq = val * val; - azint_sum[b] += val; - azint_sum2[b] += val_sq; - ++azint_count[b]; - } - ring_sum[b] += v; - ring_sum2[b] += static_cast(static_cast(v) * v); - ring_cnt[b] += 1; - if (v >= 0 && v < HIST_VALUES) - ring_hist[b * HIST_VALUES + v] += 1; - else - ring_overflow.emplace_back(b, v); - } - } else { - // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same - // test the pixels holding it would, and its pixels are added as a count. - const auto keep = [&](uint16_t b, int32_t v) { - const float lo = ring_mean[b] - clip_k * ring_sigma[b]; - const float hi = ring_mean[b] + clip_k * ring_sigma[b]; - return !(v < lo || v > hi); // exclude peaks / outliers - }; - for (size_t b = 0; b < nbins; ++b) - for (int32_t v = 0; v < HIST_VALUES; ++v) { - const uint32_t n = ring_hist[b * HIST_VALUES + v]; - if (n == 0 || !keep(static_cast(b), v)) continue; - ring_sum[b] += static_cast(n) * v; - ring_sum2[b] += static_cast(n) * static_cast(static_cast(v) * v); - ring_cnt[b] += n; - } - for (const auto &[b, v] : ring_overflow) { - if (!keep(b, v)) continue; - ring_sum[b] += v; - ring_sum2[b] += static_cast(static_cast(v) * v); - ring_cnt[b] += 1; + const auto keep = [&](uint16_t b, int32_t v) { + const float lo = ring_mean[b] - clip_k * ring_sigma[b]; + const float hi = ring_mean[b] + clip_k * ring_sigma[b]; + return !(v < lo || v > hi); // exclude peaks / outliers + }; + for (size_t b = 0; b < nbins; ++b) + for (int32_t v = 0; v < HIST_VALUES; ++v) { + const uint32_t n = ring_hist[b * HIST_VALUES + v]; + if (n == 0 || !keep(static_cast(b), v)) continue; + ring_sum[b] += static_cast(n) * v; + ring_sum2[b] += static_cast(n) * static_cast(static_cast(v) * v); + ring_cnt[b] += n; } + for (const auto &[b, v] : ring_overflow) { + if (!keep(b, v)) continue; + ring_sum[b] += v; + ring_sum2[b] += static_cast(static_cast(v) * v); + ring_cnt[b] += 1; } +} +void AdaptiveSpotFinderCPU::UpdateRingStatistics() { + const size_t nbins = ring_sum.size(); for (size_t b = 0; b < nbins; ++b) { if (ring_cnt[b] > 0) { const double m = static_cast(ring_sum[b]) / ring_cnt[b]; @@ -114,9 +129,16 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, const size_t nbins = ring_sum.size(); // --- Stage A: robust per-ring background (one plain pass + two sigma-clip passes) --- - AccumulateRings(image, 0.0f); - AccumulateRings(image, 3.0f); - AccumulateRings(image, 3.0f); + if (!rings_from_blocks) { + ResetRings(); + AccumulateRingsBlock(image, 0, static_cast(width) * height); + } + rings_from_blocks = false; + UpdateRingStatistics(); + ClipRings(3.0f); + UpdateRingStatistics(); + ClipRings(3.0f); + UpdateRingStatistics(); // --- Stage B: per-ring threshold from the single portable knob E (false pixels / frame) --- int64_t n_total = 0; @@ -153,11 +175,13 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, : adaptive_threshold::RingThreshold(ring_mean[b], ring_sigma[b], p, z); // --- Stage C: the ring threshold, intersected with the classic local-box SNR test --- - FlagRings(image); + std::fill(ring_bits.begin(), ring_bits.end(), 0); if (settings.signal_to_noise_threshold <= 0.0f) { // No local test asked for: the ring threshold alone decides, as the fixed photon floor // alone would in the classic finder. + for (int32_t row = 0; row < height; row++) + FlagRow(image, row); output_buffer = ring_bits; return; } @@ -165,17 +189,17 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, // The ring threshold IS the photon floor here, so the local pass must not apply another one. SpotFindingSettings local = settings; local.photon_count_threshold = 0; - // Only the ring pixels survive the intersection, so the local test is asked of those alone. - DetectAt(image, local, ring_bits); + // Only the ring pixels survive the intersection, so the local test is asked of those alone. They + // are flagged row by row as the local pass reaches each row, which saves reading the image for it. + DetectAt(image, local, ring_bits, [&](int32_t row) { FlagRow(image, row); }); } -void AdaptiveSpotFinderCPU::FlagRings(const ImagePreprocessorBuffer &image) { +void AdaptiveSpotFinderCPU::FlagRow(const ImagePreprocessorBuffer &image, int32_t row) { const auto &pixel_to_bin = mapping.GetPixelToBin(); const size_t nbins = ring_thr.size(); - const size_t npix = static_cast(width) * height; + const size_t first = static_cast(row) * width; - std::bitset<32> out = 0; - for (size_t pxl = 0; pxl < npix; ++pxl) { + for (size_t pxl = first; pxl < first + width; ++pxl) { const int32_t v = image[pxl]; const uint16_t b = pixel_to_bin[pxl]; bool strong = false; @@ -184,14 +208,7 @@ void AdaptiveSpotFinderCPU::FlagRings(const ImagePreprocessorBuffer &image) { else if (v != INT32_MIN && b < nbins && v >= ring_thr[b]) strong = true; - const int32_t bit = pxl % 32; if (strong) - out.set(bit); - if (bit == 31) { - ring_bits[pxl / 32] = out.to_ulong(); - out.reset(); - } + ring_bits[pxl / 32] |= 1U << (pxl % 32); } - if (npix % 32 != 0) - ring_bits[OutputSize() - 1] = out.to_ulong(); } diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h index f741988b9..d0a82aa5f 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h @@ -71,7 +71,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { std::vector ring_hist; std::vector> ring_overflow; // (ring, value) - // The azimuthal-integration profile, taken in the plain AccumulateRings pass when asked for: the + // The azimuthal-integration profile, taken in the plain ring pass when asked for: the // same pixels in the same order and the same arithmetic as AzIntEngineCPU, so the same sums, and // one pass over the image less (the CPU twin of the fused GPU engine). bool fuse_azint = false; @@ -79,15 +79,31 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { std::vector azint_sum2; std::vector azint_count; - void AccumulateRings(const ImagePreprocessorBuffer &image, float clip_k); - // Fill ring_bits from the thresholds of the current frame. - void FlagRings(const ImagePreprocessorBuffer &image); + // Set by BeginRings(): the plain ring pass of the next Detect() is being accumulated block by block. + bool rings_from_blocks = false; + + // Zero the sums of the plain ring pass (and of the fused profile). + void ResetRings(); + // One sigma-clip pass over the plain pass's values. + void ClipRings(float clip_k); + // ring_mean / ring_sigma from the current sums. + void UpdateRingStatistics(); + // Set the ring_bits of the pixels of one row from the thresholds of the current frame. ring_bits + // is zeroed before the first row. + void FlagRow(const ImagePreprocessorBuffer &image, int32_t row); public: explicit AdaptiveSpotFinderCPU(const AzimuthalIntegrationMapping &mapping); void Detect(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings) override; [[nodiscard]] const std::vector &GetRingBackground() const override { return ring_bkg; } + // The plain ring pass taken while the image is being preprocessed, so the pixels are read while + // still in cache: BeginRings(), then AccumulateRingsBlock() over every block of the image in pixel + // order - the order keeps the profile's float sums the same - and the next Detect() of that image + // starts from these sums instead of passing over the image for them. + void BeginRings(); + void AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n); + void FuseAzimuthalIntegration(bool enable) { fuse_azint = enable; } // The profile of the last Detect(), when fused. void GetProfile(AzimuthalIntegrationProfile &profile) const; diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp index e2a41bc15..cc6946cc4 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp @@ -48,24 +48,12 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid, } // namespace void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, - const std::vector &candidates) { + const std::vector &candidates, + const std::function &fill_row) { candidate_windows.clear(); - // The first pass's bits are read only inside the window of a candidate (and at the candidate), so - // it tests only the pixels of the row/column blocks such a window reaches; the bits it leaves - // unset there are never read. - const int32_t nblocks = (width + 31) / 32; - first_pass_needed.assign(static_cast(height) * nblocks, 0); - for (size_t w = 0; w < candidates.size(); w++) - for (uint32_t bits = candidates[w]; bits; bits &= bits - 1) { - const int64_t pxl = static_cast(w) * 32 + std::countr_zero(bits); - if (pxl >= static_cast(width) * height) - break; - const int32_t line = static_cast(pxl / width), col = static_cast(pxl % width); - for (int32_t y = std::max(line - NBX, 0); y <= std::min(line + NBX, height - 1); y++) - for (int32_t b = std::max(col - NBX, 0) / 32; b <= std::min(col + NBX, width - 1) / 32; b++) - first_pass_needed[static_cast(y) * nblocks + b] = 1; - } - DetectPass(image, settings, nullptr, first_pass_buffer, candidates.data()); + // Filled in by DetectPass as the candidates of each row become known. + first_pass_needed.assign(static_cast(height) * ((width + 31) / 32), 0); + DetectPass(image, settings, nullptr, first_pass_buffer, candidates.data(), fill_row); std::fill(output_buffer.begin(), output_buffer.end(), 0); const float strong2 = settings.signal_to_noise_threshold * settings.signal_to_noise_threshold; @@ -107,10 +95,39 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, const uint32_t *prev_strong, std::vector &out_buffer, - const uint32_t *candidates) { + const uint32_t *candidates, + const std::function &fill_row) { for (int i = 0; i < OutputSize(); i++) out_buffer[i] = 0; + // The first pass's bits are read only inside the window of a candidate (and at the candidate), so + // with candidates it tests only the pixels of the row/column blocks such a window reaches; the bits + // it leaves unset there are never read. Those blocks are marked from each row's candidates as the + // row enters the vertical sums below, which is before any row within NBX of it is tested - so the + // candidates themselves can be filled in a row at a time (fill_row), while the row is in cache. + const int32_t nblocks = (width + 31) / 32; + const auto new_row = [&](int32_t r) { + if (!candidates) + return; + if (fill_row) + fill_row(r); + const int32_t first = r * width; + const int32_t last = first + width - 1; + for (int32_t w = first / 32; w <= last / 32; w++) { + uint32_t bits = candidates[w]; + if (w == first / 32) + bits &= UINT32_MAX << (first % 32); + if (w == last / 32 && last % 32 != 31) + bits &= (1U << (last % 32 + 1)) - 1; + for (; bits; bits &= bits - 1) { + const int32_t col = w * 32 + std::countr_zero(bits) - first; + for (int32_t y = std::max(r - NBX, 0); y <= std::min(r + NBX, height - 1); y++) + for (int32_t b = std::max(col - NBX, 0) / 32; b <= std::min(col + NBX, width - 1) / 32; b++) + first_pass_needed[static_cast(y) * nblocks + b] = 1; + } + } + }; + // A pixel found strong by the previous pass reads as INT32_MAX, which the accumulation below // already skips and the acceptance test below already takes as strong - the same substitution // the GPU kernel makes when it reads prev_out. @@ -147,6 +164,7 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, std::vector valid_vert(width, 0); for (int line = 0; line < NBX; line++) { + new_row(line); for (int col = 0; col < width; col++) { auto pxl = line * width + col; @@ -160,6 +178,8 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, } for (int line = 0; line < height; line++) { + if (line < height - NBX) + new_row(line + NBX); for (int col = 0; col < width; col++) { if (line < height - NBX) { auto pxl = (line + NBX) * width + col; @@ -188,7 +208,6 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, // candidate lies in one). Each run starts its window from the vertical sums over the // columns it covers there - integers, so the same sums the running window holds at // that column - and slides it as below; the bits outside the runs stay 0, as below. - const int32_t nblocks = (width + 31) / 32; const uint8_t *needed = first_pass_needed.data() + static_cast(line) * nblocks; for (int32_t b = 0; b < nblocks;) { if (!needed[b]) { b++; continue; } diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.h b/image_analysis/spot_finding/ImageSpotFinderCPU.h index 39e87a270..5b754b2da 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.h +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.h @@ -5,6 +5,7 @@ #include +#include #include #include "ImageSpotFinder.h" @@ -26,10 +27,11 @@ class ImageSpotFinderCPU : public ImageSpotFinder { // fall into, and reported strong. // // With candidates set (SNR test only), the window sums of every candidate pixel are also recorded - // in candidate_windows, in pixel order. + // in candidate_windows, in pixel order, and fill_row (see DetectAt) is called. void DetectPass(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, const uint32_t *prev_strong, std::vector &out, - const uint32_t *candidates = nullptr); + const uint32_t *candidates = nullptr, + const std::function &fill_row = {}); // Local-box sums (centre pixel included) of one candidate pixel, as the first pass saw them. struct CandidateWindow { @@ -47,8 +49,12 @@ protected: // the image: a candidate's second-pass window is its first-pass window with the pixels the first // pass found strong taken out, and those are few, so they are subtracted from the sums the first // pass recorded. Integer sums and the same test, so the bits are exactly Detect's. + // + // fill_row(row), when given, sets the candidates of that row: it is called once for every row, in + // order, before any candidate of the row is read, when the pass is about to read the row anyway. void DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, - const std::vector &candidates); + const std::vector &candidates, + const std::function &fill_row = {}); public: ImageSpotFinderCPU(int32_t width, int32_t height); diff --git a/tests/ZSTDCompressorTest.cpp b/tests/ZSTDCompressorTest.cpp index 5a497601e..fc6c76ee0 100644 --- a/tests/ZSTDCompressorTest.cpp +++ b/tests/ZSTDCompressorTest.cpp @@ -291,6 +291,34 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") { REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0); } +TEST_CASE("JFJochDecompressBlocks_matches_JFJochDecompress","[ZSTD]") { + // An element count that leaves a partial block and a few unshuffled trailing elements. + const size_t nelements = 3 * 8192 + 1000 + 5; + std::vector image(nelements); + std::mt19937 g(1); + std::poisson_distribution dist(3.0); + for (auto &i: image) + i = dist(g); + + for (auto algorithm: {CompressionAlgorithm::BSHUF_LZ4, CompressionAlgorithm::BSHUF_ZSTD}) { + JFJochBitShuffleCompressor compressor(algorithm); + std::vector tmp(nelements * sizeof(uint32_t) * 2 + 12); + tmp.resize(compressor.Compress(tmp.data(), tmp.size(), image)); + + std::vector output(nelements, UINT32_MAX); + size_t next = 0; + JFJochDecompressBlocks(algorithm, reinterpret_cast(tmp.data()), tmp.size(), + nelements, sizeof(uint32_t), + [&](size_t first, const uint8_t *data, size_t n) { + REQUIRE(first == next); + memcpy(output.data() + first, data, n * sizeof(uint32_t)); + next = first + n; + }); + REQUIRE(next == nelements); + REQUIRE(output == image); + } +} + TEST_CASE("JFJochDecompressor_None","[ZSTD]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::NO_COMPRESSION).BitDepthImage(32).PixelSigned(true); From c8692e320c968680535a0209c1f664b766742ab4 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:19:31 +0200 Subject: [PATCH 07/12] CPU pixel loops: ring sums held in registers, vectorised vertical window, shared packed mask AdaptiveSpotFinderCPU::AccumulateRings: consecutive pixels mostly share a ring, so the ring's sum/sum2/count and the fused azint sums are held in locals while they do and stored when the ring changes - the same additions in the same order (az_sum2 is still contracted to the same FMA), without a store-and-reload chain through memory on every pixel. ImageSpotFinderCPU::DetectPass: the vertical-sum update (add the entering row, take out the leaving one) is one branch-free loop over the raw image that GCC vectorises (int64 lanes). A pixel strong in the previous pass used to be substituted per pixel through a bit test, which kept the loop scalar; it is now added with its row and taken out again from the few set bits of prev_strong. Integer sums, so the same totals. (A first, fully branch-free version that kept the per-pixel bit test did not vectorise on the prev_strong path and was measured slower; this is its replacement.) ImagePreprocessorCPU: the per-engine std::vector built bit by bit from the 32-bit mask (~10 core-s per cytc run, one per worker per pass) is replaced by 32-pixel mask words that PixelMask derives once beside its binary mask; each engine copies 2 MB. A branch-free rewrite of the Analyze loop was measured and dropped: the loop is bound by reading the decompressed image (330 vs 328 core-s on cytc), so only the mask test changed. Measured (perf, 499 Hz, CPU-only build, cytc, first version of this change): AccumulateRings 591 -> 539 core-s. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob, lyso (GPU). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- common/PixelMask.cpp | 7 ++ common/PixelMask.h | 4 + .../ImagePreprocessorCPU.cpp | 7 +- .../ImagePreprocessorCPU.h | 4 +- .../spot_finding/AdaptiveSpotFinderCPU.cpp | 46 ++++++++-- .../spot_finding/ImageSpotFinderCPU.cpp | 85 ++++++++++++------- 6 files changed, 107 insertions(+), 46 deletions(-) diff --git a/common/PixelMask.cpp b/common/PixelMask.cpp index a5338f4ee..8e455195e 100644 --- a/common/PixelMask.cpp +++ b/common/PixelMask.cpp @@ -59,6 +59,9 @@ void PixelMask::UpdateBinaryMask() { for (size_t i = 0; i < mask.size(); i++) binary_mask[i] = (mask[i] != 0); binary_mask_checksum = TableChecksum(binary_mask.data(), binary_mask.size()); + packed_mask.assign((mask.size() + 31) / 32, 0); + for (size_t i = 0; i < mask.size(); i++) + packed_mask[i / 32] |= static_cast(binary_mask[i]) << (i % 32); } void PixelMask::CalcEdgePixels_i(const DiffractionExperiment &experiment) { @@ -127,6 +130,10 @@ const std::vector &PixelMask::GetBinaryMask() const { return binary_mask; } +const std::vector &PixelMask::GetPackedMask() const { + return packed_mask; +} + uint64_t PixelMask::GetBinaryMaskChecksum() const { return binary_mask_checksum; } diff --git a/common/PixelMask.h b/common/PixelMask.h index 5408b0789..ddcf50be2 100644 --- a/common/PixelMask.h +++ b/common/PixelMask.h @@ -26,6 +26,9 @@ class PixelMask { // derived here once instead of in every one of those engines. std::vector binary_mask; uint64_t binary_mask_checksum = 0; + // The same flag packed 32 pixels to a word (pixel i is bit i % 32 of word i / 32) - the form the CPU + // image preprocessor reads, derived here for the same reason. + std::vector packed_mask; uint32_t LoadMask(const std::vector& mask, uint8_t bit); // Everything that follows from the mask, recomputed wherever the mask changes. @@ -68,6 +71,7 @@ public: [[nodiscard]] const std::vector &GetMask(const DiffractionExperiment& experiment) const; [[nodiscard]] const std::vector &GetMask() const; [[nodiscard]] const std::vector &GetBinaryMask() const; + [[nodiscard]] const std::vector &GetPackedMask() const; [[nodiscard]] uint64_t GetBinaryMaskChecksum() const; [[nodiscard]] std::vector GetUserMask(const DiffractionExperiment& experiment) const; diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp index 8803a73db..bb3a98931 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp @@ -5,10 +5,7 @@ ImagePreprocessorCPU::ImagePreprocessorCPU(const DiffractionExperiment &experiment, const PixelMask &mask) : ImagePreprocessor(experiment), - mask_1bit(npixels, false) { - for (int i = 0; i < npixels; i++) - mask_1bit[i] = (mask.GetMask().at(i) != 0); -} + mask_bits(mask.GetPackedMask()) {} ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *image_ptr, CompressedImageMode image_mode) { switch (image_mode) { @@ -43,7 +40,7 @@ ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed sat_pixel_val = static_cast(saturation_limit); for (int i = 0; i < npixels; i++) { - if (mask_1bit[i] != 0) { + if ((mask_bits[i / 32] >> (i % 32)) & 1U) { processed_image[i] = INT32_MIN; ++ret.masked_pixel_count; } else if (image[i] == err_pixel_val) { diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h index aa326d77b..2b07bd54a 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h @@ -7,7 +7,9 @@ class ImagePreprocessorCPU : public ImagePreprocessor { - std::vector mask_1bit; + // The pixel mask one bit per pixel, 32 to a word - PixelMask derives it once, so each engine only + // copies it. + std::vector mask_bits; template ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *input, T err_value, T sat_value); public: diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 93b7b9597..ef66199a3 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -55,26 +55,58 @@ void AdaptiveSpotFinderCPU::AccumulateRings(const ImagePreprocessorBuffer &image std::fill(azint_sum2.begin(), azint_sum2.end(), 0.0f); std::fill(azint_count.begin(), azint_count.end(), 0); } + // Consecutive pixels mostly share a ring, so the ring's sums are held in locals while they do + // and written back when the ring changes: the same additions in the same order, without a + // store and a reload of the same address on every pixel. + uint32_t *hist = ring_hist.data(); + size_t cur = nbins; // the ring held in the locals below; nbins = none + int64_t sum = 0, cnt = 0; + uint64_t sum2 = 0; + float az_sum = 0.0f, az_sum2 = 0.0f; + uint32_t az_cnt = 0; + const auto store = [&] { + if (cur == nbins) + return; + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + if (fuse_azint) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + }; for (size_t pxl = 0; pxl < npix; ++pxl) { const int32_t v = image[pxl]; if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated const uint16_t b = pixel_to_bin[pxl]; if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (b != cur) { + store(); + cur = b; + sum = ring_sum[b]; + sum2 = ring_sum2[b]; + cnt = ring_cnt[b]; + az_sum = azint_sum[b]; + az_sum2 = azint_sum2[b]; + az_cnt = azint_count[b]; + } if (fuse_azint) { const float val = static_cast(v) * corrections[pxl]; const float val_sq = val * val; - azint_sum[b] += val; - azint_sum2[b] += val_sq; - ++azint_count[b]; + az_sum += val; + az_sum2 += val_sq; + ++az_cnt; } - ring_sum[b] += v; - ring_sum2[b] += static_cast(static_cast(v) * v); - ring_cnt[b] += 1; + sum += v; + sum2 += static_cast(static_cast(v) * v); + cnt += 1; if (v >= 0 && v < HIST_VALUES) - ring_hist[b * HIST_VALUES + v] += 1; + hist[b * HIST_VALUES + v] += 1; else ring_overflow.emplace_back(b, v); } + store(); } else { // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same // test the pixels holding it would, and its pixels are added as a count. diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp index e2a41bc15..b69993fa6 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp @@ -146,42 +146,61 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, std::vector sum2_vert(width, 0); std::vector valid_vert(width, 0); - for (int line = 0; line < NBX; line++) { - for (int col = 0; col < width; col++) { - auto pxl = line * width + col; - - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] += tmp; - sum2_vert[col] += tmp * tmp; - valid_vert[col] += 1; - } + // Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is + // no row. A bad or saturated pixel adds 0. Written without branches, so the loop vectorises. + // A pixel strong in the previous pass is not counted either; it is added here with the rest + // of its row and taken out again below - there are few, and the sums are integers, so the + // totals are the same as never adding it. + const int32_t *img = image.data(); + const int32_t w = width; + int64_t *sv = sum_vert.data(); + int64_t *sv2 = sum2_vert.data(); + uint16_t *vv = valid_vert.data(); + auto slide_vert = [&](int line_in, int line_out) { + for (int32_t col = 0; col < w; col++) { + const int64_t in = line_in >= 0 ? img[line_in * w + col] : INT32_MIN; + const int64_t out = line_out >= 0 ? img[line_out * w + col] : INT32_MIN; + const bool in_ok = in != INT32_MAX && in != INT32_MIN; + const bool out_ok = out != INT32_MAX && out != INT32_MIN; + const int64_t in_v = in_ok ? in : 0; + const int64_t out_v = out_ok ? out : 0; + sv[col] += in_v - out_v; + sv2[col] += in_v * in_v - out_v * out_v; + vv[col] += static_cast(in_ok - out_ok); } - } + if (!prev_strong) + return; + // sign = +1 takes out what line_in added, -1 puts back what line_out took out. + const auto uncount_strong = [&](int line, int64_t sign) { + const int32_t first = line * w, last = first + w - 1; + for (int32_t word = first / 32; word <= last / 32; word++) { + uint32_t bits = prev_strong[word]; + if (word == first / 32) + bits &= UINT32_MAX << (first % 32); + if (word == last / 32 && last % 32 != 31) + bits &= (1U << (last % 32 + 1)) - 1; + for (; bits; bits &= bits - 1) { + const int32_t pxl = word * 32 + std::countr_zero(bits); + const int64_t v = img[pxl]; + if (v == INT32_MAX || v == INT32_MIN) + continue; + sv[pxl - first] -= sign * v; + sv2[pxl - first] -= sign * v * v; + vv[pxl - first] -= static_cast(sign); + } + } + }; + if (line_in >= 0) + uncount_strong(line_in, 1); + if (line_out >= 0) + uncount_strong(line_out, -1); + }; + + for (int line = 0; line < NBX; line++) + slide_vert(line, -1); for (int line = 0; line < height; line++) { - for (int col = 0; col < width; col++) { - if (line < height - NBX) { - auto pxl = (line + NBX) * width + col; - - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] += tmp; - sum2_vert[col] += tmp * tmp; - valid_vert[col] += 1; - } - } - - if (line >= NBX + 1) { - auto pxl = (line - (NBX + 1)) * width + col; - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] -= tmp; - sum2_vert[col] -= tmp * tmp; - valid_vert[col] -= 1; - } - } - } + slide_vert(line < height - NBX ? line + NBX : -1, line >= NBX + 1 ? line - (NBX + 1) : -1); if (candidates) { // Only the runs of 32-column blocks whose result is read (first_pass_needed; every From 59c4d6fe41c96f4547665436b26d9bf7793a7e4c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:19:51 +0200 Subject: [PATCH 08/12] BraggPredictionRot: reject far-off-frame solutions before the atan2 Most rotation solutions are far from the frame, and the existing phi prefilter rejected them only after computing phi = atan2(sin, cos). The same test is now asked of the (cos, sin) pair first: |phi| > phi_limit is cos(phi) < cos(phi_limit), i.e. c < cos(phi_limit) * |(c, s)| in double. phi_limit is the existing test's threshold at the largest rocking width any reflection can have (bandwidth term at the resolution limit) plus a 1e-3 rad margin, far above float rounding, so every solution rejected here is rejected by the existing test too, and the survivors go through it unchanged. One cos per Calc; min_zeta <= 0 disables it, as it disables the existing test. Measured (perf, 499 Hz, CPU-only build, cytc): atanf32 + atan2f32 + __atan2f_finite 196 -> 12 core-s, Calc itself 161 -> 199 core-s (the sqrt of the new test), net ~ -150 core-s. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob, lyso (GPU); Bragg* tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../bragg_prediction/BraggPredictionRot.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/image_analysis/bragg_prediction/BraggPredictionRot.cpp b/image_analysis/bragg_prediction/BraggPredictionRot.cpp index dbcd90fd4..4a516262f 100644 --- a/image_analysis/bragg_prediction/BraggPredictionRot.cpp +++ b/image_analysis/bragg_prediction/BraggPredictionRot.cpp @@ -77,6 +77,28 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys experiment.GetBraggIntegrationSettings().GetFlightPath(), geom.GetDetectorDistance_mm(), geom.GetWavelength_A()); + // The phi test after the atan2 below, taken before it: a solution with |phi| above phi_limit fails + // that test whatever its own bandwidth term, since mos_eff_rad is largest at the resolution limit. + // |phi| > phi_limit is cos(phi) < cos(phi_limit), which the (cos, sin) pair answers without the + // angle. phi_limit carries a 1e-3 rad margin, far above float rounding, so every solution rejected + // here is rejected by that test too. -1 = nothing is rejected here. + double cos_phi_limit = -1.0; + if (settings.min_zeta > 0.0f) { + double mos_max_rad = mos_angle_rad; + if (bandwidth_sigma > 0.0f) { + const double sin_theta = half_wavelength_A * static_cast(one_over_dmax); + const double dphi_bw = sin_theta < 1.0 + ? bandwidth_sigma * sin_theta / std::sqrt(1.0 - sin_theta * sin_theta) + : INFINITY; + mos_max_rad = std::sqrt(mos_max_rad * mos_max_rad + dphi_bw * dphi_bw); + } + const double phi_limit = half_wedge_angle_rad + + (settings.mosaicity_multiplier * mos_max_rad + 1e-5) / settings.min_zeta + + 1e-3; + if (phi_limit < PI) + cos_phi_limit = std::cos(phi_limit); + } + for (int h = -settings.max_h; h <= settings.max_h; h++) { // Precompute A* h contribution @@ -127,6 +149,9 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys const float cosphi = (p_m1 * p0_m1 + p_m3 * p0_m3) / rho_sq; const float sinphi = (p_m1 * p0_m3 - p_m3 * p0_m1) / rho_sq; + const double c = cosphi, s = sinphi; + if (c < cos_phi_limit * std::sqrt(c * c + s * s)) + continue; float phi = -1.0f * std::atan2(sinphi, cosphi); // Most solutions are far from this frame. zeta >= min_zeta, so the rocking-curve test From 2bad92ddd6c86a90c20977924f80ec79c11a46a6 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:19:51 +0200 Subject: [PATCH 09/12] RotationScaleMerge: the CPU partial scaling loop works on a compact copy of the partials Each iteration of the partial scaling loop passes over all partials three times (group means, per-frame G fit, corr update + rescale), each time reading a few fields of the 80-byte Obs, and the group means gathered whole Obs records through the group permutation. The loop now works on ScalingObs (frame, group, I, sigma, partiality, prescaling_corr, zeta, on_ice; 36 bytes, partials' order) with corr in its own float array, and the group means read ScalingRef (I, sigma, partiality) laid out in the CSR's group order, so only corr is gathered. corr is written back to the partials (and kept by the memo) when the loop ends. ReferenceWeight takes the fields, FitPerFrameG is a template over Obs/ScalingObs; the same values, so the same arithmetic in the same order. Transient memory: 52 bytes per partial during the loop (~0.4 GB on 8M partials). Measured (perf, 499 Hz, CPU-only build, cytc): update/rescale lambda 104 -> 35, group means 87 -> 24, FitPerFrameG 63 -> 27 core-s (-167 core-s, of which the copy costs ~10). CPU-only path; the GPU scaling path is untouched. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob, lyso (GPU); rotation scale/merge tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../scale_merge/RotationScaleMerge.cpp | 88 ++++++++++++++----- .../scale_merge/RotationScaleMerge.h | 27 +++++- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 10c5829d6..1178fe5eb 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -1336,12 +1336,13 @@ int RotationScaleMerge::ComputeAsuGroups(const HKLKeyGenerator &keygen) { return n_groups; } -double RotationScaleMerge::ReferenceWeight(const Obs &o, double min_partiality, float &I_corr) { - if (o.group < 0) return 0.0; - if (!(o.corr > 0.0f) || !std::isfinite(o.corr)) return 0.0; - if (o.partiality < min_partiality) return 0.0; - const float Ic = o.I * o.corr; - const float sigma_corr = o.sigma * o.corr; +double RotationScaleMerge::ReferenceWeight(int32_t group, float corr, float partiality, float I, float sigma, + double min_partiality, float &I_corr) { + if (group < 0) return 0.0; + if (!(corr > 0.0f) || !std::isfinite(corr)) return 0.0; + if (partiality < min_partiality) return 0.0; + const float Ic = I * corr; + const float sigma_corr = sigma * corr; if (!std::isfinite(Ic) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) return 0.0; I_corr = Ic; return 1.0 / (static_cast(sigma_corr) * sigma_corr); @@ -1383,7 +1384,30 @@ void RotationScaleMerge::ReducePartialGroupMeans(int n_groups, std::vector &obs, const std::vector &fstart, +void RotationScaleMerge::ReduceScalingGroupMeans(int n_groups, const std::vector &refs, + const std::vector &corr, + std::vector &out_mean) const { + out_mean.assign(n_groups, NAN); + ParallelChunks(n_groups, ThreadsForWork(corr.size(), nthreads), [&](int glo, int ghi) { + for (int g = glo; g < ghi; ++g) { + double sw = 0.0, swI = 0.0; + const int lo = partial_group_start[g], hi = lo + partial_group_count[g]; + for (int p = lo; p < hi; ++p) { + const ScalingRef &r = refs[p]; + float I_corr = 0.0f; + const double w = ReferenceWeight(g, corr[partial_group_perm[p]], r.partiality, r.I, r.sigma, + min_partiality, I_corr); + if (w <= 0.0) continue; + sw += w; + swI += w * I_corr; + } + if (sw > 0.0) out_mean[g] = swI / sw; + } + }); +} + +template +void RotationScaleMerge::FitPerFrameG(const std::vector &obs, const std::vector &fstart, const std::vector &fcount, const std::vector &group_mean_in, bool unity, std::vector &g) { @@ -5560,6 +5584,30 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st logger.Info("Per-frame scaling (partials): the same grouping and settings as an earlier merge, " "so its {} iterations stand", partial_loop.iterations); } else { + // The loop works on its own compact copy of the partials (ScalingObs in the header) and hands + // corr back to them when it is done. + const int n_partials = static_cast(partials.size()); + const size_t partial_threads = ThreadsForWork(partials.size(), nthreads); + std::vector scaling_obs(n_partials); + std::vector scaling_corr(n_partials); + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) { + const Obs &o = partials[i]; + scaling_obs[i] = {o.frame, o.group, o.I, o.sigma, o.partiality, o.prescaling_corr, o.zeta, + o.on_ice}; + scaling_corr[i] = o.corr; + } + }); + std::vector scaling_refs(partial_group_start.empty() + ? 0 : partial_group_start.back() + partial_group_count.back()); + ParallelChunks(n_groups, partial_threads, [&](int glo, int ghi) { + for (int g = glo; g < ghi; ++g) + for (int p = partial_group_start[g]; p < partial_group_start[g] + partial_group_count[g]; ++p) { + const ScalingObs &o = scaling_obs[partial_group_perm[p]]; + scaling_refs[p] = {o.I, o.sigma, o.partiality}; + } + }); + // RunScalingLoop rescales after every iteration before anything reads corr, so the iteration's // UpdateCorr is folded into that pass over the partials: the G and the fitted frames as the // iteration left them, then the rescale on top - the same two roundings, one pass instead of two. @@ -5567,34 +5615,32 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st std::vector frame_fitted; partial_loop = RunScalingLoop("partials", g_partial, frame_count, smooth_window, /*release_restraint=*/false, [&] { - ReducePartialGroupMeans(n_groups, partial_mean); - FitPerFrameG(partials, frame_start, frame_count, partial_mean, /*unity=*/false, g_partial); + ReduceScalingGroupMeans(n_groups, scaling_refs, scaling_corr, partial_mean); + FitPerFrameG(scaling_obs, frame_start, frame_count, partial_mean, /*unity=*/false, g_partial); g_fitted = g_partial; frame_fitted = frame_scaled_scratch; }, [&](const std::vector &apply, const std::vector &ratio) { - ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), - [&](int lo, int hi) { + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { for (int i = lo; i < hi; ++i) { - Obs &o = partials[i]; + const ScalingObs &o = scaling_obs[i]; + float &corr = scaling_corr[i]; if (frame_fitted[o.frame]) { // UpdateCorr const double denom = static_cast(o.partiality) * g_fitted[o.frame]; if (std::isfinite(o.prescaling_corr) && std::isfinite(denom) && denom > 0.0) - o.corr = static_cast(o.prescaling_corr / denom); + corr = static_cast(o.prescaling_corr / denom); else - o.corr = NAN; + corr = NAN; } - if (apply[o.frame] && std::isfinite(o.corr)) - o.corr = static_cast(o.corr * ratio[o.frame]); + if (apply[o.frame] && std::isfinite(corr)) + corr = static_cast(corr * ratio[o.frame]); } }); }); - PartialLoopMemo m{key, partial_loop, frame_scaled_scratch, g_partial, {}}; - m.corr.resize(partials.size()); - ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), - [&](int lo, int hi) { - for (int i = lo; i < hi; ++i) m.corr[i] = partials[i].corr; + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) partials[i].corr = scaling_corr[i]; }); + PartialLoopMemo m{key, partial_loop, frame_scaled_scratch, g_partial, std::move(scaling_corr)}; if (partial_loop_memos.size() == MAX_PARTIAL_LOOP_MEMOS) partial_loop_memos.erase(partial_loop_memos.begin()); partial_loop_memos.push_back(std::move(m)); diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index ebef3de15..ea1947c8f 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -446,17 +446,40 @@ private: std::vector partial_group_start, partial_group_count; std::unique_ptr partial_group_perm; + // The CPU partial scaling loop's working copy of the partials. It passes over them several times + // per iteration and reads a handful of fields each time, so it works on these rather than on the + // eighty-byte Obs: ScalingObs holds, in the partials' own order, what FitPerFrameG and the corr + // update read; ScalingRef holds, in the CSR's group order, what the group means read besides corr, + // so that only corr is gathered. The same values, so the same arithmetic in the same order. + struct ScalingObs { + int32_t frame, group; + float I, sigma, partiality, prescaling_corr, zeta; + uint8_t on_ice; + }; + struct ScalingRef { + float I, sigma, partiality; + }; + // The observation's inverse-variance weight in the scaling reference, and its scaled intensity; // 0 when the observation is not in the reference at all. - static double ReferenceWeight(const Obs &o, double min_partiality, float &I_corr); + static double ReferenceWeight(int32_t group, float corr, float partiality, float I, float sigma, + double min_partiality, float &I_corr); + static double ReferenceWeight(const Obs &o, double min_partiality, float &I_corr) { + return ReferenceWeight(o.group, o.corr, o.partiality, o.I, o.sigma, min_partiality, I_corr); + } // Inverse-variance per-group mean of I*corr over `obs` (the merge reference). void ReduceGroupMeans(const std::vector &obs, int n_groups, std::vector &out_mean) const; // The same over the partials, using the CSR above - the same sums in the same order. void ReducePartialGroupMeans(int n_groups, std::vector &out_mean) const; + // The same again over the scaling loop's copy: refs in group order, corr in the partials' order. + void ReduceScalingGroupMeans(int n_groups, const std::vector &refs, + const std::vector &corr, std::vector &out_mean) const; // Robust per-frame G fit (IRLS, Cauchy k=3), unity=false uses the rotation partiality, unity=true the // scale-fulls (partiality already folded in). Reads out_mean[group] as the reference intensity. - void FitPerFrameG(std::vector &obs, const std::vector &fstart, + // O is Obs or ScalingObs. + template + void FitPerFrameG(const std::vector &obs, const std::vector &fstart, const std::vector &fcount, const std::vector &group_mean_in, bool unity, std::vector &g); From 61d88cdefe7b9afc257d9075c5c6ca550bdd945d Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:19:51 +0200 Subject: [PATCH 10/12] BraggIntegrationEngineCPU: reflection mask and owner map kept as 16x16 tiles refl_mask (uint8) and owner (uint32) were full-frame arrays per engine, ~90 MB at 18 Mpixel, of which a call writes only the neighbourhoods of its reflections; over a sweep every page of both got touched in every worker, and the dirty-rectangle clears were row-strided memsets. They are now a TiledFrame: a per-tile directory plus a pool of 16x16 tiles allocated on first write and dropped by Clear(); reading a tile nothing wrote gives the empty value (0 / BRAGG_OWNER_NONE). Writes and reads are exactly those of the flat arrays, so the same ownership decisions. Measured (perf, 499 Hz, CPU-only build, cytc): RunImpl 1633 -> 1557 core-s, memset 88 -> 32 core-s. Peak RSS and page faults did not move measurably (myob 15.3 GB; the peak is set elsewhere). Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob, lyso (GPU); Bragg* tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../BraggIntegrationEngineCPU.cpp | 32 ++++--------- .../BraggIntegrationEngineCPU.h | 47 +++++++++++++++---- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp index 64a864868..edbe47b1b 100644 --- a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp +++ b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp @@ -44,7 +44,9 @@ struct ImageSampler { } // namespace BraggIntegrationEngineCPU::BraggIntegrationEngineCPU(const DiffractionExperiment &experiment) - : BraggIntegrationEngine(experiment) {} + : BraggIntegrationEngine(experiment), + refl_mask(static_cast(xpixel), static_cast(ypixel), 0), + owner(static_cast(xpixel), static_cast(ypixel), BRAGG_OWNER_NONE) {} template std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, @@ -71,17 +73,8 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, // signal is excluded from this reflection's r2..r3 background ring. The region is the INNER // stencil ellipse, taken in each neighbour's OWN frame - an elongated neighbour whose streak // is still marked as a disk would leak its tails into this reflection's ring. --- - // Back to empty what the previous call wrote (see the header). - for (const Rect &b : refl_mask_dirty) - for (int y = b.y0; y <= b.y1; ++y) - std::fill(refl_mask.begin() + y * W + b.x0, refl_mask.begin() + y * W + b.x1 + 1, 0); - refl_mask_dirty.clear(); - for (const Rect &b : owner_dirty) - for (int y = b.y0; y <= b.y1; ++y) - std::fill(owner.begin() + y * W + b.x0, owner.begin() + y * W + b.x1 + 1, BRAGG_OWNER_NONE); - owner_dirty.clear(); - - refl_mask.resize(npixel, 0); + refl_mask.Clear(); + owner.Clear(); for (size_t i = 0; i < npredicted; ++i) { const auto &r = predicted[i]; const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil); @@ -89,12 +82,10 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const int x1 = std::min(W - 1, static_cast(std::ceil(r.predicted_x + st.ex_in + 1.0f))); const int y0 = std::max(0, static_cast(std::floor(r.predicted_y - st.ey_in - 1.0f))); const int y1 = std::min(H - 1, static_cast(std::ceil(r.predicted_y + st.ey_in + 1.0f))); - if (x0 <= x1 && y0 <= y1) - refl_mask_dirty.push_back({x0, x1, y0, y1}); for (int y = y0; y <= y1; ++y) for (int x = x0; x <= x1; ++x) { const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y); - if (d.inner < r2_sq) refl_mask[y * W + x] = 1; + if (d.inner < r2_sq) refl_mask.At(x, y) = 1; } } @@ -106,7 +97,6 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const bool exclude = overlap == OverlapMode::Exclude && mode != IntegratorMode::BoxSum; const bool use_owner = overlap != OverlapMode::Off; if (use_owner) { - owner.resize(npixel, BRAGG_OWNER_NONE); const float claim_sq = claim * claim; for (size_t i = 0; i < npredicted; ++i) { const auto &r = predicted[i]; @@ -114,21 +104,19 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const int x1 = std::min(W - 1, static_cast(std::ceil(r.predicted_x + claim))); const int y0 = std::max(0, static_cast(std::floor(r.predicted_y - claim))); const int y1 = std::min(H - 1, static_cast(std::ceil(r.predicted_y + claim))); - if (x0 <= x1 && y0 <= y1) - owner_dirty.push_back({x0, x1, y0, y1}); for (int y = y0; y <= y1; ++y) for (int x = x0; x <= x1; ++x) { const float dx = x - r.predicted_x, dy = y - r.predicted_y; const float d2 = dx * dx + dy * dy; if (d2 >= claim_sq) continue; - uint32_t &o = owner[y * W + x]; + uint32_t &o = owner.At(x, y); o = std::min(o, BraggOwnerKey(std::sqrt(d2), inv_claim, static_cast(i))); } } } auto clean = [&](int x, int y, size_t i) { return !use_owner - || BraggOwnedBy(owner[static_cast(y) * W + x], static_cast(i)); + || BraggOwnedBy(owner.Get(x, y), static_cast(i)); }; // --- Pass A: box-sum every reflection (rough I, background, centroid, strong flag). --- @@ -213,7 +201,7 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, y_sum += y; ++n_inner_valid; } else if (d.inner >= r2_sq && d.outer < r3_sq) { - if (refl_mask[y * W + x]) { ++n_bkg_neighbour; continue; } + if (refl_mask.Get(x, y)) { ++n_bkg_neighbour; continue; } if (!valid(px)) continue; bkg_sum += static_cast(px); if (bkg_trim_frac > 0.0) bkg_vals.push_back(px); @@ -267,7 +255,7 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, for (int x = x0; x <= x1; ++x) { const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y); if (!(d.inner >= r2_sq && d.outer < r3_sq)) continue; - if (refl_mask[y * W + x]) continue; + if (refl_mask.Get(x, y)) continue; const int32_t px = img[y * W + x]; if (!valid(px)) continue; if (static_cast(px) <= thr) { diff --git a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h index ed1861599..f125f391b 100644 --- a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h +++ b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h @@ -14,15 +14,44 @@ class BraggIntegrationEngineCPU : public BraggIntegrationEngine { // Core integrator, templated on a pixel sampler so it reads either the preprocessed int32 buffer // or a raw CompressedImage of any pixel type - both presented per-pixel in the INT32_MIN(masked)/ // INT32_MAX(saturated) convention - without ever materialising a second full-image copy. - // Full-frame scratch of RunImpl: the reflection mask and the signal-region owner map. Kept from - // call to call rather than allocated per image - at 18 Mpixel that was 90 MB, a fresh mmap and a - // page fault per 4 kB every call - and put back to their empty state one rectangle at a time: the - // rectangles a call writes are recorded, and the next call clears exactly those before it starts. - struct Rect { int x0, x1, y0, y1; }; - std::vector refl_mask; - std::vector owner; - std::vector refl_mask_dirty; - std::vector owner_dirty; + // Full-frame scratch of RunImpl: the reflection mask and the signal-region owner map. A call writes + // only around its reflections, so these keep only the 16x16-pixel tiles written since the last + // Clear(): a frame-sized array was mostly never read, yet over a sweep every page of it got + // touched, in every worker. Reading a tile nothing wrote gives `empty`. + template + class TiledFrame { + static constexpr int TILE = 16; + int tiles_x; + T empty; + std::vector tile_start; // per tile: where it starts in `pixels`, -1 = not written + std::vector written; // the tiles written, for Clear() + std::vector pixels; + public: + TiledFrame(int width, int height, T empty) + : tiles_x((width + TILE - 1) / TILE), empty(empty), + tile_start(static_cast(tiles_x) * ((height + TILE - 1) / TILE), -1) {} + T Get(int x, int y) const { + const int32_t start = tile_start[(y / TILE) * tiles_x + x / TILE]; + return start < 0 ? empty : pixels[start + (y % TILE) * TILE + x % TILE]; + } + T &At(int x, int y) { + const int t = (y / TILE) * tiles_x + x / TILE; + if (tile_start[t] < 0) { + tile_start[t] = static_cast(pixels.size()); + pixels.resize(pixels.size() + TILE * TILE, empty); + written.push_back(t); + } + return pixels[tile_start[t] + (y % TILE) * TILE + x % TILE]; + } + void Clear() { + for (int t : written) + tile_start[t] = -1; + written.clear(); + pixels.clear(); + } + }; + TiledFrame refl_mask; + TiledFrame owner; template std::vector RunImpl(const Sampler &img, const std::vector &predicted, From 1fd28672bb5e7effd65aad9a700d0fa1604dc4cb Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 02:23:04 +0200 Subject: [PATCH 11/12] ModelValidation: draw the null's placements away from every orientation equivalent to the model's The null is nine fixed-seed random orientations of the model, each placed by the same rigid body as the real fit and fitted to the data AS MERGED. Where the indexing probe reindexed the data to the model's, the model's solution against the merged data is its twin-related orientation - and more generally the crystal looks the same from every orientation the lattice's rotations (the space group's, composed with the twin laws) take the model to. A draw within the rigid body's reach of one of those is walked onto it and is not a sample of the null. On 5epe (F 2 3, twin law y,x,-z, 24 equivalent orientations) replicate 8 of 9 was drawn 7.9 deg from one of them; the rigid body moved it 7.9 deg, held-out R-free 0.53 -> 0.20, R-work 0.197 against 0.56-0.58 for the other eight. The null's sd became 0.125 and the crystal's own model read +0.01 sigma, DOES NOT FIT, so the indexing it had decided at +25.8 sigma was refused and the files kept the merged indexing (R-free 0.52). This happens on every binary whose indexer picks the other hand. The fix is at the draw, not the score: a draw whose angle to the nearest equivalent orientation is under the rigid body's reach is drawn again from the same generator, so everywhere no draw falls there the nine rotations - and every output - are exactly the ones before. The reach is the rotation about the centroid that moves the model's rms-radius atom by the ladder's first zone (6 A, or d_min where coarser): 18.7 deg on 5epe, whose other draws sat at 21.7 deg and beyond and did not converge. Twin laws come from the lattice (ReindexAmbiguityOperators on the data cell and group) regardless of whether the probe ran. Redraws are capped at 1000 so that a model small enough for the reach to cover the whole of SO(3) takes its draws as they come instead of looping. Scoring the real model against the best of both hands instead was not taken: the real fit is already scored in the hand that wins, and the contamination is in the null, not in the real fit. Measured, battery commands with --model, rc173 27e154e5d vs this: 5epe 1 redraw; DOES NOT FIT +0.01 -> FITS +55.86 sigma (null 0.5730 +- 0.0072); indexing decided at +40.4 sigma and reindexed y,x,-z; R-free 0.5246 -> 0.1749 (deposited 0.1671) 9fhc 1 redraw (reach 8.8 deg; the replaced draw had not converged); files byte-identical, MODEL_FIT_SIGMA +201.27 -> +199.39, verdict unchanged 9hnc, 8sa8, 8xtg, 9bn8: no redraw; p.hkl, p.mtz, p.cif (bar the audit date), p_maps.mtz, p_model.cif and the model-validation report byte-identical [ModelValidation] tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- docs/CHANGELOG.md | 1 + rugnux/ModelValidation.cpp | 60 ++++++++++++++++++++++++++++++++++++-- rugnux/RigidBodyRefine.cpp | 18 ++++++++++++ rugnux/RigidBodyRefine.h | 6 ++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b9ab621ee..ed23eb167 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ * jfjoch_viewer's reference dataset accepts a structure-factor mmCIF as well as an MTZ, and a processing job can name an atomic model to validate the merged data against, as `rugnux --model` does. * jfjoch_viewer keeps a separate preferred dataset-info plot for grid scans, where "Spots + background" means the spot count. * jfjoch_viewer's dark theme no longer leaves navy buttons, red warnings and chart guide lines at their light-theme colours. +* `rugnux --model` draws its null's random placements away from every orientation equivalent to the model's under the space group or a twin law, so a random placement can no longer be refined onto the model's own solution and make the crystal's own model read as not fitting. * Rugnux adds beam-stop holder arms that let part of the beam through to the beam-stop mask, without changing the mask of a sweep that has none. ### 1.0.0-rc.173 diff --git a/rugnux/ModelValidation.cpp b/rugnux/ModelValidation.cpp index 84bbb1d45..1eeef5fa5 100644 --- a/rugnux/ModelValidation.cpp +++ b/rugnux/ModelValidation.cpp @@ -91,6 +91,11 @@ constexpr int NULL_REPLICATES = 9; // Fixed, so the same data and the same model give the same verdict on every run. constexpr unsigned NULL_SEED = 20260902; +// How many draws the null may throw away for lying within the rigid body's reach of the model's own +// orientation. Only a model small enough for that reach to cover most orientations gets near it, and +// there the remaining draws are taken as they come rather than the run searching forever. +constexpr int NULL_MAX_REDRAWS = 1000; + // How far above its own null a fit has to sit before the model is allowed to decide anything. The cut // is in sigma of that null and not in R: measured, the R a model that explains nothing reaches moves // with the model's atom count and B-factors as much as with the data, so no value of R separates the @@ -122,6 +127,27 @@ gemmi::Mat33 random_rotation(std::mt19937 &rng) { 2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)}; } +// A rotation of the lattice, given on fractional coordinates as a symmetry operator is, in Cartesian +// coordinates - the frame the null's orientations are drawn in. +gemmi::Mat33 cartesian_rotation(const gemmi::Op &op, const gemmi::UnitCell &cell) { + gemmi::Mat33 f; + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + f.a[i][j] = static_cast(op.rot[i][j]) / gemmi::Op::DEN; + return cell.orth.mat.multiply(f).multiply(cell.frac.mat); +} + +// The angle, in degrees, of the rotation from `rot` to the nearest of `orientations`. +double angle_to_nearest_deg(const gemmi::Mat33 &rot, const std::vector &orientations) { + double best = 180.0; + for (const gemmi::Mat33 &o : orientations) { + const gemmi::Mat33 d = rot.multiply(o.transpose()); + const double c = std::clamp((d.a[0][0] + d.a[1][1] + d.a[2][2] - 1.0) / 2.0, -1.0, 1.0); + best = std::min(best, std::acos(c) * 180.0 / PI); + } + return best; +} + // Everything one fit moves: the model itself, and the structure factors that follow it. The real // model has one of these and every null replicate gets a copy of its own, which is what lets the // replicates run at the same time without stepping on each other. @@ -740,11 +766,41 @@ ModelValidationResult ValidateAgainstModel(const std::vector & // replicates run at the same time below, and a draw taken on whichever thread reached the // generator first would make the answer depend on the scheduling. Replicate i gets rotation i // on every run, so the sigma reported is a property of the data and not of the machine. + // + // A replicate has to start from an orientation the model does NOT have, and there is more than + // one orientation to keep away from. The crystal looks the same from every orientation its + // lattice's rotations take the model to: the space group's own, and those composed with a twin + // law, which is where the model sits against data merged in the other indexing - and the + // replicates are fitted to the data as merged. A draw within the rigid body's reach of any of + // them is walked onto it and scores like the real model: measured on a cubic crystal with a + // twin law, one replicate of nine drawn 7.9 deg from the twin-related orientation was placed + // to R-work 0.20 against 0.57 for the other eight, and the spread that one gave the null was + // enough for the crystal's own model to read as not fitting. Such a draw is not a sample of + // the null, so it is drawn again. Everywhere else the draws are exactly the ones taken before. + std::vector lattice_ops{gemmi::Op::identity()}; + for (const gemmi::Op &law : ReindexAmbiguityOperators(cell, ambiguity_sg)) + lattice_ops.push_back(law); + std::vector equivalent; + for (const gemmi::Op &law : lattice_ops) + for (const gemmi::Op &s : gops.sym_ops) + equivalent.push_back(cartesian_rotation(s.combine(law), ucell)); + const double reach_deg = RigidBodyReachDeg(st.models[0], d_min); std::vector null_rotation; { std::mt19937 rng(NULL_SEED); - for (int i = 0; i < NULL_REPLICATES; i++) - null_rotation.push_back(random_rotation(rng)); + int redraws = 0; + while (static_cast(null_rotation.size()) < NULL_REPLICATES) { + const gemmi::Mat33 rot = random_rotation(rng); + if (angle_to_nearest_deg(rot, equivalent) < reach_deg && redraws < NULL_MAX_REDRAWS) { + redraws++; + continue; + } + null_rotation.push_back(rot); + } + if (redraws > 0) + logger.Info("Model validation: {} random placement(s) fell within the rigid body's reach " + "({:.1f} deg) of an orientation equivalent to the model's, and were drawn again", + redraws, reach_deg); } // Each replicate works on its own copy of the model and of its structure factors, and the // replicates share nothing else, so they run concurrently and the real model - which the diff --git a/rugnux/RigidBodyRefine.cpp b/rugnux/RigidBodyRefine.cpp index e60339c57..ccf9c8137 100644 --- a/rugnux/RigidBodyRefine.cpp +++ b/rugnux/RigidBodyRefine.cpp @@ -302,6 +302,24 @@ void SetModelPositions(gemmi::Model &model, const std::vector & // the movement to recover is the crystal's, not the molecule's. Splitting it into domains or giving a // bound ligand its own six parameters would refine against evidence this data does not separately // carry, and the ligand is what the difference map is meant to show rather than model away. +double RigidBodyReachDeg(const gemmi::Model &model, double d_min) { + const std::vector pos = ModelPositions(model); + if (pos.empty()) + return 0.0; + gemmi::Position centre; + for (const gemmi::Position &p : pos) + centre += p; + centre *= 1.0 / static_cast(pos.size()); + double r2 = 0; + for (const gemmi::Position &p : pos) + r2 += centre.dist_sq(p); + const double rms_radius = std::sqrt(r2 / static_cast(pos.size())); + // The first zone RefineRigidBody walks, which is d_min itself where d_min is coarser than all + // of the ladder. + const double first_zone = std::max(LADDER[0], d_min); + return first_zone / rms_radius * 180.0 / PI; +} + // // Some of the translation would be a gauge rather than a quantity - the origin is free in all three // directions in P1 and along the unique axis in a polar group, and |F| does not change when the whole diff --git a/rugnux/RigidBodyRefine.h b/rugnux/RigidBodyRefine.h index a17f3489b..6c15a3b3d 100644 --- a/rugnux/RigidBodyRefine.h +++ b/rugnux/RigidBodyRefine.h @@ -27,6 +27,12 @@ struct RigidBodyRefineResult { std::vector ModelPositions(const gemmi::Model &model); void SetModelPositions(gemmi::Model &model, const std::vector &pos); +// How far, in degrees, a rotation about the centroid can be from the answer and still be walked onto +// it by RefineRigidBody: the angle that moves the model's rms-radius atom by the resolution of the +// ladder's first zone. Inside it the first zone sees the rotated model overlap the density it belongs +// to, and its single broad minimum is that placement. +double RigidBodyReachDeg(const gemmi::Model &model, double d_min); + // Refine the placement of `model` as one rigid body against the observed amplitudes: an angle-axis // rotation about the model's own centroid followed by a translation, six parameters, over a // coarse-to-fine resolution ladder. Each evaluation recomputes Fcalc and the bulk-solvent mask for From 8e6ecedd5c5762ce7cd6f30767555d65810bd629 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 03:57:10 +0200 Subject: [PATCH 12/12] CPU pixel loops: keep the ring sums and the vertical slide out of memory for real Two fixes found in the post-merge profile, where the previous commit's loops did not do what they were written to do: - AccumulateRingsBlock: the ring_overflow.emplace_back call inside the loop left no callee-saved registers for the locals, so GCC kept sum/sum2/count on the stack and the store-reload chain was still there (29% of the function's samples on one stack add). The azint sums get a loop of their own over the block, and the out-of-histogram values are listed by a second loop run only when the block has any, in the same pixel order; now the sums live in registers. Same additions in the same order. - DetectPass: inlined into DetectPass's main loop the vertical slide was not vectorised (the vectorised copies GCC made were for the other call sites). It is now a free function, SlideVertical, which GCC vectorises on its own. Measured (perf, 75 s of myob, CPU-only, relative to the unchanged FlagRow/AnalyzeBlock): AccumulateRingsBlock -4..-11%, DetectPass + SlideVertical -7..-14%. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob (GPU); ImageSpotFinderCPU*, AdaptiveSpotFinder, SpotFinding, AzimuthalIntegration and portable tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../spot_finding/AdaptiveSpotFinderCPU.cpp | 89 +++++++++++++------ .../spot_finding/ImageSpotFinderCPU.cpp | 38 ++++---- 2 files changed, 83 insertions(+), 44 deletions(-) diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 4a8107b22..62474e7e3 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -60,48 +60,65 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer & const size_t nbins = ring_sum.size(); const float *corrections = mapping.Corrections().data(); - // Consecutive pixels mostly share a ring, so the ring's sums are held in locals while they do and + // Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and // written back when the ring changes: the same additions in the same order, without a store and a - // reload of the same address on every pixel. - uint32_t *hist = ring_hist.data(); - size_t cur = nbins; // the ring held in the locals below; nbins = none - int64_t sum = 0, cnt = 0; - uint64_t sum2 = 0; - float az_sum = 0.0f, az_sum2 = 0.0f; - uint32_t az_cnt = 0; - const auto store = [&] { - if (cur == nbins) - return; - ring_sum[cur] = sum; - ring_sum2[cur] = sum2; - ring_cnt[cur] = cnt; - if (fuse_azint) { + // reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own + // over the block, so that neither loop runs out of registers for its sums. + if (fuse_azint) { + size_t cur = nbins; // the ring held in the locals below; nbins = none + float az_sum = 0.0f, az_sum2 = 0.0f; + uint32_t az_cnt = 0; + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (b != cur) { + if (cur != nbins) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + cur = b; + az_sum = azint_sum[b]; + az_sum2 = azint_sum2[b]; + az_cnt = azint_count[b]; + } + const float val = static_cast(v) * corrections[pxl]; + const float val_sq = val * val; + az_sum += val; + az_sum2 += val_sq; + ++az_cnt; + } + if (cur != nbins) { azint_sum[cur] = az_sum; azint_sum2[cur] = az_sum2; azint_count[cur] = az_cnt; } - }; + } + + // Values outside the histogram are listed by a second loop, run only when the block has any: a + // call in this loop would leave the sums in memory again. + uint32_t *hist = ring_hist.data(); + bool overflow = false; + size_t cur = nbins; + int64_t sum = 0, cnt = 0; + uint64_t sum2 = 0; for (size_t pxl = first; pxl < first + n; ++pxl) { const int32_t v = image[pxl]; if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated const uint16_t b = pixel_to_bin[pxl]; if (b >= nbins) continue; // masked / out of range (UINT16_MAX) if (b != cur) { - store(); + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } cur = b; sum = ring_sum[b]; sum2 = ring_sum2[b]; cnt = ring_cnt[b]; - az_sum = azint_sum[b]; - az_sum2 = azint_sum2[b]; - az_cnt = azint_count[b]; - } - if (fuse_azint) { - const float val = static_cast(v) * corrections[pxl]; - const float val_sq = val * val; - az_sum += val; - az_sum2 += val_sq; - ++az_cnt; } sum += v; sum2 += static_cast(static_cast(v) * v); @@ -109,9 +126,23 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer & if (v >= 0 && v < HIST_VALUES) hist[b * HIST_VALUES + v] += 1; else - ring_overflow.emplace_back(b, v); + overflow = true; } - store(); + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } + + if (overflow) + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; + if (v < 0 || v >= HIST_VALUES) + ring_overflow.emplace_back(b, v); + } } // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp index ea7549f2f..1cb6773f9 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp @@ -45,6 +45,24 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid, // pixel is above SNR threshold } +// Adds row `in` to the vertical sums and takes row `out` out of them; nullptr is no row. A bad or +// saturated pixel adds 0. Written without branches, and as a function of its own so that the loop +// vectorises whatever it is inlined into; integer sums, so the same totals as pixel by pixel. +void SlideVertical(const int32_t *in, const int32_t *out, int32_t width, + int64_t *sum, int64_t *sum2, uint16_t *valid) { + for (int32_t col = 0; col < width; col++) { + const int64_t a = in ? in[col] : INT32_MIN; + const int64_t r = out ? out[col] : INT32_MIN; + const bool a_ok = a != INT32_MAX && a != INT32_MIN; + const bool r_ok = r != INT32_MAX && r != INT32_MIN; + const int64_t a_v = a_ok ? a : 0; + const int64_t r_v = r_ok ? r : 0; + sum[col] += a_v - r_v; + sum2[col] += a_v * a_v - r_v * r_v; + valid[col] += static_cast(a_ok - r_ok); + } +} + } // namespace void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, @@ -164,27 +182,17 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, std::vector valid_vert(width, 0); // Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is - // no row. A bad or saturated pixel adds 0. Written without branches, so the loop vectorises. - // A pixel strong in the previous pass is not counted either; it is added here with the rest - // of its row and taken out again below - there are few, and the sums are integers, so the - // totals are the same as never adding it. + // no row. A pixel strong in the previous pass is not counted either; SlideVertical adds it with + // the rest of its row and it is taken out again below - there are few, and the sums are + // integers, so the totals are the same as never adding it. const int32_t *img = image.data(); const int32_t w = width; int64_t *sv = sum_vert.data(); int64_t *sv2 = sum2_vert.data(); uint16_t *vv = valid_vert.data(); auto slide_vert = [&](int line_in, int line_out) { - for (int32_t col = 0; col < w; col++) { - const int64_t in = line_in >= 0 ? img[line_in * w + col] : INT32_MIN; - const int64_t out = line_out >= 0 ? img[line_out * w + col] : INT32_MIN; - const bool in_ok = in != INT32_MAX && in != INT32_MIN; - const bool out_ok = out != INT32_MAX && out != INT32_MIN; - const int64_t in_v = in_ok ? in : 0; - const int64_t out_v = out_ok ? out : 0; - sv[col] += in_v - out_v; - sv2[col] += in_v * in_v - out_v * out_v; - vv[col] += static_cast(in_ok - out_ok); - } + SlideVertical(line_in >= 0 ? img + static_cast(line_in) * w : nullptr, + line_out >= 0 ? img + static_cast(line_out) * w : nullptr, w, sv, sv2, vv); if (!prev_strong) return; // sign = +1 takes out what line_in added, -1 puts back what line_out took out.