diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index ee6cd62e2..58893e991 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -126,6 +126,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build\tests + jfjoch_portable_test.exe "[portable]" - name: Build installer (NSIS) shell: cmd run: | @@ -168,6 +181,13 @@ jobs: # JFJOCH_VIEWER_ONLY is forced on macOS. No -mcpu: the arm64 baseline is what every Apple # Silicon Mac runs, and NEON is part of it. run: cmake -S . -B build-macos -DCMAKE_BUILD_TYPE=Release -DJFJOCH_USE_CUDA=OFF -DCMAKE_PREFIX_PATH="$QT_PREFIX" + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build viewer and DMG shell: bash run: | @@ -380,6 +400,19 @@ jobs: for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" cmake --build build-rugnux + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. CPU-only: the runner has no GPU. + - name: Build portable tests + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath`) do set "VSPATH=%%i" + call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" + cmake --build build-rugnux --target jfjoch_portable_test + - name: Run portable tests + shell: cmd + run: | + cd build-rugnux\tests + jfjoch_portable_test.exe "[portable]" - name: Build rugnux zip shell: cmd run: | @@ -415,6 +448,13 @@ jobs: - name: Configure rugnux build shell: bash run: cmake -S . -B build-rugnux-macos -DJFJOCH_RUGNUX_ONLY=ON -DJFJOCH_USE_CUDA=OFF -DCMAKE_BUILD_TYPE=Release + # The quick cross-platform checks (tests/PortableTest.cpp and the [portable] cases), before + # anything is packaged. + - name: Portable tests + shell: bash + run: | + cmake --build build-rugnux-macos --target jfjoch_portable_test -j "$(sysctl -n hw.ncpu)" + cd build-rugnux-macos/tests && ./jfjoch_portable_test "[portable]" - name: Build rugnux tgz shell: bash run: cmake --build build-rugnux-macos --target package -j "$(sysctl -n hw.ncpu)" diff --git a/CMakeLists.txt b/CMakeLists.txt index 33fd421d5..27c88cba7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -365,10 +365,10 @@ SET(CMAKE_POLICY_VERSION_MINIMUM 3.5) # libzmq must be made available BEFORE sls_detector_package for the override above to take effect. FetchContent_MakeAvailable(libzmq) IF (JFJOCH_PORTABLE_ONLY) - # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib). - # Only sls_detector_package (detector) and catch2 (tests) are not built here -- and sls - # in particular does not configure under MSVC -- so skip just those two. - FetchContent_MakeAvailable(zstd hdf5 spdlog httplib) + # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2 + # for jfjoch_portable_test. Only sls_detector_package (detector) is not built here -- it does + # not configure under MSVC. + FetchContent_MakeAvailable(zstd catch2 hdf5 spdlog httplib) ELSE() FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib) ENDIF() @@ -555,6 +555,7 @@ IF (JFJOCH_RUGNUX_ONLY) ADD_SUBDIRECTORY(broker) ADD_SUBDIRECTORY(reader) ADD_SUBDIRECTORY(rugnux) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSEIF (JFJOCH_VIEWER_ONLY) # Minimal subtree: jfjoch_viewer and only the libraries it transitively links. # (broker here provides JFJochAPI only; its service targets are gated out.) @@ -571,6 +572,7 @@ ELSEIF (JFJOCH_VIEWER_ONLY) ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(viewer) ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/extract_hkl) + ADD_SUBDIRECTORY(tests) # jfjoch_portable_test only ELSE() ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) diff --git a/common/PixelMask.cpp b/common/PixelMask.cpp index a5338f4ee..8e455195e 100644 --- a/common/PixelMask.cpp +++ b/common/PixelMask.cpp @@ -59,6 +59,9 @@ void PixelMask::UpdateBinaryMask() { for (size_t i = 0; i < mask.size(); i++) binary_mask[i] = (mask[i] != 0); binary_mask_checksum = TableChecksum(binary_mask.data(), binary_mask.size()); + packed_mask.assign((mask.size() + 31) / 32, 0); + for (size_t i = 0; i < mask.size(); i++) + packed_mask[i / 32] |= static_cast(binary_mask[i]) << (i % 32); } void PixelMask::CalcEdgePixels_i(const DiffractionExperiment &experiment) { @@ -127,6 +130,10 @@ const std::vector &PixelMask::GetBinaryMask() const { return binary_mask; } +const std::vector &PixelMask::GetPackedMask() const { + return packed_mask; +} + uint64_t PixelMask::GetBinaryMaskChecksum() const { return binary_mask_checksum; } diff --git a/common/PixelMask.h b/common/PixelMask.h index 5408b0789..ddcf50be2 100644 --- a/common/PixelMask.h +++ b/common/PixelMask.h @@ -26,6 +26,9 @@ class PixelMask { // derived here once instead of in every one of those engines. std::vector binary_mask; uint64_t binary_mask_checksum = 0; + // The same flag packed 32 pixels to a word (pixel i is bit i % 32 of word i / 32) - the form the CPU + // image preprocessor reads, derived here for the same reason. + std::vector packed_mask; uint32_t LoadMask(const std::vector& mask, uint8_t bit); // Everything that follows from the mask, recomputed wherever the mask changes. @@ -68,6 +71,7 @@ public: [[nodiscard]] const std::vector &GetMask(const DiffractionExperiment& experiment) const; [[nodiscard]] const std::vector &GetMask() const; [[nodiscard]] const std::vector &GetBinaryMask() const; + [[nodiscard]] const std::vector &GetPackedMask() const; [[nodiscard]] uint64_t GetBinaryMaskChecksum() const; [[nodiscard]] std::vector GetUserMask(const DiffractionExperiment& experiment) const; diff --git a/compression/JFJochDecompress.h b/compression/JFJochDecompress.h index 73c0518f4..c5a90df63 100644 --- a/compression/JFJochDecompress.h +++ b/compression/JFJochDecompress.h @@ -21,13 +21,19 @@ extern "C" { uint64_t bshuf_read_uint64_BE(const void* buf); }; -inline size_t JFJochDecompressHperfPtr(uint8_t *output, - CompressionAlgorithm algorithm, - const uint8_t *source, - size_t source_size, - size_t nelements, - size_t elem_size, - size_t block_size) { +// Decode the bitshuffle blocks one after another and hand each to block_done(first_element, data, +// nelements), in element order. With output set, each block is decoded in place into output; with +// output = nullptr, into a block-sized buffer that is reused for the next block, so a caller that +// consumes the block straight away reads it from cache and the whole image is never written out. +template +size_t JFJochDecompressHperfBlocks(uint8_t *output, + CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + size_t block_size, + F &&block_done) { if ((algorithm != CompressionAlgorithm::BSHUF_LZ4) && (algorithm != CompressionAlgorithm::BSHUF_ZSTD) && (algorithm != CompressionAlgorithm::BSHUF_ZSTD_RLE) && @@ -39,10 +45,11 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, std::vector decompressed_block(block_size * elem_size); std::vector scratch(block_size * elem_size); + std::vector unshuffled(output ? 0 : block_size * elem_size); const uint8_t *src_ptr = source; const uint8_t *const source_end = source + source_size; - uint8_t *dst_ptr = output; + size_t first_element = 0; const size_t num_full_blocks = nelements / block_size; const size_t reminder_size = nelements - num_full_blocks * block_size; @@ -90,15 +97,17 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, throw JFJochException(JFJochExceptionCategory::Compression, "Algorithm not supported"); } + uint8_t *dst_ptr = output ? output + first_element * elem_size : unshuffled.data(); if (JFJochBitUnshuffleBlock(reinterpret_cast(dst_ptr), decompressed_block.data(), scratch.data(), current_nelements, elem_size) < 0) throw JFJochException(JFJochExceptionCategory::Compression, "bitshuffle block decode error"); + block_done(first_element, dst_ptr, current_nelements); src_ptr += compressed_size; - dst_ptr += decompressed_size; + first_element += decompressed_size / elem_size; }; for (size_t i = 0; i < num_full_blocks; ++i) @@ -111,13 +120,26 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, if (leftover_bytes > 0) { if (static_cast(source_end - src_ptr) < leftover_bytes) throw JFJochException(JFJochExceptionCategory::Compression, "Truncated trailing bytes"); + uint8_t *dst_ptr = output ? output + first_element * elem_size : unshuffled.data(); memcpy(dst_ptr, src_ptr, leftover_bytes); + block_done(first_element, dst_ptr, leftover_bytes / elem_size); src_ptr += leftover_bytes; } return static_cast(src_ptr - source); } +inline size_t JFJochDecompressHperfPtr(uint8_t *output, + CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + size_t block_size) { + return JFJochDecompressHperfBlocks(output, algorithm, source, source_size, nelements, elem_size, block_size, + [](size_t, const uint8_t *, size_t) {}); +} + // Plain LZ4, HDF5 filter 32004, as DECTRIS Eiger firmware 1.x wrote it. The framing is the same as // bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size IN BYTES, then each // block prefixed by its 32-bit big-endian compressed size - and the only difference is that no bit @@ -166,6 +188,40 @@ inline void JFJochDecompressLZ4Ptr(uint8_t *output, } } +// Check the 12-byte bitshuffle header and return the block size (in elements) it gives. +inline size_t JFJochBitshuffleHeaderBlockSize(const uint8_t *source, size_t source_size, + size_t nelements, size_t elem_size) { + // The header must be there before it can be read, and before source_size - 12 is handed to the + // decompressors. + if (source_size < 12) + throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the bitshuffle header"); + if (bshuf_read_uint64_BE(const_cast(source)) != nelements * elem_size) + throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size"); + return bshuf_read_uint32_BE(source + 8) / elem_size; +} + +inline bool JFJochIsBitshuffle(CompressionAlgorithm algorithm) { + return (algorithm == CompressionAlgorithm::BSHUF_LZ4) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD_RLE) + || (algorithm == CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF); +} + +// A bitshuffle-compressed image, handed block by block to block_done(first_element, data, nelements) +// in element order (see JFJochDecompressHperfBlocks), without the image ever being written out whole. +template +void JFJochDecompressBlocks(CompressionAlgorithm algorithm, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size, + F &&block_done) { + const size_t block_size = JFJochBitshuffleHeaderBlockSize(source, source_size, nelements, elem_size); + if (JFJochDecompressHperfBlocks(nullptr, algorithm, source + 12, source_size - 12, + nelements, elem_size, block_size, block_done) != source_size - 12) + throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error"); +} + inline void JFJochDecompressPtr(uint8_t *output, CompressionAlgorithm algorithm, const uint8_t *source, @@ -179,16 +235,8 @@ inline void JFJochDecompressPtr(uint8_t *output, } size_t block_size = 0; - if (algorithm != CompressionAlgorithm::NO_COMPRESSION) { - // The 12-byte bitshuffle header must be there before it can be read, and before - // source_size - 12 is handed to the decompressors below. - if (source_size < 12) - throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the bitshuffle header"); - if (bshuf_read_uint64_BE(const_cast(source)) != nelements * elem_size) - throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size"); - auto tmp = bshuf_read_uint32_BE(source + 8); - block_size = tmp / elem_size; - } + if (algorithm != CompressionAlgorithm::NO_COMPRESSION) + block_size = JFJochBitshuffleHeaderBlockSize(source, source_size, nelements, elem_size); switch (algorithm) { case CompressionAlgorithm::NO_COMPRESSION: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b9ab621ee..ed23eb167 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ * jfjoch_viewer's reference dataset accepts a structure-factor mmCIF as well as an MTZ, and a processing job can name an atomic model to validate the merged data against, as `rugnux --model` does. * jfjoch_viewer keeps a separate preferred dataset-info plot for grid scans, where "Spots + background" means the spot count. * jfjoch_viewer's dark theme no longer leaves navy buttons, red warnings and chart guide lines at their light-theme colours. +* `rugnux --model` draws its null's random placements away from every orientation equivalent to the model's under the space group or a twin law, so a random placement can no longer be refined onto the model's own solution and make the crystal's own model read as not fitting. * Rugnux adds beam-stop holder arms that let part of the beam through to the beam-stop mask, without changing the mask of a sweep that has none. ### 1.0.0-rc.173 diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index a861f50bf..ce3d2b173 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -50,7 +50,9 @@ MXAnalysisWithoutFPGA::MXAnalysisWithoutFPGA(const DiffractionExperiment &in_exp if (get_gpu_count() == 0) { #endif preprocessor_buffer = std::make_unique(experiment.GetPixelsNum()); - preprocessor = std::make_unique(in_experiment, in_mask); + auto cpu_preprocessor = std::make_unique(in_experiment, in_mask); + preprocessor_cpu = cpu_preprocessor.get(); + preprocessor = std::move(cpu_preprocessor); bragg_engine = std::make_unique(in_experiment); if (experiment.ROI().size() >= 1) roi = std::make_unique(experiment); @@ -128,7 +130,23 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, } const auto compression_end_time = std::chrono::steady_clock::now(); - if (!decoded_on_device) { + // The fused GPU engine (rugnux offline, GPU, adaptive detection) produces the azimuthal profile as + // a byproduct of spot finding, so the separate azint pass is skipped in that case and the profile is + // lifted out of the finder below. + const bool fused = enable_fused_adaptive_gpu && spot_finding_settings.enable + && spot_finding_settings.adaptive_threshold && fused_adaptive != nullptr; + const bool fused_cpu = spot_finding_settings.enable && spot_finding_settings.adaptive_threshold + && fused_adaptive_cpu != nullptr; + + if (preprocessor_cpu) { + const auto preprocessing_start_time = std::chrono::steady_clock::now(); + ret = PreprocessCPU(output.image, fused_cpu); + const auto preprocessing_end_time = std::chrono::steady_clock::now(); + output.preprocessing_time_s = std::chrono::duration(preprocessing_end_time - preprocessing_start_time).count(); + // Decoded inside the preprocessing pass, whose time includes it. + if (output.image.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION) + output.compression_time_s = 0.0f; + } else if (!decoded_on_device) { const uint8_t *image_ptr = Decompress(output.image); const auto decompressed_time = std::chrono::steady_clock::now(); if (output.image.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION) @@ -149,14 +167,6 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, output.preprocessing_time_s = total_s - decompress_s; } - // The fused GPU engine (rugnux offline, GPU, adaptive detection) produces the azimuthal profile as - // a byproduct of spot finding, so the separate azint pass is skipped in that case and the profile is - // lifted out of the finder below. - const bool fused = enable_fused_adaptive_gpu && spot_finding_settings.enable - && spot_finding_settings.adaptive_threshold && fused_adaptive != nullptr; - const bool fused_cpu = spot_finding_settings.enable && spot_finding_settings.adaptive_threshold - && fused_adaptive_cpu != nullptr; - if (!fused && !fused_cpu) { const auto azint_start_time = std::chrono::steady_clock::now(); AzInt().Run(*preprocessor_buffer, profile); @@ -397,6 +407,24 @@ const uint8_t *MXAnalysisWithoutFPGA::Decompress(const CompressedImage &image) { return image.GetUncompressedPtr(decompression_buffer); } +ImageStatistics MXAnalysisWithoutFPGA::PreprocessCPU(const CompressedImage &image, bool ring_pass) { + ImageStatistics ret{}; + if (ring_pass) + fused_adaptive_cpu->BeginRings(); + const auto process_block = [&](size_t first, const uint8_t *data, size_t n) { + preprocessor_cpu->AnalyzeBlock(*preprocessor_buffer, first, data, n, image.GetMode(), ret); + if (ring_pass) + fused_adaptive_cpu->AccumulateRingsBlock(*preprocessor_buffer, first, n); + }; + if (JFJochIsBitshuffle(image.GetCompressionAlgorithm())) + JFJochDecompressBlocks(image.GetCompressionAlgorithm(), image.GetCompressed(), image.GetCompressedSize(), + image.GetWidth() * image.GetHeight() * image.GetNumChannels(), image.GetByteDepth(), + process_block); + else + process_block(0, Decompress(image), npixels); + return ret; +} + void MXAnalysisWithoutFPGA::RunROIOnly(DataMessage &output) { output.roi.clear(); if (roi) diff --git a/image_analysis/MXAnalysisWithoutFPGA.h b/image_analysis/MXAnalysisWithoutFPGA.h index 20d4d5cb8..670a8b1e5 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.h +++ b/image_analysis/MXAnalysisWithoutFPGA.h @@ -20,6 +20,7 @@ #include "IndexAndRefine.h" #include "image_preprocessing/ImagePreprocessor.h" #include "image_preprocessing/ImagePreprocessorBuffer.h" +#include "image_preprocessing/ImagePreprocessorCPU.h" class CudaStream; class AdaptiveSpotFinderGPU; @@ -32,6 +33,8 @@ class MXAnalysisWithoutFPGA { std::vector decompression_buffer; std::unique_ptr preprocessor; + // The preprocessor, where it is the CPU one. + ImagePreprocessorCPU *preprocessor_cpu = nullptr; size_t npixels; size_t xpixels; @@ -68,6 +71,13 @@ class MXAnalysisWithoutFPGA { // not compressed) and return where it landed. const uint8_t *Decompress(const CompressedImage &image); + // The CPU preprocessing. A bitshuffled image is decoded a block (~32 KB) at a time and each block + // is preprocessed while it is in cache, then - with ring_pass - put through the adaptive finder's + // plain ring pass as well, so the image is never written out whole before it is preprocessed and + // the preprocessed pixels are read back from cache, not from memory. The blocks come in pixel + // order, so every sum is taken in the same order as by separate passes. + ImageStatistics PreprocessCPU(const CompressedImage &image, bool ring_pass); + // Pixels outside the resolution limits, bit-packed. Built by the integration mapping, which is // shared by every worker's engine and hands out the same mask to all of them. std::shared_ptr> mask_resolution; diff --git a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp index 52dec9c78..1adc41900 100644 --- a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp +++ b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp @@ -63,7 +63,9 @@ struct ImageSampler { } // namespace BraggIntegrationEngineCPU::BraggIntegrationEngineCPU(const DiffractionExperiment &experiment) - : BraggIntegrationEngine(experiment) {} + : BraggIntegrationEngine(experiment), + refl_mask(static_cast(xpixel), static_cast(ypixel), 0), + owner(static_cast(xpixel), static_cast(ypixel), BRAGG_OWNER_NONE) {} template std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, @@ -90,17 +92,8 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, // signal is excluded from this reflection's r2..r3 background ring. The region is the INNER // stencil ellipse, taken in each neighbour's OWN frame - an elongated neighbour whose streak // is still marked as a disk would leak its tails into this reflection's ring. --- - // Back to empty what the previous call wrote (see the header). - for (const Rect &b : refl_mask_dirty) - for (int y = b.y0; y <= b.y1; ++y) - std::fill(refl_mask.begin() + y * W + b.x0, refl_mask.begin() + y * W + b.x1 + 1, 0); - refl_mask_dirty.clear(); - for (const Rect &b : owner_dirty) - for (int y = b.y0; y <= b.y1; ++y) - std::fill(owner.begin() + y * W + b.x0, owner.begin() + y * W + b.x1 + 1, BRAGG_OWNER_NONE); - owner_dirty.clear(); - - refl_mask.resize(npixel, 0); + refl_mask.Clear(); + owner.Clear(); for (size_t i = 0; i < npredicted; ++i) { const auto &r = predicted[i]; const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil); @@ -108,12 +101,10 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const int x1 = std::min(W - 1, static_cast(std::ceil(r.predicted_x + st.ex_in + 1.0f))); const int y0 = std::max(0, static_cast(std::floor(r.predicted_y - st.ey_in - 1.0f))); const int y1 = std::min(H - 1, static_cast(std::ceil(r.predicted_y + st.ey_in + 1.0f))); - if (x0 <= x1 && y0 <= y1) - refl_mask_dirty.push_back({x0, x1, y0, y1}); for (int y = y0; y <= y1; ++y) for (int x = x0; x <= x1; ++x) { const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y); - if (d.inner < r2_sq) refl_mask[y * W + x] = 1; + if (d.inner < r2_sq) refl_mask.At(x, y) = 1; } } @@ -125,7 +116,6 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const bool exclude = overlap == OverlapMode::Exclude && mode != IntegratorMode::BoxSum; const bool use_owner = overlap != OverlapMode::Off; if (use_owner) { - owner.resize(npixel, BRAGG_OWNER_NONE); const float claim_sq = claim * claim; for (size_t i = 0; i < npredicted; ++i) { const auto &r = predicted[i]; @@ -133,21 +123,19 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, const int x1 = std::min(W - 1, static_cast(std::ceil(r.predicted_x + claim))); const int y0 = std::max(0, static_cast(std::floor(r.predicted_y - claim))); const int y1 = std::min(H - 1, static_cast(std::ceil(r.predicted_y + claim))); - if (x0 <= x1 && y0 <= y1) - owner_dirty.push_back({x0, x1, y0, y1}); for (int y = y0; y <= y1; ++y) for (int x = x0; x <= x1; ++x) { const float dx = x - r.predicted_x, dy = y - r.predicted_y; const float d2 = dx * dx + dy * dy; if (d2 >= claim_sq) continue; - uint32_t &o = owner[y * W + x]; + uint32_t &o = owner.At(x, y); o = std::min(o, BraggOwnerKey(std::sqrt(d2), inv_claim, static_cast(i))); } } } auto clean = [&](int x, int y, size_t i) { return !use_owner - || BraggOwnedBy(owner[static_cast(y) * W + x], static_cast(i)); + || BraggOwnedBy(owner.Get(x, y), static_cast(i)); }; // --- Pass A: box-sum every reflection (rough I, background, centroid, strong flag). --- @@ -182,35 +170,31 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, // The box pass A reads: the OUTER ellipse, tightly - taking the largest semi-axis in both // directions instead would read up to 60% more pixels for nothing. + struct Box { int x0, x1, y0, y1; }; auto box_of = [&](const Reflection &r, const BraggStencil &st) { - return Rect{std::max(0, static_cast(std::floor(r.predicted_x - st.ex_out - 1.0f))), + return Box{std::max(0, static_cast(std::floor(r.predicted_x - st.ex_out - 1.0f))), std::min(W - 1, static_cast(std::ceil(r.predicted_x + st.ex_out + 1.0f))), std::max(0, static_cast(std::floor(r.predicted_y - st.ey_out - 1.0f))), std::min(H - 1, static_cast(std::ceil(r.predicted_y + st.ey_out + 1.0f)))}; }; for (size_t i = 0; i < npredicted; ++i) { - // The next reflection's rows - image and reflection mask over its box, owner map over its - // claim - asked for while this one is worked on: the reflections are spread over frames far - // larger than the caches, so each box would otherwise wait on memory row by row. (Written - // out here: GCC drops a call to a function that does nothing but prefetch.) + // The next reflection's image rows, asked for while this one is worked on: the reflections + // are spread over a frame far larger than the caches, so each box would otherwise wait on + // memory row by row. (Written out here: GCC drops a call to a function that does nothing but + // prefetch.) if (i + 1 < npredicted) { const Reflection &n = predicted[i + 1]; - const Rect b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil)); - const int ox = static_cast(std::lround(n.predicted_x)), oy = static_cast(std::lround(n.predicted_y)); - for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y) { - const size_t row = static_cast(y) * W; - prefetch_row(img.address(row + b.x0), img.address(row + b.x1)); - prefetch_row(&refl_mask[row + b.x0], &refl_mask[row + b.x1]); - if (use_owner && std::abs(y - oy) <= R) - prefetch_row(&owner[row + std::max(b.x0, ox - R)], &owner[row + std::min(b.x1, ox + R)]); - } + const Box b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil)); + for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y) + prefetch_row(img.address(static_cast(y) * W + b.x0), + img.address(static_cast(y) * W + b.x1)); } const auto &r = predicted[i]; Rough out; // This reflection's stencil: the r1 signal disk, and the r2..r3 ring elongated radially by // the analytic smear. const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil); - const Rect box = box_of(r, st); + const Box box = box_of(r, st); const int x0 = box.x0, x1 = box.x1, y0 = box.y0, y1 = box.y1; // Both from the stencil's own radius, so nothing downstream is derived from a second one. @@ -253,7 +237,7 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, y_sum += y; ++n_inner_valid; } else if (d.inner >= r2_sq && d.outer < r3_sq) { - if (refl_mask[y * W + x]) { ++n_bkg_neighbour; continue; } + if (refl_mask.Get(x, y)) { ++n_bkg_neighbour; continue; } if (!valid(px)) continue; bkg_sum += static_cast(px); if (bkg_trim_frac > 0.0) bkg_vals.push_back(px); @@ -307,7 +291,7 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, for (int x = x0; x <= x1; ++x) { const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y); if (!(d.inner >= r2_sq && d.outer < r3_sq)) continue; - if (refl_mask[y * W + x]) continue; + if (refl_mask.Get(x, y)) continue; const int32_t px = img[y * W + x]; if (!valid(px)) continue; if (static_cast(px) <= thr) { @@ -549,14 +533,12 @@ std::vector BraggIntegrationEngineCPU::RunImpl(const Sampler &img, // needed) radially elongated, on a grid grown to hold the streak. const double fx = predicted[i].predicted_x - rh.cx, fy = predicted[i].predicted_y - rh.cy; Rf = elong ? std::min(3 * R, static_cast(std::ceil(r2 + 2.0 * std::sqrt(s2r)))) : R; - // The grid's image and owner rows, asked for now (as in pass A): building the profile - // below takes long enough for them to arrive before the fit reads them. + // The grid's image rows, asked for now (as in pass A): building the profile below takes + // long enough for them to arrive before the fit reads them. const int gx0 = std::max(0, rh.cx - Rf), gx1 = std::min(W - 1, rh.cx + Rf); - for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y) { - const size_t row = static_cast(y) * W; - prefetch_row(img.address(row + gx0), img.address(row + gx1)); - if (use_owner) prefetch_row(&owner[row + gx0], &owner[row + gx1]); - } + for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y) + prefetch_row(img.address(static_cast(y) * W + gx0), + img.address(static_cast(y) * W + gx1)); const int Gf = 2 * Rf + 1; Pbuf.assign(static_cast(Gf) * Gf, 0.0); double gs = 0.0; diff --git a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h index ed1861599..f125f391b 100644 --- a/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h +++ b/image_analysis/bragg_integration/BraggIntegrationEngineCPU.h @@ -14,15 +14,44 @@ class BraggIntegrationEngineCPU : public BraggIntegrationEngine { // Core integrator, templated on a pixel sampler so it reads either the preprocessed int32 buffer // or a raw CompressedImage of any pixel type - both presented per-pixel in the INT32_MIN(masked)/ // INT32_MAX(saturated) convention - without ever materialising a second full-image copy. - // Full-frame scratch of RunImpl: the reflection mask and the signal-region owner map. Kept from - // call to call rather than allocated per image - at 18 Mpixel that was 90 MB, a fresh mmap and a - // page fault per 4 kB every call - and put back to their empty state one rectangle at a time: the - // rectangles a call writes are recorded, and the next call clears exactly those before it starts. - struct Rect { int x0, x1, y0, y1; }; - std::vector refl_mask; - std::vector owner; - std::vector refl_mask_dirty; - std::vector owner_dirty; + // Full-frame scratch of RunImpl: the reflection mask and the signal-region owner map. A call writes + // only around its reflections, so these keep only the 16x16-pixel tiles written since the last + // Clear(): a frame-sized array was mostly never read, yet over a sweep every page of it got + // touched, in every worker. Reading a tile nothing wrote gives `empty`. + template + class TiledFrame { + static constexpr int TILE = 16; + int tiles_x; + T empty; + std::vector tile_start; // per tile: where it starts in `pixels`, -1 = not written + std::vector written; // the tiles written, for Clear() + std::vector pixels; + public: + TiledFrame(int width, int height, T empty) + : tiles_x((width + TILE - 1) / TILE), empty(empty), + tile_start(static_cast(tiles_x) * ((height + TILE - 1) / TILE), -1) {} + T Get(int x, int y) const { + const int32_t start = tile_start[(y / TILE) * tiles_x + x / TILE]; + return start < 0 ? empty : pixels[start + (y % TILE) * TILE + x % TILE]; + } + T &At(int x, int y) { + const int t = (y / TILE) * tiles_x + x / TILE; + if (tile_start[t] < 0) { + tile_start[t] = static_cast(pixels.size()); + pixels.resize(pixels.size() + TILE * TILE, empty); + written.push_back(t); + } + return pixels[tile_start[t] + (y % TILE) * TILE + x % TILE]; + } + void Clear() { + for (int t : written) + tile_start[t] = -1; + written.clear(); + pixels.clear(); + } + }; + TiledFrame refl_mask; + TiledFrame owner; template std::vector RunImpl(const Sampler &img, const std::vector &predicted, diff --git a/image_analysis/bragg_prediction/BraggPredictionRot.cpp b/image_analysis/bragg_prediction/BraggPredictionRot.cpp index dbcd90fd4..4a516262f 100644 --- a/image_analysis/bragg_prediction/BraggPredictionRot.cpp +++ b/image_analysis/bragg_prediction/BraggPredictionRot.cpp @@ -77,6 +77,28 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys experiment.GetBraggIntegrationSettings().GetFlightPath(), geom.GetDetectorDistance_mm(), geom.GetWavelength_A()); + // The phi test after the atan2 below, taken before it: a solution with |phi| above phi_limit fails + // that test whatever its own bandwidth term, since mos_eff_rad is largest at the resolution limit. + // |phi| > phi_limit is cos(phi) < cos(phi_limit), which the (cos, sin) pair answers without the + // angle. phi_limit carries a 1e-3 rad margin, far above float rounding, so every solution rejected + // here is rejected by that test too. -1 = nothing is rejected here. + double cos_phi_limit = -1.0; + if (settings.min_zeta > 0.0f) { + double mos_max_rad = mos_angle_rad; + if (bandwidth_sigma > 0.0f) { + const double sin_theta = half_wavelength_A * static_cast(one_over_dmax); + const double dphi_bw = sin_theta < 1.0 + ? bandwidth_sigma * sin_theta / std::sqrt(1.0 - sin_theta * sin_theta) + : INFINITY; + mos_max_rad = std::sqrt(mos_max_rad * mos_max_rad + dphi_bw * dphi_bw); + } + const double phi_limit = half_wedge_angle_rad + + (settings.mosaicity_multiplier * mos_max_rad + 1e-5) / settings.min_zeta + + 1e-3; + if (phi_limit < PI) + cos_phi_limit = std::cos(phi_limit); + } + for (int h = -settings.max_h; h <= settings.max_h; h++) { // Precompute A* h contribution @@ -127,6 +149,9 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys const float cosphi = (p_m1 * p0_m1 + p_m3 * p0_m3) / rho_sq; const float sinphi = (p_m1 * p0_m3 - p_m3 * p0_m1) / rho_sq; + const double c = cosphi, s = sinphi; + if (c < cos_phi_limit * std::sqrt(c * c + s * s)) + continue; float phi = -1.0f * std::atan2(sinphi, cosphi); // Most solutions are far from this frame. zeta >= min_zeta, so the rocking-curve test diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp index 8803a73db..863843797 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.cpp @@ -5,63 +5,65 @@ ImagePreprocessorCPU::ImagePreprocessorCPU(const DiffractionExperiment &experiment, const PixelMask &mask) : ImagePreprocessor(experiment), - mask_1bit(npixels, false) { - for (int i = 0; i < npixels; i++) - mask_1bit[i] = (mask.GetMask().at(i) != 0); -} + mask_bits(mask.GetPackedMask()) {} ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *image_ptr, CompressedImageMode image_mode) { + if (processed_image.size() != npixels) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Processed image size mismatch"); + + ImageStatistics ret{}; + AnalyzeBlock(processed_image, 0, image_ptr, npixels, image_mode, ret); + return ret; +} + +void ImagePreprocessorCPU::AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, CompressedImageMode image_mode, ImageStatistics &stats) { switch (image_mode) { case CompressedImageMode::Int8: - return Analyze(processed_image, image_ptr, INT8_MIN, INT8_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT8_MIN, INT8_MAX, stats); case CompressedImageMode::Int16: - return Analyze(processed_image, image_ptr, INT16_MIN, INT16_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT16_MIN, INT16_MAX, stats); case CompressedImageMode::Int32: - return Analyze(processed_image, image_ptr, INT32_MIN, INT32_MAX); + return AnalyzeBlock(processed_image, first, input, n, INT32_MIN, INT32_MAX, stats); case CompressedImageMode::Uint8: - return Analyze(processed_image, image_ptr, UINT8_MAX, UINT8_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT8_MAX, UINT8_MAX, stats); case CompressedImageMode::Uint16: - return Analyze(processed_image, image_ptr, UINT16_MAX, UINT16_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT16_MAX, UINT16_MAX, stats); case CompressedImageMode::Uint32: - return Analyze(processed_image, image_ptr, UINT32_MAX, UINT32_MAX); + return AnalyzeBlock(processed_image, first, input, n, UINT32_MAX, UINT32_MAX, stats); default: throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "RGB/float mode not supported"); } } template -ImageStatistics ImagePreprocessorCPU::Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *input, T err_pixel_val, T sat_pixel_val) { - - if (processed_image.size() != npixels) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Processed image size mismatch"); - +void ImagePreprocessorCPU::AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, T err_pixel_val, T sat_pixel_val, ImageStatistics &ret) { auto image = reinterpret_cast(input); - - ImageStatistics ret{}; + int32_t *out = processed_image.data() + first; if (sat_pixel_val > saturation_limit) sat_pixel_val = static_cast(saturation_limit); - for (int i = 0; i < npixels; i++) { - if (mask_1bit[i] != 0) { - processed_image[i] = INT32_MIN; + for (size_t i = 0; i < n; i++) { + if ((mask_bits[(first + i) / 32] >> ((first + i) % 32)) & 1U) { + out[i] = INT32_MIN; ++ret.masked_pixel_count; } else if (image[i] == err_pixel_val) { // Error/invalid marker = the pixel type's extreme value (0xFFFFFFFF for EIGER uint32). // Tested before saturation, since for unsigned types the marker also exceeds sat_pixel_val // (which is clipped above to the HDF5 saturation_value). - processed_image[i] = INT32_MIN; + out[i] = INT32_MIN; ++ret.error_pixel_count; } else if (image[i] >= sat_pixel_val) { - processed_image[i] = INT32_MAX; + out[i] = INT32_MAX; ++ret.saturated_pixel_count; } else { - processed_image[i] = static_cast(image[i]); + out[i] = static_cast(image[i]); if (image[i] > ret.max_value) ret.max_value = image[i]; if (image[i] < ret.min_value) ret.min_value = image[i]; } } - return ret; } diff --git a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h index aa326d77b..c0e1b80d6 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorCPU.h +++ b/image_analysis/image_preprocessing/ImagePreprocessorCPU.h @@ -7,10 +7,17 @@ class ImagePreprocessorCPU : public ImagePreprocessor { - std::vector mask_1bit; + // The pixel mask one bit per pixel, 32 to a word - PixelMask derives it once, so each engine only + // copies it. + std::vector mask_bits; - template ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *input, T err_value, T sat_value); + template void AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, + size_t n, T err_value, T sat_value, ImageStatistics &stats); public: ImagePreprocessorCPU(const DiffractionExperiment &experiment, const PixelMask &mask); ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *decompressed_image, CompressedImageMode image_mode) override; + // Analyze for the n pixels from pixel `first` on, whose values `input` holds, adding to stats. The + // blocks of an image can come in any order and together give Analyze()'s result. + void AnalyzeBlock(ImagePreprocessorBuffer &processed_image, size_t first, const uint8_t *input, size_t n, + CompressedImageMode image_mode, ImageStatistics &stats); }; diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 10c5829d6..1178fe5eb 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -1336,12 +1336,13 @@ int RotationScaleMerge::ComputeAsuGroups(const HKLKeyGenerator &keygen) { return n_groups; } -double RotationScaleMerge::ReferenceWeight(const Obs &o, double min_partiality, float &I_corr) { - if (o.group < 0) return 0.0; - if (!(o.corr > 0.0f) || !std::isfinite(o.corr)) return 0.0; - if (o.partiality < min_partiality) return 0.0; - const float Ic = o.I * o.corr; - const float sigma_corr = o.sigma * o.corr; +double RotationScaleMerge::ReferenceWeight(int32_t group, float corr, float partiality, float I, float sigma, + double min_partiality, float &I_corr) { + if (group < 0) return 0.0; + if (!(corr > 0.0f) || !std::isfinite(corr)) return 0.0; + if (partiality < min_partiality) return 0.0; + const float Ic = I * corr; + const float sigma_corr = sigma * corr; if (!std::isfinite(Ic) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) return 0.0; I_corr = Ic; return 1.0 / (static_cast(sigma_corr) * sigma_corr); @@ -1383,7 +1384,30 @@ void RotationScaleMerge::ReducePartialGroupMeans(int n_groups, std::vector &obs, const std::vector &fstart, +void RotationScaleMerge::ReduceScalingGroupMeans(int n_groups, const std::vector &refs, + const std::vector &corr, + std::vector &out_mean) const { + out_mean.assign(n_groups, NAN); + ParallelChunks(n_groups, ThreadsForWork(corr.size(), nthreads), [&](int glo, int ghi) { + for (int g = glo; g < ghi; ++g) { + double sw = 0.0, swI = 0.0; + const int lo = partial_group_start[g], hi = lo + partial_group_count[g]; + for (int p = lo; p < hi; ++p) { + const ScalingRef &r = refs[p]; + float I_corr = 0.0f; + const double w = ReferenceWeight(g, corr[partial_group_perm[p]], r.partiality, r.I, r.sigma, + min_partiality, I_corr); + if (w <= 0.0) continue; + sw += w; + swI += w * I_corr; + } + if (sw > 0.0) out_mean[g] = swI / sw; + } + }); +} + +template +void RotationScaleMerge::FitPerFrameG(const std::vector &obs, const std::vector &fstart, const std::vector &fcount, const std::vector &group_mean_in, bool unity, std::vector &g) { @@ -5560,6 +5584,30 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st logger.Info("Per-frame scaling (partials): the same grouping and settings as an earlier merge, " "so its {} iterations stand", partial_loop.iterations); } else { + // The loop works on its own compact copy of the partials (ScalingObs in the header) and hands + // corr back to them when it is done. + const int n_partials = static_cast(partials.size()); + const size_t partial_threads = ThreadsForWork(partials.size(), nthreads); + std::vector scaling_obs(n_partials); + std::vector scaling_corr(n_partials); + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) { + const Obs &o = partials[i]; + scaling_obs[i] = {o.frame, o.group, o.I, o.sigma, o.partiality, o.prescaling_corr, o.zeta, + o.on_ice}; + scaling_corr[i] = o.corr; + } + }); + std::vector scaling_refs(partial_group_start.empty() + ? 0 : partial_group_start.back() + partial_group_count.back()); + ParallelChunks(n_groups, partial_threads, [&](int glo, int ghi) { + for (int g = glo; g < ghi; ++g) + for (int p = partial_group_start[g]; p < partial_group_start[g] + partial_group_count[g]; ++p) { + const ScalingObs &o = scaling_obs[partial_group_perm[p]]; + scaling_refs[p] = {o.I, o.sigma, o.partiality}; + } + }); + // RunScalingLoop rescales after every iteration before anything reads corr, so the iteration's // UpdateCorr is folded into that pass over the partials: the G and the fitted frames as the // iteration left them, then the rescale on top - the same two roundings, one pass instead of two. @@ -5567,34 +5615,32 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st std::vector frame_fitted; partial_loop = RunScalingLoop("partials", g_partial, frame_count, smooth_window, /*release_restraint=*/false, [&] { - ReducePartialGroupMeans(n_groups, partial_mean); - FitPerFrameG(partials, frame_start, frame_count, partial_mean, /*unity=*/false, g_partial); + ReduceScalingGroupMeans(n_groups, scaling_refs, scaling_corr, partial_mean); + FitPerFrameG(scaling_obs, frame_start, frame_count, partial_mean, /*unity=*/false, g_partial); g_fitted = g_partial; frame_fitted = frame_scaled_scratch; }, [&](const std::vector &apply, const std::vector &ratio) { - ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), - [&](int lo, int hi) { + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { for (int i = lo; i < hi; ++i) { - Obs &o = partials[i]; + const ScalingObs &o = scaling_obs[i]; + float &corr = scaling_corr[i]; if (frame_fitted[o.frame]) { // UpdateCorr const double denom = static_cast(o.partiality) * g_fitted[o.frame]; if (std::isfinite(o.prescaling_corr) && std::isfinite(denom) && denom > 0.0) - o.corr = static_cast(o.prescaling_corr / denom); + corr = static_cast(o.prescaling_corr / denom); else - o.corr = NAN; + corr = NAN; } - if (apply[o.frame] && std::isfinite(o.corr)) - o.corr = static_cast(o.corr * ratio[o.frame]); + if (apply[o.frame] && std::isfinite(corr)) + corr = static_cast(corr * ratio[o.frame]); } }); }); - PartialLoopMemo m{key, partial_loop, frame_scaled_scratch, g_partial, {}}; - m.corr.resize(partials.size()); - ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), - [&](int lo, int hi) { - for (int i = lo; i < hi; ++i) m.corr[i] = partials[i].corr; + ParallelChunks(n_partials, partial_threads, [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) partials[i].corr = scaling_corr[i]; }); + PartialLoopMemo m{key, partial_loop, frame_scaled_scratch, g_partial, std::move(scaling_corr)}; if (partial_loop_memos.size() == MAX_PARTIAL_LOOP_MEMOS) partial_loop_memos.erase(partial_loop_memos.begin()); partial_loop_memos.push_back(std::move(m)); diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index ebef3de15..ea1947c8f 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -446,17 +446,40 @@ private: std::vector partial_group_start, partial_group_count; std::unique_ptr partial_group_perm; + // The CPU partial scaling loop's working copy of the partials. It passes over them several times + // per iteration and reads a handful of fields each time, so it works on these rather than on the + // eighty-byte Obs: ScalingObs holds, in the partials' own order, what FitPerFrameG and the corr + // update read; ScalingRef holds, in the CSR's group order, what the group means read besides corr, + // so that only corr is gathered. The same values, so the same arithmetic in the same order. + struct ScalingObs { + int32_t frame, group; + float I, sigma, partiality, prescaling_corr, zeta; + uint8_t on_ice; + }; + struct ScalingRef { + float I, sigma, partiality; + }; + // The observation's inverse-variance weight in the scaling reference, and its scaled intensity; // 0 when the observation is not in the reference at all. - static double ReferenceWeight(const Obs &o, double min_partiality, float &I_corr); + static double ReferenceWeight(int32_t group, float corr, float partiality, float I, float sigma, + double min_partiality, float &I_corr); + static double ReferenceWeight(const Obs &o, double min_partiality, float &I_corr) { + return ReferenceWeight(o.group, o.corr, o.partiality, o.I, o.sigma, min_partiality, I_corr); + } // Inverse-variance per-group mean of I*corr over `obs` (the merge reference). void ReduceGroupMeans(const std::vector &obs, int n_groups, std::vector &out_mean) const; // The same over the partials, using the CSR above - the same sums in the same order. void ReducePartialGroupMeans(int n_groups, std::vector &out_mean) const; + // The same again over the scaling loop's copy: refs in group order, corr in the partials' order. + void ReduceScalingGroupMeans(int n_groups, const std::vector &refs, + const std::vector &corr, std::vector &out_mean) const; // Robust per-frame G fit (IRLS, Cauchy k=3), unity=false uses the rotation partiality, unity=true the // scale-fulls (partiality already folded in). Reads out_mean[group] as the reference intensity. - void FitPerFrameG(std::vector &obs, const std::vector &fstart, + // O is Obs or ScalingObs. + template + void FitPerFrameG(const std::vector &obs, const std::vector &fstart, const std::vector &fcount, const std::vector &group_mean_in, bool unity, std::vector &g); diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 93b7b9597..62474e7e3 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only #include -#include #include #include "AdaptiveSpotFinderCPU.h" @@ -34,71 +33,150 @@ void AdaptiveSpotFinderCPU::GetProfile(AzimuthalIntegrationProfile &profile) con // Per-ring background statistics with iterated peak exclusion following peakfinder8: // Barty et al. (2014) J. Appl. Cryst. 47, 1118-1131 -// Accumulate per-ring mean/variance from the raw (photon) image. clip_k <= 0 -> use every valid -// pixel (first pass); clip_k > 0 -> keep only pixels within clip_k sigma of the current ring mean, -// which removes the Bragg peaks from the background estimate. -void AdaptiveSpotFinderCPU::AccumulateRings(const ImagePreprocessorBuffer &image, float clip_k) { +// Per-ring mean/variance from the raw (photon) image: a plain pass over every valid pixel, then +// sigma-clip passes keeping only pixels within clip_k sigma of the current ring mean, which removes +// the Bragg peaks from the background estimate. +void AdaptiveSpotFinderCPU::ResetRings() { + std::fill(ring_sum.begin(), ring_sum.end(), 0); + std::fill(ring_sum2.begin(), ring_sum2.end(), 0); + std::fill(ring_cnt.begin(), ring_cnt.end(), 0); + std::fill(ring_hist.begin(), ring_hist.end(), 0); + ring_overflow.clear(); + if (fuse_azint) { + std::fill(azint_sum.begin(), azint_sum.end(), 0.0f); + std::fill(azint_sum2.begin(), azint_sum2.end(), 0.0f); + std::fill(azint_count.begin(), azint_count.end(), 0); + } +} + +void AdaptiveSpotFinderCPU::BeginRings() { + ResetRings(); + rings_from_blocks = true; +} + +// The plain pass over pixels [first, first + n). +void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n) { const auto &pixel_to_bin = mapping.GetPixelToBin(); const size_t nbins = ring_sum.size(); - const size_t npix = static_cast(width) * height; + const float *corrections = mapping.Corrections().data(); + + // Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and + // written back when the ring changes: the same additions in the same order, without a store and a + // reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own + // over the block, so that neither loop runs out of registers for its sums. + if (fuse_azint) { + size_t cur = nbins; // the ring held in the locals below; nbins = none + float az_sum = 0.0f, az_sum2 = 0.0f; + uint32_t az_cnt = 0; + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (b != cur) { + if (cur != nbins) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + cur = b; + az_sum = azint_sum[b]; + az_sum2 = azint_sum2[b]; + az_cnt = azint_count[b]; + } + const float val = static_cast(v) * corrections[pxl]; + const float val_sq = val * val; + az_sum += val; + az_sum2 += val_sq; + ++az_cnt; + } + if (cur != nbins) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + } + + // Values outside the histogram are listed by a second loop, run only when the block has any: a + // call in this loop would leave the sums in memory again. + uint32_t *hist = ring_hist.data(); + bool overflow = false; + size_t cur = nbins; + int64_t sum = 0, cnt = 0; + uint64_t sum2 = 0; + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (b != cur) { + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } + cur = b; + sum = ring_sum[b]; + sum2 = ring_sum2[b]; + cnt = ring_cnt[b]; + } + sum += v; + sum2 += static_cast(static_cast(v) * v); + cnt += 1; + if (v >= 0 && v < HIST_VALUES) + hist[b * HIST_VALUES + v] += 1; + else + overflow = true; + } + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } + + if (overflow) + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; + if (v < 0 || v >= HIST_VALUES) + ring_overflow.emplace_back(b, v); + } +} + +// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the +// pixels holding it would, and its pixels are added as a count. +void AdaptiveSpotFinderCPU::ClipRings(float clip_k) { + const size_t nbins = ring_sum.size(); std::fill(ring_sum.begin(), ring_sum.end(), 0); std::fill(ring_sum2.begin(), ring_sum2.end(), 0); std::fill(ring_cnt.begin(), ring_cnt.end(), 0); - if (clip_k <= 0.0f) { - std::fill(ring_hist.begin(), ring_hist.end(), 0); - ring_overflow.clear(); - const float *corrections = mapping.Corrections().data(); - if (fuse_azint) { - std::fill(azint_sum.begin(), azint_sum.end(), 0.0f); - std::fill(azint_sum2.begin(), azint_sum2.end(), 0.0f); - std::fill(azint_count.begin(), azint_count.end(), 0); - } - for (size_t pxl = 0; pxl < npix; ++pxl) { - const int32_t v = image[pxl]; - if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated - const uint16_t b = pixel_to_bin[pxl]; - if (b >= nbins) continue; // masked / out of range (UINT16_MAX) - if (fuse_azint) { - const float val = static_cast(v) * corrections[pxl]; - const float val_sq = val * val; - azint_sum[b] += val; - azint_sum2[b] += val_sq; - ++azint_count[b]; - } - ring_sum[b] += v; - ring_sum2[b] += static_cast(static_cast(v) * v); - ring_cnt[b] += 1; - if (v >= 0 && v < HIST_VALUES) - ring_hist[b * HIST_VALUES + v] += 1; - else - ring_overflow.emplace_back(b, v); - } - } else { - // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same - // test the pixels holding it would, and its pixels are added as a count. - const auto keep = [&](uint16_t b, int32_t v) { - const float lo = ring_mean[b] - clip_k * ring_sigma[b]; - const float hi = ring_mean[b] + clip_k * ring_sigma[b]; - return !(v < lo || v > hi); // exclude peaks / outliers - }; - for (size_t b = 0; b < nbins; ++b) - for (int32_t v = 0; v < HIST_VALUES; ++v) { - const uint32_t n = ring_hist[b * HIST_VALUES + v]; - if (n == 0 || !keep(static_cast(b), v)) continue; - ring_sum[b] += static_cast(n) * v; - ring_sum2[b] += static_cast(n) * static_cast(static_cast(v) * v); - ring_cnt[b] += n; - } - for (const auto &[b, v] : ring_overflow) { - if (!keep(b, v)) continue; - ring_sum[b] += v; - ring_sum2[b] += static_cast(static_cast(v) * v); - ring_cnt[b] += 1; + const auto keep = [&](uint16_t b, int32_t v) { + const float lo = ring_mean[b] - clip_k * ring_sigma[b]; + const float hi = ring_mean[b] + clip_k * ring_sigma[b]; + return !(v < lo || v > hi); // exclude peaks / outliers + }; + for (size_t b = 0; b < nbins; ++b) + for (int32_t v = 0; v < HIST_VALUES; ++v) { + const uint32_t n = ring_hist[b * HIST_VALUES + v]; + if (n == 0 || !keep(static_cast(b), v)) continue; + ring_sum[b] += static_cast(n) * v; + ring_sum2[b] += static_cast(n) * static_cast(static_cast(v) * v); + ring_cnt[b] += n; } + for (const auto &[b, v] : ring_overflow) { + if (!keep(b, v)) continue; + ring_sum[b] += v; + ring_sum2[b] += static_cast(static_cast(v) * v); + ring_cnt[b] += 1; } +} +void AdaptiveSpotFinderCPU::UpdateRingStatistics() { + const size_t nbins = ring_sum.size(); for (size_t b = 0; b < nbins; ++b) { if (ring_cnt[b] > 0) { const double m = static_cast(ring_sum[b]) / ring_cnt[b]; @@ -114,9 +192,16 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, const size_t nbins = ring_sum.size(); // --- Stage A: robust per-ring background (one plain pass + two sigma-clip passes) --- - AccumulateRings(image, 0.0f); - AccumulateRings(image, 3.0f); - AccumulateRings(image, 3.0f); + if (!rings_from_blocks) { + ResetRings(); + AccumulateRingsBlock(image, 0, static_cast(width) * height); + } + rings_from_blocks = false; + UpdateRingStatistics(); + ClipRings(3.0f); + UpdateRingStatistics(); + ClipRings(3.0f); + UpdateRingStatistics(); // --- Stage B: per-ring threshold from the single portable knob E (false pixels / frame) --- int64_t n_total = 0; @@ -153,11 +238,13 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, : adaptive_threshold::RingThreshold(ring_mean[b], ring_sigma[b], p, z); // --- Stage C: the ring threshold, intersected with the classic local-box SNR test --- - FlagRings(image); + std::fill(ring_bits.begin(), ring_bits.end(), 0); if (settings.signal_to_noise_threshold <= 0.0f) { // No local test asked for: the ring threshold alone decides, as the fixed photon floor // alone would in the classic finder. + for (int32_t row = 0; row < height; row++) + FlagRow(image, row); output_buffer = ring_bits; return; } @@ -165,17 +252,17 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, // The ring threshold IS the photon floor here, so the local pass must not apply another one. SpotFindingSettings local = settings; local.photon_count_threshold = 0; - // Only the ring pixels survive the intersection, so the local test is asked of those alone. - DetectAt(image, local, ring_bits); + // Only the ring pixels survive the intersection, so the local test is asked of those alone. They + // are flagged row by row as the local pass reaches each row, which saves reading the image for it. + DetectAt(image, local, ring_bits, [&](int32_t row) { FlagRow(image, row); }); } -void AdaptiveSpotFinderCPU::FlagRings(const ImagePreprocessorBuffer &image) { +void AdaptiveSpotFinderCPU::FlagRow(const ImagePreprocessorBuffer &image, int32_t row) { const auto &pixel_to_bin = mapping.GetPixelToBin(); const size_t nbins = ring_thr.size(); - const size_t npix = static_cast(width) * height; + const size_t first = static_cast(row) * width; - std::bitset<32> out = 0; - for (size_t pxl = 0; pxl < npix; ++pxl) { + for (size_t pxl = first; pxl < first + width; ++pxl) { const int32_t v = image[pxl]; const uint16_t b = pixel_to_bin[pxl]; bool strong = false; @@ -184,14 +271,7 @@ void AdaptiveSpotFinderCPU::FlagRings(const ImagePreprocessorBuffer &image) { else if (v != INT32_MIN && b < nbins && v >= ring_thr[b]) strong = true; - const int32_t bit = pxl % 32; if (strong) - out.set(bit); - if (bit == 31) { - ring_bits[pxl / 32] = out.to_ulong(); - out.reset(); - } + ring_bits[pxl / 32] |= 1U << (pxl % 32); } - if (npix % 32 != 0) - ring_bits[OutputSize() - 1] = out.to_ulong(); } diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h index f741988b9..d0a82aa5f 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h @@ -71,7 +71,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { std::vector ring_hist; std::vector> ring_overflow; // (ring, value) - // The azimuthal-integration profile, taken in the plain AccumulateRings pass when asked for: the + // The azimuthal-integration profile, taken in the plain ring pass when asked for: the // same pixels in the same order and the same arithmetic as AzIntEngineCPU, so the same sums, and // one pass over the image less (the CPU twin of the fused GPU engine). bool fuse_azint = false; @@ -79,15 +79,31 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { std::vector azint_sum2; std::vector azint_count; - void AccumulateRings(const ImagePreprocessorBuffer &image, float clip_k); - // Fill ring_bits from the thresholds of the current frame. - void FlagRings(const ImagePreprocessorBuffer &image); + // Set by BeginRings(): the plain ring pass of the next Detect() is being accumulated block by block. + bool rings_from_blocks = false; + + // Zero the sums of the plain ring pass (and of the fused profile). + void ResetRings(); + // One sigma-clip pass over the plain pass's values. + void ClipRings(float clip_k); + // ring_mean / ring_sigma from the current sums. + void UpdateRingStatistics(); + // Set the ring_bits of the pixels of one row from the thresholds of the current frame. ring_bits + // is zeroed before the first row. + void FlagRow(const ImagePreprocessorBuffer &image, int32_t row); public: explicit AdaptiveSpotFinderCPU(const AzimuthalIntegrationMapping &mapping); void Detect(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings) override; [[nodiscard]] const std::vector &GetRingBackground() const override { return ring_bkg; } + // The plain ring pass taken while the image is being preprocessed, so the pixels are read while + // still in cache: BeginRings(), then AccumulateRingsBlock() over every block of the image in pixel + // order - the order keeps the profile's float sums the same - and the next Detect() of that image + // starts from these sums instead of passing over the image for them. + void BeginRings(); + void AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n); + void FuseAzimuthalIntegration(bool enable) { fuse_azint = enable; } // The profile of the last Detect(), when fused. void GetProfile(AzimuthalIntegrationProfile &profile) const; diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp index e2a41bc15..1cb6773f9 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp @@ -45,27 +45,33 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid, // pixel is above SNR threshold } +// Adds row `in` to the vertical sums and takes row `out` out of them; nullptr is no row. A bad or +// saturated pixel adds 0. Written without branches, and as a function of its own so that the loop +// vectorises whatever it is inlined into; integer sums, so the same totals as pixel by pixel. +void SlideVertical(const int32_t *in, const int32_t *out, int32_t width, + int64_t *sum, int64_t *sum2, uint16_t *valid) { + for (int32_t col = 0; col < width; col++) { + const int64_t a = in ? in[col] : INT32_MIN; + const int64_t r = out ? out[col] : INT32_MIN; + const bool a_ok = a != INT32_MAX && a != INT32_MIN; + const bool r_ok = r != INT32_MAX && r != INT32_MIN; + const int64_t a_v = a_ok ? a : 0; + const int64_t r_v = r_ok ? r : 0; + sum[col] += a_v - r_v; + sum2[col] += a_v * a_v - r_v * r_v; + valid[col] += static_cast(a_ok - r_ok); + } +} + } // namespace void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, - const std::vector &candidates) { + const std::vector &candidates, + const std::function &fill_row) { candidate_windows.clear(); - // The first pass's bits are read only inside the window of a candidate (and at the candidate), so - // it tests only the pixels of the row/column blocks such a window reaches; the bits it leaves - // unset there are never read. - const int32_t nblocks = (width + 31) / 32; - first_pass_needed.assign(static_cast(height) * nblocks, 0); - for (size_t w = 0; w < candidates.size(); w++) - for (uint32_t bits = candidates[w]; bits; bits &= bits - 1) { - const int64_t pxl = static_cast(w) * 32 + std::countr_zero(bits); - if (pxl >= static_cast(width) * height) - break; - const int32_t line = static_cast(pxl / width), col = static_cast(pxl % width); - for (int32_t y = std::max(line - NBX, 0); y <= std::min(line + NBX, height - 1); y++) - for (int32_t b = std::max(col - NBX, 0) / 32; b <= std::min(col + NBX, width - 1) / 32; b++) - first_pass_needed[static_cast(y) * nblocks + b] = 1; - } - DetectPass(image, settings, nullptr, first_pass_buffer, candidates.data()); + // Filled in by DetectPass as the candidates of each row become known. + first_pass_needed.assign(static_cast(height) * ((width + 31) / 32), 0); + DetectPass(image, settings, nullptr, first_pass_buffer, candidates.data(), fill_row); std::fill(output_buffer.begin(), output_buffer.end(), 0); const float strong2 = settings.signal_to_noise_threshold * settings.signal_to_noise_threshold; @@ -107,10 +113,39 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, const uint32_t *prev_strong, std::vector &out_buffer, - const uint32_t *candidates) { + const uint32_t *candidates, + const std::function &fill_row) { for (int i = 0; i < OutputSize(); i++) out_buffer[i] = 0; + // The first pass's bits are read only inside the window of a candidate (and at the candidate), so + // with candidates it tests only the pixels of the row/column blocks such a window reaches; the bits + // it leaves unset there are never read. Those blocks are marked from each row's candidates as the + // row enters the vertical sums below, which is before any row within NBX of it is tested - so the + // candidates themselves can be filled in a row at a time (fill_row), while the row is in cache. + const int32_t nblocks = (width + 31) / 32; + const auto new_row = [&](int32_t r) { + if (!candidates) + return; + if (fill_row) + fill_row(r); + const int32_t first = r * width; + const int32_t last = first + width - 1; + for (int32_t w = first / 32; w <= last / 32; w++) { + uint32_t bits = candidates[w]; + if (w == first / 32) + bits &= UINT32_MAX << (first % 32); + if (w == last / 32 && last % 32 != 31) + bits &= (1U << (last % 32 + 1)) - 1; + for (; bits; bits &= bits - 1) { + const int32_t col = w * 32 + std::countr_zero(bits) - first; + for (int32_t y = std::max(r - NBX, 0); y <= std::min(r + NBX, height - 1); y++) + for (int32_t b = std::max(col - NBX, 0) / 32; b <= std::min(col + NBX, width - 1) / 32; b++) + first_pass_needed[static_cast(y) * nblocks + b] = 1; + } + } + }; + // A pixel found strong by the previous pass reads as INT32_MAX, which the accumulation below // already skips and the acceptance test below already takes as strong - the same substitution // the GPU kernel makes when it reads prev_out. @@ -146,49 +181,61 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, std::vector sum2_vert(width, 0); std::vector valid_vert(width, 0); - for (int line = 0; line < NBX; line++) { - for (int col = 0; col < width; col++) { - auto pxl = line * width + col; - - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] += tmp; - sum2_vert[col] += tmp * tmp; - valid_vert[col] += 1; + // Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is + // no row. A pixel strong in the previous pass is not counted either; SlideVertical adds it with + // the rest of its row and it is taken out again below - there are few, and the sums are + // integers, so the totals are the same as never adding it. + const int32_t *img = image.data(); + const int32_t w = width; + int64_t *sv = sum_vert.data(); + int64_t *sv2 = sum2_vert.data(); + uint16_t *vv = valid_vert.data(); + auto slide_vert = [&](int line_in, int line_out) { + SlideVertical(line_in >= 0 ? img + static_cast(line_in) * w : nullptr, + line_out >= 0 ? img + static_cast(line_out) * w : nullptr, w, sv, sv2, vv); + if (!prev_strong) + return; + // sign = +1 takes out what line_in added, -1 puts back what line_out took out. + const auto uncount_strong = [&](int line, int64_t sign) { + const int32_t first = line * w, last = first + w - 1; + for (int32_t word = first / 32; word <= last / 32; word++) { + uint32_t bits = prev_strong[word]; + if (word == first / 32) + bits &= UINT32_MAX << (first % 32); + if (word == last / 32 && last % 32 != 31) + bits &= (1U << (last % 32 + 1)) - 1; + for (; bits; bits &= bits - 1) { + const int32_t pxl = word * 32 + std::countr_zero(bits); + const int64_t v = img[pxl]; + if (v == INT32_MAX || v == INT32_MIN) + continue; + sv[pxl - first] -= sign * v; + sv2[pxl - first] -= sign * v * v; + vv[pxl - first] -= static_cast(sign); + } } - } + }; + if (line_in >= 0) + uncount_strong(line_in, 1); + if (line_out >= 0) + uncount_strong(line_out, -1); + }; + + for (int line = 0; line < NBX; line++) { + new_row(line); + slide_vert(line, -1); } for (int line = 0; line < height; line++) { - for (int col = 0; col < width; col++) { - if (line < height - NBX) { - auto pxl = (line + NBX) * width + col; - - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] += tmp; - sum2_vert[col] += tmp * tmp; - valid_vert[col] += 1; - } - } - - if (line >= NBX + 1) { - auto pxl = (line - (NBX + 1)) * width + col; - int64_t tmp = value_at(pxl); - if (tmp != INT32_MAX && tmp != INT32_MIN) { - sum_vert[col] -= tmp; - sum2_vert[col] -= tmp * tmp; - valid_vert[col] -= 1; - } - } - } + if (line < height - NBX) + new_row(line + NBX); + slide_vert(line < height - NBX ? line + NBX : -1, line >= NBX + 1 ? line - (NBX + 1) : -1); if (candidates) { // Only the runs of 32-column blocks whose result is read (first_pass_needed; every // candidate lies in one). Each run starts its window from the vertical sums over the // columns it covers there - integers, so the same sums the running window holds at // that column - and slides it as below; the bits outside the runs stay 0, as below. - const int32_t nblocks = (width + 31) / 32; const uint8_t *needed = first_pass_needed.data() + static_cast(line) * nblocks; for (int32_t b = 0; b < nblocks;) { if (!needed[b]) { b++; continue; } diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.h b/image_analysis/spot_finding/ImageSpotFinderCPU.h index 39e87a270..5b754b2da 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.h +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.h @@ -5,6 +5,7 @@ #include +#include #include #include "ImageSpotFinder.h" @@ -26,10 +27,11 @@ class ImageSpotFinderCPU : public ImageSpotFinder { // fall into, and reported strong. // // With candidates set (SNR test only), the window sums of every candidate pixel are also recorded - // in candidate_windows, in pixel order. + // in candidate_windows, in pixel order, and fill_row (see DetectAt) is called. void DetectPass(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, const uint32_t *prev_strong, std::vector &out, - const uint32_t *candidates = nullptr); + const uint32_t *candidates = nullptr, + const std::function &fill_row = {}); // Local-box sums (centre pixel included) of one candidate pixel, as the first pass saw them. struct CandidateWindow { @@ -47,8 +49,12 @@ protected: // the image: a candidate's second-pass window is its first-pass window with the pixels the first // pass found strong taken out, and those are few, so they are subtracted from the sums the first // pass recorded. Integer sums and the same test, so the bits are exactly Detect's. + // + // fill_row(row), when given, sets the candidates of that row: it is called once for every row, in + // order, before any candidate of the row is read, when the pass is about to read the row anyway. void DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, - const std::vector &candidates); + const std::vector &candidates, + const std::function &fill_row = {}); public: ImageSpotFinderCPU(int32_t width, int32_t height); diff --git a/rugnux/ModelValidation.cpp b/rugnux/ModelValidation.cpp index 0138440fb..1eeef5fa5 100644 --- a/rugnux/ModelValidation.cpp +++ b/rugnux/ModelValidation.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,7 +31,6 @@ #include "../common/CorrelationCoefficient.h" #include "../common/JFJochMath.h" // PI (M_PI is not standard, and MSVC does not define it) #include "../common/Logger.h" -#include "../common/ParallelFor.h" // ParallelFor #include "../image_analysis/scale_merge/ReindexAmbiguity.h" // ReindexReflections #include "../image_analysis/scale_merge/CrystalSetting.h" // CellMappingOperators #include "ModelFFT.h" @@ -90,6 +91,11 @@ constexpr int NULL_REPLICATES = 9; // Fixed, so the same data and the same model give the same verdict on every run. constexpr unsigned NULL_SEED = 20260902; +// How many draws the null may throw away for lying within the rigid body's reach of the model's own +// orientation. Only a model small enough for that reach to cover most orientations gets near it, and +// there the remaining draws are taken as they come rather than the run searching forever. +constexpr int NULL_MAX_REDRAWS = 1000; + // How far above its own null a fit has to sit before the model is allowed to decide anything. The cut // is in sigma of that null and not in R: measured, the R a model that explains nothing reaches moves // with the model's atom count and B-factors as much as with the data, so no value of R separates the @@ -121,6 +127,27 @@ gemmi::Mat33 random_rotation(std::mt19937 &rng) { 2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)}; } +// A rotation of the lattice, given on fractional coordinates as a symmetry operator is, in Cartesian +// coordinates - the frame the null's orientations are drawn in. +gemmi::Mat33 cartesian_rotation(const gemmi::Op &op, const gemmi::UnitCell &cell) { + gemmi::Mat33 f; + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + f.a[i][j] = static_cast(op.rot[i][j]) / gemmi::Op::DEN; + return cell.orth.mat.multiply(f).multiply(cell.frac.mat); +} + +// The angle, in degrees, of the rotation from `rot` to the nearest of `orientations`. +double angle_to_nearest_deg(const gemmi::Mat33 &rot, const std::vector &orientations) { + double best = 180.0; + for (const gemmi::Mat33 &o : orientations) { + const gemmi::Mat33 d = rot.multiply(o.transpose()); + const double c = std::clamp((d.a[0][0] + d.a[1][1] + d.a[2][2] - 1.0) / 2.0, -1.0, 1.0); + best = std::min(best, std::acos(c) * 180.0 / PI); + } + return best; +} + // Everything one fit moves: the model itself, and the structure factors that follow it. The real // model has one of these and every null replicate gets a copy of its own, which is what lets the // replicates run at the same time without stepping on each other. @@ -739,17 +766,47 @@ ModelValidationResult ValidateAgainstModel(const std::vector & // replicates run at the same time below, and a draw taken on whichever thread reached the // generator first would make the answer depend on the scheduling. Replicate i gets rotation i // on every run, so the sigma reported is a property of the data and not of the machine. + // + // A replicate has to start from an orientation the model does NOT have, and there is more than + // one orientation to keep away from. The crystal looks the same from every orientation its + // lattice's rotations take the model to: the space group's own, and those composed with a twin + // law, which is where the model sits against data merged in the other indexing - and the + // replicates are fitted to the data as merged. A draw within the rigid body's reach of any of + // them is walked onto it and scores like the real model: measured on a cubic crystal with a + // twin law, one replicate of nine drawn 7.9 deg from the twin-related orientation was placed + // to R-work 0.20 against 0.57 for the other eight, and the spread that one gave the null was + // enough for the crystal's own model to read as not fitting. Such a draw is not a sample of + // the null, so it is drawn again. Everywhere else the draws are exactly the ones taken before. + std::vector lattice_ops{gemmi::Op::identity()}; + for (const gemmi::Op &law : ReindexAmbiguityOperators(cell, ambiguity_sg)) + lattice_ops.push_back(law); + std::vector equivalent; + for (const gemmi::Op &law : lattice_ops) + for (const gemmi::Op &s : gops.sym_ops) + equivalent.push_back(cartesian_rotation(s.combine(law), ucell)); + const double reach_deg = RigidBodyReachDeg(st.models[0], d_min); std::vector null_rotation; { std::mt19937 rng(NULL_SEED); - for (int i = 0; i < NULL_REPLICATES; i++) - null_rotation.push_back(random_rotation(rng)); + int redraws = 0; + while (static_cast(null_rotation.size()) < NULL_REPLICATES) { + const gemmi::Mat33 rot = random_rotation(rng); + if (angle_to_nearest_deg(rot, equivalent) < reach_deg && redraws < NULL_MAX_REDRAWS) { + redraws++; + continue; + } + null_rotation.push_back(rot); + } + if (redraws > 0) + logger.Info("Model validation: {} random placement(s) fell within the rigid body's reach " + "({:.1f} deg) of an orientation equivalent to the model's, and were drawn again", + redraws, reach_deg); } // Each replicate works on its own copy of the model and of its structure factors, and the // replicates share nothing else, so they run concurrently and the real model - which the // maps and the atom-density readout below are still taken from - is never moved. std::vector null_r_work(NULL_REPLICATES), null_margin(NULL_REPLICATES); - ParallelFor(NULL_REPLICATES, nthreads, [&](int i) { + auto run_replicate = [&](int i) { ModelState rep; rep.st = mdl.st; SetModelPositions(rep.st.models[0], as_read); @@ -762,7 +819,22 @@ ModelValidationResult ValidateAgainstModel(const std::vector & identity_fit = std::move(probe.identity_fit); // replicates are placed against obs } null_r_work[i] = place_and_fit(rep, obs, std::move(identity_fit)).fit.r_work; - }); + }; + // On threads of their own, not on ParallelFor's pool. A parallel pass reached from a pool + // worker runs inline (ParallelFor.h), so there the scale fits and the rigid-body Jacobian of + // every replicate but the one on the calling thread ran serially, and the null lasted as long + // as its slowest serial replicate. From threads outside the pool those passes spread over the + // pool as the real fit's do. Each pass splits its work the same way wherever it runs, so the + // numbers are the same. + std::atomic next_replicate{0}; + std::vector> replicate_threads; + for (size_t t = 0; t < std::min(std::max(nthreads, 1), NULL_REPLICATES); t++) + replicate_threads.push_back(std::async(std::launch::async, [&] { + for (int i = next_replicate++; i < NULL_REPLICATES; i = next_replicate++) + run_replicate(i); + })); + for (std::future &f : replicate_threads) + f.get(); const auto [null_mean, null_sd] = mean_sd(null_r_work); result.fit_tested = true; result.null_replicates = NULL_REPLICATES; @@ -770,13 +842,14 @@ ModelValidationResult ValidateAgainstModel(const std::vector & result.null_r_work_sd = null_sd; auto record_fit = [&](const Placement &p) { - // Guarded at a floor, not at zero: sd == 0 and sd == 1e-7 are one ulp apart and land on - // opposite verdicts, and a null with no spread - a model too small or too symmetric for a - // rotation about its own centroid to move |Fcalc| - is the case that produces it. Measured - // nulls sit near 0.60 with sd ~0.009, so this is nowhere near anything genuine. + // The spread is floored, not trusted down to zero: sd == 0 and sd == 1e-7 are one ulp apart + // and would give sigmas of nothing and of millions. A null with no spread - a model too + // small or too symmetric for a rotation about its own centroid to move |Fcalc| - fits no + // better than its own random placements, so it still reads near zero here. The floor is + // not a gate: a large, well-measured crystal has a null as narrow as 0.0006-0.0010 + // (measured), and a real fit 0.3 below it is hundreds of sigma, not none. constexpr double NULL_SD_FLOOR = 1e-3; - result.r_work_sigma = - null_sd >= NULL_SD_FLOOR ? (null_mean - p.fit.r_work) / null_sd : 0.0; + result.r_work_sigma = (null_mean - p.fit.r_work) / std::max(null_sd, NULL_SD_FLOOR); result.model_fits = result.r_work_sigma >= MODEL_FIT_SIGMA; }; record_fit(real); @@ -794,8 +867,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & result.indexing_margin_null_mean = margin_mean; result.indexing_margin_null_sd = margin_sd; constexpr double MARGIN_SD_FLOOR = 1e-4; - result.indexing_margin_sigma = - margin_sd >= MARGIN_SD_FLOOR ? (indexing.margin - margin_mean) / margin_sd : 0.0; + result.indexing_margin_sigma = (indexing.margin - margin_mean) / std::max(margin_sd, MARGIN_SD_FLOOR); // The margin is judged against the margin a random placement produces, not against a value. // A random model also picks a winner, and on these data it picks one by a comparable lead // (measured), so the raw margin says nothing on its own. diff --git a/rugnux/RigidBodyRefine.cpp b/rugnux/RigidBodyRefine.cpp index 8b36f44d5..ccf9c8137 100644 --- a/rugnux/RigidBodyRefine.cpp +++ b/rugnux/RigidBodyRefine.cpp @@ -214,8 +214,17 @@ public: bool Evaluate(double const *const *parameters, double *residuals, double **jacobians) const override { const int n = num_residuals(); - if (!ev_.Residuals(parameters[0], residuals)) - return false; + // Ceres evaluates every step it accepts twice: the residuals alone to decide whether to take + // it, then residuals and Jacobian together at the same point. The residuals are a function of + // the placement alone, so the second time they are the first time's, copied. + if (!last_residuals_.empty() && std::equal(parameters[0], parameters[0] + 6, last_q_.begin())) { + std::copy(last_residuals_.begin(), last_residuals_.end(), residuals); + } else { + if (!ev_.Residuals(parameters[0], residuals)) + return false; + std::copy(parameters[0], parameters[0] + 6, last_q_.begin()); + last_residuals_.assign(residuals, residuals + n); + } if (jacobians != nullptr && jacobians[0] != nullptr) { const double step = ev_.JacobianStep(); std::vector> shifted(6, std::vector(n)); @@ -244,6 +253,8 @@ private: Evaluator &ev_; std::vector &columns_; size_t nthreads_; + mutable std::array last_q_{}; + mutable std::vector last_residuals_; // at last_q_; empty until an evaluation succeeds }; // The directions in which this space group's origin is free. Translating the whole cell content along @@ -291,6 +302,24 @@ void SetModelPositions(gemmi::Model &model, const std::vector & // the movement to recover is the crystal's, not the molecule's. Splitting it into domains or giving a // bound ligand its own six parameters would refine against evidence this data does not separately // carry, and the ligand is what the difference map is meant to show rather than model away. +double RigidBodyReachDeg(const gemmi::Model &model, double d_min) { + const std::vector pos = ModelPositions(model); + if (pos.empty()) + return 0.0; + gemmi::Position centre; + for (const gemmi::Position &p : pos) + centre += p; + centre *= 1.0 / static_cast(pos.size()); + double r2 = 0; + for (const gemmi::Position &p : pos) + r2 += centre.dist_sq(p); + const double rms_radius = std::sqrt(r2 / static_cast(pos.size())); + // The first zone RefineRigidBody walks, which is d_min itself where d_min is coarser than all + // of the ladder. + const double first_zone = std::max(LADDER[0], d_min); + return first_zone / rms_radius * 180.0 / PI; +} + // // Some of the translation would be a gauge rather than a quantity - the origin is free in all three // directions in P1 and along the unique axis in a polar group, and |F| does not change when the whole diff --git a/rugnux/RigidBodyRefine.h b/rugnux/RigidBodyRefine.h index a17f3489b..6c15a3b3d 100644 --- a/rugnux/RigidBodyRefine.h +++ b/rugnux/RigidBodyRefine.h @@ -27,6 +27,12 @@ struct RigidBodyRefineResult { std::vector ModelPositions(const gemmi::Model &model); void SetModelPositions(gemmi::Model &model, const std::vector &pos); +// How far, in degrees, a rotation about the centroid can be from the answer and still be walked onto +// it by RefineRigidBody: the angle that moves the model's rms-radius atom by the resolution of the +// ladder's first zone. Inside it the first zone sees the rotated model overlap the density it belongs +// to, and its single broad minimum is that placement. +double RigidBodyReachDeg(const gemmi::Model &model, double d_min); + // Refine the placement of `model` as one rigid body against the observed amplitudes: an angle-axis // rotation about the model's own centroid followed by a translation, six parameters, over a // coarse-to-fine resolution ladder. Each evaluation recomputes Fcalc and the bulk-solvent mask for diff --git a/tests/AdaptiveSpotFinderCPUTest.cpp b/tests/AdaptiveSpotFinderCPUTest.cpp index 0db0270a6..14e2cbc61 100644 --- a/tests/AdaptiveSpotFinderCPUTest.cpp +++ b/tests/AdaptiveSpotFinderCPUTest.cpp @@ -28,7 +28,7 @@ SpotFindingSettings AdaptiveSettings() { // Raw (untransformed) geometry: the mapping is built over the raw module layout, which is smaller // than the converted image. The finder has to walk the raw image, so a spot planted at a raw pixel // comes back at that pixel. -TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder]") { +TEST_CASE("AdaptiveSpotFinderCPU_RawGeometry", "[AdaptiveSpotFinder][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(80).BeamX_pxl(1030).BeamY_pxl(1080); x.QSpacingForAzimInt_recipA(0.05).QRangeForAzimInt_recipA(0.05, 5.0); diff --git a/tests/AzimuthalIntegrationTest.cpp b/tests/AzimuthalIntegrationTest.cpp index d5adf25fc..5151c4faf 100644 --- a/tests/AzimuthalIntegrationTest.cpp +++ b/tests/AzimuthalIntegrationTest.cpp @@ -5,6 +5,7 @@ #include "../common/AzimuthalIntegrationProfile.h" #include "../common/AzimuthalIntegrationMapping.h" +#include "../common/JFJochMath.h" TEST_CASE("AzimuthalIntegrationMapping_Constructor","[AzimuthalIntegration]") { DiffractionExperiment x(DetJF4M()); @@ -39,7 +40,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetBinNumber_mask","[AzimuthalIntegration for (int row = 0; row < x.GetYPixelsNum(); row++) { for (int col = 0; col < x.GetXPixelsNum(); col++) { float d = geom.PxlToRes(col, row); - float q = 2 * M_PI / d; + float q = 2 * PI / d; if (q >= 3.1) pixel_mask[row * x.GetXPixelsNum() + col] = 1; } @@ -147,7 +148,7 @@ TEST_CASE("AzimuthalIntegrationMapping_GetMapping","[AzimuthalIntegration]") { CHECK(map[x.GetXPixelsNum() * 1500 + 1500] / 40 == 0); } -TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationMapping_QToBin","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4); @@ -400,7 +401,7 @@ TEST_CASE("AzimuthalIntegrationProfile_GetMeanValueOfBins","[AzimuthalIntegratio REQUIRE(profile.GetBkgEstimate(x.GetAzimuthalIntegrationSettings()) == Catch::Approx((sum[0] + sum[1] + sum[2]) / double(count[0] + count[1] + count[2]))); } -TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration]") { +TEST_CASE("AzimuthalIntegrationProfile_GetResult1D","[AzimuthalIntegration][portable]") { DiffractionExperiment x(DetJF4M()); x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000); diff --git a/tests/BraggIntegrationEngineCompressedImageTest.cpp b/tests/BraggIntegrationEngineCompressedImageTest.cpp index 46344bb22..881c4b4b8 100644 --- a/tests/BraggIntegrationEngineCompressedImageTest.cpp +++ b/tests/BraggIntegrationEngineCompressedImageTest.cpp @@ -93,7 +93,7 @@ void RequireIdentical(const std::vector &a, const std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -491,7 +491,7 @@ TEST_CASE("CBORSerialize_Calibration", "[CBOR]") { CHECK(memcmp(output_message.GetCompressed(), calib1.data(), 256 * sizeof(float)) == 0); } -TEST_CASE("CBORSerialize_End", "[CBOR]") { +TEST_CASE("CBORSerialize_End", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -897,7 +897,7 @@ TEST_CASE("CBORSerialize_Image_Rgb", "[CBOR]") { REQUIRE(memcmp(image_array.image.GetCompressed(), test.data(), test.size() * sizeof(rgb)) == 0); } -TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); @@ -1026,7 +1026,7 @@ TEST_CASE("CBORSerialize_Start_ROIMap", "[CBOR]") { } -TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { +TEST_CASE("CBORSerialize_Image_Spots", "[CBOR][portable]") { std::vector buffer(8 * 1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b386a5a4..7ce420525 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,42 @@ +# Quick cross-platform checks: the results that depend on the architecture, the compiler or the +# standard library (bitshuffle/LZ4 SIMD paths, HDF5 read-back, CBF parsing, CPU spot finding, FFTW +# indexing, gemmi MTZ/CIF). Built in every configuration, the portable (viewer/rugnux-only) ones +# included, and linked only against what those build - so it runs on the macOS and Windows jobs. +# Only the cases tagged [portable] are meant to run there (a few seconds in all); the rest of these +# files is compiled in because the tagged cases live next to them. +# EXCLUDE_FROM_ALL: a product build does not pay for it, CI builds the target by name. +ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL + PortableTest.cpp + TestData.h + ZSTDCompressorTest.cpp + JFJochReaderTest.cpp + SweepLayoutTest.cpp + CBORTest.cpp + ImageSpotFinderCPUTest.cpp + AdaptiveSpotFinderCPUTest.cpp + AzimuthalIntegrationTest.cpp + CalcBraggPredictionTest.cpp + BraggStencilTest.cpp + BraggIntegrationEngineCompressedImageTest.cpp + WriteReflectionsTest.cpp + LoadReferenceMtzFreeFlagsTest.cpp + LoadReferenceSfCifTest.cpp + RotationIndexerTest.cpp +) +target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis + JFJochWriter JFJochCommon) +target_include_directories(jfjoch_portable_test PRIVATE .) +# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically. +IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) + SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) +ENDIF() + +IF (JFJOCH_PORTABLE_ONLY) + RETURN() +ENDIF() + ADD_EXECUTABLE(jfjoch_test + PortableTest.cpp DiffractionExperimentTest.cpp RawToConvertedGeometryTest.cpp ../common/RawToConvertedGeometry.h diff --git a/tests/CalcBraggPredictionTest.cpp b/tests/CalcBraggPredictionTest.cpp index 9d94c9a90..48f0c1661 100644 --- a/tests/CalcBraggPredictionTest.cpp +++ b/tests/CalcBraggPredictionTest.cpp @@ -4,6 +4,7 @@ #include #include "../image_analysis/bragg_prediction/BraggPrediction.h" +#include "../common/JFJochMath.h" #include #include "../image_analysis/SensorAbsorption.h" #include "../image_analysis/bragg_prediction/BraggPredictionRot.h" @@ -40,11 +41,11 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { // crossed strictly more of the medium. float prev = 1.0f; for (double alpha_deg : {10.0, 20.0, 30.0, 40.0, 50.0, 55.0}) { - const float f = a18.Factor(static_cast(std::cos(alpha_deg * M_PI / 180.0))); + const float f = a18.Factor(static_cast(std::cos(alpha_deg * PI / 180.0))); CHECK(f > prev); prev = f; } - const float cos55 = static_cast(std::cos(55.0 * M_PI / 180.0)); + const float cos55 = static_cast(std::cos(55.0 * PI / 180.0)); // 18 keV over 160 mm at 55 degrees: +1.5%. CHECK(a18.Factor(cos55) == Catch::Approx(1.0147).epsilon(0.002)); @@ -88,7 +89,7 @@ TEST_CASE("BraggPrediction_FlightPathAttenuation", "[air_path]") { CHECK(vac.WilsonBShift_A2(3.02, 50.0, lambda_of(3.76)) == 0.0); } -TEST_CASE("BraggPrediction_11keV") { +TEST_CASE("BraggPrediction_11keV", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(11.0); @@ -160,10 +161,10 @@ TEST_CASE("BraggPrediction_15keV") { } } -TEST_CASE("BraggPrediction_Rot1_Rot2") { +TEST_CASE("BraggPrediction_Rot1_Rot2", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); @@ -224,7 +225,7 @@ TEST_CASE("BraggPrediction_backscattering") { } } -TEST_CASE("BraggPrediction_systematic_absences") { +TEST_CASE("BraggPrediction_systematic_absences", "[portable]") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(120.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) .IncidentEnergy_keV(12.0); @@ -404,7 +405,7 @@ TEST_CASE("BraggPredictionGPU") { TEST_CASE("BraggPredictionGPU_Rot1_Rot2") { DiffractionExperiment experiment(DetJF4M()); experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0) - .PoniRot1_rad(2.0/180.0 * M_PI).PoniRot2_rad(3.0/180.0 * M_PI) + .PoniRot1_rad(2.0/180.0 * PI).PoniRot2_rad(3.0/180.0 * PI) .IncidentEnergy_keV(11.0); DiffractionGeometry geom = experiment.GetDiffractionGeometry(); diff --git a/tests/ImageSpotFinderCPUTest.cpp b/tests/ImageSpotFinderCPUTest.cpp index fe680a6cb..35d06773e 100644 --- a/tests/ImageSpotFinderCPUTest.cpp +++ b/tests/ImageSpotFinderCPUTest.cpp @@ -4,7 +4,7 @@ #include #include "../image_analysis/spot_finding/ImageSpotFinderCPU.h" -TEST_CASE("ImageSpotFinderCPU_SignalToNoise") { +TEST_CASE("ImageSpotFinderCPU_SignalToNoise", "[portable]") { size_t width = 100; size_t height = 100; @@ -108,7 +108,7 @@ TEST_CASE("ImageSpotFinderCPU_CountThreshold_Resolution") { REQUIRE(spots[1].RawCoord().y == 75); } -TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask") { +TEST_CASE("ImageSpotFinderCPU_CountThreshold_Mask", "[portable]") { size_t width = 100; size_t height = 100; diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index ba8d01780..9ade44cb7 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -608,7 +608,7 @@ TEST_CASE("JFJochReader_GridScan", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test09").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -774,7 +774,7 @@ TEST_CASE("JFJochReader_DataI16_OldMasterFormat", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU16", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test10").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -846,7 +846,7 @@ TEST_CASE("JFJochReader_DataU16", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataI32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test11").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -913,7 +913,7 @@ TEST_CASE("JFJochReader_DataI32", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_DataU32", "[HDF5][Full]") { +TEST_CASE("JFJochReader_DataU32", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test12").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1786,7 +1786,7 @@ TEST_CASE("JFJochReader_NXmxIntegrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1906,7 +1906,7 @@ TEST_CASE("JFJochReader_GetRawImage_Concurrent", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -1965,7 +1965,7 @@ TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true); @@ -2022,7 +2022,7 @@ TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_integrated").ImagesPerTrigger(5).OverwriteExistingFiles(true); @@ -2070,7 +2070,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_Integrated", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_vds_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -2131,7 +2131,7 @@ TEST_CASE("JFJochReader_HDF5DataSource_VDS", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full]") { +TEST_CASE("JFJochReader_HDF5DataSource_Legacy", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.FilePrefix("source_legacy_mapping").ImagesPerTrigger(5).ImagesPerFile(2).OverwriteExistingFiles(true); @@ -3434,7 +3434,7 @@ TEST_CASE("JFJochReader_AxisRecovery", "[HDF5][Full]") { // saturation_value is written inclusive and used exclusive, so a read has to add the count back. It // did not, and the value fell by one on every write-read-write cycle - unbounded, and compounding // whenever a _process.h5 was reprocessed. Nothing caught it: no test asserted the read-back limit. -TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full]") { +TEST_CASE("JFJochReader_SaturationSurvivesRoundTrip", "[HDF5][Full][portable]") { DiffractionExperiment x(DetJF(1)); x.ImagesPerTrigger(2).OverwriteExistingFiles(true).FilePrefix("test_satrt"); x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) @@ -3958,7 +3958,7 @@ namespace { } } -TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a pixel at Count_cutoff is saturated") { @@ -4081,7 +4081,7 @@ namespace { // Three ways a CBF that is not a detector image, or is one with a hole in its head, used to be // opened anyway - each of them silently, which is the failure this project cares most about. -TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full]") { +TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("a byte-offset CBF with no PILATUS header is not ours to read") { @@ -4128,7 +4128,7 @@ TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Fu } } -TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full]") { +TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full][portable]") { // The header is read with a small first probe, and again with the large one when it does not end // inside it; what stands past the first 16 kB must come out exactly as before. std::string filler; @@ -4261,7 +4261,7 @@ namespace { } } -TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full][portable]") { const int64_t nx = 24, ny = 16; SECTION("the header's geometry reaches the experiment in the units the rest of the code uses") { @@ -4425,7 +4425,7 @@ namespace { } } -TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full]") { +TEST_CASE("JFJochSMVReader_Geometry", "[HDF5][Full][portable]") { SECTION("a pixel at CCD_IMAGE_SATURATION is saturated, stated or not") { // The pixels are handed out as 32-bit, so without an explicit limit the fallback would be // INT32_MAX and no 16-bit image could ever reach it. diff --git a/tests/LoadReferenceMtzFreeFlagsTest.cpp b/tests/LoadReferenceMtzFreeFlagsTest.cpp index c2c5b811f..12bc73e43 100644 --- a/tests/LoadReferenceMtzFreeFlagsTest.cpp +++ b/tests/LoadReferenceMtzFreeFlagsTest.cpp @@ -42,7 +42,7 @@ namespace { bool ExpectedFree(int h, int k, int l) { return (h * 7 + k * 13 + l * 17) % 10 == 0; } } -TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz imports FreeR (CCP4 convention: test set = 0)", "[reference_mtz][portable]") { const auto path = (std::filesystem::temp_directory_path() / "rugnux_freer_ccp4.mtz").string(); WriteMtzWithFreeR(path, /*free_value=*/0); const auto ref = LoadReferenceMtz(path); diff --git a/tests/LoadReferenceSfCifTest.cpp b/tests/LoadReferenceSfCifTest.cpp index b56a2dff6..c8027fb15 100644 --- a/tests/LoadReferenceSfCifTest.cpp +++ b/tests/LoadReferenceSfCifTest.cpp @@ -100,7 +100,7 @@ _refln.intensity_sigma } } -TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz]") { +TEST_CASE("LoadReferenceMtz reads a structure-factor mmCIF", "[reference_mtz][portable]") { // Named without a .cif extension: the format is recognised by content. const auto path = WriteFile("rugnux_ref_sf.dat", SF_CIF); const auto ref = LoadReferenceMtz(path); diff --git a/tests/PortableTest.cpp b/tests/PortableTest.cpp new file mode 100644 index 000000000..3c67eed15 --- /dev/null +++ b/tests/PortableTest.cpp @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Checks that the results which depend on the architecture, the compiler or the standard library are +// the same everywhere: the bitshuffle/LZ4 path (hperf SSE2/AVX2 on x86, NEON on aarch64, SSE2 under +// MSVC) and an end-to-end run of the analysis. Golden values were taken on x86-64 Linux; an arm64 +// or MSVC build that disagrees has a portability defect, not a new reference. + +#include + +#include +#include + +#include "TestData.h" +#include "../compression/JFJochCompressor.h" +#include "../compression/JFJochDecompress.h" +#include "../common/JFJochMath.h" +#include "../writer/HDF5Objects.h" +#include "../image_analysis/indexing/IndexerFactory.h" +#include "../reader/JFJochHDF5Reader.h" +#include "../rugnux/Rugnux.h" + +namespace { + // FNV-1a, spelled out: std::hash is not the same function on every standard library. + uint64_t Fnv1a(const void *data, size_t size) { + auto p = static_cast(data); + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < size; i++) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; + } + + uint64_t SplitMix64(uint64_t &state) { + uint64_t z = (state += 0x9e3779b97f4a7c15ULL); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; + z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; + return z ^ (z >> 31); + } +} + +// compression_benchmark.h5 holds 25 int16 frames, bitshuffle/LZ4 (HDF5 filter 32008). One frame is +// read three ways - the raw chunk decoded by the hperf path and by the classic bitshuffle, and +// through the HDF5 filter - and all three must give the frame whose hash was recorded on x86. +TEST_CASE("Portable_BSLZ4_BenchmarkFrameGoldenHash", "[portable][compression]") { + RegisterHDF5Filter(); + HDF5ReadOnlyFile file("../../tests/test_data/compression_benchmark.h5"); + HDF5DataSet dataset(file, "/entry/data/data"); + HDF5DataSpace file_space(dataset); + const auto dims = file_space.GetDimensions(); + REQUIRE(dims.size() == 3); + const size_t npixel = dims[1] * dims[2]; + + const std::pair golden[] = { + {0, 0xaffea29c511b6ec2ULL}, + {24, 0xe46913009c95a1f1ULL} + }; + + for (const auto &[frame, hash]: golden) { + std::vector chunk; + dataset.ReadDirectChunk(chunk, {frame, 0, 0}); + + std::vector hperf, classic, filtered(npixel); + JFJochDecompress(hperf, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, true); + JFJochDecompress(classic, CompressionAlgorithm::BSHUF_LZ4, chunk, npixel, false); + dataset.ReadVector(filtered, {frame, 0, 0}, {1, dims[1], dims[2]}); + + CHECK(Fnv1a(hperf.data(), hperf.size() * sizeof(int16_t)) == hash); + CHECK(hperf == classic); + CHECK(hperf == filtered); + } +} + +// The encoder must write the same bytes everywhere: a file is decoded on a different machine from +// the one that wrote it, and the header of every block is big-endian regardless of the host. +TEST_CASE("Portable_BSLZ4_EncodeGoldenHash", "[portable][compression]") { + uint64_t state = 42; + std::vector image(1024 * 1024 + 13); // a tail that is not a whole block + for (auto &v: image) + v = (SplitMix64(state) % 16 == 0) ? SplitMix64(state) % 200 : 0; + + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + auto compressed = compressor.Compress(image); + CHECK(compressed.size() == 562349); + CHECK(Fnv1a(compressed.data(), compressed.size()) == 0x874e45eb17a51ed2ULL); + + std::vector decoded; + JFJochDecompress(decoded, CompressionAlgorithm::BSHUF_LZ4, compressed, image.size()); + CHECK(decoded == image); +} + +// One bitshuffle block through the shipped selector (hperf on x86, classic NEON on aarch64) against +// the classic reference transform, over the element sizes and block tails the file format allows. +TEST_CASE("Portable_BitShuffleBlock_MatchesReference", "[portable][compression]") { + uint64_t state = 7; + for (size_t elem: {1, 2, 4, 8}) { + for (size_t size: {8, 16, 24, 136, 1032, 2040, 2048, 8192}) { + for (int pattern = 0; pattern < 3; pattern++) { + std::vector orig(size * elem), ref(size * elem), enc(size * elem), + scratch(size * elem), out(size * elem); + for (size_t i = 0; i < orig.size(); i++) { + if (pattern == 0) orig[i] = static_cast(SplitMix64(state)); // noise + else if (pattern == 1) orig[i] = (i % elem == 0) ? static_cast(i % 7) : 0; // counts + else orig[i] = static_cast((i % elem == elem - 1) ? 0x80 : 0); // sign bit + } + INFO("elem " << elem << " size " << size << " pattern " << pattern); + REQUIRE(bshuf_trans_bit_elem(orig.data(), ref.data(), size, elem) >= 0); + REQUIRE(JFJochBitShuffleBlock(enc.data(), orig.data(), scratch.data(), size, elem) >= 0); + CHECK(enc == ref); + REQUIRE(JFJochBitUnshuffleBlock(out.data(), ref.data(), scratch.data(), size, elem) >= 0); + CHECK(out == orig); + } + } + } +} + +// The CPU indexer (FFTW, NEON codelets on arm64) on a synthetic orthorhombic lattice. Named +// explicitly rather than through Auto, so a CUDA build on a runner without a GPU tests it too. +TEST_CASE("Portable_FFTWIndexer_SyntheticLattice", "[portable][Indexing]") { + const UnitCell uc(39, 45, 78, 90, 90, 90); + const CrystalLattice cl(uc); + + DiffractionExperiment experiment; + IndexingSettings settings; + settings.Algorithm(IndexingAlgorithmEnum::FFTW).FFT_MaxUnitCell_A(250.0).FFT_HighResolution_A(2 * PI / 3.0); + experiment.ImportIndexingSettings(settings).SetUnitCell(uc); + auto indexer = CreateIndexer(experiment); + REQUIRE(indexer); + + std::vector spots; + for (int h = -2; h < 10; h++) + for (int k = -5; k < 10; k++) + for (int l = -3; l < 10; l++) + spots.push_back(h * cl.Astar() + k * cl.Bstar() + l * cl.Cstar()); + + const auto result = indexer->Run(spots); + REQUIRE(result.lattice.size() == 1); + const auto out = result.lattice[0].GetUnitCell(); + std::array lengths = {out.a, out.b, out.c}; + std::sort(lengths.begin(), lengths.end()); + CHECK(lengths[0] == Catch::Approx(39.0)); + CHECK(lengths[1] == Catch::Approx(45.0)); + CHECK(lengths[2] == Catch::Approx(78.0)); + CHECK(out.alpha == Catch::Approx(90.0)); + CHECK(out.beta == Catch::Approx(90.0)); + CHECK(out.gamma == Catch::Approx(90.0)); +} + +// End to end on the git-LFS rotation dataset (tests/data, see README.md there): HDF5 read through +// external links, CPU spot finding, the default indexer (FFTW on a CPU build), refinement and +// integration, on a short wedge. SKIPs when LFS was not pulled. +TEST_CASE("Portable_Rugnux_RotationWedge", "[portable][lfs]") { + const auto master = jfjoch_test::LargeDataFile("rotation_master.h5"); + if (!master) + SKIP("rotation_master.h5 not available (git-lfs data not pulled)"); + + RegisterHDF5Filter(); + JFJochHDF5Reader reader; + REQUIRE_NOTHROW(reader.ReadFile(*master)); + auto dataset = reader.GetDataset(); + REQUIRE(dataset); + + DiffractionExperiment experiment(dataset->experiment); + IndexingSettings indexing; + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + experiment.ImportIndexingSettings(indexing); + + ProcessConfig config; + config.mode = ProcessMode::FullAnalysis; + config.nthreads = std::max(1u, std::thread::hardware_concurrency()); + config.end_image = 4; + config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); + config.spot_finding.indexing = true; + + Rugnux process(reader, experiment, *dataset->pixel_mask, config); + ProcessResult result; + REQUIRE_NOTHROW(result = process.Run()); + + CHECK(result.images_processed == 4); + REQUIRE(result.indexing_rate.has_value()); + CHECK(result.indexing_rate.value() > 0.5f); + reader.Close(); +} diff --git a/tests/RotationIndexerTest.cpp b/tests/RotationIndexerTest.cpp index 73cfa44e4..a98f74fe6 100644 --- a/tests/RotationIndexerTest.cpp +++ b/tests/RotationIndexerTest.cpp @@ -18,13 +18,7 @@ TEST_CASE("RotationIndexer") { .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_i.ImportIndexingSettings(settings); @@ -113,13 +107,7 @@ TEST_CASE("RotationIndexer::RefineConstrained puts a free metric back on its cla .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_i.ImportIndexingSettings(settings); @@ -195,13 +183,7 @@ static std::optional IndexOnTiltedDetector(double true_ti .ImagesPerTrigger(50); IndexingSettings settings; -#ifdef JFJOCH_USE_CUDA - settings.Algorithm(IndexingAlgorithmEnum::FFT); -#elif JFJOCH_USE_FFTW - settings.Algorithm(IndexingAlgorithmEnum::FFTW); -#else - return {}; -#endif + settings.Algorithm(IndexingAlgorithmEnum::Auto); // rotation: FFT with a GPU, FFTW without settings.RotationIndexing(true).RotationIndexingAngularStride_deg(1.0).RotationIndexingMinAngularRange_deg(30.0); exp_header.ImportIndexingSettings(settings); @@ -278,7 +260,7 @@ TEST_CASE("RotationIndexer keeps a tilt a mounting can have") { // index; the result records the walk, both counts, the verdict, and a geometry that matches it. The // lattice is not checked: a 1.5 deg detector error on 6 A data already puts the FFT on a different // cell before any fit. -TEST_CASE("RotationIndexer judges a tilt no mounting can have on the spots") { +TEST_CASE("RotationIndexer judges a tilt no mounting can have on the spots", "[portable]") { const auto ret = IndexOnTiltedDetector(1.5, 6.0f); REQUIRE(ret.has_value()); REQUIRE(ret->tilt_walk.has_value()); diff --git a/tests/SweepLayoutTest.cpp b/tests/SweepLayoutTest.cpp index a3ef530d3..88663dde5 100644 --- a/tests/SweepLayoutTest.cpp +++ b/tests/SweepLayoutTest.cpp @@ -21,7 +21,7 @@ std::vector Series(const std::vector &angles, double incre } // namespace -TEST_CASE("SweepLayout_Contiguous") { +TEST_CASE("SweepLayout_Contiguous", "[portable]") { std::vector angles; for (int i = 0; i < 100; i++) angles.push_back(20.0 + 0.1 * i); @@ -34,7 +34,7 @@ TEST_CASE("SweepLayout_Contiguous") { CHECK(l.files[i] == "f" + std::to_string(i) + ".cbf"); } -TEST_CASE("SweepLayout_Gapped") { +TEST_CASE("SweepLayout_Gapped", "[portable]") { // Frames 3, 4 and 7 of a ten-frame sweep never made it into the archive. The sweep is still ten // steps wide and every frame keeps its own angle. const auto l = sweep::Place(Series({0.0, 0.1, 0.2, 0.5, 0.6, 0.8, 0.9}, 0.1), "test"); diff --git a/tests/WriteReflectionsTest.cpp b/tests/WriteReflectionsTest.cpp index 8609e410e..b8e6d06c4 100644 --- a/tests/WriteReflectionsTest.cpp +++ b/tests/WriteReflectionsTest.cpp @@ -31,7 +31,7 @@ namespace { } } -TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections]") { +TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[write_reflections][portable]") { jfjoch_test::SyntheticMergeParams params; params.true_space_group = "P 43 21 2"; params.twin_supergroup = "P 43 21 2"; @@ -74,7 +74,7 @@ TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[w } TEST_CASE("Unmerged MTZ: LP is Lorentz-polarization, QE the sensor efficiency, FLIGHT the flight path", - "[write_reflections]") { + "[write_reflections][portable]") { // The whole point of the split: LP must mean what XDS and DIALS mean by it, and the raw count // sum must still be recoverable from the file alone, as I / LP * QE * FLIGHT. auto experiment = TestExperiment(); diff --git a/tests/ZSTDCompressorTest.cpp b/tests/ZSTDCompressorTest.cpp index 5a497601e..9e2295b16 100644 --- a/tests/ZSTDCompressorTest.cpp +++ b/tests/ZSTDCompressorTest.cpp @@ -229,7 +229,7 @@ TEST_CASE("JFjochZstdCompressor_Frame_ones","[ZSTD]") { REQUIRE(memcmp(input_shuffled.data(), output.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_ZSTD).BitDepthImage(32).PixelSigned(true); @@ -269,7 +269,7 @@ TEST_CASE("JFJochCompressor_DestTooSmall_Throws","[ZSTD]") { REQUIRE_NOTHROW(compressor.Compress(big.data(), big.size(), image)); } -TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") { +TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD][portable]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(32).PixelSigned(true); @@ -291,6 +291,34 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") { REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0); } +TEST_CASE("JFJochDecompressBlocks_matches_JFJochDecompress","[ZSTD]") { + // An element count that leaves a partial block and a few unshuffled trailing elements. + const size_t nelements = 3 * 8192 + 1000 + 5; + std::vector image(nelements); + std::mt19937 g(1); + std::poisson_distribution dist(3.0); + for (auto &i: image) + i = dist(g); + + for (auto algorithm: {CompressionAlgorithm::BSHUF_LZ4, CompressionAlgorithm::BSHUF_ZSTD}) { + JFJochBitShuffleCompressor compressor(algorithm); + std::vector tmp(nelements * sizeof(uint32_t) * 2 + 12); + tmp.resize(compressor.Compress(tmp.data(), tmp.size(), image)); + + std::vector output(nelements, UINT32_MAX); + size_t next = 0; + JFJochDecompressBlocks(algorithm, reinterpret_cast(tmp.data()), tmp.size(), + nelements, sizeof(uint32_t), + [&](size_t first, const uint8_t *data, size_t n) { + REQUIRE(first == next); + memcpy(output.data() + first, data, n * sizeof(uint32_t)); + next = first + n; + }); + REQUIRE(next == nelements); + REQUIRE(output == image); + } +} + TEST_CASE("JFJochDecompressor_None","[ZSTD]") { DiffractionExperiment x(DetJF4M()); x.Compression(CompressionAlgorithm::NO_COMPRESSION).BitDepthImage(32).PixelSigned(true); @@ -396,7 +424,7 @@ TEST_CASE("ZstdHuff_MaskLike", "[ZSTD]") { RequireHuffRoundTrip(image); } -TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD]") { +TEST_CASE("ZstdHuff_BenchmarkImage", "[ZSTD][portable]") { RegisterHDF5Filter(); // bitshuffle filter, needed to read the compressed benchmark dataset HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); HDF5DataSet dataset(data, "/entry/data/data"); diff --git a/tests/data/README.md b/tests/data/README.md index 3227cd93b..b1353efcf 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -23,7 +23,8 @@ git lfs pull # or: git lfs pull --include "tests/data/*.h5" | `rotation_master.h5` | protein rotation series (~1800 images) | yes (LFS) | `rotation_master.h5` (plus its `_data_NNNNNN.h5` files) is fetched by `git lfs pull` and -drives `Rugnux_Rotation`. A separate serial dataset is intentionally **not** shipped +drives `Rugnux_Rotation`, and its first four frames `Portable_Rugnux_RotationWedge` (the +`[portable]` end-to-end check, which skips on the macOS/Windows jobs because they do not pull LFS). A separate serial dataset is intentionally **not** shipped to keep the repository small — the rotation series can be run in serial mode (full analysis without rotation indexing) to exercise that path. To add your own dataset, drop the master + its data files here as real files (not symlinks, if you intend to commit them via LFS); the master