Jungfraujoch/tests/RugnuxLargeTest.cpp

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

// End-to-end Rugnux runs over real JUNGFRAU datasets that are kept in git-LFS under
// tests/data. They are tagged [large] and SKIP() when the data is not present (e.g. LFS not
// pulled), so the default test run stays fast and CI without the data still passes.

#include <catch2/catch_all.hpp>
#include <catch2/reporters/catch_reporter_event_listener.hpp>
#include <catch2/reporters/catch_reporter_registrars.hpp>

#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <thread>

#include "TestData.h"
#include "../common/DiffractionExperiment.h"
#include "../common/HostMemory.h"
#include "../common/IndexingSettings.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../rugnux/Rugnux.h"
#include "../rugnux/RugnuxOutputFiles.h"

namespace {
    // Start-up hook: report once whether the large datasets are available, so it is obvious why
    // the [large] tests skip when they do.
    class LargeDataListener : public Catch::EventListenerBase {
    public:
        using Catch::EventListenerBase::EventListenerBase;
        void testRunStarting(Catch::TestRunInfo const &) override {
            const bool rot = jfjoch_test::LargeDataFile("rotation_master.h5").has_value();
            std::cout << "[jfjoch_test] large dataset in " << jfjoch_test::LargeDataDir()
                      << ": rotation=" << (rot ? "yes" : "no")
                      << " ([large] tests skip when absent)" << std::endl;
        }
    };

    int default_threads() {
        const unsigned hc = std::thread::hardware_concurrency();
        return hc == 0 ? 4 : static_cast<int>(hc);
    }
}

CATCH_REGISTER_LISTENER(LargeDataListener)

TEST_CASE("Rugnux_Rotation", "[large]") {
    const auto master = jfjoch_test::LargeDataFile("rotation_master.h5");
    if (!master)
        SKIP("rotation_master.h5 not available (git-lfs data not pulled)");

    RegisterHDF5Filter();
    JFJochHDF5Reader reader;
    REQUIRE_NOTHROW(reader.ReadFile(*master));
    auto dataset = reader.GetDataset();
    REQUIRE(dataset);

    DiffractionExperiment experiment(dataset->experiment);
    IndexingSettings indexing;
    indexing.Algorithm(IndexingAlgorithmEnum::Auto);
    indexing.RotationIndexing(true);
    indexing.GeomRefinementAlgorithm(GeomRefinementAlgorithmEnum::BeamCenter);
    experiment.ImportIndexingSettings(indexing);

    ProcessConfig config;
    config.mode = ProcessMode::FullAnalysis;
    config.nthreads = default_threads();
    config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings();
    config.spot_finding.indexing = true;
    config.rotation_indexing = true;
    config.two_pass_rotation = true;

    Rugnux process(reader, experiment, *dataset->pixel_mask, config);
    ProcessResult result;
    REQUIRE_NOTHROW(result = process.Run());

    CHECK_FALSE(result.cancelled);
    CHECK(result.images_processed == reader.GetNumberOfImages());
    REQUIRE(result.indexing_rate.has_value());
    CHECK(result.indexing_rate.value() > 0.1f);          // a real rotation series indexes well
    CHECK(result.consensus_cell.has_value());

    reader.Close();
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

// A rotation run reuses what its first pass found - an indexing probe's validation evidence, each
// frame's spot list, the rotation indexer's outcome - wherever it asks the same question again: in the
// rotation-scale walk's probes and in the canonical pass after them. verify_first_pass_memo recomputes
// at every one of those hits instead and throws where the recomputation differs, so this run passing
// is the check that the reuse is exact. The geometry post-refinement and the scaling it runs inside
// are on because they are what starts the walk.
TEST_CASE("Rugnux_FirstPassMemoMatchesRecomputation", "[large]") {
    const auto master = jfjoch_test::LargeDataFile("rotation_master.h5");
    if (!master)
        SKIP("rotation_master.h5 not available (git-lfs data not pulled)");

    RegisterHDF5Filter();
    JFJochHDF5Reader reader;
    REQUIRE_NOTHROW(reader.ReadFile(*master));
    auto dataset = reader.GetDataset();
    REQUIRE(dataset);

    DiffractionExperiment experiment(dataset->experiment);
    IndexingSettings indexing;
    indexing.Algorithm(IndexingAlgorithmEnum::Auto);
    indexing.RotationIndexing(true);
    experiment.ImportIndexingSettings(indexing);

    ProcessConfig config;
    config.mode = ProcessMode::FullAnalysis;
    config.nthreads = default_threads();
    config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings();
    config.spot_finding.indexing = true;
    config.rotation_indexing = true;
    config.two_pass_rotation = true;
    config.rotation_postrefine_geometry = true;
    config.run_scaling = true;
    config.verify_first_pass_memo = true;

    Rugnux process(reader, experiment, *dataset->pixel_mask, config);
    ProcessResult result;
    REQUIRE_NOTHROW(result = process.Run());
    CHECK(result.consensus_cell.has_value());

    reader.Close();
}

// The host memory budget decides only what runs side by side - the quality guard's merge beside the
// ingest, the post-refinement beside the merge chain, the two search merges beside each other, how many
// large per-worker buffers are held at once - and never what is computed. So a run planned against a
// budget nothing fits in (everything in series) writes what a run with room for everything writes, to
// the byte.
TEST_CASE("Rugnux_HostBudgetChangesOnlyTheSchedule", "[large]") {
    const auto master = jfjoch_test::LargeDataFile("rotation_master.h5");
    if (!master)
        SKIP("rotation_master.h5 not available (git-lfs data not pulled)");

    RegisterHDF5Filter();
    const auto slurp = [](const std::string &path) {
        std::ifstream f(path, std::ios::binary);
        return std::string(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
    };
    const auto run = [&](size_t budget, const std::string &prefix) {
        JFJochHDF5Reader reader;
        REQUIRE_NOTHROW(reader.ReadFile(*master));
        auto dataset = reader.GetDataset();
        REQUIRE(dataset);
        DiffractionExperiment experiment(dataset->experiment);
        IndexingSettings indexing;
        indexing.Algorithm(IndexingAlgorithmEnum::Auto);
        indexing.RotationIndexing(true);
        experiment.ImportIndexingSettings(indexing);

        ProcessConfig config;
        config.mode = ProcessMode::FullAnalysis;
        config.nthreads = default_threads();
        config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings();
        config.spot_finding.indexing = true;
        config.rotation_indexing = true;
        config.two_pass_rotation = true;
        config.rotation_postrefine_geometry = true;
        config.run_scaling = true;
        config.host_memory_budget_bytes = budget;

        Rugnux process(reader, experiment, *dataset->pixel_mask, config);
        ProcessResult result;
        REQUIRE_NOTHROW(result = process.Run());
        reader.Close();
        // The library writes nothing: the files are the CLI's, written from the result
        Logger logger("Rugnux_HostBudgetChangesOnlyTheSchedule");
        WriteRugnuxOutputs(result, prefix, config.provenance, config.nthreads, logger);
        return result;
    };
    const std::filesystem::path dir = std::filesystem::temp_directory_path() / "rugnux_host_budget_test";
    std::filesystem::create_directories(dir);
    const ProcessResult roomy = run(size_t(1) << 40, (dir / "roomy").string());
    const ProcessResult tight = run(size_t(1) << 20, (dir / "tight").string());

    REQUIRE(roomy.has_merge_statistics);
    CHECK(tight.merge_statistics_text == roomy.merge_statistics_text);
    CHECK(tight.merged_i_sigma == roomy.merged_i_sigma);
    const std::string hkl = slurp((dir / "roomy.hkl").string());
    CHECK_FALSE(hkl.empty());
    CHECK(slurp((dir / "tight.hkl").string()) == hkl);
    std::filesystem::remove_all(dir);
    HostWorkerPool::Instance().SetSize(0);   // process-wide: back to the machine's for the tests after this one
    HostWorkerPool::Instance().SetBudget(0);
}