leonarski_fandClaude Fable 5
dc8a80f3e0
The viewer could grow to ~100 GB RAM when live-following an HTTP broker. The rc.153 images_in_flight backpressure only throttled imageLoaded; the heavy per-frame payload rides datasetLoaded, fanned out over ~10 queued cross-thread connections with no cap. In HTTPSyncDataset follow mode (entered when an operator clicks an image while following live) RefreshDatasetOnly_i emits a fresh full dataset every autoload tick with no imageLoaded, so the gate never engaged and the queued events - each pinning a full JFJochReaderDataset (full-detector PixelMask + per-image plots) - accumulated without bound. Backpressure datasetLoaded the same way as imageLoaded: a datasets_in_flight counter (cap 2), all emits routed through EmitDatasetLoaded_i, and AutoLoadTimerExpired gated on it (covers HTTPSyncDataset). The window routes the worker's datasetLoaded through a single OnDatasetReady sink that fans out synchronously via datasetReady and acks with datasetConsumed. Under load stale datasets are dropped; the next tick sends the latest. Share the pixel mask instead of deep-copying it: JFJochReaderDataset::pixel_mask is now shared_ptr<const PixelMask>, so per-frame dataset copies share the ~72 MB mask. UpdateUserMask does copy-on-write; JFJochHttpReader caches the mask by arm_date so a live refresh reuses one shared mask per acquisition. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
83 lines
3.2 KiB
C++
83 lines
3.2 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
// End-to-end Rugnux runs over real JUNGFRAU datasets that are kept in git-LFS under
|
|
// tests/data. They are tagged [large] and SKIP() when the data is not present (e.g. LFS not
|
|
// pulled), so the default test run stays fast and CI without the data still passes.
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
#include <catch2/reporters/catch_reporter_event_listener.hpp>
|
|
#include <catch2/reporters/catch_reporter_registrars.hpp>
|
|
|
|
#include <iostream>
|
|
#include <thread>
|
|
|
|
#include "TestData.h"
|
|
#include "../common/DiffractionExperiment.h"
|
|
#include "../common/IndexingSettings.h"
|
|
#include "../reader/JFJochHDF5Reader.h"
|
|
#include "../rugnux/Rugnux.h"
|
|
|
|
namespace {
|
|
// Start-up hook: report once whether the large datasets are available, so it is obvious why
|
|
// the [large] tests skip when they do.
|
|
class LargeDataListener : public Catch::EventListenerBase {
|
|
public:
|
|
using Catch::EventListenerBase::EventListenerBase;
|
|
void testRunStarting(Catch::TestRunInfo const &) override {
|
|
const bool rot = jfjoch_test::LargeDataFile("lyso_rotation_master.h5").has_value();
|
|
std::cout << "[jfjoch_test] large dataset in " << jfjoch_test::LargeDataDir()
|
|
<< ": rotation=" << (rot ? "yes" : "no")
|
|
<< " ([large] tests skip when absent)" << std::endl;
|
|
}
|
|
};
|
|
|
|
int default_threads() {
|
|
const unsigned hc = std::thread::hardware_concurrency();
|
|
return hc == 0 ? 4 : static_cast<int>(hc);
|
|
}
|
|
}
|
|
|
|
CATCH_REGISTER_LISTENER(LargeDataListener)
|
|
|
|
TEST_CASE("Rugnux_LysoRotation", "[large]") {
|
|
const auto master = jfjoch_test::LargeDataFile("lyso_rotation_master.h5");
|
|
if (!master)
|
|
SKIP("lyso_rotation_master.h5 not available (git-lfs data not pulled)");
|
|
|
|
RegisterHDF5Filter();
|
|
JFJochHDF5Reader reader;
|
|
REQUIRE_NOTHROW(reader.ReadFile(*master));
|
|
auto dataset = reader.GetDataset();
|
|
REQUIRE(dataset);
|
|
|
|
DiffractionExperiment experiment(dataset->experiment);
|
|
IndexingSettings indexing;
|
|
indexing.Algorithm(IndexingAlgorithmEnum::Auto);
|
|
indexing.RotationIndexing(true);
|
|
indexing.GeomRefinementAlgorithm(GeomRefinementAlgorithmEnum::BeamCenter);
|
|
experiment.ImportIndexingSettings(indexing);
|
|
|
|
ProcessConfig config;
|
|
config.mode = ProcessMode::FullAnalysis;
|
|
config.nthreads = default_threads();
|
|
config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings();
|
|
config.spot_finding.indexing = true;
|
|
config.rotation_indexing = true;
|
|
config.two_pass_rotation = true;
|
|
config.reuse_rotation_spots = false; // redo spot finding (raw dataset may carry no spots)
|
|
|
|
Rugnux process(reader, experiment, *dataset->pixel_mask, config);
|
|
ProcessResult result;
|
|
REQUIRE_NOTHROW(result = process.Run());
|
|
|
|
CHECK_FALSE(result.cancelled);
|
|
CHECK(result.images_processed == reader.GetNumberOfImages());
|
|
REQUIRE(result.indexing_rate.has_value());
|
|
CHECK(result.indexing_rate.value() > 0.1f); // a real rotation series indexes well
|
|
CHECK(result.consensus_cell.has_value());
|
|
|
|
reader.Close();
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|