viewer: fix HTTP live-follow OOM via datasetLoaded backpressure + shared pixel mask

The viewer could grow to ~100 GB RAM when live-following an HTTP broker. The
rc.153 images_in_flight backpressure only throttled imageLoaded; the heavy
per-frame payload rides datasetLoaded, fanned out over ~10 queued cross-thread
connections with no cap. In HTTPSyncDataset follow mode (entered when an operator
clicks an image while following live) RefreshDatasetOnly_i emits a fresh full
dataset every autoload tick with no imageLoaded, so the gate never engaged and
the queued events - each pinning a full JFJochReaderDataset (full-detector
PixelMask + per-image plots) - accumulated without bound.

Backpressure datasetLoaded the same way as imageLoaded: a datasets_in_flight
counter (cap 2), all emits routed through EmitDatasetLoaded_i, and
AutoLoadTimerExpired gated on it (covers HTTPSyncDataset). The window routes the
worker's datasetLoaded through a single OnDatasetReady sink that fans out
synchronously via datasetReady and acks with datasetConsumed. Under load stale
datasets are dropped; the next tick sends the latest.

Share the pixel mask instead of deep-copying it: JFJochReaderDataset::pixel_mask
is now shared_ptr<const PixelMask>, so per-frame dataset copies share the ~72 MB
mask. UpdateUserMask does copy-on-write; JFJochHttpReader caches the mask by
arm_date so a live refresh reuses one shared mask per acquisition.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

reader/HDF5MetadataSource.cpp

@@ -692,7 +692,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
                 );
             if (mask_tmp.empty())
                 mask_tmp = std::vector<uint32_t>(image_size_x * image_size_y);
-            dataset->pixel_mask = PixelMask(mask_tmp);
+            dataset->pixel_mask = std::make_shared<const PixelMask>(mask_tmp);
         }
 
         ReadROIMetadata(*master_file, *dataset);

reader/JFJochHttpReader.cpp

@@ -19,6 +19,8 @@ void JFJochHttpReader::Close() {
     SetStartMessage({});
     last_image_buffer_counter = {};
     last_op_http_sync = false;
+    cached_pixel_mask.reset();
+    cached_pixel_mask_arm_date.clear();
 }
 
 ImageBufferStatus JFJochHttpReader::GetImageBufferStatus() const {
@@ -162,8 +164,17 @@ std::shared_ptr<JFJochReaderDataset> JFJochHttpReader::UpdateDataset_i() {
                 std::chrono::microseconds(std::lround(msg->start_message->count_time * 1e6))
         );
 
-        if (!msg->start_message->pixel_mask.empty())
-            dataset->pixel_mask = PixelMask(msg->start_message->pixel_mask.begin()->second);
+        if (!msg->start_message->pixel_mask.empty()) {
+            // The pixel mask is constant for an acquisition; build the full-detector (~tens of MB)
+            // PixelMask once per arm and share it across the per-refresh dataset snapshots, so a live
+            // dataset refresh does not reconstruct and copy it on every tick.
+            if (!cached_pixel_mask || cached_pixel_mask_arm_date != dataset->arm_date) {
+                cached_pixel_mask =
+                    std::make_shared<const PixelMask>(msg->start_message->pixel_mask.begin()->second);
+                cached_pixel_mask_arm_date = dataset->arm_date;
+            }
+            dataset->pixel_mask = cached_pixel_mask;
+        }
 
         dataset->experiment.NumTriggers(1);
         dataset->experiment.ImagesPerTrigger(msg->start_message->number_of_images);

reader/JFJochHttpReader.h

@@ -15,6 +15,11 @@ class JFJochHttpReader : public JFJochReader {
     std::optional<int64_t> last_image_buffer_counter;
     bool last_op_http_sync = false;
 
+    // Cache of the (constant-per-acquisition) pixel mask, keyed on arm date, so the per-refresh
+    // dataset rebuild reuses one shared mask instead of reconstructing it every tick.
+    std::shared_ptr<const PixelMask> cached_pixel_mask;
+    std::string cached_pixel_mask_arm_date;
+
     ImageBufferStatus GetImageBufferStatus() const;
 
     bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,

reader/JFJochReader.cpp

@@ -95,7 +95,10 @@ void JFJochReader::UpdateUserMask(const std::vector<uint32_t> &mask) {
     if (!dataset)
         return;
     auto new_dataset = std::make_shared<JFJochReaderDataset>(*dataset);
-    new_dataset->pixel_mask.LoadUserMask(dataset->experiment, mask);
+    // Copy-on-write: the mask is shared with the old snapshot, so edit a fresh copy, not in place.
+    auto new_mask = std::make_shared<PixelMask>(*dataset->pixel_mask);
+    new_mask->LoadUserMask(dataset->experiment, mask);
+    new_dataset->pixel_mask = new_mask;
     dataset = new_dataset;
 }
 

reader/JFJochReaderDataset.h

@@ -7,6 +7,7 @@
 #include <optional>
 #include <vector>
 #include <map>
+#include <memory>
 
 #include "../common/DiffractionGeometry.h"
 #include "../common/DiffractionExperiment.h"
@@ -17,7 +18,10 @@ struct JFJochReaderDataset {
     std::string arm_date;
 
     DiffractionExperiment experiment;
-    PixelMask pixel_mask;
+    // Shared, not copied, across dataset snapshots: the mask is constant for a run, so a per-frame
+    // dataset refresh / mutable copy must not clone the full-detector (~tens of MB) mask. Held as
+    // shared_ptr<const>; the rare edit (user mask) builds a fresh mask (copy-on-write). Never null.
+    std::shared_ptr<const PixelMask> pixel_mask = std::make_shared<const PixelMask>();
 
     std::optional<int64_t> error_value;
 

reader/JFJochReaderImage.cpp

@@ -99,7 +99,7 @@ void JFJochReaderImage::ProcessInputImage(const void *data, size_t npixel, int64
     top_pixels.reserve(top_pixels_acc.Capacity());
 
     bool has_input_mask = false;
-    const auto &mask = dataset->pixel_mask.GetMask();
+    const auto &mask = dataset->pixel_mask->GetMask();
     if (mask.size() == npixel)
         has_input_mask = true;
 

tests/JFJochReaderTest.cpp

@@ -239,8 +239,8 @@ TEST_CASE("JFJochReader_PixelMask", "[HDF5][Full]") {
         JFJochHDF5Reader reader;
         reader.ReadFile("test16_master.h5");
         auto dataset = reader.GetDataset();
-        REQUIRE(dataset->pixel_mask.GetMask().size() == x.GetPixelsNum());
-        CHECK(dataset->pixel_mask.GetMask() == pixel_mask);
+        REQUIRE(dataset->pixel_mask->GetMask().size() == x.GetPixelsNum());
+        CHECK(dataset->pixel_mask->GetMask() == pixel_mask);
         std::shared_ptr<JFJochReaderImage> reader_image;
 
         REQUIRE_NOTHROW(reader_image = reader.LoadImage(0));

tests/RugnuxLargeTest.cpp

@@ -67,7 +67,7 @@ TEST_CASE("Rugnux_LysoRotation", "[large]") {
     config.two_pass_rotation = true;
     config.reuse_rotation_spots = false;   // redo spot finding (raw dataset may carry no spots)
 
-    Rugnux process(reader, experiment, dataset->pixel_mask, config);
+    Rugnux process(reader, experiment, *dataset->pixel_mask, config);
     ProcessResult result;
     REQUIRE_NOTHROW(result = process.Run());
 

tests/RugnuxTest.cpp

@@ -57,7 +57,7 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") {
     config.nthreads = 2;
     config.output_prefix = "process_azint_out";
 
-    Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config);
+    Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
     ProcessResult result;
     REQUIRE_NOTHROW(result = process.Run());
 
@@ -95,7 +95,7 @@ TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") {
     config.mode = ProcessMode::AzimuthalIntegration;
     config.nthreads = 3;
 
-    Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config);
+    Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
     auto result = process.Run();
 
     CHECK_FALSE(result.cancelled);
@@ -119,7 +119,7 @@ TEST_CASE("Rugnux_Cancel", "[HDF5][Full]") {
     config.mode = ProcessMode::AzimuthalIntegration;
     config.nthreads = 2;
 
-    Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config);
+    Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
     process.Cancel();   // cancel before running: the worker loop stops immediately
     auto result = process.Run();
 

tools/rugnux_cli.cpp

@@ -1035,7 +1035,7 @@ int main(int argc, char **argv) {
         config.nthreads = nthreads;
         config.output_prefix = output_prefix;
 
-        Rugnux process(reader, experiment, dataset->pixel_mask, config);
+        Rugnux process(reader, experiment, *dataset->pixel_mask, config);
         g_active_process = &process;
         std::signal(SIGINT, handle_sigint);
 
@@ -1199,7 +1199,7 @@ int main(int argc, char **argv) {
     // _process.h5 unless explicitly requested. Without merging, the _process.h5 is the only output.
     config.write_process_h5 = run_scaling ? write_process_h5_flag : true;
 
-    Rugnux process(reader, experiment, dataset->pixel_mask, config);
+    Rugnux process(reader, experiment, *dataset->pixel_mask, config);
 
     g_active_process = &process;
     std::signal(SIGINT, handle_sigint);

viewer/JFJochImageReadingWorker.cpp

@@ -293,7 +293,7 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
             UpdateAzint_i(dataset.get());
         }
 
-        emit datasetLoaded(dataset);
+        EmitDatasetLoaded_i(dataset);
         EmitReferenceInfo_i();   // refresh the reference's consistency check against the new dataset
 
         // Reset the run collection for the freshly opened file (file mode only has snapshots).
@@ -313,7 +313,7 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
         emit fileLoadError("File Load Error", QString::fromStdString(e.what()));
         ResetFileOpenRetry_i();
 
-        emit datasetLoaded({});
+        EmitDatasetLoaded_i({});
         EmitImageLoaded_i({});
     }
 }
@@ -345,7 +345,7 @@ void JFJochImageReadingWorker::CloseFile() {
     total_images = 0;
     current_file = "";
     EmitImageLoaded_i({});
-    emit datasetLoaded({});
+    EmitDatasetLoaded_i({});
 }
 
 void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation) {
@@ -363,9 +363,9 @@ void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation
 
 void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) {
     if (dataset) {
-        azint_mapping = std::make_unique<AzimuthalIntegrationMapping>(curr_experiment, dataset->pixel_mask);
+        azint_mapping = std::make_unique<AzimuthalIntegrationMapping>(curr_experiment, *dataset->pixel_mask);
         index_and_refine = std::make_unique<IndexAndRefine>(curr_experiment, indexing.get());
-        image_analysis = std::make_unique<MXAnalysisWithoutFPGA>(curr_experiment, *azint_mapping, dataset->pixel_mask,
+        image_analysis = std::make_unique<MXAnalysisWithoutFPGA>(curr_experiment, *azint_mapping, *dataset->pixel_mask,
             *index_and_refine.get());
 
         last_profile_.reset();
@@ -391,7 +391,7 @@ void JFJochImageReadingWorker::LoadImage_i(int64_t image_number, int64_t summati
             }
             current_image_ptr = tmp_image_ptr;
             total_images = http_reader.GetNumberOfImages();
-            emit datasetLoaded(http_reader.GetDataset());
+            EmitDatasetLoaded_i(http_reader.GetDataset());
         } else {
             if (image_number < 0 || image_number + summation > total_images)
                 return;
@@ -483,7 +483,7 @@ void JFJochImageReadingWorker::UpdateDataset_i(const std::optional<DiffractionEx
     curr_experiment.ImportScalingSettings(scaling_settings);
     UpdateAzint_i(dataset.get());
 
-    emit datasetLoaded(dataset);
+    EmitDatasetLoaded_i(dataset);
 
     current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);
     ApplyROIOverrideToImage_i();
@@ -701,7 +701,7 @@ void JFJochImageReadingWorker::MaskFromSelectedROI(QString name, bool add) {
     if (!elem)
         return;
 
-    auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
+    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
     auto geom = exp.GetDiffractionGeometry();
     const int64_t width = exp.GetXPixelsNum();
     const int64_t height = exp.GetYPixelsNum();
@@ -765,7 +765,7 @@ void JFJochImageReadingWorker::SetROIDefinition_i(const ROIDefinition &rois) {
         }
     }
 
-    emit datasetLoaded(dataset);
+    EmitDatasetLoaded_i(dataset);
     EmitImageLoaded_i(current_image_ptr);
 }
 
@@ -795,7 +795,7 @@ void JFJochImageReadingWorker::UpdateUserMask_i(const std::vector<uint32_t> &mas
 
     UpdateAzint_i(dataset.get());
 
-    emit datasetLoaded(dataset);
+    EmitDatasetLoaded_i(dataset);
 
     current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);
 
@@ -817,7 +817,7 @@ void JFJochImageReadingWorker::SaveUserMaskTIFF(QString filename) {
     QMutexLocker locker(&m);
     if (!current_image_ptr)
         return;
-    auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
+    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
     CompressedImage mask_image(user_mask, current_image_ptr->Dataset().experiment.GetXPixelsNum(), current_image_ptr->Dataset().experiment.GetYPixelsNum());
     WriteTIFFToFile(filename.toStdString(), mask_image);
 }
@@ -848,7 +848,7 @@ void JFJochImageReadingWorker::LoadUserMaskTIFF(QString filename, bool replace)
 
     // replace: start from an empty mask; add: keep the current one. A non-zero TIFF pixel masks.
     auto user_mask = replace ? std::vector<uint32_t>(width * height, 0)
-                             : current_image_ptr->Dataset().pixel_mask.GetUserMask();
+                             : current_image_ptr->Dataset().pixel_mask->GetUserMask();
     for (size_t i = 0; i < tiff.size(); i++)
         if (tiff[i] != 0)
             user_mask[i] = 1;
@@ -861,7 +861,7 @@ void JFJochImageReadingWorker::UploadUserMask() {
     if (!current_image_ptr)
         return;
 
-    auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
+    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
     if (http_mode)
         http_reader.UploadUserMask(user_mask);
 }
@@ -891,6 +891,10 @@ void JFJochImageReadingWorker::AutoLoadTimerExpired() {
     // skipped, which for a live monitor is exactly right (the next fetch grabs the newest image).
     if (images_in_flight >= max_images_in_flight)
         return;
+    // Same gate for datasets: covers HTTPSyncDataset (which emits no image), so its per-tick
+    // datasetLoaded flood is throttled to the GUI's dataset drain rate instead of piling up.
+    if (datasets_in_flight >= max_datasets_in_flight)
+        return;
 
     switch (autoload_mode) {
         case AutoloadMode::HTTPSync:
@@ -925,6 +929,19 @@ void JFJochImageReadingWorker::ImageConsumed() {
     --images_in_flight;
 }
 
+void JFJochImageReadingWorker::EmitDatasetLoaded_i(const std::shared_ptr<const JFJochReaderDataset>& dataset) {
+    // Assumes m locked! Dataset-stream analogue of EmitImageLoaded_i. Each queued datasetLoaded event
+    // pins a whole dataset; DatasetConsumed() decrements once the GUI has fanned it out and rendered
+    // it, and AutoLoadTimerExpired refuses to produce another live refresh while the cap is reached.
+    ++datasets_in_flight;
+    emit datasetLoaded(dataset);
+}
+
+void JFJochImageReadingWorker::DatasetConsumed() {
+    QMutexLocker locker(&m);
+    --datasets_in_flight;
+}
+
 void JFJochImageReadingWorker::SetHttpConnected_i(bool connected, const QString &addr) {
     // Assumes m locked! Only signals on a real change so the status bar does not flicker.
     if (connected == http_connected)
@@ -948,7 +965,7 @@ void JFJochImageReadingWorker::RefreshDatasetOnly_i() {
                 emit imageNumberChanged(total_images, current_image.value());
         }
         if (dataset)
-            emit datasetLoaded(dataset);
+            EmitDatasetLoaded_i(dataset);
     } catch (std::exception &e) {
         logger.Debug("Dataset refresh failed: {}", e.what());
         SetHttpConnected_i(false, current_file);
@@ -1022,7 +1039,7 @@ ReprocessingInputs JFJochImageReadingWorker::GetReprocessingInputs() const {
         return in;
     in.file = current_file;
     in.experiment = curr_experiment;
-    in.pixel_mask = current_image_ptr->Dataset().pixel_mask;
+    in.pixel_mask = *current_image_ptr->Dataset().pixel_mask;
     in.spot_finding = spot_finding_settings;
     if (reference_)
         in.reference_data = reference_->reflections;
@@ -1041,7 +1058,7 @@ void JFJochImageReadingWorker::ActivateSnapshot_i(const QString &name) {
     curr_experiment.ImportIndexingSettings(indexing_settings);
     curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings);
     UpdateAzint_i(dataset.get());
-    emit datasetLoaded(dataset);
+    EmitDatasetLoaded_i(dataset);
 
     QStringList names;
     for (const auto &n: file_reader.SnapshotNames())

viewer/JFJochImageReadingWorker.h

@@ -131,6 +131,14 @@ private:
     int images_in_flight = 0;
     int max_images_in_flight = 4;
 
+    // Same backpressure for the dataset stream (see EmitDatasetLoaded_i / DatasetConsumed): each
+    // queued datasetLoaded event pins a full dataset (mask + per-image plots), so cap how many may
+    // be in flight to the GUI. This also coalesces dataset-only follow (HTTPSyncDataset), which emits
+    // no image and so was never gated by images_in_flight - stale datasets are dropped, the next tick
+    // sends the latest. Without this the datasetLoaded backlog grew without bound (100 GB OOM).
+    int datasets_in_flight = 0;
+    int max_datasets_in_flight = 2;
+
     // File open retry/back-off (GPFS via NFSv4 visibility lag)
     struct PendingLoadRequest {
         QString filename;
@@ -153,6 +161,7 @@ private:
     void LoadFile_i(const QString &filename, qint64 image_number, qint64 summation, bool retry);
     void LoadImage_i(int64_t image_number, int64_t summation);
     void EmitImageLoaded_i(const std::shared_ptr<const JFJochReaderImage>& image);
+    void EmitDatasetLoaded_i(const std::shared_ptr<const JFJochReaderDataset>& dataset);
     void RefreshDatasetOnly_i();
     void SetHttpConnected_i(bool connected, const QString &addr);
     void ReanalyzeImage_i();
@@ -239,6 +248,10 @@ public slots:
     // autoload backpressure so the reader can't outrun the viewer (see images_in_flight).
     void ImageConsumed();
 
+    // GUI ack that it has consumed (fanned out + rendered) one dataset, freeing an in-flight slot.
+    // Drives the datasetLoaded backpressure so the reader can't outrun the viewer's dataset drain.
+    void DatasetConsumed();
+
     // Register a reprocessing result (_process.h5) as a metadata snapshot over the same images and
     // make it the active dataset; SetActiveSnapshot switches between registered snapshots.
     void RegisterProcessingSnapshot(QString id, QString label, QString master_path);

viewer/JFJochViewerWindow.cpp

@@ -192,6 +192,14 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
     connect(this, &JFJochViewerWindow::imageConsumed,
             reading_worker, &JFJochImageReadingWorker::ImageConsumed);
 
+    // Dataset stream routed the same way, so it gets the same backpressure: the worker's datasetLoaded
+    // crosses the thread boundary once into OnDatasetReady, which re-emits datasetReady to every GUI
+    // consumer synchronously and then acks the worker (datasetConsumed).
+    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+            this, &JFJochViewerWindow::OnDatasetReady);
+    connect(this, &JFJochViewerWindow::datasetConsumed,
+            reading_worker, &JFJochImageReadingWorker::DatasetConsumed);
+
     connect(this, &JFJochViewerWindow::imageReady,
             viewer, &JFJochDiffractionImage::loadImage);
 
@@ -257,13 +265,13 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
     connect(menuBar, &JFJochViewerMenu::uploadUserMaskSelected,
             reading_worker, &JFJochImageReadingWorker::UploadUserMask);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             tableWindow, &JFJochViewerImageListWindow::datasetLoaded);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             spotWindow, &JFJochViewerSpotListWindow::datasetLoaded);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             metadataWindow, &JFJochViewerMetadataWindow::datasetLoaded);
 
     connect(this, &JFJochViewerWindow::imageReady,
@@ -297,7 +305,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
             reading_worker, &JFJochImageReadingWorker::FindCenter);
 
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             reflectionWindow, &JFJochViewerReflectionListWindow::datasetLoaded);
 
     connect(this, &JFJochViewerWindow::imageReady,
@@ -360,12 +368,12 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
     connect(calibrationWindow, &JFJochCalibrationWindow::loadCalibration,
             reading_worker, &JFJochImageReadingWorker::LoadCalibration);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             calibrationWindow, &JFJochCalibrationWindow::datasetLoaded);
     connect(reading_worker, &JFJochImageReadingWorker::simpleImageLoaded,
             calibrationWindow, &JFJochCalibrationWindow::calibrationLoaded);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             reciprocalWindow, &JFJochViewerReciprocalSpaceWindow::datasetLoaded);
     connect(this, &JFJochViewerWindow::imageReady,
             reciprocalWindow, &JFJochViewerReciprocalSpaceWindow::imageLoaded);
@@ -395,9 +403,8 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
     // Ensure worker is deleted in its own thread when the thread stops
     connect(reading_thread, &QThread::finished, reading_worker, &QObject::deleteLater);
 
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded, this,
-            [this](std::shared_ptr<const JFJochReaderDataset> ds) { lastDataset = std::move(ds); });
-    // lastImage is captured in OnImageReady, which sees every frame before fanning it out.
+    // lastDataset and lastImage are captured in OnDatasetReady / OnImageReady, which see every
+    // dataset / frame before fanning them out.
 
         connect(this, &JFJochViewerWindow::imageReady,
             toolBarDisplay, &JFJochViewerToolbarDisplay::imageLoaded);
@@ -488,7 +495,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
             reading_worker, &JFJochImageReadingWorker::UpdateDataset);
     connect(settingsPanel, &JFJochViewerSettingsDock::findBeamCenter,
             reading_worker, &JFJochImageReadingWorker::FindCenter);
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             settingsPanel, &JFJochViewerSettingsDock::datasetLoaded);
     connect(this, &JFJochViewerWindow::imageReady,
             settingsPanel, &JFJochViewerSettingsDock::loadImage);
@@ -534,7 +541,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString
         resizeDocks({lastDatasetInfoDock, imageStripDock}, {260, 140}, Qt::Vertical);
     }
     menuBar->AddDockEntry(imageStripDock, "Image strip");
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             imageStrip, &JFJochViewerImageStrip::datasetLoaded);
     connect(reading_worker, &JFJochImageReadingWorker::fileOpened,
             imageStrip, &JFJochViewerImageStrip::resetForNewFile);
@@ -616,6 +623,12 @@ void JFJochViewerWindow::OnImageReady(std::shared_ptr<const JFJochReaderImage> i
     emit imageConsumed();       // ack the worker so it may fetch/produce the next frame
 }
 
+void JFJochViewerWindow::OnDatasetReady(std::shared_ptr<const JFJochReaderDataset> dataset) {
+    lastDataset = dataset;      // newest dataset, for dataset-info docks opened later
+    emit datasetReady(dataset); // fan out to every GUI consumer synchronously (same thread)
+    emit datasetConsumed();     // ack the worker so it may produce the next dataset refresh
+}
+
 void JFJochViewerWindow::NewDatasetInfo() {
     auto info = new JFJochViewerDatasetInfo(this);
     info->datasetLoaded(lastDataset);
@@ -637,7 +650,7 @@ void JFJochViewerWindow::NewDatasetInfo() {
     lastDatasetInfoDock = dock;
 
     // Wire signals like the initial dataset_info
-    connect(reading_worker, &JFJochImageReadingWorker::datasetLoaded,
+    connect(this, &JFJochViewerWindow::datasetReady,
             info, &JFJochViewerDatasetInfo::datasetLoaded);
     connect(this, &JFJochViewerWindow::imageReady,
             info, &JFJochViewerDatasetInfo::imageLoaded);

viewer/JFJochViewerWindow.h

@@ -61,6 +61,12 @@ private slots:
     // (synchronously, same thread) via imageReady, then acks the worker so it may produce the next.
     void OnImageReady(std::shared_ptr<const JFJochReaderImage> image);
 
+    // Single cross-thread sink for the worker's datasetLoaded, mirroring OnImageReady: fans out to
+    // every GUI consumer synchronously via datasetReady, then acks the worker (datasetConsumed) so the
+    // worker can cap how many datasets are in flight - without this the queued datasetLoaded backlog
+    // (each event pinning a full dataset) grew without bound in live follow (100 GB OOM).
+    void OnDatasetReady(std::shared_ptr<const JFJochReaderDataset> dataset);
+
 public slots:
     void LoadFile(const QString &filename, qint64 image_number, qint64 summation, bool retry);
     void LoadImage(qint64 image_number, qint64 summation);
@@ -74,6 +80,10 @@ signals:
     void imageReady(std::shared_ptr<const JFJochReaderImage> image);
     void imageConsumed();   // ack to the worker that one frame has been consumed (backpressure)
 
+    // Same GUI-thread fan-out + ack for the dataset stream (see OnDatasetReady).
+    void datasetReady(std::shared_ptr<const JFJochReaderDataset> dataset);
+    void datasetConsumed();   // ack to the worker that one dataset has been consumed (backpressure)
+
     void LoadFileRequest(const QString &filename, qint64 image_number, qint64 summation, bool retry);
     void LoadImageRequest(int64_t image_number, int64_t summation);
     void adjustForegroundButton(bool input);

viewer/widgets/JFJochViewerImageStatistics.cpp

@@ -307,7 +307,7 @@ void JFJochViewerImageStatistics::loadImage(std::shared_ptr<const JFJochReaderIm
     if (total_pixels == 0)
         total_pixels = 1;
 
-    auto mask_stats = image->Dataset().pixel_mask.GetStatistics();
+    auto mask_stats = image->Dataset().pixel_mask->GetStatistics();
     text = QString("<b>%1</b>").arg(mask_stats.total_masked);
     masked_pixels->setText(text);
     masked_pixels->setToolTip(

viewer/windows/JFJochCalibrationWindow.cpp

@@ -96,7 +96,7 @@ void JFJochCalibrationWindow::calibrationLoaded(std::shared_ptr<const SimpleImag
 void JFJochCalibrationWindow::LoadMask() {
     if (dataset) {
         auto mask = std::make_shared<SimpleImage>();
-        auto tmp =  dataset->pixel_mask.GetMask();
+        auto tmp =  dataset->pixel_mask->GetMask();
         mask->buffer.resize(tmp.size() * sizeof(uint32_t));
         memcpy(mask->buffer.data(), tmp.data(), tmp.size() * sizeof(uint32_t));
         mask->image = CompressedImage(mask->buffer.data(), mask->buffer.size(),