mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 66 Wiki Activity

viewer: fix HTTP live-follow OOM via datasetLoaded backpressure + shared pixel mask

Browse Source 
The viewer could grow to ~100 GB RAM when live-following an HTTP broker. The
rc.153 images_in_flight backpressure only throttled imageLoaded; the heavy
per-frame payload rides datasetLoaded, fanned out over ~10 queued cross-thread
connections with no cap. In HTTPSyncDataset follow mode (entered when an operator
clicks an image while following live) RefreshDatasetOnly_i emits a fresh full
dataset every autoload tick with no imageLoaded, so the gate never engaged and
the queued events - each pinning a full JFJochReaderDataset (full-detector
PixelMask + per-image plots) - accumulated without bound.

Backpressure datasetLoaded the same way as imageLoaded: a datasets_in_flight
counter (cap 2), all emits routed through EmitDatasetLoaded_i, and
AutoLoadTimerExpired gated on it (covers HTTPSyncDataset). The window routes the
worker's datasetLoaded through a single OnDatasetReady sink that fans out
synchronously via datasetReady and acks with datasetConsumed. Under load stale
datasets are dropped; the next tick sends the latest.

Share the pixel mask instead of deep-copying it: JFJochReaderDataset::pixel_mask
is now shared_ptr<const PixelMask>, so per-frame dataset copies share the ~72 MB
mask. UpdateUserMask does copy-on-write; JFJochHttpReader caches the mask by
arm_date so a live refresh reuses one shared mask per acquisition.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
leonarski_f
2026-07-09 10:58:10 +02:00
co-authored by Claude Fable 5
parent b4a29ebb32
commit dc8a80f3e0
16 changed files with 120 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
viewer/JFJochImageReadingWorker.cpp
+33 -16
View File
@@ -293,7 +293,7 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
UpdateAzint_i(dataset.get());
}
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
EmitReferenceInfo_i(); // refresh the reference's consistency check against the new dataset
// Reset the run collection for the freshly opened file (file mode only has snapshots).
@@ -313,7 +313,7 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
emit fileLoadError("File Load Error", QString::fromStdString(e.what()));
ResetFileOpenRetry_i();
emit datasetLoaded({});
EmitDatasetLoaded_i({});
EmitImageLoaded_i({});
}
}
@@ -345,7 +345,7 @@ void JFJochImageReadingWorker::CloseFile() {
total_images = 0;
current_file = "";
EmitImageLoaded_i({});
emit datasetLoaded({});
EmitDatasetLoaded_i({});
}
void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation) {
@@ -363,9 +363,9 @@ void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation
void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) {
if (dataset) {
azint_mapping = std::make_unique<AzimuthalIntegrationMapping>(curr_experiment, dataset->pixel_mask);
azint_mapping = std::make_unique<AzimuthalIntegrationMapping>(curr_experiment, *dataset->pixel_mask);
index_and_refine = std::make_unique<IndexAndRefine>(curr_experiment, indexing.get());
image_analysis = std::make_unique<MXAnalysisWithoutFPGA>(curr_experiment, *azint_mapping, dataset->pixel_mask,
image_analysis = std::make_unique<MXAnalysisWithoutFPGA>(curr_experiment, *azint_mapping, *dataset->pixel_mask,
*index_and_refine.get());
last_profile_.reset();
@@ -391,7 +391,7 @@ void JFJochImageReadingWorker::LoadImage_i(int64_t image_number, int64_t summati
}
current_image_ptr = tmp_image_ptr;
total_images = http_reader.GetNumberOfImages();
emit datasetLoaded(http_reader.GetDataset());
EmitDatasetLoaded_i(http_reader.GetDataset());
} else {
if (image_number < 0 || image_number + summation > total_images)
return;
@@ -483,7 +483,7 @@ void JFJochImageReadingWorker::UpdateDataset_i(const std::optional<DiffractionEx
curr_experiment.ImportScalingSettings(scaling_settings);
UpdateAzint_i(dataset.get());
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);
ApplyROIOverrideToImage_i();
@@ -701,7 +701,7 @@ void JFJochImageReadingWorker::MaskFromSelectedROI(QString name, bool add) {
if (!elem)
return;
auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
auto geom = exp.GetDiffractionGeometry();
const int64_t width = exp.GetXPixelsNum();
const int64_t height = exp.GetYPixelsNum();
@@ -765,7 +765,7 @@ void JFJochImageReadingWorker::SetROIDefinition_i(const ROIDefinition &rois) {
}
}
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
EmitImageLoaded_i(current_image_ptr);
}
@@ -795,7 +795,7 @@ void JFJochImageReadingWorker::UpdateUserMask_i(const std::vector<uint32_t> &mas
UpdateAzint_i(dataset.get());
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);
@@ -817,7 +817,7 @@ void JFJochImageReadingWorker::SaveUserMaskTIFF(QString filename) {
QMutexLocker locker(&m);
if (!current_image_ptr)
return;
auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
CompressedImage mask_image(user_mask, current_image_ptr->Dataset().experiment.GetXPixelsNum(), current_image_ptr->Dataset().experiment.GetYPixelsNum());
WriteTIFFToFile(filename.toStdString(), mask_image);
}
@@ -848,7 +848,7 @@ void JFJochImageReadingWorker::LoadUserMaskTIFF(QString filename, bool replace)
// replace: start from an empty mask; add: keep the current one. A non-zero TIFF pixel masks.
auto user_mask = replace ? std::vector<uint32_t>(width * height, 0)
: current_image_ptr->Dataset().pixel_mask.GetUserMask();
: current_image_ptr->Dataset().pixel_mask->GetUserMask();
for (size_t i = 0; i < tiff.size(); i++)
if (tiff[i] != 0)
user_mask[i] = 1;
@@ -861,7 +861,7 @@ void JFJochImageReadingWorker::UploadUserMask() {
if (!current_image_ptr)
return;
auto user_mask = current_image_ptr->Dataset().pixel_mask.GetUserMask();
auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
if (http_mode)
http_reader.UploadUserMask(user_mask);
}
@@ -891,6 +891,10 @@ void JFJochImageReadingWorker::AutoLoadTimerExpired() {
// skipped, which for a live monitor is exactly right (the next fetch grabs the newest image).
if (images_in_flight >= max_images_in_flight)
return;
// Same gate for datasets: covers HTTPSyncDataset (which emits no image), so its per-tick
// datasetLoaded flood is throttled to the GUI's dataset drain rate instead of piling up.
if (datasets_in_flight >= max_datasets_in_flight)
return;
switch (autoload_mode) {
case AutoloadMode::HTTPSync:
@@ -925,6 +929,19 @@ void JFJochImageReadingWorker::ImageConsumed() {
--images_in_flight;
}
void JFJochImageReadingWorker::EmitDatasetLoaded_i(const std::shared_ptr<const JFJochReaderDataset>& dataset) {
// Assumes m locked! Dataset-stream analogue of EmitImageLoaded_i. Each queued datasetLoaded event
// pins a whole dataset; DatasetConsumed() decrements once the GUI has fanned it out and rendered
// it, and AutoLoadTimerExpired refuses to produce another live refresh while the cap is reached.
++datasets_in_flight;
emit datasetLoaded(dataset);
}
void JFJochImageReadingWorker::DatasetConsumed() {
QMutexLocker locker(&m);
--datasets_in_flight;
}
void JFJochImageReadingWorker::SetHttpConnected_i(bool connected, const QString &addr) {
// Assumes m locked! Only signals on a real change so the status bar does not flicker.
if (connected == http_connected)
@@ -948,7 +965,7 @@ void JFJochImageReadingWorker::RefreshDatasetOnly_i() {
emit imageNumberChanged(total_images, current_image.value());
}
if (dataset)
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
} catch (std::exception &e) {
logger.Debug("Dataset refresh failed: {}", e.what());
SetHttpConnected_i(false, current_file);
@@ -1022,7 +1039,7 @@ ReprocessingInputs JFJochImageReadingWorker::GetReprocessingInputs() const {
return in;
in.file = current_file;
in.experiment = curr_experiment;
in.pixel_mask = current_image_ptr->Dataset().pixel_mask;
in.pixel_mask = *current_image_ptr->Dataset().pixel_mask;
in.spot_finding = spot_finding_settings;
if (reference_)
in.reference_data = reference_->reflections;
@@ -1041,7 +1058,7 @@ void JFJochImageReadingWorker::ActivateSnapshot_i(const QString &name) {
curr_experiment.ImportIndexingSettings(indexing_settings);
curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings);
UpdateAzint_i(dataset.get());
emit datasetLoaded(dataset);
EmitDatasetLoaded_i(dataset);
QStringList names;
for (const auto &n: file_reader.SnapshotNames())