Jungfraujoch/viewer/JFJochImageReadingWorker.cpp

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#include <cerrno>
#include <cstring>
#include <fstream>
#include <iterator>
#include "../common/JFJochMath.h"
#ifndef _WIN32
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#endif

#include "JFJochImageReadingWorker.h"
#include "../reader/JFJochReaderImage.h"   // JFJochReaderImage + GAP/ERROR/SATURATED sentinels
#include "../image_analysis/geom_refinement/AssignSpotsToRings.h"
#include "../image_analysis/spot_finding/StrongPixelSet.h"
#include "../image_analysis/spot_finding/SpotUtils.h"
#include "../image_analysis/spot_finding/ImageSpotFinder.h"
#include <QVector>
#include <QMutexLocker>
#include <QFileInfo>
#include <QPainter>
#include <QFontMetrics>
#include <algorithm>

#include "../preview/JFJochTIFF.h"

namespace {
    enum class PreflightResult {
        Ok,
        NotYetVisible,      // ENOENT, ESTALE, etc.
        PermissionDenied,   // EACCES, EPERM
        IsDirectory,
        OtherError
    };

    PreflightResult preflight_open_ro(const QString& filename, std::string& reason_out) {
        reason_out.clear();

#ifndef _WIN32
        const QByteArray path = filename.toLocal8Bit();
        errno = 0;
        const int fd = ::open(path.constData(), O_RDONLY | O_CLOEXEC);
        if (fd >= 0) {
            struct stat st;
            if (fstat(fd, &st) == 0 && S_ISDIR(st.st_mode)) {
                ::close(fd);
                reason_out = "Path is a directory";
                return PreflightResult::IsDirectory;
            }
            ::close(fd);
            return PreflightResult::Ok;
        }

        const int e = errno;
        reason_out = fmt::format("{} (errno={} {})", std::strerror(e), e, std::strerror(e));

        // NFS can transiently report missing/stale entries.
        if (e == ENOENT || e == ESTALE || e == EIO || e == ETIMEDOUT || e == ENOTCONN)
            return PreflightResult::NotYetVisible;

        if (e == EACCES || e == EPERM)
            return PreflightResult::PermissionDenied;

        if (e == EISDIR)
            return PreflightResult::IsDirectory;

        return PreflightResult::OtherError;
#else
        // No NFS-style transient-errno semantics off POSIX; use a portable check.
        const QFileInfo info(filename);
        if (!info.exists()) {
            reason_out = "File not visible yet";
            return PreflightResult::NotYetVisible;
        }
        if (info.isDir()) {
            reason_out = "Path is a directory";
            return PreflightResult::IsDirectory;
        }
        if (!info.isReadable()) {
            reason_out = "Permission denied";
            return PreflightResult::PermissionDenied;
        }
        return PreflightResult::Ok;
#endif
    }
}

JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &settings,
                                                   const DiffractionExperiment &experiment, QObject *parent)
    : QObject(parent),
      indexing_settings(experiment.GetIndexingSettings()),
      azint_settings(experiment.GetAzimuthalIntegrationSettings()),
      bragg_settings(experiment.GetBraggIntegrationSettings()),
      scaling_settings(experiment.GetScalingSettings()) {
    qRegisterMetaType<QVector<QRect>>("QVector<QRect>");
    qRegisterMetaType<BrokerStatus>("BrokerStatus");
    qRegisterMetaType<ROIDefinition>("ROIDefinition");
    qRegisterMetaType<BraggIntegrationSettings>("BraggIntegrationSettings");
    qRegisterMetaType<ScalingSettings>("ScalingSettings");
    qRegisterMetaType<RunData>("RunData");
    qRegisterMetaType<QVector<RunData>>("QVector<RunData>");
    qRegisterMetaType<QVector<qint64>>("QVector<qint64>");
    qRegisterMetaType<ReferenceMtzInfo>("ReferenceMtzInfo");
    spot_finding_settings = settings;

    indexing = std::make_unique<IndexerThreadPool>(indexing_settings);
    http_reader.Experiment(experiment);
    file_reader.Experiment(experiment);
    thumb_color_scale_.Select(ColorScaleEnum::Indigo);

    autoload_timer = new QTimer(this);
    autoload_timer->setInterval(autoload_interval);
    connect(autoload_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::AutoLoadTimerExpired);

    status_timer = new QTimer(this);
    status_timer->setInterval(status_interval);
    connect(status_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::StatusTimerExpired);

    file_open_retry_timer = new QTimer(this);
    file_open_retry_timer->setSingleShot(true);
    connect(file_open_retry_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::FileOpenRetryTimerExpired);
}


void JFJochImageReadingWorker::ResetFileOpenRetry_i() {
    // Assumes m locked!
    if (file_open_retry_timer)
        file_open_retry_timer->stop();

    // Signal UI to close the dialog if we were active
    if (file_open_retry_active)
        emit fileLoadRetryStatus(false, "");

    file_open_retry_active = false;
    file_open_retry_warned = false;
    file_open_retry_attempts = 0;
    file_open_retry_delay_ms = 50;
    file_open_retry_elapsed.invalidate();
    pending_load = {};
}

void JFJochImageReadingWorker::ScheduleFileOpenRetry_i(const QString& reason) {
    // Assumes m locked!
    if (!file_open_retry_active) {
        file_open_retry_active = true;
        file_open_retry_warned = false;
        file_open_retry_attempts = 0;
        file_open_retry_delay_ms = 50;
        file_open_retry_elapsed.restart();
    }

    if (!file_open_retry_warned) {
        file_open_retry_warned = true;
        logger.Warning(fmt::format(
            "File '{}' not available yet (GPFS/NFS). Retrying with back-off up to 10 s. Reason: {}",
            pending_load.filename.toStdString(), reason.toStdString()));

        // Signal UI to show the dialog
        emit fileLoadRetryStatus(true, fmt::format("Waiting for file {} to appear on disk...", pending_load.filename.toStdString()).c_str());
    } else {
        logger.Debug(fmt::format(
            "Retry pending for file '{}'. Reason: {}",
            pending_load.filename.toStdString(), reason.toStdString()));
    }

    if (file_open_retry_elapsed.isValid() && file_open_retry_elapsed.elapsed() >= 10'000) {
        std::string msg = fmt::format(
            "Timed out waiting for file '{}' after 10 s ({} attempt(s))",
            pending_load.filename.toStdString(), file_open_retry_attempts);

        logger.Error(msg);

        // Reset first (closes the progress dialog)
        ResetFileOpenRetry_i();

        // Then show the error dialog
        emit fileLoadError("File Open Timeout", QString::fromStdString(msg));
        return;
    }

    const int delay = file_open_retry_delay_ms;
    file_open_retry_delay_ms = std::min(file_open_retry_delay_ms * 2, file_open_retry_delay_max_ms);

    if (file_open_retry_timer)
        file_open_retry_timer->start(delay);
}

void JFJochImageReadingWorker::FileOpenRetryTimerExpired() {
    PendingLoadRequest req;

    QMutexLocker ul(&m);

    if (!file_open_retry_active)
        return;

    req = pending_load;

    // Re-trigger LoadFile, but keep retry=true so we stay in the retry loop.
    LoadFile_i(req.filename, req.image_number, req.summation, true);
}

void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_number, qint64 summation, bool retry) {
       try {
        std::shared_ptr<const JFJochReaderDataset> dataset;
        auto start = std::chrono::high_resolution_clock::now();

        if (filename.startsWith("http://")) {
            http_mode = true;
            http_reader.ReadURL(filename.toStdString());
            total_images = http_reader.GetNumberOfImages();
            dataset = http_reader.GetDataset();
            SetHttpConnected_i(true, filename);
            status_timer->start();
            if (image_number < 0)
                setAutoLoadMode_i(AutoloadMode::HTTPSync);
            else
                setAutoLoadMode_i(AutoloadMode::None);

        } else {
            http_mode = false;
            status_timer->stop();
            SetHttpConnected_i(false, "");

            if (retry) {
                pending_load.filename = filename;
                pending_load.image_number = image_number;
                pending_load.summation = summation;

                std::string reason;
                const PreflightResult pr = preflight_open_ro(filename, reason);

                switch (pr) {
                    case PreflightResult::Ok:
                        break;
                    case PreflightResult::NotYetVisible:
                        ScheduleFileOpenRetry_i(QString::fromStdString(reason));
                        return; // IMPORTANT: do not try to open the file yet
                    case PreflightResult::PermissionDenied:
                        logger.Error(fmt::format(
                            "Permission denied opening '{}' (read-only preflight failed: {}). Not retrying.",
                            filename.toStdString(), reason));
                        emit fileLoadError("Permission Denied", QString::fromStdString(reason));
                        ResetFileOpenRetry_i();
                        return;
                    case PreflightResult::IsDirectory:
                        logger.Error(fmt::format(
                            "Error opening '{}': {}. Not retrying.",
                            filename.toStdString(), reason));
                        emit fileLoadError("Cannot open directory", QString::fromStdString(reason));
                        ResetFileOpenRetry_i();
                        return;
                    case PreflightResult::OtherError:
                        logger.Error(fmt::format(
                            "Other error '{}' (read-only preflight failed: {}). Not retrying.",
                            filename.toStdString(), reason));
                        emit fileLoadError("File Open Error", QString::fromStdString(reason));
                        ResetFileOpenRetry_i();
                        return;
                }

                // At this point we will attempt the real open.
                file_open_retry_attempts++;
            }

            file_reader.ReadFile(filename.toStdString());
            total_images = file_reader.GetNumberOfImages();
            dataset = file_reader.GetDataset();
            setAutoLoadMode_i(AutoloadMode::None);

            if (retry && file_open_retry_active) {
                logger.Info(fmt::format(
                    "File '{}' opened after {} attempt(s), waited {} ms",
                    filename.toStdString(),
                    file_open_retry_attempts,
                    file_open_retry_elapsed.isValid() ? file_open_retry_elapsed.elapsed() : 0));
            }
            ResetFileOpenRetry_i();
        }

        current_image.reset();
        current_summation = 1;
        current_file = filename;
        if (dataset) {
            curr_experiment = dataset->experiment;
            roi_override_.reset();   // new file: use its ROIs, forget any earlier edits
            curr_experiment.ImportIndexingSettings(indexing_settings);
            curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings);
            curr_experiment.ImportBraggIntegrationSettings(bragg_settings);
            curr_experiment.ImportScalingSettings(scaling_settings);
            UpdateAzint_i(dataset.get());
        }

        EmitDatasetLoaded_i(dataset);
        EmitReferenceInfo_i();   // refresh the reference's consistency check against the new dataset

        // Reset the run collection for the freshly opened file (file mode only has snapshots).
        run_labels_.clear();
        if (!http_mode)
            run_labels_["Original"] = "Original";
        EmitRuns_i();
        emit fileOpened();   // listeners (e.g. the jobs table) reset their per-file state

        auto end = std::chrono::high_resolution_clock::now();
        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
        logger.Info(fmt::format("Loaded file {} in {} ms", filename.toStdString(), duration));

        LoadImage_i(image_number, summation);
    } catch (std::exception &e) {
        logger.Error(fmt::format("Error loading file {} {}", filename.toStdString(), e.what()));
        emit fileLoadError("File Load Error", QString::fromStdString(e.what()));
        ResetFileOpenRetry_i();

        EmitDatasetLoaded_i({});
        EmitImageLoaded_i({});
    }
}

void JFJochImageReadingWorker::LoadFile(const QString &filename, qint64 image_number, qint64 summation, bool retry) {
    QMutexLocker ul(&m);

    ResetFileOpenRetry_i();
    LoadFile_i(filename, image_number, summation, retry);
}

void JFJochImageReadingWorker::CloseFile() {
    QMutexLocker ul(&m);

    ResetFileOpenRetry_i();

    if (http_mode)
        http_reader.Close();
    else
        file_reader.Close();

    status_timer->stop();
    setAutoLoadMode_i(AutoloadMode::None);
    SetHttpConnected_i(false, "");

    current_image_ptr.reset();
    current_image.reset();
    current_summation = 1;
    total_images = 0;
    current_file = "";
    EmitImageLoaded_i({});
    EmitDatasetLoaded_i({});
}

void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation) {
    QMutexLocker ul(&m);
    // Manually selecting an image while following live drops to dataset-only follow:
    // the chosen image stays put while plots and image count keep updating.
    if (autoload_mode == AutoloadMode::HTTPSync || autoload_mode == AutoloadMode::HTTPSyncDataset)
        setAutoLoadMode_i(AutoloadMode::HTTPSyncDataset);
    else
        setAutoLoadMode_i(AutoloadMode::None);
    if ((image_number == current_image) && (current_summation == summation))
        return;
    LoadImage_i(image_number, summation);
}

void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) {
    if (dataset) {
        azint_mapping = std::make_unique<AzimuthalIntegrationMapping>(curr_experiment, *dataset->pixel_mask);
        // Interactive/live analysis only needs the current image's result (returned per image); it
        // never scales the accumulated run, so don't retain the whole-run integration_outcome vector.
        index_and_refine = std::make_unique<IndexAndRefine>(curr_experiment, indexing.get(), /*retain_outcomes=*/false);
        image_analysis = std::make_unique<MXAnalysisWithoutFPGA>(curr_experiment, *azint_mapping, *dataset->pixel_mask,
            *index_and_refine.get());

        last_profile_.reset();
    }
}

void JFJochImageReadingWorker::LoadImage_i(int64_t image_number, int64_t summation) {
    // Assumes m locked!
    try {
        if (summation <= 0)
            return;

        std::vector<int32_t> image;
        auto start = std::chrono::high_resolution_clock::now();

        if (http_mode) {
            if (image_number < 0 && summation != 1)
                return;
            const auto tmp_image_ptr = http_reader.LoadImage(image_number, summation);
            if (image_number < 0 && tmp_image_ptr == nullptr) {
                logger.Debug("No change in online buffer, not updating viewer");
                return; // Do nothing, since there is no update in the file
            }
            current_image_ptr = tmp_image_ptr;
            total_images = http_reader.GetNumberOfImages();
            EmitDatasetLoaded_i(http_reader.GetDataset());
        } else {
            if (image_number < 0 || image_number + summation > total_images)
                return;
            current_image_ptr = file_reader.LoadImage(image_number, summation);
        }

        if (!current_image_ptr) {
            EmitImageLoaded_i({});
            return;
        }

        ApplyROIOverrideToImage_i();   // edited ROIs win over the file's for every image

        current_image = current_image_ptr->ImageData().number;
        current_summation = summation;

        auto end = std::chrono::high_resolution_clock::now();

        if (auto_reanalyze) {
            ReanalyzeImage_i();
        } else if (image_analysis && !curr_experiment.ROI().empty()) {
            // ROIs are wanted even when a full re-analysis is not; compute only those.
            try {
                image_analysis->AnalyzeROIOnly(current_image_ptr->ImageData());
            } catch (const std::exception &e) {
                logger.Error("ROI-only analysis failed: {}", e.what());
            }
        }

        auto end_analysis = std::chrono::high_resolution_clock::now();

        auto duration_1 = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
        auto duration_2 = std::chrono::duration_cast<std::chrono::milliseconds>(end_analysis - end).count();

        // Adapt autoload interval from moving average of (load + analysis) time
        if (autoload_mode != AutoloadMode::None) {
            const int total_ms = static_cast<int>(duration_1 + duration_2);
            autoload_ms_ma.Add(static_cast<float>(total_ms));

            const auto avg_ms = autoload_ms_ma.Read();
            if (avg_ms.has_value()) {
                int proposed_ms = static_cast<int>(avg_ms.value() * autoload_safety_factor);
                proposed_ms = std::clamp(proposed_ms, autoload_interval_min_ms, autoload_interval_max_ms);

                if (proposed_ms != autoload_interval) {
                    autoload_interval = proposed_ms;
                    autoload_timer->setInterval(autoload_interval);
                }
            }

            if (autoload_mode == AutoloadMode::HTTPSync)
                emit liveRateChanged(autoload_interval > 0 ? 1000.0 / autoload_interval : 0.0);
        }

        logger.Info("Loaded image {} in {}/{} ms Autoload timer set to {} ms", image_number, duration_1, duration_2, autoload_interval);
        emit imageNumberChanged(total_images, current_image.value());
        EmitImageLoaded_i(current_image_ptr);
    } catch (std::exception &e) {
        logger.Error("Error loading image {}: {}", image_number, e.what());
    }
}

void JFJochImageReadingWorker::UpdateDataset_i(const std::optional<DiffractionExperiment> &experiment) {
    if (!current_image_ptr)
        return;

    std::shared_ptr<const JFJochReaderDataset> dataset;

    if (http_mode) {
        if (experiment)
            http_reader.UpdateGeomMetadata(experiment.value());
        dataset = http_reader.GetDataset();
    } else {
        if (experiment)
            file_reader.UpdateGeomMetadata(experiment.value());
        dataset = file_reader.GetDataset();
    }
    if (!dataset) {
        logger.Error("UpdateDataset_i: dataset is null (http_mode={}) - skipping update to avoid crash", http_mode);
        return;
    }

    curr_experiment = dataset->experiment;
    if (roi_override_)
        curr_experiment.ROI().SetROI(*roi_override_);   // keep edited ROIs across settings changes
    curr_experiment.ImportIndexingSettings(indexing_settings);
    curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings);
    curr_experiment.ImportBraggIntegrationSettings(bragg_settings);
    curr_experiment.ImportScalingSettings(scaling_settings);
    UpdateAzint_i(dataset.get());

    EmitDatasetLoaded_i(dataset);

    current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);
    ApplyROIOverrideToImage_i();

    if (auto_reanalyze)
        ReanalyzeImage_i();
    EmitImageLoaded_i(current_image_ptr);
}

void JFJochImageReadingWorker::UpdateDataset(const DiffractionExperiment &experiment) {
    QMutexLocker ul(&m);
    UpdateDataset_i(experiment);
}

void JFJochImageReadingWorker::ReanalyzeImage_i() {
    if (!current_image_ptr || !azint_mapping || !image_analysis)
        return;

    auto start_time = std::chrono::high_resolution_clock::now();
    auto new_image = std::make_shared<JFJochReaderImage>(*current_image_ptr);
    auto new_image_dataset = new_image->CreateMutableDataset();

    new_image_dataset->experiment.ImportIndexingSettings(indexing_settings);
    new_image_dataset->experiment.ImportAzimuthalIntegrationSettings(azint_settings);
    new_image_dataset->az_int_bin_to_phi = azint_mapping->GetBinToPhi();
    new_image_dataset->az_int_bin_to_q = azint_mapping->GetBinToQ();
    new_image_dataset->azimuthal_bins = azint_mapping->GetAzimuthalBinCount();
    new_image_dataset->q_bins = azint_mapping->GetQBinCount();

    // Azimuthal profile for the analysis/display pipeline. AzimuthalIntegrationProfile
    // holds a mutex (non-copyable), so keep it via unique_ptr re-created each analysis.
    last_profile_ = std::make_unique<AzimuthalIntegrationProfile>(*azint_mapping);
    image_analysis->Analyze(new_image->ImageData(), *last_profile_, spot_finding_settings);

    current_image_ptr = new_image;
    auto end_time = std::chrono::high_resolution_clock::now();
    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
    logger.Info("Analysis of image in {} ms", duration.count());
}

void JFJochImageReadingWorker::Analyze() {
    QMutexLocker locker(&m);
    ReanalyzeImage_i();
    EmitImageLoaded_i(current_image_ptr);
}

void JFJochImageReadingWorker::FindCenter(const UnitCell& calibrant, bool guess) {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;
    logger.Info("Finding center");

    DiffractionGeometry geom = current_image_ptr->Dataset().experiment.GetDiffractionGeometry();
    try {
        if (guess)
            GuessGeometry(geom, current_image_ptr->ImageData().spots, calibrant);
        else
            OptimizeGeometry(geom, current_image_ptr->ImageData().spots, calibrant);
    } catch (const JFJochException &e) {
        logger.ErrorException(e);
        return;
    }

    logger.Info("Geometry found X: {} pxl Y: {} pxl Dist: {} mm", geom.GetBeamX_pxl(), geom.GetBeamY_pxl(),
                geom.GetDetectorDistance_mm());

    DiffractionExperiment new_experiment = current_image_ptr->Dataset().experiment;
    new_experiment.BeamX_pxl(geom.GetBeamX_pxl()).BeamY_pxl(geom.GetBeamY_pxl())
            .DetectorDistance_mm(geom.GetDetectorDistance_mm())
            .PoniRot1_rad(geom.GetPoniRot1_rad())
            .PoniRot2_rad(geom.GetPoniRot2_rad())
            .PoniRot3_rad(geom.GetPoniRot3_rad());
    UpdateDataset_i(new_experiment);

    std::vector<float> ring_Q = CalculateXtalRings(calibrant);

    QVector<float> rings;
    for (int i = 0; i < 15 && i < ring_Q.size(); i++) {
        rings.push_back(2 * PI / ring_Q[i]);
    }
    emit setRings(rings);
}

void JFJochImageReadingWorker::UpdateSpotFindingSettings(const SpotFindingSettings &settings,
                                                         const IndexingSettings &indexing,
                                                         int64_t max_spots) {
    QMutexLocker locker(&m);
    spot_finding_settings = settings;

    indexing_settings.Algorithm(indexing.GetAlgorithm());
    indexing_settings.GeomRefinementAlgorithm(indexing.GetGeomRefinementAlgorithm());
    indexing_settings.RotationIndexing(indexing.GetRotationIndexing());   // "Process as stills" drives this
    indexing_settings.Tolerance(indexing.GetTolerance());
    indexing_settings.ViableCellMinSpots(indexing.GetViableCellMinSpots());
    indexing_settings.IndexIceRings(indexing.GetIndexIceRings());
    indexing_settings.UnitCellDistTolerance(indexing.GetUnitCellDistTolerance());
    curr_experiment.ImportIndexingSettings(indexing_settings);
    curr_experiment.MaxSpotCount(max_spots);

    if (auto_reanalyze) {
        ReanalyzeImage_i();
        EmitImageLoaded_i(current_image_ptr);
    }
}

void JFJochImageReadingWorker::ReanalyzeImages(bool input) {
    QMutexLocker locker(&m);
    auto_reanalyze = input;

    if (auto_reanalyze) {
        ReanalyzeImage_i();
        EmitImageLoaded_i(current_image_ptr);
    }
}


void JFJochImageReadingWorker::UpdateAzintSettings(const AzimuthalIntegrationSettings &settings) {
    QMutexLocker locker(&m);
    azint_settings = settings;
    UpdateDataset_i(std::nullopt);
}

void JFJochImageReadingWorker::UpdateBraggIntegrationSettings(BraggIntegrationSettings settings) {
    QMutexLocker locker(&m);
    bragg_settings = settings;
    UpdateDataset_i(std::nullopt);
}

void JFJochImageReadingWorker::UpdateScalingSettings(ScalingSettings settings) {
    QMutexLocker locker(&m);
    scaling_settings = settings;
    // Scaling only affects the processing-job post-pass (jobs read curr_experiment via
    // GetReprocessingInputs), not interactive single-image analysis - just store + import it.
    curr_experiment.ImportScalingSettings(scaling_settings);
}

void JFJochImageReadingWorker::SetReferenceMtz(QString path, QString column) {
    QMutexLocker locker(&m);
    try {
        reference_ = LoadReferenceMtz(
            path.toStdString(),
            column.isEmpty() ? std::nullopt : std::optional<std::string>(column.toStdString()));
        logger.Info(fmt::format("Loaded {} reference reflections from {} (column {})",
                                reference_->reflections.size(), path.toStdString(),
                                reference_->used_column));
    } catch (const std::exception &e) {
        reference_.reset();
        logger.Error(fmt::format("Error reading reference MTZ {}: {}", path.toStdString(), e.what()));
        ReferenceMtzInfo info;
        info.warning = QString("Load failed: ") + e.what();   // loaded stays false
        emit referenceMtzChanged(info);
        return;
    }
    EmitReferenceInfo_i();
}

void JFJochImageReadingWorker::ClearReferenceMtz() {
    QMutexLocker locker(&m);
    reference_.reset();
    EmitReferenceInfo_i();
}

void JFJochImageReadingWorker::EmitReferenceInfo_i() {
    // Assumes m locked. Builds the display summary from the stored reference and the current data,
    // so the consistency warning reflects whichever dataset is loaded now.
    ReferenceMtzInfo info;
    if (!reference_) {
        emit referenceMtzChanged(info);   // loaded = false: "No reference loaded"
        return;
    }

    info.loaded = true;
    for (const auto &c : reference_->candidate_columns)
        info.columns << QString::fromStdString(c.label);
    info.used_column = QString::fromStdString(reference_->used_column);

    QString summary = QString::number(static_cast<qulonglong>(reference_->reflections.size())) + " refl";
    if (reference_->d_max > 0.0)
        summary += QString::asprintf(", %.2f-%.2f A", reference_->d_max, reference_->d_min);
    if (!reference_->space_group_name.empty())
        summary += ", " + QString::fromStdString(reference_->space_group_name);
    if (reference_->cell)
        summary += QString::asprintf(", %.1f %.1f %.1f %.1f %.1f %.1f",
                                     reference_->cell->a, reference_->cell->b, reference_->cell->c,
                                     reference_->cell->alpha, reference_->cell->beta, reference_->cell->gamma);
    info.summary = summary;

    const auto data_sg = curr_experiment.GetSpaceGroupNumber().has_value()
        ? std::optional<int>(static_cast<int>(*curr_experiment.GetSpaceGroupNumber()))
        : std::nullopt;
    info.warning = QString::fromStdString(
        ReferenceConsistencyWarning(*reference_, curr_experiment.GetUnitCell(), data_sg));

    emit referenceMtzChanged(info);
}

void JFJochImageReadingWorker::SetROIDefinition(const ROIDefinition &rois) {
    QMutexLocker locker(&m);
    SetROIDefinition_i(rois);
}

void JFJochImageReadingWorker::MaskFromSelectedROI(QString name, bool add) {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;

    const auto &exp = current_image_ptr->Dataset().experiment;
    const auto rois = exp.ROI().GetROIDefinition();
    const std::string n = name.toStdString();

    const ROIElement *elem = nullptr;
    for (const auto &b : rois.boxes)     if (b.GetName() == n) { elem = &b; break; }
    if (!elem) for (const auto &c : rois.circles)   if (c.GetName() == n) { elem = &c; break; }
    if (!elem) for (const auto &a : rois.azimuthal) if (a.GetName() == n) { elem = &a; break; }
    if (!elem)
        return;

    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
    auto geom = exp.GetDiffractionGeometry();
    const int64_t width = exp.GetXPixelsNum();
    const int64_t height = exp.GetYPixelsNum();
    for (int64_t y = 0; y < height; y++) {
        for (int64_t x = 0; x < width; x++) {
            const float res = geom.PxlToRes(x, y);
            const float phi = geom.Phi_rad(x, y) * 180.0f / static_cast<float>(PI);
            if (elem->CheckROI(x, y, res, phi))
                user_mask[x + y * width] = add ? 1 : 0;
        }
    }
    UpdateUserMask_i(user_mask);
}

void JFJochImageReadingWorker::DownloadROIsFromServer() {
    QMutexLocker locker(&m);
    if (!http_mode)
        return;
    try {
        SetROIDefinition_i(http_reader.GetROIDefinitions());
    } catch (const std::exception &e) {
        logger.Error("Download ROIs from server failed: {}", e.what());
    }
}

void JFJochImageReadingWorker::UploadROIsToServer() {
    QMutexLocker locker(&m);
    if (!http_mode)
        return;
    try {
        http_reader.UploadROIDefinitions(curr_experiment.ROI().GetROIDefinition());
    } catch (const std::exception &e) {
        logger.Error("Upload ROIs to server failed: {}", e.what());
    }
}

void JFJochImageReadingWorker::SetROIDefinition_i(const ROIDefinition &rois) {
    // The worker experiment is the source of truth; the analysis ROI engine is rebuilt
    // for it so newly loaded images pick up the change automatically. From now on the
    // edited ROIs override whatever the file carried.
    roi_override_ = rois;
    curr_experiment.ROI().SetROI(rois);
    if (image_analysis)
        image_analysis->RebuildROI();

    if (!current_image_ptr)
        return;

    // Mutate the dataset too, so the canvas (which draws image->Dataset().experiment.ROI())
    // reflects the edit, then recompute only the ROIs for the current image.
    auto mutable_dataset = current_image_ptr->CreateMutableDataset();
    mutable_dataset->experiment.ROI().SetROI(rois);
    std::shared_ptr<const JFJochReaderDataset> dataset = mutable_dataset;
    current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);

    if (image_analysis) {
        try {
            image_analysis->RunROIOnly(current_image_ptr->ImageData());
        } catch (const std::exception &e) {
            logger.Error("ROI-only analysis failed: {}", e.what());
        }
    }

    EmitDatasetLoaded_i(dataset);
    EmitImageLoaded_i(current_image_ptr);
}

void JFJochImageReadingWorker::ApplyROIOverrideToImage_i() {
    if (!roi_override_ || !current_image_ptr)
        return;
    auto md = current_image_ptr->CreateMutableDataset();
    md->experiment.ROI().SetROI(*roi_override_);
    current_image_ptr = std::make_shared<JFJochReaderImage>(
        current_image_ptr->ImageData(), std::shared_ptr<const JFJochReaderDataset>(md));
}

void JFJochImageReadingWorker::UpdateUserMask_i(const std::vector<uint32_t> &mask) {
    std::shared_ptr<const JFJochReaderDataset> dataset;
    if (http_mode) {
        http_reader.UpdateUserMask(mask);
        dataset = http_reader.GetDataset();
    } else {
        file_reader.UpdateUserMask(mask);
        dataset = file_reader.GetDataset();
    }

    if (!dataset) {
        logger.Error("UpdateUserMask_i: dataset is null (http_mode={}) - skipping update to avoid crash", http_mode);
        return;
    }

    UpdateAzint_i(dataset.get());

    EmitDatasetLoaded_i(dataset);

    current_image_ptr = std::make_shared<JFJochReaderImage>(current_image_ptr->ImageData(), dataset);

    if (current_image.has_value())
        LoadImage_i(current_image.value(), current_summation);
}


void JFJochImageReadingWorker::ClearUserMask() {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;

    auto user_mask = std::vector<uint32_t>(current_image_ptr->Dataset().experiment.GetXPixelsNum() * current_image_ptr->Dataset().experiment.GetYPixelsNum(), 0);
    UpdateUserMask_i(user_mask);
}

void JFJochImageReadingWorker::SaveUserMaskTIFF(QString filename) {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;
    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
    CompressedImage mask_image(user_mask, current_image_ptr->Dataset().experiment.GetXPixelsNum(), current_image_ptr->Dataset().experiment.GetYPixelsNum());
    WriteTIFFToFile(filename.toStdString(), mask_image);
}

void JFJochImageReadingWorker::LoadUserMaskTIFF(QString filename, bool replace) {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;

    std::ifstream f(filename.toStdString(), std::ios::binary);
    const std::string content((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());

    uint32_t cols = 0, lines = 0;
    std::vector<uint16_t> tiff;
    try {
        tiff = ReadTIFFFromString16(content, cols, lines);
    } catch (const std::exception &e) {
        logger.Error("Could not read mask TIFF {}: {}", filename.toStdString(), e.what());
        return;
    }

    const int64_t width = current_image_ptr->Dataset().experiment.GetXPixelsNum();
    const int64_t height = current_image_ptr->Dataset().experiment.GetYPixelsNum();
    if (static_cast<int64_t>(cols) != width || static_cast<int64_t>(lines) != height) {
        logger.Error("Mask TIFF is {}x{} but the detector is {}x{}", cols, lines, width, height);
        return;
    }

    // replace: start from an empty mask; add: keep the current one. A non-zero TIFF pixel masks.
    auto user_mask = replace ? std::vector<uint32_t>(width * height, 0)
                             : current_image_ptr->Dataset().pixel_mask->GetUserMask();
    for (size_t i = 0; i < tiff.size(); i++)
        if (tiff[i] != 0)
            user_mask[i] = 1;

    UpdateUserMask_i(user_mask);
}

void JFJochImageReadingWorker::UploadUserMask() {
    QMutexLocker locker(&m);
    if (!current_image_ptr)
        return;

    auto user_mask = current_image_ptr->Dataset().pixel_mask->GetUserMask();
    if (http_mode)
        http_reader.UploadUserMask(user_mask);
}

void JFJochImageReadingWorker::LoadCalibration(QString dataset) {
    if (!http_mode) {
        auto tmp = std::make_shared<SimpleImage>();

        try {
            tmp->image = file_reader.ReadCalibration(tmp->buffer, dataset.toStdString());

            std::shared_ptr<const SimpleImage> ctmp = tmp;
            emit simpleImageLoaded(ctmp);
        } catch (const std::exception &e) {
            logger.Info("Error loading calibration: {}", e.what());
        }
    } else
        logger.Info("HTTP mode doesn't allow to read calibration (at the moment");
}

void JFJochImageReadingWorker::AutoLoadTimerExpired() {
    QMutexLocker locker(&m);

    // Backpressure (all auto modes): don't hand the GUI more frames than it has rendered - it acks
    // each one via ImageConsumed(). Without this a fast producer (live follow or movie playback)
    // outruns rendering and the queued imageLoaded events pile up unbounded. Intermediate frames are
    // skipped, which for a live monitor is exactly right (the next fetch grabs the newest image).
    if (images_in_flight >= max_images_in_flight)
        return;
    // Same gate for datasets: covers HTTPSyncDataset (which emits no image), so its per-tick
    // datasetLoaded flood is throttled to the GUI's dataset drain rate instead of piling up.
    if (datasets_in_flight >= max_datasets_in_flight)
        return;

    switch (autoload_mode) {
        case AutoloadMode::HTTPSync:
            if (http_mode)
                LoadImage_i(-1 , 1);
            break;
        case AutoloadMode::HTTPSyncDataset:
            if (http_mode)
                RefreshDatasetOnly_i();
            break;
        case AutoloadMode::Movie: {
            if (total_images == 0 || !current_image)
                return;
            int64_t new_image = (current_image.value() + jump_value) % total_images;
            LoadImage_i(new_image, current_summation);
            break;
        }
        case AutoloadMode::None:
            break;
    }
}

void JFJochImageReadingWorker::EmitImageLoaded_i(const std::shared_ptr<const JFJochReaderImage>& image) {
    // Assumes m locked! Count every frame handed to the GUI; ImageConsumed() decrements it once the
    // frame has been rendered. AutoLoadTimerExpired caps how many frames may be in flight at once.
    ++images_in_flight;
    emit imageLoaded(image);
}

void JFJochImageReadingWorker::ImageConsumed() {
    QMutexLocker locker(&m);
    --images_in_flight;
}

void JFJochImageReadingWorker::EmitDatasetLoaded_i(const std::shared_ptr<const JFJochReaderDataset>& dataset) {
    // Assumes m locked! Dataset-stream analogue of EmitImageLoaded_i. Each queued datasetLoaded event
    // pins a whole dataset; DatasetConsumed() decrements once the GUI has fanned it out and rendered
    // it, and AutoLoadTimerExpired refuses to produce another live refresh while the cap is reached.
    ++datasets_in_flight;
    emit datasetLoaded(dataset);
}

void JFJochImageReadingWorker::DatasetConsumed() {
    QMutexLocker locker(&m);
    --datasets_in_flight;
}

void JFJochImageReadingWorker::SetHttpConnected_i(bool connected, const QString &addr) {
    // Assumes m locked! Only signals on a real change so the status bar does not flicker.
    if (connected == http_connected)
        return;
    http_connected = connected;
    emit httpConnectionChanged(connected, addr);
}

void JFJochImageReadingWorker::RefreshDatasetOnly_i() {
    // Assumes m locked! Updates dataset/plots and image count without touching the displayed image.
    if (!http_mode)
        return;
    try {
        int64_t num_images = 0;
        auto dataset = http_reader.RefreshDatasetIfChanged(num_images);
        SetHttpConnected_i(true, current_file);

        if (num_images != total_images) {
            total_images = num_images;
            if (current_image.has_value())
                emit imageNumberChanged(total_images, current_image.value());
        }
        if (dataset)
            EmitDatasetLoaded_i(dataset);
    } catch (std::exception &e) {
        logger.Debug("Dataset refresh failed: {}", e.what());
        SetHttpConnected_i(false, current_file);
    }
}

void JFJochImageReadingWorker::StatusTimerExpired() {
    QMutexLocker locker(&m);
    if (!http_mode)
        return;
    try {
        BrokerStatus status = http_reader.GetBrokerStatus();
        SetHttpConnected_i(true, current_file);
        emit brokerStatusUpdated(status);
    } catch (std::exception &e) {
        SetHttpConnected_i(false, current_file);
    }
}

void JFJochImageReadingWorker::setAutoLoadMode_i(AutoloadMode in_mode) {
    autoload_mode = in_mode;
    if (autoload_mode == AutoloadMode::None)
        autoload_timer->stop();
    else
        autoload_timer->start();
    if (autoload_mode != AutoloadMode::HTTPSync)
        emit liveRateChanged(0.0);
    emit autoloadChanged(autoload_mode);
}

void JFJochImageReadingWorker::setAutoLoadMode(AutoloadMode mode) {
    QMutexLocker ul(&m);

    switch (mode) {
        case AutoloadMode::HTTPSync:
        case AutoloadMode::HTTPSyncDataset:
            if (http_mode)
                setAutoLoadMode_i(mode);
            else
                setAutoLoadMode_i(AutoloadMode::None);
            break;
        case AutoloadMode::Movie:
            setAutoLoadMode_i(mode);
            break;
        case AutoloadMode::None:
            setAutoLoadMode_i(mode);
            break;
    }
}

void JFJochImageReadingWorker::setAutoLoadJump(int64_t val) {
    QMutexLocker ul(&m);
    if (val > 0)
        jump_value = val;
}

void JFJochImageReadingWorker::LoadSpots(int64_t start_image, int64_t end_image, int64_t stride) {
    QMutexLocker ul(&m);
    std::shared_ptr<JFJochReaderSpots> result;
    if (http_mode)
        result = http_reader.ReadAllSpots(start_image, end_image, stride);
    else
        result = file_reader.ReadAllSpots(start_image, end_image, stride);
    emit spotsLoaded(result);
}

ReprocessingInputs JFJochImageReadingWorker::GetReprocessingInputs() const {
    QMutexLocker ul(&m);
    ReprocessingInputs in;
    if (http_mode || current_file.isEmpty() || !current_image_ptr)
        return in;
    in.file = current_file;
    in.experiment = curr_experiment;
    in.pixel_mask = *current_image_ptr->Dataset().pixel_mask;
    in.spot_finding = spot_finding_settings;
    if (reference_)
        in.reference_data = reference_->reflections;
    in.valid = true;
    return in;
}

void JFJochImageReadingWorker::ActivateSnapshot_i(const QString &name) {
    // Assumes m locked! Switch the reader's active metadata snapshot and refresh the view.
    file_reader.SetActiveSnapshot(name.toStdString());
    auto dataset = file_reader.GetDataset();
    if (!dataset)
        return;

    curr_experiment = dataset->experiment;
    curr_experiment.ImportIndexingSettings(indexing_settings);
    curr_experiment.ImportAzimuthalIntegrationSettings(azint_settings);
    UpdateAzint_i(dataset.get());
    EmitDatasetLoaded_i(dataset);

    QStringList names;
    for (const auto &n: file_reader.SnapshotNames())
        names << QString::fromStdString(n);
    emit snapshotsChanged(names, QString::fromStdString(file_reader.ActiveSnapshot()));
    EmitRuns_i();

    if (current_image.has_value()) {
        // A snapshot carries its own per-image results; just re-read the image against the new
        // metadata source, never re-run analysis (that would re-index on every snapshot switch).
        const bool prev_reanalyze = auto_reanalyze;
        auto_reanalyze = false;
        LoadImage_i(current_image.value(), current_summation);
        auto_reanalyze = prev_reanalyze;
    }
}

void JFJochImageReadingWorker::EmitRuns_i() {
    // Assumes m locked! Build the run list (every snapshot dataset + its display label).
    QVector<RunData> runs;
    if (!http_mode) {
        for (const auto &[id, dataset]: file_reader.AllSnapshotDatasets()) {
            const auto it = run_labels_.find(id);
            const QString label = (it != run_labels_.end()) ? it->second : QString::fromStdString(id);
            runs.push_back(RunData{QString::fromStdString(id), label, dataset});
        }
    }
    emit runsChanged(runs, http_mode ? QString() : QString::fromStdString(file_reader.ActiveSnapshot()));
}

void JFJochImageReadingWorker::RenameRun(QString id, QString label) {
    QMutexLocker ul(&m);
    run_labels_[id.toStdString()] = label;
    EmitRuns_i();
}

void JFJochImageReadingWorker::RemoveRun(QString id) {
    QMutexLocker ul(&m);
    if (http_mode || id == "Original")
        return;
    file_reader.RemoveSnapshot(id.toStdString());
    run_labels_.erase(id.toStdString());
    // Refresh the (possibly now-Original) active view; ActivateSnapshot_i also re-emits runsChanged.
    ActivateSnapshot_i(QString::fromStdString(file_reader.ActiveSnapshot()));
}

void JFJochImageReadingWorker::RegisterProcessingSnapshot(QString id, QString label, QString master_path) {
    QMutexLocker ul(&m);
    if (http_mode) {
        logger.Error("Processing snapshots are only available for files");
        return;
    }
    try {
        file_reader.RegisterSnapshot(id.toStdString(), master_path.toStdString());
        run_labels_[id.toStdString()] = label;
        ActivateSnapshot_i(id);   // activates + emits datasetLoaded / snapshotsChanged / runsChanged
    } catch (const std::exception &e) {
        logger.Error("Failed to register snapshot {}: {}", id.toStdString(), e.what());
        emit fileLoadError("Snapshot error", QString::fromStdString(e.what()));
    }
}

void JFJochImageReadingWorker::SetActiveSnapshot(QString name) {
    QMutexLocker ul(&m);
    if (http_mode)
        return;
    try {
        ActivateSnapshot_i(name);
    } catch (const std::exception &e) {
        logger.Error("Failed to activate snapshot {}: {}", name.toStdString(), e.what());
    }
}


void JFJochImageReadingWorker::SetThumbnailColorMap(int color_map) {
    QMutexLocker locker(&m);
    thumb_color_scale_.Select(static_cast<ColorScaleEnum>(color_map));
}

void JFJochImageReadingWorker::SetThumbnailFeatureColor(QColor c) {
    QMutexLocker locker(&m);
    thumb_feature_color_ = c;
}

void JFJochImageReadingWorker::SetThumbnailSpotColor(QColor c) {
    QMutexLocker locker(&m);
    thumb_spot_color_ = c;
}

void JFJochImageReadingWorker::RenderThumbnails(QVector<qint64> image_numbers, bool show_spots) {
    // File mode only; live HTTP cannot address arbitrary images cheaply.
    if (http_mode)
        return;
    for (qint64 n : image_numbers) {
        QImage thumb = RenderThumbnail_i(n, show_spots);
        if (!thumb.isNull())
            emit thumbnailReady(n, thumb);
    }
}

QImage JFJochImageReadingWorker::RenderThumbnail_i(int64_t image_number, bool show_spots) {
    // Everything runs inside one try: an uncaught exception here would be on the worker thread and
    // would std::terminate the whole app. On any failure we just skip this thumbnail.
    try {
        std::shared_ptr<JFJochReaderImage> img;
        {
            QMutexLocker locker(&m);
            if (http_mode || image_number < 0 || image_number >= total_images)
                return {};
            img = file_reader.LoadImage(image_number, 1);
        }
        if (!img)
            return {};

        const int W = static_cast<int>(img->Dataset().experiment.GetXPixelsNum());
        const int H = static_cast<int>(img->Dataset().experiment.GetYPixelsNum());
        const auto &px = img->Image();
        if (W <= 0 || H <= 0 || static_cast<int64_t>(px.size()) < static_cast<int64_t>(W) * H)
            return {};

        const auto &lut = thumb_color_scale_.LUTData();
        const int lutSize = static_cast<int>(lut.size());
        if (lutSize <= 0)
            return {};

        // Downsample by block-maximum (keeps sharp Bragg spots), then map through the LUT.
        constexpr int maxDim = 120;
        const double s = static_cast<double>(maxDim) / std::max(W, H);
        const int tw = std::max(1, static_cast<int>(W * s));
        const int th = std::max(1, static_cast<int>(H * s));
        const int bx = std::max(1, W / tw);
        const int by = std::max(1, H / th);

        const float fg = std::max(1.0f, static_cast<float>(img->GetAutoContrastValue()));
        const float invRange = (lutSize - 1) / fg;
        const rgb gap = thumb_color_scale_.Apply(ColorScaleSpecial::Gap);

        QImage out(tw, th, QImage::Format_RGB32);
        for (int ty = 0; ty < th; ++ty) {
            QRgb *line = reinterpret_cast<QRgb *>(out.scanLine(ty));
            const int y1 = std::min(H, ty * by + by);
            for (int tx = 0; tx < tw; ++tx) {
                const int x1 = std::min(W, tx * bx + bx);
                int32_t best = 0;
                bool any = false;
                for (int yy = ty * by; yy < y1; ++yy)
                    for (int xx = tx * bx; xx < x1; ++xx) {
                        const int32_t v = px[yy * W + xx];
                        if (v == GAP_PXL_VALUE || v == ERROR_PXL_VALUE) continue;
                        if (v == SATURATED_PXL_VALUE) { best = static_cast<int32_t>(fg); any = true; continue; }
                        if (!any || v > best) { best = v; any = true; }
                    }
                rgb c;
                if (!any) {
                    c = gap;
                } else {
                    int idx = static_cast<int>(std::max(0, best) * invRange + 0.5f);
                    idx = std::clamp(idx, 0, lutSize - 1);
                    c = lut[idx];
                }
                line[tx] = qRgb(c.r, c.g, c.b);
            }
        }

        if (show_spots) {
            QPainter p(&out);
            p.setRenderHint(QPainter::Antialiasing);
            for (const auto &sp : img->ImageData().spots) {
                p.setPen(QPen(sp.indexed ? thumb_feature_color_ : thumb_spot_color_, 1.0));
                p.drawEllipse(QPointF(sp.x * s, sp.y * s), 1.6, 1.6);
            }
        }

        // Image-number badge in the top-left, so the strip needs no text label (and loses no height).
        {
            QPainter p(&out);
            p.setRenderHint(QPainter::Antialiasing);
            QFont f = p.font();
            f.setBold(true);
            f.setPixelSize(11);
            p.setFont(f);
            const QString label = QString::number(image_number + 1);
            const QRect tb = QFontMetrics(f).boundingRect(label);
            const QRect badge(2, 2, tb.width() + 8, tb.height() + 2);
            p.setPen(Qt::NoPen);
            p.setBrush(QColor(0xFA, 0x72, 0x68));        // coral
            p.drawRoundedRect(badge, 3, 3);
            p.setPen(Qt::white);
            p.drawText(badge, Qt::AlignCenter, label);
        }
        return out;
    } catch (const std::exception &e) {
        logger.Debug("Thumbnail render failed for image {}: {}", image_number, e.what());
        return {};
    } catch (...) {
        return {};
    }
}