Jungfraujoch/viewer/JFJochImageReadingWorker.h

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#pragma once

#include <QObject>
#include <QString>
#include <QVector>
#include <QRect>
#include <QMutex>
#include <QTimer>
#include <QElapsedTimer>
#include <QImage>
#include <QColor>

#include "../common/ColorScale.h"

#include "../reader/JFJochHDF5Reader.h"
#include "../common/Logger.h"
#include "../reader/JFJochHttpReader.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
#include "../common/BraggIntegrationSettings.h"
#include "../common/ScalingSettings.h"
#include "../image_analysis/bragg_prediction/BraggPrediction.h"
#include "../image_analysis/LoadFCalcFromMtz.h"   // ReferenceMtzData + LoadReferenceMtz
#include "SimpleImage.h"
#include "ReferenceMtzInfo.h"
#include "../common/MovingAverage.h"

Q_DECLARE_METATYPE(std::shared_ptr<const JFJochReaderDataset>)
Q_DECLARE_METATYPE(std::shared_ptr<const JFJochReaderImage>)
Q_DECLARE_METATYPE(std::shared_ptr<const JFJochReaderSpots>)

Q_DECLARE_METATYPE(DiffractionExperiment)
Q_DECLARE_METATYPE(SpotFindingSettings)
Q_DECLARE_METATYPE(IndexingSettings)
Q_DECLARE_METATYPE(AzimuthalIntegrationSettings)
Q_DECLARE_METATYPE(BraggIntegrationSettings)
Q_DECLARE_METATYPE(ScalingSettings)

Q_DECLARE_METATYPE(UnitCell)
Q_DECLARE_METATYPE(std::shared_ptr<const SimpleImage>)
Q_DECLARE_METATYPE(BrokerStatus)
Q_DECLARE_METATYPE(ROIDefinition)

// Everything a reprocessing job needs, captured from the worker's current state in one locked read.
struct ReprocessingInputs {
    bool valid = false;
    QString file;
    DiffractionExperiment experiment;
    PixelMask pixel_mask;
    SpotFindingSettings spot_finding;
    std::vector<MergedReflection> reference_data;   // empty unless a reference MTZ is loaded
};

#include "RunData.h"

class JFJochImageReadingWorker : public QObject {
    Q_OBJECT

public:
    // HTTPSync: follow latest image + dataset. HTTPSyncDataset: follow dataset/plots/image count
    // but keep the displayed image frozen (entered by manually selecting an image while following).
    enum class AutoloadMode {HTTPSync, HTTPSyncDataset, Movie, None};
    Q_ENUM(AutoloadMode)
private:
    mutable QMutex m;
    bool http_mode = false;

    AutoloadMode autoload_mode = AutoloadMode::None;

    JFJochHDF5Reader file_reader;
    JFJochHttpReader http_reader;

    QString current_file;

    DiffractionExperiment curr_experiment;
    // Once the user edits ROIs they override whatever the file carried, for this and
    // every subsequently loaded image, until a new file is opened.
    std::optional<ROIDefinition> roi_override_;
    std::map<std::string, QString> run_labels_;   // snapshot id -> editable display label
    IndexingSettings indexing_settings;
    AzimuthalIntegrationSettings azint_settings;
    BraggIntegrationSettings bragg_settings;
    ScalingSettings scaling_settings;
    // Reference reflections (from a reference MTZ), kept independent of the loaded dataset - they
    // survive file switches and feed reference-based scaling / CCref reprocessing jobs.
    std::optional<ReferenceMtzData> reference_;
    std::unique_ptr<IndexerThreadPool> indexing;

    std::shared_ptr<JFJochReaderImage> current_image_ptr;
    std::unique_ptr<AzimuthalIntegrationMapping> azint_mapping;
    std::unique_ptr<MXAnalysisWithoutFPGA> image_analysis;
    std::unique_ptr<IndexAndRefine> index_and_refine;

    // Azimuthal profile buffer of the most recently analyzed image (filled by Analyze).
    std::unique_ptr<AzimuthalIntegrationProfile> last_profile_;

    SpotFindingSettings spot_finding_settings;

    ColorScale thumb_color_scale_;   // colour map used for the image-strip thumbnails
    QColor thumb_feature_color_ = Qt::magenta;   // indexed-spot overlay (matches the main viewer)
    QColor thumb_spot_color_ = Qt::green;        // non-indexed spot overlay
    QImage RenderThumbnail_i(int64_t image_number, bool show_spots);

    std::optional<int64_t> current_image;
    int64_t current_summation = 1;
    int64_t total_images = 0;
    int64_t jump_value = 1;
    Logger logger{"jfjoch_viewer"};

    bool auto_reanalyze = false;

    QTimer *autoload_timer;
    int autoload_interval = 500; // milliseconds

    // Broker status polling (independent of image sync, runs whenever connected over HTTP)
    QTimer *status_timer = nullptr;
    int status_interval = 1000; // milliseconds
    bool http_connected = false;

    // Adaptive autoload interval based on recent load+analysis time
    MovingAverage autoload_ms_ma{8};          // window size (tune as needed)
    int autoload_interval_min_ms = 50;        // 20 Hz is the top performance!
    int autoload_interval_max_ms = 2000;      // 0.5 Hz as bottom limit
    float autoload_safety_factor = 2.0f;

    // Backpressure: cap how many frames may be "in flight" to the GUI (emitted but not yet rendered
    // and acked via ImageConsumed). The autoload timer won't produce another frame while the cap is
    // reached, so a fast producer can't grow the queued-event backlog without bound.
    int images_in_flight = 0;
    int max_images_in_flight = 4;

    // Same backpressure for the dataset stream (see EmitDatasetLoaded_i / DatasetConsumed): each
    // queued datasetLoaded event pins a full dataset (mask + per-image plots), so cap how many may
    // be in flight to the GUI. This also coalesces dataset-only follow (HTTPSyncDataset), which emits
    // no image and so was never gated by images_in_flight - stale datasets are dropped, the next tick
    // sends the latest. Without this the datasetLoaded backlog grew without bound (100 GB OOM).
    int datasets_in_flight = 0;
    int max_datasets_in_flight = 2;

    // File open retry/back-off (GPFS via NFSv4 visibility lag)
    struct PendingLoadRequest {
        QString filename;
        qint64 image_number = 0;
        qint64 summation = 1;
    };

    QTimer *file_open_retry_timer = nullptr;
    PendingLoadRequest pending_load;
    bool file_open_retry_active = false;
    bool file_open_retry_warned = false;
    int file_open_retry_attempts = 0;
    int file_open_retry_delay_ms = 100;
    int file_open_retry_delay_max_ms = 10000;
    QElapsedTimer file_open_retry_elapsed;

    void ResetFileOpenRetry_i();
    void ScheduleFileOpenRetry_i(const QString& reason);

    void LoadFile_i(const QString &filename, qint64 image_number, qint64 summation, bool retry);
    void LoadImage_i(int64_t image_number, int64_t summation);
    void EmitImageLoaded_i(const std::shared_ptr<const JFJochReaderImage>& image);
    void EmitDatasetLoaded_i(const std::shared_ptr<const JFJochReaderDataset>& dataset);
    void RefreshDatasetOnly_i();
    void SetHttpConnected_i(bool connected, const QString &addr);
    void ReanalyzeImage_i();
    void UpdateDataset_i(const std::optional<DiffractionExperiment>& experiment);
    void UpdateAzint_i(const JFJochReaderDataset *dataset);
    void UpdateUserMask_i(const std::vector<uint32_t> &mask);
    void SetROIDefinition_i(const ROIDefinition &rois);
    void ApplyROIOverrideToImage_i();   // rewrite current_image_ptr's dataset with roi_override_
    void setAutoLoadMode_i(AutoloadMode mode);
    void ActivateSnapshot_i(const QString &name);
    void EmitRuns_i();   // emit runsChanged with every snapshot dataset + its label
    void EmitReferenceInfo_i();   // emit referenceMtzChanged from reference_ + current data

signals:
    void datasetLoaded(std::shared_ptr<const JFJochReaderDataset>);
    void imageLoaded(std::shared_ptr<const JFJochReaderImage>);
    void spotsLoaded(std::shared_ptr<const JFJochReaderSpots>);
    void imageStatsUpdated(std::shared_ptr<const JFJochReaderImage>);
    void imageNumberChanged(int64_t total_images, int64_t current_image);
    void setRings(const QVector<float> &v);
    void simpleImageLoaded(std::shared_ptr<const SimpleImage> image);
    void autoloadChanged(AutoloadMode mode);
    void fileLoadError(QString title, QString message);
    void fileLoadRetryStatus(bool active, QString message);
    void brokerStatusUpdated(BrokerStatus status);
    void httpConnectionChanged(bool connected, QString addr);
    void liveRateChanged(double hz);
    void snapshotsChanged(QStringList names, QString active);
    void runsChanged(QVector<RunData> runs, QString active_id);
    void fileOpened();   // a new file/stream was loaded - listeners should reset per-file state
    void thumbnailReady(qint64 image_number, QImage thumb);   // for the image-strip / hit feed
    void referenceMtzChanged(ReferenceMtzInfo info);   // a reference MTZ was (un)loaded - update the dock

public:
    JFJochImageReadingWorker(const SpotFindingSettings &settings, const DiffractionExperiment& experiment, QObject *parent = nullptr);
    ~JFJochImageReadingWorker() override = default;

    // Captures the current file + experiment + mask + spot-finding settings so a processing job
    // uses exactly the same settings as interactive analysis. Thread-safe (locks the worker mutex).
    ReprocessingInputs GetReprocessingInputs() const;

private slots:
    void AutoLoadTimerExpired();
    void StatusTimerExpired();
    void FileOpenRetryTimerExpired();

public slots:
    void LoadFile(const QString &filename, qint64 image_number, qint64 summation, bool retry);
    void CloseFile();
    void LoadImage(int64_t image_number, int64_t summation);
    void LoadSpots(int64_t start_image, int64_t end_image, int64_t stride);

    void UpdateDataset(const DiffractionExperiment& experiment);

    void FindCenter(const UnitCell& calibrant, bool guess);

    void Analyze();

    void UpdateSpotFindingSettings(const SpotFindingSettings &settings, const IndexingSettings &indexing, int64_t max_spots);
    void ReanalyzeImages(bool input);
    void UpdateAzintSettings(const AzimuthalIntegrationSettings& settings);
    void UpdateBraggIntegrationSettings(BraggIntegrationSettings settings);
    void UpdateScalingSettings(ScalingSettings settings);

    // Load (column empty = auto-select) or clear a reference MTZ; kept independent of the dataset.
    void SetReferenceMtz(QString path, QString column);
    void ClearReferenceMtz();

    void SetROIDefinition(const ROIDefinition &rois);
    void DownloadROIsFromServer();
    void UploadROIsToServer();
    void MaskFromSelectedROI(QString name, bool add);   // rasterise an ROI into the user mask

    void SaveUserMaskTIFF(QString filename);
    void LoadUserMaskTIFF(QString filename, bool replace);
    void UploadUserMask();
    void ClearUserMask();

    void LoadCalibration(QString dataset);
    void setAutoLoadMode(AutoloadMode mode);
    void setAutoLoadJump(int64_t val);

    // GUI acks that it has consumed (rendered) one frame, freeing an in-flight slot. Drives the
    // autoload backpressure so the reader can't outrun the viewer (see images_in_flight).
    void ImageConsumed();

    // GUI ack that it has consumed (fanned out + rendered) one dataset, freeing an in-flight slot.
    // Drives the datasetLoaded backpressure so the reader can't outrun the viewer's dataset drain.
    void DatasetConsumed();

    // Register a reprocessing result (_process.h5) as a metadata snapshot over the same images and
    // make it the active dataset; SetActiveSnapshot switches between registered snapshots.
    void RegisterProcessingSnapshot(QString id, QString label, QString master_path);
    void SetActiveSnapshot(QString name);
    void RenameRun(QString id, QString label);   // change a run's display label (legend)
    void RemoveRun(QString id);                  // drop a reprocessing snapshot (not "Original")

    // Render downsampled thumbnails for the given images (file mode only), emitting thumbnailReady
    // for each; non-disruptive (does not change the displayed image).
    void RenderThumbnails(QVector<qint64> image_numbers, bool show_spots);
    void SetThumbnailColorMap(int color_map);
    void SetThumbnailFeatureColor(QColor c);
    void SetThumbnailSpotColor(QColor c);
};