Jungfraujoch/image_analysis/hot_pixels/HotPixels.h

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#pragma once

// =============================================================================
// HotPixelFinder - pixels the pre-scan's own frames show to be defective
// =============================================================================
//
// A pixel that reads high on frame after frame, whatever the crystal is doing, is not recording
// diffraction. Unmasked, it is integrated as part of whichever reflection's box it falls in, and
// under rotation that is one pixel at one resolution collecting a different reflection on every
// frame that reaches it - a merged intensity hundreds or thousands of times its shell mean,
// supported by one or two observations the merge's outlier test cannot judge. The file's own
// mask misses such pixels on many detectors, and which pixels are hot moves with the threshold
// setting, so they are measured per dataset, on the pre-scan's sample of frames.
//
// On each frame a pixel is LIT when it is Poisson-significantly above the level of its own
// resolution ring - the larger of the median of the 2 px iso-2theta ring and of the ring's
// sixteenth in azimuth, so an ice or powder ring, the polarization dip and a partly shadowed ring
// all set their own level. It is PERSISTENT when it is lit on more of the sampled frames than
// either explanation that is not a defect allows:
//
//   * one Bragg reflection: it stays on a pixel for (oscillation + rocking width) / |zeta| of
//     rotation, i.e. on at most 1 + ceil(that / frame spacing) of frames sampled that far apart;
//   * chance: independent reflections and noise light a pixel of this ring on a fraction q of the
//     frames (measured, on the ring's pixels that are not candidates), and the number of lit frames
//     that fewer than 0.01 of all the detector's pixels would reach by chance is the binomial bound.
//
// A persistent pixel is masked if it reads on average at least ten times its ring and its mean excess
// is above the Poisson bound: a weaker one cannot make an outlier. Alone or as part of a patch - a
// patch lit through the whole sweep is stationary in the lab, so it is not a reflection of the
// rotating crystal, and a reflection crossing it reads the patch. A pixel holding the
// detector's error value on most frames is masked with it: every frame already treats it as invalid,
// but the static mask did not know.
//
// A DEAD pixel is the mirror of a hot one: it reads on average at most half of its own sector's level,
// and its mean deficit is above the bound a hot pixel's excess has to clear. Nothing but a defect keeps
// a pixel dark on frame after frame - a reflection only adds counts - and every reflection whose disk
// covers it is integrated short by the deficit. The detector's outermost row and column get no rule of
// their own - a dead edge row, or a hot pixel there, is caught like any other.
//
// A CCD is read with an offset and is not Poisson, but neither test needs it to be: a dead CCD pixel
// reads its offset, far below a ring with any background on it, and a hot one ten times its ring.
// =============================================================================

#include <atomic>
#include <cstdint>
#include <memory>
#include <mutex>
#include <vector>

#include "../../common/DiffractionExperiment.h"
#include "../../common/PixelMask.h"
#ifdef JFJOCH_USE_CUDA
#include "HotPixelsGPU.h"
#endif

class HotPixelFinder {
public:
    // Sixteen sectors of the ring; the level is the larger of the two medians.
    static constexpr int SECTORS = 16;
    // Ring width in 2theta: this many pixels where the beam meets the detector, wider further out.
    static constexpr float RING_WIDTH_PX = 2.0f;
    // A ring needs this many valid pixels to have a level, a sector this many to have one of its own.
    static constexpr int MIN_RING_PIXELS = 16;
    static constexpr int MIN_SECTOR_PIXELS = 8;
    // Lit: above level + LIT_NSIGMA * noise + LIT_OFFSET (noise = the larger of sqrt(level) and the
    // ring's robust spread; the offset keeps a pixel in a ring with no background from being lit by
    // one or two photons).
    static constexpr float LIT_NSIGMA = 3.3f;
    static constexpr float LIT_OFFSET = 2.0f;
    // How long one reflection can stay on a pixel, degrees of rotation beyond the oscillation: +-3
    // sigma of a rocking curve 2 degrees FWHM, wider than the crystals this is meant for.
    static constexpr float ROCKING_WIDTH_DEG = 5.0f;
    // Pixels expected to be called persistent by chance on the whole detector.
    static constexpr double FAMILY_WISE_RATE = 0.01;
    // A persistent pixel is masked only where it reads this many times its ring on average: the
    // strength at which the pixels behind the merged outliers were found, and well clear of a pixel
    // that is merely a little over-responding.
    static constexpr double STRONG_RATIO = 10.0;
    // The most frames a finder can be given: its per-pixel counters are 16-bit.
    static constexpr int MAX_SAMPLED_FRAMES = UINT16_MAX;

    struct Result {
        std::vector<uint32_t> mask;   // non-zero = masked (1 hot, 2 error value, 3 dead), converted geometry
        size_t hot = 0;               // persistent above the ring
        size_t error = 0;             // holding the error value on most frames
        size_t dead = 0;              // reading at most half of its sector's level
        uint32_t frames = 0;
    };

    HotPixelFinder(const DiffractionExperiment &experiment, const PixelMask &mask, size_t nthreads);
    ~HotPixelFinder();

    // One preprocessed frame (INT32_MIN = masked or error value, INT32_MAX = saturated). Thread
    // safe; `scratch` is the caller's own, reused across its calls.
    void AddImage(const int32_t *image, std::vector<int32_t> &scratch);

#ifdef JFJOCH_USE_CUDA
    // The same for a frame preprocessed on the device, on `frame`'s stream - the caller's own, as
    // `scratch` above, on the calling thread's card. The per-pixel sums are then kept on the cards and
    // replace the host's when the mask is read, so a finder is fed one way or the other, not both.
    // Thread safe.
    void AddDeviceImage(const int32_t *device_image, HotPixelFinderGPU::Frame &frame);

    // Build the device halves, one per card, now rather than with the first device frame, so the
    // workers do not wait on them one behind the other.
    void PrepareDevice();
#endif

    // The mask, from the frames added so far. oscillation_deg is the rotation per image and
    // spacing_deg the rotation between two consecutive sampled frames.
    [[nodiscard]] Result GetMask(double oscillation_deg, double spacing_deg, size_t nthreads);

private:
    const DiffractionGeometry geometry;
    const Coord axis;   // rotation axis; zero where there is none
    const size_t width, height;
    int nrings = 0;

    // Ring * SECTORS + sector of every pixel, -1 where the pixel is already masked. The per-pixel
    // arrays are hundreds of megabytes on a large detector, so they are left unwritten when allocated
    // and first written by the constructor's parallel pass rather than zeroed on one thread.
    std::unique_ptr<int32_t[]> key;
    // Where each ring-sector's pixels start in a scratch buffer laid out by key.
    std::vector<uint32_t> key_begin;
    size_t unmasked = 0;

    // The per-pixel sums are guarded by bands of rows, so workers adding frames at the same time
    // meet only when they reach the same band.
    static constexpr size_t BANDS = 64;
    mutable std::mutex band_mutex[BANDS];
    std::atomic<size_t> next_band{0};
    mutable std::mutex m;
    uint32_t frames = 0;
    std::unique_ptr<uint16_t[]> n_lit, n_error;
    std::unique_ptr<int64_t[]> sum_value;
    // A pixel's valid frames and the sum of its ring-sector level over them, kept per ring-sector - the
    // frames its ring had a level on, and the sum of the level over those - less what the pixel's own
    // error frames among them took out. Integers, so exactly the per-pixel sums, at a fraction of the
    // memory traffic per frame.
    std::vector<uint16_t> key_frames;
    std::vector<int64_t> key_level_sum;
    // The same sum for the level a pixel is judged DARK against: its own sector's median, the ring's
    // where the sector has too few pixels. Every pixel of a key has the same, so it needs no per-pixel
    // correction - a pixel with error frames is not judged dark.
    std::vector<int64_t> key_local_level_sum;
    std::unique_ptr<uint16_t[]> n_error_ring_ok;
    std::unique_ptr<int64_t[]> error_level_sum;
#ifdef JFJOCH_USE_CUDA
    // By card, built by PrepareDevice or the first device frame. Each card's frames add to its own sums,
    // and the cards are folded into the first when the mask is read.
    std::vector<std::unique_ptr<HotPixelFinderGPU>> gpus;
    // The first card's device half with every other card's sums added to it, or null without one.
    HotPixelFinderGPU *FoldedGpu();
#endif

    // Each ring-sector's level and lit threshold, from the frame's order statistics, and the frame's
    // share of the per-key sums. The device does the same in HotPixelsGPU.cu, with the same roundings.
    void AddLevels(const std::vector<int32_t> &sector_level, const std::vector<int32_t> &ring_level,
                   const std::vector<float> &ring_spread, const std::vector<char> &ring_ok,
                   const std::vector<int32_t> &local_level,
                   std::vector<int32_t> &level, std::vector<float> &threshold);
    [[nodiscard]] int n_valid(size_t i) const { return key_frames[key[i]] - n_error_ring_ok[i]; }
    [[nodiscard]] int64_t sum_level(size_t i) const { return key_level_sum[key[i]] - error_level_sum[i]; }
};