// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #pragma once // ============================================================================= // HotPixelFinder - pixels the pre-scan's own frames show to be defective // ============================================================================= // // A pixel that reads high on frame after frame, whatever the crystal is doing, is not recording // diffraction. Unmasked, it is integrated as part of whichever reflection's box it falls in, and // under rotation that is one pixel at one resolution collecting a different reflection on every // frame that reaches it - a merged intensity hundreds or thousands of times its shell mean, // supported by one or two observations the merge's outlier test cannot judge. The file's own // mask misses such pixels on many detectors, and which pixels are hot moves with the threshold // setting, so they are measured per dataset, on the pre-scan's sample of frames. // // On each frame a pixel is LIT when it is Poisson-significantly above the level of its own // resolution ring - the larger of the median of the 2 px iso-2theta ring and of the ring's // sixteenth in azimuth, so an ice or powder ring, the polarization dip and a partly shadowed ring // all set their own level. It is PERSISTENT when it is lit on more of the sampled frames than // either explanation that is not a defect allows: // // * one Bragg reflection: it stays on a pixel for (oscillation + rocking width) / |zeta| of // rotation, i.e. on at most 1 + ceil(that / frame spacing) of frames sampled that far apart; // * chance: independent reflections and noise light a pixel of this ring on a fraction q of the // frames (measured, on the ring's pixels that are not candidates), and the number of lit frames // that fewer than 0.01 of all the detector's pixels would reach by chance is the binomial bound. // // A persistent pixel is masked if it reads on average at least ten times its ring and its mean excess // is above the Poisson bound: a weaker one cannot make an outlier. Alone or as part of a patch - a // patch lit through the whole sweep is stationary in the lab, so it is not a reflection of the // rotating crystal, and a reflection crossing it reads the patch. A pixel holding the // detector's error value or its saturation limit on most frames is masked with it: every frame already // treats it as invalid, but the static mask did not know. A pixel stuck at the limit is no overload - // a CCD can have thousands, at 65535 on every frame - and unmasked, every reflection whose disk it // falls in is flagged as overloaded. // // A DEAD pixel is the mirror of a hot one: it reads on average at most half of its own sector's level, // and its mean deficit is above the bound a hot pixel's excess has to clear. Nothing but a defect keeps // a pixel dark on frame after frame - a reflection only adds counts - and every reflection whose disk // covers it is integrated short by the deficit. The detector's outermost row and column get no rule of // their own - a dead edge row, or a hot pixel there, is caught like any other. // // A CCD is read with an offset and is not Poisson, but neither test needs it to be: a dead CCD pixel // reads its offset, far below a ring with any background on it, and a hot one ten times its ring. // ============================================================================= #include #include #include #include #include #include "../../common/DiffractionExperiment.h" #include "../../common/PixelMask.h" #ifdef JFJOCH_USE_CUDA #include "HotPixelsGPU.h" #endif class HotPixelFinder { public: // Sixteen sectors of the ring; the level is the larger of the two medians. static constexpr int SECTORS = 16; // Ring width in 2theta: this many pixels where the beam meets the detector, wider further out. static constexpr float RING_WIDTH_PX = 2.0f; // A ring needs this many valid pixels to have a level, a sector this many to have one of its own. static constexpr int MIN_RING_PIXELS = 16; static constexpr int MIN_SECTOR_PIXELS = 8; // Lit: above level + LIT_NSIGMA * noise + LIT_OFFSET (noise = the larger of sqrt(level) and the // ring's robust spread; the offset keeps a pixel in a ring with no background from being lit by // one or two photons). In photons: on frames in detector units the Poisson noise is sqrt(gain * // level) and the offset LIT_OFFSET photons, gain of them each (BraggIntegrationSettings::pixel_gain). static constexpr float LIT_NSIGMA = 3.3f; static constexpr float LIT_OFFSET = 2.0f; // How long one reflection can stay on a pixel, degrees of rotation beyond the oscillation: +-3 // sigma of a rocking curve 2 degrees FWHM, wider than the crystals this is meant for. static constexpr float ROCKING_WIDTH_DEG = 5.0f; // Pixels expected to be called persistent by chance on the whole detector. static constexpr double FAMILY_WISE_RATE = 0.01; // A persistent pixel is masked only where it reads this many times its ring on average: the // strength at which the pixels behind the merged outliers were found, and well clear of a pixel // that is merely a little over-responding. static constexpr double STRONG_RATIO = 10.0; // The most frames a finder can be given: its per-pixel counters are 16-bit. static constexpr int MAX_SAMPLED_FRAMES = UINT16_MAX; struct Result { std::vector mask; // non-zero = masked (1 hot, 2 error value, 3 dead), converted geometry size_t hot = 0; // persistent above the ring size_t error = 0; // holding the error value or the saturation limit on most frames size_t dead = 0; // reading at most half of its sector's level uint32_t frames = 0; }; HotPixelFinder(const DiffractionExperiment &experiment, const PixelMask &mask, size_t nthreads); ~HotPixelFinder(); // One preprocessed frame (INT32_MIN = masked or error value, INT32_MAX = saturated). Thread // safe; `scratch` is the caller's own, reused across its calls. void AddImage(const int32_t *image, std::vector &scratch); #ifdef JFJOCH_USE_CUDA // The same for a frame preprocessed on the device, on `frame`'s stream - the caller's own, as // `scratch` above, on the calling thread's card. The per-pixel sums are then kept on the cards and // replace the host's when the mask is read, so a finder is fed one way or the other, not both. // Thread safe. void AddDeviceImage(const int32_t *device_image, HotPixelFinderGPU::Frame &frame); // Build the device halves, one per card, now rather than with the first device frame, so the // workers do not wait on them one behind the other. void PrepareDevice(); #endif // The mask, from the frames added so far. oscillation_deg is the rotation per image and // spacing_deg the rotation between two consecutive sampled frames. [[nodiscard]] Result GetMask(double oscillation_deg, double spacing_deg, size_t nthreads); private: const DiffractionGeometry geometry; const DetectorModules modules; // where the pixels are on the geometry's plane; empty = as imaged const Coord axis; // rotation axis; zero where there is none const float gain; // detector units per photon of the frames (BraggIntegrationSettings::pixel_gain) const size_t width, height; int nrings = 0; // Ring * SECTORS + sector of every pixel, -1 where the pixel is already masked. The per-pixel // arrays are hundreds of megabytes on a large detector, so they are left unwritten when allocated // and first written by the constructor's parallel pass rather than zeroed on one thread. std::unique_ptr key; // Where each ring-sector's pixels start in a scratch buffer laid out by key. std::vector key_begin; size_t unmasked = 0; // The per-pixel sums are guarded by bands of rows, so workers adding frames at the same time // meet only when they reach the same band. static constexpr size_t BANDS = 64; mutable std::mutex band_mutex[BANDS]; std::atomic next_band{0}; mutable std::mutex m; uint32_t frames = 0; std::unique_ptr n_lit, n_error; std::unique_ptr sum_value; // A pixel's valid frames and the sum of its ring-sector level over them, kept per ring-sector - the // frames its ring had a level on, and the sum of the level over those - less what the pixel's own // error frames among them took out. Integers, so exactly the per-pixel sums, at a fraction of the // memory traffic per frame. std::vector key_frames; std::vector key_level_sum; // The same sum for the level a pixel is judged DARK against: its own sector's median, the ring's // where the sector has too few pixels. Every pixel of a key has the same, so it needs no per-pixel // correction - a pixel with error frames is not judged dark. std::vector key_local_level_sum; std::unique_ptr n_error_ring_ok; std::unique_ptr error_level_sum; #ifdef JFJOCH_USE_CUDA // By card, built by PrepareDevice or the first device frame. Each card's frames add to its own sums, // and the cards are folded into the first when the mask is read. std::vector> gpus; // The first card's device half with every other card's sums added to it, or null without one. HotPixelFinderGPU *FoldedGpu(); #endif // Each ring-sector's level and lit threshold, from the frame's order statistics, and the frame's // share of the per-key sums. The device does the same in HotPixelsGPU.cu, with the same roundings. void AddLevels(const std::vector §or_level, const std::vector &ring_level, const std::vector &ring_spread, const std::vector &ring_ok, const std::vector &local_level, std::vector &level, std::vector &threshold); [[nodiscard]] int n_valid(size_t i) const { return key_frames[key[i]] - n_error_ring_ok[i]; } [[nodiscard]] int64_t sum_level(size_t i) const { return key_level_sum[key[i]] - error_level_sum[i]; } };