Jungfraujoch/reader/JFJochCBFReader.cpp

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#include "JFJochCBFReader.h"

#include <algorithm>
#include <cctype>
#include <cstring>
#include <filesystem>
#include <array>
#include <map>
#include <optional>
#include <tuple>

#include "../common/JFJochException.h"
#include "../common/Logger.h"
#include "../common/JFJochMath.h"
#include "SweepLayout.h"

namespace {

// ".cbf", or ".cbf.gz" - EMBL Hamburg's beamlines write the gzipped form by default, and the
// reader decompresses it in place, so a sweep of those is named here rather than converted first.
// Returns the suffix that matched, because the sweep template needs its length.
std::optional<std::string> CBFSuffix(const std::filesystem::path &p) {
    std::string name = p.filename().string();
    std::transform(name.begin(), name.end(), name.begin(),
                   [](unsigned char c) { return std::tolower(c); });
    for (const char *suffix : {".cbf.gz", ".cbf"}) {
        const std::string s(suffix);
        if (name.size() > s.size() && name.compare(name.size() - s.size(), s.size(), s) == 0)
            return s;
    }
    return {};
}

bool HasCBFExtension(const std::filesystem::path &p) {
    return CBFSuffix(p).has_value();
}

// The sweep a file belongs to, as a template: everything before the trailing run of digits, the
// number of digits, and the extension. "o8_1_0042.cbf" -> {"o8_1_", 4}. A directory can hold several
// sweeps ("o8_1_*" beside "o8_2_*"), so collecting every .cbf in it would silently splice two
// crystals together; matching the template is what makes "point at any frame" safe.
struct Template {
    std::string prefix;
    size_t digits = 0;
    std::string suffix = ".cbf";     // ".cbf" or ".cbf.gz"; the two are not one sweep

    bool Matches(const std::string &name) const {
        if (name.size() != prefix.size() + digits + suffix.size())
            return false;
        if (name.compare(0, prefix.size(), prefix) != 0)
            return false;
        if (name.compare(prefix.size() + digits, suffix.size(), suffix) != 0)
            return false;
        for (size_t i = 0; i < digits; i++)
            if (!std::isdigit(static_cast<unsigned char>(name[prefix.size() + i])))
                return false;
        return true;
    }
};

std::optional<Template> TemplateOf(const std::string &filename) {
    const std::filesystem::path p(filename);
    const auto suffix = CBFSuffix(p);
    if (!suffix.has_value())
        return {};
    // The stem is the name with the whole matched suffix removed, which std::filesystem cannot do
    // for ".cbf.gz" - its extension() there is ".gz" and its stem() still ends in ".cbf".
    const std::string name = p.filename().string();
    const std::string stem = name.substr(0, name.size() - suffix->size());
    size_t end = stem.size();
    while (end > 0 && std::isdigit(static_cast<unsigned char>(stem[end - 1])))
        end--;
    if (end == stem.size())
        return {};                                   // no trailing number: not part of a numbered sweep
    return Template{stem.substr(0, end), stem.size() - end, *suffix};
}

std::vector<std::string> CollectSweep(const std::string &path) {
    std::filesystem::path p(path);
    const bool is_dir = std::filesystem::is_directory(p);
    // A frame named with no directory at all is in this one - parent_path() of a bare filename is
    // empty, and iterating an empty path finds nothing, so naming a frame from inside its own
    // directory found no sweep.
    std::filesystem::path dir = is_dir ? p : p.parent_path();
    if (dir.empty())
        dir = ".";

    // Naming a frame selects ITS sweep. Naming a directory selects the sweep with the most frames in
    // it, which is the one a user pointing at a data directory means.
    std::optional<Template> want;
    if (!is_dir)
        want = TemplateOf(p.filename().string());

    std::map<std::tuple<std::string, size_t, std::string>, std::vector<std::string>> sweeps;
    std::error_code ec;
    for (const auto &e : std::filesystem::directory_iterator(dir, ec)) {
        if (!e.is_regular_file() || !HasCBFExtension(e.path()))
            continue;
        const std::string name = e.path().filename().string();
        const auto t = TemplateOf(name);
        if (!t.has_value())
            continue;
        if (want.has_value() && !want->Matches(name))
            continue;
        sweeps[{t->prefix, t->digits, t->suffix}].push_back(e.path().string());
    }

    std::vector<std::string> out;
    for (auto &[key, files] : sweeps)
        if (files.size() > out.size())
            out = std::move(files);

    // The frame number is zero-padded in every PILATUS naming scheme in use, so within one template a
    // plain sort is the collection order.
    std::sort(out.begin(), out.end());
    return out;
}

// An imgCIF laboratory direction in the internal frame. imgCIF puts Z from the sample towards the
// source and Y opposite gravity, while the internal frame has z along the beam and y along increasing
// row, so the two differ by a half turn about x - a rotation and not a mirror, so an axis carried
// through it turns the same way by the same angle. (writer/HDF5NXmx.cpp states the same relation from
// the other side, where it separates this from the McStas one, which is a half turn about z.)
Coord ImgCIFToInternal(const std::array<double, 3> &v) {
    return {static_cast<float>(v[0]), static_cast<float>(-v[1]), static_cast<float>(-v[2])};
}

// The base rotation axis where a header states nothing about it, in the internal frame (x along
// increasing detector column, y along increasing row, z along the beam). It is a convention: such a
// header names its rotation axis but gives no direction, so this is the sign an NXmx master writes
// for the same instruments, and a file that needs the other one is settled from the data by the run's
// axis-sign rescue.
const Coord ASSUMED_BASE_AXIS(-1.0f, 0.0f, 0.0f);

// The base rotation axis of the instrument, in the internal frame.
//
// Two headers in every corpus examined here state it and were being overruled by the assumption. One
// beamline's PILATUS turns about the VERTICAL: its axis table says so outright, and its "# Oscillation
// _axis" line says so a second way, by naming the image direction the spindle runs along rather than a
// vector. Assuming the horizontal axis put the spindle 90 degrees out - which no amount of refinement
// recovers, and which a sign rescue cannot reach either, since it is not a sign - and the run indexed
// nothing. The sign is still the rescue's business; the DIRECTION is the file's.
Coord BaseAxis(const minicbf::Header &h) {
    if (h.spindle_axis.has_value())
        return ImgCIFToInternal(*h.spindle_axis);
    if (h.spindle_along_slow)
        return {0.0f, -1.0f, 0.0f};     // minus the slow direction, as the default is minus the fast
    return ASSUMED_BASE_AXIS;
}

// How the stored image sits in the detector plane, where the header's axis table states it - the same
// thing the NXmx module directions say, in the form this format says it. Nothing comes back when the
// header carries no table, or when what it states is not one of the eight discrete orientations.
std::optional<DetectorOrientation> ImageOrientation(const minicbf::Header &h) {
    if (!h.fast_direction.has_value() || !h.slow_direction.has_value())
        return {};
    return DetectorOrientation::Match(ImgCIFToInternal(*h.fast_direction),
                                      ImgCIFToInternal(*h.slow_direction));
}

// The axis a miniCBF sweep turns about, in the internal frame (x along increasing detector column,
// y along increasing row, z along the beam).
//
// The base axis comes from the header where it states one (BaseAxis above) and is otherwise assumed.
// The sense of the omega rotation below follows the base axis, so a sweep parked at a non-zero omega
// inherits whichever direction it turns out to have.
//
// The head is base -> chi -> phi, so only the axes OUTSIDE the scanned one can tilt it. An omega
// scan turns about the base axis however the cradle is set - which is why a header carrying a large
// fixed chi still comes out as the base axis here - and only a phi scan is carried by chi and by
// omega. Chi turns about the beam, as the imgCIF axis convention has it, pointing back at the
// source; internal z points the other way, hence the minus. A head whose inner circle is inclined -
// a fixed-chi stage, a kappa arm - states that inclination in its axis table and nowhere else.
Coord RotationAxis(const minicbf::Header &h) {
    const Coord base = BaseAxis(h);
    if (!minicbf::ScansPhi(h))
        return base;

    const auto rad = [](double deg) { return static_cast<float>(deg * PI / 180.0); };
    // Where the table states the inner axis outright, that IS the inclination, and the driven-circle
    // angles do not describe it: a fixed-chi stage carries phi at a standing angle to the base with
    // no chi circle to report, so "# Chi" reads zero while the axis is tens of degrees away.
    const Coord chi_axis(0.0f, 0.0f, -1.0f);
    const Coord tilted = h.inner_spindle_axis.has_value()
                             ? ImgCIFToInternal(*h.inner_spindle_axis)
                             : RotMatrix(rad(h.chi_deg.value_or(0.0)), chi_axis) * base;
    return RotMatrix(rad(h.omega_deg.value_or(0.0)), base) * tilted;
}

}  // namespace

bool JFJochCBFReader::CanRead(const std::string &path) {
    std::error_code ec;
    if (std::filesystem::is_directory(path, ec))
        return !CollectSweep(path).empty();
    if (!HasCBFExtension(std::filesystem::path(path)))
        return false;
    try {
        // A pixel size as well as the compression: a byte-offset CBF with no "# Pixel_size" line has
        // no geometry, and XDS writes its correction files in exactly that shape. Claiming one here
        // would open it with a pixel size of zero.
        const auto h = minicbf::ReadHeader(path);
        return h.byte_offset && h.pixel_x_m > 0.0 && h.pixel_y_m > 0.0;
    } catch (const JFJochException &) {
        return false;
    }
}

void JFJochCBFReader::ReadFiles(const std::string &path) {
    files_ = CollectSweep(path);
    if (files_.empty())
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "No CBF images found for " + path);

    header0_ = minicbf::ReadHeader(files_[0]);
    if (!header0_.byte_offset)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "Unsupported CBF compression (only x-CBF_BYTE_OFFSET)");
    // Beside the compression, because a pixel size is what makes the file an IMAGE: every resolution,
    // every scattering vector and the beam centre in millimetres scale by it, and the default of 0
    // collapses all of them without a word. A byte-offset CBF with no "# Pixel_size" line is not a
    // detector image from this family - XDS writes its correction files in exactly that shape.
    if (!(header0_.pixel_x_m > 0.0) || !(header0_.pixel_y_m > 0.0))
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              files_[0] + " has no pixel size in its header (no '# Pixel_size' line); "
                              "it carries a CBF binary section but is not a detector image");

    dataset_ = std::make_shared<JFJochReaderDataset>();
    dataset_->experiment = default_experiment;

    DetectorSetup detector = DetDECTRIS(header0_.nx, header0_.ny, header0_.detector, {});
    detector.PixelSize_um(static_cast<int64_t>(std::lround(header0_.pixel_x_m * 1e6)));
    detector.SensorThickness_um(static_cast<int64_t>(std::lround(header0_.thickness_m * 1e6)));
    detector.SensorMaterial(header0_.material);
    // Only when the header states one. An absent line reads as zero, and a limit of zero would mark
    // EVERY pixel saturated - the integration accept gate then drops the whole reflection and the run
    // comes out empty for a reason nothing reports. Left unset, DiffractionExperiment::
    // GetSaturationLimit() falls back to the container's own overflow, which is the safe direction:
    // it can only fail to call a pixel saturated.
    // The count-rate correction CLIPS an over-range pixel to Count_cutoff rather than marking it
    // (Trueb et al. (2015) J. Synchrotron Rad. 22, 701-707), so the cutoff is a value a pixel can
    // hold and the limit is exclusive - unlike NXmx's saturation_value, which is the last VALID
    // count. Three of the miniCBF sets in the corpus carry such a clipped pixel in every frame, in
    // flat background, hundreds of times the next-highest count. XDS and DIALS read this field as
    // the last valid count and so pass those pixels through; imgCIF's own overload item agrees with
    // us that the valid values are the ones below it.
    if (header0_.count_cutoff > 0)
        detector.SaturationLimit(header0_.count_cutoff);
    else
        Logger("CBFReader").Warning("{} states no Count_cutoff, so no pixel will be called "
                                    "saturated; if this detector overloads, its strongest "
                                    "reflections will be integrated as if they were valid.",
                                    files_[0]);
    // Images are handed out as signed 32-bit whatever the file stored, so that is the depth the rest
    // of the code must see; the real overflow is the header's Count_cutoff, set above.
    detector.BitDepthImage(32);
    if (const auto orientation = ImageOrientation(header0_))
        detector.ImageOrientation(orientation.value());
    detector.MinFrameTime(std::chrono::microseconds(0));
    detector.MinCountTime(std::chrono::microseconds(0));
    detector.ReadOutTime(std::chrono::nanoseconds(0));
    dataset_->experiment.Detector(detector);

    dataset_->experiment.BeamX_pxl(static_cast<float>(header0_.beam_x_px));
    dataset_->experiment.BeamY_pxl(static_cast<float>(header0_.beam_y_px));
    dataset_->experiment.DetectorDistance_mm(static_cast<float>(header0_.distance_m * 1000.0));

    // A detector swung out on a 2theta arm, which small-molecule collection uses routinely. The arm
    // turns the detector about the sample, so it carries the square-on geometry with it: the header's
    // Detector_distance stays the distance along the detector normal and Beam_xy stays the point of
    // normal incidence, neither of which the swing moves - which is exactly what the PONI convention
    // wants, so the swing is a PONI rotation and nothing else in the header changes. It turns about
    // the axis the header's table gives the arm, and otherwise about the base spindle axis: on the
    // four-circle geometry these headers describe the arm and the spindle are one axis, and the one
    // header here that states both states them with the same vector.
    if (header0_.two_theta_deg != 0.0) {
        const Coord axis = header0_.detector_axis.has_value()
                               ? ImgCIFToInternal(*header0_.detector_axis) : BaseAxis(header0_);
        float rot1 = 0, rot2 = 0, rot3 = 0;
        PoniAnglesFromMatrix(RotMatrix(static_cast<float>(header0_.two_theta_deg * PI / 180.0), axis),
                             rot1, rot2, rot3);
        dataset_->experiment.PoniRot1_rad(rot1).PoniRot2_rad(rot2).PoniRot3_rad(rot3);
    }

    dataset_->experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / static_cast<float>(header0_.wavelength_A));
    dataset_->experiment.FrameTime(
        std::chrono::duration_cast<std::chrono::nanoseconds>(
            std::chrono::duration<double>(header0_.period_s)),
        std::chrono::duration_cast<std::chrono::nanoseconds>(
            std::chrono::duration<double>(header0_.exposure_s)));

    // Where every image sits on the spindle, and how the instrument stood, from its own header.
    // Parsing one costs a few hundred microseconds - it is two dozen regular expressions - so on a
    // sweep of several thousand frames this is seconds of startup before a single image is read, and
    // the files are independent.
    std::vector<sweep::Frame> frames(files_.size());
    sweep::ForEachInOrder(files_.size(), [&](size_t i) {
        const auto h = minicbf::ReadHeader(files_[i]);
        frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m,
                     h.beam_x_px, h.beam_y_px, h.wavelength_A};
    });

    // The sweep the headers describe, which is not always the files laid out end to end: a deposited
    // series can be missing frames, and those are gaps in the rotation rather than images to close up.
    const auto layout = sweep::Place(frames, "CBFReader");
    files_ = layout.files;
    dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name,
                                                   static_cast<float>(layout.start_deg),
                                                   static_cast<float>(layout.increment_deg),
                                                   RotationAxis(header0_), {}));

    dataset_->error_value = -1;
    dataset_->experiment.ImagesPerTrigger(static_cast<int64_t>(files_.size()));

    // The untrusted pixels a PILATUS marks with a negative value: module gaps and the bad-pixel map.
    // They are the same on every frame of a sweep, so frame 0 defines the mask.
    std::vector<int32_t> first;
    minicbf::Read(files_[0], first);
    std::vector<uint32_t> mask(first.size(), 0);
    for (size_t i = 0; i < first.size(); i++)
        if (first[i] < 0)
            mask[i] = 1;
    dataset_->pixel_mask = std::make_shared<const PixelMask>(mask);

    SetStartMessage(dataset_);
}

uint64_t JFJochCBFReader::GetNumberOfImages() const {
    return files_.size();
}

void JFJochCBFReader::Close() {
    files_.clear();
    dataset_.reset();
}

template <class Buffer>
CompressedImage JFJochCBFReader::DecodeInto(int64_t image_number, Buffer &buffer,
                                            std::vector<uint8_t> &scratch) const {
    if (image_number < 0 || static_cast<size_t>(image_number) >= files_.size())
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "Image number out of range");
    if (files_[image_number].empty())
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "No image at this point of the sweep");

    const size_t npixel = static_cast<size_t>(header0_.nx) * static_cast<size_t>(header0_.ny);
    buffer.resize(npixel * sizeof(int32_t));

    // Decode straight into the caller's bytes: the pixels are plain int32 and nothing downstream has
    // to decompress them, so NO_COMPRESSION over that buffer is the whole image.
    const auto h = minicbf::ReadInto(files_[image_number],
                                     reinterpret_cast<int32_t *>(buffer.data()), npixel, scratch);
    if (static_cast<size_t>(h.nelem) != npixel)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "CBF image size differs from the first image of the sweep");

    return CompressedImage(buffer.data(), buffer.size(),
                           static_cast<size_t>(header0_.nx), static_cast<size_t>(header0_.ny),
                           CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION);
}

bool JFJochCBFReader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
                                  DataMessage &message,
                                  std::vector<uint8_t> &buffer,
                                  int64_t image_number,
                                  bool update_dataset) {
    (void) update_dataset;
    if (!dataset)
        return false;

    if (!HasImage(image_number))
        return false;

    // The image must outlive this call, so it is decoded straight into the caller's buffer - the same
    // thing the argument is for on the HDF5 path - and message.image only points at it.
    std::vector<uint8_t> scratch;
    message.image = DecodeInto(image_number, buffer, scratch);
    message.number = image_number;
    return true;
}

// A slot the series has no file for is a missing image, not an error: every image loop in the
// pipeline already treats "nothing to read" as a frame to pass over, which is exactly what a gap in
// a deposited sweep is.
bool JFJochCBFReader::HasImage(int64_t image_number) const {
    return image_number >= 0 && static_cast<size_t>(image_number) < files_.size()
           && !files_[image_number].empty();
}

bool JFJochCBFReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) {
    if (!HasImage(image_number))
        return false;
    // The file as it is stored: the consumer decodes the byte-offset stream, on the device where there
    // is one, so the host neither decodes nor uploads a four-byte image. The stream sits in the image's
    // own buffer, which the image keeps (read_buffer is not part of it).
    size_t start = 0;
    const auto h = minicbf::ReadCompressed(files_[image_number], image.image_buffer, start);
    if (h.nx != header0_.nx || h.ny != header0_.ny)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "CBF image size differs from the first image of the sweep");
    image.image = CompressedImage(image.image_buffer.data() + start, image.image_buffer.size() - start,
                                  static_cast<size_t>(header0_.nx), static_cast<size_t>(header0_.ny),
                                  CompressedImageMode::Int32, CompressionAlgorithm::CBF_BYTE_OFFSET);
    return true;
}

std::vector<SpotToSave> JFJochCBFReader::ReadSpots(int64_t) const {
    return {};   // a raw CBF stores no analysis results
}