XFEL reader: integer frames stay integer, one gain, placement by panel

DecodeInto no longer turns every stored pixel into a float and back:
- the gain is one value (adu_per_photon_), not a float per stored pixel
  - it always was one value; only --gain or a measured gain set it;
- a 16-bit signed frame (Cheetah CXI, SACLA) without a gain is handed
  up as Int16, in its own type; other integer frames are widened to
  int32 without a float; only a float frame or a gain other than one
  goes through lround(v / gain), exactly as before;
- placement is per panel (start, size, destination steps) instead of a
  64-bit destination per stored pixel; a panel square with the image
  is copied row by row (PAL: 1440x1440 into 1444x1444 - the two-pixel
  margin is Place()'s, kept so the geometry does not move);
- the decode buffers (chunk, inflated, unshuffled) are reused per
  thread, the bitshuffle decode is not copied again;
- a chunk whose HDF5 filter_mask is not zero (a filter skipped) is
  read by HDF5 instead of being decoded as if it were not;
- the never-set value flags (flag_lessthan/morethan/equal) are gone
  from the layout and from the per-pixel loop.

MeasureGain in the CLI now reads Int16 frames as well, so a SACLA run
still has its gain measured.

Tests: JFJochXFELReader_SACLA/_CXI updated for the mode, new
JFJochXFELReader_CXI_Int16 (deflate without shuffle, a chunk with its
filter skipped) and JFJochXTCReader_StoredPixels (synthetic XTC,
stored hand-up and the offset path).

Measured on two PAL-XFEL Rayonix sets (int16, deflate; RTX 5080,
5950X, page cache warm; reports and .hkl identical to the baseline):
  main pass, 2000 frames:  296 -> 799 frames/s (pal86),
                           302 -> 755 frames/s (pal90)
  wall, 2000 frames:       26.4 -> 13.6 s, 32.1 -> 19.4 s
  (the geometry-refinement steps, which read frames too, halve)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-21 10:50:34 +02:00
1 parent eb47013d4f
commit 95e51f868d
6 files changed
+293 -130

No files matched your search

+129 -9
View File
@@ -2,26 +2,31 @@
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <hdf5.h>
#include <string>
#include <vector>
#include "../reader/JFJochXFELReader.h"
#include "../reader/JFJochXTCReader.h"
// Synthetic SACLA Cheetah and Cheetah CXI files, written with the HDF5 C API the way those pipelines
// write them: whole frames as single deflated (and byte-shuffled) chunks, so the direct-chunk path is
// what is read.
namespace {
void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector<hsize_t> dims) {
void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector<hsize_t> dims,
bool shuffle = true) {
const hid_t space = H5Screate_simple(static_cast<int>(dims.size()), dims.data(), nullptr);
std::vector<hsize_t> chunk = dims;
if (chunk.size() == 3)
chunk[0] = 1;
const hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_chunk(dcpl, static_cast<int>(chunk.size()), chunk.data());
H5Pset_shuffle(dcpl);
if (shuffle)
H5Pset_shuffle(dcpl);
H5Pset_deflate(dcpl, 4);
const hid_t ds = H5Dcreate2(loc, name.c_str(), type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
H5Dwrite(ds, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
@@ -47,10 +52,17 @@ void WriteScalar(hid_t loc, const std::string &name, double v) {
H5Sclose(space);
}
int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row) {
// A pixel of the flat image, as the reader hands it out: a 16-bit signed frame without a gain in its own
// type, anything else as int32.
int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row,
CompressedImageMode mode = CompressedImageMode::Int32) {
auto raw = reader.GetRawImage(image);
REQUIRE(raw);
return reinterpret_cast<const int32_t *>(raw->image_buffer.data())[row * raw->image.GetWidth() + col];
REQUIRE(raw->image.GetMode() == mode);
const size_t i = row * raw->image.GetWidth() + col;
if (mode == CompressedImageMode::Int16)
return reinterpret_cast<const int16_t *>(raw->image_buffer.data())[i];
return reinterpret_cast<const int32_t *>(raw->image_buffer.data())[i];
}
} // namespace
@@ -103,12 +115,14 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") {
// a whole column: the beam at 1.5, the pixel at column and row 2.
CHECK(x.GetBeamX_pxl() == Catch::Approx(1.5));
CHECK(x.GetBeamY_pxl() == Catch::Approx(1.5));
CHECK(PixelAt(reader, 0, 2, 2) == 20);
CHECK(PixelAt(reader, 1, 2, 2) == 21);
constexpr auto int16 = CompressedImageMode::Int16;
CHECK(PixelAt(reader, 0, 2, 2, int16) == 20);
CHECK(PixelAt(reader, 1, 2, 2, int16) == 21);
// Second sensor: (fs 0, ss 0) centre at x = 20 - 0.5, y = 0.5; fs runs along +y, ss along -x.
CHECK(PixelAt(reader, 0, 21, 2) == -5);
CHECK(PixelAt(reader, 0, 21, 3) == 33);
CHECK(PixelAt(reader, 0, 20, 2) == 44);
CHECK(PixelAt(reader, 0, 21, 2, int16) == -5);
CHECK(PixelAt(reader, 0, 21, 3, int16) == 33);
CHECK(PixelAt(reader, 0, 20, 2, int16) == 44);
CHECK(PixelAt(reader, 0, 0, 0, int16) == 0); // between the panels
// The file's mask holds the first sensor's (fs 1, ss 0), at column 3, row 2.
const auto &mask = reader.GetDataset()->pixel_mask->GetMask();
CHECK(mask[2 * x.GetXPixelsNum() + 3] != 0);
@@ -117,6 +131,8 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") {
reader.ApplyGain(2.0f);
CHECK(reader.CountsInPhotons());
CHECK(PixelAt(reader, 0, 2, 2) == 10);
CHECK(PixelAt(reader, 0, 21, 2) == -3); // -2.5, rounded away from zero
CHECK(PixelAt(reader, 0, 21, 3) == 17); // 16.5, likewise
reader.Close();
std::filesystem::remove(name);
}
@@ -182,3 +198,107 @@ TEST_CASE("JFJochXFELReader_CXI_TiledRefused", "[portable]") {
CHECK(refusal->find("tiled detector") != std::string::npos);
std::filesystem::remove(name);
}
TEST_CASE("JFJochXFELReader_CXI_Int16", "[portable]") {
// A single-panel Cheetah CXI file as a Rayonix one is written: 16-bit signed frames, deflated without
// a shuffle. The second frame's chunk is stored with its deflate skipped (filter mask bit 0), which the
// direct-chunk decoding does not expect: HDF5 reads that one.
const std::string name = "xfel_reader_int16.cxi";
{
const hid_t f = H5Fcreate(name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
for (const char *g: {"entry_1", "entry_1/data_1", "entry_1/instrument_1", "entry_1/instrument_1/detector_1",
"entry_1/instrument_1/source_1"})
H5Gclose(H5Gcreate2(f, g, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT));
std::vector<int16_t> frames(2 * 16, 3);
frames[1] = 32767; // frame 0, pixel (1, 0)
frames[4 + 2] = -4; // frame 0, pixel (2, 1)
WriteFrames(f, "entry_1/data_1/data", H5T_NATIVE_INT16, frames.data(), {2, 4, 4}, false);
std::vector<int16_t> raw(16, 9);
raw[5] = 1234; // frame 1, pixel (1, 1)
const hid_t ds = H5Dopen2(f, "entry_1/data_1/data", H5P_DEFAULT);
const hsize_t offset[3] = {1, 0, 0};
REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 1, offset, raw.size() * sizeof(int16_t), raw.data()) >= 0);
H5Dclose(ds);
const std::string det = "entry_1/instrument_1/detector_1/";
WriteVector(f, det + "x_pixel_size", {100e-6, 100e-6});
WriteVector(f, "entry_1/instrument_1/source_1/energy", std::vector<double>(2, 12000.0 * 1.602176634e-19));
H5Fclose(f);
}
REQUIRE(JFJochXFELReader::CanReadCXI(name));
JFJochXFELReader reader;
REQUIRE_NOTHROW(reader.ReadCXI({name}));
// Corner at (-2, -2): the pixel (fs, ss) lands at column and row 2 + fs, 2 + ss, in its stored type.
constexpr auto int16 = CompressedImageMode::Int16;
CHECK(PixelAt(reader, 0, 3, 2, int16) == 32767);
CHECK(PixelAt(reader, 0, 4, 3, int16) == -4);
CHECK(PixelAt(reader, 0, 5, 5, int16) == 3);
CHECK(PixelAt(reader, 0, 1, 1, int16) == 0); // the margin
CHECK(PixelAt(reader, 1, 3, 3, int16) == 1234);
CHECK(PixelAt(reader, 1, 2, 2, int16) == 9);
CHECK(reader.GetDataset()->experiment.GetSaturationLimit() == 32767);
reader.Close();
std::filesystem::remove(name);
}
namespace {
// An LCLS-I XTC stream of the shape JFJochXTCReader reads: a Configure datagram, then one event (an
// L1Accept) per image, each a container holding one 16-bit camera frame (Camera::FrameV1).
void WriteXTC(const std::string &name, uint32_t width, uint32_t height, uint32_t pixel_offset,
const std::vector<std::vector<uint16_t>> &images) {
std::ofstream f(name, std::ios::binary);
auto words = [&](std::initializer_list<uint32_t> w) {
for (uint32_t v: w)
f.write(reinterpret_cast<const char *>(&v), 4);
};
constexpr uint32_t ID_XTC = 1, ID_FRAME = 2, XTC_HEADER = 20;
// clock (2 words), timestamp (2 words, the transition in bits 24-27 of the first), env, Xtc header
words({0, 0, 0u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER});
const uint32_t frame_extent = XTC_HEADER + 16 + 2 * width * height;
for (const auto &image: images) {
words({0, 0, 12u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER + frame_extent});
words({0, 0, 7, ID_FRAME | (1u << 16), frame_extent});
words({width, height, 16, pixel_offset});
f.write(reinterpret_cast<const char *>(image.data()), static_cast<std::streamsize>(2 * image.size()));
}
}
} // namespace
TEST_CASE("JFJochXTCReader_StoredPixels", "[portable]") {
// Two streams of 3 x 2 images. Without an offset to take off, the main loop's read hands the pixels up
// as stored (UINT16_WIDENED), the same values the decoded int32 image holds; with one, it is taken off.
const std::string a = "xtc_reader_a.xtc", b = "xtc_reader_b.xtc", c = "xtc_reader_c.xtc";
WriteXTC(a, 3, 2, 0, {{1, 2, 3, 4, 5, 65535}, {7, 7, 7, 7, 7, 7}});
WriteXTC(b, 3, 2, 0, {{9, 8, 7, 6, 5, 4}});
WriteXTC(c, 3, 2, 10, {{11, 12, 13, 14, 15, 9}});
REQUIRE(JFJochXTCReader::CanRead(a));
JFJochXTCReader reader;
REQUIRE_NOTHROW(reader.ReadFiles({a, b}));
REQUIRE(reader.GetNumberOfImages() == 3);
auto raw = reader.GetRawImage(2);
REQUIRE(raw);
CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::UINT16_WIDENED);
std::vector<uint8_t> buffer;
auto pixels = reinterpret_cast<const int32_t *>(raw->image.GetUncompressedPtr(buffer));
CHECK(pixels[0] == 9);
CHECK(pixels[5] == 4);
raw = reader.GetRawImage(0);
REQUIRE(raw);
pixels = reinterpret_cast<const int32_t *>(raw->image.GetUncompressedPtr(buffer));
CHECK(pixels[5] == 65535);
REQUIRE_NOTHROW(reader.ReadFiles({c}));
raw = reader.GetRawImage(0);
REQUIRE(raw);
CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
CHECK(raw->image.GetMode() == CompressedImageMode::Int32);
pixels = reinterpret_cast<const int32_t *>(raw->image_buffer.data());
CHECK(pixels[0] == 1);
CHECK(pixels[5] == -1);
reader.Close();
for (const auto &n: {a, b, c})
std::filesystem::remove(n);
}