XFEL reader: integer frames stay integer, one gain, placement by panel
DecodeInto no longer turns every stored pixel into a float and back:
- the gain is one value (adu_per_photon_), not a float per stored pixel
- it always was one value; only --gain or a measured gain set it;
- a 16-bit signed frame (Cheetah CXI, SACLA) without a gain is handed
up as Int16, in its own type; other integer frames are widened to
int32 without a float; only a float frame or a gain other than one
goes through lround(v / gain), exactly as before;
- placement is per panel (start, size, destination steps) instead of a
64-bit destination per stored pixel; a panel square with the image
is copied row by row (PAL: 1440x1440 into 1444x1444 - the two-pixel
margin is Place()'s, kept so the geometry does not move);
- the decode buffers (chunk, inflated, unshuffled) are reused per
thread, the bitshuffle decode is not copied again;
- a chunk whose HDF5 filter_mask is not zero (a filter skipped) is
read by HDF5 instead of being decoded as if it were not;
- the never-set value flags (flag_lessthan/morethan/equal) are gone
from the layout and from the per-pixel loop.
MeasureGain in the CLI now reads Int16 frames as well, so a SACLA run
still has its gain measured.
Tests: JFJochXFELReader_SACLA/_CXI updated for the mode, new
JFJochXFELReader_CXI_Int16 (deflate without shuffle, a chunk with its
filter skipped) and JFJochXTCReader_StoredPixels (synthetic XTC,
stored hand-up and the offset path).
Measured on two PAL-XFEL Rayonix sets (int16, deflate; RTX 5080,
5950X, page cache warm; reports and .hkl identical to the baseline):
main pass, 2000 frames: 296 -> 799 frames/s (pal86),
302 -> 755 frames/s (pal90)
wall, 2000 frames: 26.4 -> 13.6 s, 32.1 -> 19.4 s
(the geometry-refinement steps, which read frames too, halve)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
1 parent
eb47013d4f
commit
95e51f868d
6 files changed
+293
-130
No files matched your search
+129
-9
@@ -2,26 +2,31 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <hdf5.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../reader/JFJochXFELReader.h"
|
||||
#include "../reader/JFJochXTCReader.h"
|
||||
|
||||
// Synthetic SACLA Cheetah and Cheetah CXI files, written with the HDF5 C API the way those pipelines
|
||||
// write them: whole frames as single deflated (and byte-shuffled) chunks, so the direct-chunk path is
|
||||
// what is read.
|
||||
namespace {
|
||||
|
||||
void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector<hsize_t> dims) {
|
||||
void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector<hsize_t> dims,
|
||||
bool shuffle = true) {
|
||||
const hid_t space = H5Screate_simple(static_cast<int>(dims.size()), dims.data(), nullptr);
|
||||
std::vector<hsize_t> chunk = dims;
|
||||
if (chunk.size() == 3)
|
||||
chunk[0] = 1;
|
||||
const hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
|
||||
H5Pset_chunk(dcpl, static_cast<int>(chunk.size()), chunk.data());
|
||||
H5Pset_shuffle(dcpl);
|
||||
if (shuffle)
|
||||
H5Pset_shuffle(dcpl);
|
||||
H5Pset_deflate(dcpl, 4);
|
||||
const hid_t ds = H5Dcreate2(loc, name.c_str(), type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
|
||||
H5Dwrite(ds, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
|
||||
@@ -47,10 +52,17 @@ void WriteScalar(hid_t loc, const std::string &name, double v) {
|
||||
H5Sclose(space);
|
||||
}
|
||||
|
||||
int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row) {
|
||||
// A pixel of the flat image, as the reader hands it out: a 16-bit signed frame without a gain in its own
|
||||
// type, anything else as int32.
|
||||
int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row,
|
||||
CompressedImageMode mode = CompressedImageMode::Int32) {
|
||||
auto raw = reader.GetRawImage(image);
|
||||
REQUIRE(raw);
|
||||
return reinterpret_cast<const int32_t *>(raw->image_buffer.data())[row * raw->image.GetWidth() + col];
|
||||
REQUIRE(raw->image.GetMode() == mode);
|
||||
const size_t i = row * raw->image.GetWidth() + col;
|
||||
if (mode == CompressedImageMode::Int16)
|
||||
return reinterpret_cast<const int16_t *>(raw->image_buffer.data())[i];
|
||||
return reinterpret_cast<const int32_t *>(raw->image_buffer.data())[i];
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -103,12 +115,14 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") {
|
||||
// a whole column: the beam at 1.5, the pixel at column and row 2.
|
||||
CHECK(x.GetBeamX_pxl() == Catch::Approx(1.5));
|
||||
CHECK(x.GetBeamY_pxl() == Catch::Approx(1.5));
|
||||
CHECK(PixelAt(reader, 0, 2, 2) == 20);
|
||||
CHECK(PixelAt(reader, 1, 2, 2) == 21);
|
||||
constexpr auto int16 = CompressedImageMode::Int16;
|
||||
CHECK(PixelAt(reader, 0, 2, 2, int16) == 20);
|
||||
CHECK(PixelAt(reader, 1, 2, 2, int16) == 21);
|
||||
// Second sensor: (fs 0, ss 0) centre at x = 20 - 0.5, y = 0.5; fs runs along +y, ss along -x.
|
||||
CHECK(PixelAt(reader, 0, 21, 2) == -5);
|
||||
CHECK(PixelAt(reader, 0, 21, 3) == 33);
|
||||
CHECK(PixelAt(reader, 0, 20, 2) == 44);
|
||||
CHECK(PixelAt(reader, 0, 21, 2, int16) == -5);
|
||||
CHECK(PixelAt(reader, 0, 21, 3, int16) == 33);
|
||||
CHECK(PixelAt(reader, 0, 20, 2, int16) == 44);
|
||||
CHECK(PixelAt(reader, 0, 0, 0, int16) == 0); // between the panels
|
||||
// The file's mask holds the first sensor's (fs 1, ss 0), at column 3, row 2.
|
||||
const auto &mask = reader.GetDataset()->pixel_mask->GetMask();
|
||||
CHECK(mask[2 * x.GetXPixelsNum() + 3] != 0);
|
||||
@@ -117,6 +131,8 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") {
|
||||
reader.ApplyGain(2.0f);
|
||||
CHECK(reader.CountsInPhotons());
|
||||
CHECK(PixelAt(reader, 0, 2, 2) == 10);
|
||||
CHECK(PixelAt(reader, 0, 21, 2) == -3); // -2.5, rounded away from zero
|
||||
CHECK(PixelAt(reader, 0, 21, 3) == 17); // 16.5, likewise
|
||||
reader.Close();
|
||||
std::filesystem::remove(name);
|
||||
}
|
||||
@@ -182,3 +198,107 @@ TEST_CASE("JFJochXFELReader_CXI_TiledRefused", "[portable]") {
|
||||
CHECK(refusal->find("tiled detector") != std::string::npos);
|
||||
std::filesystem::remove(name);
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochXFELReader_CXI_Int16", "[portable]") {
|
||||
// A single-panel Cheetah CXI file as a Rayonix one is written: 16-bit signed frames, deflated without
|
||||
// a shuffle. The second frame's chunk is stored with its deflate skipped (filter mask bit 0), which the
|
||||
// direct-chunk decoding does not expect: HDF5 reads that one.
|
||||
const std::string name = "xfel_reader_int16.cxi";
|
||||
{
|
||||
const hid_t f = H5Fcreate(name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
|
||||
for (const char *g: {"entry_1", "entry_1/data_1", "entry_1/instrument_1", "entry_1/instrument_1/detector_1",
|
||||
"entry_1/instrument_1/source_1"})
|
||||
H5Gclose(H5Gcreate2(f, g, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT));
|
||||
std::vector<int16_t> frames(2 * 16, 3);
|
||||
frames[1] = 32767; // frame 0, pixel (1, 0)
|
||||
frames[4 + 2] = -4; // frame 0, pixel (2, 1)
|
||||
WriteFrames(f, "entry_1/data_1/data", H5T_NATIVE_INT16, frames.data(), {2, 4, 4}, false);
|
||||
std::vector<int16_t> raw(16, 9);
|
||||
raw[5] = 1234; // frame 1, pixel (1, 1)
|
||||
const hid_t ds = H5Dopen2(f, "entry_1/data_1/data", H5P_DEFAULT);
|
||||
const hsize_t offset[3] = {1, 0, 0};
|
||||
REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 1, offset, raw.size() * sizeof(int16_t), raw.data()) >= 0);
|
||||
H5Dclose(ds);
|
||||
const std::string det = "entry_1/instrument_1/detector_1/";
|
||||
WriteVector(f, det + "x_pixel_size", {100e-6, 100e-6});
|
||||
WriteVector(f, "entry_1/instrument_1/source_1/energy", std::vector<double>(2, 12000.0 * 1.602176634e-19));
|
||||
H5Fclose(f);
|
||||
}
|
||||
|
||||
REQUIRE(JFJochXFELReader::CanReadCXI(name));
|
||||
JFJochXFELReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadCXI({name}));
|
||||
// Corner at (-2, -2): the pixel (fs, ss) lands at column and row 2 + fs, 2 + ss, in its stored type.
|
||||
constexpr auto int16 = CompressedImageMode::Int16;
|
||||
CHECK(PixelAt(reader, 0, 3, 2, int16) == 32767);
|
||||
CHECK(PixelAt(reader, 0, 4, 3, int16) == -4);
|
||||
CHECK(PixelAt(reader, 0, 5, 5, int16) == 3);
|
||||
CHECK(PixelAt(reader, 0, 1, 1, int16) == 0); // the margin
|
||||
CHECK(PixelAt(reader, 1, 3, 3, int16) == 1234);
|
||||
CHECK(PixelAt(reader, 1, 2, 2, int16) == 9);
|
||||
CHECK(reader.GetDataset()->experiment.GetSaturationLimit() == 32767);
|
||||
reader.Close();
|
||||
std::filesystem::remove(name);
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
// An LCLS-I XTC stream of the shape JFJochXTCReader reads: a Configure datagram, then one event (an
|
||||
// L1Accept) per image, each a container holding one 16-bit camera frame (Camera::FrameV1).
|
||||
void WriteXTC(const std::string &name, uint32_t width, uint32_t height, uint32_t pixel_offset,
|
||||
const std::vector<std::vector<uint16_t>> &images) {
|
||||
std::ofstream f(name, std::ios::binary);
|
||||
auto words = [&](std::initializer_list<uint32_t> w) {
|
||||
for (uint32_t v: w)
|
||||
f.write(reinterpret_cast<const char *>(&v), 4);
|
||||
};
|
||||
constexpr uint32_t ID_XTC = 1, ID_FRAME = 2, XTC_HEADER = 20;
|
||||
// clock (2 words), timestamp (2 words, the transition in bits 24-27 of the first), env, Xtc header
|
||||
words({0, 0, 0u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER});
|
||||
const uint32_t frame_extent = XTC_HEADER + 16 + 2 * width * height;
|
||||
for (const auto &image: images) {
|
||||
words({0, 0, 12u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER + frame_extent});
|
||||
words({0, 0, 7, ID_FRAME | (1u << 16), frame_extent});
|
||||
words({width, height, 16, pixel_offset});
|
||||
f.write(reinterpret_cast<const char *>(image.data()), static_cast<std::streamsize>(2 * image.size()));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("JFJochXTCReader_StoredPixels", "[portable]") {
|
||||
// Two streams of 3 x 2 images. Without an offset to take off, the main loop's read hands the pixels up
|
||||
// as stored (UINT16_WIDENED), the same values the decoded int32 image holds; with one, it is taken off.
|
||||
const std::string a = "xtc_reader_a.xtc", b = "xtc_reader_b.xtc", c = "xtc_reader_c.xtc";
|
||||
WriteXTC(a, 3, 2, 0, {{1, 2, 3, 4, 5, 65535}, {7, 7, 7, 7, 7, 7}});
|
||||
WriteXTC(b, 3, 2, 0, {{9, 8, 7, 6, 5, 4}});
|
||||
WriteXTC(c, 3, 2, 10, {{11, 12, 13, 14, 15, 9}});
|
||||
REQUIRE(JFJochXTCReader::CanRead(a));
|
||||
|
||||
JFJochXTCReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles({a, b}));
|
||||
REQUIRE(reader.GetNumberOfImages() == 3);
|
||||
auto raw = reader.GetRawImage(2);
|
||||
REQUIRE(raw);
|
||||
CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::UINT16_WIDENED);
|
||||
std::vector<uint8_t> buffer;
|
||||
auto pixels = reinterpret_cast<const int32_t *>(raw->image.GetUncompressedPtr(buffer));
|
||||
CHECK(pixels[0] == 9);
|
||||
CHECK(pixels[5] == 4);
|
||||
raw = reader.GetRawImage(0);
|
||||
REQUIRE(raw);
|
||||
pixels = reinterpret_cast<const int32_t *>(raw->image.GetUncompressedPtr(buffer));
|
||||
CHECK(pixels[5] == 65535);
|
||||
|
||||
REQUIRE_NOTHROW(reader.ReadFiles({c}));
|
||||
raw = reader.GetRawImage(0);
|
||||
REQUIRE(raw);
|
||||
CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION);
|
||||
CHECK(raw->image.GetMode() == CompressedImageMode::Int32);
|
||||
pixels = reinterpret_cast<const int32_t *>(raw->image_buffer.data());
|
||||
CHECK(pixels[0] == 1);
|
||||
CHECK(pixels[5] == -1);
|
||||
reader.Close();
|
||||
for (const auto &n: {a, b, c})
|
||||
std::filesystem::remove(n);
|
||||
}
|
||||
Reference in new issue
Block a user