From 95e51f868d0e717001a40da338196f7d00fc04ad Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 21 Oct 2026 10:50:34 +0200 Subject: [PATCH] XFEL reader: integer frames stay integer, one gain, placement by panel DecodeInto no longer turns every stored pixel into a float and back: - the gain is one value (adu_per_photon_), not a float per stored pixel - it always was one value; only --gain or a measured gain set it; - a 16-bit signed frame (Cheetah CXI, SACLA) without a gain is handed up as Int16, in its own type; other integer frames are widened to int32 without a float; only a float frame or a gain other than one goes through lround(v / gain), exactly as before; - placement is per panel (start, size, destination steps) instead of a 64-bit destination per stored pixel; a panel square with the image is copied row by row (PAL: 1440x1440 into 1444x1444 - the two-pixel margin is Place()'s, kept so the geometry does not move); - the decode buffers (chunk, inflated, unshuffled) are reused per thread, the bitshuffle decode is not copied again; - a chunk whose HDF5 filter_mask is not zero (a filter skipped) is read by HDF5 instead of being decoded as if it were not; - the never-set value flags (flag_lessthan/morethan/equal) are gone from the layout and from the per-pixel loop. MeasureGain in the CLI now reads Int16 frames as well, so a SACLA run still has its gain measured. Tests: JFJochXFELReader_SACLA/_CXI updated for the mode, new JFJochXFELReader_CXI_Int16 (deflate without shuffle, a chunk with its filter skipped) and JFJochXTCReader_StoredPixels (synthetic XTC, stored hand-up and the offset path). Measured on two PAL-XFEL Rayonix sets (int16, deflate; RTX 5080, 5950X, page cache warm; reports and .hkl identical to the baseline): main pass, 2000 frames: 296 -> 799 frames/s (pal86), 302 -> 755 frames/s (pal90) wall, 2000 frames: 26.4 -> 13.6 s, 32.1 -> 19.4 s (the geometry-refinement steps, which read frames too, halve) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu --- reader/DetectorLayout.h | 2 - reader/JFJochXFELReader.cpp | 242 ++++++++++++++++++--------------- reader/JFJochXFELReader.h | 29 +++- rugnux/rugnux_cli.cpp | 11 +- tests/CrystFELGeometryTest.cpp | 1 - tests/XFELReaderTest.cpp | 138 +++++++++++++++++-- 6 files changed, 293 insertions(+), 130 deletions(-) diff --git a/reader/DetectorLayout.h b/reader/DetectorLayout.h index 3b992b516..7f594eb4c 100644 --- a/reader/DetectorLayout.h +++ b/reader/DetectorLayout.h @@ -70,8 +70,6 @@ struct Layout { std::string data = "/data/data"; std::optional photon_energy; // eV std::optional wavelength; // Angstrom; used where no photon_energy is stated - // A stored value below, above or at these marks a pixel unusable in that frame. - std::optional flag_lessthan, flag_morethan, flag_equal; }; } // namespace xfel diff --git a/reader/JFJochXFELReader.cpp b/reader/JFJochXFELReader.cpp index d59e524b4..fae829168 100644 --- a/reader/JFJochXFELReader.cpp +++ b/reader/JFJochXFELReader.cpp @@ -407,9 +407,10 @@ JFJochXFELReader::Layout JFJochXFELReader::ReadLayout(HDF5ReadOnlyFile &file, co l.stored_size = H5Tget_size(type); l.stored_float = H5Tget_class(type) == H5T_FLOAT; l.stored_signed = H5Tget_sign(type) == H5T_SGN_2; - ok &= ((H5Tget_class(type) == H5T_INTEGER && (l.stored_size == 2 || l.stored_size == 4)) - || (l.stored_float && l.stored_size == 4)) - && H5Tget_order(type) == H5T_ORDER_LE; + l.plain_type = ((H5Tget_class(type) == H5T_INTEGER && (l.stored_size == 2 || l.stored_size == 4)) + || (l.stored_float && l.stored_size == 4)) + && H5Tget_order(type) == H5T_ORDER_LE; + ok &= l.plain_type; H5Tclose(type); l.deflated = deflate; l.direct_chunks = ok && (deflate || (bitshuffle && l.algorithm != CompressionAlgorithm::NO_COMPRESSION)); @@ -573,9 +574,13 @@ void JFJochXFELReader::Place() { // Everything starts as a gap; the panels' pixels are put on top, less the bad regions. std::vector mask(width_ * height_, 1u << PixelMask::ModuleGapPixelBit); std::vector taken(width_ * height_, 0); - destination_.assign(frame_width_ * frame_height_, -1); - adu_per_photon_.assign(frame_width_ * frame_height_, 1.0f); - counts_in_photons_ = true; + blocks_.clear(); + // One gain for every panel: what the run is given (ReadFiles, ApplyGain) is set on every panel alike. + const auto &p0 = geom_.panels[0]; + counts_in_photons_ = p0.adu_per_photon.has_value() || p0.adu_per_eV.has_value(); + const double gain = p0.adu_per_photon.has_value() ? p0.adu_per_photon.value() + : p0.adu_per_eV.has_value() ? p0.adu_per_eV.value() * photon_energy_eV_ : 1.0; + adu_per_photon_ = static_cast(gain); std::optional saturation; std::vector modules; @@ -585,13 +590,6 @@ void JFJochXFELReader::Place() { if (p.max_fs >= static_cast(frame_width_) || p.max_ss >= static_cast(frame_height_)) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Panel " + p.name + " lies outside the stored frame"); - double gain = 1.0; - if (p.adu_per_photon.has_value()) - gain = p.adu_per_photon.value(); - else if (p.adu_per_eV.has_value()) - gain = p.adu_per_eV.value() * photon_energy_eV_; - else - counts_in_photons_ = false; if (p.max_adu.has_value()) saturation = std::min(saturation.value_or(INT64_MAX), p.max_adu.value() / gain); if (p.no_index) @@ -624,6 +622,9 @@ void JFJochXFELReader::Place() { m.sx = scale * (static_cast(bl.fy) * p.fs_x + static_cast(bl.sy) * p.ss_x); m.sy = scale * (static_cast(bl.fy) * p.fs_y + static_cast(bl.sy) * p.ss_y); modules.push_back(m); + blocks_.push_back({static_cast(p.min_ss) * frame_width_ + static_cast(p.min_fs), w + 1, h + 1, + r0 * static_cast(width_) + c0, bl.fy * static_cast(width_) + bl.fx, + bl.sy * static_cast(width_) + bl.sx}); const int64_t e = p.mask_edge_pixels; for (int64_t ss = p.min_ss; ss <= p.max_ss; ss++) { @@ -648,10 +649,7 @@ void JFJochXFELReader::Place() { if (bad) continue; - const size_t src = ss * frame_width_ + fs; - destination_[src] = static_cast(dst); - adu_per_photon_[src] = static_cast(gain); - mask[dst] = masked_by_file[src] ? (1u << PixelMask::ErrorPixelBit) : 0; + mask[dst] = masked_by_file[ss * frame_width_ + fs] ? (1u << PixelMask::ErrorPixelBit) : 0; } } } @@ -668,7 +666,7 @@ void JFJochXFELReader::Place() { if (!saturation.has_value() && !layout_.stored_float && layout_.stored_size > 0) { const double type_max = std::exp2(8.0 * static_cast(layout_.stored_size) - (layout_.stored_signed ? 1 : 0)) - 1; - saturation = type_max / *std::max_element(adu_per_photon_.begin(), adu_per_photon_.end()); + saturation = type_max / adu_per_photon_; } if (saturation.has_value()) detector.SaturationLimit(static_cast(std::floor(saturation.value()))); @@ -730,86 +728,101 @@ void JFJochXFELReader::Close() { dataset_.reset(); } -template -void JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffer) const { - const auto &frame = frames_.at(image_number); - std::vector stored(frame_width_ * frame_height_); - if (layout_.direct_chunks) { - // Under the lock only where the chunk is (a lookup in its index); the read and the decoding not. - std::shared_ptr raw_file; - uint64_t address = 0, chunk_size = 0; - { - std::unique_lock ul(hdf5_mutex); - HDF5DataSet dataset(FileAt(frame.file), frame.dataset); - const hsize_t coord_3d[3] = {static_cast(frame.index), 0, 0}; - const hsize_t coord_2d[2] = {0, 0}; - unsigned filter_mask = 0; - haddr_t addr = HADDR_UNDEF; - hsize_t size = 0; - if (H5Dget_chunk_info_by_coord(dataset.GetID(), frame.index >= 0 ? coord_3d : coord_2d, &filter_mask, - &addr, &size) < 0 || addr == HADDR_UNDEF || size == 0) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "No stored chunk for " + frame.dataset + " of " + files_[frame.file]); - raw_file = RawFileAt(frame.file); - address = user_block_[frame.file] + addr; - chunk_size = size; - } - std::vector compressed(chunk_size); - raw_file->ReadAt(compressed.data(), compressed.size(), address); - - const size_t n = stored.size(); - const size_t size = layout_.stored_size; - std::vector raw(n * size); - if (layout_.deflated) { - uLongf raw_size = raw.size(); - if (uncompress(raw.data(), &raw_size, compressed.data(), compressed.size()) != Z_OK - || raw_size != raw.size()) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Cannot inflate " + frame.dataset + " of " + files_[frame.file]); - } else { - const CompressedImage image(compressed.data(), compressed.size(), frame_width_, frame_height_, - CalcImageMode(size, layout_.stored_float, layout_.stored_signed), - layout_.algorithm); - std::vector decoded; - std::memcpy(raw.data(), image.GetUncompressedPtr(decoded), raw.size()); - } - if (layout_.shuffled) { - std::vector tmp(raw.size()); - for (size_t i = 0; i < n; i++) - for (size_t b = 0; b < size; b++) - tmp[i * size + b] = raw[b * n + i]; - raw.swap(tmp); - } - for (size_t i = 0; i < n; i++) { - const uint8_t *v = raw.data() + i * size; - if (layout_.stored_float) { - float x; - std::memcpy(&x, v, 4); - stored[i] = x; - } else if (size == 2) { - uint16_t x; - std::memcpy(&x, v, 2); - stored[i] = layout_.stored_signed ? static_cast(static_cast(x)) : x; - } else { - uint32_t x; - std::memcpy(&x, v, 4); - stored[i] = layout_.stored_signed ? static_cast(static_cast(x)) - : static_cast(x); - } - } - } else { +void JFJochXFELReader::ReadStored(const Frame &frame, std::vector &stored) const { + const std::vector start = frame.index >= 0 + ? std::vector{static_cast(frame.index), 0, 0} + : std::vector{0, 0}; + const std::vector count = frame.index >= 0 ? std::vector{1, frame_height_, frame_width_} + : std::vector{frame_height_, frame_width_}; + // Under the lock only where the chunk is (a lookup in its index); the read and the decoding not. + std::shared_ptr raw_file; + uint64_t address = 0, chunk_size = 0; + { std::unique_lock ul(hdf5_mutex); HDF5DataSet dataset(FileAt(frame.file), frame.dataset); - if (frame.index >= 0) - dataset.ReadVector(stored, {static_cast(frame.index), 0, 0}, {1, frame_height_, frame_width_}); - else - dataset.ReadVector(stored, {0, 0}, {frame_height_, frame_width_}); + unsigned filter_mask = 0; + if (layout_.direct_chunks) { + haddr_t addr = HADDR_UNDEF; + hsize_t size = 0; + if (H5Dget_chunk_info_by_coord(dataset.GetID(), start.data(), &filter_mask, &addr, &size) < 0 + || addr == HADDR_UNDEF || size == 0) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "No stored chunk for " + frame.dataset + " of " + files_[frame.file]); + address = addr; + chunk_size = size; + } + // A chunk stored with one of its filters skipped (its filter mask) is not the encoding decoded + // below: HDF5 reads it, as it reads a frame that is not stored as one chunk. + if (!layout_.direct_chunks || filter_mask != 0) { + if (layout_.plain_type) + dataset.ReadVectorToU8(stored, start, count); + else { + std::vector v(frame_width_ * frame_height_); + dataset.ReadVector(v, start, count); + stored.resize(v.size() * sizeof(float)); + std::memcpy(stored.data(), v.data(), stored.size()); + } + return; + } + raw_file = RawFileAt(frame.file); + address += user_block_[frame.file]; // HDF5's chunk addresses count from the end of the user block } + thread_local std::vector compressed, shuffled; + compressed.resize(chunk_size); + raw_file->ReadAt(compressed.data(), compressed.size(), address); + + const size_t n = frame_width_ * frame_height_; + const size_t size = layout_.stored_size; + if (layout_.deflated) { + std::vector &inflated = layout_.shuffled ? shuffled : stored; + inflated.resize(n * size); + uLongf inflated_size = inflated.size(); + if (uncompress(inflated.data(), &inflated_size, compressed.data(), compressed.size()) != Z_OK + || inflated_size != inflated.size()) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Cannot inflate " + frame.dataset + " of " + files_[frame.file]); + if (layout_.shuffled) { + stored.resize(n * size); + for (size_t i = 0; i < n; i++) + for (size_t b = 0; b < size; b++) + stored[i * size + b] = shuffled[b * n + i]; + } + } else { + const CompressedImage image(compressed.data(), compressed.size(), frame_width_, frame_height_, + CalcImageMode(size, layout_.stored_float, layout_.stored_signed), + layout_.algorithm); + image.GetUncompressed(stored); + } +} + +template +void JFJochXFELReader::PlaceInto(const In *in, const std::vector &masked, Out err, Buffer &buffer, + Convert convert) const { + buffer.resize(width_ * height_ * sizeof(Out)); + auto out = reinterpret_cast(buffer.data()); + std::fill(out, out + width_ * height_, Out{0}); // between the panels + for (const auto &b: blocks_) + for (int64_t ss = 0; ss < b.height; ss++) { + const size_t src = b.src + static_cast(ss) * frame_width_; + Out *dst = out + b.dst + ss * b.step_ss; + if (masked.empty() && b.step_fs == 1) + std::transform(in + src, in + src + b.width, dst, [&](In v) { return static_cast(convert(v)); }); + else + for (int64_t fs = 0; fs < b.width; fs++) + dst[fs * b.step_fs] = !masked.empty() && masked[src + fs] ? err : static_cast(convert(in[src + fs])); + } +} + +template +CompressedImage JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffer) const { + const auto &frame = frames_.at(image_number); + thread_local std::vector stored; + ReadStored(frame, stored); // A pixel a per-frame mask marks in this frame is handed out as no measurement. std::vector masked; if (!frame_masks_.empty() && frame.index >= 0) { - masked.assign(stored.size(), 0); + masked.assign(frame_width_ * frame_height_, 0); std::unique_lock ul(hdf5_mutex); for (const auto &m: frame_masks_) { const auto v = FileAt(frame.file).ReadVector(m.data, {static_cast(frame.index), 0, 0}, @@ -820,19 +833,36 @@ void JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffer) const { } } - // And so is one whose value the layout flags as no measurement. - const auto flagged = [this](float v) { - return (geom_.flag_lessthan && v < *geom_.flag_lessthan) || (geom_.flag_morethan && v > *geom_.flag_morethan) - || (geom_.flag_equal && v == *geom_.flag_equal); + const float gain = adu_per_photon_; + const auto photons = [gain](auto v) { return static_cast(std::lround(static_cast(v) / gain)); }; + const auto as_is = [](auto v) { return v; }; + const auto image = [&](CompressedImageMode mode) { + return CompressedImage(buffer.data(), buffer.size(), width_, height_, mode, CompressionAlgorithm::NO_COMPRESSION); + }; + const auto integers = [&](auto convert) { + const uint8_t *p = stored.data(); + if (layout_.stored_size == 2 && layout_.stored_signed) + PlaceInto(reinterpret_cast(p), masked, INT32_MIN, buffer, convert); + else if (layout_.stored_size == 2) + PlaceInto(reinterpret_cast(p), masked, INT32_MIN, buffer, convert); + else if (layout_.stored_signed) + PlaceInto(reinterpret_cast(p), masked, INT32_MIN, buffer, convert); + else + PlaceInto(reinterpret_cast(p), masked, INT32_MIN, buffer, convert); }; - buffer.assign(width_ * height_ * sizeof(int32_t), 0); - auto out = reinterpret_cast(buffer.data()); - for (size_t i = 0; i < stored.size(); i++) - if (destination_[i] >= 0) - out[destination_[i]] = (!masked.empty() && masked[i]) || flagged(stored[i]) - ? INT32_MIN - : static_cast(std::lround(stored[i] / adu_per_photon_[i])); + // Floats become photons, rounded. Integers stay integers: divided only by a gain other than one, and a + // 16-bit signed frame - as Cheetah writes - is handed out in its own type. + if (layout_.stored_float || !layout_.plain_type) + PlaceInto(reinterpret_cast(stored.data()), masked, INT32_MIN, buffer, photons); + else if (gain != 1.0f) + integers(photons); + else if (layout_.stored_size == 2 && layout_.stored_signed) { + PlaceInto(reinterpret_cast(stored.data()), masked, INT16_MIN, buffer, as_is); + return image(CompressedImageMode::Int16); + } else + integers([](auto v) { return static_cast(v); }); + return image(CompressedImageMode::Int32); } bool JFJochXFELReader::LoadImage_i(std::shared_ptr &dataset, @@ -843,9 +873,7 @@ bool JFJochXFELReader::LoadImage_i(std::shared_ptr &dataset (void) update_dataset; if (!dataset || image_number < 0 || static_cast(image_number) >= frames_.size()) return false; - DecodeInto(image_number, buffer); - message.image = CompressedImage(buffer.data(), buffer.size(), width_, height_, - CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION); + message.image = DecodeInto(image_number, buffer); message.number = image_number; return true; } @@ -853,9 +881,7 @@ bool JFJochXFELReader::LoadImage_i(std::shared_ptr &dataset bool JFJochXFELReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) { if (image_number < 0 || static_cast(image_number) >= frames_.size()) return false; - DecodeInto(image_number, image.image_buffer); - image.image = CompressedImage(image.image_buffer.data(), image.image_buffer.size(), width_, height_, - CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION); + image.image = DecodeInto(image_number, image.image_buffer); return true; } diff --git a/reader/JFJochXFELReader.h b/reader/JFJochXFELReader.h index 2f9059b56..84fb09923 100644 --- a/reader/JFJochXFELReader.h +++ b/reader/JFJochXFELReader.h @@ -29,9 +29,9 @@ class HDF5ReadOnlyFile; // Each panel is put on the flat image whole, as a block, its axes turned to the nearest image axis and its // first pixel at a whole pixel, with everything between the panels and every masked or bad region masked. // Where the panel really lies (the fraction of a pixel, a small turn) is its module's placement on the -// detector plane (DetectorModules), which the run maps its spots and predictions through. Pixel values are converted to photons -// with the gain where one is known (the geometry's, or one given or measured later), and are otherwise -// left in detector units. +// detector plane (DetectorModules), which the run maps its spots and predictions through. Pixel values are +// converted to photons with the gain where one is given or measured, one value for every panel, and are +// otherwise left in detector units: integers then stay integers, a 16-bit signed frame in its own type. // // One wavelength and one camera length hold for the whole run: where the file states one per shot, it // is the mean over every frame of the run. @@ -61,9 +61,16 @@ class JFJochXFELReader : public JFJochReader { size_t frame_width_ = 0, frame_height_ = 0; // the stored frame size_t width_ = 0, height_ = 0; // the flat image - // For every stored pixel, where it lands on the flat image (-1: nowhere) and what one photon reads. - std::vector destination_; - std::vector adu_per_photon_; + // Where each panel's pixels land on the flat image: its first stored pixel, its size, and the flat-image + // index of its first pixel and the steps of +1 fs and +1 ss there. + struct Block { + size_t src = 0; + int64_t width = 0, height = 0; + int64_t dst = 0, step_fs = 1, step_ss = 0; + }; + std::vector blocks_; + // What one photon reads, the same on every panel; 1 where none is known. + float adu_per_photon_ = 1.0f; bool counts_in_photons_ = false; double clen_m_ = 0, distance_mm_ = 0, photon_energy_eV_ = 0; // Masks stored with one plane per frame (a Cheetah CXI's per-shot mask), applied as each frame is read. @@ -80,6 +87,9 @@ class JFJochXFELReader : public JFJochReader { CompressionAlgorithm algorithm = CompressionAlgorithm::NO_COMPRESSION; // where not deflated bool stored_float = false, stored_signed = false; size_t stored_size = 0; + // A 2- or 4-byte little-endian integer or a 4-byte float, read as stored; any other type HDF5 reads + // as float. + bool plain_type = false; bool operator==(const Layout &) const = default; }; Layout layout_; @@ -103,8 +113,13 @@ class JFJochXFELReader : public JFJochReader { int64_t image_number, bool update_dataset) override; + // The frame's pixels as stored (plain_type), or as float, into stored. + void ReadStored(const Frame &frame, std::vector &stored) const; + // The stored pixels placed on the flat image as Out, each through convert; a pixel masked marks is err. + template + void PlaceInto(const In *in, const std::vector &masked, Out err, Buffer &buffer, Convert convert) const; template - void DecodeInto(int64_t image_number, Buffer &buffer) const; + CompressedImage DecodeInto(int64_t image_number, Buffer &buffer) const; public: JFJochXFELReader(); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 4ba3045b2..32629a6ee 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -726,12 +726,17 @@ std::optional MeasureGain(JFJochReader &reader) { std::vector buffer; std::vector pixels; for (size_t i = 0; i < frames; i++) { - if (!reader.ReadRawImage(static_cast(i * n / frames), raw) - || raw.image.GetMode() != CompressedImageMode::Int32) + if (!reader.ReadRawImage(static_cast(i * n / frames), raw)) continue; const size_t w = raw.image.GetWidth(), h = raw.image.GetHeight(); pixels.resize(w * h); - std::memcpy(pixels.data(), raw.image.GetUncompressedPtr(buffer), w * h * sizeof(int32_t)); + const uint8_t *ptr = raw.image.GetUncompressedPtr(buffer); + if (raw.image.GetMode() == CompressedImageMode::Int32) + std::memcpy(pixels.data(), ptr, w * h * sizeof(int32_t)); + else if (raw.image.GetMode() == CompressedImageMode::Int16) + std::copy_n(reinterpret_cast(ptr), w * h, pixels.begin()); + else + continue; levels.push_back(background_gain::MeasureFrame(pixels, w, h, dataset->pixel_mask->GetMask(), dataset->experiment.GetSaturationLimit())); } diff --git a/tests/CrystFELGeometryTest.cpp b/tests/CrystFELGeometryTest.cpp index baa2a7235..adfc2a419 100644 --- a/tests/CrystFELGeometryTest.cpp +++ b/tests/CrystFELGeometryTest.cpp @@ -30,7 +30,6 @@ TEST_CASE("CrystFELGeometry_OldSyntax", "[CrystFELGeometry][portable]") { // Only the module geometry is read: gain, photon energy, value flags and bad regions are ignored. CHECK(!geom.panels[0].adu_per_photon.has_value()); CHECK(!geom.photon_energy.has_value()); - CHECK(!geom.flag_lessthan.has_value()); CHECK(geom.bad.empty()); CHECK(std::find(ignored.begin(), ignored.end(), "photon_energy") != ignored.end()); } diff --git a/tests/XFELReaderTest.cpp b/tests/XFELReaderTest.cpp index 8bd0caa49..16c63bc63 100644 --- a/tests/XFELReaderTest.cpp +++ b/tests/XFELReaderTest.cpp @@ -2,26 +2,31 @@ // SPDX-License-Identifier: GPL-3.0-only #include +#include #include +#include #include #include #include #include "../reader/JFJochXFELReader.h" +#include "../reader/JFJochXTCReader.h" // Synthetic SACLA Cheetah and Cheetah CXI files, written with the HDF5 C API the way those pipelines // write them: whole frames as single deflated (and byte-shuffled) chunks, so the direct-chunk path is // what is read. namespace { -void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector dims) { +void WriteFrames(hid_t loc, const std::string &name, hid_t type, const void *data, std::vector dims, + bool shuffle = true) { const hid_t space = H5Screate_simple(static_cast(dims.size()), dims.data(), nullptr); std::vector chunk = dims; if (chunk.size() == 3) chunk[0] = 1; const hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); H5Pset_chunk(dcpl, static_cast(chunk.size()), chunk.data()); - H5Pset_shuffle(dcpl); + if (shuffle) + H5Pset_shuffle(dcpl); H5Pset_deflate(dcpl, 4); const hid_t ds = H5Dcreate2(loc, name.c_str(), type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite(ds, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); @@ -47,10 +52,17 @@ void WriteScalar(hid_t loc, const std::string &name, double v) { H5Sclose(space); } -int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row) { +// A pixel of the flat image, as the reader hands it out: a 16-bit signed frame without a gain in its own +// type, anything else as int32. +int32_t PixelAt(JFJochXFELReader &reader, int64_t image, size_t col, size_t row, + CompressedImageMode mode = CompressedImageMode::Int32) { auto raw = reader.GetRawImage(image); REQUIRE(raw); - return reinterpret_cast(raw->image_buffer.data())[row * raw->image.GetWidth() + col]; + REQUIRE(raw->image.GetMode() == mode); + const size_t i = row * raw->image.GetWidth() + col; + if (mode == CompressedImageMode::Int16) + return reinterpret_cast(raw->image_buffer.data())[i]; + return reinterpret_cast(raw->image_buffer.data())[i]; } } // namespace @@ -103,12 +115,14 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") { // a whole column: the beam at 1.5, the pixel at column and row 2. CHECK(x.GetBeamX_pxl() == Catch::Approx(1.5)); CHECK(x.GetBeamY_pxl() == Catch::Approx(1.5)); - CHECK(PixelAt(reader, 0, 2, 2) == 20); - CHECK(PixelAt(reader, 1, 2, 2) == 21); + constexpr auto int16 = CompressedImageMode::Int16; + CHECK(PixelAt(reader, 0, 2, 2, int16) == 20); + CHECK(PixelAt(reader, 1, 2, 2, int16) == 21); // Second sensor: (fs 0, ss 0) centre at x = 20 - 0.5, y = 0.5; fs runs along +y, ss along -x. - CHECK(PixelAt(reader, 0, 21, 2) == -5); - CHECK(PixelAt(reader, 0, 21, 3) == 33); - CHECK(PixelAt(reader, 0, 20, 2) == 44); + CHECK(PixelAt(reader, 0, 21, 2, int16) == -5); + CHECK(PixelAt(reader, 0, 21, 3, int16) == 33); + CHECK(PixelAt(reader, 0, 20, 2, int16) == 44); + CHECK(PixelAt(reader, 0, 0, 0, int16) == 0); // between the panels // The file's mask holds the first sensor's (fs 1, ss 0), at column 3, row 2. const auto &mask = reader.GetDataset()->pixel_mask->GetMask(); CHECK(mask[2 * x.GetXPixelsNum() + 3] != 0); @@ -117,6 +131,8 @@ TEST_CASE("JFJochXFELReader_SACLA", "[portable]") { reader.ApplyGain(2.0f); CHECK(reader.CountsInPhotons()); CHECK(PixelAt(reader, 0, 2, 2) == 10); + CHECK(PixelAt(reader, 0, 21, 2) == -3); // -2.5, rounded away from zero + CHECK(PixelAt(reader, 0, 21, 3) == 17); // 16.5, likewise reader.Close(); std::filesystem::remove(name); } @@ -182,3 +198,107 @@ TEST_CASE("JFJochXFELReader_CXI_TiledRefused", "[portable]") { CHECK(refusal->find("tiled detector") != std::string::npos); std::filesystem::remove(name); } + +TEST_CASE("JFJochXFELReader_CXI_Int16", "[portable]") { + // A single-panel Cheetah CXI file as a Rayonix one is written: 16-bit signed frames, deflated without + // a shuffle. The second frame's chunk is stored with its deflate skipped (filter mask bit 0), which the + // direct-chunk decoding does not expect: HDF5 reads that one. + const std::string name = "xfel_reader_int16.cxi"; + { + const hid_t f = H5Fcreate(name.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + for (const char *g: {"entry_1", "entry_1/data_1", "entry_1/instrument_1", "entry_1/instrument_1/detector_1", + "entry_1/instrument_1/source_1"}) + H5Gclose(H5Gcreate2(f, g, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)); + std::vector frames(2 * 16, 3); + frames[1] = 32767; // frame 0, pixel (1, 0) + frames[4 + 2] = -4; // frame 0, pixel (2, 1) + WriteFrames(f, "entry_1/data_1/data", H5T_NATIVE_INT16, frames.data(), {2, 4, 4}, false); + std::vector raw(16, 9); + raw[5] = 1234; // frame 1, pixel (1, 1) + const hid_t ds = H5Dopen2(f, "entry_1/data_1/data", H5P_DEFAULT); + const hsize_t offset[3] = {1, 0, 0}; + REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 1, offset, raw.size() * sizeof(int16_t), raw.data()) >= 0); + H5Dclose(ds); + const std::string det = "entry_1/instrument_1/detector_1/"; + WriteVector(f, det + "x_pixel_size", {100e-6, 100e-6}); + WriteVector(f, "entry_1/instrument_1/source_1/energy", std::vector(2, 12000.0 * 1.602176634e-19)); + H5Fclose(f); + } + + REQUIRE(JFJochXFELReader::CanReadCXI(name)); + JFJochXFELReader reader; + REQUIRE_NOTHROW(reader.ReadCXI({name})); + // Corner at (-2, -2): the pixel (fs, ss) lands at column and row 2 + fs, 2 + ss, in its stored type. + constexpr auto int16 = CompressedImageMode::Int16; + CHECK(PixelAt(reader, 0, 3, 2, int16) == 32767); + CHECK(PixelAt(reader, 0, 4, 3, int16) == -4); + CHECK(PixelAt(reader, 0, 5, 5, int16) == 3); + CHECK(PixelAt(reader, 0, 1, 1, int16) == 0); // the margin + CHECK(PixelAt(reader, 1, 3, 3, int16) == 1234); + CHECK(PixelAt(reader, 1, 2, 2, int16) == 9); + CHECK(reader.GetDataset()->experiment.GetSaturationLimit() == 32767); + reader.Close(); + std::filesystem::remove(name); +} + +namespace { + +// An LCLS-I XTC stream of the shape JFJochXTCReader reads: a Configure datagram, then one event (an +// L1Accept) per image, each a container holding one 16-bit camera frame (Camera::FrameV1). +void WriteXTC(const std::string &name, uint32_t width, uint32_t height, uint32_t pixel_offset, + const std::vector> &images) { + std::ofstream f(name, std::ios::binary); + auto words = [&](std::initializer_list w) { + for (uint32_t v: w) + f.write(reinterpret_cast(&v), 4); + }; + constexpr uint32_t ID_XTC = 1, ID_FRAME = 2, XTC_HEADER = 20; + // clock (2 words), timestamp (2 words, the transition in bits 24-27 of the first), env, Xtc header + words({0, 0, 0u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER}); + const uint32_t frame_extent = XTC_HEADER + 16 + 2 * width * height; + for (const auto &image: images) { + words({0, 0, 12u << 24, 0, 0, 0, 0, 0, ID_XTC, XTC_HEADER + frame_extent}); + words({0, 0, 7, ID_FRAME | (1u << 16), frame_extent}); + words({width, height, 16, pixel_offset}); + f.write(reinterpret_cast(image.data()), static_cast(2 * image.size())); + } +} + +} // namespace + +TEST_CASE("JFJochXTCReader_StoredPixels", "[portable]") { + // Two streams of 3 x 2 images. Without an offset to take off, the main loop's read hands the pixels up + // as stored (UINT16_WIDENED), the same values the decoded int32 image holds; with one, it is taken off. + const std::string a = "xtc_reader_a.xtc", b = "xtc_reader_b.xtc", c = "xtc_reader_c.xtc"; + WriteXTC(a, 3, 2, 0, {{1, 2, 3, 4, 5, 65535}, {7, 7, 7, 7, 7, 7}}); + WriteXTC(b, 3, 2, 0, {{9, 8, 7, 6, 5, 4}}); + WriteXTC(c, 3, 2, 10, {{11, 12, 13, 14, 15, 9}}); + REQUIRE(JFJochXTCReader::CanRead(a)); + + JFJochXTCReader reader; + REQUIRE_NOTHROW(reader.ReadFiles({a, b})); + REQUIRE(reader.GetNumberOfImages() == 3); + auto raw = reader.GetRawImage(2); + REQUIRE(raw); + CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::UINT16_WIDENED); + std::vector buffer; + auto pixels = reinterpret_cast(raw->image.GetUncompressedPtr(buffer)); + CHECK(pixels[0] == 9); + CHECK(pixels[5] == 4); + raw = reader.GetRawImage(0); + REQUIRE(raw); + pixels = reinterpret_cast(raw->image.GetUncompressedPtr(buffer)); + CHECK(pixels[5] == 65535); + + REQUIRE_NOTHROW(reader.ReadFiles({c})); + raw = reader.GetRawImage(0); + REQUIRE(raw); + CHECK(raw->image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION); + CHECK(raw->image.GetMode() == CompressedImageMode::Int32); + pixels = reinterpret_cast(raw->image_buffer.data()); + CHECK(pixels[0] == 1); + CHECK(pixels[5] == -1); + reader.Close(); + for (const auto &n: {a, b, c}) + std::filesystem::remove(n); +}