From d67f10cda1c531df348bcede5777791a7262859d Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 1 Nov 2026 08:56:31 +0100 Subject: [PATCH] rugnux: a _process.h5 carrying its images is typed as the images are stored Where the input is not a single HDF5 file the process file can link to - several masters read as one (JFJochMultiReader), CBF, CXI/XFEL input - the _process.h5 carries the images itself. Its start message was then typed by the experiment, which is pinned to signed 32-bit (the container images are DECODED into), while the images are passed on as stored: 16-bit signed from a JUNGFRAU in photons, unsigned 16/32-bit from an EIGER. The writer refuses an image that contradicts the declared format, so --write-process-h5 on two Cristallina (int16) runs failed with "Image is 16-bit signed, but the start message declares 32-bit signed". The pixel format, error and underload values are now taken from the first image as the reader hands it out, exactly as the linked-HDF5 case takes them from the source files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37 --- rugnux/Rugnux.cpp | 40 +++++++++++++++++++++------------------ tests/RugnuxTest.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index cf9722375..d7fc2d31c 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1034,32 +1034,36 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // ask for the links only when the source is HDF5, and otherwise write the images into the file - // the per-image analysis is the point of it either way, and without pictures nothing downstream // could show the frame a reflection came from. + // Everything describing the pixels must describe the images the file links to or carries. experiment_ + // cannot: it is pinned to signed 32-bit, the container the reader DECODES images into, while the images + // themselves are passed on as stored - 16-bit signed from a JUNGFRAU in photons, unsigned 16- or 32-bit + // from an EIGER. Typed by experiment_, the virtual dataset would convert every value on read, and the + // images the file carries would contradict its header (refused by the writer). + StoredPixelFormat stored{static_cast(experiment_.GetByteDepthImage() * 8), experiment_.IsPixelSigned()}; if (auto *hdf5 = dynamic_cast(&reader_)) { // With the stride: image i of the process file is source image start_image + i * stride, so // the linked pictures line up with the per-image analysis written beside them. start_message.hdf5_source_data = hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride); - - // Everything describing the pixels must describe those linked files. experiment_ cannot: it - // is pinned to signed 32-bit, the container the reader hands images out in, which would type - // the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every - // value on read. Only consumers going through the virtual view ever saw it - our own reader - // resolves the mapping and opens the source itself. - const auto stored = hdf5->GetStoredPixelFormat(); - start_message.bit_depth_image = stored.bit_depth; - // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever - // the field is present, and on a signed image those land inside the trusted range. - start_message.bit_depth_readout.reset(); - if (!stored.is_signed) - start_message.bit_depth_readout = stored.bit_depth; - start_message.pixel_signed = stored.is_signed; - start_message.error_value = stored.is_signed - ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN - : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX - start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; + stored = hdf5->GetStoredPixelFormat(); } else { start_message.write_images = true; // no file to link to; the process file carries them + // As the reader hands the images out, which is how they are written. + JFJochReaderRawImage first; + if (reader_.ReadRawImage(start_image, first)) + stored = {static_cast(first.image.GetByteDepth() * 8), first.image.IsSigned()}; } + start_message.bit_depth_image = stored.bit_depth; + // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever + // the field is present, and on a signed image those land inside the trusted range. + start_message.bit_depth_readout.reset(); + if (!stored.is_signed) + start_message.bit_depth_readout = stored.bit_depth; + start_message.pixel_signed = stored.is_signed; + start_message.error_value = stored.is_signed + ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN + : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX + start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing // but the process file reads them. Copy them where the start message is handed on rather than here: diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index 7114e22f3..dc257c7b9 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -12,6 +12,7 @@ #include "../common/ScanResultGenerator.h" #include "../writer/FileWriter.h" #include "../reader/JFJochHDF5Reader.h" +#include "../reader/JFJochMultiReader.h" #include "../rugnux/Rugnux.h" #include "../rugnux/RugnuxCommandLine.h" #include "../rugnux/RugnuxOutputFiles.h" @@ -94,6 +95,50 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } +TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") { + // A reader with no HDF5 file to link to (here several inputs read as one) has its images written + // into the _process.h5, typed as they are stored - 16-bit signed - not as the 32-bit container + // they are decoded into. + WriteTestDataset("process_carry_in", 4); + + std::vector> inputs; + auto input = std::make_unique(); + REQUIRE_NOTHROW(input->ReadFile("process_carry_in_master.h5")); + inputs.push_back(std::move(input)); + JFJochMultiReader reader(std::move(inputs)); + auto dataset = reader.GetDataset(); + REQUIRE(dataset); + + ProcessConfig config; + config.mode = ProcessMode::AzimuthalIntegration; + config.nthreads = 2; + ProcessFileSink sink("process_carry_out"); + config.image_sink = &sink; + + Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config); + ProcessResult result; + REQUIRE_NOTHROW(result = process.Run()); + CHECK(result.images_processed == 4); + + { + JFJochHDF5Reader out; + REQUIRE_NOTHROW(out.ReadFile("process_carry_out_process.h5")); + CHECK(out.GetNumberOfImages() == 4); + CHECK(out.GetStoredPixelFormat().bit_depth == 16); + CHECK(out.GetStoredPixelFormat().is_signed); + std::shared_ptr img; + REQUIRE_NOTHROW(img = out.LoadImage(0)); + REQUIRE(img); + CHECK(img->Image()[100] == 5); + } + + reader.Close(); + remove("process_carry_in_master.h5"); + remove("process_carry_in_data_000001.h5"); + remove("process_carry_out_process.h5"); + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} + TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") { WriteTestDataset("process_noout_in", 6);