rugnux: a _process.h5 carrying its images is typed as the images are stored

Where the input is not a single HDF5 file the process file can link to - several masters read
as one (JFJochMultiReader), CBF, CXI/XFEL input - the _process.h5 carries the images itself.
Its start message was then typed by the experiment, which is pinned to signed 32-bit (the
container images are DECODED into), while the images are passed on as stored: 16-bit signed
from a JUNGFRAU in photons, unsigned 16/32-bit from an EIGER. The writer refuses an image that
contradicts the declared format, so --write-process-h5 on two Cristallina (int16) runs failed
with "Image is 16-bit signed, but the start message declares 32-bit signed".

The pixel format, error and underload values are now taken from the first image as the
reader hands it out, exactly as the linked-HDF5 case takes them from the source files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37

rugnux/Rugnux.cpp

@@ -1034,32 +1034,36 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     // ask for the links only when the source is HDF5, and otherwise write the images into the file -
     // the per-image analysis is the point of it either way, and without pictures nothing downstream
     // could show the frame a reflection came from.
+    // Everything describing the pixels must describe the images the file links to or carries. experiment_
+    // cannot: it is pinned to signed 32-bit, the container the reader DECODES images into, while the images
+    // themselves are passed on as stored - 16-bit signed from a JUNGFRAU in photons, unsigned 16- or 32-bit
+    // from an EIGER. Typed by experiment_, the virtual dataset would convert every value on read, and the
+    // images the file carries would contradict its header (refused by the writer).
+    StoredPixelFormat stored{static_cast<int64_t>(experiment_.GetByteDepthImage() * 8), experiment_.IsPixelSigned()};
     if (auto *hdf5 = dynamic_cast<JFJochHDF5Reader *>(&reader_)) {
         // With the stride: image i of the process file is source image start_image + i * stride, so
         // the linked pictures line up with the per-image analysis written beside them.
         start_message.hdf5_source_data =
             hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride);
-
+        stored = hdf5->GetStoredPixelFormat();
-        // Everything describing the pixels must describe those linked files. experiment_ cannot: it
-        // is pinned to signed 32-bit, the container the reader hands images out in, which would type
-        // the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every
-        // value on read. Only consumers going through the virtual view ever saw it - our own reader
-        // resolves the mapping and opens the source itself.
-        const auto stored = hdf5->GetStoredPixelFormat();
-        start_message.bit_depth_image = stored.bit_depth;
-        // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
-        // the field is present, and on a signed image those land inside the trusted range.
-        start_message.bit_depth_readout.reset();
-        if (!stored.is_signed)
-            start_message.bit_depth_readout = stored.bit_depth;
-        start_message.pixel_signed = stored.is_signed;
-        start_message.error_value = stored.is_signed
-                                    ? -(int64_t(1) << (stored.bit_depth - 1))      // INTx_MIN
-                                    : (int64_t(1) << stored.bit_depth) - 1;        // UINTx_MAX
-        start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
     } else {
         start_message.write_images = true;   // no file to link to; the process file carries them
+        // As the reader hands the images out, which is how they are written.
+        JFJochReaderRawImage first;
+        if (reader_.ReadRawImage(start_image, first))
+            stored = {static_cast<int64_t>(first.image.GetByteDepth() * 8), first.image.IsSigned()};
     }
+    start_message.bit_depth_image = stored.bit_depth;
+    // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
+    // the field is present, and on a signed image those land inside the trusted range.
+    start_message.bit_depth_readout.reset();
+    if (!stored.is_signed)
+        start_message.bit_depth_readout = stored.bit_depth;
+    start_message.pixel_signed = stored.is_signed;
+    start_message.error_value = stored.is_signed
+                                ? -(int64_t(1) << (stored.bit_depth - 1))      // INTx_MIN
+                                : (int64_t(1) << stored.bit_depth) - 1;        // UINTx_MAX
+    start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
 
     // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
     // but the process file reads them. Copy them where the start message is handed on rather than here:

tests/RugnuxTest.cpp

@@ -12,6 +12,7 @@
 #include "../common/ScanResultGenerator.h"
 #include "../writer/FileWriter.h"
 #include "../reader/JFJochHDF5Reader.h"
+#include "../reader/JFJochMultiReader.h"
 #include "../rugnux/Rugnux.h"
 #include "../rugnux/RugnuxCommandLine.h"
 #include "../rugnux/RugnuxOutputFiles.h"
@@ -94,6 +95,50 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") {
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
 
+TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") {
+    // A reader with no HDF5 file to link to (here several inputs read as one) has its images written
+    // into the _process.h5, typed as they are stored - 16-bit signed - not as the 32-bit container
+    // they are decoded into.
+    WriteTestDataset("process_carry_in", 4);
+
+    std::vector<std::unique_ptr<JFJochReader>> inputs;
+    auto input = std::make_unique<JFJochHDF5Reader>();
+    REQUIRE_NOTHROW(input->ReadFile("process_carry_in_master.h5"));
+    inputs.push_back(std::move(input));
+    JFJochMultiReader reader(std::move(inputs));
+    auto dataset = reader.GetDataset();
+    REQUIRE(dataset);
+
+    ProcessConfig config;
+    config.mode = ProcessMode::AzimuthalIntegration;
+    config.nthreads = 2;
+    ProcessFileSink sink("process_carry_out");
+    config.image_sink = &sink;
+
+    Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
+    ProcessResult result;
+    REQUIRE_NOTHROW(result = process.Run());
+    CHECK(result.images_processed == 4);
+
+    {
+        JFJochHDF5Reader out;
+        REQUIRE_NOTHROW(out.ReadFile("process_carry_out_process.h5"));
+        CHECK(out.GetNumberOfImages() == 4);
+        CHECK(out.GetStoredPixelFormat().bit_depth == 16);
+        CHECK(out.GetStoredPixelFormat().is_signed);
+        std::shared_ptr<JFJochReaderImage> img;
+        REQUIRE_NOTHROW(img = out.LoadImage(0));
+        REQUIRE(img);
+        CHECK(img->Image()[100] == 5);
+    }
+
+    reader.Close();
+    remove("process_carry_in_master.h5");
+    remove("process_carry_in_data_000001.h5");
+    remove("process_carry_out_process.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
 TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") {
     WriteTestDataset("process_noout_in", 6);