rugnux: a _process.h5 of an input it cannot link to holds no image data

Where the input is anything but a single HDF5 master the process file can link to (several masters
read as one, CBF, XTC, CXI, ...), the _process.h5 no longer copies the images into
/entry/data/data: the copy made the file as large as the data. It now holds only the per-image
analysis, and the earlier fix (d67f10cda), which typed that copy as the images are stored, goes
with it - the int16 multi-master CrMX run that failed there writes its file without images.

The reader takes such a file (no /entry/data/data, no linked data files) as one with
/entry/detector/number images: the master's per-image analysis is read as for an integrated master
and the reflections as always, so --mode scale reads it like any other, while LoadImage and
ReadRawImage return no image instead of failing on the missing pixels (the viewer then shows none).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37

docs/HDF5.md

@@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data.
 
 This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead
 of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data`
-VDS still resolves to the original images when they are available. `rugnux` can also process
+VDS still resolves to the original images when they are available. Where the input is anything but a
+single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no
+`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process
 an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set.
 
 ## 3. NXmx-standard content

reader/HDF5MetadataSource.cpp

@@ -538,6 +538,75 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
         // Of a multichannel master; only its first channel is read
         std::string first_channel;
 
+        // The per-image analysis and metadata a master holds itself (VDS, integrated, or a _process.h5 that
+        // holds no images at all).
+        auto read_master_per_image = [&] {
+        if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"))
+            dataset->efficiency = master_file->ReadVector<float>(
+                "/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image");
+        else
+            dataset->efficiency = std::vector<float>(number_of_images, 1.0);
+
+        if (master_file->Exists("/entry/roi"))
+            dataset->roi = master_file->FindLeafs("/entry/roi");
+
+        for (const auto &s: dataset->roi) {
+            dataset->roi_max.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/max"));
+            dataset->roi_sum.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum"));
+            dataset->roi_sum_sq.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum_sq"));
+            dataset->roi_npixel.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/npixel"));
+            dataset->roi_x.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/x"));
+            dataset->roi_y.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/y"));
+        }
+
+        if (master_file->Exists("/entry/MX")) {
+            if (master_file->Exists("/entry/MX/peakCountUnfiltered"))
+                dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/peakCountUnfiltered");
+            else
+                dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/nPeaks");
+
+            dataset->spot_count_low_res = master_file->ReadOptVector<float>("/entry/MX/peakCountLowRes");
+            dataset->spot_count_indexed = master_file->ReadOptVector<float>("/entry/MX/peakCountIndexed");
+            dataset->spot_count_ice_rings = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingRes");
+            dataset->spot_count_ice_control = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingControl");
+
+            dataset->indexing_result = master_file->ReadOptVector<float>("/entry/MX/imageIndexed");
+            dataset->bkg_estimate = master_file->ReadOptVector<float>("/entry/MX/bkgEstimate");
+            dataset->spindle_blind_fraction = master_file->ReadOptVector<float>("/entry/MX/spindleBlindFraction");
+            dataset->ice_ring_score = master_file->ReadOptVector<float>("/entry/MX/iceRingScore");
+            dataset->resolution_estimate = master_file->ReadOptVector<float>("/entry/MX/resolutionEstimate");
+            dataset->profile_radius = master_file->ReadOptVector<float>("/entry/MX/profileRadius");
+            // Master files write indexedLatticeCount; data files / the per-file MX
+            // plugin use indexingLatticeCount. Accept either for backward compatibility.
+            dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexedLatticeCount");
+            if (dataset->indexing_lattice_count.empty())
+                dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexingLatticeCount");
+            dataset->mosaicity_deg = master_file->ReadOptVector<float>("/entry/MX/mosaicity");
+            dataset->b_factor = master_file->ReadOptVector<float>("/entry/MX/bFactor");
+            dataset->image_scale_factor = master_file->ReadOptVector<float>("/entry/MX/imageScaleFactor");
+            dataset->image_scale_cc = master_file->ReadOptVector<float>("/entry/MX/imageScaleCC");
+            dataset->integrated_reflections = master_file->ReadOptVector<float>("/entry/MX/integratedReflections");
+            dataset->sweep_quality = master_file->ReadOptVector<uint8_t>("/entry/MX/sweepQuality");
+            if (master_file->Exists("/entry/MX/sweepQualityReasons")) {
+                const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons");
+                for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
+                    dataset->sweep_quality_reasons.push_back(
+                        master_file->ReadElement<std::string>("/entry/MX/sweepQualityReasons", i)
+                                .value_or(""));
+            }
+            dataset->frame_disposition = master_file->ReadOptVector<uint8_t>("/entry/MX/frameDisposition");
+            if (master_file->Exists("/entry/MX/frameDispositionCodes")) {
+                const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes");
+                for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
+                    dataset->frame_disposition_codes.push_back(
+                        master_file->ReadElement<std::string>("/entry/MX/frameDispositionCodes", i)
+                                .value_or(""));
+            }
+        }
+        if (master_file->Exists("/entry/image"))
+            dataset->max_value = master_file->ReadOptVector<int64_t>("/entry/image/max_value");
+        };
+
         if (master_file->Exists("/entry/data/data")) {
             HDF5DataSet data_dataset(*master_file, "/entry/data/data");
             HDF5Dcpl dcpl(data_dataset);
@@ -554,70 +623,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
             if (data_layout == HDF5DataSetLayout::VIRTUAL)
                 vds_data_mappings = ReadVDSImageMappings(*master_file, "/entry/data/data");
 
-            if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"))
-                dataset->efficiency = master_file->ReadVector<float>(
-                    "/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image");
-            else
-                dataset->efficiency = std::vector<float>(number_of_images, 1.0);
-
-            if (master_file->Exists("/entry/roi"))
-                dataset->roi = master_file->FindLeafs("/entry/roi");
-
-            for (const auto &s: dataset->roi) {
-                dataset->roi_max.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/max"));
-                dataset->roi_sum.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum"));
-                dataset->roi_sum_sq.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum_sq"));
-                dataset->roi_npixel.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/npixel"));
-                dataset->roi_x.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/x"));
-                dataset->roi_y.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/y"));
-            }
-
-            if (master_file->Exists("/entry/MX")) {
-                if (master_file->Exists("/entry/MX/peakCountUnfiltered"))
-                    dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/peakCountUnfiltered");
-                else
-                    dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/nPeaks");
-
-                dataset->spot_count_low_res = master_file->ReadOptVector<float>("/entry/MX/peakCountLowRes");
-                dataset->spot_count_indexed = master_file->ReadOptVector<float>("/entry/MX/peakCountIndexed");
-                dataset->spot_count_ice_rings = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingRes");
-                dataset->spot_count_ice_control = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingControl");
-
-                dataset->indexing_result = master_file->ReadOptVector<float>("/entry/MX/imageIndexed");
-                dataset->bkg_estimate = master_file->ReadOptVector<float>("/entry/MX/bkgEstimate");
-                dataset->spindle_blind_fraction = master_file->ReadOptVector<float>("/entry/MX/spindleBlindFraction");
-                dataset->ice_ring_score = master_file->ReadOptVector<float>("/entry/MX/iceRingScore");
-                dataset->resolution_estimate = master_file->ReadOptVector<float>("/entry/MX/resolutionEstimate");
-                dataset->profile_radius = master_file->ReadOptVector<float>("/entry/MX/profileRadius");
-                // Master files write indexedLatticeCount; data files / the per-file MX
-                // plugin use indexingLatticeCount. Accept either for backward compatibility.
-                dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexedLatticeCount");
-                if (dataset->indexing_lattice_count.empty())
-                    dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexingLatticeCount");
-                dataset->mosaicity_deg = master_file->ReadOptVector<float>("/entry/MX/mosaicity");
-                dataset->b_factor = master_file->ReadOptVector<float>("/entry/MX/bFactor");
-                dataset->image_scale_factor = master_file->ReadOptVector<float>("/entry/MX/imageScaleFactor");
-                dataset->image_scale_cc = master_file->ReadOptVector<float>("/entry/MX/imageScaleCC");
-                dataset->integrated_reflections = master_file->ReadOptVector<float>("/entry/MX/integratedReflections");
-                dataset->sweep_quality = master_file->ReadOptVector<uint8_t>("/entry/MX/sweepQuality");
-                if (master_file->Exists("/entry/MX/sweepQualityReasons")) {
-                    const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons");
-                    for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
-                        dataset->sweep_quality_reasons.push_back(
-                            master_file->ReadElement<std::string>("/entry/MX/sweepQualityReasons", i)
-                                    .value_or(""));
-                }
-                dataset->frame_disposition = master_file->ReadOptVector<uint8_t>("/entry/MX/frameDisposition");
-                if (master_file->Exists("/entry/MX/frameDispositionCodes")) {
-                    const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes");
-                    for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
-                        dataset->frame_disposition_codes.push_back(
-                            master_file->ReadElement<std::string>("/entry/MX/frameDispositionCodes", i)
-                                    .value_or(""));
-                }
-            }
-            if (master_file->Exists("/entry/image"))
-                dataset->max_value = master_file->ReadOptVector<int64_t>("/entry/image/max_value");
+            read_master_per_image();
 
             format = FileWriterFormat::NXmxVDS;
         } else if (master_file->Exists("/entry/data/data_000001")) {
@@ -794,6 +800,13 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
             image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector");
             image_size_y = master_file->GetInt("/entry/instrument/detector/detectorSpecific/y_pixels_in_detector");
             number_of_images = 0;
+            // A _process.h5 of an input it could not link to holds the per-image analysis without the
+            // images: as many images as it has per-image entries, none of them with pixels.
+            if (master_file->Exists("/entry/detector/number")) {
+                number_of_images = master_file->GetDimension("/entry/detector/number")[0];
+                images_per_file = number_of_images;
+                read_master_per_image();
+            }
         }
 
         if (master_file->Exists("/entry/MX")) {

reader/JFJochHDF5Reader.cpp

@@ -15,6 +15,7 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) {
         auto metadata = std::make_shared<HDF5MetadataSource>();
         auto open_result = metadata->Open(filename, default_experiment);
 
+        has_pixels = open_result.image_layout.format != FileWriterFormat::NoFile;
         image_source_.Configure(std::move(open_result.image_layout));
         // Original file: per-image metadata is co-located with the pixels.
         metadata->UseImageSourceForMetadata(&image_source_);
@@ -67,6 +68,8 @@ bool JFJochHDF5Reader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &
         if (!active_metadata_)
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                                   "Cannot load image if file not loaded");
+        if (!has_pixels)
+            return false;
 
         auto loc = GetImageLocation(image_number);
         chunk = image_source_.PrepareDirectRead(loc);
@@ -88,7 +91,7 @@ bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset
     std::unique_lock ul(hdf5_mutex);
     (void) update_dataset;
 
-    if (!dataset)
+    if (!dataset || !has_pixels)
         return false;
     if (!active_metadata_)
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,

reader/JFJochHDF5Reader.h

@@ -31,6 +31,9 @@ class JFJochHDF5Reader : public JFJochReader {
     std::shared_ptr<HDF5MetadataSource> active_metadata_;
     std::string active_snapshot_;
     size_t number_of_images = 0;
+    // False for a file of per-image analysis without the images (a _process.h5 of an input it could
+    // not link to): its images have metadata, but no pixels to load.
+    bool has_pixels = false;
 
     bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
                      DataMessage &message,

rugnux/Rugnux.cpp

@@ -1035,41 +1035,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     start_message.file_format = FileWriterFormat::NXmxIntegrated;
     start_message.write_master_file = true;
     start_message.write_images = false;
-    // The process file LINKS to the original images rather than writing its own, which only an HDF5
-    // input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs. So
-    // ask for the links only when the source is HDF5, and otherwise write the images into the file -
-    // the per-image analysis is the point of it either way, and without pictures nothing downstream
-    // could show the frame a reflection came from.
-    // Everything describing the pixels must describe the images the file links to or carries. experiment_
-    // cannot: it is pinned to signed 32-bit, the container the reader DECODES images into, while the images
-    // themselves are passed on as stored - 16-bit signed from a JUNGFRAU in photons, unsigned 16- or 32-bit
-    // from an EIGER. Typed by experiment_, the virtual dataset would convert every value on read, and the
-    // images the file carries would contradict its header (refused by the writer).
-    StoredPixelFormat stored{static_cast<int64_t>(experiment_.GetByteDepthImage() * 8), experiment_.IsPixelSigned()};
+    // The process file LINKS to the original images rather than writing its own, which only a single
+    // HDF5 input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs,
+    // an XTC stream or several masters read as one. Such a process file holds no /entry/data/data at
+    // all - a copy of the images would make it as large as the data, and the per-image analysis is
+    // the point of it.
     if (auto *hdf5 = dynamic_cast<JFJochHDF5Reader *>(&reader_)) {
         // With the stride: image i of the process file is source image start_image + i * stride, so
         // the linked pictures line up with the per-image analysis written beside them.
         start_message.hdf5_source_data =
             hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride);
-        stored = hdf5->GetStoredPixelFormat();
-    } else {
-        start_message.write_images = true;   // no file to link to; the process file carries them
-        // As the reader hands the images out, which is how they are written.
-        JFJochReaderRawImage first;
-        if (reader_.ReadRawImage(start_image, first))
-            stored = {static_cast<int64_t>(first.image.GetByteDepth() * 8), first.image.IsSigned()};
+
+        // Everything describing the pixels must describe those linked files. experiment_ cannot: it
+        // is pinned to signed 32-bit, the container the reader hands images out in, which would type
+        // the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every
+        // value on read. Only consumers going through the virtual view ever saw it - our own reader
+        // resolves the mapping and opens the source itself.
+        const auto stored = hdf5->GetStoredPixelFormat();
+        start_message.bit_depth_image = stored.bit_depth;
+        // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
+        // the field is present, and on a signed image those land inside the trusted range.
+        start_message.bit_depth_readout.reset();
+        if (!stored.is_signed)
+            start_message.bit_depth_readout = stored.bit_depth;
+        start_message.pixel_signed = stored.is_signed;
+        start_message.error_value = stored.is_signed
+                                    ? -(int64_t(1) << (stored.bit_depth - 1))      // INTx_MIN
+                                    : (int64_t(1) << stored.bit_depth) - 1;        // UINTx_MAX
+        start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
     }
-    start_message.bit_depth_image = stored.bit_depth;
-    // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
-    // the field is present, and on a signed image those land inside the trusted range.
-    start_message.bit_depth_readout.reset();
-    if (!stored.is_signed)
-        start_message.bit_depth_readout = stored.bit_depth;
-    start_message.pixel_signed = stored.is_signed;
-    start_message.error_value = stored.is_signed
-                                ? -(int64_t(1) << (stored.bit_depth - 1))      // INTx_MIN
-                                : (int64_t(1) << stored.bit_depth) - 1;        // UINTx_MAX
-    start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
 
     // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
     // but the process file reads them. Copy them where the start message is handed on rather than here:

tests/RugnuxTest.cpp

@@ -95,15 +95,15 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") {
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
 
-TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") {
-    // A reader with no HDF5 file to link to (here several inputs read as one) has its images written
-    // into the _process.h5, typed as they are stored - 16-bit signed - not as the 32-bit container
-    // they are decoded into.
-    WriteTestDataset("process_carry_in", 4);
+TEST_CASE("Rugnux_ProcessFileWithoutImages", "[HDF5][Full]") {
+    // A reader with no single HDF5 file to link to (here several inputs read as one) gets a _process.h5
+    // with the per-image analysis and no /entry/data/data, which reads back with its images' metadata
+    // and no pixels.
+    WriteTestDataset("process_noimg_in", 4);
 
     std::vector<std::unique_ptr<JFJochReader>> inputs;
     auto input = std::make_unique<JFJochHDF5Reader>();
-    REQUIRE_NOTHROW(input->ReadFile("process_carry_in_master.h5"));
+    REQUIRE_NOTHROW(input->ReadFile("process_noimg_in_master.h5"));
     inputs.push_back(std::move(input));
     JFJochMultiReader reader(std::move(inputs));
     auto dataset = reader.GetDataset();
@@ -112,7 +112,7 @@ TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") {
     ProcessConfig config;
     config.mode = ProcessMode::AzimuthalIntegration;
     config.nthreads = 2;
-    ProcessFileSink sink("process_carry_out");
+    ProcessFileSink sink("process_noimg_out");
     config.image_sink = &sink;
 
     Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
@@ -120,22 +120,22 @@ TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") {
     REQUIRE_NOTHROW(result = process.Run());
     CHECK(result.images_processed == 4);
 
+    {
+        HDF5ReadOnlyFile file("process_noimg_out_process.h5");
+        CHECK_FALSE(file.Exists("/entry/data/data"));
+    }
     {
         JFJochHDF5Reader out;
-        REQUIRE_NOTHROW(out.ReadFile("process_carry_out_process.h5"));
+        REQUIRE_NOTHROW(out.ReadFile("process_noimg_out_process.h5"));
         CHECK(out.GetNumberOfImages() == 4);
-        CHECK(out.GetStoredPixelFormat().bit_depth == 16);
-        CHECK(out.GetStoredPixelFormat().is_signed);
-        std::shared_ptr<JFJochReaderImage> img;
-        REQUIRE_NOTHROW(img = out.LoadImage(0));
-        REQUIRE(img);
-        CHECK(img->Image()[100] == 5);
+        CHECK(out.LoadImage(0) == nullptr);
+        CHECK_NOTHROW(out.ReadReflections(0, 3));
     }
 
     reader.Close();
-    remove("process_carry_in_master.h5");
-    remove("process_carry_in_data_000001.h5");
-    remove("process_carry_out_process.h5");
+    remove("process_noimg_in_master.h5");
+    remove("process_noimg_in_data_000001.h5");
+    remove("process_noimg_out_process.h5");
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }