From bb94e97f8e3ce45bfc78656e1d7dec1af8d4e871 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 1 Nov 2026 11:22:03 +0100 Subject: [PATCH] rugnux: a _process.h5 of an input it cannot link to holds no image data Where the input is anything but a single HDF5 master the process file can link to (several masters read as one, CBF, XTC, CXI, ...), the _process.h5 no longer copies the images into /entry/data/data: the copy made the file as large as the data. It now holds only the per-image analysis, and the earlier fix (d67f10cda), which typed that copy as the images are stored, goes with it - the int16 multi-master CrMX run that failed there writes its file without images. The reader takes such a file (no /entry/data/data, no linked data files) as one with /entry/detector/number images: the master's per-image analysis is read as for an integrated master and the reflections as always, so --mode scale reads it like any other, while LoadImage and ReadRawImage return no image instead of failing on the missing pixels (the viewer then shows none). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37 --- docs/HDF5.md | 4 +- reader/HDF5MetadataSource.cpp | 141 +++++++++++++++++++--------------- reader/JFJochHDF5Reader.cpp | 5 +- reader/JFJochHDF5Reader.h | 3 + rugnux/Rugnux.cpp | 52 ++++++------- tests/RugnuxTest.cpp | 34 ++++---- 6 files changed, 127 insertions(+), 112 deletions(-) diff --git a/docs/HDF5.md b/docs/HDF5.md index e6908eab6..aee90d853 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data. This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data` -VDS still resolves to the original images when they are available. `rugnux` can also process +VDS still resolves to the original images when they are available. Where the input is anything but a +single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no +`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. ## 3. NXmx-standard content diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index 0d6fa3b6d..f0c668375 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -538,6 +538,75 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen // Of a multichannel master; only its first channel is read std::string first_channel; + // The per-image analysis and metadata a master holds itself (VDS, integrated, or a _process.h5 that + // holds no images at all). + auto read_master_per_image = [&] { + if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image")) + dataset->efficiency = master_file->ReadVector( + "/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"); + else + dataset->efficiency = std::vector(number_of_images, 1.0); + + if (master_file->Exists("/entry/roi")) + dataset->roi = master_file->FindLeafs("/entry/roi"); + + for (const auto &s: dataset->roi) { + dataset->roi_max.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/max")); + dataset->roi_sum.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/sum")); + dataset->roi_sum_sq.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/sum_sq")); + dataset->roi_npixel.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/npixel")); + dataset->roi_x.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/x")); + dataset->roi_y.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/y")); + } + + if (master_file->Exists("/entry/MX")) { + if (master_file->Exists("/entry/MX/peakCountUnfiltered")) + dataset->spot_count = master_file->ReadOptVector("/entry/MX/peakCountUnfiltered"); + else + dataset->spot_count = master_file->ReadOptVector("/entry/MX/nPeaks"); + + dataset->spot_count_low_res = master_file->ReadOptVector("/entry/MX/peakCountLowRes"); + dataset->spot_count_indexed = master_file->ReadOptVector("/entry/MX/peakCountIndexed"); + dataset->spot_count_ice_rings = master_file->ReadOptVector("/entry/MX/peakCountIceRingRes"); + dataset->spot_count_ice_control = master_file->ReadOptVector("/entry/MX/peakCountIceRingControl"); + + dataset->indexing_result = master_file->ReadOptVector("/entry/MX/imageIndexed"); + dataset->bkg_estimate = master_file->ReadOptVector("/entry/MX/bkgEstimate"); + dataset->spindle_blind_fraction = master_file->ReadOptVector("/entry/MX/spindleBlindFraction"); + dataset->ice_ring_score = master_file->ReadOptVector("/entry/MX/iceRingScore"); + dataset->resolution_estimate = master_file->ReadOptVector("/entry/MX/resolutionEstimate"); + dataset->profile_radius = master_file->ReadOptVector("/entry/MX/profileRadius"); + // Master files write indexedLatticeCount; data files / the per-file MX + // plugin use indexingLatticeCount. Accept either for backward compatibility. + dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexedLatticeCount"); + if (dataset->indexing_lattice_count.empty()) + dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexingLatticeCount"); + dataset->mosaicity_deg = master_file->ReadOptVector("/entry/MX/mosaicity"); + dataset->b_factor = master_file->ReadOptVector("/entry/MX/bFactor"); + dataset->image_scale_factor = master_file->ReadOptVector("/entry/MX/imageScaleFactor"); + dataset->image_scale_cc = master_file->ReadOptVector("/entry/MX/imageScaleCC"); + dataset->integrated_reflections = master_file->ReadOptVector("/entry/MX/integratedReflections"); + dataset->sweep_quality = master_file->ReadOptVector("/entry/MX/sweepQuality"); + if (master_file->Exists("/entry/MX/sweepQualityReasons")) { + const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons"); + for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++) + dataset->sweep_quality_reasons.push_back( + master_file->ReadElement("/entry/MX/sweepQualityReasons", i) + .value_or("")); + } + dataset->frame_disposition = master_file->ReadOptVector("/entry/MX/frameDisposition"); + if (master_file->Exists("/entry/MX/frameDispositionCodes")) { + const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes"); + for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++) + dataset->frame_disposition_codes.push_back( + master_file->ReadElement("/entry/MX/frameDispositionCodes", i) + .value_or("")); + } + } + if (master_file->Exists("/entry/image")) + dataset->max_value = master_file->ReadOptVector("/entry/image/max_value"); + }; + if (master_file->Exists("/entry/data/data")) { HDF5DataSet data_dataset(*master_file, "/entry/data/data"); HDF5Dcpl dcpl(data_dataset); @@ -554,70 +623,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen if (data_layout == HDF5DataSetLayout::VIRTUAL) vds_data_mappings = ReadVDSImageMappings(*master_file, "/entry/data/data"); - if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image")) - dataset->efficiency = master_file->ReadVector( - "/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"); - else - dataset->efficiency = std::vector(number_of_images, 1.0); - - if (master_file->Exists("/entry/roi")) - dataset->roi = master_file->FindLeafs("/entry/roi"); - - for (const auto &s: dataset->roi) { - dataset->roi_max.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/max")); - dataset->roi_sum.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/sum")); - dataset->roi_sum_sq.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/sum_sq")); - dataset->roi_npixel.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/npixel")); - dataset->roi_x.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/x")); - dataset->roi_y.emplace_back(master_file->ReadVector("/entry/roi/" + s + "/y")); - } - - if (master_file->Exists("/entry/MX")) { - if (master_file->Exists("/entry/MX/peakCountUnfiltered")) - dataset->spot_count = master_file->ReadOptVector("/entry/MX/peakCountUnfiltered"); - else - dataset->spot_count = master_file->ReadOptVector("/entry/MX/nPeaks"); - - dataset->spot_count_low_res = master_file->ReadOptVector("/entry/MX/peakCountLowRes"); - dataset->spot_count_indexed = master_file->ReadOptVector("/entry/MX/peakCountIndexed"); - dataset->spot_count_ice_rings = master_file->ReadOptVector("/entry/MX/peakCountIceRingRes"); - dataset->spot_count_ice_control = master_file->ReadOptVector("/entry/MX/peakCountIceRingControl"); - - dataset->indexing_result = master_file->ReadOptVector("/entry/MX/imageIndexed"); - dataset->bkg_estimate = master_file->ReadOptVector("/entry/MX/bkgEstimate"); - dataset->spindle_blind_fraction = master_file->ReadOptVector("/entry/MX/spindleBlindFraction"); - dataset->ice_ring_score = master_file->ReadOptVector("/entry/MX/iceRingScore"); - dataset->resolution_estimate = master_file->ReadOptVector("/entry/MX/resolutionEstimate"); - dataset->profile_radius = master_file->ReadOptVector("/entry/MX/profileRadius"); - // Master files write indexedLatticeCount; data files / the per-file MX - // plugin use indexingLatticeCount. Accept either for backward compatibility. - dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexedLatticeCount"); - if (dataset->indexing_lattice_count.empty()) - dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexingLatticeCount"); - dataset->mosaicity_deg = master_file->ReadOptVector("/entry/MX/mosaicity"); - dataset->b_factor = master_file->ReadOptVector("/entry/MX/bFactor"); - dataset->image_scale_factor = master_file->ReadOptVector("/entry/MX/imageScaleFactor"); - dataset->image_scale_cc = master_file->ReadOptVector("/entry/MX/imageScaleCC"); - dataset->integrated_reflections = master_file->ReadOptVector("/entry/MX/integratedReflections"); - dataset->sweep_quality = master_file->ReadOptVector("/entry/MX/sweepQuality"); - if (master_file->Exists("/entry/MX/sweepQualityReasons")) { - const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons"); - for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++) - dataset->sweep_quality_reasons.push_back( - master_file->ReadElement("/entry/MX/sweepQualityReasons", i) - .value_or("")); - } - dataset->frame_disposition = master_file->ReadOptVector("/entry/MX/frameDisposition"); - if (master_file->Exists("/entry/MX/frameDispositionCodes")) { - const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes"); - for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++) - dataset->frame_disposition_codes.push_back( - master_file->ReadElement("/entry/MX/frameDispositionCodes", i) - .value_or("")); - } - } - if (master_file->Exists("/entry/image")) - dataset->max_value = master_file->ReadOptVector("/entry/image/max_value"); + read_master_per_image(); format = FileWriterFormat::NXmxVDS; } else if (master_file->Exists("/entry/data/data_000001")) { @@ -794,6 +800,13 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector"); image_size_y = master_file->GetInt("/entry/instrument/detector/detectorSpecific/y_pixels_in_detector"); number_of_images = 0; + // A _process.h5 of an input it could not link to holds the per-image analysis without the + // images: as many images as it has per-image entries, none of them with pixels. + if (master_file->Exists("/entry/detector/number")) { + number_of_images = master_file->GetDimension("/entry/detector/number")[0]; + images_per_file = number_of_images; + read_master_per_image(); + } } if (master_file->Exists("/entry/MX")) { diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index 4009a8cdf..22ffdb7ea 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -15,6 +15,7 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) { auto metadata = std::make_shared(); auto open_result = metadata->Open(filename, default_experiment); + has_pixels = open_result.image_layout.format != FileWriterFormat::NoFile; image_source_.Configure(std::move(open_result.image_layout)); // Original file: per-image metadata is co-located with the pixels. metadata->UseImageSourceForMetadata(&image_source_); @@ -67,6 +68,8 @@ bool JFJochHDF5Reader::ReadRawImage(int64_t image_number, JFJochReaderRawImage & if (!active_metadata_) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot load image if file not loaded"); + if (!has_pixels) + return false; auto loc = GetImageLocation(image_number); chunk = image_source_.PrepareDirectRead(loc); @@ -88,7 +91,7 @@ bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr &dataset std::unique_lock ul(hdf5_mutex); (void) update_dataset; - if (!dataset) + if (!dataset || !has_pixels) return false; if (!active_metadata_) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index 175fedd89..053213873 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -31,6 +31,9 @@ class JFJochHDF5Reader : public JFJochReader { std::shared_ptr active_metadata_; std::string active_snapshot_; size_t number_of_images = 0; + // False for a file of per-image analysis without the images (a _process.h5 of an input it could + // not link to): its images have metadata, but no pixels to load. + bool has_pixels = false; bool LoadImage_i(std::shared_ptr &dataset, DataMessage &message, diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 482229d17..ae8213b00 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1035,41 +1035,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b start_message.file_format = FileWriterFormat::NXmxIntegrated; start_message.write_master_file = true; start_message.write_images = false; - // The process file LINKS to the original images rather than writing its own, which only an HDF5 - // input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs. So - // ask for the links only when the source is HDF5, and otherwise write the images into the file - - // the per-image analysis is the point of it either way, and without pictures nothing downstream - // could show the frame a reflection came from. - // Everything describing the pixels must describe the images the file links to or carries. experiment_ - // cannot: it is pinned to signed 32-bit, the container the reader DECODES images into, while the images - // themselves are passed on as stored - 16-bit signed from a JUNGFRAU in photons, unsigned 16- or 32-bit - // from an EIGER. Typed by experiment_, the virtual dataset would convert every value on read, and the - // images the file carries would contradict its header (refused by the writer). - StoredPixelFormat stored{static_cast(experiment_.GetByteDepthImage() * 8), experiment_.IsPixelSigned()}; + // The process file LINKS to the original images rather than writing its own, which only a single + // HDF5 input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs, + // an XTC stream or several masters read as one. Such a process file holds no /entry/data/data at + // all - a copy of the images would make it as large as the data, and the per-image analysis is + // the point of it. if (auto *hdf5 = dynamic_cast(&reader_)) { // With the stride: image i of the process file is source image start_image + i * stride, so // the linked pictures line up with the per-image analysis written beside them. start_message.hdf5_source_data = hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride); - stored = hdf5->GetStoredPixelFormat(); - } else { - start_message.write_images = true; // no file to link to; the process file carries them - // As the reader hands the images out, which is how they are written. - JFJochReaderRawImage first; - if (reader_.ReadRawImage(start_image, first)) - stored = {static_cast(first.image.GetByteDepth() * 8), first.image.IsSigned()}; + + // Everything describing the pixels must describe those linked files. experiment_ cannot: it + // is pinned to signed 32-bit, the container the reader hands images out in, which would type + // the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every + // value on read. Only consumers going through the virtual view ever saw it - our own reader + // resolves the mapping and opens the source itself. + const auto stored = hdf5->GetStoredPixelFormat(); + start_message.bit_depth_image = stored.bit_depth; + // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever + // the field is present, and on a signed image those land inside the trusted range. + start_message.bit_depth_readout.reset(); + if (!stored.is_signed) + start_message.bit_depth_readout = stored.bit_depth; + start_message.pixel_signed = stored.is_signed; + start_message.error_value = stored.is_signed + ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN + : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX + start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; } - start_message.bit_depth_image = stored.bit_depth; - // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever - // the field is present, and on a signed image those land inside the trusted range. - start_message.bit_depth_readout.reset(); - if (!stored.is_signed) - start_message.bit_depth_readout = stored.bit_depth; - start_message.pixel_signed = stored.is_signed; - start_message.error_value = stored.is_signed - ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN - : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX - start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing // but the process file reads them. Copy them where the start message is handed on rather than here: diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index dc257c7b9..fa971397c 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -95,15 +95,15 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") { - // A reader with no HDF5 file to link to (here several inputs read as one) has its images written - // into the _process.h5, typed as they are stored - 16-bit signed - not as the 32-bit container - // they are decoded into. - WriteTestDataset("process_carry_in", 4); +TEST_CASE("Rugnux_ProcessFileWithoutImages", "[HDF5][Full]") { + // A reader with no single HDF5 file to link to (here several inputs read as one) gets a _process.h5 + // with the per-image analysis and no /entry/data/data, which reads back with its images' metadata + // and no pixels. + WriteTestDataset("process_noimg_in", 4); std::vector> inputs; auto input = std::make_unique(); - REQUIRE_NOTHROW(input->ReadFile("process_carry_in_master.h5")); + REQUIRE_NOTHROW(input->ReadFile("process_noimg_in_master.h5")); inputs.push_back(std::move(input)); JFJochMultiReader reader(std::move(inputs)); auto dataset = reader.GetDataset(); @@ -112,7 +112,7 @@ TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") { ProcessConfig config; config.mode = ProcessMode::AzimuthalIntegration; config.nthreads = 2; - ProcessFileSink sink("process_carry_out"); + ProcessFileSink sink("process_noimg_out"); config.image_sink = &sink; Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config); @@ -120,22 +120,22 @@ TEST_CASE("Rugnux_ProcessFileCarriesImagesAsStored", "[HDF5][Full]") { REQUIRE_NOTHROW(result = process.Run()); CHECK(result.images_processed == 4); + { + HDF5ReadOnlyFile file("process_noimg_out_process.h5"); + CHECK_FALSE(file.Exists("/entry/data/data")); + } { JFJochHDF5Reader out; - REQUIRE_NOTHROW(out.ReadFile("process_carry_out_process.h5")); + REQUIRE_NOTHROW(out.ReadFile("process_noimg_out_process.h5")); CHECK(out.GetNumberOfImages() == 4); - CHECK(out.GetStoredPixelFormat().bit_depth == 16); - CHECK(out.GetStoredPixelFormat().is_signed); - std::shared_ptr img; - REQUIRE_NOTHROW(img = out.LoadImage(0)); - REQUIRE(img); - CHECK(img->Image()[100] == 5); + CHECK(out.LoadImage(0) == nullptr); + CHECK_NOTHROW(out.ReadReflections(0, 3)); } reader.Close(); - remove("process_carry_in_master.h5"); - remove("process_carry_in_data_000001.h5"); - remove("process_carry_out_process.h5"); + remove("process_noimg_in_master.h5"); + remove("process_noimg_in_data_000001.h5"); + remove("process_noimg_out_process.h5"); REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); }