From d01e69e4072c66d9f1807451cccf45e8e4fd9902 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 22 Aug 2026 18:24:31 +0200 Subject: [PATCH] rugnux: describe the linked images in _process.h5, not the processing container _process.h5 is an NXmxIntegrated master that links to the ORIGINAL image files rather than writing images of its own (write_images = false). Its pixel metadata therefore has to describe those files - but it was filled from experiment_, which Rugnux pins to signed 32-bit because that is the container HDF5MetadataSource hands images out in. The virtual dataset was consequently typed int32 over unsigned 16- or 32-bit sources, so HDF5 converted every value on read: the 0xFFFFFFFF error marker of a uint32 source does not survive, and error_value and underload_value described a container the file does not contain. Our own reader never saw it, because it resolves the mapping and opens the source file itself. Only consumers that go through the virtual view - DIALS, XDS via Durin, plain h5py - read the converted values. Take the format from the reader instead. GetStoredPixelFormat() reports the bit depth and signedness of /entry/data/data as stored, which is deliberately not the same thing as the experiment's image format, and rugnux fills the start message from it. The writer is left alone on purpose: it must be able to produce a master before the mapped files exist, or when they are not readable, and the mapping strips the source directory, so it cannot open them to ask. Measured on a 20-image set with int16 sources: the _process.h5 virtual dataset goes from H5T_STD_I32LE to H5T_STD_I16LE, matching the source, and DIALS reports trusted_range (-32767, 32765) instead of (-2147483647, 32765). Co-Authored-By: Claude Opus 5 (1M context) --- docs/CHANGELOG.md | 1 + reader/HDF5ImageSource.cpp | 7 +++++++ reader/HDF5ImageSource.h | 11 +++++++++++ reader/JFJochHDF5Reader.cpp | 10 ++++++++++ reader/JFJochHDF5Reader.h | 4 ++++ rugnux/Rugnux.cpp | 15 +++++++++++++++ 6 files changed, 48 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 68e417c1..fbde6cd2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ This is an UNSTABLE release. It includes many experimental features, as well as * `bit_depth_readout` now reports the bit depth of the stored image rather than the detector's electronic readout depth, which is what NXmx readers expect - DIALS derives its masking markers from that field and cannot read a 32-bit image without it. * NXmx `underload_value` (lowest valid value) is now written, so a reader masks the error-pixel marker instead of treating it as an intensity. * A DECTRIS detector sending signed images is no longer declared unsigned in the outgoing stream and in HDF5. +* rugnux: `_process.h5` describes the pixel format of the images it links to, instead of the signed 32-bit container rugnux processes in - its virtual dataset was typed `int32` over unsigned 16- or 32-bit data, so any reader going through it saw converted values. ### 1.0.0-rc.161 This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. diff --git a/reader/HDF5ImageSource.cpp b/reader/HDF5ImageSource.cpp index fe624134..4bd1b119 100644 --- a/reader/HDF5ImageSource.cpp +++ b/reader/HDF5ImageSource.cpp @@ -16,6 +16,13 @@ HDF5ImageLocator::Location HDF5ImageSource::Resolve(int64_t global) const { return locator_.Resolve(global); } +StoredPixelFormat HDF5ImageSource::GetStoredPixelFormat() const { + auto loc = locator_.Resolve(0); + HDF5DataSet dataset(*loc.file, "/entry/data/data"); + HDF5DataType datatype(dataset); + return {static_cast(datatype.GetElemSize()) * 8, datatype.IsSigned()}; +} + std::vector HDF5ImageSource::GetSourceMapping(uint64_t first_image, std::optional image_count, uint64_t total_images, diff --git a/reader/HDF5ImageSource.h b/reader/HDF5ImageSource.h index 8b1a9176..9bc18ddd 100644 --- a/reader/HDF5ImageSource.h +++ b/reader/HDF5ImageSource.h @@ -14,11 +14,22 @@ // HDF5ImageLocator to find the file (with its open-file cache). This is the part whose "links // to files stay" constant: switching which master the per-image metadata is read from never // touches it. Caller must hold the global hdf5_mutex (HDF5 is not thread-safe). +// Bit depth and signedness of /entry/data/data as it is STORED. Deliberately not the same thing as +// the experiment's image format: the reader hands every image out in a signed 32-bit container +// whatever the file holds (see HDF5MetadataSource), so an output file that links to the original +// images rather than writing its own must describe them with this, not with the experiment. +struct StoredPixelFormat { + int64_t bit_depth = 0; + bool is_signed = false; +}; + class HDF5ImageSource { public: void Configure(HDF5ImageLocator::Layout layout); void Clear(); + [[nodiscard]] StoredPixelFormat GetStoredPixelFormat() const; + // Where image `global` physically lives. Also used by the metadata source to find the data // file that holds a legacy/VDS image's per-image metadata. HDF5ImageLocator::Location Resolve(int64_t global) const; diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index 953d130b..3ddc3e60 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -103,6 +103,16 @@ std::vector JFJochHDF5Reader::GetHDF5DataSource(uint64_t return image_source_.GetSourceMapping(first_image, image_count, number_of_images, stride); } +StoredPixelFormat JFJochHDF5Reader::GetStoredPixelFormat() const { + std::unique_lock ul(hdf5_mutex); + + if (!active_metadata_) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Cannot read stored pixel format if file not loaded"); + + return image_source_.GetStoredPixelFormat(); +} + std::vector JFJochHDF5Reader::ReadReflections(size_t start_image, std::optional end_image) const { std::unique_lock ul(hdf5_mutex); diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index d4653e0d..b18a45d0 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -53,6 +53,10 @@ public: uint64_t stride = 1 ) const; + // Pixel format of the stored images. Needed by anything that links to them instead of writing + // its own, since the experiment describes the reader's container rather than the file. + [[nodiscard]] StoredPixelFormat GetStoredPixelFormat() const; + std::vector ReadReflections(size_t start_image = 0, std::optional end_image = {}) const; std::vector ReadSpots(int64_t image) const override; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 632b3fd7..87e058b7 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -931,6 +931,21 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b start_message.hdf5_source_data = reader_.GetHDF5DataSource(start_image, images_to_process, config_.stride); + // This file links to the ORIGINAL images instead of writing its own (write_images = false), so + // everything describing the pixels must describe those files. experiment_ cannot: it is pinned + // to signed 32-bit, the container the reader hands images out in, which would type the virtual + // dataset int32 over unsigned 16- or 32-bit sources and silently convert every value on read. + // Only consumers going through the virtual view ever saw it - our own reader resolves the + // mapping and opens the source itself. + const auto stored = reader_.GetStoredPixelFormat(); + start_message.bit_depth_image = stored.bit_depth; + start_message.bit_depth_readout = stored.bit_depth; + start_message.pixel_signed = stored.is_signed; + start_message.error_value = stored.is_signed + ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN + : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX + start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; + std::unique_ptr writer; if (write_files && config_.write_process_h5) writer = std::make_unique(start_message, /*check_overwrite_at_start=*/true,