diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 68e417c1..fbde6cd2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ This is an UNSTABLE release. It includes many experimental features, as well as * `bit_depth_readout` now reports the bit depth of the stored image rather than the detector's electronic readout depth, which is what NXmx readers expect - DIALS derives its masking markers from that field and cannot read a 32-bit image without it. * NXmx `underload_value` (lowest valid value) is now written, so a reader masks the error-pixel marker instead of treating it as an intensity. * A DECTRIS detector sending signed images is no longer declared unsigned in the outgoing stream and in HDF5. +* rugnux: `_process.h5` describes the pixel format of the images it links to, instead of the signed 32-bit container rugnux processes in - its virtual dataset was typed `int32` over unsigned 16- or 32-bit data, so any reader going through it saw converted values. ### 1.0.0-rc.161 This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. diff --git a/reader/HDF5ImageSource.cpp b/reader/HDF5ImageSource.cpp index fe624134..4bd1b119 100644 --- a/reader/HDF5ImageSource.cpp +++ b/reader/HDF5ImageSource.cpp @@ -16,6 +16,13 @@ HDF5ImageLocator::Location HDF5ImageSource::Resolve(int64_t global) const { return locator_.Resolve(global); } +StoredPixelFormat HDF5ImageSource::GetStoredPixelFormat() const { + auto loc = locator_.Resolve(0); + HDF5DataSet dataset(*loc.file, "/entry/data/data"); + HDF5DataType datatype(dataset); + return {static_cast(datatype.GetElemSize()) * 8, datatype.IsSigned()}; +} + std::vector HDF5ImageSource::GetSourceMapping(uint64_t first_image, std::optional image_count, uint64_t total_images, diff --git a/reader/HDF5ImageSource.h b/reader/HDF5ImageSource.h index 8b1a9176..9bc18ddd 100644 --- a/reader/HDF5ImageSource.h +++ b/reader/HDF5ImageSource.h @@ -14,11 +14,22 @@ // HDF5ImageLocator to find the file (with its open-file cache). This is the part whose "links // to files stay" constant: switching which master the per-image metadata is read from never // touches it. Caller must hold the global hdf5_mutex (HDF5 is not thread-safe). +// Bit depth and signedness of /entry/data/data as it is STORED. Deliberately not the same thing as +// the experiment's image format: the reader hands every image out in a signed 32-bit container +// whatever the file holds (see HDF5MetadataSource), so an output file that links to the original +// images rather than writing its own must describe them with this, not with the experiment. +struct StoredPixelFormat { + int64_t bit_depth = 0; + bool is_signed = false; +}; + class HDF5ImageSource { public: void Configure(HDF5ImageLocator::Layout layout); void Clear(); + [[nodiscard]] StoredPixelFormat GetStoredPixelFormat() const; + // Where image `global` physically lives. Also used by the metadata source to find the data // file that holds a legacy/VDS image's per-image metadata. HDF5ImageLocator::Location Resolve(int64_t global) const; diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index 953d130b..3ddc3e60 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -103,6 +103,16 @@ std::vector JFJochHDF5Reader::GetHDF5DataSource(uint64_t return image_source_.GetSourceMapping(first_image, image_count, number_of_images, stride); } +StoredPixelFormat JFJochHDF5Reader::GetStoredPixelFormat() const { + std::unique_lock ul(hdf5_mutex); + + if (!active_metadata_) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Cannot read stored pixel format if file not loaded"); + + return image_source_.GetStoredPixelFormat(); +} + std::vector JFJochHDF5Reader::ReadReflections(size_t start_image, std::optional end_image) const { std::unique_lock ul(hdf5_mutex); diff --git a/reader/JFJochHDF5Reader.h b/reader/JFJochHDF5Reader.h index d4653e0d..b18a45d0 100644 --- a/reader/JFJochHDF5Reader.h +++ b/reader/JFJochHDF5Reader.h @@ -53,6 +53,10 @@ public: uint64_t stride = 1 ) const; + // Pixel format of the stored images. Needed by anything that links to them instead of writing + // its own, since the experiment describes the reader's container rather than the file. + [[nodiscard]] StoredPixelFormat GetStoredPixelFormat() const; + std::vector ReadReflections(size_t start_image = 0, std::optional end_image = {}) const; std::vector ReadSpots(int64_t image) const override; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 632b3fd7..87e058b7 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -931,6 +931,21 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b start_message.hdf5_source_data = reader_.GetHDF5DataSource(start_image, images_to_process, config_.stride); + // This file links to the ORIGINAL images instead of writing its own (write_images = false), so + // everything describing the pixels must describe those files. experiment_ cannot: it is pinned + // to signed 32-bit, the container the reader hands images out in, which would type the virtual + // dataset int32 over unsigned 16- or 32-bit sources and silently convert every value on read. + // Only consumers going through the virtual view ever saw it - our own reader resolves the + // mapping and opens the source itself. + const auto stored = reader_.GetStoredPixelFormat(); + start_message.bit_depth_image = stored.bit_depth; + start_message.bit_depth_readout = stored.bit_depth; + start_message.pixel_signed = stored.is_signed; + start_message.error_value = stored.is_signed + ? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN + : (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX + start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0; + std::unique_ptr writer; if (write_files && config_.write_process_h5) writer = std::make_unique(start_message, /*check_overwrite_at_start=*/true,