diff --git a/reader/JFJochXFELReader.cpp b/reader/JFJochXFELReader.cpp index 44aca7411..f0da13dfa 100644 --- a/reader/JFJochXFELReader.cpp +++ b/reader/JFJochXFELReader.cpp @@ -425,6 +425,28 @@ JFJochXFELReader::Layout JFJochXFELReader::ReadLayout(HDF5ReadOnlyFile &file, co return l; } +// A per-frame mask is read as the frames are where it is stored in plain unsigned integers - read as +// stored, they are what HDF5 would convert them to - of the frame's size, in every file the way the +// first file stores it. Any other is read by HDF5. +JFJochXFELReader::Layout JFJochXFELReader::FrameMaskLayout(const xfel::Mask &m) const { + Layout layout; + for (size_t i = 0; i < frames_.size(); i++) { + if (i > 0 && frames_[i].file == frames_[i - 1].file) + continue; // a file's first frame stands for all of its frames, as for the frames themselves + const Frame mask{frames_[i].file, ReplaceEvent(m.data, frames_[i].group), frames_[i].index, frames_[i].group}; + if (!FileAt(mask.file).Exists(mask.dataset)) + return {}; + const Layout l = ReadLayout(FileAt(mask.file), mask); + if (i == 0) + layout = l; + else if (!(l == layout)) + return {}; + } + if (layout.stored_float || layout.stored_signed || layout.height != frame_height_ || layout.width != frame_width_) + return {}; + return layout; +} + void JFJochXFELReader::Open(const std::vector &files) { files_ = files; geom_.masks.insert(geom_.masks.end(), extra_masks_.begin(), extra_masks_.end()); @@ -515,11 +537,13 @@ void JFJochXFELReader::Place() { { std::unique_lock ul(hdf5_mutex); frame_masks_.clear(); + frame_mask_layouts_.clear(); for (const auto &m: geom_.masks) { std::vector v; if (m.file.empty() && (m.data.find('%') != std::string::npos || FileAt(0).GetDimension(m.data).size() == 3)) { frame_masks_.push_back(m); // one per shot or one plane per frame: applied as each is read + frame_mask_layouts_.push_back(FrameMaskLayout(m)); continue; } if (!m.file.empty()) { @@ -775,7 +799,8 @@ void JFJochXFELReader::Close() { } template -bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const { +bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, + bool keep_compressed) const { const std::vector start = frame.index >= 0 ? std::vector{static_cast(frame.index), 0, 0} : std::vector{0, 0}; @@ -788,7 +813,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_ std::unique_lock ul(hdf5_mutex); HDF5DataSet dataset(FileAt(frame.file), frame.dataset); unsigned filter_mask = 0; - if (layout_.direct_chunks) { + if (layout.direct_chunks) { haddr_t addr = HADDR_UNDEF; hsize_t size = 0; if (H5Dget_chunk_info_by_coord(dataset.GetID(), start.data(), &filter_mask, &addr, &size) < 0 @@ -800,8 +825,8 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_ } // A chunk stored with one of its filters skipped (its filter mask) is not the encoding decoded // below: HDF5 reads it, as it reads a frame that is not stored as one chunk. - if (!layout_.direct_chunks || filter_mask != 0) { - if (layout_.plain_type) + if (!layout.direct_chunks || filter_mask != 0) { + if (layout.plain_type) dataset.ReadVectorToU8(stored, start, count); else { std::vector v(frame_width_ * frame_height_); @@ -814,7 +839,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_ raw_file = RawFileAt(frame.file); address += user_block_[frame.file]; // HDF5's chunk addresses count from the end of the user block } - if (keep_compressed && !layout_.deflated) { + if (keep_compressed && !layout.deflated) { stored.resize(chunk_size); raw_file->ReadAt(stored.data(), stored.size(), address); return true; @@ -824,19 +849,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_ raw_file->ReadAt(compressed.data(), compressed.size(), address); const size_t n = frame_width_ * frame_height_; - const size_t size = layout_.stored_size; - if (layout_.deflated) { - if (!layout_.shuffled) + const size_t size = layout.stored_size; + if (layout.deflated) { + if (!layout.shuffled) stored.resize(n * size); else shuffled.resize(n * size); - uint8_t *inflated = layout_.shuffled ? shuffled.data() : stored.data(); + uint8_t *inflated = layout.shuffled ? shuffled.data() : stored.data(); uLongf inflated_size = n * size; if (uncompress(inflated, &inflated_size, compressed.data(), compressed.size()) != Z_OK || inflated_size != n * size) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot inflate " + frame.dataset + " of " + files_[frame.file]); - if (layout_.shuffled) { + if (layout.shuffled) { stored.resize(n * size); for (size_t i = 0; i < n; i++) for (size_t b = 0; b < size; b++) @@ -844,7 +869,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_ } } else { stored.resize(n * size); - JFJochDecompressPtr(stored.data(), layout_.algorithm, compressed.data(), compressed.size(), n, size); + JFJochDecompressPtr(stored.data(), layout.algorithm, compressed.data(), compressed.size(), n, size); } return false; } @@ -884,24 +909,37 @@ CompressedImage JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffe const bool as_stored = blocks_.empty() && layout_.plain_type && !layout_.stored_float && layout_.stored_signed && adu_per_photon_ == 1.0f; if (as_stored && frame_masks_.empty()) { - const bool compressed = ReadStored(frame, buffer, true); + const bool compressed = ReadStored(frame, layout_, buffer, true); return CompressedImage(buffer.data(), buffer.size(), width_, height_, stored_mode, compressed ? layout_.algorithm : CompressionAlgorithm::NO_COMPRESSION); } thread_local std::vector stored; - ReadStored(frame, stored, false); + ReadStored(frame, layout_, stored, false); // A pixel a per-frame mask marks in this frame is handed out as no measurement. std::vector masked; if (!frame_masks_.empty()) { masked.assign(frame_width_ * frame_height_, 0); - std::unique_lock ul(hdf5_mutex); - for (const auto &m: frame_masks_) { - const auto path = ReplaceEvent(m.data, frame.group); - const auto v = frame.index >= 0 - ? FileAt(frame.file).ReadVector(path, {static_cast(frame.index), 0, 0}, - {1, frame_height_, frame_width_}) - : FileAt(frame.file).ReadVector(path, {0, 0}, {frame_height_, frame_width_}); + for (size_t i = 0; i < frame_masks_.size(); i++) { + const auto &m = frame_masks_[i]; + const auto &mask_layout = frame_mask_layouts_[i]; + const Frame mask{frame.file, ReplaceEvent(m.data, frame.group), frame.index, frame.group}; + std::vector v(frame_width_ * frame_height_); + if (mask_layout.plain_type) { + thread_local std::vector mask_stored; + ReadStored(mask, mask_layout, mask_stored, false); + if (mask_layout.stored_size == 2) { + const auto *p = reinterpret_cast(mask_stored.data()); + std::copy(p, p + v.size(), v.begin()); + } else + std::memcpy(v.data(), mask_stored.data(), v.size() * sizeof(uint32_t)); + } else { + std::unique_lock ul(hdf5_mutex); + v = frame.index >= 0 + ? FileAt(frame.file).ReadVector(mask.dataset, {static_cast(frame.index), 0, 0}, + {1, frame_height_, frame_width_}) + : FileAt(frame.file).ReadVector(mask.dataset, {0, 0}, {frame_height_, frame_width_}); + } MarkBad(m, v, masked); } } diff --git a/reader/JFJochXFELReader.h b/reader/JFJochXFELReader.h index 9a07adc5d..868e5210f 100644 --- a/reader/JFJochXFELReader.h +++ b/reader/JFJochXFELReader.h @@ -96,6 +96,9 @@ class JFJochXFELReader : public JFJochReader { bool operator==(const Layout &) const = default; }; Layout layout_; + // For each of frame_masks_, how it is stored where it is read as the frames are (ReadStored): plain + // unsigned integers of the frame's size, stored alike in every file; plain_type false where HDF5 reads it. + std::vector frame_mask_layouts_; // Positional-read handles on the files, opened as first needed (under hdf5_mutex), with the size of // each file's user block, which HDF5 chunk addresses count from. Like open_files_, only the most // recently used MAX_OPEN_FILES are held (raw_order_); a reader still using a released one keeps it. @@ -103,6 +106,7 @@ class JFJochXFELReader : public JFJochReader { mutable std::vector user_block_; mutable std::vector raw_order_; static Layout ReadLayout(HDF5ReadOnlyFile &file, const Frame &frame); + Layout FrameMaskLayout(const xfel::Mask &m) const; // under hdf5_mutex // The files, in the order given, every frame in them, and the run's mean of every value the geometry // takes from the files. @@ -116,10 +120,11 @@ class JFJochXFELReader : public JFJochReader { int64_t image_number, bool update_dataset) override; - // The frame's pixels as stored (plain_type), or as float, into stored; with keep_compressed, a chunk - // decoded by CompressedImage (not deflated) is left as stored in the file, and true returned. + // The frame's pixels, stored as layout says, as stored (plain_type), or as float, into stored; with + // keep_compressed, a chunk decoded by CompressedImage (not deflated) is left as stored in the file, and + // true returned. template - bool ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const; + bool ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, bool keep_compressed) const; // The stored pixels as the image handed out (as stored, or placed), as Out, each through convert; a pixel masked marks is err. template void PlaceInto(const In *in, const std::vector &masked, Out err, Buffer &buffer, Convert convert) const; diff --git a/tests/XFELReaderTest.cpp b/tests/XFELReaderTest.cpp index 323a0501d..d722031c8 100644 --- a/tests/XFELReaderTest.cpp +++ b/tests/XFELReaderTest.cpp @@ -280,6 +280,11 @@ TEST_CASE("JFJochXFELReader_GeometryMasks", "[portable]") { std::vector mask(2 * 24, 0); mask[1 * 6 + 1] = 0x20; // frame 0 only, (fs 1, ss 1): a bit Cheetah's own set does not include WriteFrames(f, "entry_1/data_1/mask", H5T_NATIVE_UINT16, mask.data(), {2, 4, 6}); + // Frame 1's mask chunk is stored with both its filters skipped: HDF5 reads that one. + const hid_t ds = H5Dopen2(f, "entry_1/data_1/mask", H5P_DEFAULT); + const hsize_t offset[3] = {1, 0, 0}; + REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 0x3, offset, 24 * sizeof(uint16_t), mask.data() + 24) >= 0); + H5Dclose(ds); const std::string det = "entry_1/instrument_1/detector_1/"; std::vector shared(24, 0); shared[3 * 6 + 5] = 8; // (fs 5, ss 3) dead to Cheetah