mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 93 Wiki Activity

XFEL reader: a per-frame mask is read the way the frames are, its chunk inflated outside the HDF5 lock

Browse Source 
The Cheetah CXI per-frame mask (N x H x W uint16, one gzip chunk per frame) was read with
ReadVector<uint32_t> under hdf5_mutex: the inflate and HDF5's element-wise type conversion serialised
every reading thread and held an AGIPD run to ~40 frames/s. It now goes through ReadStored, as the
data does: only the chunk lookup is under the lock; the read, inflate and unshuffle are not, and the
stored uint16/uint32 values are widened to uint32 exactly as HDF5 converted them.

The direct path is taken where the mask is plain unsigned integers of the frame's size, stored alike
in every file (checked once, in Place); a chunk stored with a filter skipped is read by HDF5 as for
the data, and any other mask keeps the locked HDF5 read. Images and pixel mask are bit-identical to
before (hash of every pixel of 2000 images on two AGIPD runs and one MPCCD run).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-22 17:45:58 +02:00
1 parent 4858816e2a
commit 9507b064b4
3 files changed
+71 -23
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

reader/JFJochXFELReader.cpp
+58 -20
View File
@@ -425,6 +425,28 @@ JFJochXFELReader::Layout JFJochXFELReader::ReadLayout(HDF5ReadOnlyFile &file, co
return l;
}
// A per-frame mask is read as the frames are where it is stored in plain unsigned integers - read as
// stored, they are what HDF5 would convert them to - of the frame's size, in every file the way the
// first file stores it. Any other is read by HDF5.
JFJochXFELReader::Layout JFJochXFELReader::FrameMaskLayout(const xfel::Mask &m) const {
Layout layout;
for (size_t i = 0; i < frames_.size(); i++) {
if (i > 0 && frames_[i].file == frames_[i - 1].file)
continue; // a file's first frame stands for all of its frames, as for the frames themselves
const Frame mask{frames_[i].file, ReplaceEvent(m.data, frames_[i].group), frames_[i].index, frames_[i].group};
if (!FileAt(mask.file).Exists(mask.dataset))
return {};
const Layout l = ReadLayout(FileAt(mask.file), mask);
if (i == 0)
layout = l;
else if (!(l == layout))
return {};
}
if (layout.stored_float || layout.stored_signed || layout.height != frame_height_ || layout.width != frame_width_)
return {};
return layout;
}
void JFJochXFELReader::Open(const std::vector<std::string> &files) {
files_ = files;
geom_.masks.insert(geom_.masks.end(), extra_masks_.begin(), extra_masks_.end());
@@ -515,11 +537,13 @@ void JFJochXFELReader::Place() {
{
std::unique_lock ul(hdf5_mutex);
frame_masks_.clear();
frame_mask_layouts_.clear();
for (const auto &m: geom_.masks) {
std::vector<uint32_t> v;
if (m.file.empty() && (m.data.find('%') != std::string::npos
|| FileAt(0).GetDimension(m.data).size() == 3)) {
frame_masks_.push_back(m); // one per shot or one plane per frame: applied as each is read
frame_mask_layouts_.push_back(FrameMaskLayout(m));
continue;
}
if (!m.file.empty()) {
@@ -775,7 +799,8 @@ void JFJochXFELReader::Close() {
}
template <class Buffer>
bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const {
bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buffer &stored,
bool keep_compressed) const {
const std::vector<hsize_t> start = frame.index >= 0
? std::vector<hsize_t>{static_cast<hsize_t>(frame.index), 0, 0}
: std::vector<hsize_t>{0, 0};
@@ -788,7 +813,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
std::unique_lock ul(hdf5_mutex);
HDF5DataSet dataset(FileAt(frame.file), frame.dataset);
unsigned filter_mask = 0;
if (layout_.direct_chunks) {
if (layout.direct_chunks) {
haddr_t addr = HADDR_UNDEF;
hsize_t size = 0;
if (H5Dget_chunk_info_by_coord(dataset.GetID(), start.data(), &filter_mask, &addr, &size) < 0
@@ -800,8 +825,8 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
}
// A chunk stored with one of its filters skipped (its filter mask) is not the encoding decoded
// below: HDF5 reads it, as it reads a frame that is not stored as one chunk.
if (!layout_.direct_chunks || filter_mask != 0) {
if (layout_.plain_type)
if (!layout.direct_chunks || filter_mask != 0) {
if (layout.plain_type)
dataset.ReadVectorToU8(stored, start, count);
else {
std::vector<float> v(frame_width_ * frame_height_);
@@ -814,7 +839,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
raw_file = RawFileAt(frame.file);
address += user_block_[frame.file]; // HDF5's chunk addresses count from the end of the user block
}
if (keep_compressed && !layout_.deflated) {
if (keep_compressed && !layout.deflated) {
stored.resize(chunk_size);
raw_file->ReadAt(stored.data(), stored.size(), address);
return true;
@@ -824,19 +849,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
raw_file->ReadAt(compressed.data(), compressed.size(), address);
const size_t n = frame_width_ * frame_height_;
const size_t size = layout_.stored_size;
if (layout_.deflated) {
if (!layout_.shuffled)
const size_t size = layout.stored_size;
if (layout.deflated) {
if (!layout.shuffled)
stored.resize(n * size);
else
shuffled.resize(n * size);
uint8_t *inflated = layout_.shuffled ? shuffled.data() : stored.data();
uint8_t *inflated = layout.shuffled ? shuffled.data() : stored.data();
uLongf inflated_size = n * size;
if (uncompress(inflated, &inflated_size, compressed.data(), compressed.size()) != Z_OK
|| inflated_size != n * size)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Cannot inflate " + frame.dataset + " of " + files_[frame.file]);
if (layout_.shuffled) {
if (layout.shuffled) {
stored.resize(n * size);
for (size_t i = 0; i < n; i++)
for (size_t b = 0; b < size; b++)
@@ -844,7 +869,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
}
} else {
stored.resize(n * size);
JFJochDecompressPtr(stored.data(), layout_.algorithm, compressed.data(), compressed.size(), n, size);
JFJochDecompressPtr(stored.data(), layout.algorithm, compressed.data(), compressed.size(), n, size);
}
return false;
}
@@ -884,24 +909,37 @@ CompressedImage JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffe
const bool as_stored = blocks_.empty() && layout_.plain_type && !layout_.stored_float && layout_.stored_signed
&& adu_per_photon_ == 1.0f;
if (as_stored && frame_masks_.empty()) {
const bool compressed = ReadStored(frame, buffer, true);
const bool compressed = ReadStored(frame, layout_, buffer, true);
return CompressedImage(buffer.data(), buffer.size(), width_, height_, stored_mode,
compressed ? layout_.algorithm : CompressionAlgorithm::NO_COMPRESSION);
}
thread_local std::vector<uint8_t> stored;
ReadStored(frame, stored, false);
ReadStored(frame, layout_, stored, false);
// A pixel a per-frame mask marks in this frame is handed out as no measurement.
std::vector<uint8_t> masked;
if (!frame_masks_.empty()) {
masked.assign(frame_width_ * frame_height_, 0);
std::unique_lock ul(hdf5_mutex);
for (const auto &m: frame_masks_) {
const auto path = ReplaceEvent(m.data, frame.group);
const auto v = frame.index >= 0
? FileAt(frame.file).ReadVector<uint32_t>(path, {static_cast<hsize_t>(frame.index), 0, 0},
{1, frame_height_, frame_width_})
: FileAt(frame.file).ReadVector<uint32_t>(path, {0, 0}, {frame_height_, frame_width_});
for (size_t i = 0; i < frame_masks_.size(); i++) {
const auto &m = frame_masks_[i];
const auto &mask_layout = frame_mask_layouts_[i];
const Frame mask{frame.file, ReplaceEvent(m.data, frame.group), frame.index, frame.group};
std::vector<uint32_t> v(frame_width_ * frame_height_);
if (mask_layout.plain_type) {
thread_local std::vector<uint8_t> mask_stored;
ReadStored(mask, mask_layout, mask_stored, false);
if (mask_layout.stored_size == 2) {
const auto *p = reinterpret_cast<const uint16_t *>(mask_stored.data());
std::copy(p, p + v.size(), v.begin());
} else
std::memcpy(v.data(), mask_stored.data(), v.size() * sizeof(uint32_t));
} else {
std::unique_lock ul(hdf5_mutex);
v = frame.index >= 0
? FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {static_cast<hsize_t>(frame.index), 0, 0},
{1, frame_height_, frame_width_})
: FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {0, 0}, {frame_height_, frame_width_});
}
MarkBad(m, v, masked);
}
}