Merge agipd-mask-speed into rc178: XFEL per-frame mask read outside the HDF5 lock
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
commit
a6835fe9d3
3 files changed
+71
-23
No files matched your search
+58
-20
@@ -425,6 +425,28 @@ JFJochXFELReader::Layout JFJochXFELReader::ReadLayout(HDF5ReadOnlyFile &file, co
|
||||
return l;
|
||||
}
|
||||
|
||||
// A per-frame mask is read as the frames are where it is stored in plain unsigned integers - read as
|
||||
// stored, they are what HDF5 would convert them to - of the frame's size, in every file the way the
|
||||
// first file stores it. Any other is read by HDF5.
|
||||
JFJochXFELReader::Layout JFJochXFELReader::FrameMaskLayout(const xfel::Mask &m) const {
|
||||
Layout layout;
|
||||
for (size_t i = 0; i < frames_.size(); i++) {
|
||||
if (i > 0 && frames_[i].file == frames_[i - 1].file)
|
||||
continue; // a file's first frame stands for all of its frames, as for the frames themselves
|
||||
const Frame mask{frames_[i].file, ReplaceEvent(m.data, frames_[i].group), frames_[i].index, frames_[i].group};
|
||||
if (!FileAt(mask.file).Exists(mask.dataset))
|
||||
return {};
|
||||
const Layout l = ReadLayout(FileAt(mask.file), mask);
|
||||
if (i == 0)
|
||||
layout = l;
|
||||
else if (!(l == layout))
|
||||
return {};
|
||||
}
|
||||
if (layout.stored_float || layout.stored_signed || layout.height != frame_height_ || layout.width != frame_width_)
|
||||
return {};
|
||||
return layout;
|
||||
}
|
||||
|
||||
void JFJochXFELReader::Open(const std::vector<std::string> &files) {
|
||||
files_ = files;
|
||||
geom_.masks.insert(geom_.masks.end(), extra_masks_.begin(), extra_masks_.end());
|
||||
@@ -515,11 +537,13 @@ void JFJochXFELReader::Place() {
|
||||
{
|
||||
std::unique_lock ul(hdf5_mutex);
|
||||
frame_masks_.clear();
|
||||
frame_mask_layouts_.clear();
|
||||
for (const auto &m: geom_.masks) {
|
||||
std::vector<uint32_t> v;
|
||||
if (m.file.empty() && (m.data.find('%') != std::string::npos
|
||||
|| FileAt(0).GetDimension(m.data).size() == 3)) {
|
||||
frame_masks_.push_back(m); // one per shot or one plane per frame: applied as each is read
|
||||
frame_mask_layouts_.push_back(FrameMaskLayout(m));
|
||||
continue;
|
||||
}
|
||||
if (!m.file.empty()) {
|
||||
@@ -775,7 +799,8 @@ void JFJochXFELReader::Close() {
|
||||
}
|
||||
|
||||
template <class Buffer>
|
||||
bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const {
|
||||
bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buffer &stored,
|
||||
bool keep_compressed) const {
|
||||
const std::vector<hsize_t> start = frame.index >= 0
|
||||
? std::vector<hsize_t>{static_cast<hsize_t>(frame.index), 0, 0}
|
||||
: std::vector<hsize_t>{0, 0};
|
||||
@@ -788,7 +813,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
|
||||
std::unique_lock ul(hdf5_mutex);
|
||||
HDF5DataSet dataset(FileAt(frame.file), frame.dataset);
|
||||
unsigned filter_mask = 0;
|
||||
if (layout_.direct_chunks) {
|
||||
if (layout.direct_chunks) {
|
||||
haddr_t addr = HADDR_UNDEF;
|
||||
hsize_t size = 0;
|
||||
if (H5Dget_chunk_info_by_coord(dataset.GetID(), start.data(), &filter_mask, &addr, &size) < 0
|
||||
@@ -800,8 +825,8 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
|
||||
}
|
||||
// A chunk stored with one of its filters skipped (its filter mask) is not the encoding decoded
|
||||
// below: HDF5 reads it, as it reads a frame that is not stored as one chunk.
|
||||
if (!layout_.direct_chunks || filter_mask != 0) {
|
||||
if (layout_.plain_type)
|
||||
if (!layout.direct_chunks || filter_mask != 0) {
|
||||
if (layout.plain_type)
|
||||
dataset.ReadVectorToU8(stored, start, count);
|
||||
else {
|
||||
std::vector<float> v(frame_width_ * frame_height_);
|
||||
@@ -814,7 +839,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
|
||||
raw_file = RawFileAt(frame.file);
|
||||
address += user_block_[frame.file]; // HDF5's chunk addresses count from the end of the user block
|
||||
}
|
||||
if (keep_compressed && !layout_.deflated) {
|
||||
if (keep_compressed && !layout.deflated) {
|
||||
stored.resize(chunk_size);
|
||||
raw_file->ReadAt(stored.data(), stored.size(), address);
|
||||
return true;
|
||||
@@ -824,19 +849,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
|
||||
raw_file->ReadAt(compressed.data(), compressed.size(), address);
|
||||
|
||||
const size_t n = frame_width_ * frame_height_;
|
||||
const size_t size = layout_.stored_size;
|
||||
if (layout_.deflated) {
|
||||
if (!layout_.shuffled)
|
||||
const size_t size = layout.stored_size;
|
||||
if (layout.deflated) {
|
||||
if (!layout.shuffled)
|
||||
stored.resize(n * size);
|
||||
else
|
||||
shuffled.resize(n * size);
|
||||
uint8_t *inflated = layout_.shuffled ? shuffled.data() : stored.data();
|
||||
uint8_t *inflated = layout.shuffled ? shuffled.data() : stored.data();
|
||||
uLongf inflated_size = n * size;
|
||||
if (uncompress(inflated, &inflated_size, compressed.data(), compressed.size()) != Z_OK
|
||||
|| inflated_size != n * size)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"Cannot inflate " + frame.dataset + " of " + files_[frame.file]);
|
||||
if (layout_.shuffled) {
|
||||
if (layout.shuffled) {
|
||||
stored.resize(n * size);
|
||||
for (size_t i = 0; i < n; i++)
|
||||
for (size_t b = 0; b < size; b++)
|
||||
@@ -844,7 +869,7 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, Buffer &stored, bool keep_
|
||||
}
|
||||
} else {
|
||||
stored.resize(n * size);
|
||||
JFJochDecompressPtr(stored.data(), layout_.algorithm, compressed.data(), compressed.size(), n, size);
|
||||
JFJochDecompressPtr(stored.data(), layout.algorithm, compressed.data(), compressed.size(), n, size);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -884,24 +909,37 @@ CompressedImage JFJochXFELReader::DecodeInto(int64_t image_number, Buffer &buffe
|
||||
const bool as_stored = blocks_.empty() && layout_.plain_type && !layout_.stored_float && layout_.stored_signed
|
||||
&& adu_per_photon_ == 1.0f;
|
||||
if (as_stored && frame_masks_.empty()) {
|
||||
const bool compressed = ReadStored(frame, buffer, true);
|
||||
const bool compressed = ReadStored(frame, layout_, buffer, true);
|
||||
return CompressedImage(buffer.data(), buffer.size(), width_, height_, stored_mode,
|
||||
compressed ? layout_.algorithm : CompressionAlgorithm::NO_COMPRESSION);
|
||||
}
|
||||
thread_local std::vector<uint8_t> stored;
|
||||
ReadStored(frame, stored, false);
|
||||
ReadStored(frame, layout_, stored, false);
|
||||
|
||||
// A pixel a per-frame mask marks in this frame is handed out as no measurement.
|
||||
std::vector<uint8_t> masked;
|
||||
if (!frame_masks_.empty()) {
|
||||
masked.assign(frame_width_ * frame_height_, 0);
|
||||
std::unique_lock ul(hdf5_mutex);
|
||||
for (const auto &m: frame_masks_) {
|
||||
const auto path = ReplaceEvent(m.data, frame.group);
|
||||
const auto v = frame.index >= 0
|
||||
? FileAt(frame.file).ReadVector<uint32_t>(path, {static_cast<hsize_t>(frame.index), 0, 0},
|
||||
{1, frame_height_, frame_width_})
|
||||
: FileAt(frame.file).ReadVector<uint32_t>(path, {0, 0}, {frame_height_, frame_width_});
|
||||
for (size_t i = 0; i < frame_masks_.size(); i++) {
|
||||
const auto &m = frame_masks_[i];
|
||||
const auto &mask_layout = frame_mask_layouts_[i];
|
||||
const Frame mask{frame.file, ReplaceEvent(m.data, frame.group), frame.index, frame.group};
|
||||
std::vector<uint32_t> v(frame_width_ * frame_height_);
|
||||
if (mask_layout.plain_type) {
|
||||
thread_local std::vector<uint8_t> mask_stored;
|
||||
ReadStored(mask, mask_layout, mask_stored, false);
|
||||
if (mask_layout.stored_size == 2) {
|
||||
const auto *p = reinterpret_cast<const uint16_t *>(mask_stored.data());
|
||||
std::copy(p, p + v.size(), v.begin());
|
||||
} else
|
||||
std::memcpy(v.data(), mask_stored.data(), v.size() * sizeof(uint32_t));
|
||||
} else {
|
||||
std::unique_lock ul(hdf5_mutex);
|
||||
v = frame.index >= 0
|
||||
? FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {static_cast<hsize_t>(frame.index), 0, 0},
|
||||
{1, frame_height_, frame_width_})
|
||||
: FileAt(frame.file).ReadVector<uint32_t>(mask.dataset, {0, 0}, {frame_height_, frame_width_});
|
||||
}
|
||||
MarkBad(m, v, masked);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,6 +96,9 @@ class JFJochXFELReader : public JFJochReader {
|
||||
bool operator==(const Layout &) const = default;
|
||||
};
|
||||
Layout layout_;
|
||||
// For each of frame_masks_, how it is stored where it is read as the frames are (ReadStored): plain
|
||||
// unsigned integers of the frame's size, stored alike in every file; plain_type false where HDF5 reads it.
|
||||
std::vector<Layout> frame_mask_layouts_;
|
||||
// Positional-read handles on the files, opened as first needed (under hdf5_mutex), with the size of
|
||||
// each file's user block, which HDF5 chunk addresses count from. Like open_files_, only the most
|
||||
// recently used MAX_OPEN_FILES are held (raw_order_); a reader still using a released one keeps it.
|
||||
@@ -103,6 +106,7 @@ class JFJochXFELReader : public JFJochReader {
|
||||
mutable std::vector<uint64_t> user_block_;
|
||||
mutable std::vector<size_t> raw_order_;
|
||||
static Layout ReadLayout(HDF5ReadOnlyFile &file, const Frame &frame);
|
||||
Layout FrameMaskLayout(const xfel::Mask &m) const; // under hdf5_mutex
|
||||
|
||||
// The files, in the order given, every frame in them, and the run's mean of every value the geometry
|
||||
// takes from the files.
|
||||
@@ -116,10 +120,11 @@ class JFJochXFELReader : public JFJochReader {
|
||||
int64_t image_number,
|
||||
bool update_dataset) override;
|
||||
|
||||
// The frame's pixels as stored (plain_type), or as float, into stored; with keep_compressed, a chunk
|
||||
// decoded by CompressedImage (not deflated) is left as stored in the file, and true returned.
|
||||
// The frame's pixels, stored as layout says, as stored (plain_type), or as float, into stored; with
|
||||
// keep_compressed, a chunk decoded by CompressedImage (not deflated) is left as stored in the file, and
|
||||
// true returned.
|
||||
template <class Buffer>
|
||||
bool ReadStored(const Frame &frame, Buffer &stored, bool keep_compressed) const;
|
||||
bool ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, bool keep_compressed) const;
|
||||
// The stored pixels as the image handed out (as stored, or placed), as Out, each through convert; a pixel masked marks is err.
|
||||
template <class Out, class In, class Buffer, class Convert>
|
||||
void PlaceInto(const In *in, const std::vector<uint8_t> &masked, Out err, Buffer &buffer, Convert convert) const;
|
||||
|
||||
@@ -280,6 +280,11 @@ TEST_CASE("JFJochXFELReader_GeometryMasks", "[portable]") {
|
||||
std::vector<uint16_t> mask(2 * 24, 0);
|
||||
mask[1 * 6 + 1] = 0x20; // frame 0 only, (fs 1, ss 1): a bit Cheetah's own set does not include
|
||||
WriteFrames(f, "entry_1/data_1/mask", H5T_NATIVE_UINT16, mask.data(), {2, 4, 6});
|
||||
// Frame 1's mask chunk is stored with both its filters skipped: HDF5 reads that one.
|
||||
const hid_t ds = H5Dopen2(f, "entry_1/data_1/mask", H5P_DEFAULT);
|
||||
const hsize_t offset[3] = {1, 0, 0};
|
||||
REQUIRE(H5Dwrite_chunk(ds, H5P_DEFAULT, 0x3, offset, 24 * sizeof(uint16_t), mask.data() + 24) >= 0);
|
||||
H5Dclose(ds);
|
||||
const std::string det = "entry_1/instrument_1/detector_1/";
|
||||
std::vector<uint16_t> shared(24, 0);
|
||||
shared[3 * 6 + 5] = 8; // (fs 5, ss 3) dead to Cheetah
|
||||
|
||||
Reference in new issue
Block a user