Build Packages / Create release (push) Successful in 21s
Build Packages / build:viewer:macos-arm64:nocuda (push) Successful in 3m1s
Build Packages / build:rugnux:macos-arm64:nocuda (push) Successful in 2m18s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 8m34s
Build Packages / build:rugnux:linux-x86_64:cuda (push) Successful in 9m22s
Build Packages / build:viewer:linux-x86_64:nocuda (push) Successful in 10m35s
Build Packages / build:viewer:linux-x86_64:cuda (push) Successful in 11m40s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 16m1s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 16m31s
Build Packages / build:viewer:windows-x86_64:nocuda (push) Successful in 17m47s
Build Packages / build:viewer:windows-x86_64:cuda (push) Successful in 20m21s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 24m42s
Build Packages / build:rugnux:windows-x86_64:cuda (push) Successful in 11m12s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 20m31s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 20m10s
Build Packages / Generate python client (push) Successful in 35s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 20m59s
Build Packages / Build documentation (push) Successful in 1m27s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 20m49s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 18m10s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 18m20s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 14m31s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 11m3s
Build Packages / Unit tests (push) Successful in 1h20m31s
rugnux will mostly run on GPFS/Lustre-type storage. GPFS caches in its own
fixed-size pagepool, often smaller than a dataset, so streaming the files
ahead of the consumer can evict data before it is used; and choosing the
behaviour by the underlying filesystem is not wanted. This removes the
ReadAhead streamer of aa3fa6b9c: reader/ReadAhead.{h,cpp},
JFJochReader::StartReadAhead/NoteImageRead, the DataFiles() lists that only
it used (HDF5 reader, HDF5ImageSource, HDF5ImageLocator, CBF/marCCD/SMV),
and its start in rugnux_cli.cpp.
Kept from the same commit: the in-order CBF/marCCD/SMV header scan
(sweep::ForEachInOrder) and the 16 kB -> 256 kB CBF header probe, with its
test. They only change the order and size of reads the program makes
anyway, hold no memory and read less (cold 2400-frame CBF header scan
18 s -> 6 s on the measured HDD).
Output unchanged: myob p.hkl, p.mtz, p_P1.mtz and p_unmerged.mtz
byte-identical to the rc173 reference (17.1 s wall warm, 3.8 GB peak RSS).
Reader tests: [HDF5] 108 cases, CBF/marCCD/SMV/sweep/VDS/GetRawImage cases
all pass.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
141 lines
5.7 KiB
C++
141 lines
5.7 KiB
C++
// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "JFJochReader.h"
|
|
#include <future>
|
|
|
|
JFJochReader &JFJochReader::Experiment(const DiffractionExperiment &experiment) {
|
|
std::unique_lock ul(m);
|
|
default_experiment = experiment;
|
|
return *this;
|
|
}
|
|
|
|
void JFJochReader::SummationThread(int64_t image0, int64_t n_image, int64_t image_jump, JFJochReaderImage &image) {
|
|
std::vector<uint8_t> buffer;
|
|
DataMessage msg;
|
|
for (int64_t i = image0; i < n_image; i += image_jump) {
|
|
bool ret = LoadImage_i(dataset, msg, buffer, i, false);
|
|
if (ret) {
|
|
auto image_sum = std::make_shared<JFJochReaderImage>(msg, dataset);
|
|
{
|
|
std::unique_lock ul(summation_mutex);
|
|
image.AddImage(*image_sum);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::shared_ptr<JFJochReaderImage> JFJochReader::LoadImage(int64_t image_number, int64_t summation_factor) {
|
|
// It would be a mess to load two images at the same time
|
|
// so loading is protected via mutex
|
|
// yet copying share_ptr pointer is atomic and needs no mutex protection
|
|
std::unique_lock ul(m);
|
|
|
|
std::vector<uint8_t> buffer;
|
|
DataMessage msg;
|
|
|
|
if (LoadImage_i(dataset, msg, buffer, image_number, true)) {
|
|
auto image = std::make_shared<JFJochReaderImage>(msg, dataset);
|
|
if (summation_factor > 4) {
|
|
int64_t nthread = std::min<int64_t>(summation_factor - 1, 8);
|
|
|
|
std::vector<std::future<void>> futures;
|
|
for (int i = 0; i < nthread; i++)
|
|
futures.emplace_back(std::async(std::launch::async,
|
|
&JFJochReader::SummationThread, this,
|
|
image_number + 1 + i,
|
|
image_number + summation_factor,
|
|
nthread,
|
|
std::ref(*image)));
|
|
for (auto &f: futures)
|
|
f.get();
|
|
} else if (summation_factor > 1) {
|
|
SummationThread(image_number + 1, image_number + summation_factor, 1, *image);
|
|
}
|
|
return image;
|
|
}
|
|
return {};
|
|
}
|
|
|
|
std::shared_ptr<JFJochReaderRawImage> JFJochReader::GetRawImage(int64_t image_number) {
|
|
auto ret = std::make_shared<JFJochReaderRawImage>();
|
|
if (!ReadRawImage(image_number, *ret))
|
|
return {};
|
|
return ret;
|
|
}
|
|
|
|
void JFJochReader::SetStartMessage(const std::shared_ptr<JFJochReaderDataset> &val) {
|
|
std::unique_lock ul(m);
|
|
dataset = val;
|
|
}
|
|
|
|
std::shared_ptr<const JFJochReaderDataset> JFJochReader::GetDataset() const {
|
|
std::unique_lock ul(m);
|
|
return dataset;
|
|
}
|
|
|
|
void JFJochReader::UpdateGeomMetadata(const DiffractionExperiment &experiment) {
|
|
std::unique_lock ul(m);
|
|
if (!dataset)
|
|
return;
|
|
|
|
auto new_dataset = std::make_shared<JFJochReaderDataset>(*dataset);
|
|
// At the moment subset of options is limited to safe ones...need to change it in the future
|
|
|
|
new_dataset->experiment.BeamX_pxl(experiment.GetBeamX_pxl());
|
|
new_dataset->experiment.BeamY_pxl(experiment.GetBeamY_pxl());
|
|
new_dataset->experiment.DetectorDistance_mm(experiment.GetDetectorDistance_mm());
|
|
new_dataset->experiment.IncidentEnergy_keV(experiment.GetIncidentEnergy_keV());
|
|
new_dataset->experiment.PoniRot1_rad(experiment.GetDatasetSettings().GetPoniRot1_rad());
|
|
new_dataset->experiment.PoniRot2_rad(experiment.GetDatasetSettings().GetPoniRot2_rad());
|
|
new_dataset->experiment.PoniRot3_rad(experiment.GetDatasetSettings().GetPoniRot3_rad());
|
|
new_dataset->experiment.SetUnitCell(experiment.GetUnitCell());
|
|
new_dataset->experiment.SetSpaceGroup(experiment.GetGemmiSpaceGroup());
|
|
new_dataset->experiment.PolarizationFactor(experiment.GetPolarizationFactor());
|
|
new_dataset->experiment.Goniometer(experiment.GetGoniometer());
|
|
new_dataset->experiment.GridScan(experiment.GetGridScan());
|
|
new_dataset->experiment.ImportIndexingSettings(experiment.GetIndexingSettings());
|
|
new_dataset->experiment.ImportBraggIntegrationSettings(experiment.GetBraggIntegrationSettings());
|
|
new_dataset->experiment.DetectIceRings(experiment.IsDetectIceRings());
|
|
dataset = new_dataset;
|
|
}
|
|
|
|
void JFJochReader::UpdateUserMask(const std::vector<uint32_t> &mask) {
|
|
std::unique_lock ul(m);
|
|
if (!dataset)
|
|
return;
|
|
auto new_dataset = std::make_shared<JFJochReaderDataset>(*dataset);
|
|
// Copy-on-write: the mask is shared with the old snapshot, so edit a fresh copy, not in place.
|
|
auto new_mask = std::make_shared<PixelMask>(*dataset->pixel_mask);
|
|
new_mask->LoadUserMask(dataset->experiment, mask);
|
|
new_dataset->pixel_mask = new_mask;
|
|
dataset = new_dataset;
|
|
}
|
|
|
|
std::shared_ptr<JFJochReaderSpots> JFJochReader::ReadAllSpots(int64_t start_image, int64_t end_image,
|
|
int64_t stride) const {
|
|
|
|
if (start_image < 0)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Start image must be non-negative");
|
|
|
|
if (start_image > end_image)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Start image number is greater than end image number");
|
|
|
|
if (stride == 0)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Stride cannot be zero");
|
|
|
|
size_t nelems = (end_image - start_image) / stride + 1;
|
|
|
|
auto ret = std::make_shared<JFJochReaderSpots>();
|
|
ret->start_image = static_cast<int64_t>(start_image);
|
|
ret->stride = static_cast<int64_t>(stride);
|
|
ret->spots.reserve(nelems);
|
|
|
|
for (int i = 0; i < nelems; i++)
|
|
ret->spots.emplace_back(ReadSpots(start_image + i * stride));
|
|
return ret;
|
|
}
|