Files
Jungfraujoch/common/DatasetSettings.cpp
T
leonarski_fandClaude Opus 5 1f4f77fe42 Choose images_per_file from the acquisition when it is not given
The value means three different things at once. It is the unit of writer
parallelism - whole files go round-robin to the writers,
(image_number / images_per_file) % socket.size() in ZMQStream2Pusher::SendImage
and TCPStreamPusher - it multiplies writer memory linearly, since every data-file
plugin reserves per file, and it decides whether a legacy master is readable at
all, because dxtbx follows only the first data file of one.

That last point is what makes a flat default wrong. Measured with DIALS on a
2500-image rotation sweep written as legacy: split into five files it reports
2500 images and then raises IndexError beyond image 499, so it half-works
silently; in one file all 2500 read. AutoPROC does not read VDS, so legacy has to
stay the default, which leaves the file count as the only lever.

So make it optional and resolve it from the acquisition. A rotation sweep of at
most 20000 images goes into one data file - rotation datasets are small enough,
and one writer keeps up with them. A grid scan splits on whole fast-axis rows, so
a file is a meaningful piece of the grid. Stills and serial keep 1000, where the
image count far exceeds it and the parallelism and the bounded writer memory are
what matter. An explicit value is always taken literally.

GetImagesPerFile is the single place this is resolved, and it must always return
a fixed non-zero number, because everything downstream - receiver, pusher, puller,
writer - requires one. That was already true of the old 0 = "one file" spelling;
0 is now gone from the API and omitting the field says the same thing better.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-22 21:57:12 +02:00

519 lines
15 KiB
C++

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <cmath>
#include "DatasetSettings.h"
#include "Definitions.h"
#include "JFJochException.h"
#include "CheckPath.h"
#define check_max(param, val, max) if ((val) > (max)) throw JFJochException(JFJochExceptionCategory::InputParameterAboveMax, param)
#define check_min(param, val, min) if ((val) < (min)) throw JFJochException(JFJochExceptionCategory::InputParameterBelowMin, param)
#define check_finite(param, val) if (!std::isfinite(val)) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, param)
DatasetSettings::DatasetSettings() {
photon_energy_keV = WVL_1A_IN_KEV;
detector_distance_mm = 100;
beam_x_pxl = 0.0;
beam_y_pxl = 0.0;
file_prefix = "test";
ntrigger = 1;
images_per_trigger = 1;
compression = CompressionAlgorithm::BSHUF_LZ4;
data_reduction_factor_serialmx = 1.0;
write_nxmx_hdf5_master = true;
spot_finding_enable = true;
poni_rot_1_rad = 0.0f;
poni_rot_2_rad = 0.0f;
poni_rot_3_rad = 0.0f;
max_spot_count = std::min(MAX_SPOT_COUNT, 250);
detect_ice_rings = false;
}
DatasetSettings &DatasetSettings::ImagesPerTrigger(int64_t input) {
check_max("Total number of images", input, 10*1000*1000);
check_min("Total number of images", input, 0);
images_per_trigger = input;
return *this;
}
DatasetSettings &DatasetSettings::NumTriggers(int64_t input) {
check_max("Total number of triggers", input, 10*1000*1000);
check_min("Total number of triggers", input, 1);
ntrigger = input;
return *this;
}
DatasetSettings &DatasetSettings::PhotonEnergy_keV(float input) {
check_finite("Energy (keV)", input);
check_min("Energy (keV)", input, MIN_ENERGY_KEV);
check_max("Energy (keV)", input, MAX_ENERGY_KEV);
photon_energy_keV = input;
return *this;
}
DatasetSettings &DatasetSettings::BeamX_pxl(float input) {
check_finite("Beam center x", input);
beam_x_pxl = input;
return *this;
}
DatasetSettings &DatasetSettings::BeamY_pxl(float input) {
check_finite("Beam center y", input);
beam_y_pxl = input;
return *this;
}
DatasetSettings &DatasetSettings::DetectorDistance_mm(float input) {
check_finite("Detector distance (mm)", input);
check_min("Detector distance (mm)", input, 1);
detector_distance_mm = input;
return *this;
}
DatasetSettings &DatasetSettings::FilePrefix(std::string input) {
CheckPath(input); // multi-user guard: no absolute path, no '..' traversal
return FilePrefixTrusted(std::move(input));
}
DatasetSettings &DatasetSettings::FilePrefixTrusted(std::string input) {
// Offline/local callers (rugnux, the viewer's processing) supply their own output path, so the
// CheckPath guard applied by FilePrefix() is deliberately skipped here - an absolute path is
// allowed. Not reachable from the broker/writer remote-input path.
if ((input.find("_master.h5") == input.length() - 10) && (input.length() > 10))
file_prefix = input.substr(0, input.length() - 10);
else
file_prefix = input;
return *this;
}
DatasetSettings &DatasetSettings::Compression(CompressionAlgorithm input) {
switch (input) {
case CompressionAlgorithm::NO_COMPRESSION:
case CompressionAlgorithm::BSHUF_LZ4:
case CompressionAlgorithm::BSHUF_ZSTD:
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
case CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF:
compression = input;
break;
default:
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Invalid value for compression enum parameter");
}
return *this;
}
DatasetSettings &DatasetSettings::SetUnitCell(const std::optional<UnitCell> &cell) {
// If one or more cell parameters are zero - assume cell is not provided
// But if all parameters non-zero - throw exception for non-sense values
if (cell && (cell->a != 0) && (cell->b != 0) && (cell->c != 0)
&& (cell->alpha != 0) && (cell->beta != 0) && (cell->gamma != 0)) {
check_min("Unit cell a", cell->a, 0.1);
check_min("Unit cell b", cell->b, 0.1);
check_min("Unit cell c", cell->c, 0.1);
check_min("Angle alpha", cell->alpha, 5.0);
check_min("Angle beta", cell->beta, 5.0);
check_min("Angle gamma", cell->gamma, 5.0);
check_max("Angle alpha", cell->alpha, 355.0);
check_max("Angle beta", cell->beta, 355.0);
check_max("Angle gamma", cell->gamma, 360);
unit_cell = cell;
} else
unit_cell.reset();
return *this;
}
DatasetSettings &DatasetSettings::SpaceGroupNumber(std::optional<int64_t> input) {
if (input) {
check_min("Space group number", input, 1);
check_max("Space group number", input, 230);
}
space_group_number = input;
return *this;
}
DatasetSettings &DatasetSettings::SampleName(std::string input) {
sample_name = input;
return *this;
}
DatasetSettings &DatasetSettings::AttenuatorTransmission(const std::optional<float> &input) {
if (input) {
check_finite("Attenuator transmission", input.value());
check_max("Attenuator transmission", input.value(), 1.0);
check_min("Attenuator transmission", input.value(), 0.0);
}
attenuator_transmission = input;
return *this;
}
DatasetSettings &DatasetSettings::TotalFlux(const std::optional<float> &input) {
if (input) {
check_finite("Total flux", input.value());
check_min("Total flux", input.value(), 0.0);
}
total_flux = input;
return *this;
}
DatasetSettings &DatasetSettings::Goniometer(const std::optional<GoniometerAxis> &input) {
goniometer = input;
return *this;
}
DatasetSettings &DatasetSettings::HeaderAppendix(const nlohmann::json &input) {
header_appendix = input;
return *this;
}
DatasetSettings &DatasetSettings::ImageAppendix(const nlohmann::json &input) {
image_appendix = input;
return *this;
}
std::optional<float> DatasetSettings::GetAttenuatorTransmission() const {
return attenuator_transmission;
}
std::optional<float> DatasetSettings::GetTotalFlux() const {
return total_flux;
}
const std::optional<GoniometerAxis> &DatasetSettings::GetGoniometer() const {
return goniometer;
}
std::optional<GoniometerAxis> &DatasetSettings::Goniometer() {
return goniometer;
}
const nlohmann::json& DatasetSettings::GetHeaderAppendix() const {
return header_appendix;
}
const nlohmann::json& DatasetSettings::GetImageAppendix() const {
return image_appendix;
}
std::optional<UnitCell> DatasetSettings::GetUnitCell() const {
return unit_cell;
}
std::optional<int64_t> DatasetSettings::GetSpaceGroupNumber() const {
return space_group_number;
}
std::string DatasetSettings::GetSampleName() const {
return sample_name;
}
float DatasetSettings::GetPhotonEnergy_keV() const {
return photon_energy_keV;
}
float DatasetSettings::GetBeamX_pxl() const {
return beam_x_pxl;
}
float DatasetSettings::GetBeamY_pxl() const {
return beam_y_pxl;
}
float DatasetSettings::GetDetectorDistance_mm() const {
return detector_distance_mm;
}
Coord DatasetSettings::GetScatteringVector() const {
return {0, 0, photon_energy_keV / WVL_1A_IN_KEV};
}
std::string DatasetSettings::GetFilePrefix() const {
return file_prefix;
}
CompressionAlgorithm DatasetSettings::GetCompressionAlgorithm() const {
return compression;
}
int64_t DatasetSettings::GetNumTriggers() const {
return ntrigger;
}
int64_t DatasetSettings::GetImageNumPerTrigger() const {
return images_per_trigger;
}
DatasetSettings &DatasetSettings::ImagesPerFile(const std::optional<int64_t> &input) {
if (input.has_value())
check_min("Images per file", input.value(), 1);
images_per_file = input;
return *this;
}
std::optional<int64_t> DatasetSettings::GetImagesPerFile() const {
return images_per_file;
}
DatasetSettings &DatasetSettings::LossyCompressionSerialMX(float input) {
check_min("Data reduction factor for serial MX", input, 0.0);
check_max("Data reduction factor for serial MX", input, 1.0);
data_reduction_factor_serialmx = input;
return *this;
}
float DatasetSettings::GetLossyCompressionSerialMX() const {
return data_reduction_factor_serialmx;
}
DatasetSettings &DatasetSettings::RunNumber(const std::optional<uint64_t> &input) {
if (input) {
check_min("Run number", input, 0);
check_max("Run number", input, INT64_MAX);
}
run_number = input;
return *this;
}
DatasetSettings & DatasetSettings::RunName(const std::optional<std::string> &input) {
if (input && input.value().empty())
run_name = {};
else
run_name = input;
return *this;
}
DatasetSettings &DatasetSettings::ExperimentGroup(const std::string &input) {
group = input;
return *this;
}
std::optional<uint64_t> DatasetSettings::GetRunNumber() const {
return run_number;
}
std::optional<std::string> DatasetSettings::GetRunName() const {
return run_name;
}
std::string DatasetSettings::GetExperimentGroup() const {
return group;
}
std::optional<std::chrono::nanoseconds> DatasetSettings::GetImageTime() const {
return image_time;
}
DatasetSettings &DatasetSettings::ImageTime(const std::optional<std::chrono::nanoseconds> &input) {
if (input && (input.value().count() == 0))
image_time = {};
else
image_time = input;
return *this;
}
DatasetSettings &DatasetSettings::LossyCompressionPoisson(std::optional<int64_t> input) {
if (!input || (input == 0))
compression_poisson_factor = {};
else {
check_min("Poisson compression factor", input.value(), 1);
check_max("Poisson compression factor", input.value(), 16);
compression_poisson_factor = input;
}
return *this;
}
std::optional<int64_t> DatasetSettings::GetLossyCompressionPoisson() const {
return compression_poisson_factor;
}
DatasetSettings &DatasetSettings::PixelValueLowThreshold(const std::optional<int64_t> &input) {
if (!input || (input == 0))
pixel_value_low_threshold = {};
else {
check_min("Pixel value low threshold", input.value(), 0);
check_max("Pixel value low threshold", input.value(), INT24_MAX - 1);
pixel_value_low_threshold = input;
}
return *this;
}
DatasetSettings &DatasetSettings::PixelValueHighThreshold(const std::optional<int64_t> &input) {
if (!input || (input == 0))
pixel_value_high_threshold = {};
else {
check_min("Pixel value high threshold", input.value(), 1);
check_max("Pixel value high threshold", input.value(), INT32_MAX);
pixel_value_high_threshold = input;
}
return *this;
}
std::optional<int64_t> DatasetSettings::GetPixelValueLowThreshold() const {
return pixel_value_low_threshold;
}
std::optional<int64_t> DatasetSettings::GetPixelValueHighThreshold() const {
return pixel_value_high_threshold;
}
bool DatasetSettings::IsWriteNXmxHDF5Master() const {
return write_nxmx_hdf5_master;
}
DatasetSettings &DatasetSettings::WriteNXmxHDF5Master(bool input) {
write_nxmx_hdf5_master = input;
return *this;
}
std::optional<bool> DatasetSettings::IsSaveCalibration() const {
return save_calibration;
}
DatasetSettings &DatasetSettings::SaveCalibration(std::optional<bool> input) {
save_calibration = input;
return *this;
}
DatasetSettings &DatasetSettings::GridScan(const std::optional<GridScanSettings> &input) {
grid_scan = input;
return *this;
}
std::optional<GridScanSettings> &DatasetSettings::GridScan() {
return grid_scan;
}
const std::optional<GridScanSettings> &DatasetSettings::GetGridScan() const {
return grid_scan;
}
std::optional<float> DatasetSettings::GetPolarizationFactor() const {
return polarization_factor;
}
DatasetSettings &DatasetSettings::BandwidthFWHM(const std::optional<float> &input) {
bandwidth_fwhm = input;
return *this;
}
std::optional<float> DatasetSettings::GetBandwidthFWHM() const {
return bandwidth_fwhm;
}
float DatasetSettings::GetPoniRot3_rad() const {
return poni_rot_3_rad;
}
float DatasetSettings::GetPoniRot2_rad() const {
return poni_rot_2_rad;
}
float DatasetSettings::GetPoniRot1_rad() const {
return poni_rot_1_rad;
}
DatasetSettings &DatasetSettings::PoniRot1_rad(float input) {
check_finite("PONI rotation 1 (radians)", input);
poni_rot_1_rad = input;
return *this;
}
DatasetSettings &DatasetSettings::PoniRot2_rad(float input) {
check_finite("PONI rotation 2 (radians)", input);
poni_rot_2_rad = input;
return *this;
}
DatasetSettings &DatasetSettings::PoniRot3_rad(float input) {
check_finite("PONI rotation 3 (radians)", input);
poni_rot_3_rad = input;
return *this;
}
DatasetSettings &DatasetSettings::PolarizationFactor(const std::optional<float> &input) {
if (input.has_value()) {
check_finite("Polarization factor", input.value());
check_min("Polarization factor", input.value(), -1.0);
check_max("Polarization factor", input.value(), 1.0);
}
polarization_factor = input;
return *this;
}
std::optional<float> DatasetSettings::GetRingCurrent_mA() const {
return ring_current_mA;
}
DatasetSettings &DatasetSettings::RingCurrent_mA(const std::optional<float> &input) {
if (input.has_value()) {
check_min("Ring current (mA)", input, 0.0);
}
ring_current_mA = input;
return *this;
}
std::optional<float> DatasetSettings::GetSampleTemperature_K() const {
return sample_temperature_K;
}
DatasetSettings &DatasetSettings::SampleTemperature_K(const std::optional<float> &input) {
if (input.has_value()) {
check_min("Sample temperature (K)", input, 0.0);
check_max("Sample temperature (K)", input, 1000.0);
}
sample_temperature_K = input;
return *this;
}
DatasetSettings &DatasetSettings::SpotFindingEnable(bool input) {
spot_finding_enable = input;
return *this;
}
bool DatasetSettings::IsSpotFindingEnabled() const {
return spot_finding_enable;
}
DatasetSettings &DatasetSettings::MaxSpotCount(int64_t input) {
check_min("Max spot count", input, 10);
check_max("Max spot count", input, MAX_SPOT_COUNT);
max_spot_count = input;
return *this;
}
DatasetSettings & DatasetSettings::DetectIceRings(bool input) {
detect_ice_rings = input;
return *this;
}
bool DatasetSettings::IsDetectIceRings() const {
return detect_ice_rings;
}
DatasetSettings &DatasetSettings::FluorescenceSpectrum(const XrayFluorescenceSpectrum &input) {
fluorescence_spectrum = input;
return *this;
}
const XrayFluorescenceSpectrum & DatasetSettings::GetFluorescenceSpectrum() const {
return fluorescence_spectrum;
}
int64_t DatasetSettings::GetMaxSpotCount() const {
return max_spot_count;
}
std::optional<SmargonPosition> DatasetSettings::GetSmargonPosition() const {
return smargon_position;
}
DatasetSettings &DatasetSettings::Smargon(const std::optional<SmargonPosition> &input) {
smargon_position = input;
return *this;
}