From 1f4f77fe4202f4e89331a1115bf8ecf7a0521d25 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 22 Aug 2026 21:57:12 +0200 Subject: [PATCH] Choose images_per_file from the acquisition when it is not given The value means three different things at once. It is the unit of writer parallelism - whole files go round-robin to the writers, (image_number / images_per_file) % socket.size() in ZMQStream2Pusher::SendImage and TCPStreamPusher - it multiplies writer memory linearly, since every data-file plugin reserves per file, and it decides whether a legacy master is readable at all, because dxtbx follows only the first data file of one. That last point is what makes a flat default wrong. Measured with DIALS on a 2500-image rotation sweep written as legacy: split into five files it reports 2500 images and then raises IndexError beyond image 499, so it half-works silently; in one file all 2500 read. AutoPROC does not read VDS, so legacy has to stay the default, which leaves the file count as the only lever. So make it optional and resolve it from the acquisition. A rotation sweep of at most 20000 images goes into one data file - rotation datasets are small enough, and one writer keeps up with them. A grid scan splits on whole fast-axis rows, so a file is a meaningful piece of the grid. Stills and serial keep 1000, where the image count far exceeds it and the parallelism and the bounded writer memory are what matter. An explicit value is always taken literally. GetImagesPerFile is the single place this is resolved, and it must always return a fixed non-zero number, because everything downstream - receiver, pusher, puller, writer - requires one. That was already true of the old 0 = "one file" spelling; 0 is now gone from the API and omitting the field says the same thing better. Co-Authored-By: Claude Opus 5 (1M context) --- broker/OpenAPIConvert.cpp | 3 +- broker/gen/model/Dataset_settings.cpp | 6 +-- broker/gen/model/Dataset_settings.h | 2 +- broker/jfjoch_api.yaml | 10 ++-- broker/redoc-static.html | 10 ++-- common/DatasetSettings.cpp | 8 +-- common/DatasetSettings.h | 6 +-- common/Definitions.h | 6 +++ common/DiffractionExperiment.cpp | 38 ++++++++++++--- common/DiffractionExperiment.h | 2 +- docs/CHANGELOG.md | 4 ++ docs/python_client/docs/DatasetSettings.md | 2 +- frontend/src/client/types.gen.ts | 7 ++- frontend/src/client/zod.gen.ts | 2 +- tests/DiffractionExperimentTest.cpp | 57 +++++++++++++++++++++- 15 files changed, 133 insertions(+), 30 deletions(-) diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index 7fe98181..cf59c923 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -675,7 +675,8 @@ DatasetSettings Convert(const org::openapitools::server::model::Dataset_settings ret.SampleName(input.getSampleName()); ret.HeaderAppendix(input.getHeaderAppendix()); ret.ImageAppendix(input.getImageAppendix()); - ret.ImagesPerFile(input.getImagesPerFile()); + if (input.imagesPerFileIsSet()) + ret.ImagesPerFile(input.getImagesPerFile()); if (input.dataReductionFactorSerialmxIsSet()) ret.LossyCompressionSerialMX(input.getDataReductionFactorSerialmx()); diff --git a/broker/gen/model/Dataset_settings.cpp b/broker/gen/model/Dataset_settings.cpp index b4059782..0b5e912c 100644 --- a/broker/gen/model/Dataset_settings.cpp +++ b/broker/gen/model/Dataset_settings.cpp @@ -33,7 +33,7 @@ Dataset_settings::Dataset_settings() m_Incident_energy_keV = 0.0f; m_File_prefix = ""; m_File_prefixIsSet = false; - m_Images_per_file = 1000L; + m_Images_per_file = 0L; m_Images_per_fileIsSet = false; m_Space_group_number = 0L; m_Space_group_numberIsSet = false; @@ -192,10 +192,10 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP const std::string currentValuePath = _pathPrefix + ".imagesPerFile"; - if (value < 0ll) + if (value < 1ll) { success = false; - msg << currentValuePath << ": must be greater than or equal to 0;"; + msg << currentValuePath << ": must be greater than or equal to 1;"; } } diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index d80efd53..40e31cb0 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -113,7 +113,7 @@ public: bool filePrefixIsSet() const; void unsetFile_prefix(); /// - /// Number of files in a single HDF5 data file (0 = write all images to a single data file). + /// Number of images in a single HDF5 data file, taken literally when given. If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images goes into a single data file, so the dataset stays readable by programs that follow only the first data file of a legacy master; a grid scan is split on a whole number of fast-axis rows; anything else (stills, serial) uses 1000. /// int64_t getImagesPerFile() const; void setImagesPerFile(int64_t const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index faf752dc..4aa8c9de 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -429,9 +429,13 @@ components: images_per_file: type: integer format: int64 - minimum: 0 - default: 1000 - description: Number of files in a single HDF5 data file (0 = write all images to a single data file). + minimum: 1 + description: | + Number of images in a single HDF5 data file, taken literally when given. + If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images + goes into a single data file, so the dataset stays readable by programs that follow only + the first data file of a legacy master; a grid scan is split on a whole number of fast-axis + rows; anything else (stills, serial) uses 1000. space_group_number: type: integer format: int64 diff --git a/broker/redoc-static.html b/broker/redoc-static.html index ede61c2a..d6e9454d 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -453,7 +453,11 @@ Beam center in X direction [pixels]

incident_energy_keV
required
number <float> [ 0.001 .. 500 ]

Used to calculate /entry/beam/incident_wavelength in NXmx Incident particle (photon, electron) energy in keV

file_prefix
string
Default: ""

Prefix for filenames. If left empty, no file will be saved.

-
images_per_file
integer <int64> >= 0
Default: 1000

Number of files in a single HDF5 data file (0 = write all images to a single data file).

+
images_per_file
integer <int64> >= 1

Number of images in a single HDF5 data file, taken literally when given. +If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images +goes into a single data file, so the dataset stays readable by programs that follow only +the first data file of a legacy master; a grid scan is split on a whole number of fast-axis +rows; anything else (stills, serial) uses 1000.

space_group_number
integer <int64> [ 1 .. 194 ]

Number of space group for the crystal. Currently used solely as metadata, not relevant for image processing done in Jungfraujoch.

sample_name
string
Default: ""

/entry/sample/name in NXmx Sample name

@@ -512,7 +516,7 @@ Assuming that Smargon is used as static positioner and not moving during the sca

Request samples

Content type
application/json
{
  • "images_per_trigger": 1,
  • "ntrigger": 1,
  • "image_time_us": 0,
  • "beam_x_pxl": 0.1,
  • "beam_y_pxl": 0.1,
  • "detector_distance_mm": 0.1,
  • "incident_energy_keV": 0.001,
  • "file_prefix": "",
  • "images_per_file": 1000,
  • "space_group_number": 1,
  • "sample_name": "",
  • "compression": "bslz4",
  • "total_flux": 0.1,
  • "transmission": 1,
  • "goniometer": {
    },
  • "grid_scan": {
    },
  • "header_appendix": null,
  • "image_appendix": null,
  • "data_reduction_factor_serialmx": 1,
  • "pixel_value_low_threshold": 0,
  • "run_number": 0,
  • "run_name": "string",
  • "experiment_group": "string",
  • "poisson_compression": 16,
  • "write_nxmx_hdf5_master": true,
  • "save_calibration": true,
  • "polarization_factor": -1,
  • "ring_current_mA": 0.1,
  • "sample_temperature_K": 0.1,
  • "poni_rot1_rad": 0,
  • "poni_rot2_rad": 0,
  • "poni_rot3_rad": 0,
  • "unit_cell": {
    },
  • "spot_finding": true,
  • "smargon": {
    },
  • "max_spot_count": 250,
  • "detect_ice_rings": true,
  • "async_start": false,
  • "xray_fluorescence_spectrum": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition running

Block execution of external script till detector and Jungfraujoch are ready to collect data. +

http://localhost:5232/start

Request samples

Content type
application/json
{
  • "images_per_trigger": 1,
  • "ntrigger": 1,
  • "image_time_us": 0,
  • "beam_x_pxl": 0.1,
  • "beam_y_pxl": 0.1,
  • "detector_distance_mm": 0.1,
  • "incident_energy_keV": 0.001,
  • "file_prefix": "",
  • "images_per_file": 1,
  • "space_group_number": 1,
  • "sample_name": "",
  • "compression": "bslz4",
  • "total_flux": 0.1,
  • "transmission": 1,
  • "goniometer": {
    },
  • "grid_scan": {
    },
  • "header_appendix": null,
  • "image_appendix": null,
  • "data_reduction_factor_serialmx": 1,
  • "pixel_value_low_threshold": 0,
  • "run_number": 0,
  • "run_name": "string",
  • "experiment_group": "string",
  • "poisson_compression": 16,
  • "write_nxmx_hdf5_master": true,
  • "save_calibration": true,
  • "polarization_factor": -1,
  • "ring_current_mA": 0.1,
  • "sample_temperature_K": 0.1,
  • "poni_rot1_rad": 0,
  • "poni_rot2_rad": 0,
  • "poni_rot3_rad": 0,
  • "unit_cell": {
    },
  • "spot_finding": true,
  • "smargon": {
    },
  • "max_spot_count": 250,
  • "detect_ice_rings": true,
  • "async_start": false,
  • "xray_fluorescence_spectrum": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Wait for acquisition running

Block execution of external script till detector and Jungfraujoch are ready to collect data. To not block web server for a indefinite period of time, the procedure is provided with a timeout. Extending timeout is possible, but requires to ensure safety that client will not close the connection and retry the connection.

query Parameters
timeout
integer [ 0 .. 3600 ]
Default: 60

Timeout in seconds (0 == immediate response)

@@ -977,7 +981,7 @@ then image might be replaced in the buffer between calling /images and /image.cb