Choose images_per_file from the acquisition when it is not given

The value means three different things at once. It is the unit of writer
parallelism - whole files go round-robin to the writers,
(image_number / images_per_file) % socket.size() in ZMQStream2Pusher::SendImage
and TCPStreamPusher - it multiplies writer memory linearly, since every data-file
plugin reserves per file, and it decides whether a legacy master is readable at
all, because dxtbx follows only the first data file of one.

That last point is what makes a flat default wrong. Measured with DIALS on a
2500-image rotation sweep written as legacy: split into five files it reports
2500 images and then raises IndexError beyond image 499, so it half-works
silently; in one file all 2500 read. AutoPROC does not read VDS, so legacy has to
stay the default, which leaves the file count as the only lever.

So make it optional and resolve it from the acquisition. A rotation sweep of at
most 20000 images goes into one data file - rotation datasets are small enough,
and one writer keeps up with them. A grid scan splits on whole fast-axis rows, so
a file is a meaningful piece of the grid. Stills and serial keep 1000, where the
image count far exceeds it and the parallelism and the bounded writer memory are
what matter. An explicit value is always taken literally.

GetImagesPerFile is the single place this is resolved, and it must always return
a fixed non-zero number, because everything downstream - receiver, pusher, puller,
writer - requires one. That was already true of the old 0 = "one file" spelling;
0 is now gone from the API and omitting the field says the same thing better.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

broker/OpenAPIConvert.cpp

@@ -675,7 +675,8 @@ DatasetSettings Convert(const org::openapitools::server::model::Dataset_settings
     ret.SampleName(input.getSampleName());
     ret.HeaderAppendix(input.getHeaderAppendix());
     ret.ImageAppendix(input.getImageAppendix());
-    ret.ImagesPerFile(input.getImagesPerFile());
+    if (input.imagesPerFileIsSet())
+        ret.ImagesPerFile(input.getImagesPerFile());
 
     if (input.dataReductionFactorSerialmxIsSet())
         ret.LossyCompressionSerialMX(input.getDataReductionFactorSerialmx());

broker/gen/model/Dataset_settings.cpp

@@ -33,7 +33,7 @@ Dataset_settings::Dataset_settings()
     m_Incident_energy_keV = 0.0f;
     m_File_prefix = "";
     m_File_prefixIsSet = false;
-    m_Images_per_file = 1000L;
+    m_Images_per_file = 0L;
     m_Images_per_fileIsSet = false;
     m_Space_group_number = 0L;
     m_Space_group_numberIsSet = false;
@@ -192,10 +192,10 @@ bool Dataset_settings::validate(std::stringstream& msg, const std::string& pathP
         const std::string currentValuePath = _pathPrefix + ".imagesPerFile";
                 
         
-        if (value < 0ll)
+        if (value < 1ll)
         {
             success = false;
-            msg << currentValuePath << ": must be greater than or equal to 0;";
+            msg << currentValuePath << ": must be greater than or equal to 1;";
         }
 
     }

broker/gen/model/Dataset_settings.h

@@ -113,7 +113,7 @@ public:
     bool filePrefixIsSet() const;
     void unsetFile_prefix();
     /// <summary>
-    /// Number of files in a single HDF5 data file (0 &#x3D; write all images to a single data file).
+    /// Number of images in a single HDF5 data file, taken literally when given. If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images goes into a single data file, so the dataset stays readable by programs that follow only the first data file of a legacy master; a grid scan is split on a whole number of fast-axis rows; anything else (stills, serial) uses 1000. 
     /// </summary>
     int64_t getImagesPerFile() const;
     void setImagesPerFile(int64_t const value);

broker/jfjoch_api.yaml

@@ -429,9 +429,13 @@ components:
         images_per_file:
           type: integer
           format: int64
-          minimum: 0
-          default: 1000
-          description: Number of files in a single HDF5 data file (0 = write all images to a single data file).
+          minimum: 1
+          description: |
+            Number of images in a single HDF5 data file, taken literally when given.
+            If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images
+            goes into a single data file, so the dataset stays readable by programs that follow only
+            the first data file of a legacy master; a grid scan is split on a whole number of fast-axis
+            rows; anything else (stills, serial) uses 1000.
         space_group_number:
           type: integer
           format: int64

common/DatasetSettings.cpp

@@ -21,7 +21,6 @@ DatasetSettings::DatasetSettings() {
     ntrigger = 1;
     images_per_trigger = 1;
     compression = CompressionAlgorithm::BSHUF_LZ4;
-    images_per_file = 1000;
     data_reduction_factor_serialmx = 1.0;
     write_nxmx_hdf5_master = true;
     spot_finding_enable = true;
@@ -249,13 +248,14 @@ int64_t DatasetSettings::GetImageNumPerTrigger() const {
     return images_per_trigger;
 }
 
-DatasetSettings &DatasetSettings::ImagesPerFile(int64_t input) {
-    check_min("Images per file", input, 0);
+DatasetSettings &DatasetSettings::ImagesPerFile(const std::optional<int64_t> &input) {
+    if (input.has_value())
+        check_min("Images per file", input.value(), 1);
     images_per_file = input;
     return *this;
 }
 
-int64_t DatasetSettings::GetImagesPerFile() const {
+std::optional<int64_t> DatasetSettings::GetImagesPerFile() const {
     return images_per_file;
 }
 

common/DatasetSettings.h

@@ -23,7 +23,7 @@ class DatasetSettings {
     float photon_energy_keV;
 
     std::string file_prefix;
-    int64_t images_per_file;
+    std::optional<int64_t> images_per_file;
 
     CompressionAlgorithm compression;
 
@@ -89,7 +89,7 @@ public:
     DatasetSettings& GridScan(const std::optional<GridScanSettings>& input);
     DatasetSettings& HeaderAppendix(const nlohmann::json& input);
     DatasetSettings& ImageAppendix(const nlohmann::json& input);
-    DatasetSettings& ImagesPerFile(int64_t input);
+    DatasetSettings& ImagesPerFile(const std::optional<int64_t> &input);
     DatasetSettings& RunNumber(const std::optional<uint64_t> &run_number);
     DatasetSettings& RunName(const std::optional<std::string> &input);
     DatasetSettings& ExperimentGroup(const std::string &group);
@@ -139,7 +139,7 @@ public:
     CompressionAlgorithm GetCompressionAlgorithm() const;
     int64_t GetNumTriggers() const;
     int64_t GetImageNumPerTrigger() const;
-    int64_t GetImagesPerFile() const;
+    std::optional<int64_t> GetImagesPerFile() const;
 
     std::optional<uint64_t> GetRunNumber() const;
     std::optional<std::string> GetRunName() const;

common/Definitions.h

@@ -20,6 +20,12 @@ constexpr size_t JUNGFRAU_PACKET_SIZE_BYTES = 8192;
 
 constexpr int MAX_IMAGE_NUMBER = 2*1024*1024;
 
+// Defaults for images_per_file when the acquisition does not ask for a particular number; see
+// DiffractionExperiment::GetImagesPerFile.
+constexpr int64_t DEFAULT_IMAGES_PER_FILE = 1000;
+// Above this, a rotation sweep is split rather than written to one file.
+constexpr int64_t ROTATION_SINGLE_FILE_IMAGE_LIMIT = 20000;
+
 constexpr std::chrono::nanoseconds MIN_COUNT_TIME = std::chrono::microseconds(3);
 constexpr std::chrono::nanoseconds MIN_STORAGE_CELL_DELAY = std::chrono::nanoseconds(2100);
 constexpr std::chrono::nanoseconds MIN_FRAME_TIME_JUNGFRAU_HALF_SPEED = std::chrono::microseconds(1000);

common/DiffractionExperiment.cpp

@@ -1113,19 +1113,43 @@ std::vector<uint16_t> DiffractionExperiment::ExportROIMap() const {
     return roi_mask.GetROIMap(GetDiffractionGeometry(), GetXPixelsNumConv(), GetYPixelsNumConv());
 }
 
-DiffractionExperiment &DiffractionExperiment::ImagesPerFile(int64_t input) {
+DiffractionExperiment &DiffractionExperiment::ImagesPerFile(const std::optional<int64_t> &input) {
     dataset.ImagesPerFile(input);
     return *this;
 }
 
+// Images per file is optional in the settings, because at setup time the acquisition it should suit
+// is not necessarily known yet. Everything downstream of here - receiver, pusher, puller, writer -
+// requires a fixed, non-zero number, so this is the one place that resolves it and it must always
+// return one.
 int64_t DiffractionExperiment::GetImagesPerFile() const {
-    auto tmp = dataset.GetImagesPerFile();
+    // A single self-contained file holds the whole run whatever the setting says.
+    if (file_writer.GetFileFormat() == FileWriterFormat::NXmxIntegrated)
+        return std::max<int64_t>(GetImageNum(), 1);
 
-    if (tmp == 0
-        || file_writer.GetFileFormat() == FileWriterFormat::NXmxIntegrated)
-        return GetImageNum();
-    else
-        return tmp;
+    if (const auto requested = dataset.GetImagesPerFile())
+        return requested.value();   // taken literally
+
+    const int64_t image_num = GetImageNum();
+
+    // Rotation: keep the sweep in one data file. A legacy master splits over several files is read
+    // by some programs as only its first one, and a rotation dataset is small enough that the file
+    // is not a problem - past the limit the writer parallelism is worth more, one file going to one
+    // writer (see ZMQStream2Pusher::SendImage).
+    const auto goniometer = GetGoniometer();
+    if (goniometer.has_value() && (goniometer->GetIncrement_deg() != 0.0f)
+        && (image_num > 0) && (image_num <= ROTATION_SINGLE_FILE_IMAGE_LIMIT))
+        return image_num;
+
+    // Grid scan: split on whole rows of the fast axis, so a file is a meaningful piece of the grid
+    // rather than an arbitrary cut across it.
+    if (const auto grid_scan = GetGridScan()) {
+        const int64_t n_fast = grid_scan->GetNFast();
+        if (n_fast > 0)
+            return ((DEFAULT_IMAGES_PER_FILE + n_fast - 1) / n_fast) * n_fast;
+    }
+
+    return DEFAULT_IMAGES_PER_FILE;   // stills, serial
 }
 
 int64_t DiffractionExperiment::GetImageBufferLocationSize() const {

common/DiffractionExperiment.h

@@ -137,7 +137,7 @@ public:
     DiffractionExperiment& ImageAppendix(const nlohmann::json& input);
     DiffractionExperiment& Summation(int64_t input);
     DiffractionExperiment& MaxSpotCount(int64_t input);
-    DiffractionExperiment& ImagesPerFile(int64_t input);
+    DiffractionExperiment& ImagesPerFile(const std::optional<int64_t> &input);
     DiffractionExperiment& LossyCompressionSerialMX(float input);
     DiffractionExperiment& LossyCompressionPoisson(const std::optional<int64_t> &input);
     DiffractionExperiment& SaveCalibration(const std::optional<bool> &input);

docs/CHANGELOG.md

@@ -15,6 +15,10 @@ This is an UNSTABLE release. It includes many experimental features, as well as
 * rugnux: the detector geometry is also logged in XDS's convention (`ORGX`/`ORGY`, detector axis vectors, rotation axis), so it can be compared directly with an XDS refinement.
 * HDF5: a data file missing next to a VDS master now reads as the error-pixel marker instead of zero counts, so those frames are masked rather than silently integrated as blank.
 * The writer refuses a stream whose start message declares a different pixel format than its images carry, instead of writing a master that does not describe its own data.
+* `images_per_file` is now chosen from the acquisition when it is not given: a rotation sweep of at most 20000 images goes into a single data file, a grid scan splits on whole fast-axis rows, and stills and serial keep 1000.
+
+**Breaking change to OpenAPI** - regenerate the client (`jfjoch-client` 1.0.0-rc.162, `frontend/src/client`):
+* `dataset_settings.images_per_file` is no longer `default: 1000` and no longer accepts `0`; it is optional, and its minimum is 1. A client sending `0` (previously "one file for the whole run") is now rejected - omit the field instead, which for a rotation sweep gives the same single file.
 
 ### 1.0.0-rc.161
 This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

docs/python_client/docs/DatasetSettings.md

@@ -13,7 +13,7 @@ Name | Type | Description | Notes
 **detector_distance_mm** | **float** | /entry/detector/distance in NXmx Detector distance [mm] | 
 **incident_energy_ke_v** | **float** | Used to calculate /entry/beam/incident_wavelength in NXmx Incident particle (photon, electron) energy in keV  | 
 **file_prefix** | **str** | Prefix for filenames. If left empty, no file will be saved. | [optional] [default to '']
-**images_per_file** | **int** | Number of files in a single HDF5 data file (0 = write all images to a single data file). | [optional] [default to 1000]
+**images_per_file** | **int** | Number of images in a single HDF5 data file, taken literally when given. If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images goes into a single data file, so the dataset stays readable by programs that follow only the first data file of a legacy master; a grid scan is split on a whole number of fast-axis rows; anything else (stills, serial) uses 1000.  | [optional] 
 **space_group_number** | **int** | Number of space group for the crystal. Currently used solely as metadata, not relevant for image processing done in Jungfraujoch. | [optional] 
 **sample_name** | **str** | /entry/sample/name in NXmx Sample name  | [optional] [default to '']
 **compression** | **str** | Compression type for the images transferred over ZeroMQ and saved to HDF5 file.  | [optional] [default to 'bslz4']

frontend/src/client/types.gen.ts

@@ -191,7 +191,12 @@ export type dataset_settings = {
      */
     file_prefix?: string;
     /**
-     * Number of files in a single HDF5 data file (0 = write all images to a single data file).
+     * Number of images in a single HDF5 data file, taken literally when given.
+     * If omitted, it is chosen from the acquisition: a rotation sweep of at most 20000 images
+     * goes into a single data file, so the dataset stays readable by programs that follow only
+     * the first data file of a legacy master; a grid scan is split on a whole number of fast-axis
+     * rows; anything else (stills, serial) uses 1000.
+     *
      */
     images_per_file?: number;
     /**

frontend/src/client/zod.gen.ts

@@ -111,7 +111,7 @@ export const zDatasetSettings = z.object({
     detector_distance_mm: z.number().gte(0),
     incident_energy_keV: z.number().gte(0.001).lte(500),
     file_prefix: z.string().optional().default(''),
-    images_per_file: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1000)),
+    images_per_file: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
     space_group_number: z.coerce.bigint().gte(BigInt(1)).lte(BigInt(194)).optional(),
     sample_name: z.string().optional().default(''),
     compression: z.enum([

tests/DiffractionExperimentTest.cpp

@@ -1077,4 +1077,59 @@ TEST_CASE("DiffractionExperiment_PedestalRun","[DiffractionExperiment]") {
     x.Mode(DetectorMode::PedestalG2);
     REQUIRE(x.IsPedestalRun());
     REQUIRE(!x.IsApplyPixelMask());
-}
+}
+TEST_CASE("DiffractionExperiment_ImagesPerFile_Defaults", "[DiffractionExperiment]") {
+    // images_per_file is optional in the settings, because at setup time the acquisition it should
+    // suit is not necessarily known. GetImagesPerFile() is the single place that resolves it, and
+    // everything downstream (receiver, pusher, puller, writer) requires the answer to be a fixed
+    // non-zero number - so every branch below must produce one.
+    const GoniometerAxis rotation("omega", 0.0f, 0.1f, {-1, 0, 0}, {});
+    const GoniometerAxis stationary("omega", 0.0f, 0.0f, {-1, 0, 0}, {});
+
+    SECTION("taken literally when given") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(5000).NumTriggers(1).Goniometer(rotation).ImagesPerFile(250);
+        CHECK(x.GetImagesPerFile() == 250);
+    }
+
+    SECTION("stills default to 1000") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(50000).NumTriggers(1);
+        CHECK(x.GetImagesPerFile() == DEFAULT_IMAGES_PER_FILE);
+    }
+
+    SECTION("a rotation sweep goes into one file") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(3600).NumTriggers(1).Goniometer(rotation);
+        CHECK(x.GetImagesPerFile() == 3600);
+    }
+
+    SECTION("a long rotation sweep is split again") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(ROTATION_SINGLE_FILE_IMAGE_LIMIT + 1).NumTriggers(1).Goniometer(rotation);
+        CHECK(x.GetImagesPerFile() == DEFAULT_IMAGES_PER_FILE);
+    }
+
+    SECTION("a goniometer that does not turn is not a rotation sweep") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(3600).NumTriggers(1).Goniometer(stationary);
+        CHECK(x.GetImagesPerFile() == DEFAULT_IMAGES_PER_FILE);
+    }
+
+    SECTION("a grid scan splits on whole fast-axis rows") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(10000).NumTriggers(1)
+         .GridScan(GridScanSettings(30, 10.0f, 10.0f, false, false));
+        const int64_t images_per_file = x.GetImagesPerFile();
+        CHECK(images_per_file % 30 == 0);
+        CHECK(images_per_file >= DEFAULT_IMAGES_PER_FILE);
+        CHECK(images_per_file < DEFAULT_IMAGES_PER_FILE + 30);
+    }
+
+    SECTION("a single self-contained file holds the whole run") {
+        DiffractionExperiment x(DetJF4M());
+        x.ImagesPerTrigger(5000).NumTriggers(1)
+         .SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).ImagesPerFile(250);
+        CHECK(x.GetImagesPerFile() == 5000);
+    }
+}