From 95108276d136fd3157ce77401c3100bcd40d8614 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 27 Aug 2026 16:04:52 +0200 Subject: [PATCH] Pre-flight: check the master file, not every data file of the run 5f838bdb2 widened CheckOutputFilesAvailable from the master file to every data file the run will write, so a stale _data_000001.h5 would be caught up front rather than at the rename. The enumeration is unbounded: it is one std::filesystem::exists per planned file, so a long run with a small images_per_file - it can be set to 1 - is thousands of lookups, and they run inside the 5 s PREFLIGHT ACK budget that BUSY heartbeats do not extend. On a shared filesystem that turns a writable output into an "ACK timeout", which is both wrong and unreadable. It also called HDF5Metadata::DataFileName for every planned file, and that throws past file 999999 (9999 on SwissFEL), refusing a run up front on a file number the acquisition would never reach. Back to the master file alone, which is what the constructor checked before. A data file in the way is caught per-writer at finalize again, as it was. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016L1qig74oYQzfUJJZbbxFh --- writer/FileWriter.cpp | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index a566cc01..6e55f9b5 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -222,37 +222,24 @@ void FileWriter::CheckOutputFilesAvailable(const StartMessage &msg, FileWriterFo if (msg.overwrite.value_or(false)) return; - // Checked by the single writer that owns the master file (write_master_file - index 0 in a - // multi-writer TCP/ZMQ setup), and it checks the whole run, its siblings' data files included. - // Data files are staggered across writers by file number, so letting each writer check its own - // would race the siblings already creating them; one writer can speak for all of them because - // the master links the data files by relative name, so they share a directory by construction. + // Only the master file is checked, and only by the single writer that owns it (write_master_file + // - index 0 in a multi-writer TCP/ZMQ setup). Enumerating the data files instead would make one + // writer stat every file of the run: thousands of lookups for a long run with a small + // images_per_file, which on a network filesystem overruns the pre-flight's ACK budget and refuses + // an output that was perfectly writable. Those conflicts are caught per-writer at finalize. if (!msg.write_master_file.value_or(false)) return; - auto refuse = [](const std::string &name) { - if (std::filesystem::exists(name)) - throw JFJochException(JFJochExceptionCategory::FileWriteError, - "Output file already exists and overwrite is off: " + name); - }; - const bool nxmx = format == FileWriterFormat::NXmxLegacy || format == FileWriterFormat::NXmxVDS || format == FileWriterFormat::NXmxIntegrated; - - if (nxmx) - refuse(HDF5Metadata::MasterFileName(msg)); - - // NXmxIntegrated puts the images in the master file, so there are no data files to check. - if (format == FileWriterFormat::NXmxIntegrated || format == FileWriterFormat::NoFile) + if (!nxmx) return; - const int64_t per_file = msg.images_per_file > 0 - ? msg.images_per_file - : static_cast(default_images_per_file); - const int64_t total_images = static_cast(msg.number_of_images); - for (int64_t file_number = 0; file_number * per_file < total_images; file_number++) - refuse(HDF5Metadata::DataFileName(msg, file_number)); + const std::string name = HDF5Metadata::MasterFileName(msg); + if (std::filesystem::exists(name)) + throw JFJochException(JFJochExceptionCategory::FileWriteError, + "Output file already exists and overwrite is off: " + name); } void FileWriter::Preflight(const StartMessage &request, bool trusted_path) {