From 6b713cf3daa4d4e083782a0b75c831cdc8904a35 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 3 Aug 2026 16:33:24 +0200 Subject: [PATCH] rugnux: size the written datasets by the ordinals reached, not by the successes Each image is written at its own ordinal, so a frame that fails to load or analyse leaves a HOLE - the frames after it keep their positions rather than shifting up. The end message nevertheless reported the number of frames that SUCCEEDED as the image count, and the writer sizes /entry/data/data and every per-image array from that. So one failed frame in the middle of a run made the declared extent one short, and the image it dropped was the LAST one written, not the one that failed. Two failures dropped two, and so on: the file quietly ends before the data does, with the per-image metadata still carrying rows for images the VDS no longer maps. It also under-counted the data files when the run was split. Track the highest ordinal actually written and use that. A hole then reads as the fill value, which is what a frame that was never written should look like, and the counts of collected and written images stay counts. Co-Authored-By: Claude Opus 5 (1M context) --- rugnux/Rugnux.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 1bf6de90..c158e196 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -893,6 +893,17 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // are serialized by the global hdf5_mutex; the analysis runs in parallel. std::atomic next_ordinal = 0; std::atomic finished_count = 0; + // Images are written at their own ordinal, so a frame that fails leaves a HOLE rather than + // shifting everything after it up. The extent of the written datasets is therefore the highest + // ordinal reached, not the number that succeeded - counting successes makes the file one image + // short for every failure, and the image it loses is the LAST one, not the one that failed. + std::atomic max_written_ordinal = -1; + auto note_written = [&max_written_ordinal](int ordinal) { + int prev = max_written_ordinal.load(std::memory_order_relaxed); + while (prev < ordinal + && !max_written_ordinal.compare_exchange_weak(prev, ordinal, std::memory_order_relaxed)) { + } + }; std::atomic total_uncompressed_bytes = 0; auto azint_worker = [&]() { @@ -946,6 +957,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b plots.Add(msg, profile); if (writer) writer->Write(msg); + note_written(ordinal); if (observer) observer->OnImageProcessed(msg); const int done = finished_count.fetch_add(1) + 1; if (observer) observer->OnProgress(done, images_to_process); @@ -993,6 +1005,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b plots.Add(msg, profile); if (writer) writer->Write(msg); + note_written(ordinal); if (observer) observer->OnImageProcessed(msg); const int done = finished_count.fetch_add(1) + 1; if (observer) observer->OnProgress(done, images_to_process); @@ -1031,7 +1044,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // End message (also written to the file). EndMessage end_msg; - end_msg.max_image_number = result.images_processed; + // The EXTENT of the per-image datasets (see note_written), not the success count: those two + // differ by exactly the number of frames that failed, and the writer sizes its virtual datasets + // from this. + end_msg.max_image_number = static_cast(max_written_ordinal.load() + 1); end_msg.images_collected_count = result.images_processed; end_msg.images_sent_to_write_count = result.images_processed; end_msg.end_date = time_UTC(std::chrono::system_clock::now());