Merge failed-perimage into rc176: a rotation run that finds no lattice still reports every image (spots, resolution, background; not indexed), live and offline

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

docs/JFJOCH_BROKER.md

@@ -211,7 +211,8 @@ once with that, and the live status records the cut (`deadline` or `cancel`) and
 state, the images received, the step it is on and how far into it, the steps so far with what each
 found, and once it is over the verdict, the summary, warnings and the time from the last image to
 every result written. The web frontend shows it on the dashboard. It needs the dataset's bearer token
-like the routes above.
+like the routes above. A collection in which rugnux finds no lattice still gets the spot count, resolution
+estimate and background of every image in `GET /result/scan` and the master file, marked as not indexed.
 
 ## Setting up a local test for Jungfraujoch
 For development, it is possible to set up a local installation of Jungfraujoch. 

docs/RUGNUX.md

@@ -5,7 +5,9 @@ data-processing half of the system (see [Naming](NAMING.md) for where the name c
 It takes a stored dataset, runs the full analysis pipeline — spot finding, indexing, geometry
 refinement, Bragg integration, scaling and merging — and writes the merged reflections
 (`.mtz`/`.cif`/`.hkl`), the unmerged MTZ and the results report; the per-image `_process.h5` is
-written only on request (`--write-process-h5`), or as the only output with `--no-merge`.
+written only on request (`--write-process-h5`), or as the only output with `--no-merge`. A rotation run
+that finds no lattice still spot-finds every image and writes their per-image analysis there, then exits
+as failed.
 
 It runs the *same* analysis code as the online and interactive tools, driven from the command line
 over files.

rugnux/Rugnux.cpp

@@ -408,6 +408,16 @@ void RaiseFFTBoundForKnownCell(IndexingSettings &settings, const DiffractionExpe
 }
 
 
+ProcessResult Rugnux::SpotFindingPass(RugnuxObserver *observer, const DiffractionExperiment &header_experiment,
+                                      const SpotFindingSettings &spot_finding) {
+    Logger("Rugnux").Info("No lattice: one pass over every image with spot finding and no indexing, for the "
+                          "per-image analysis");
+    experiment_ = header_experiment;
+    config_.spot_finding = spot_finding;
+    config_.spot_finding.indexing = false;
+    return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
+}
+
 ProcessResult Rugnux::Run(RugnuxObserver *observer) {
     // Each pass times only itself, so the canonical result's processing_time_s is the last pass alone -
     // on the rotation two-pass that is under a third of what the run actually took, the pre-scan and the
@@ -424,25 +434,35 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) {
     again->near_tie_rechecks_ = std::make_shared<NearTieRechecks>();
     again->decide_late_ = true;
     again->decide_all_late_ = true;
+    const SpotFindingSettings spot_finding = config_.spot_finding;
     Rugnux *run = this;
     ProcessResult result;
-    bool lost = false;
     try {
-        result = RunAllPasses(observer);
-    } catch (const NoLatticeFound &) {
-        // A run that found no lattice after deciding a near tie early is made again deciding late.
-        if (!near_tie_rechecks_->carried || config_.fast_live)
-            throw;
-        lost = true;
-    }
-    if (lost || (!result.cancelled && !config_.fast_live && NearTieOverturned())) {
-        JoinBeamCenterCapture();
-        Logger logger("Rugnux");
-        logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, "
-                    "deciding near ties on all the frames");
-        result.outputs.reset();   // replaced: not held through the run made again
-        result = again->RunAllPasses(observer);
-        run = again.get();
+        bool lost = false;
+        try {
+            result = RunAllPasses(observer);
+        } catch (const NoLatticeFound &) {
+            // A run that found no lattice after deciding a near tie early is made again deciding late.
+            if (!near_tie_rechecks_->carried || config_.fast_live)
+                throw;
+            lost = true;
+        }
+        if (lost || (!result.cancelled && !config_.fast_live && NearTieOverturned())) {
+            JoinBeamCenterCapture();
+            Logger logger("Rugnux");
+            logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, "
+                        "deciding near ties on all the frames");
+            result.outputs.reset();   // replaced: not held through the run made again
+            result = again->RunAllPasses(observer);
+            run = again.get();
+        }
+    } catch (NoLatticeFound &e) {
+        // No lattice is a statement about every frame, so every frame is reported: what it diffracted, and
+        // that it did not index. Only where someone reads the images - the live lattice probe passes no
+        // observer.
+        if (observer && !cancelled_)
+            e.per_image = std::make_shared<ProcessResult>(SpotFindingPass(observer, header_experiment, spot_finding));
+        throw;
     }
     // Cut short in fast_live: a pass cut before its image loop read anything has no lattice and no merge,
     // so the run's answer is the last pass that ran to its end. (A canonical pass cut inside its image
@@ -779,6 +799,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     }
 
     const bool full = (config_.mode == ProcessMode::FullAnalysis);
+    // A full analysis with indexing off is the per-image pass of a run that found no lattice
+    // (SpotFindingPass): the image loop and nothing that needs a lattice.
+    const bool indexing = full && config_.spot_finding.indexing;
     // Powder calibration runs the same per-image engine as the full analysis (spot finding lives inside
     // MXAnalysisWithoutFPGA) with indexing switched off in its settings. By rings it needs only the
     // azimuthal profile and the azint worker would do - but the spots are worth their half second even
@@ -935,6 +958,11 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     // The output pass streams its per-image analysis to the caller's image sink, where there is one.
     RugnuxImageSink *const sink = write_output && !geometry_prepass ? config_.image_sink : nullptr;
     std::unique_ptr<ProcessFileWriter> writer_queue;
+    // An output pass with no sink keeps the per-image scalars of its images, for the end message a run
+    // that indexed nothing is written from (the _process.h5 below).
+    std::optional<ScanResultGenerator> scan_result;
+    if (result.outputs && !sink)
+        scan_result.emplace(experiment_);
     // Whether anything is in place to break the indexing ambiguity, for the warning further down: a
     // reference MTZ, or (serial stills) the reference computed from a model just below.
     bool ambiguity_reference = !config_.reference_data.empty();
@@ -1053,19 +1081,19 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             .indexing_bound_A = indexing_bound_A, .census_future = census_future, .superseded = superseded,
             .ambiguity_reference = ambiguity_reference, .start_time = start_time,
             .total_uncompressed_bytes = total_uncompressed_bytes, .end_msg = end_msg,
-            .unmerged_mtz = unmerged_mtz};
+            .scan_result = scan_result, .unmerged_mtz = unmerged_mtz};
 
     // First pass of two-pass rotation indexing (full analysis only).
-    if (full && force_rotation_result_.has_value()) {
+    if (indexing && force_rotation_result_.has_value()) {
         // Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
         // search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
         // integrates markedly worse.
         indexer->ForceRotationIndexerResult(*force_rotation_result_);
         logger.Info("Rotation indexer result forced from pass 1 (lattice + geometry) - skipping first pass");
-    } else if (full && config_.forced_rotation_lattice.has_value()) {
+    } else if (indexing && config_.forced_rotation_lattice.has_value()) {
         indexer->ForceRotationIndexerLattice(*config_.forced_rotation_lattice);
         logger.Info("Rotation indexer lattice forced externally - skipping first pass");
-    } else if (full && config_.rotation_indexing && config_.two_pass_rotation) {
+    } else if (indexing && config_.rotation_indexing && config_.two_pass_rotation) {
         TimingMark mark(logger, "first-pass indexing");
         const bool probe_done = FirstPassRotationIndexing(p);
         if (recheck && near_tie_carried_)
@@ -1094,7 +1122,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     //
     // Only an output pass probes: the answer the run returns is always one (RunAllPasses), and no other
     // pass's probe is read - the pre-pass, the beam-centre arm and the probe passes leave theirs unread.
-    if (full && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
+    if (indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
         std::vector<int64_t> frames;
         for (const int o : select_equally_spaced_image_ordinals(images_to_process, 60))
             frames.push_back(o);
@@ -1167,6 +1195,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
                        "kept for a _process.h5 instead");
         fill_writer_maps();
         result.outputs->process_start = start_message;
+        scan_result->FillEndMessage(end_msg);
     }
 
     // The per-frame sigma_M the merge wrote back onto the outcomes (the _plots.txt column), for the report.

rugnux/Rugnux.h

@@ -326,12 +326,18 @@ RotationScaleWalk WalkRotationScale(double first_fit,
                                     const std::function<std::optional<double>(float)> &refit_at,
                                     int max_rounds, int64_t sweep_images);
 
+struct ProcessResult;
+
 // The first pass found no lattice of the crystal in the frames - what a sample with no crystal in the
 // beam ends in. A statement about the data, not an error in the input: the CLI writes a report with
 // a FAILED verdict for it. `reason` is the explanation without the exception category.
 class NoLatticeFound : public JFJochException {
 public:
     std::string reason;
+    // The per-image pass Run() makes of every frame after it (SpotFindingPass): spot finding and the
+    // per-image analysis without indexing, its _process.h5 handed back as on a run that indexed nothing.
+    // None where Run() had no observer or was cancelled.
+    std::shared_ptr<ProcessResult> per_image;
     explicit NoLatticeFound(const std::string &r)
             : JFJochException(JFJochExceptionCategory::InputParameterInvalid, r), reason(r) {}
 };
@@ -1386,6 +1392,11 @@ class Rugnux {
 
     // Every pass of the run, returning the canonical one. Run() wraps this to time the whole thing.
     ProcessResult RunAllPasses(RugnuxObserver *observer);
+    // After a run that found no lattice: one output pass over every frame with spot finding and the
+    // per-image analysis but no indexing, so each frame still reports its spots, resolution and
+    // background, and shows it did not index. At the file's geometry and the run's own spot finding.
+    ProcessResult SpotFindingPass(RugnuxObserver *observer, const DiffractionExperiment &header_experiment,
+                                  const SpotFindingSettings &spot_finding);
 
     // The rotation-scale walk of the rotation two-pass and the refined passes after it run side by side
     // (see RunAllPasses): the refined passes on a copy of the run, at the stored angles, and the walk

rugnux/RugnuxFirstPassAccept.cpp

@@ -94,9 +94,8 @@ namespace {
     // They are one situation from the user's side - no crystal was found in these frames - so
     // both name the cause and then give the same remedies.
     constexpr const char *next_steps =
-            " Nothing was integrated. Check the beam centre (--beam-x / --beam-y), raise "
-            "--max-spots if the frames are well populated with spots, and use --mode azint to "
-            "write the per-image analysis for inspection.";
+            " Nothing was integrated. Check the beam centre (--beam-x / --beam-y), and raise "
+            "--max-spots if the frames are well populated with spots.";
 }
 
 void Rugnux::FirstPassRun::RequireLattice(const FirstPass &best) const {

rugnux/RugnuxImagePass.cpp

@@ -307,6 +307,10 @@ void Rugnux::ImagePass(PipelineLocals &p) {
             msg.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now() - t0).count();
             msg.run_number = experiment_.GetRunNumber();
             msg.run_name = experiment_.GetRunName();
+            // A full analysis with indexing off is the pass of a run that found no lattice (SpotFindingPass):
+            // the image was not indexed, which is said rather than left unasked.
+            if (full && !config_.spot_finding.indexing)
+                msg.indexing_result = false;
 
             if (calibration) {
                 std::lock_guard lock(calibration_spot_m);
@@ -325,6 +329,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
                                         experiment_.GetBeamY_pxl(), worker_offsets);
 
             plots.Add(msg, profile);
+            if (p.scan_result) p.scan_result->Add(msg);
             if (writer_queue) writer_queue->Post(msg, img);
             note_written(ordinal);
             if (observer) observer->OnImageProcessed(msg);
@@ -361,6 +366,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     Step(fmt::format("{}{}, {} images",
                      !per_image_analysis ? "Azimuthal integration"
                      : calibration ? "Image pass"
+                     : !config_.spot_finding.indexing ? "Spot finding"
                      : experiment_.IsRotationIndexing() ? "Integration" : "Indexing and integration",
                      geometry_prepass ? " (geometry pre-pass)"
                      : full && prepass_detector_geometry_ ? " (refined geometry)" : "",
@@ -654,7 +660,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     end_msg.az_int_result["dataset"] = plots.GetAzIntProfile();
     end_msg.indexing_rate = result.indexing_rate;
 
-    if (full && !stopped()) {
+    if (full && config_.spot_finding.indexing && !stopped()) {
         const auto rot = indexer->FinalizeRotationIndexing();
         indexer->RethrowIndexerError();
         if (rot.has_value()) {

rugnux/RugnuxPipeline.h

@@ -21,6 +21,7 @@
 #include "Rugnux.h"
 #include "../common/CUDAWrapper.h"
 #include "../common/Logger.h"
+#include "../common/ScanResultGenerator.h"
 #include "../reader/JFJochReader.h"
 #include "../image_analysis/IndexAndRefine.h"
 #include "../image_analysis/indexing/IndexerThreadPool.h"
@@ -174,6 +175,8 @@ struct PipelineLocals {
     const std::chrono::steady_clock::time_point start_time;
     std::atomic<uint64_t> &total_uncompressed_bytes;
     EndMessage &end_msg;
+    // The per-image scalars of an output pass with no image sink, for its end message (RunPipeline).
+    std::optional<ScanResultGenerator> &scan_result;
     std::future<MtzTable> &unmerged_mtz;
     // What the main line holds through its merge (RunPipeline), for the merge to say how much of the card
     // it holds once that is known; null in a probe pass.

rugnux/rugnux_cli.cpp

@@ -3207,6 +3207,16 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) {
         provenance.wall_time_s = std::chrono::duration<double>(
                 std::chrono::steady_clock::now() - invocation_start).count();
         sink.reset();   // exit() runs no destructor: the _process.h5 is closed here
+        // The per-image analysis of every frame (NoLatticeFound::per_image), where a run that indexed
+        // nothing puts it: the _process.h5 and the per-image table.
+        if (e.per_image && !output_prefix.empty()) {
+            try {
+                WriteRugnuxOutputs(*e.per_image, output_prefix, output_provenance,
+                                   static_cast<size_t>(std::max(1, nthreads)), logger);
+            } catch (const std::exception &w) {
+                logger.Error("Writing the per-image analysis failed: {}", FailureText(w));
+            }
+        }
         const ReportDocument doc = WriteResultReport(output_prefix, input_file, experiment, failed, logger,
                                                      provenance);
         console->Fail();

tests/RugnuxTest.cpp

@@ -5,6 +5,8 @@
 #include <catch2/catch_all.hpp>
 #include <cmath>
 #include <filesystem>
+#include <mutex>
+#include <set>
 
 #include "../common/DiffractionExperiment.h"
 #include "../common/ScanResultGenerator.h"
@@ -20,10 +22,12 @@
 namespace {
     // Write a small VDS dataset of `n` flat images and return nothing (prefix_master.h5 +
     // prefix_data_000001.h5 land in the test working directory).
-    void WriteTestDataset(const std::string &prefix, int n) {
+    void WriteTestDataset(const std::string &prefix, int n, bool rotation = false) {
         RegisterHDF5Filter();
 
         DiffractionExperiment x(DetJF(1));
+        if (rotation)
+            x.Goniometer(GoniometerAxis("omega", 0.0f, 1.0f, Coord(1, 0, 0), std::nullopt));
         x.FilePrefix(prefix).ImagesPerTrigger(n).OverwriteExistingFiles(true);
         x.BitDepthImage(16).ImagesPerFile(n).SetFileWriterFormat(FileWriterFormat::NXmxVDS).PixelSigned(true);
         x.Compression(CompressionAlgorithm::NO_COMPRESSION);
@@ -138,6 +142,66 @@ TEST_CASE("Rugnux_Cancel", "[HDF5][Full]") {
     REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }
 
+// A rotation run that finds no lattice still reports every image - its spots, resolution and background,
+// and that it did not index - as the canonical pass, and hands the same back with the failure.
+TEST_CASE("Rugnux_NoLatticeReportsEveryImage", "[HDF5][Full]") {
+    WriteTestDataset("process_nolattice_in", 8, true);
+
+    JFJochHDF5Reader reader;
+    REQUIRE_NOTHROW(reader.ReadFile("process_nolattice_in_master.h5"));
+    auto dataset = reader.GetDataset();
+    DiffractionExperiment x = dataset->experiment;
+    auto indexing = x.GetIndexingSettings();
+    indexing.RotationIndexing(true);
+    indexing.Algorithm(IndexingAlgorithmEnum::Auto);
+    x.ImportIndexingSettings(indexing);
+    REQUIRE(x.IsRotationIndexing());
+
+    struct Observer : RugnuxObserver {
+        std::atomic<bool> canonical{false};
+        std::mutex m;
+        std::set<int64_t> images;
+        bool complete = true;
+        void OnImagePass(bool c) override { canonical = c; }
+        void OnImageProcessed(const DataMessage &msg) override {
+            if (!canonical)
+                return;
+            std::lock_guard lock(m);
+            images.insert(msg.number);
+            complete &= msg.spot_count.has_value() && msg.bkg_estimate.has_value()
+                        && msg.indexing_result == false;
+        }
+    } observer;
+
+    ProcessConfig config;
+    config.nthreads = 2;
+    config.rotation_indexing = true;
+    config.rotation_postrefine_geometry = true;
+    config.spot_finding = RugnuxDefaultSpotFinding(true);
+
+    Rugnux process(reader, x, *dataset->pixel_mask, config);
+    std::shared_ptr<ProcessResult> per_image;
+    try {
+        process.Run(&observer);
+    } catch (const NoLatticeFound &e) {
+        per_image = e.per_image;
+    }
+    REQUIRE(per_image);
+    CHECK(per_image->images_processed == 8);
+    CHECK(per_image->end_message.max_image_number == 8);
+    REQUIRE(per_image->outputs);
+    CHECK(per_image->outputs->process_start.has_value());
+    CHECK(per_image->end_message.spot_count.size() == 8);
+    CHECK(per_image->end_message.v_bkg_estimate.size() == 8);
+    CHECK(observer.images.size() == 8);
+    CHECK(observer.complete);
+
+    reader.Close();
+    remove("process_nolattice_in_master.h5");
+    remove("process_nolattice_in_data_000001.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
 // A calibration re-integrates its images binned about the geometry it fitted, and fits again. That
 // second profile must be the sum over every image: here each of six frames carries a different sixth
 // of the LaB6 rings, so a profile missing a frame is missing rings. The header's beam centre is 5 px