From c6f6396e462006185edd1e6ad30c072449052a37 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 10 Oct 2026 18:56:00 +0200 Subject: [PATCH] Indexer failures are thrown, not read as "no lattice"; GPU memory errors say what to do A CUDA out-of-memory in an indexer worker was turned into an IndexerResult carrying only the message string. The rotation indexer then returned no lattice, the first pass treated that as a genuine negative and the rescues acted on it: in a stress run (two rugnux on one card) a run had both first-pass schemes fail with out-of-memory, the axis-sign rescue compared 0 frames at the flipped sign with the -1 "no lattice" sentinel, adopted the flipped sign, and the beam-centre rescue then moved the centre - a failure of the machine used as evidence about the crystal. The CUDA category was lost on the way, so the fail-fast check (IsFatalResourceError) never saw it. - IndexerResult::error is now the std::exception_ptr the worker caught. RotationIndexer::RunIndexing rethrows it before touching any state, so nothing is memoised for a failed run and the exception keeps its category: rugnux stops with the CUDA message at once (verified with a forced indexer failure: exit 1 after 1.9 s, "CUDA (GPU) error (...)"). FirstPass::indexer_error, RotationIndexer::GetIndexerError and RequireLattice's machine-failure branch are gone. - The beam-centre arm in RunAllPasses rethrows fatal resource errors like its sibling catches; it had caught the out-of-memory and carried on at the file's centre. - IndexAndRefine keeps per-frame indexer failures apart from frames that did not index (count + first exception). The frame stays "not attempted" as before; the receiver logs the count and the first error at the end of a collection (no cancel), rugnux's image pass and stills geometry refinement rethrow it and stop. A rotation-indexer failure inside IndexAndRefine (online rotation indexing, FinalizeRotationIndexing) is recorded the same way, so a live collection is not cancelled by it. - A failed device allocation or cuFFT plan now says how much was asked, how much of the card is free, and - when the card is merely full - that another program is using the GPU or the data set needs more than the card has, with the CPU remedy; previously "Failed to allocate device memory (out of memory)" / "Failed to create CUFFT plan with cufftPlanMany". Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/IndexAndRefine.cpp | 37 +++++++++++++++-- image_analysis/IndexAndRefine.h | 13 ++++++ image_analysis/indexing/CUDAMemHelpers.h | 40 ++++++++++++++++--- image_analysis/indexing/Indexer.h | 12 +++--- image_analysis/indexing/IndexerThreadPool.cpp | 6 ++- .../rotation_indexer/RotationIndexer.cpp | 16 +++----- .../rotation_indexer/RotationIndexer.h | 8 +--- receiver/JFJochReceiver.cpp | 11 +++++ rugnux/RugnuxFirstPass.cpp | 2 - rugnux/RugnuxFirstPass.h | 7 ++-- rugnux/RugnuxFirstPassAccept.cpp | 14 +------ rugnux/RugnuxImagePass.cpp | 7 +++- rugnux/RugnuxPasses.cpp | 4 +- rugnux/RugnuxScaleMerge.cpp | 4 +- rugnux/RugnuxStills.cpp | 1 + tests/CudaHandledErrorTest.cpp | 4 +- tests/RotationIndexerTest.cpp | 3 -- 17 files changed, 131 insertions(+), 58 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 145dad247..ef313f71f 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -88,6 +88,24 @@ std::optional IndexAndRefine::RotationAngle(int64_t image) const { return std::nullopt; } +void IndexAndRefine::RecordIndexerError(std::exception_ptr e) { + std::lock_guard lock(indexer_error_mutex_); + if (!first_indexer_error_) + first_indexer_error_ = e; + ++indexer_errors_; +} + +int64_t IndexAndRefine::GetIndexerErrorCount() const { + std::lock_guard lock(indexer_error_mutex_); + return indexer_errors_; +} + +void IndexAndRefine::RethrowIndexerError() const { + std::lock_guard lock(indexer_error_mutex_); + if (first_indexer_error_) + std::rethrow_exception(first_indexer_error_); +} + void IndexAndRefine::AddImageToRotationIndexer(DataMessage &msg) { if (rotation_indexer) rotation_indexer->ProcessImage(msg.number, msg.spots, RotationAngle(msg.number)); @@ -105,8 +123,13 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetryRotat auto rot_cnt = rotation_indexer_counter.Process(msg.number); if (rot_cnt.first) rotation_indexer->ProcessImage(msg.number, msg.spots, RotationAngle(msg.number)); - if (rot_cnt.second) - rotation_indexer->RunIndexing(); + if (rot_cnt.second) { + try { + rotation_indexer->RunIndexing(); + } catch (...) { + RecordIndexerError(std::current_exception()); + } + } result = rotation_indexer->GetLattice(); } @@ -187,6 +210,8 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetry(Data } } auto res = indexer_->Run(experiment, recip); + if (res.error) + RecordIndexerError(res.error); any_executed |= res.executed; // Kept from whichever run could measure it - the largest seed that answered - rather than from // the run whose lattice won: the severity describes the frame's lattice rows, not the cell that @@ -247,6 +272,8 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetry(Data const auto algorithm = experiment.GetIndexingAlgorithm(); if (algorithm == IndexingAlgorithmEnum::FFT || algorithm == IndexingAlgorithmEnum::FFTW) { const auto res = indexer_->Run(experiment, recip, /*severity_only=*/true); + if (res.error) + RecordIndexerError(res.error); spindle_blind_fraction = res.spindle_blind_fraction; } if (!spindle_blind_fraction && !indexer_result.lattice.empty()) { @@ -847,7 +874,11 @@ std::optional IndexAndRefine::FinalizeRotationIndexing() if (const auto latt = rotation_indexer->GetLattice()) return latt; - rotation_indexer->RunIndexing(); + try { + rotation_indexer->RunIndexing(); + } catch (...) { + RecordIndexerError(std::current_exception()); + } return rotation_indexer->GetLattice(); } return {}; diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index 541603d40..e6bee48e5 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -173,6 +174,13 @@ class IndexAndRefine { const IndexingOutcome &outcome, const BraggIntegrateFn &probe_integrate); + // Where the indexer failed rather than found nothing: how often, and the first failure as thrown. + // The frame is left unindexed, as one the indexer never got to; the caller decides the rest. + mutable std::mutex indexer_error_mutex_; + int64_t indexer_errors_ = 0; + std::exception_ptr first_indexer_error_; + void RecordIndexerError(std::exception_ptr e); + std::unique_ptr reindex_resolver; void ScaleImage(DataMessage &msg, IntegrationOutcome& outcome); @@ -186,6 +194,11 @@ public: bool real_time = false); void AddImageToRotationIndexer(DataMessage &msg); + // An indexer that fails says nothing about the frame and everything about the machine, so it is + // not a frame that did not index. Online, a collection goes on and reports how often it happened + // (GetIndexerErrorCount); offline, the run stops on it (RethrowIndexerError). + int64_t GetIndexerErrorCount() const; + void RethrowIndexerError() const; // the first failure, if there was one void ForceRotationIndexerLattice(const CrystalLattice& lattice); void ForceRotationIndexerResult(const RotationIndexerResult& result); // Supply a per-frame (by image number) mosaicity for prediction, overriding the per-image estimate. diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 27e167491..8945c0fc1 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -5,13 +5,37 @@ #include #include +#include #include #include #include +#include #include #include "../common/JFJochException.h" #include "../common/CUDAWrapper.h" +// The message for GPU memory that could not be had: what was asked (bytes, 0 where unknown), what the +// card has free and, where it is merely full, the usual causes and the remedy - the same advice the +// scaling's own out-of-memory gives. A context lost to another fault fails here too and keeps its reason. +inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_memory, const char *reason) { + char buf[512]; + if (bytes > 0) + std::snprintf(buf, sizeof buf, "%s (%.1f MB) failed (%s)", what, bytes / 1e6, reason); + else + std::snprintf(buf, sizeof buf, "%s failed (%s)", what, reason); + std::string msg(buf); + if (!out_of_memory) + return msg; + size_t free_b = 0, total_b = 0; + if (cudaMemGetInfo(&free_b, &total_b) == cudaSuccess) { + std::snprintf(buf, sizeof buf, ", with %.1f of the %.1f GB card free", free_b / 1e9, total_b / 1e9); + msg += buf; + } + return msg + ". Either another program is using this GPU, or this data set needs more GPU memory than " + "the card has - free the GPU and run again, or run on the CPU (CUDA_VISIBLE_DEVICES= rugnux " + "..., or a CPU-only build)"; +} + class CudaStream { cudaStream_t stream_ = nullptr; public: @@ -94,12 +118,15 @@ public: const int* onembed, int ostride, int odist, cufftType type, int batch) { - if (cufftPlanMany(&plan_, rank, const_cast(n), - const_cast(inembed), istride, idist, - const_cast(onembed), ostride, odist, - type, batch) != CUFFT_SUCCESS) + const cufftResult r = cufftPlanMany(&plan_, rank, const_cast(n), + const_cast(inembed), istride, idist, + const_cast(onembed), ostride, odist, + type, batch); + if (r != CUFFT_SUCCESS) throw JFJochException(JFJochExceptionCategory::GPUCUDAError, - "Failed to create CUFFT plan with cufftPlanMany"); + GPUMemoryFailure("Creating the CUFFT plan (cufftPlanMany)", 0, + r == CUFFT_ALLOC_FAILED, + ("cuFFT error " + std::to_string(r)).c_str())); } // Convenience overload for vector input @@ -314,7 +341,8 @@ public: // as a card that is merely full. if (const cudaError_t err = cudaMalloc(&ptr_, count * sizeof(T)); err != cudaSuccess) throw JFJochException(JFJochExceptionCategory::GPUCUDAError, - std::string("Failed to allocate device memory (") + cudaGetErrorString(err) + ")"); + GPUMemoryFailure("Allocating GPU memory", count * sizeof(T), + err == cudaErrorMemoryAllocation, cudaGetErrorString(err))); } ~CudaDevicePtr() { release(); diff --git a/image_analysis/indexing/Indexer.h b/image_analysis/indexing/Indexer.h index cd2190145..8ed46ee6f 100644 --- a/image_analysis/indexing/Indexer.h +++ b/image_analysis/indexing/Indexer.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include "../../common/CrystalLattice.h" @@ -16,11 +17,12 @@ struct IndexerResult { std::vector lattice; float indexing_time_s; bool executed = false; // If indexing was not performed (due to indexer being not-available) mark it, so indexing result is marked accordingly - // What the indexer threw, when it threw. A THIRD outcome, not a flavour of the other two: an - // empty lattice list with no error is a real negative result about the frame - it was examined - // and no lattice fits it - whereas an error says nothing about the frame at all, the indexer - // never having got to look, and will recur on the next one. - std::optional error; + // What the indexer threw, when it threw - the exception itself, so its category (a CUDA + // out-of-memory, say) reaches whoever decides what to do about it. A THIRD outcome, not a flavour + // of the other two: an empty lattice list with no error is a real negative result about the + // frame - it was examined and no lattice fits it - whereas an error says nothing about the frame + // at all, the indexer never having got to look, and will recur on the next one. + std::exception_ptr error; // How much of a single sweep's blind cone this orientation makes unrecoverable, in [0,1]. // Absent when the frame gives nothing to decide on (no spindle defined, no lattice rows). std::optional spindle_blind_fraction; diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index ea1acdc62..4afe38a97 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -155,7 +155,8 @@ void IndexerThread::Worker(int threadid) { // downstream as "this frame did not index". spdlog::error("Indexer thread {} failed: {}", threadid, e.what()); tmp_result = std::make_unique(IndexerResult{ - .lattice = {}, .indexing_time_s = 0, .executed = false, .error = e.what()}); + .lattice = {}, .indexing_time_s = 0, .executed = false, + .error = std::current_exception()}); // This thread goes on to the next frame, so a CUDA error left behind here would be // reported by the first cudaGetLastError() of every frame that follows - one failed // attempt would read as an indexer that never works again. @@ -295,7 +296,8 @@ IndexerResult IndexerThreadPool::Run(const DiffractionExperiment &experiment, co } catch (const std::exception &e) { spdlog::error("Indexer thread failed: {}", e.what()); result = std::make_unique(IndexerResult{ - .lattice = {}, .indexing_time_s = 0, .executed = false, .error = e.what()}); + .lattice = {}, .indexing_time_s = 0, .executed = false, + .error = std::current_exception()}); } { std::unique_lock lock(m); diff --git a/image_analysis/rotation_indexer/RotationIndexer.cpp b/image_analysis/rotation_indexer/RotationIndexer.cpp index e077d828b..dbc1e168a 100644 --- a/image_analysis/rotation_indexer/RotationIndexer.cpp +++ b/image_analysis/rotation_indexer/RotationIndexer.cpp @@ -203,7 +203,7 @@ namespace { lattice(kb, b.indexed_lattice); for (const auto &l : a.extra_lattices) lattice(ka, l); for (const auto &l : b.extra_lattices) lattice(kb, l); - return ka == kb && a.indexer_error == b.indexer_error && a.tilt_walk.has_value() == b.tilt_walk.has_value() + return ka == kb && a.tilt_walk.has_value() == b.tilt_walk.has_value() && (a.unconstrained != nullptr) == (b.unconstrained != nullptr); } } @@ -236,8 +236,7 @@ std::vector RotationIndexer::InputKey() const { } RotationIndexer::IndexingOutcome RotationIndexer::Outcome() const { - return {axis_, updated_geom_, search_result_, extra_lattices_, unconstrained_, tilt_walk_, indexed_lattice, - indexer_error_}; + return {axis_, updated_geom_, search_result_, extra_lattices_, unconstrained_, tilt_walk_, indexed_lattice}; } void RotationIndexer::RunIndexing() { @@ -263,7 +262,6 @@ void RotationIndexer::RunIndexing() { unconstrained_ = kept->unconstrained; tilt_walk_ = kept->tilt_walk; indexed_lattice = kept->indexed_lattice; - indexer_error_ = kept->indexer_error; return; } @@ -295,7 +293,10 @@ void RotationIndexer::RunIndexingLocked() { coords.emplace_back(rot * s.ReciprocalCoord(geom_)); } const auto indexer_result = indexer_.Run(experiment, coords); - indexer_error_ = indexer_result.error; + // An indexer that failed never looked at the cloud, so there is no outcome to keep, and none is + // memoised (RunIndexing): the failure goes to the caller as the indexer threw it. + if (indexer_result.error) + std::rethrow_exception(indexer_result.error); tilt_walk_.reset(); if (!indexer_result.lattice.empty() && indexer_result.lattice[0].CalcVolume() > 1.0) { @@ -891,11 +892,6 @@ std::optional RotationIndexer::GetLattice() const { }; } -std::optional RotationIndexer::GetIndexerError() const { - std::unique_lock ul(m); - return indexer_error_; -} - void RotationIndexer::ForceResult(const RotationIndexerResult &result) { std::unique_lock ul(m); indexed_lattice = result.lattice; diff --git a/image_analysis/rotation_indexer/RotationIndexer.h b/image_analysis/rotation_indexer/RotationIndexer.h index fe83d261c..e17e22315 100644 --- a/image_analysis/rotation_indexer/RotationIndexer.h +++ b/image_analysis/rotation_indexer/RotationIndexer.h @@ -71,7 +71,6 @@ public: std::shared_ptr unconstrained; std::optional tilt_walk; std::optional indexed_lattice; - std::optional indexer_error; }; private: mutable std::mutex m; @@ -108,7 +107,6 @@ private: size_t accumulated_spots = 0; std::optional indexed_lattice; - std::optional indexer_error_; // The accumulated per-image spots as one reciprocal-space cloud, back-rotated to phi = 0 under the // given geometry and axis. The caller holds the lock. @@ -132,12 +130,10 @@ public: // `image` is used (only valid when `image` is the goniometer's own image index). void ProcessImage(int64_t image, const std::vector& spots, std::optional angle_deg = std::nullopt); + // Throws what the indexer threw, where it failed rather than found no lattice: the two are one + // about the machine and one about the crystal, and only the second is an answer. void RunIndexing(); std::optional GetLattice() const; - // What the indexer threw on the last RunIndexing(), if it threw. Empty when it simply found no - // lattice: a caller that reports the failure to the user needs the two apart, one being about - // the crystal and the other about the machine. - [[nodiscard]] std::optional GetIndexerError() const; void ForceLattice(const CrystalLattice& lattice); // Adopt a complete result (lattice + refined geometry/axis/symmetry) found by another indexer - // used when several first-pass sampling schemes are run and the best is selected. diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 8c4aca8b3..abb36ea42 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -181,6 +181,17 @@ void JFJochReceiver::SendEndMessage() { rotation_indexing_lattice_type = message.rotation_lattice_type; } message.unit_cell = indexer.GetConsensusUnitCell(); + // The collection does not stop for an indexer that failed - the images are written regardless - but + // its indexing rate is then not a statement about the crystal, and the log has to say so. + if (const int64_t n = indexer.GetIndexerErrorCount(); n > 0) { + try { + indexer.RethrowIndexerError(); + } catch (const std::exception &e) { + logger.Error("Indexing failed {} times in this collection, first with: {}. Those attempts are " + "left out of the indexing rate, and a rotation lattice may be missing for the same " + "reason", n, e.what()); + } + } for (int i = 0; i < adu_histogram_module.size(); i++) message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram(); diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index d2f814092..8aa476a60 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -817,8 +817,6 @@ auto Rugnux::FirstPassRun::ScoreSchemes(const SchemeIndexers &ris, IndexAndRefin for (size_t i = 0; i < ris.size(); i++) { if (rx.cancelled_) break; - if (!bp.indexer_error) - bp.indexer_error = ris[i]->GetIndexerError(); auto found = ris[i]->GetLattice(); if (!found.has_value()) continue; diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index 7ca66ce90..d9f6cc7c0 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -28,13 +28,12 @@ #include "../image_analysis/MXAnalysisWithoutFPGA.h" namespace rugnux_internal { - // A candidate lattice of the first pass and what it scored on the validation frames. - // indexer_error: what the indexer threw, if it threw. An empty result with no error is a - // verdict on the crystal; an empty result with one is a verdict on the machine. + // A candidate lattice of the first pass and what it scored on the validation frames; no result + // (score -1) where nothing indexed. An indexer that failed is not that: it throws (RunIndexing). // larger_settled: the result is the larger cell of an axis-harmonic pair, and the arbiter in // the scheme comparison established it on the spots. struct FirstPass { std::optional result; int score = -1; std::string name; double vol = 0.0; - std::optional indexer_error; bool larger_settled = false; }; + bool larger_settled = false; }; // Two results are one lattice when their Niggli-reduced primitive edges agree within 2 % (the // battery's lattice-identity test), whatever setting or class each is held in. diff --git a/rugnux/RugnuxFirstPassAccept.cpp b/rugnux/RugnuxFirstPassAccept.cpp index 55fd883f5..1df384d0b 100644 --- a/rugnux/RugnuxFirstPassAccept.cpp +++ b/rugnux/RugnuxFirstPassAccept.cpp @@ -103,20 +103,8 @@ namespace { } void Rugnux::FirstPassRun::RequireLattice(const FirstPass &best) const { - if (!best.result.has_value()) { - // An indexer that FAILED is a third outcome, and none of those remedies fit it: it - // never examined the frames, so nothing about the data or the processing options is - // implicated and changing them cannot help. Name the failure instead. - if (best.indexer_error) - throw JFJochException(JFJochExceptionCategory::IndexerError, - fmt::format("Rotation indexing could not run: {}. Nothing was " - "integrated. The indexer never examined the frames, " - "so this is a failure of the machine and not of the " - "data - free the resource it needs (a GPU shared with " - "another process is the usual cause) and run again.", - *best.indexer_error)); + if (!best.result.has_value()) throw NoLatticeFound(std::string("Two-pass rotation indexing found no lattice.") + next_steps); - } } // Whether the run has a lattice at all is decided here, on the pooled spots rather than on diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index 05e24de27..caa246819 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -269,6 +269,9 @@ void Rugnux::ImagePass(PipelineLocals &p) { logger.Error("Error analyzing image {}: {}", image_idx, e.what()); continue; } + // An indexer that failed left this frame unindexed without looking at it; the run stops + // rather than merge what the machine dropped. + indexer->RethrowIndexerError(); pinned.reset(); msg.processing_time_s = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); msg.run_number = experiment_.GetRunNumber(); @@ -571,7 +574,9 @@ void Rugnux::ImagePass(PipelineLocals &p) { end_msg.indexing_rate = result.indexing_rate; if (full && !cancelled_) { - if (const auto rot = indexer->FinalizeRotationIndexing(); rot.has_value()) { + const auto rot = indexer->FinalizeRotationIndexing(); + indexer->RethrowIndexerError(); + if (rot.has_value()) { last_rotation_result_ = *rot; end_msg.rotation_lattice = rot->lattice; // The refined geometry, not the nominal StartMessage values. Two things about these five diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 16836812b..19f87c98f 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -441,7 +441,9 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { } catch (const std::exception &e) { // A hypothesis that does not survive its own pass is not a failure of the run: the // file's centre has already produced a complete first pass, and this arm exists only - // to be compared against it. + // to be compared against it. A machine that ran out of a resource is, though: the + // file's centre would meet it next. + if (IsFatalResourceError(e)) throw; logger.Warning("Beam centre check: the pass at the measured centre did not complete " "({}) - the run keeps the file's centre", e.what()); } diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 1b6dd982f..6fd346d02 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -749,7 +749,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { } } if (result.consensus_cell.has_value()) { - if (const auto rot = indexer->FinalizeRotationIndexing(); rot && rot->axis) { + const auto rot = indexer->FinalizeRotationIndexing(); + indexer->RethrowIndexerError(); + if (rot && rot->axis) { if (geometry_prepass) prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback PostRefineSettings prs; diff --git a/rugnux/RugnuxStills.cpp b/rugnux/RugnuxStills.cpp index e2e6ab916..2c938876d 100644 --- a/rugnux/RugnuxStills.cpp +++ b/rugnux/RugnuxStills.cpp @@ -191,6 +191,7 @@ void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_ if (IsFatalResourceError(e)) throw; continue; } + indexer.RethrowIndexerError(); // a frame the indexer failed on is not one that did not index if (!msg.indexing_result.value_or(false) || !msg.indexing_lattice.has_value()) continue; diff --git a/tests/CudaHandledErrorTest.cpp b/tests/CudaHandledErrorTest.cpp index 9889ddfc1..101dd9f0d 100644 --- a/tests/CudaHandledErrorTest.cpp +++ b/tests/CudaHandledErrorTest.cpp @@ -29,7 +29,9 @@ TEST_CASE("CudaDevicePtr_FailedAllocationLeavesErrorUntilCleared", "[CUDAMemHelp SKIP("No CUDA GPU present"); cuda_clear_error(); - REQUIRE_THROWS(CudaDevicePtr(std::numeric_limits::max(), CudaAlloc::Synchronous)); + // Said as what it is, with the remedy, not as a bare CUDA status. + REQUIRE_THROWS_WITH(CudaDevicePtr(std::numeric_limits::max(), CudaAlloc::Synchronous), + Catch::Matchers::ContainsSubstring("another program is using this GPU")); // Work that goes fine in between changes nothing: the error is still waiting. CudaDevicePtr fine(1 << 16); diff --git a/tests/RotationIndexerTest.cpp b/tests/RotationIndexerTest.cpp index a98f74fe6..5bf029ed6 100644 --- a/tests/RotationIndexerTest.cpp +++ b/tests/RotationIndexerTest.cpp @@ -75,9 +75,6 @@ TEST_CASE("RotationIndexer") { } CHECK(cnt == 20); - // An indexer that ran records no error; only one that threw does. A caller reporting "no lattice" - // to the user tells the two apart on this. - CHECK_FALSE(indexer.GetIndexerError().has_value()); auto ret = indexer.GetLattice(); REQUIRE(ret.has_value());