From 373fda05a459892b917003acac2aecac22453c11 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 20:10:43 +0200 Subject: [PATCH] CUDA: a handled failure discharges the error it leaves behind rc.172 stopped two CUDA failures from staying behind as the thread's last error, both inside CUDAMemHelpers.h: a pooled allocation that fell back to cudaMalloc, and a stream that could not be created. The reason those mattered was general, and four other places have it too. Nothing but cudaGetLastError() takes an error away. A later successful call neither clears it nor returns it - measured: after a cudaMalloc that cannot be served, cudaStreamSynchronize, cudaMemcpyAsync and a small cudaMalloc all report success, and the "out of memory" is still waiting for whoever calls cudaGetLastError() next. In this code that is the check after a kernel launch. So wherever a CUDA failure is caught and the work carries on by another route, the error it left is reported over the route that went fine: - the device decoder in MXAnalysisWithoutFPGA::Analyze, whose failure is answered by decompressing on the host - and the host route runs kernels of its own. This is the broker's path, and it is the pair of log lines a long-running broker showed, one warning naming the device route and one fatal error naming nothing. - the indexer worker, which turns a failed attempt into a result and goes on to the next frame - on the same thread, so one failure would read as an indexer that never works again. - the beam stop's GPU accumulate and the FFT beam-centre capture, both of which fall back to the host. Each discharges it now, through cuda_clear_error() in CUDAWrapper (a no-op without CUDA). A sticky error is not cleared by this and nothing here pretends otherwise. tests/CudaHandledErrorTest.cpp: that a failed allocation leaves an error which survives later successful work until it is cleared, and that an image whose device decode fails comes out of Analyze whole and leaves nothing behind for the next one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- common/CUDAWrapper.cpp | 2 + common/CUDAWrapper.cu | 4 + common/CUDAWrapper.h | 8 ++ image_analysis/MXAnalysisWithoutFPGA.cpp | 5 ++ image_analysis/beam_stop/ShadowFinder.cpp | 1 + .../geom_refinement/BeamCenterFFT.cpp | 1 + image_analysis/indexing/IndexerThreadPool.cpp | 4 + tests/CMakeLists.txt | 1 + tests/CudaHandledErrorTest.cpp | 85 +++++++++++++++++++ 9 files changed, 111 insertions(+) create mode 100644 tests/CudaHandledErrorTest.cpp diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 3cedde9d0..22d12c221 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -38,4 +38,6 @@ void set_gpu(int32_t dev_id) {} void pin_gpu() {} +void cuda_clear_error() {} + #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 51e203cf9..5cc0669db 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -61,3 +61,7 @@ void pin_gpu() { if (dev_count > 0) set_gpu(counter.fetch_add(1) % dev_count); } + +void cuda_clear_error() { + cudaGetLastError(); +} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index ccbd4a758..2b078ba3f 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -22,3 +22,11 @@ void set_gpu(int32_t dev_id); // (counter++ % get_gpu_count()). Call once per thread; no thread id needed. No-op when no GPU // is visible. Honours CUDA_VISIBLE_DEVICES via get_gpu_count(). void pin_gpu(); + +// Drop the error CUDA has recorded for the calling thread. Call it where a CUDA failure has been +// HANDLED - a device route that fell back to the host, an indexing attempt whose failure was turned +// into a result - because the error otherwise stays as the thread's last error and the next +// cuda_err(cudaGetLastError()) after some later kernel launch reports it, over work that went fine. +// A sticky error (an illegal access, say) is not cleared by this, and nothing here pretends it is: +// the context is gone in that case and every later call fails on its own. No-op without CUDA. +void cuda_clear_error(); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 740798420..93a3d736d 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -8,6 +8,7 @@ #include "spot_finding/StrongPixelSet.h" #include "../compression/JFJochDecompress.h" +#include "../common/CUDAWrapper.h" #include "spot_finding/SpotUtils.h" #include "bragg_prediction/BraggPredictionFactory.h" @@ -107,6 +108,10 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, // rather than the acquisition. spdlog::warn("Device decoding failed ({}), falling back to host decompression", e.what()); decoded_on_device = false; + // Handled here, so it must not stay behind as the thread's last error: the host route below + // runs kernels of its own and checks cudaGetLastError() after them, which would report this + // failure again - fatally - over an image that decoded perfectly well on the host. + cuda_clear_error(); } const auto compression_end_time = std::chrono::steady_clock::now(); diff --git a/image_analysis/beam_stop/ShadowFinder.cpp b/image_analysis/beam_stop/ShadowFinder.cpp index a0d3559a3..6a6816173 100644 --- a/image_analysis/beam_stop/ShadowFinder.cpp +++ b/image_analysis/beam_stop/ShadowFinder.cpp @@ -447,6 +447,7 @@ void ShadowFinder::AddImage(const DataMessage &data, std::vector &buffe return; } catch (const std::exception &e) { spdlog::warn("Beam stop: GPU accumulate failed ({}), falling back to the host", e.what()); + cuda_clear_error(); // handled - see cuda_clear_error() } } #endif diff --git a/image_analysis/geom_refinement/BeamCenterFFT.cpp b/image_analysis/geom_refinement/BeamCenterFFT.cpp index 8ebf71bd9..8c2f60366 100644 --- a/image_analysis/geom_refinement/BeamCenterFFT.cpp +++ b/image_analysis/geom_refinement/BeamCenterFFT.cpp @@ -236,6 +236,7 @@ BeamCenterFFTResult BeamCenterFFTScore(int64_t width, int64_t height, // The card is shared with this run's analysis workers, so the memory the check above // saw can be gone by the time it is asked for. A capture is not worth failing a run // over: score it on the CPU instead. + cuda_clear_error(); // handled - see cuda_clear_error() } } #endif diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 56b925c18..5b119957a 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -156,6 +156,10 @@ void IndexerThread::Worker(int threadid) { spdlog::error("Indexer thread {} failed: {}", threadid, e.what()); tmp_result = std::make_unique(IndexerResult{ .lattice = {}, .indexing_time_s = 0, .executed = false, .error = e.what()}); + // This thread goes on to the next frame, so a CUDA error left behind here would be + // reported by the first cudaGetLastError() of every frame that follows - one failed + // attempt would read as an indexer that never works again. + cuda_clear_error(); } { std::unique_lock lock(m); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6e90cd2bf..c0e10383c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -80,6 +80,7 @@ ADD_EXECUTABLE(jfjoch_test AdaptiveSpotFinderGPUTest.cpp SpotExtractorGPUParityTest.cpp CUDAMemHelpersTest.cpp + CudaHandledErrorTest.cpp CalcBraggPredictionTest.cpp SpotUtilsTest.cpp LatticeSearchTest.cpp diff --git a/tests/CudaHandledErrorTest.cpp b/tests/CudaHandledErrorTest.cpp new file mode 100644 index 000000000..9889ddfc1 --- /dev/null +++ b/tests/CudaHandledErrorTest.cpp @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include "../common/CUDAWrapper.h" + +#ifdef JFJOCH_USE_CUDA + +#include +#include + +#include "../common/AzimuthalIntegrationMapping.h" +#include "../common/AzimuthalIntegrationProfile.h" +#include "../common/DiffractionExperiment.h" +#include "../common/PixelMask.h" +#include "../compression/JFJochCompressor.h" +#include "../image_analysis/IndexAndRefine.h" +#include "../image_analysis/MXAnalysisWithoutFPGA.h" +#include "../image_analysis/indexing/CUDAMemHelpers.h" + +// What makes a handled CUDA failure dangerous: the failed call leaves the error behind as the +// thread's last error, and nothing but cudaGetLastError() takes it away - later successful calls +// neither clear it nor return it. So a route that failed, was caught and was replaced by another +// one hands its error to the next cudaGetLastError() in the code that followed, which is the check +// after a kernel launch. That is how a broker logged a raw "out of memory" over an image whose +// buffers were all allocated. cuda_clear_error() is what a handled failure discharges it with. +TEST_CASE("CudaDevicePtr_FailedAllocationLeavesErrorUntilCleared", "[CUDAMemHelpers]") { + if (get_gpu_count() == 0) + SKIP("No CUDA GPU present"); + + cuda_clear_error(); + REQUIRE_THROWS(CudaDevicePtr(std::numeric_limits::max(), CudaAlloc::Synchronous)); + + // Work that goes fine in between changes nothing: the error is still waiting. + CudaDevicePtr fine(1 << 16); + REQUIRE(cudaStreamSynchronize(cuda_allocation_stream()) == cudaSuccess); + + cuda_clear_error(); + CHECK(cudaGetLastError() == cudaSuccess); +} + +// The device decoder is allowed to fail - the host decompresses instead - but the host route runs +// kernels of its own and checks cudaGetLastError() after them. An image must come out of that +// whole, and with nothing left behind for the next one. +TEST_CASE("MXAnalysis_HandledDeviceDecodeFailureLeavesNoError", "[CUDAMemHelpers]") { + if (get_gpu_count() == 0) + SKIP("No CUDA GPU present"); + + constexpr size_t width = 1030, height = 1064; + + DiffractionExperiment experiment(DetDECTRIS(width, height, "Test", {})); + experiment.ImagesPerTrigger(1).NumTriggers(1).PixelSigned(false) + .DetectorDistance_mm(75).BeamX_pxl(515).BeamY_pxl(532).IncidentEnergy_keV(12.4); + + PixelMask pixel_mask(experiment); + AzimuthalIntegrationMapping mapping(experiment, pixel_mask); + IndexAndRefine index_and_refine(experiment, nullptr); + MXAnalysisWithoutFPGA analysis(experiment, mapping, pixel_mask, index_and_refine); + + std::vector pixels(width * height, 0); + for (size_t i = 0; i < pixels.size(); i += 997) + pixels[i] = 500; + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + const std::vector compressed = compressor.Compress(pixels); + REQUIRE(!compressed.empty()); + + DataMessage message{}; + message.number = 0; + message.image = CompressedImage(compressed.data(), compressed.size(), width, height, + CompressedImageMode::Uint16, CompressionAlgorithm::BSHUF_LZ4); + + SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); + settings.enable = false; + + // Stage the device route into failing, by leaving behind the error a request the card cannot + // serve leaves. + void *unservable = nullptr; + REQUIRE(cudaMalloc(&unservable, std::numeric_limits::max()) != cudaSuccess); + + AzimuthalIntegrationProfile profile(mapping); + REQUIRE_NOTHROW(analysis.Analyze(message, profile, settings)); + CHECK(cudaGetLastError() == cudaSuccess); +} + +#endif