diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 3cedde9d0..22d12c221 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -38,4 +38,6 @@ void set_gpu(int32_t dev_id) {} void pin_gpu() {} +void cuda_clear_error() {} + #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 51e203cf9..5cc0669db 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -61,3 +61,7 @@ void pin_gpu() { if (dev_count > 0) set_gpu(counter.fetch_add(1) % dev_count); } + +void cuda_clear_error() { + cudaGetLastError(); +} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index ccbd4a758..2b078ba3f 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -22,3 +22,11 @@ void set_gpu(int32_t dev_id); // (counter++ % get_gpu_count()). Call once per thread; no thread id needed. No-op when no GPU // is visible. Honours CUDA_VISIBLE_DEVICES via get_gpu_count(). void pin_gpu(); + +// Drop the error CUDA has recorded for the calling thread. Call it where a CUDA failure has been +// HANDLED - a device route that fell back to the host, an indexing attempt whose failure was turned +// into a result - because the error otherwise stays as the thread's last error and the next +// cuda_err(cudaGetLastError()) after some later kernel launch reports it, over work that went fine. +// A sticky error (an illegal access, say) is not cleared by this, and nothing here pretends it is: +// the context is gone in that case and every later call fails on its own. No-op without CUDA. +void cuda_clear_error(); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 740798420..93a3d736d 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -8,6 +8,7 @@ #include "spot_finding/StrongPixelSet.h" #include "../compression/JFJochDecompress.h" +#include "../common/CUDAWrapper.h" #include "spot_finding/SpotUtils.h" #include "bragg_prediction/BraggPredictionFactory.h" @@ -107,6 +108,10 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, // rather than the acquisition. spdlog::warn("Device decoding failed ({}), falling back to host decompression", e.what()); decoded_on_device = false; + // Handled here, so it must not stay behind as the thread's last error: the host route below + // runs kernels of its own and checks cudaGetLastError() after them, which would report this + // failure again - fatally - over an image that decoded perfectly well on the host. + cuda_clear_error(); } const auto compression_end_time = std::chrono::steady_clock::now(); diff --git a/image_analysis/beam_stop/ShadowFinder.cpp b/image_analysis/beam_stop/ShadowFinder.cpp index a0d3559a3..6a6816173 100644 --- a/image_analysis/beam_stop/ShadowFinder.cpp +++ b/image_analysis/beam_stop/ShadowFinder.cpp @@ -447,6 +447,7 @@ void ShadowFinder::AddImage(const DataMessage &data, std::vector &buffe return; } catch (const std::exception &e) { spdlog::warn("Beam stop: GPU accumulate failed ({}), falling back to the host", e.what()); + cuda_clear_error(); // handled - see cuda_clear_error() } } #endif diff --git a/image_analysis/geom_refinement/BeamCenterFFT.cpp b/image_analysis/geom_refinement/BeamCenterFFT.cpp index 8ebf71bd9..8c2f60366 100644 --- a/image_analysis/geom_refinement/BeamCenterFFT.cpp +++ b/image_analysis/geom_refinement/BeamCenterFFT.cpp @@ -236,6 +236,7 @@ BeamCenterFFTResult BeamCenterFFTScore(int64_t width, int64_t height, // The card is shared with this run's analysis workers, so the memory the check above // saw can be gone by the time it is asked for. A capture is not worth failing a run // over: score it on the CPU instead. + cuda_clear_error(); // handled - see cuda_clear_error() } } #endif diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 56b925c18..5b119957a 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -156,6 +156,10 @@ void IndexerThread::Worker(int threadid) { spdlog::error("Indexer thread {} failed: {}", threadid, e.what()); tmp_result = std::make_unique(IndexerResult{ .lattice = {}, .indexing_time_s = 0, .executed = false, .error = e.what()}); + // This thread goes on to the next frame, so a CUDA error left behind here would be + // reported by the first cudaGetLastError() of every frame that follows - one failed + // attempt would read as an indexer that never works again. + cuda_clear_error(); } { std::unique_lock lock(m); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6e90cd2bf..c0e10383c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -80,6 +80,7 @@ ADD_EXECUTABLE(jfjoch_test AdaptiveSpotFinderGPUTest.cpp SpotExtractorGPUParityTest.cpp CUDAMemHelpersTest.cpp + CudaHandledErrorTest.cpp CalcBraggPredictionTest.cpp SpotUtilsTest.cpp LatticeSearchTest.cpp diff --git a/tests/CudaHandledErrorTest.cpp b/tests/CudaHandledErrorTest.cpp new file mode 100644 index 000000000..9889ddfc1 --- /dev/null +++ b/tests/CudaHandledErrorTest.cpp @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include "../common/CUDAWrapper.h" + +#ifdef JFJOCH_USE_CUDA + +#include +#include + +#include "../common/AzimuthalIntegrationMapping.h" +#include "../common/AzimuthalIntegrationProfile.h" +#include "../common/DiffractionExperiment.h" +#include "../common/PixelMask.h" +#include "../compression/JFJochCompressor.h" +#include "../image_analysis/IndexAndRefine.h" +#include "../image_analysis/MXAnalysisWithoutFPGA.h" +#include "../image_analysis/indexing/CUDAMemHelpers.h" + +// What makes a handled CUDA failure dangerous: the failed call leaves the error behind as the +// thread's last error, and nothing but cudaGetLastError() takes it away - later successful calls +// neither clear it nor return it. So a route that failed, was caught and was replaced by another +// one hands its error to the next cudaGetLastError() in the code that followed, which is the check +// after a kernel launch. That is how a broker logged a raw "out of memory" over an image whose +// buffers were all allocated. cuda_clear_error() is what a handled failure discharges it with. +TEST_CASE("CudaDevicePtr_FailedAllocationLeavesErrorUntilCleared", "[CUDAMemHelpers]") { + if (get_gpu_count() == 0) + SKIP("No CUDA GPU present"); + + cuda_clear_error(); + REQUIRE_THROWS(CudaDevicePtr(std::numeric_limits::max(), CudaAlloc::Synchronous)); + + // Work that goes fine in between changes nothing: the error is still waiting. + CudaDevicePtr fine(1 << 16); + REQUIRE(cudaStreamSynchronize(cuda_allocation_stream()) == cudaSuccess); + + cuda_clear_error(); + CHECK(cudaGetLastError() == cudaSuccess); +} + +// The device decoder is allowed to fail - the host decompresses instead - but the host route runs +// kernels of its own and checks cudaGetLastError() after them. An image must come out of that +// whole, and with nothing left behind for the next one. +TEST_CASE("MXAnalysis_HandledDeviceDecodeFailureLeavesNoError", "[CUDAMemHelpers]") { + if (get_gpu_count() == 0) + SKIP("No CUDA GPU present"); + + constexpr size_t width = 1030, height = 1064; + + DiffractionExperiment experiment(DetDECTRIS(width, height, "Test", {})); + experiment.ImagesPerTrigger(1).NumTriggers(1).PixelSigned(false) + .DetectorDistance_mm(75).BeamX_pxl(515).BeamY_pxl(532).IncidentEnergy_keV(12.4); + + PixelMask pixel_mask(experiment); + AzimuthalIntegrationMapping mapping(experiment, pixel_mask); + IndexAndRefine index_and_refine(experiment, nullptr); + MXAnalysisWithoutFPGA analysis(experiment, mapping, pixel_mask, index_and_refine); + + std::vector pixels(width * height, 0); + for (size_t i = 0; i < pixels.size(); i += 997) + pixels[i] = 500; + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + const std::vector compressed = compressor.Compress(pixels); + REQUIRE(!compressed.empty()); + + DataMessage message{}; + message.number = 0; + message.image = CompressedImage(compressed.data(), compressed.size(), width, height, + CompressedImageMode::Uint16, CompressionAlgorithm::BSHUF_LZ4); + + SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); + settings.enable = false; + + // Stage the device route into failing, by leaving behind the error a request the card cannot + // serve leaves. + void *unservable = nullptr; + REQUIRE(cudaMalloc(&unservable, std::numeric_limits::max()) != cudaSuccess); + + AzimuthalIntegrationProfile profile(mapping); + REQUIRE_NOTHROW(analysis.Analyze(message, profile, settings)); + CHECK(cudaGetLastError() == cudaSuccess); +} + +#endif