From ea9d0adcde686a3fef6138eceb7f5a66357f3bdb Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 17:11:38 +0200 Subject: [PATCH] Stop on a lost CUDA context instead of falling back to the host decoder A sticky error (an illegal address, say) is reported by whichever worker synchronises next - usually the bslz4 device decode - and MXAnalysisWithoutFPGA::Analyze and Rugnux::MaskDefectivePixels then logged "falling back to host" and carried on, although the context is gone and the run fails anyway, later and less clearly. cuda_throw_if_context_lost() now throws a GPUCUDAError ("CUDA device unusable after an unrecoverable error: ...") there first, which the image loops already treat as fatal (IsFatalResourceError), like a CUDA out-of-memory. The last error cannot tell: once cudaGetLastError() has returned a sticky error, it and cudaPeekAtLastError() both report success. cudaFree(nullptr) frees nothing and does not synchronise, but returns the sticky error (checked: after an illegal-address kernel it returns cudaErrorIllegalAddress; with only a pending out-of-memory it returns success), so a handled, non-sticky decode failure still falls back - MXAnalysis_HandledDeviceDecodeFailureLeavesNoError passes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- common/CUDAWrapper.cpp | 2 ++ common/CUDAWrapper.cu | 10 ++++++++++ common/CUDAWrapper.h | 6 ++++++ image_analysis/MXAnalysisWithoutFPGA.cpp | 3 ++- rugnux/Rugnux.cpp | 1 + 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index e57813c12..4bb7e826d 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -46,4 +46,6 @@ void set_gpu_blocking_sync() {} void cuda_clear_error() {} +void cuda_throw_if_context_lost() {} + #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 782a4374e..2cfedf594 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -114,3 +114,13 @@ void set_gpu_blocking_sync() { void cuda_clear_error() { cudaGetLastError(); } + +void cuda_throw_if_context_lost() { + // cudaFree(nullptr) frees nothing, but a lost context fails it. The last error cannot say the same: + // once cudaGetLastError() has returned a sticky error, it and cudaPeekAtLastError() report success. + const cudaError_t err = cudaFree(nullptr); + if (err != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, + std::string("CUDA device unusable after an unrecoverable error: ") + + cudaGetErrorString(err)); +} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index 3feda3068..737acd4b4 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -44,3 +44,9 @@ void set_gpu_blocking_sync(); // A sticky error (an illegal access, say) is not cleared by this, and nothing here pretends it is: // the context is gone in that case and every later call fails on its own. No-op without CUDA. void cuda_clear_error(); + +// Throw if the CUDA context is lost - a sticky error (an illegal access, say) that every later call +// on this device will fail with. For a device route about to fall back to the host: there is no +// fallback from that, and falling back only moves the failure somewhere less clear. A handled, +// non-sticky failure passes. No-op without CUDA. +void cuda_throw_if_context_lost(); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 7466943a7..a861f50bf 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -117,7 +117,8 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, // The device route must never be the reason a frame fails: whatever it could not handle, the // host decoder gets its turn. If the data really is bad the host throws too and the caller // sees the same error it saw before any of this existed - but a GPU-side problem costs speed - // rather than the acquisition. + // rather than the acquisition. Unless the context is lost, which the host route cannot survive. + cuda_throw_if_context_lost(); spdlog::warn("Device decoding failed ({}), falling back to host decompression", e.what()); decoded_on_device = false; // Handled here, so it must not stay behind as the thread's last error: the host route below diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index f060ddd0e..c8119f214 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1710,6 +1710,7 @@ void Rugnux::MaskDefectivePixels(int start_image, const std::vector &sample try { decoded_on_device = preprocessor->AnalyzeCompressed(*image, raw.image, stats); } catch (const JFJochException &e) { + cuda_throw_if_context_lost(); logger.Warning("Defective pixels: device decoding of image {} failed ({}), " "decompressing it on the host", image_idx, e.what()); cuda_clear_error();