diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index e57813c12..4bb7e826d 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -46,4 +46,6 @@ void set_gpu_blocking_sync() {} void cuda_clear_error() {} +void cuda_throw_if_context_lost() {} + #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 782a4374e..2cfedf594 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -114,3 +114,13 @@ void set_gpu_blocking_sync() { void cuda_clear_error() { cudaGetLastError(); } + +void cuda_throw_if_context_lost() { + // cudaFree(nullptr) frees nothing, but a lost context fails it. The last error cannot say the same: + // once cudaGetLastError() has returned a sticky error, it and cudaPeekAtLastError() report success. + const cudaError_t err = cudaFree(nullptr); + if (err != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, + std::string("CUDA device unusable after an unrecoverable error: ") + + cudaGetErrorString(err)); +} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index 3feda3068..737acd4b4 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -44,3 +44,9 @@ void set_gpu_blocking_sync(); // A sticky error (an illegal access, say) is not cleared by this, and nothing here pretends it is: // the context is gone in that case and every later call fails on its own. No-op without CUDA. void cuda_clear_error(); + +// Throw if the CUDA context is lost - a sticky error (an illegal access, say) that every later call +// on this device will fail with. For a device route about to fall back to the host: there is no +// fallback from that, and falling back only moves the failure somewhere less clear. A handled, +// non-sticky failure passes. No-op without CUDA. +void cuda_throw_if_context_lost(); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 7466943a7..a861f50bf 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -117,7 +117,8 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, // The device route must never be the reason a frame fails: whatever it could not handle, the // host decoder gets its turn. If the data really is bad the host throws too and the caller // sees the same error it saw before any of this existed - but a GPU-side problem costs speed - // rather than the acquisition. + // rather than the acquisition. Unless the context is lost, which the host route cannot survive. + cuda_throw_if_context_lost(); spdlog::warn("Device decoding failed ({}), falling back to host decompression", e.what()); decoded_on_device = false; // Handled here, so it must not stay behind as the thread's last error: the host route below diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index f060ddd0e..c8119f214 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1710,6 +1710,7 @@ void Rugnux::MaskDefectivePixels(int start_image, const std::vector &sample try { decoded_on_device = preprocessor->AnalyzeCompressed(*image, raw.image, stats); } catch (const JFJochException &e) { + cuda_throw_if_context_lost(); logger.Warning("Defective pixels: device decoding of image {} failed ({}), " "decompressing it on the host", image_idx, e.what()); cuda_clear_error();