diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 57849fb15..740798420 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -350,8 +350,12 @@ void MXAnalysisWithoutFPGA::AnalyzeROIOnly(DataMessage &output) { const uint8_t *MXAnalysisWithoutFPGA::Decompress(const CompressedImage &image) { // An uncompressed image is read straight out of the message and never touches decompression_buffer, - // so it stays in pageable memory - the buffer is only worth page-locking when it is actually used. - if (image.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION) + // so what the upload reads is the reader's own bytes: page-lock those instead. Pageable memory is + // staged by the driver a chunk at a time, which on a 4-byte 16 Mpx frame is 12.9 ms against 4.0 ms + // pinned - and a miniCBF sweep takes this path for every image. + if (image.GetCompressionAlgorithm() == CompressionAlgorithm::NO_COMPRESSION) + preprocessor->PinInputRegion(image.GetCompressed(), image.GetUncompressedSize()); + else preprocessor->PinInputBuffer(decompression_buffer, image.GetUncompressedSize()); return image.GetUncompressedPtr(decompression_buffer); } diff --git a/image_analysis/image_preprocessing/ImagePreprocessor.h b/image_analysis/image_preprocessing/ImagePreprocessor.h index 2264b649b..c9a60b97b 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessor.h +++ b/image_analysis/image_preprocessing/ImagePreprocessor.h @@ -49,4 +49,11 @@ public: // pinned pool, which is a host-side copy on the calling thread: it does not overlap and it degrades // badly with the number of workers. Nothing to do on the CPU. virtual void PinInputBuffer(std::vector &buffer, size_t size) {} + + // The same for a region the caller already owns. An uncompressed image is never decompressed + // into a buffer of ours - the upload reads the reader's own bytes - so that is what has to be + // page-locked. The region is remembered, so a worker handing over the same buffer on every frame + // registers it once. The registration is dropped when this object is destroyed, which is why an + // engine must not outlive the buffer it was given. + virtual void PinInputRegion(const void *ptr, size_t bytes) {} }; diff --git a/image_analysis/image_preprocessing/ImagePreprocessorGPU.cu b/image_analysis/image_preprocessing/ImagePreprocessorGPU.cu index 5fc0e46f3..643058cb6 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorGPU.cu +++ b/image_analysis/image_preprocessing/ImagePreprocessorGPU.cu @@ -338,6 +338,25 @@ float ImagePreprocessorGPU::GetLastDecompressionTime_s() const { return bslz4_decoder ? bslz4_decoder->GetDecodeTime_s() : 0.0f; } +ImagePreprocessorGPU::~ImagePreprocessorGPU() { + // Unchecked, as everywhere else in this teardown path: a destructor is noexcept and the region + // may already be gone with the device. + if (pinned_input) + cudaHostUnregister(const_cast(pinned_input)); +} + +void ImagePreprocessorGPU::PinInputRegion(const void *ptr, size_t bytes) { + if (ptr == pinned_input && bytes == pinned_input_bytes) + return; + if (pinned_input) + cuda_err(cudaHostUnregister(const_cast(pinned_input))); + pinned_input = nullptr; + pinned_input_bytes = 0; + cuda_err(cudaHostRegister(const_cast(ptr), bytes, cudaHostRegisterDefault)); + pinned_input = ptr; + pinned_input_bytes = bytes; +} + void ImagePreprocessorGPU::PinInputBuffer(std::vector &buffer, size_t size) { if (buffer.size() == size) return; diff --git a/image_analysis/image_preprocessing/ImagePreprocessorGPU.h b/image_analysis/image_preprocessing/ImagePreprocessorGPU.h index 60679c7ec..c28f4a48c 100644 --- a/image_analysis/image_preprocessing/ImagePreprocessorGPU.h +++ b/image_analysis/image_preprocessing/ImagePreprocessorGPU.h @@ -26,6 +26,9 @@ class ImagePreprocessorGPU : public ImagePreprocessor { std::vector cpu_stats; CudaRegisteredVector cpu_stats_reg; CudaRegisteredVector input_reg; // page-locks the caller's decompression buffer + // The uncompressed-image counterpart of input_reg: the region PinInputRegion last page-locked. + const void *pinned_input = nullptr; + size_t pinned_input_bytes = 0; std::vector cpu_image; @@ -46,10 +49,12 @@ public: // per pixel it is the single largest transfer in the pipeline, so the caller says whether it wants it. ImagePreprocessorGPU(const DiffractionExperiment &experiment, const PixelMask &mask, std::shared_ptr stream, bool copy_image_to_host = true); + ~ImagePreprocessorGPU() override; ImageStatistics Analyze(ImagePreprocessorBuffer &processed_image, const uint8_t *decompressed_image, CompressedImageMode image_mode) override; bool AnalyzeCompressed(ImagePreprocessorBuffer &processed_image, const CompressedImage &image, ImageStatistics &stats) override; [[nodiscard]] float GetLastDecompressionTime_s() const override; void PinInputBuffer(std::vector &buffer, size_t size) override; + void PinInputRegion(const void *ptr, size_t bytes) override; }; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index b53322e9f..42d3f7731 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1412,10 +1412,12 @@ void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_ auto worker = [&]() { pin_gpu(); // round-robin per worker thread; must precede engine construction + // Before the analysis engine, which page-locks these bytes for its uploads and unregisters + // them when it is destroyed: it must not outlive the buffer it was given. + JFJochReaderRawImage img; MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, indexer, /*enable_fused_adaptive_gpu=*/true); AzimuthalIntegrationProfile profile(mapping); - JFJochReaderRawImage img; for (int stripe = next_stripe.fetch_add(1); stripe < STRIPES && !cancelled_; stripe = next_stripe.fetch_add(1)) { @@ -2286,9 +2288,11 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b ? std::min(config_.nthreads, std::max(8, 4 * get_gpu_count())) : config_.nthreads; struct SpotEngine { + // Before the analysis engine, which page-locks these bytes for its uploads and + // unregisters them when it is destroyed: it must not outlive the buffer it was given. + JFJochReaderRawImage raw_image; std::unique_ptr analysis; std::unique_ptr profile; - JFJochReaderRawImage raw_image; }; std::vector engines(std::max(spot_workers, 1)); const auto use_gpu_of_worker = [](size_t t) { @@ -3472,11 +3476,13 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b auto full_worker = [&]() { pin_gpu(); // round-robin per worker thread; must precede engine construction + // Before the analysis engine, which page-locks these bytes for its uploads and unregisters + // them when it is destroyed: it must not outlive the buffer it was given. + std::shared_ptr img; MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, *indexer, /*enable_fused_adaptive_gpu=*/true); AzimuthalIntegrationProfile profile(mapping); HarmonicEvidence worker_harmonic; - std::shared_ptr img; while (!cancelled_) { const int ordinal = next_ordinal.fetch_add(1);