GPU: kernels read host buffers in place on unified-memory systems

gpu_direct_host_read() (CUDAWrapper) is true where the device accesses pageable
memory through the host's page tables (cudaDevAttrPageableMemoryAccess and
...UsesHostPageTables: GB10, GH200, GB200), asked once per device;
JFJOCH_FORCE_DIRECT_HOST_READ forces it on for testing with page-locked inputs.
Where it holds:
- BSLZ4DecoderGPU: the LZ4/fused kernels and the tail copy read the reader's
  compressed chunk directly, no pinned staging memcpy and no H2D copy;
- ImagePreprocessorGPU: Analyze and the UINT16_WIDENED path hand the host
  buffer to preprocess_kernel (if aligned to the pixel type), and nothing is
  page-locked;
- RotationScaleMergeGPU::IngestUpload: IngestExtractKernel reads the pinned
  stage in place, without device staging buffers; the stage is reused only
  after the kernel has run.
Discrete cards take the unchanged path. Every caller synchronises its stream
before the host buffer goes away. DirectHostReadGPUTest checks the three
sites against the CPU on page-locked inputs, in either mode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

image_analysis/image_preprocessing/BSLZ4DecoderGPU.cu

@@ -7,6 +7,7 @@
 
 #include "BSLZ4DecoderGPU.h"
 #include "BSLZ4DecodeWarp.h"
+#include "../../common/CUDAWrapper.h"
 #include "../../common/JFJochException.h"
 #include "../../compression/JFJochDecompress.h"   // BSHUF_BLOCKED_MULT and the container layout
 
@@ -118,6 +119,7 @@ bool BSLZ4DecoderGPU::Supports(const CompressedImage &image) {
 
 BSLZ4DecoderGPU::BSLZ4DecoderGPU(size_t in_max_uncompressed_bytes, std::shared_ptr<CudaStream> in_stream)
     : stream(std::move(in_stream)),
+      direct_host_read(gpu_direct_host_read()),
       max_uncompressed_bytes(in_max_uncompressed_bytes) {
     gpu_status = CudaDevicePtr<uint32_t>(1);
     host_status = CudaHostPtr<uint32_t>(1);
@@ -211,7 +213,8 @@ BSLZ4ShuffledImage BSLZ4DecoderGPU::PrepareChunk(const CompressedImage &image) {
     if (nblocks_needed > (clen - 12) / MIN_BLOCK_BYTES_ON_WIRE)
         throw JFJochException(JFJochExceptionCategory::Compression, "bslz4 chunk too short for the blocks its header implies");
 
-    EnsureCompressedCapacity(clen);
+    if (!direct_host_read)
+        EnsureCompressedCapacity(clen);
     EnsureBlockCapacity(nblocks_needed);
 
     size_t nblk = 0, off = 12, out_off = 0;
@@ -240,8 +243,14 @@ BSLZ4ShuffledImage BSLZ4DecoderGPU::PrepareChunk(const CompressedImage &image) {
     host_status.get()[0] = 0;
     cuda_err(cudaMemcpyAsync(gpu_status.get(), host_status.get(), sizeof(uint32_t),
                              cudaMemcpyHostToDevice, *stream));
-    memcpy(host_compressed.get(), src, clen);
-    cuda_err(cudaMemcpyAsync(gpu_compressed.get(), host_compressed.get(), clen, cudaMemcpyHostToDevice, *stream));
+    // Where the card reads host memory directly, the kernels read the reader's own buffer, which the
+    // caller keeps until it has synchronised the stream.
+    const uint8_t *compressed = src;
+    if (!direct_host_read) {
+        memcpy(host_compressed.get(), src, clen);
+        cuda_err(cudaMemcpyAsync(gpu_compressed.get(), host_compressed.get(), clen, cudaMemcpyHostToDevice, *stream));
+        compressed = gpu_compressed.get();
+    }
 
     const int nb = static_cast<int>(nblk);
     if (nb > 0)
@@ -249,7 +258,7 @@ BSLZ4ShuffledImage BSLZ4DecoderGPU::PrepareChunk(const CompressedImage &image) {
                                  cudaMemcpyHostToDevice, *stream));
 
     BSLZ4ShuffledImage ret;
-    ret.compressed = gpu_compressed.get();
+    ret.compressed = compressed;
     ret.desc = gpu_desc.get();
     ret.status = gpu_status.get();
     ret.nblocks = nb;
@@ -257,7 +266,7 @@ BSLZ4ShuffledImage BSLZ4DecoderGPU::PrepareChunk(const CompressedImage &image) {
     ret.block_bytes = block_bytes;
     ret.tail_elems = static_cast<uint32_t>(leftover_bytes / elem_size);
     ret.tail_elem0 = static_cast<uint32_t>(out_off / elem_size);
-    ret.tail_src = leftover_bytes > 0 ? gpu_compressed.get() + off : nullptr;
+    ret.tail_src = leftover_bytes > 0 ? compressed + off : nullptr;
     return ret;
 }
 
@@ -306,11 +315,12 @@ void BSLZ4DecoderGPU::Decode(const CompressedImage &image, uint8_t *gpu_out) {
         cuda_err(cudaGetLastError());
     }
 
-    // The verbatim tail is already on the device inside the uploaded chunk.
+    // The verbatim tail is already on the device inside the uploaded chunk - or in host memory the card
+    // reads directly, hence cudaMemcpyDefault.
     if (s.tail_elems > 0)
         cuda_err(cudaMemcpyAsync(gpu_out + static_cast<size_t>(s.tail_elem0) * s.elem_size, s.tail_src,
                                  static_cast<size_t>(s.tail_elems) * s.elem_size,
-                                 cudaMemcpyDeviceToDevice, *stream));
+                                 cudaMemcpyDefault, *stream));
 }
 
 void BSLZ4DecoderGPU::ThrowIfDecodeFailed() {