From cc925b26683526ad18890f1f5ba4ec33fa5d91e4 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 20:25:36 +0200 Subject: [PATCH] Remove NUMAHWPolicy and the libnuma dependency NUMA CPU/memory pinning is no longer worthwhile: the FPGA DMA buffers are placed device-local by the kernel (dma_alloc_coherent), the big RAM ring buffer is random-access (first-touch handles placement), and GPU work is already spread across all visible devices. So drop the pinning entirely and with it libnuma. - Delete NUMAHWPolicy; the only concern worth keeping - GPU selection - is done directly via pin_gpu() (round-robin over visible GPUs) in the indexer pool and the Lite analysis threads. CPU-only threads (FPGA acquire/pedestal/summation/frame-transform) no longer bind anything. - Drop get_gpu_numa_node() (sysfs lookup) - only SelectGPUAndItsNUMA used it. - numa_policy broker setting is deprecated and ignored (kept in the API for backward compatibility; warns once on startup). - Remove NUMA_LIBRARY / numa.h / numaif.h detection from CMake. - Docs: drop the NUMA dependency, remove the numa_policy config example, and document running multiple brokers on disjoint GPUs via CUDA_VISIBLE_DEVICES. - Remove NUMA_GPU_REVIEW.md (the planning note; this work is now done). Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 5 - NUMA_GPU_REVIEW.md | 145 ------------------ broker/JFJochBrokerParser.cpp | 4 - broker/jfjoch_api.yaml | 3 +- broker/jfjoch_broker.cpp | 2 + common/CMakeLists.txt | 9 -- common/CUDAWrapper.cpp | 4 - common/CUDAWrapper.cu | 54 ------- common/CUDAWrapper.h | 1 - common/NUMAHWPolicy.cpp | 125 --------------- common/NUMAHWPolicy.h | 33 ---- docs/DEPLOYMENT.md | 8 +- docs/JFJOCH_BROKER.md | 1 - docs/SOFTWARE.md | 1 - image_analysis/indexing/IndexerThreadPool.cpp | 10 +- image_analysis/indexing/IndexerThreadPool.h | 1 - receiver/JFJochReceiver.cpp | 2 - receiver/JFJochReceiver.h | 4 - receiver/JFJochReceiverFPGA.cpp | 24 --- receiver/JFJochReceiverFPGA.h | 2 - receiver/JFJochReceiverLite.cpp | 11 +- receiver/JFJochReceiverLite.h | 2 - receiver/JFJochReceiverService.cpp | 12 -- receiver/JFJochReceiverService.h | 4 - receiver/JFJochReceiverTest.cpp | 8 +- receiver/JFJochReceiverTest.h | 2 - tools/jfjoch_fpga_test.cpp | 6 +- tools/jfjoch_lite_perf_test.cpp | 4 +- 28 files changed, 24 insertions(+), 463 deletions(-) delete mode 100644 NUMA_GPU_REVIEW.md delete mode 100644 common/NUMAHWPolicy.cpp delete mode 100644 common/NUMAHWPolicy.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d675e610..f81c5ea0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,11 +73,6 @@ IF(HAS_FFTW3_H AND FFTWF_LIBRARY) ENDIF() INCLUDE_DIRECTORIES(include) -INCLUDE(CheckIncludeFile) - -FIND_LIBRARY(NUMA_LIBRARY NAMES numa DOC "NUMA Library") -CHECK_INCLUDE_FILE(numaif.h HAS_NUMAIF) -CHECK_INCLUDE_FILE(numa.h HAS_NUMA_H) include(FetchContent) diff --git a/NUMA_GPU_REVIEW.md b/NUMA_GPU_REVIEW.md deleted file mode 100644 index 840d86ac..00000000 --- a/NUMA_GPU_REVIEW.md +++ /dev/null @@ -1,145 +0,0 @@ -# NUMA / GPU usage — current state, goals, remaining work - -Working note on the NUMA/GPU direction. Several items are now **implemented and committed** on -branch `2606-pixel-refine` (see §3/§5); this note tracks what landed, what's left, and — importantly -— the corrected mental model of *where NUMA actually matters*. File:line anchors are from -`2606-pixel-refine`. - -**Headline:** after the committed work, **libnuma is used in exactly one file** -(`common/NUMAHWPolicy.cpp`). Everything else (GPU node lookup, the big RAM buffer, the FPGA DMA -buffers, the simulator) is libnuma-free. - -## 1. Current state (the mind map) - -### 1a. `ImageBuffer` — the big RAM ring buffer -- **One instance**, a member of the receiver: `JFJochReceiverService::image_buffer` - (`receiver/JFJochReceiverService.h:21`), sized `image_buffer_MiB` from broker config - (`broker/jfjoch_broker.cpp:104` → ctor `receiver/JFJochReceiverService.cpp:15`). The 150–200 GB - allocation. -- **Allocation (DONE, commit `d373ba04`):** plain `std::malloc` + a **parallel first-touch `memset`** - (`hardware_concurrency()` threads, unpinned) in `common/ImageBuffer.cpp`. Each page is - first-touched — and thus NUMA-placed — by whichever node the scheduler ran the zeroing thread on: - approximates the old `numa_alloc_interleaved` placement for the random-access buffer, *and* - pre-faults every page (no first-use fault in the hot path), *and* speeds up startup, *and* drops - libnuma here. (Was `numa_alloc_interleaved` + single-threaded `memset`.) -- **Producers/consumers**: receiver/decompression threads write frames into slots; consumers are - preview/TIFF/JPEG/HTTP retrieval (`GetImage`) and the ZMQ/file sender. Access is random and - unpinned (any thread → any slot) — which is *why* interleave/first-touch placement (not per-node - binding) is the right model. -- **Not used by `jfjoch_process` or `jfjoch_viewer`** — they read HDF5 through the reader, never - instantiate `ImageBuffer`. Broker/receiver-only (it only needs to *compile* for the viewer). - -### 1b. FPGA DMA buffers — placement is a *kernel* concern, not libnuma *(this section was missing)* -The real per-frame DMA buffers are **allocated and NUMA-placed by the kernel driver**, with zero -userspace/libnuma involvement. The earlier draft framed the userspace `mbind` as "the real -hardware-locality win" — that was wrong; the win is in the kernel: -- `fpga/pcie_driver/jfjoch_memory.c:28` — `dma_alloc_coherent(&pdev->dev, FPGA_BUFFER_LOCATION_SIZE, - …)` × `nbuffer` (512). Gives **physically contiguous, DMA-coherent** pages (required: each buffer's - bus address is written into the FPGA address table at `:37-38`). Placement is **device-local by - construction** — the kernel DMA/page allocator uses `dev_to_node(&pdev->dev)`. This is exactly why - it can't be a userspace `malloc`+`mbind`: userspace virtual memory is neither physically contiguous - nor a valid DMA target. -- `jfjoch_memory.c:99` — `dma_mmap_coherent` maps those *same physical pages* into userspace; - `JungfraujochDevice::MapKernelBuffer` (`fpga/host_library/JungfraujochDevice.cpp:166`) is a plain - `mmap` of the char dev. No second allocation, no first-touch, no migration. -- `IOCTL_JFJOCH_NUMA` (`fpga/pcie_driver/jfjoch_ioctl.c:133`) just **reports** - `drvdata->pdev->dev.numa_node`. Userspace's only NUMA action on the real path is to pin the - *acquire thread's CPU* to that node (the `RunOnNode` calls in §1c). -- **Simulated path (DONE, commit `042df308`):** `HLSSimulatedDevice` used to *emulate* device locality - with a userspace `mmap`+`mbind` (``) — the only libnuma user in `acquisition_device/`. It - now uses **plain zeroed heap buffers** (`std::vector`, not performance-critical). Buffer ownership - was refactored so each device owns its lifecycle: `PCIExpressDevice` `mmap`s/`munmap`s the kernel - DMA buffers; `HLSSimulatedDevice` owns the heap buffers; the base `AcquisitionDevice` is just a - non-owning address view. No libnuma, no mmap on the sim path. - -### 1c. `NUMAHWPolicy` — bundles three concerns per worker thread (the last libnuma user) -Built from the broker config `numa_policy` string (e.g. `n2g2`, `n8g4`, `n8g4_hbm`) into a table of -`NUMABinding{cpu_node, mem_node, gpu}`; `GetBinding(thread) = bindings[thread % nbindings]` -(round-robin, `common/NUMAHWPolicy.cpp:50`). `Bind(thread)` does **all three** at once: -1. **CPU pin** — `RunOnNode` / `numa_run_on_node` ← libnuma -2. **Memory bind** — `MemOnNode` / `numa_set_membind` (`n8g4_hbm` → HBM nodes) ← libnuma -3. **GPU select** — `SelectGPU` / `set_gpu` (= `cudaSetDevice`, **not** libnuma) - -Call sites: -- `receiver/JFJochReceiverFPGA.cpp:180/217/264` — acquire (data-stream) threads - `RunOnNode(acquisition_device.GetNUMANode())` → the kernel-reported device node (§1b). CPU pin only. -- `receiver/JFJochReceiverFPGA.cpp:299`, `receiver/JFJochReceiverLite.cpp:234` — analysis worker - threads `numa_policy.Bind(threadid)` → cpu + mem + GPU per the policy table. -- `image_analysis/indexing/IndexerThreadPool.cpp:34` — each indexer thread - `SelectGPUAndItsNUMA(threadid % gpu_count)` (GPU round-robin + that GPU's NUMA node via sysfs; - independent of the `numa_policy` table). - -The GPU NUMA node is now read from sysfs (`/sys/bus/pci/devices//numa_node`, -`common/CUDAWrapper.cu:75`), **not** libnuma. So the only remaining libnuma calls are `RunOnNode` and -`MemOnNode` in this file. - -### 1d. GPU dispatch — `jfjoch_process` now uses all visible GPUs (was the root cause) -- `get_gpu_count()` = `cudaGetDeviceCount()` (`common/CUDAWrapper.*`), so it already honours - **`CUDA_VISIBLE_DEVICES`**. All dispatch is `% gpu_count`. -- **Broker/receiver**: worker threads spread over GPUs via `numa_policy.Bind` (the `gpu` field) + - indexer pool via `threadid % gpu_count`. → uses all visible GPUs. -- **`jfjoch_process` (DONE, G2):** its worker (`tools/jfjoch_process.cpp:854`) now calls `pin_gpu()` - before building `MXAnalysisWithoutFPGA`, so each worker's CUDA streams/engines land on a distinct - device. Previously everything ran on GPU 0 (only the indexer pool spread). - -## 2. Goals - -- **G1 — multiple brokers, disjoint GPUs.** Run >1 `jfjoch_broker` on one machine, each confined to a - subset of GPUs, code transparently using "all it can see" (no hard-coded indices). Workload - control, no security requirement. -- **G2 — `jfjoch_process` should use all visible GPUs**, not just GPU 0. **DONE** (§1d). -- **G3 — drop the libnuma dependency.** Annoying dep; also a blocker for the long-term Windows/MSVC - viewer. Mostly done — one file (`NUMAHWPolicy.cpp`) left. -- **G4 — reassess whether NUMA CPU/mem pinning is still worth it.** Note the corrected model: FPGA DMA - buffer *placement* is the kernel's job (§1b), so the only behavioural NUMA op left on real data flow - is the **CPU pin** of the acquire/analysis threads to the kernel-reported device node. That's the - thing to A/B. - -**Key separability insight:** dependency removal (G3) and behaviour removal (G4) are *independent -axes*. Dropping libnuma does **not** require dropping any NUMA behaviour — `numa_run_on_node` → -`sched_setaffinity`, `mbind`/`set_mempolicy` are raw syscalls. So G3 can complete regardless of how -the G4 measurement turns out. - -## 3. Status of changes - -- **G1 (zero code) — TODO (docs).** Launch each broker under `CUDA_DEVICE_ORDER=PCI_BUS_ID - CUDA_VISIBLE_DEVICES= jfjoch_broker …`. `get_gpu_count()`/`% gpu_count` do the rest. Set - `CUDA_DEVICE_ORDER=PCI_BUS_ID` so indices are stable across boots. Action: write the deployment note. -- **G2 — DONE.** `pin_gpu()` in `CUDAWrapper` (process-wide round-robin `counter++ % get_gpu_count()`, - no-op when no GPU); `jfjoch_process` worker calls it once. Caller-agnostic, reusable. -- **`ImageBuffer` — DONE (`d373ba04`).** malloc + parallel first-touch (§1a). -- **`acquisition_device` — DONE (`042df308`).** Simulator off NUMA/mmap → plain heap; per-device - buffer ownership; libnuma gone from `acquisition_device/` (§1b). -- **`CUDAWrapper` `numa_node` — DONE.** sysfs lookup, not libnuma (§1c). -- **`NUMAHWPolicy` — REMAINING.** Split the bundled concerns: - - **Memory bind** (`MemOnNode`) — **drop.** HBM out of scope (the one Xeon MAX box didn't pay off; - treat every host as plain multi-socket). - - **CPU pin** (`RunOnNode`) — gated on **G4**. If kept, reimplement with `sched_setaffinity` + - node→cpulist from `/sys/devices/system/node/nodeN/cpulist` (no libnuma). Note it pins to the - *kernel-reported device node* (§1b) — well-founded if any pinning is kept. - - **GPU select** (`SelectGPU`) — **keep** (`cudaSetDevice`). `SelectGPUAndItsNUMA` then collapses to - `set_gpu` (+ optional CPU pin); `Bind` collapses to `SelectGPU` (+ optional CPU pin). - - Result: `NUMA_LIBRARY` leaves the CMake (`CMakeLists.txt:78-80`, - `common/CMakeLists.txt:155-161`). - -## 4. Open questions / to validate before deleting `NUMAHWPolicy` pinning - -- **G4 is empirical.** A/B at production frame rate (CPU pin on vs off): sustained throughput, dropped - frames, latency jitter. Reasoning predicts "no regression," but measure on one real box first — - production systems are currently tuned around this. -- Confirm `vm.zone_reclaim_mode` is `0` on the broker hosts (`cat /proc/sys/vm/zone_reclaim_mode`). - Non-zero turns `ImageBuffer`'s first-touch into synchronous local reclaim (page-cache eviction / - writeback) → startup latency stalls; `0` just spills the page to a remote node. -- `ImageBuffer` first-touch placement is *approximate* (depends on the scheduler spreading the zeroing - threads); fine with RAM headroom, but not the guaranteed 50/50 of `mbind` interleave. Deterministic - per-page interleave, if ever needed, is a raw `mbind(MPOL_INTERLEAVE)` syscall — still libnuma-free. - -## 5. Remaining order (low-risk first) - -1. **G1** — document the `CUDA_VISIBLE_DEVICES` launch (no code). -2. **G4 A/B** on a real broker — the gate for dropping the `RunOnNode` CPU pin. -3. **`NUMAHWPolicy`** — drop `MemOnNode`; drop/replace `RunOnNode` per G4; keep `SelectGPU`. Then - remove `NUMA_LIBRARY` from CMake → G3 complete. - -*(DONE so far: G2 `pin_gpu`, `ImageBuffer` first-touch, `acquisition_device` de-NUMA, `CUDAWrapper` -sysfs node lookup.)* diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 68b000ae..ffb3d092 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -234,10 +234,6 @@ void ParseAcquisitionDeviceGroup(const org::openapitools::server::model::Jfjoch_ } void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochReceiverService &service) { - std::string numa_policy = input.getNumaPolicy(); - if (!numa_policy.empty()) - service.NUMAPolicy(numa_policy); - // Using default in case service.NumThreads(input.getReceiverThreads()); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 67e80aa8..6e2b42db 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -2264,7 +2264,8 @@ components: description: Number of threads used by the receiver numa_policy: type: string - description: NUMA policy to bind CPUs + deprecated: true + description: Ignored value frontend_directory: type: string description: Location of built JavaScript web frontend diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index dde40cbe..e8997992 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -74,6 +74,8 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } + if (settings.numaPolicyIsSet()) + logger.Warning("NUMA policy is deprecated and ignored - processed are split over all the CPUs/GPUs in the system"); if (settings.verboseIsSet() && settings.isVerbose()) logger.Verbose(true); else diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f6273dc4..06a365c2 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -55,7 +55,6 @@ ADD_LIBRARY(JFJochCommon STATIC DetectorModuleGeometry.cpp DetectorModuleGeometry.h DetectorSetup.h DetectorSetup.cpp ZeroCopyReturnValue.h Histogram.h DiffractionGeometry.h CUDAWrapper.cpp CUDAWrapper.h - NUMAHWPolicy.cpp NUMAHWPolicy.h ADUHistogram.cpp ADUHistogram.h RawToConvertedGeometryCore.h Plot.h @@ -151,11 +150,3 @@ IF (JFJOCH_CUDA_AVAILABLE) TARGET_LINK_LIBRARIES(JFJochCommon CUDA::cudart_static ${CMAKE_DL_LIBS} $<$:rt>) ENDIF() - -IF(HAS_NUMAIF AND HAS_NUMA_H AND NUMA_LIBRARY) - TARGET_COMPILE_DEFINITIONS(JFJochCommon PUBLIC JFJOCH_USE_NUMA) - TARGET_LINK_LIBRARIES(JFJochCommon ${NUMA_LIBRARY}) - MESSAGE(STATUS "NUMA memory/CPU pinning enabled") -ELSE() - MESSAGE(WARNING "NUMA memory/CPU pinning disabled") -ENDIF() diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index d73e3760..ab3e2e6c 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -13,8 +13,4 @@ void set_gpu(int32_t dev_id) {} void pin_gpu() {} -int get_gpu_numa_node(int dev_id) { - return -1; -} - #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 684f8b82..554c54c6 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -45,57 +45,3 @@ void pin_gpu() { if (dev_count > 0) set_gpu(counter.fetch_add(1) % dev_count); } - -// Return CUDA device PCI Bus ID as "domain:bus:device.function", e.g., "0000:65:00.0" -static std::string get_cuda_device_pci_bus_id(int dev_id) { - // CUDA API provides cudaDeviceGetPCIBusId - char buf[64] = {0}; - cudaDeviceProp prop; - cudaError_t st = cudaGetDeviceProperties(&prop, dev_id); - if (st != cudaSuccess) { - throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(st)); - } - // Prefer cudaDeviceGetPCIBusId for full id including domain and function - cudaError_t st2 = cudaDeviceGetPCIBusId(buf, static_cast(sizeof(buf)), dev_id); - if (st2 == cudaSuccess) { - return std::string(buf); - } - // Fallback: synthesize from properties (domain may be missing on very old drivers) - // Note: function is typically ".0" - char alt[64]; - std::snprintf(alt, sizeof(alt), "%04x:%02x:%02x.%u", - prop.pciDomainID, prop.pciBusID, prop.pciDeviceID, 0u); - return std::string(alt); -} - -// Resolve NUMA node from PCI address using Linux sysfs -// Returns: -// >=0 NUMA node index -// -1 if NUMA node is not available/unknown -int get_gpu_numa_node(int dev_id) { - auto dev_count = get_gpu_count(); - if (dev_count <= 0) return -1; - if (dev_id < 0 || dev_id >= dev_count) { - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Invalid CUDA device ID"); - } - - // We don't need to call cudaSetDevice here; querying by id is sufficient. - const std::string pci_bus_id = get_cuda_device_pci_bus_id(dev_id); // "dddd:bb:dd.f" - - // sysfs path for PCI device. Examples: - // - /sys/bus/pci/devices/0000:65:00.0/numa_node - const std::string sysfs_path = std::string("/sys/bus/pci/devices/") + pci_bus_id + "/numa_node"; - - std::ifstream f(sysfs_path); - if (!f.is_open()) { - // On some systems, the symlink may be via /sys/class/drm or nvidia, but primary path should exist. - return -1; - } - - int numa = -1; - f >> numa; - if (!f.good()) { - return -1; - } - return numa; -} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index dcc97f3c..4b288ee0 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -7,7 +7,6 @@ int32_t get_gpu_count(); void set_gpu(int32_t dev_id); -int get_gpu_numa_node(int dev_id); // Pin the calling thread to the next GPU in round-robin order, using a process-wide counter // (counter++ % get_gpu_count()). Call once per thread; no thread id needed. No-op when no GPU diff --git a/common/NUMAHWPolicy.cpp b/common/NUMAHWPolicy.cpp deleted file mode 100644 index ac6621ae..00000000 --- a/common/NUMAHWPolicy.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "NUMAHWPolicy.h" - -#include "../common/CUDAWrapper.h" -#include "JFJochException.h" - -#ifdef JFJOCH_USE_NUMA -#include -#endif - -NUMAHWPolicy::NUMAHWPolicy(const std::string &policy) : name(policy) { - if ((policy.empty()) || (policy == "none")) { - name = "none"; - } else if (policy == "n2g2") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 1}); - } else if (policy == "n2g4") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 3}); - } else if (policy == "n2g2_hbm") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 1}); - } else if (policy == "n2g4_hbm") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 3}); - } else if (policy == "n8g4") { - for (int32_t i = 0; i < 8; i++) - bindings.emplace_back(NUMABinding{.cpu_node = i, .mem_node = i, .gpu = i/2}); - } else if (policy == "n8g4_hbm") { - for (int32_t i = 0; i < 8; i++) - bindings.emplace_back(NUMABinding{.cpu_node = i, .mem_node = i + 8, .gpu = i / 2}); - } else if (policy == "g2") { - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 1}); - } else if (policy == "g4") { - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 3}); - } else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown NUMA policy"); -} - -NUMABinding NUMAHWPolicy::GetBinding(uint32_t thread) const { - if (bindings.empty()) - return NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = -1}; - else - return bindings.at(thread % bindings.size()); -} - -void NUMAHWPolicy::Bind(uint32_t thread) const { - Bind(GetBinding(thread)); -} - -void NUMAHWPolicy::Bind(const NUMABinding &binding) { - RunOnNode(binding.cpu_node); - MemOnNode(binding.mem_node); - SelectGPU(binding.gpu); -} - -void NUMAHWPolicy::RunOnNode(int32_t cpu_node) { -#ifdef JFJOCH_USE_NUMA - if (numa_available() != -1) { - auto max_nodes = numa_num_configured_nodes(); - - if (cpu_node >= 0) { - if (cpu_node < max_nodes) - numa_run_on_node(cpu_node); - else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "CPU NUMA node out of bounds"); - } - } -#endif -} - -void NUMAHWPolicy::MemOnNode(int32_t mem_node) { -#ifdef JFJOCH_USE_NUMA - if (numa_available() != -1) { - auto max_nodes = numa_num_configured_nodes(); - - if (mem_node >= 0) { - if (mem_node < max_nodes) { - struct bitmask *mask = numa_allocate_nodemask(); - numa_bitmask_setbit(mask, mem_node); - numa_set_membind(mask); - numa_bitmask_free(mask); - } else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Memory NUMA node out of bounds"); - } - } -#endif -} - -void NUMAHWPolicy::SelectGPU(int32_t gpu) { - auto gpu_count = get_gpu_count(); - - if ((gpu_count > 0) && (gpu >= 0)) { - if (gpu < gpu_count) - set_gpu(gpu); - else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "GPU device out of bounds"); - } -} - -void NUMAHWPolicy::SelectGPUAndItsNUMA(int32_t gpu) { - int numa = get_gpu_numa_node(gpu); - if (numa >= 0) { - RunOnNode(numa); - MemOnNode(numa); - } - set_gpu(gpu); -} - - -const std::string &NUMAHWPolicy::GetName() const { - return name; -} - - diff --git a/common/NUMAHWPolicy.h b/common/NUMAHWPolicy.h deleted file mode 100644 index 6c1375dd..00000000 --- a/common/NUMAHWPolicy.h +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include -#include -#include -#include - -struct NUMABinding { - int32_t cpu_node; - int32_t mem_node; - int32_t gpu; -}; - -class NUMAHWPolicy { - std::string name; - std::vector bindings; -public: - NUMAHWPolicy() = default; - explicit NUMAHWPolicy(const std::string& policy); - NUMABinding GetBinding(uint32_t thread) const; - - const std::string &GetName() const; - - void Bind(uint32_t thread) const; - static void Bind(const NUMABinding &binding); - static void RunOnNode(int32_t cpu_node); - static void MemOnNode(int32_t mem_node); - static void SelectGPU(int32_t gpu); - static void SelectGPUAndItsNUMA(int32_t gpu); -}; diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index fb91982c..1a3e757b 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -110,6 +110,12 @@ Example JSON files are placed in `etc/` folder. JSON file format is also explain When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring. +Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one `jfjoch_broker` on a single machine, each confined to a disjoint subset of GPUs, set `CUDA_VISIBLE_DEVICES`; setting `CUDA_DEVICE_ORDER=PCI_BUS_ID` keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host: +``` +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232 +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233 +``` + To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. These need to be obtained from the PSI Detector Group. @@ -118,7 +124,7 @@ These need to be obtained from the PSI Detector Group. To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_fpga_test` tool. For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs: ``` -jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 -Pn2g2 +jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 ``` Or 1M pixel system with one FPGA card: ``` diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index d7e58f0b..76e3f2b7 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -129,7 +129,6 @@ Example with all fields: }, "image_buffer_MiB": 2048, "receiver_threads": 64, - "numa_policy": "n2g2", "frontend_directory": "/usr/share/jfjoch/frontend", "image_pusher": "ZeroMQ", "zeromq_metadata": { diff --git a/docs/SOFTWARE.md b/docs/SOFTWARE.md index d64671e8..8b144dbe 100644 --- a/docs/SOFTWARE.md +++ b/docs/SOFTWARE.md @@ -21,7 +21,6 @@ Required: Optional: * CUDA compiler version 12.7 or newer - required for MX fast feedback indexer * FFTW library - for indexing if GPU/CUDA is absent -* NUMA library - to pin threads to nodes/CPUs * Node.js - to make frontend * Qt version 6 (for jfjoch_viewer) diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 86900f12..51913d9a 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -28,15 +28,11 @@ IndexerThread::IndexerThread(const IndexingSettings &settings, int threadid) { void IndexerThread::Worker(const IndexingSettings &settings, int threadid) { try { -#ifdef JFJOCH_USE_CUDA - auto gpu_count = get_gpu_count(); - if (gpu_count > 0) - NUMAHWPolicy::SelectGPUAndItsNUMA(threadid % gpu_count); -#endif + pin_gpu(); } catch (const std::exception &e) { - spdlog::error("Failed to bind thread to NUMA node: {}", e.what()); + spdlog::error("Failed to pin to GPU {}", e.what()); } catch (...) { - // NUMA policy errors are not critical and should be ignored for the time being. + // GPU pinning errors are not critical and should be ignored for the time being. } std::unique_ptr fft_indexer, ffbidx_indexer, fftw_indexer; diff --git a/image_analysis/indexing/IndexerThreadPool.h b/image_analysis/indexing/IndexerThreadPool.h index 224e2943..fd7e0a04 100644 --- a/image_analysis/indexing/IndexerThreadPool.h +++ b/image_analysis/indexing/IndexerThreadPool.h @@ -17,7 +17,6 @@ #include "../common/JFJochMessages.h" #include "../common/DiffractionSpot.h" #include "../common/DiffractionExperiment.h" -#include "../common/NUMAHWPolicy.h" #include "Indexer.h" class IndexerThread { diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 64e82e33..752013ff 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -13,7 +13,6 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, JFJochReceiverPlots &in_plots, const SpotFindingSettings &spot_finding_settings, Logger &logger, - const NUMAHWPolicy &in_numa_policy, const PixelMask &in_pixel_mask, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, @@ -29,7 +28,6 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, plots(in_plots), scan_result(in_experiment), serialmx_filter(in_experiment), - numa_policy(in_numa_policy), pixel_mask(in_pixel_mask), indexer(experiment, indexing_thread_pool) { logger.Info("Initializing receiver"); diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 83918987..8c2a1362 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -19,7 +19,6 @@ #include "JFJochReceiverPlots.h" #include "LossyFilter.h" #include "../common/MovingAverage.h" -#include "../common/NUMAHWPolicy.h" #include "../common/ScanResultGenerator.h" #include "../image_analysis/indexing/IndexerThreadPool.h" #include "../image_analysis/IndexAndRefine.h" @@ -75,8 +74,6 @@ protected: std::optional images_written; - NUMAHWPolicy numa_policy; - std::vector> adu_histogram_module; PixelMask pixel_mask; @@ -104,7 +101,6 @@ public: JFJochReceiverPlots &plots, const SpotFindingSettings &spot_finding_settings, Logger &logger, - const NUMAHWPolicy &numa_policy, const PixelMask &pixel_mask, ZMQPreviewSocket *zmq_preview_socket = nullptr, ZMQMetadataSocket *zmq_metadata_socket = nullptr, diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 802b7faf..3b2cbbb8 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -6,7 +6,6 @@ #include #include "ImageMetadata.h" -#include "../common/CUDAWrapper.h" JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_pixel_mask, @@ -14,7 +13,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen AcquisitionDeviceGroup &in_aq_device, ImagePusher &in_image_sender, Logger &in_logger, int64_t in_forward_and_sum_nthreads, - const NUMAHWPolicy &in_numa_policy, const SpotFindingSettings &in_spot_finding_settings, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, @@ -29,7 +27,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen in_plots, in_spot_finding_settings, in_logger, - in_numa_policy, in_pixel_mask, in_zmq_preview_socket, in_zmq_metadata_socket, @@ -60,8 +57,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Number of threads must be more than zero"); - logger.Info("NUMA policy: {}", numa_policy.GetName()); - expected_packets_per_image = 0; for (int d = 0; d < ndatastreams; d++) { acquisition_device[d].PrepareAction(experiment); @@ -176,12 +171,6 @@ void JFJochReceiverFPGA::SendCalibration() { } void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Warning("NUMA bind error {} for device thread {} - continuing without binding", e.what(), data_stream); - } - try { LoadCalibrationToFPGA(data_stream); frame_transformation_ready.wait(); @@ -213,12 +202,6 @@ void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) { void JFJochReceiverFPGA::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell, uint32_t threadid, bool ignore) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Error("HW bind error {}", e.what()); - } - JFPedestalCalc pedestal_calc(experiment); uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber(); @@ -260,12 +243,6 @@ int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream, uint16_t module_number, uint32_t threadid, ModuleSummation &summation) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Error("HW bind error {}", e.what()); - } - ModuleSummation local_summation(experiment); int64_t starting_frame = image_number * experiment.GetSummation(); @@ -296,7 +273,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { std::unique_ptr analyzer; try { - numa_policy.Bind(threadid); analyzer = std::make_unique(experiment, indexer); } catch (const JFJochException &e) { frame_transformation_ready.count_down(); diff --git a/receiver/JFJochReceiverFPGA.h b/receiver/JFJochReceiverFPGA.h index c6146c91..a34af2c0 100644 --- a/receiver/JFJochReceiverFPGA.h +++ b/receiver/JFJochReceiverFPGA.h @@ -18,7 +18,6 @@ #include "../image_pusher/ImagePusher.h" #include "../common/Logger.h" #include "../common/ThreadSafeFIFO.h" -#include "../common/NUMAHWPolicy.h" #include "../common/ImageBuffer.h" #include "../common/PixelMask.h" @@ -97,7 +96,6 @@ public: AcquisitionDeviceGroup &acquisition_devices, ImagePusher &image_pusher, Logger &logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &numa_policy, const SpotFindingSettings &spot_finding_settings, JFJochReceiverCurrentStatus ¤t_status, JFJochReceiverPlots &plots, diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index e986956e..cc8fbcad 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -3,6 +3,7 @@ #include "JFJochReceiverLite.h" #include "../image_analysis/indexing/IndexerFactory.h" +#include "../common/CUDAWrapper.h" using namespace std::chrono_literals; @@ -34,7 +35,6 @@ JFJochReceiverLite::JFJochReceiverLite(const DiffractionExperiment &in_experimen ImagePusher &in_image_pusher, Logger &in_logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &in_numa_policy, const SpotFindingSettings &in_spot_finding_settings, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, @@ -49,7 +49,6 @@ JFJochReceiverLite::JFJochReceiverLite(const DiffractionExperiment &in_experimen in_plots, in_spot_finding_settings, in_logger, - in_numa_policy, in_pixel_mask, in_zmq_preview_socket, in_zmq_metadata_socket, @@ -231,14 +230,12 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { logger.Debug("Thread {} started", id); try { - numa_policy.Bind(id); - data_analysis_started.count_down(); + pin_gpu(); } catch (const JFJochException &e) { - data_analysis_started.count_down(); - Cancel(e); - return; + logger.Warning("Error pinning GPU {}", e.what()); } + data_analysis_started.count_down(); measurement_started.wait(); try { diff --git a/receiver/JFJochReceiverLite.h b/receiver/JFJochReceiverLite.h index b8c1b546..33a0d0aa 100644 --- a/receiver/JFJochReceiverLite.h +++ b/receiver/JFJochReceiverLite.h @@ -12,7 +12,6 @@ #include "../common/ImageBuffer.h" #include "../common/Logger.h" #include "../image_pusher/ImagePusher.h" -#include "../common/NUMAHWPolicy.h" #include "../preview/PreviewImage.h" #include "../preview/ZMQPreviewSocket.h" #include "../preview/ZMQMetadataSocket.h" @@ -53,7 +52,6 @@ public: ImagePuller &image_puller, ImagePusher &image_pusher, Logger &logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &numa_policy, const SpotFindingSettings &spot_finding_settings, JFJochReceiverCurrentStatus ¤t_status, JFJochReceiverPlots &plots, diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index a6e92f49..63b7b5b4 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -24,16 +24,6 @@ JFJochReceiverService &JFJochReceiverService::NumThreads(int64_t input) { return *this; } -JFJochReceiverService &JFJochReceiverService::NUMAPolicy(const NUMAHWPolicy &policy) { - numa_policy = policy; - return *this; -} - -JFJochReceiverService &JFJochReceiverService::NUMAPolicy(const std::string &policy) { - numa_policy = NUMAHWPolicy(policy); - return *this; -} - void JFJochReceiverService::FinalizeMeasurementChangeState() { std::unique_lock ul(state_mutex); state = ReceiverState::Idle; @@ -80,7 +70,6 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, aq_devices, image_pusher, logger, nthreads_local, - numa_policy, spot_finding_settings, receiver_status, plots, @@ -102,7 +91,6 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, image_pusher, logger, nthreads_local, - numa_policy, spot_finding_settings, receiver_status, plots, diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index 1edec952..5c4748fd 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -9,12 +9,10 @@ #include "JFJochReceiver.h" #include "../acquisition_device/AcquisitionDeviceGroup.h" -#include "../common/NUMAHWPolicy.h" #include "../preview/ZMQMetadataSocket.h" #include "../preview/ZMQPreviewSocket.h" class JFJochReceiverService { - NUMAHWPolicy numa_policy; std::unique_ptr receiver; AcquisitionDeviceGroup &aq_devices; Logger &logger; @@ -45,8 +43,6 @@ public: ImagePusher &pusher, size_t send_buffer_size_MiB = 512); JFJochReceiverService& NumThreads(int64_t input); - JFJochReceiverService& NUMAPolicy(const NUMAHWPolicy& policy); - JFJochReceiverService& NUMAPolicy(const std::string& policy); JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); JFJochReceiverService& MetadataSocket(const std::string &addr); diff --git a/receiver/JFJochReceiverTest.cpp b/receiver/JFJochReceiverTest.cpp index 1e00e7ab..3bd87105 100644 --- a/receiver/JFJochReceiverTest.cpp +++ b/receiver/JFJochReceiverTest.cpp @@ -38,7 +38,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const DiffractionExperiment &x, const PixelMask &pixel_mask, uint16_t nthreads, - const std::string &numa_policy, size_t send_buf_size_MiB, bool quick_integrate) { @@ -47,8 +46,8 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, for (int i = 0; i < raw_expected_image.size(); i++) raw_expected_image[i] = i % 65536; - return JFJochReceiverTest(output, logger, aq_devices, x, pixel_mask, raw_expected_image, nthreads, numa_policy, send_buf_size_MiB, - quick_integrate); + return JFJochReceiverTest(output, logger, aq_devices, x, pixel_mask, raw_expected_image, nthreads, + send_buf_size_MiB, quick_integrate); } bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, @@ -57,7 +56,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const PixelMask &pixel_mask, const std::vector &raw_expected_image, uint16_t nthreads, - const std::string &numa_policy, size_t send_buf_size_MiB, bool quick_integrate) { std::vector raw_expected_image_with_mask = raw_expected_image; @@ -79,7 +77,7 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, TestImagePusher pusher(image_number); JFJochReceiverService service(aq_devices, logger, pusher, send_buf_size_MiB); - service.NumThreads(nthreads).NUMAPolicy(numa_policy); + service.NumThreads(nthreads); service.LoadInternalGeneratorImage(x, raw_expected_image, 0); service.Indexing(x.GetIndexingSettings()); diff --git a/receiver/JFJochReceiverTest.h b/receiver/JFJochReceiverTest.h index 89281494..2cc03ee0 100644 --- a/receiver/JFJochReceiverTest.h +++ b/receiver/JFJochReceiverTest.h @@ -10,7 +10,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const DiffractionExperiment &x, const PixelMask &pixel_mask, uint16_t nthreads, - const std::string &numa_policy = "", size_t send_buf_size_MiB = 1024, bool quick_integrate = false); @@ -20,6 +19,5 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const PixelMask &pixel_mask, const std::vector &data, uint16_t nthreads, - const std::string &numa_policy = "", size_t send_buf_size_MiB = 1024, bool quick_integrate = false); diff --git a/tools/jfjoch_fpga_test.cpp b/tools/jfjoch_fpga_test.cpp index 9cd98e3a..b4359252 100644 --- a/tools/jfjoch_fpga_test.cpp +++ b/tools/jfjoch_fpga_test.cpp @@ -23,7 +23,6 @@ void print_usage(Logger &logger) { logger.Info(" -m Number of modules"); logger.Info(" -i Number of images"); logger.Info(" -N Number of image processing threads"); - logger.Info(" -P NUMA Policy (none|n2g2|n8g4|n8g4_hbm), none is default"); logger.Info(" -B Size of send buffer in MiB (default 2048)"); logger.Info(" -q Use Poisson lossy compression, with square root of counts"); logger.Info(" -T Use thresholding for low counts"); @@ -53,7 +52,6 @@ int main(int argc, char **argv) { std::optional fft_num_vectors; bool verbose = false; - std::string numa_policy_name; bool raw_data = false; bool force_32bit = false; bool force_8bit = false; @@ -100,7 +98,7 @@ int main(int argc, char **argv) { verbose = true; break; case 'P': - numa_policy_name = std::string(optarg); + logger.Warning("NUMA policy is deprecated and ignored"); break; case 'R': raw_data = true; @@ -247,7 +245,7 @@ int main(int argc, char **argv) { bool ret; std::thread run_thread([&] { try { - ret = JFJochReceiverTest(output, logger, aq_devices, x, mask, input, nthreads, numa_policy_name, + ret = JFJochReceiverTest(output, logger, aq_devices, x, mask, input, nthreads, send_buffer_size_MiB, quick_integrate); } catch (std::exception &e) { logger.Error(e.what()); diff --git a/tools/jfjoch_lite_perf_test.cpp b/tools/jfjoch_lite_perf_test.cpp index 5840d25f..13ba0103 100644 --- a/tools/jfjoch_lite_perf_test.cpp +++ b/tools/jfjoch_lite_perf_test.cpp @@ -18,7 +18,6 @@ void print_usage(Logger &logger) { logger.Info("Options:"); logger.Info(" -i Number of images"); logger.Info(" -N Number of image processing threads (default: 8)"); - logger.Info(" -P NUMA policy: none|n2g2|n8g4|n8g4_hbm (default: none)"); logger.Info(" -F{} Write file, optional parameter is name (default: lyso_lite_perf_test)"); logger.Info(" -X Indexing (none|fft|fftw|ffbidx), ffbidx is default"); logger.Info(" -t Indexing thread pool size (default: 4)"); @@ -58,7 +57,7 @@ int main(int argc, char **argv) { nthreads = atol(optarg); break; case 'P': - numa_policy_name = std::string(optarg); + logger.Warning("NUMA policy is deprecated and ignored"); break; case 'i': nimages = atol(optarg); @@ -166,7 +165,6 @@ int main(int argc, char **argv) { AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); - service.NUMAPolicy(numa_policy_name); service.NumThreads(nthreads); IndexingSettings i_settings;