mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2026-10-31 11:01:27 +01:00
Add nsys probe scripts for ClusterFinderCUDA
Per-engine GPU time, duty cycle and roofline on the campaign configs (3x3 cap 3000, 9x9 cap 1700, 1 and 4 streams, 20k frames).
This commit is contained in:
kferjaoui committed
1 parent
5d20e1d65e
commit
da3f563872
10 files changed
+535
-11
No files matched your search
@@ -141,7 +141,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Pedestal (1000 frames): 1.23s\n"
|
||||
"Pedestal (1000 frames): 1.34s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -175,7 +175,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Reading 100000 frames: 33.56s (2,980 FPS, 32.00 GB at 0.95 GB/s)\n"
|
||||
"Reading 100000 frames: 82.39s (1,214 FPS, 32.00 GB at 0.39 GB/s)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -207,14 +207,14 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████| 100000/100000 [00:23<00:00, 4249.96it/s]\n"
|
||||
"100%|██████████████████████████████████████████████████| 100000/100000 [00:23<00:00, 4191.14it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU MT (24 threads): 23.53s 4,249 FPS 233,099,227 clusters (2331.0/frame)\n"
|
||||
"CPU MT (24 threads): 23.86s 4,190 FPS 233,099,227 clusters (2331.0/frame)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -309,10 +309,10 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" s FPS us/frame clusters\n",
|
||||
"CPU MT (24 thr) 23.53 4,249 235.3 233,099,227\n",
|
||||
"CUDA batched 3.53 28,325 35.3 233,093,554\n",
|
||||
"CPU MT (24 thr) 23.86 4,190 238.6 233,099,227\n",
|
||||
"CUDA batched 3.34 29,926 33.4 233,093,554\n",
|
||||
"\n",
|
||||
"speedup: 6.67x\n"
|
||||
"speedup: 7.14x\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -413,10 +413,26 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 10,
|
||||
"id": "sets",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"CPU 233,099,227 clusters\n",
|
||||
"CUDA 233,093,554 clusters net -5,673 (0.0024%)\n",
|
||||
"\n",
|
||||
"exact-position mismatches CPU-only 135,742 CUDA-only 130,069\n",
|
||||
" of which shifted by 1 px 104,347 104,198\n",
|
||||
" genuine add / drop 31,395 25,871\n",
|
||||
"\n",
|
||||
"frames touched by a genuine difference: 41,569 of 100,000\n",
|
||||
"[40.6s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def cluster_keys(vectors):\n",
|
||||
" \"\"\"One int64 key per cluster: frame * rows*cols + y * cols + x.\n",
|
||||
@@ -501,10 +517,29 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 11,
|
||||
"id": "drift",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"image/png":
|
||||
"iVBORw0KGgoAAAANSUhEUgAAA3kAAAFeCAYAAAA4xtZyAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbExJREFUeJzt3Qd4U1X/B/BvS9l7770REQEZArK38LpQkKk4APmzRUBlyVIBeUEBURQVFAfoCwoKspQpewiKInvvvZv/8z3X2yZpWtI2bZP0+3metMnNyc3Jzcm993fPCnE4HA6IiIiIiIhIUAhN6gyIiIiIiIiI7yjIExERERERCSIK8kRERERERIKIgjwREREREZEgoiBPREREREQkiCjIExERERERCSIK8kRERERERIKIgjwREREREZEgoiBPREREREQkiCjIk6Cyfft2PPPMMyhatCjSpEmDDBkyoFKlSnjrrbdw9uzZiHR169Y1t4QyZcoUzJw5M8HWH4h8tU1CQkIwbNiwiMdcJ5ft37/fJd1rr72GQoUKISwsDFmyZDHLbt68ia5duyJv3rxIkSIFKlasGO/8SOBzL1Peunr1qnndihUrojwXXbkUV59++inatGmD0qVLIzQ0FEWKFIl2E12+fBm9e/dGvnz5zP6dv985c+Z4TLt582Y0bNjQHAP4+3/sscfwzz//eL35Y/P6yZMno0yZMkidOrU59gwfPhy3bt2Kku7kyZPo3LkzcuTIgXTp0qFGjRpYunSp13nytE9LTn799VezjQ8cOBCxLLp9OssRt3VCiu9vnK/j6z3doivXnuzcuROtW7dGzpw5zfbhZ+/evbtLGi6L7r34W3L23HPPoXz58qaMpU2bFqVKlcLLL7+M06dPu6SbMWMG8ufPjytXrsTp80vCC0uE9xBJFB988IHZsfFkgTukcuXKmQPtxo0bMW3aNKxduxbffvttogU0PJAn9EEmkCTUNmnRooX5bnmQt/3vf//DqFGj8Oqrr6JZs2bmwEdTp07F+++/b07KKleubE7gROKKQR5P6Mn9opGncilRffbZZzh+/DiqVq2K8PBwj8GRjYHWhg0bMHbsWHPi+fnnn6Nt27bmdU8//XREuj/++MN8Hzzh/+qrr3D9+nUMGTIEtWvXxtatW83JcExi83ruZ15//XUMHDgQjRs3NvljMHbkyBFMnz49It2NGzfQoEEDnD9/Hv/973+RK1cuvPfee2jatCl+/vln1KlTJ8Y8RbdPSy4cDocJ8J9//nkULlw4Ynl0+3Qe6zNlyoRA8H//938u5ZdKlizp1WuXL19u9jUsmzzP4TH24MGD2LJli0s6bg+WQWdM99RTT+HRRx91Wc6g7YUXXkCJEiVMAMhzKJa9hQsXmvWmSpXKpOvUqRPefPNNcxHd3g+Kn3GIBIE1a9Y4UqRI4WjatKnj+vXrUZ6/ceOG43//+1/E4zp16phbQrnnnnt8vv7w8HDH1atXHYHKV9uEu62hQ4fGmGbkyJEm3YkTJ1yWP/fcc460adM6fOnKlSs+XZ8kPm/KlCenTp2K82uTi5s3bzpu3boV7fN37tyJuN+iRQtH4cKFPab74YcfzLb+/PPPXZY3atTIkS9fPsft27cjlrVu3dqRI0cOx4ULFyKW7d+/35EyZUrHgAED7ppnb19/+vRpR5o0aRwvvPCCy+tHjRrlCAkJcfz+++8Ry9577z2Tfx6rbNwu5cqVc1StWvWueYpun+ZJIB8norNw4ULz+f/4448E36d76+OPPzZ52rdvX5xez9fx9W+//Xacjz158+Y1vxueH8TWsGHDzPv//PPPd007ZcoUk3bp0qUuy8eNG+fInDmzjoN+SkGeBIWHH37YERYW5jh48KBX6d2DvOXLl5sdGP972glzZ27bu3ev46mnnjI711SpUjly5crlqF+/vmPLli3meZ6k8DXON+cTF5449OvXz1GkSBFz0sATlF69ejkuX77s8t583UsvveSYOnWqo0yZMiYt70eHwe3w4cNN2tSpUzuyZcvmqFu3rmP16tURaa5du+YYOHCgy3t3797dce7cuSjv7enElZ+jU6dOUQ5yy5Ytc3Tt2tWRPXt2876PPvqo48iRIy6vi2mbeMLtxAM415c+fXpHkyZNHH/++WeUvLkfaD29F9O7L3P+XnmA5EnYfffdZ07asmTJ4nj88cfNd+2MZYbB6sqVKx01atQwJxcsC3H5Xj/99FPzXXEdFSpUcCxYsCDKNti9e7ejTZs2poyxrBUsWNDRoUMHlwsZx44dMyeZ+fPnN+/L9+fBO6YTa9ucOXPMSXKePHnM52Z+XnnllSh5pnXr1pnfGb8Plq9ixYqZzxeb/NrfgzcnS/weefLC7VKxYsWI/Nnbia/h43Tp0jkeeOABx4YNG6J8V54uKrD8upc99zJ18uRJR7du3Rxly5Y1ZS9nzpyOevXqOX755Zco+wb3m/378PSZ7PLz22+/OWrVqmW++6JFizrGjBnjEuzQzp07zXfDNAw4+Dv9/vvvPe6nPPn111/NfilDhgxmHSyvfL1t69atZl0ffvhhtCfUzhfG9uzZ42jbtq3ZFvxuue3fffddl9fZ+1GW7b59+5rfAIMdlgtvxBTkcV/Az+Jerhn08T3t/Ryf5+d98cUXo6yjcePGjpIlS8aYh9i8ftasWea9165d65Lu6NGjZjmDPVvDhg0dpUuXjrLO0aNHm7SHDx+ONk/R7dPs57jd5s6da34n/G3yN0z8fmrXrm2+M/5Oypcv73jzzTdN4O3MLpcMQFlO+Fvjej/66CPzPMvN/fffb7YL17Fo0aIoefSmfLCMv/HGG45SpUqZ92BwcO+99zomTpzouJuWLVua37mzmPbp7scqu2yyvAwePNgcvzNmzOho0KBBlMBx8eLFjlatWpl9Krdn8eLFzT6WF3X8KcibOXOmef2KFSti/Voe87iNuB/3JkD8+uuvzXvx2OeMxx/+xmfMmBHrPEjCU3NNCXh37tzBsmXLTFONggULJvj7NW/e3LwnmyiwfwTbqa9Zs8Y0w7GbRTzxxBPInDmzaaJIdtMaNu9is5zDhw9j8ODBqFChAn7//XfTFGjHjh2m2Q7byNu+++470w+Bz+fJk8c08fHk9u3bpgkP07JJS/369c2ydevWmSYZDz74oGnu8sgjj5g+IIMGDTLNO9iHcejQoaZZGW9xbQLENvxsMsLmU4cOHTLNZdu3b2++l7ttE0/svHK78rM/8MADWL16tfmMd8P3YjMo9hf48ccfzXsWKFDANIt64403TPMWO1/Fixc3/1988UXTv6Jnz56m+Qn7b44YMcJst23btiF37twR6z927Jj5bAMGDMDo0aNNP6LYfq8//PCDadbF92DzIpYlNpn5888/UaxYMZOG71urVi3T/Ibp2HyH7z1//nzTD4Xbz27mxjzwvfh5+D2OHDnS9Pf4+OOPY9xWf/31lynPLDPp06c3zdT4+X/77beIbUQ//fQTWrZsibJly2LChAmm3HP9ixcvjkjjTX5ji+tkWWUTNX6PbBLEJntcxnLM7c/t+sorr+Dhhx/Gvn37TB+S+LL77/K3wd8d+4KxXLEJH9+X/9kMk+WL5apLly7mN0B3awrI76xdu3bo16+fWT/Xy8/DfmYdO3Y0abjdWJ74nbA5Gn/3X3zxBXr06OFV/leuXIlGjRqZcsjfAbc9f3f8DrkeNtG67777cP/995sywvw742+B78myQbt27TK/BX7v48ePN9uEZYK/F+7/+Dmc8fOwvxmbj7FsRrffig32O2L5Y380Z/yM9vPM4969e3Ht2rWI5e5plyxZYppfuvdDssXm9XxPuvfee13SsWzwd2A/b+eP+1xP6yTuL9i/KTb7NOf+g7t37zbNRNknkOXG/ixsBshlbGLH3xOb3fF3/tFHH0Upl+zPzv0a183mj88++6zZn3/zzTdmv8b35W+b+2b2T2SZjU354H6OfViZz4ceesg0zWVe7GNndLj/4D6UzRqdcV8X3T49OvwcNWvWxIcffoiLFy+afQd/F9x+7NNnbzeWX/6m+Zm5r+N+j/s37s9TpkwJX2LzY+aLZZtjCPA7aNWq1V1f98svv5j/PB9h3rjf5nfPfRK/B/v78YTbk30beaxwPjY54zkEm3iyiTKbJPM9uO2c8btmf1Qe01hexM8kQiApkqCOHz9urjCxBsFbca3JY/McPr7blcfomibyin1oaGiUWodvvvnGrJdX0G18zCudZ8+evevn4ZVzpv/ggw+iTfPjjz+aNG+99ZbL8i+//NIsnz59epxr8ljL4IzvweW8yheX5pq8UszX//e//3VZzivjd6vJc64xcr/yyryzZsYZr8Iz7fjx412WHzp0yFy5dm6exfx7arIS2+81d+7cjosXL7qUYb6e67GxFoY1iqxVig5rG1i7ceDAgShNaPg+zs3F7oZXc1mLwSu1fO22bdsinuOVbN5YExwdb/Ib25o8bn/nGg679olX4Z2byX733Xdm+fz5831Sk+eOTQG5bXjVn7XU3jTXjK4mj8vWr1/vkpZN9lhTbXv55ZejNPcjpvGmJq969eqmNvXSpUsun4G1MAUKFIi4cj9p0iSzPtaQ27i/Ye0Fa6Wd35evc26+SD169DA1MvY+yt6PPvTQQ464iKkmjzVoztvIvdaMNWLEGj0+/uKLL6KtNeNrohOb1z///PNmW3nC2irW/NlYy+6pdpC1Z56aobqLbp/G7cWuCs7foSesRWMZ5rGC6Z2PK3a53LhxY8SyM2fOmHT8DTq3yrB/gyw7sS0fbAnA2sbY4u+F78mWB+487dNjqslr3ry5S7qvvvrKY22s+36R+1j32u341uSxHLEMMQ+seZ89e7b57d7tWO6+P+B+l8cptqiZNm2aaVFTokSJGJtQsgUKv9/oapDt46J943ZzPmY5a9eunTmmif/R6JoisZAtWzZzpfDtt982V/bYCZLine truncated
|
||||
"text/plain": [
|
||||
"<Figure size 900x360 with 1 Axes>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"first half: -2,032 second half: -3,641\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"cnt_cpu = np.bincount(fr_cpu, minlength=N)\n",
|
||||
"cnt_cuda = np.bincount(fr_cuda, minlength=N)\n",
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# ClusterFinderCUDA nsys probes
|
||||
|
||||
Per-engine GPU time per frame (kernel, H2D, D2H) for the CUDA cluster finder,
|
||||
traced with Nsight Systems on real MOENCH data. The tallest engine is the
|
||||
roofline: the engines overlap and PCIe is full-duplex, so throughput is bound by
|
||||
the max, not the sum.
|
||||
|
||||
These are *not* throughput numbers. nsys inflates wall clock several-fold by
|
||||
tracing every CUDA call. Use them to answer "what does the kernel cost" and
|
||||
"which engine binds", and measure sustained FPS separately without a profiler.
|
||||
|
||||
## Files
|
||||
|
||||
| file | role |
|
||||
|---|---|
|
||||
| `run_probes.py` | driver: runs each config under nsys, exports SQLite, writes `probes.csv` |
|
||||
| `nsys_kernel_probe.py` | the traced process: pedestal, one batched pass, exit |
|
||||
| `gpu_span.py` | reads the SQLite export, computes busy time, duty cycle, roofline |
|
||||
| `common.py` | paths, guards (fresh build, idle GPU), `env.json` stamp |
|
||||
|
||||
## Requirements
|
||||
|
||||
- A built tree with the CUDA extension: `build/aare/_aare_cuda*.so`. Run the
|
||||
scripts with a Python of the same minor version as the build, with numpy and
|
||||
matplotlib installed (`aare/__init__.py` imports both). On pc-moench-04 the
|
||||
build is 3.11 and the `py` conda environment works:
|
||||
`/home/ferjao_k/.conda/envs/py/bin/python`.
|
||||
- Nsight Systems. Default path `/opt/nvidia/nsight-systems/2024.5.1/bin/nsys`,
|
||||
override with `NSYS=...`.
|
||||
- The MOENCH dataset under `/mnt/sls_det_storage/.../2026032408/process/xrf/`
|
||||
(override with `AARE_PERF_DATA=<dir>`). 1000 pedestal frames, 20 000 data
|
||||
frames, 5 sigma.
|
||||
- An idle GPU. The driver aborts above 5 % utilisation: a competing process
|
||||
leaves per-op averages intact but ruins duty cycles and rooflines.
|
||||
|
||||
Paths are resolved relative to this folder. `AARE_REPO` and `AARE_BUILD`
|
||||
override the repo root and build directory.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
cd python/tests/perf
|
||||
python run_probes.py # full sweep: 4 configs, 20k frames each (~5 min)
|
||||
python run_probes.py --frames 2000 # smoke test only; clocks do not ramp, kernel reads ~7 % slow
|
||||
python run_probes.py --only 3x3_s1_uncontended 9x9_s1_uncontended --tag mychange
|
||||
```
|
||||
|
||||
Configs, matching the campaign in `docs/ClusterFinderCUDA_benchmark_results.md`
|
||||
on branch `bench/cuda_cf`:
|
||||
|
||||
| label | cluster | cap | streams | batch | frames |
|
||||
|---|---|---|---|---|---|
|
||||
| `3x3_s4` | 3x3 | 3000 | 4 | 2000 | 20 000 |
|
||||
| `3x3_s1_uncontended` | 3x3 | 3000 | 1 | 2000 | 20 000 |
|
||||
| `9x9_s4` | 9x9 | 1700 | 4 | 2000 | 20 000 |
|
||||
| `9x9_s1_uncontended` | 9x9 | 1700 | 1 | 2000 | 20 000 |
|
||||
|
||||
Keep the cap: at 9x9 the D2H slot is `4 + cap * sizeof(cluster)` and is copied
|
||||
whole, so the cap sets the D2H bar.
|
||||
|
||||
Output goes to `results/<date>_<f32|f64>[_tag]/`: `env.json` (git rev, dirty
|
||||
flag, driver, pedestal type), one `.nsys-rep` + `.sqlite` per config, and
|
||||
`probes.csv`. Re-running one config replaces its own row and keeps the others.
|
||||
|
||||
To inspect one trace by hand:
|
||||
|
||||
```bash
|
||||
nsys stats --report cuda_gpu_sum results/<dir>/probe_9x9_s1_uncontended_cap1700.nsys-rep
|
||||
python gpu_span.py results/<dir>/probe_9x9_s1_uncontended_cap1700.sqlite 20000
|
||||
```
|
||||
|
||||
`probes.csv` and `env.json` are small and can be committed with a result;
|
||||
the `.nsys-rep` and `.sqlite` traces are tens of MB each and are gitignored.
|
||||
|
||||
## Reading the numbers
|
||||
|
||||
- **Quote kernel changes from the `s1_uncontended` rows.** With one stream
|
||||
nothing overlaps, so the kernel column is the kernel alone. The 1-stream
|
||||
kernel and transfer numbers reproduce across sessions to about 1 %.
|
||||
- **4-stream transfer columns drift by up to 20 % between sessions** on an
|
||||
identical build (H2D and D2H fight for the PCIe link). Only compare s4
|
||||
rows taken in the same session, and read them for the bottleneck, not for
|
||||
a percentage.
|
||||
- A kernel below the tallest transfer bar is invisible end to end. Further
|
||||
kernel work only pays once it is the tallest bar.
|
||||
|
||||
## Reference, 2026-09-10, RTX 4090, driver 595.71.05, float pedestal
|
||||
|
||||
Kernel us/frame. "before" is commit 7177f00 (pre-refactor kernel, rebuilt and
|
||||
re-probed the same day, matching its August numbers): `results/2026-09-10_f32_ref7177/`.
|
||||
"after" is the kernel with the cooperative tile load, size-dependent unroll
|
||||
and two-phase load: `results/2026-09-10_f32/`.
|
||||
|
||||
| config | before | after | roofline after | binds |
|
||||
|---|--:|--:|--:|---|
|
||||
| 3x3, 1 stream | 4.32 | 4.65 | 13.13 | H2D |
|
||||
| 3x3, 4 streams | 5.00 | 5.64 | 16.35 | H2D |
|
||||
| 9x9, 1 stream | 23.80 | 15.29 | 22.00 | D2H |
|
||||
| 9x9, 4 streams | 24.72 | 15.59 | 25.39 | D2H |
|
||||
|
||||
Both sizes are transfer-bound; sustained throughput is unchanged (3x3 about
|
||||
61 400 FPS, 9x9 about 39 600 FPS on the zero-copy path).
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Shared plumbing for the nsys probes: paths, guards, environment stamp.
|
||||
|
||||
Two guards are enforced here because both have produced wrong numbers before:
|
||||
a stale extension (header edited after the build) and a busy GPU (another
|
||||
process leaves per-op averages intact but destroys duty cycles).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REPO = Path(os.environ.get("AARE_REPO", HERE.parents[2]))
|
||||
BUILD = Path(os.environ.get("AARE_BUILD", REPO / "build"))
|
||||
sys.path.insert(0, str(BUILD))
|
||||
|
||||
DATA_DIR = Path(os.environ.get(
|
||||
"AARE_PERF_DATA",
|
||||
"/mnt/sls_det_storage/moench_data/2603_MaxIVBeamtime/2026032408/process/xrf/"))
|
||||
DATA_FILE = DATA_DIR / "Cu_factor_10_data_master_0.json"
|
||||
PEDESTAL_FILE = DATA_DIR / "Cu_factor_10_pedestal_master_0.json"
|
||||
|
||||
N_PEDESTAL_FRAMES = 1000
|
||||
N_SIGMA = 5
|
||||
|
||||
NSYS = os.environ.get("NSYS", "/opt/nvidia/nsight-systems/2024.5.1/bin/nsys")
|
||||
|
||||
|
||||
def _sh(cmd: str, default: str = "unknown") -> str:
|
||||
try:
|
||||
return subprocess.run(cmd, shell=True, capture_output=True, text=True,
|
||||
timeout=30).stdout.strip() or default
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def device_ped_type() -> str:
|
||||
"""DEVICE_PED_TYPE from the kernel header (the float/double build axis)."""
|
||||
hdr = REPO / "include/aare/clusterfinder_kernel.cuh"
|
||||
for line in hdr.read_text().splitlines()[:40]:
|
||||
if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"):
|
||||
return line.split("=")[1].split(";")[0].strip()
|
||||
return "unknown"
|
||||
|
||||
|
||||
def assert_build_fresh() -> None:
|
||||
"""Abort if a CUDA header is newer than the installed extension."""
|
||||
so = list((BUILD / "aare").glob("_aare_cuda*.so"))
|
||||
if not so:
|
||||
raise RuntimeError(f"no _aare_cuda*.so under {BUILD / 'aare'}")
|
||||
built = max(f.stat().st_mtime for f in so)
|
||||
headers = ["clusterfinder_kernel.cuh", "clusterfinder_algo.cuh",
|
||||
"ClusterFinderCUDA.hpp"]
|
||||
stale = [h for h in headers
|
||||
if (REPO / "include/aare" / h).stat().st_mtime > built]
|
||||
if stale:
|
||||
raise RuntimeError(f"{', '.join(stale)} newer than the extension: rebuild first")
|
||||
|
||||
|
||||
def assert_idle_gpu(max_pct: int = 5) -> None:
|
||||
pct = _sh("nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader")
|
||||
try:
|
||||
val = int(pct.split()[0])
|
||||
except Exception:
|
||||
return
|
||||
if val > max_pct:
|
||||
raise SystemExit(f"GPU is {val}% busy: numbers would not be quotable. Aborting.")
|
||||
|
||||
|
||||
def capture_env() -> dict:
|
||||
import aare
|
||||
|
||||
return {
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"host": _sh("hostname"),
|
||||
"git_rev": _sh(f"git -C {REPO} rev-parse --short HEAD"),
|
||||
"git_branch": _sh(f"git -C {REPO} rev-parse --abbrev-ref HEAD"),
|
||||
"git_dirty": bool(_sh(f"git -C {REPO} status --porcelain", default="")),
|
||||
"aare_version": getattr(aare, "__version__", "unknown"),
|
||||
"device_ped_type": device_ped_type(),
|
||||
"gpu": _sh("nvidia-smi --query-gpu=name --format=csv,noheader"),
|
||||
"driver": _sh("nvidia-smi --query-gpu=driver_version --format=csv,noheader"),
|
||||
"nvcc": _sh("nvcc --version | tail -1"),
|
||||
"python": sys.version.split()[0],
|
||||
}
|
||||
|
||||
|
||||
def results_dir(tag: str = "") -> Path:
|
||||
"""perf/results/<date>_<f32|f64>[_tag]/"""
|
||||
short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk")
|
||||
name = f"{time.strftime('%Y-%m-%d')}_{short}" + (f"_{tag}" if tag else "")
|
||||
d = HERE / "results" / name
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def write_env(path: Path, env: dict) -> None:
|
||||
path.write_text(json.dumps(env, indent=2) + "\n")
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Per-engine busy time and duty cycle from an nsys SQLite export.
|
||||
|
||||
nsys stats --force-export=true --report cuda_gpu_sum <rep>.nsys-rep
|
||||
python gpu_span.py <rep>.sqlite <n_frames>
|
||||
|
||||
Per engine (kernel / H2D / D2H): sum of op durations, busy = union of the
|
||||
intervals (overlaps not double-counted), duty = busy / processing window.
|
||||
The window is first kernel start to last D2H end, so the pedestal-upload
|
||||
copies that precede any kernel are excluded.
|
||||
|
||||
Roofline = tallest engine's busy time per frame. PCIe is full-duplex and the
|
||||
engines overlap, so the floor is the max, never the sum.
|
||||
"""
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
Q = {"kernel": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_KERNEL ORDER BY start",
|
||||
"H2D": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=1 ORDER BY start",
|
||||
"D2H": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=2 ORDER BY start"}
|
||||
|
||||
|
||||
def union(rows):
|
||||
busy, cs, ce = 0, *rows[0]
|
||||
for s, e in rows[1:]:
|
||||
if s > ce:
|
||||
busy += ce - cs
|
||||
cs, ce = s, e
|
||||
else:
|
||||
ce = max(ce, e)
|
||||
return busy + ce - cs
|
||||
|
||||
|
||||
def analyze(sqlite_path, n_frames) -> dict:
|
||||
db = sqlite3.connect(str(sqlite_path))
|
||||
ops = {k: db.execute(q).fetchall() for k, q in Q.items()}
|
||||
lo = ops["kernel"][0][0]
|
||||
hi = max(ops["kernel"][-1][1], ops["D2H"][-1][1])
|
||||
span = hi - lo
|
||||
|
||||
out = {"n_frames": n_frames, "window_ms": span / 1e6,
|
||||
"window_us_per_frame": span / 1e3 / n_frames}
|
||||
for k, rows in ops.items():
|
||||
rows = [r for r in rows if r[1] > lo]
|
||||
tot, busy = sum(e - s for s, e in rows), union(rows)
|
||||
out[f"{k}_n"] = len(rows)
|
||||
out[f"{k}_sum_ms"] = tot / 1e6
|
||||
out[f"{k}_busy_ms"] = busy / 1e6
|
||||
out[f"{k}_overlap"] = tot / busy
|
||||
out[f"{k}_duty_pct"] = 100 * busy / span
|
||||
out[f"{k}_us_per_frame"] = busy / 1e3 / n_frames
|
||||
tallest = max(("kernel", "H2D", "D2H"), key=lambda k: out[f"{k}_us_per_frame"])
|
||||
out["bottleneck"] = tallest
|
||||
out["roofline_us_per_frame"] = out[f"{tallest}_us_per_frame"]
|
||||
out["roofline_fps"] = 1e6 / out["roofline_us_per_frame"]
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
N = int(sys.argv[2]) if len(sys.argv) > 2 else 2000
|
||||
r = analyze(sys.argv[1], N)
|
||||
print(f"processing window: {r['window_ms']:.1f} ms / {N} frames = "
|
||||
f"{r['window_us_per_frame']:.1f} us/frame")
|
||||
for k in ("kernel", "H2D", "D2H"):
|
||||
print(f" {k:6s} sum={r[k + '_sum_ms']:7.2f} ms busy={r[k + '_busy_ms']:7.2f} ms "
|
||||
f"overlap={r[k + '_overlap']:4.2f}x duty={r[k + '_duty_pct']:5.1f}% "
|
||||
f"per-frame={r[k + '_us_per_frame']:5.2f} us")
|
||||
print(f" roofline = {r['bottleneck']} at {r['roofline_us_per_frame']:.2f} us/frame "
|
||||
f"= {r['roofline_fps']:,.0f} FPS")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""The process nsys traces: train the pedestal, run one batched pass, exit.
|
||||
|
||||
nsys_kernel_probe.py <n_streams> <n_frames> <cluster_dim> <cap> <batch>
|
||||
|
||||
Run by run_probes.py; run directly only under `nsys profile --trace=cuda`.
|
||||
The wall time printed is profiler-inflated and is not a throughput number.
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import common # noqa: E402 (puts the build dir on sys.path)
|
||||
|
||||
from aare import File, ClusterFinderCUDA # noqa: E402
|
||||
|
||||
n_streams, N, cdim, cap, batch = (int(a) for a in sys.argv[1:6])
|
||||
|
||||
f = File(common.DATA_FILE)
|
||||
pd = File(common.PEDESTAL_FILE)
|
||||
|
||||
cf = ClusterFinderCUDA((f.rows, f.cols), (cdim, cdim), n_sigma=common.N_SIGMA,
|
||||
max_clusters_per_frame=cap, n_streams=n_streams)
|
||||
for _ in range(common.N_PEDESTAL_FRAMES):
|
||||
cf.push_pedestal_frame(pd.read_frame().copy())
|
||||
|
||||
data = f.read_n(N)
|
||||
cf.register_input_buffer(data)
|
||||
# Pin the output slots without processing frames: a warm-up pass would advance
|
||||
# the device pedestal and make this run incomparable with the next.
|
||||
cf.reserve_output_slots(cf.chunk_size_for(min(N, batch)))
|
||||
|
||||
t0 = time.perf_counter()
|
||||
n = 0
|
||||
for start in range(0, N, batch):
|
||||
for cv in cf.find_clusters_batched(data[start:min(start + batch, N)],
|
||||
first_frame=start):
|
||||
n += cv.size
|
||||
t = time.perf_counter() - t0
|
||||
cf.unregister_input_buffer()
|
||||
|
||||
h2d = f.rows * f.cols * 2
|
||||
slot = 2 + 2 + cdim * cdim * 4 # x, y (uint16) + data (int32)
|
||||
print(f"n_streams={n_streams} N={N} cluster={cdim}x{cdim} cap={cap} batch={batch}")
|
||||
print(f" H2D/frame={h2d:,} B D2H/frame={cap * slot:,} B "
|
||||
f"({slot} B/slot, {100 * n / N / cap:.0f}% filled)")
|
||||
print(f" wall={t:.3f}s ({N / t:.0f} FPS, profiler-inflated) clusters/frame={n / N:.2f}")
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp": "2026-09-10 13:37:02",
|
||||
"host": "pc-moench-04.psi.ch",
|
||||
"git_rev": "272b483",
|
||||
"git_branch": "testing/cuda_cf_api",
|
||||
"git_dirty": true,
|
||||
"aare_version": "2026.7.2",
|
||||
"device_ped_type": "float",
|
||||
"gpu": "NVIDIA GeForce RTX 4090",
|
||||
"driver": "595.71.05",
|
||||
"gpu_busy_pct": "0 %",
|
||||
"nvcc": "Build cuda_12.4.r12.4/compiler.34097967_0",
|
||||
"python": "3.11.15"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
n_frames,window_ms,window_us_per_frame,window_fps,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type
|
||||
20000,467.758071,23.38790355,42757.145712617756,20000,112.903327,112.777756,1.0011134376534323,24.110274732170254,5.6388878,19999,326.931385,326.931385,1.0,69.89326433236467,16.34656925,20000,152.729782,152.729782,1.0,32.65144771815172,7.6364891,H2D,16.34656925,61174.91595369468,3x3_s4,3,3000,4,2000,float
|
||||
20000,861.486383,43.07431915,23215.689063305832,20000,92.96108,92.96108,1.0,10.790777641345493,4.648054,19999,262.616382,262.616382,1.0,30.484101337211733,13.130819099999998,20000,105.867334,105.867334,1.0,12.288915540525728,5.2933667,H2D,13.130819099999998,76156.71135093166,3x3_s1_uncontended,3,3000,1,2000,float
|
||||
20000,1502.075647,75.10378235,13314.908633226778,20000,312.858126,311.891228,1.0031001128380566,20.76401602162451,15.5945614,19999,414.382371,414.382371,1.0,27.58731704542441,20.719118549999997,20000,507.805309,507.805309,1.0,33.806906464012464,25.39026545,D2H,25.39026545,39385.17310775103,9x9_s4,9,1700,4,2000,float
|
||||
20000,1652.407506,82.6203753,12103.551894661994,20000,305.830789,305.830789,1.0,18.508194128234614,15.29153945,19999,264.610496,264.610496,1.0,16.01363435104125,13.2305248,20000,440.039995,440.039995,1.0,26.630234576046522,22.00199975,D2H,22.00199975,45450.41411519878,9x9_s1_uncontended,9,1700,1,2000,float
|
||||
|
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp": "2026-09-10 12:31:37",
|
||||
"host": "pc-moench-04.psi.ch",
|
||||
"git_rev": "7177f00",
|
||||
"git_branch": "HEAD",
|
||||
"git_dirty": true,
|
||||
"aare_version": "2026.7.2",
|
||||
"device_ped_type": "float",
|
||||
"gpu": "NVIDIA GeForce RTX 4090",
|
||||
"driver": "595.71.05",
|
||||
"gpu_busy_pct": "0 %",
|
||||
"nvcc": "Build cuda_12.4.r12.4/compiler.34097967_0",
|
||||
"python": "3.11.15"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
n_frames,window_ms,window_us_per_frame,window_fps,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type
|
||||
20000,657.208045,32.86040225,30431.763810803626,20000,100.861444,100.083645,1.0077714895375762,15.228609229821586,5.00418225,19999,316.797856,316.797856,1.0,48.20358764780489,15.839892800000001,20000,144.76269,144.76269,1.0,22.02691995348292,7.2381345,H2D,15.839892800000001,63131.740386525846,3x3_s4,3,3000,4,2000,float
|
||||
20000,1151.676025,57.58380124999999,17365.99492031624,20000,86.388419,86.388419,1.0,7.501104227640755,4.31942095,19999,262.127428,262.127428,1.0,22.760517915617807,13.1063714,20000,105.9432,105.9432,1.0,9.199045365210237,5.29716,H2D,13.1063714,76298.76870420443,3x3_s1_uncontended,3,3000,1,2000,float
|
||||
20000,1845.553098,92.2776549,10836.85970437465,20000,511.593396,494.436337,1.0347002388701865,26.790686084069524,24.72181685,19999,342.849695,342.849695,1.0,18.577070222013194,17.14248475,20000,495.751951,495.751951,1.0,26.86197170578508,24.78759755,D2H,24.78759755,40342.75600863949,9x9_s4,9,1700,4,2000,float
|
||||
20000,2824.450123,141.22250615000002,7081.0243158965495,20000,475.987403,475.987403,1.0,16.852391873517252,23.799370149999998,19999,262.138573,262.138573,1.0,9.281048047737114,13.10692865,20000,439.610736,439.610736,1.0,15.564471555725893,21.9805368,kernel,23.799370149999998,42017.918696894594,9x9_s1_uncontended,9,1700,1,2000,float
|
||||
|
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""nsys probe sweep: per-engine GPU time per frame, duty cycles, roofline.
|
||||
|
||||
python3.11 run_probes.py # full sweep, 20k frames
|
||||
python3.11 run_probes.py --frames 2000 # quick check (clocks may not ramp)
|
||||
python3.11 run_probes.py --only 9x9_s1_uncontended --tag mytag
|
||||
|
||||
Per config: an .nsys-rep + .sqlite in results/<date>_<f32|f64>[_tag]/ and one
|
||||
row in probes.csv (re-running a config replaces its own row only).
|
||||
|
||||
20 000 frames, not fewer: over a short run the GPU clocks never fully ramp and
|
||||
the kernel reads ~10 % slow.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import common # noqa: E402
|
||||
import gpu_span # noqa: E402
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
# (cluster_dim, cap, n_streams, label)
|
||||
# s4 = the production configuration.
|
||||
# s1 = uncontended control: one stream, so H2D and D2H never coexist and the
|
||||
# kernel number is the kernel alone. Quote kernel changes from s1.
|
||||
# The cap sets the D2H bar (the slot is copied whole regardless of fill), so a
|
||||
# probe at a different cap is a different measurement.
|
||||
CONFIGS = [
|
||||
(3, 3000, 4, "3x3_s4"),
|
||||
(3, 3000, 1, "3x3_s1_uncontended"),
|
||||
(9, 1700, 4, "9x9_s4"),
|
||||
(9, 1700, 1, "9x9_s1_uncontended"),
|
||||
]
|
||||
|
||||
|
||||
def run_one(cdim, cap, streams, label, n_frames, batch, outdir) -> dict | None:
|
||||
rep = outdir / f"probe_{label}_cap{cap}"
|
||||
print(f"\n--- {label}: {cdim}x{cdim} cap={cap} streams={streams} N={n_frames} ---")
|
||||
|
||||
prof = [common.NSYS, "profile", "--trace=cuda", "--sample=none",
|
||||
"--cpuctxsw=none", "--force-overwrite=true", "-o", str(rep),
|
||||
sys.executable, str(HERE / "nsys_kernel_probe.py"),
|
||||
str(streams), str(n_frames), str(cdim), str(cap), str(batch)]
|
||||
p = subprocess.run(prof, capture_output=True, text=True)
|
||||
for line in p.stdout.splitlines():
|
||||
if line.strip().startswith(("n_streams", "H2D/frame", "wall")):
|
||||
print(" ", line.strip())
|
||||
if not rep.with_suffix(".nsys-rep").exists():
|
||||
print(f" FAILED: {p.stderr.strip()[-400:]}")
|
||||
return None
|
||||
|
||||
subprocess.run([common.NSYS, "stats", "--force-export=true", "--report",
|
||||
"cuda_gpu_sum", str(rep.with_suffix(".nsys-rep"))],
|
||||
capture_output=True, text=True)
|
||||
sq = rep.with_suffix(".sqlite")
|
||||
if not sq.exists():
|
||||
print(" FAILED: no sqlite export")
|
||||
return None
|
||||
|
||||
r = gpu_span.analyze(sq, n_frames)
|
||||
r.update(label=label, cluster_dim=cdim, cap=cap, n_streams=streams,
|
||||
batch=batch, device_ped_type=common.device_ped_type())
|
||||
print(f" kernel {r['kernel_us_per_frame']:5.2f} us (duty {r['kernel_duty_pct']:4.1f}%) "
|
||||
f"H2D {r['H2D_us_per_frame']:5.2f} ({r['H2D_duty_pct']:4.1f}%) "
|
||||
f"D2H {r['D2H_us_per_frame']:5.2f} ({r['D2H_duty_pct']:4.1f}%)")
|
||||
print(f" -> roofline: {r['bottleneck']}-bound at "
|
||||
f"{r['roofline_us_per_frame']:.2f} us/frame = {r['roofline_fps']:,.0f} FPS")
|
||||
return r
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--frames", type=int, default=20_000)
|
||||
ap.add_argument("--batch", type=int, default=2000)
|
||||
ap.add_argument("--tag", default="")
|
||||
ap.add_argument("--only", nargs="+", default=None, help="subset of labels")
|
||||
ap.add_argument("--cap", type=int, default=None, help="override every cap")
|
||||
args = ap.parse_args()
|
||||
|
||||
common.assert_build_fresh()
|
||||
common.assert_idle_gpu()
|
||||
env = common.capture_env()
|
||||
outdir = common.results_dir(args.tag)
|
||||
common.write_env(outdir / "env.json", env)
|
||||
print(f"build: DEVICE_PED_TYPE={env['device_ped_type']} git={env['git_rev']}"
|
||||
f"{' (dirty)' if env['git_dirty'] else ''}")
|
||||
print(f"out: {outdir}")
|
||||
|
||||
rows = []
|
||||
for cdim, cap, streams, label in CONFIGS:
|
||||
if args.only and label not in args.only:
|
||||
continue
|
||||
r = run_one(cdim, args.cap or cap, streams, label, args.frames, args.batch, outdir)
|
||||
if r:
|
||||
rows.append(r)
|
||||
|
||||
if not rows:
|
||||
return 1
|
||||
out = outdir / "probes.csv"
|
||||
|
||||
def key(r):
|
||||
return (str(r["label"]), str(r["cap"]), str(r["n_streams"]))
|
||||
|
||||
prior = list(csv.DictReader(out.open())) if out.exists() else []
|
||||
fresh = {key(r) for r in rows}
|
||||
merged = [r for r in prior if key(r) not in fresh] + rows
|
||||
with out.open("w", newline="") as fh:
|
||||
w = csv.DictWriter(fh, fieldnames=list(rows[0]))
|
||||
w.writeheader()
|
||||
w.writerows(merged)
|
||||
|
||||
print(f"\n=== rooflines ({env['device_ped_type']} build, us/frame) ===")
|
||||
print(f"{'config':<22} {'kernel':>8} {'H2D':>8} {'D2H':>8} "
|
||||
f"{'bottleneck':<10} {'roofline':>9} {'FPS':>10}")
|
||||
for r in rows:
|
||||
print(f"{r['label']:<22} {r['kernel_us_per_frame']:8.2f} "
|
||||
f"{r['H2D_us_per_frame']:8.2f} {r['D2H_us_per_frame']:8.2f} "
|
||||
f"{r['bottleneck']:<10} {r['roofline_us_per_frame']:9.2f} "
|
||||
f"{r['roofline_fps']:10,.0f}")
|
||||
print(f"\n-> {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user