From b7b786f423d36495a6ef3e4b27b50caddb325dc0 Mon Sep 17 00:00:00 2001 From: kferjaoui Date: Tue, 27 Oct 2026 12:22:18 +0100 Subject: [PATCH] perf: move ClusterFinderCUDA probes into perf/cf/ Make room for probes of other CUDA classes. common.py and gpu_span.py stay in perf/ and become shared; the cluster-finder driver, traced process, README and results move to perf/cf/. - common.py: assert_build_fresh() takes the header list, capture_env() takes extra fields, results_dir() takes the probe folder. - cf/cf_common.py: pedestal frames, n_sigma, rebuild-guard headers, DEVICE_PED_TYPE and the _ results naming. - gpu_span.py: optional kernel-name filter; tolerates an engine with no ops. Output without a filter is unchanged. Co-Authored-By: Claude Opus 5.5 --- python/tests/perf/README.md | 107 ++---------------- python/tests/perf/cf/README.md | 103 +++++++++++++++++ python/tests/perf/cf/cf_common.py | 35 ++++++ .../tests/perf/{ => cf}/nsys_kernel_probe.py | 7 +- .../{ => cf}/results/2026-09-10_f32/env.json | 0 .../results/2026-09-10_f32/probes.csv | 0 .../results/2026-09-10_f32_ref7177/env.json | 0 .../results/2026-09-10_f32_ref7177/probes.csv | 0 python/tests/perf/{ => cf}/run_probes.py | 11 +- python/tests/perf/common.py | 45 +++----- python/tests/perf/gpu_span.py | 32 ++++-- 11 files changed, 200 insertions(+), 140 deletions(-) create mode 100644 python/tests/perf/cf/README.md create mode 100644 python/tests/perf/cf/cf_common.py rename python/tests/perf/{ => cf}/nsys_kernel_probe.py (87%) rename python/tests/perf/{ => cf}/results/2026-09-10_f32/env.json (100%) rename python/tests/perf/{ => cf}/results/2026-09-10_f32/probes.csv (100%) rename python/tests/perf/{ => cf}/results/2026-09-10_f32_ref7177/env.json (100%) rename python/tests/perf/{ => cf}/results/2026-09-10_f32_ref7177/probes.csv (100%) rename python/tests/perf/{ => cf}/run_probes.py (93%) diff --git a/python/tests/perf/README.md b/python/tests/perf/README.md index c20c9ef1..dc001f8c 100644 --- a/python/tests/perf/README.md +++ b/python/tests/perf/README.md @@ -1,102 +1,19 @@ -# ClusterFinderCUDA nsys probes +# GPU performance probes -Per-engine GPU time per frame (kernel, H2D, D2H) for the CUDA cluster finder, -traced with Nsight Systems on real MOENCH data. The tallest engine is the -roofline: the engines overlap and PCIe is full-duplex, so throughput is bound by -the max, not the sum. +Nsight Systems probes for the CUDA classes: per-engine GPU time per frame +(kernel, H2D, D2H), duty cycles and the roofline. One folder per class, each +with its own driver, traced process, README and `results/`: -These are *not* throughput numbers. nsys inflates wall clock several-fold by -tracing every CUDA call. Use them to answer "what does the kernel cost" and -"which engine binds", and measure sustained FPS separately without a profiler. +| folder | class | +|---|---| +| [`cf/`](cf/README.md) | `ClusterFinderCUDA` | -## Files +Shared by all of them: | file | role | |---|---| -| `run_probes.py` | driver: runs each config under nsys, exports SQLite, writes `probes.csv` | -| `nsys_kernel_probe.py` | the traced process: pedestal, one batched pass, exit | -| `gpu_span.py` | reads the SQLite export, computes busy time, duty cycle, roofline | -| `common.py` | paths, guards (fresh build, idle GPU), `env.json` stamp | +| `common.py` | paths (`AARE_REPO`, `AARE_BUILD`, `AARE_PERF_DATA`, `NSYS`), guards (fresh build, idle GPU), `env.json` stamp, results folder | +| `gpu_span.py` | reads an nsys SQLite export: busy time, duty cycle and roofline per engine, optionally for one kernel name only | -## Requirements - -- A built tree with the CUDA extension: `build/aare/_aare_cuda*.so`. Run the - scripts with a Python of the same minor version as the build, with numpy and - matplotlib installed (`aare/__init__.py` imports both). On pc-moench-04 the - build is 3.11 and the `py` conda environment works: - `/home/ferjao_k/.conda/envs/py/bin/python`. -- Nsight Systems. Default path `/opt/nvidia/nsight-systems/2024.5.1/bin/nsys`, - override with `NSYS=...`. -- The MOENCH dataset under `/mnt/sls_det_storage/.../2026032408/process/xrf/` - (override with `AARE_PERF_DATA=`). 1000 pedestal frames, 20 000 data - frames, 5 sigma. -- An idle GPU. The driver aborts above 5 % utilisation: a competing process - leaves per-op averages intact but ruins duty cycles and rooflines. - -Paths are resolved relative to this folder. `AARE_REPO` and `AARE_BUILD` -override the repo root and build directory. - -## Run - -```bash -cd python/tests/perf -python run_probes.py # full sweep: 4 configs, 20k frames each (~5 min) -python run_probes.py --frames 2000 # smoke test only; clocks do not ramp, kernel reads ~7 % slow -python run_probes.py --only 3x3_s1_uncontended 9x9_s1_uncontended --tag mychange -``` - -Configs, matching the campaign in `docs/ClusterFinderCUDA_benchmark_results.md` -on branch `bench/cuda_cf`: - -| label | cluster | cap | streams | batch | frames | -|---|---|---|---|---|---| -| `3x3_s4` | 3x3 | 3000 | 4 | 2000 | 20 000 | -| `3x3_s1_uncontended` | 3x3 | 3000 | 1 | 2000 | 20 000 | -| `9x9_s4` | 9x9 | 1700 | 4 | 2000 | 20 000 | -| `9x9_s1_uncontended` | 9x9 | 1700 | 1 | 2000 | 20 000 | - -Keep the cap: at 9x9 the D2H slot is `4 + cap * sizeof(cluster)` and is copied -whole, so the cap sets the D2H bar. - -Output goes to `results/_[_tag]/`: `env.json` (git rev, dirty -flag, driver, pedestal type), one `.nsys-rep` + `.sqlite` per config, and -`probes.csv`. Re-running one config replaces its own row and keeps the others. - -To inspect one trace by hand: - -```bash -nsys stats --report cuda_gpu_sum results//probe_9x9_s1_uncontended_cap1700.nsys-rep -python gpu_span.py results//probe_9x9_s1_uncontended_cap1700.sqlite 20000 -``` - -`probes.csv` and `env.json` are small and can be committed with a result; -the `.nsys-rep` and `.sqlite` traces are tens of MB each and are gitignored. - -## Reading the numbers - -- **Quote kernel changes from the `s1_uncontended` rows.** With one stream - nothing overlaps, so the kernel column is the kernel alone. The 1-stream - kernel and transfer numbers reproduce across sessions to about 1 %. -- **4-stream transfer columns drift by up to 20 % between sessions** on an - identical build (H2D and D2H fight for the PCIe link). Only compare s4 - rows taken in the same session, and read them for the bottleneck, not for - a percentage. -- A kernel below the tallest transfer bar is invisible end to end. Further - kernel work only pays once it is the tallest bar. - -## Reference, 2026-09-10, RTX 4090, driver 595.71.05, float pedestal - -Kernel us/frame. "before" is commit 7177f00 (pre-refactor kernel, rebuilt and -re-probed the same day, matching its August numbers): `results/2026-09-10_f32_ref7177/`. -"after" is the kernel with the cooperative tile load, size-dependent unroll -and two-phase load: `results/2026-09-10_f32/`. - -| config | before | after | roofline after | binds | -|---|--:|--:|--:|---| -| 3x3, 1 stream | 4.32 | 4.65 | 13.13 | H2D | -| 3x3, 4 streams | 5.00 | 5.64 | 16.35 | H2D | -| 9x9, 1 stream | 23.80 | 15.29 | 22.00 | D2H | -| 9x9, 4 streams | 24.72 | 15.59 | 25.39 | D2H | - -Both sizes are transfer-bound; sustained throughput is unchanged (3x3 about -61 400 FPS, 9x9 about 39 600 FPS on the zero-copy path). +Requirements (a CUDA build of the Python extension, Nsight Systems, the +dataset, an idle GPU) are listed in each folder's README. diff --git a/python/tests/perf/cf/README.md b/python/tests/perf/cf/README.md new file mode 100644 index 00000000..b67ef2f0 --- /dev/null +++ b/python/tests/perf/cf/README.md @@ -0,0 +1,103 @@ +# ClusterFinderCUDA nsys probes + +Per-engine GPU time per frame (kernel, H2D, D2H) for the CUDA cluster finder, +traced with Nsight Systems on real MOENCH data. The tallest engine is the +roofline: the engines overlap and PCIe is full-duplex, so throughput is bound by +the max, not the sum. + +These are *not* throughput numbers. nsys inflates wall clock several-fold by +tracing every CUDA call. Use them to answer "what does the kernel cost" and +"which engine binds", and measure sustained FPS separately without a profiler. + +## Files + +| file | role | +|---|---| +| `run_probes.py` | driver: runs each config under nsys, exports SQLite, writes `probes.csv` | +| `nsys_kernel_probe.py` | the traced process: pedestal, one batched pass, exit | +| `cf_common.py` | cluster-finder settings: pedestal frames, n_sigma, headers for the rebuild guard, `DEVICE_PED_TYPE` | +| `../gpu_span.py` | shared: reads the SQLite export, computes busy time, duty cycle, roofline | +| `../common.py` | shared: paths, guards (fresh build, idle GPU), `env.json` stamp | + +## Requirements + +- A built tree with the CUDA extension: `build/aare/_aare_cuda*.so`. Run the + scripts with a Python of the same minor version as the build, with numpy and + matplotlib installed (`aare/__init__.py` imports both). On pc-moench-04 the + build is 3.11 and the `py` conda environment works: + `/home/ferjao_k/.conda/envs/py/bin/python`. +- Nsight Systems. Default path `/opt/nvidia/nsight-systems/2024.5.1/bin/nsys`, + override with `NSYS=...`. +- The MOENCH dataset under `/mnt/sls_det_storage/.../2026032408/process/xrf/` + (override with `AARE_PERF_DATA=`). 1000 pedestal frames, 20 000 data + frames, 5 sigma. +- An idle GPU. The driver aborts above 5 % utilisation: a competing process + leaves per-op averages intact but ruins duty cycles and rooflines. + +Paths are resolved relative to `python/tests/perf/`. `AARE_REPO` and `AARE_BUILD` +override the repo root and build directory. + +## Run + +```bash +cd python/tests/perf/cf +python run_probes.py # full sweep: 4 configs, 20k frames each (~5 min) +python run_probes.py --frames 2000 # smoke test only; clocks do not ramp, kernel reads ~7 % slow +python run_probes.py --only 3x3_s1_uncontended 9x9_s1_uncontended --tag mychange +``` + +Configs, matching the campaign in `docs/ClusterFinderCUDA_benchmark_results.md` +on branch `bench/cuda_cf`: + +| label | cluster | cap | streams | batch | frames | +|---|---|---|---|---|---| +| `3x3_s4` | 3x3 | 3000 | 4 | 2000 | 20 000 | +| `3x3_s1_uncontended` | 3x3 | 3000 | 1 | 2000 | 20 000 | +| `9x9_s4` | 9x9 | 1700 | 4 | 2000 | 20 000 | +| `9x9_s1_uncontended` | 9x9 | 1700 | 1 | 2000 | 20 000 | + +Keep the cap: at 9x9 the D2H slot is `4 + cap * sizeof(cluster)` and is copied +whole, so the cap sets the D2H bar. + +Output goes to `results/_[_tag]/`: `env.json` (git rev, dirty +flag, driver, pedestal type), one `.nsys-rep` + `.sqlite` per config, and +`probes.csv`. Re-running one config replaces its own row and keeps the others. + +To inspect one trace by hand: + +```bash +nsys stats --report cuda_gpu_sum results//probe_9x9_s1_uncontended_cap1700.nsys-rep +python ../gpu_span.py results//probe_9x9_s1_uncontended_cap1700.sqlite 20000 +``` + +`probes.csv` and `env.json` are small and can be committed with a result; +the `.nsys-rep` and `.sqlite` traces are tens of MB each and are gitignored. + +## Reading the numbers + +- **Quote kernel changes from the `s1_uncontended` rows.** With one stream + nothing overlaps, so the kernel column is the kernel alone. The 1-stream + kernel and transfer numbers reproduce across sessions to about 1 %. +- **4-stream transfer columns drift by up to 20 % between sessions** on an + identical build (H2D and D2H fight for the PCIe link). Only compare s4 + rows taken in the same session, and read them for the bottleneck, not for + a percentage. +- A kernel below the tallest transfer bar is invisible end to end. Further + kernel work only pays once it is the tallest bar. + +## Reference, 2026-09-10, RTX 4090, driver 595.71.05, float pedestal + +Kernel us/frame. "before" is commit 7177f00 (pre-refactor kernel, rebuilt and +re-probed the same day, matching its August numbers): `results/2026-09-10_f32_ref7177/`. +"after" is the kernel with the cooperative tile load, size-dependent unroll +and two-phase load: `results/2026-09-10_f32/`. + +| config | before | after | roofline after | binds | +|---|--:|--:|--:|---| +| 3x3, 1 stream | 4.32 | 4.65 | 13.13 | H2D | +| 3x3, 4 streams | 5.00 | 5.64 | 16.35 | H2D | +| 9x9, 1 stream | 23.80 | 15.29 | 22.00 | D2H | +| 9x9, 4 streams | 24.72 | 15.59 | 25.39 | D2H | + +Both sizes are transfer-bound; sustained throughput is unchanged (3x3 about +61 400 FPS, 9x9 about 39 600 FPS on the zero-copy path). diff --git a/python/tests/perf/cf/cf_common.py b/python/tests/perf/cf/cf_common.py new file mode 100644 index 00000000..ddc4fd86 --- /dev/null +++ b/python/tests/perf/cf/cf_common.py @@ -0,0 +1,35 @@ +"""ClusterFinderCUDA-specific settings for the probes in this folder. + +Importing this module also makes ../common.py and ../gpu_span.py importable. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +import common # noqa: E402 + +N_PEDESTAL_FRAMES = 1000 +N_SIGMA = 5 + +# Rebuild-guard: the extension must be newer than all of these. +HEADERS = ["include/aare/cf/clusterfinder_kernel.cuh", + "include/aare/cf/clusterfinder_algo.cuh", + "include/aare/cf/ClusterFinderCUDA.hpp"] + + +def device_ped_type() -> str: + """DEVICE_PED_TYPE from the kernel header (the float/double build axis).""" + hdr = common.REPO / "include/aare/cf/clusterfinder_kernel.cuh" + for line in hdr.read_text().splitlines()[:40]: + if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"): + return line.split("=")[1].split(";")[0].strip() + return "unknown" + + +def results_dir(tag: str = "") -> Path: + """cf/results/_[_tag]/""" + short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk") + return common.results_dir(HERE, short, tag) diff --git a/python/tests/perf/nsys_kernel_probe.py b/python/tests/perf/cf/nsys_kernel_probe.py similarity index 87% rename from python/tests/perf/nsys_kernel_probe.py rename to python/tests/perf/cf/nsys_kernel_probe.py index 0481a17d..3c56600e 100644 --- a/python/tests/perf/nsys_kernel_probe.py +++ b/python/tests/perf/cf/nsys_kernel_probe.py @@ -10,7 +10,8 @@ import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -import common # noqa: E402 (puts the build dir on sys.path) +import cf_common # noqa: E402 (puts ../ and the build dir on sys.path) +from cf_common import common # noqa: E402 from aare import File, ClusterFinderCUDA # noqa: E402 @@ -20,9 +21,9 @@ f = File(common.DATA_FILE) pd = File(common.PEDESTAL_FILE) cf = ClusterFinderCUDA((f.rows, f.cols), cluster_size=(cdim, cdim), - n_sigma=common.N_SIGMA, max_clusters_per_frame=cap, + n_sigma=cf_common.N_SIGMA, max_clusters_per_frame=cap, n_streams=n_streams) -for _ in range(common.N_PEDESTAL_FRAMES): +for _ in range(cf_common.N_PEDESTAL_FRAMES): cf.push_pedestal_frame(pd.read_frame().copy()) data = f.read_n(N) diff --git a/python/tests/perf/results/2026-09-10_f32/env.json b/python/tests/perf/cf/results/2026-09-10_f32/env.json similarity index 100% rename from python/tests/perf/results/2026-09-10_f32/env.json rename to python/tests/perf/cf/results/2026-09-10_f32/env.json diff --git a/python/tests/perf/results/2026-09-10_f32/probes.csv b/python/tests/perf/cf/results/2026-09-10_f32/probes.csv similarity index 100% rename from python/tests/perf/results/2026-09-10_f32/probes.csv rename to python/tests/perf/cf/results/2026-09-10_f32/probes.csv diff --git a/python/tests/perf/results/2026-09-10_f32_ref7177/env.json b/python/tests/perf/cf/results/2026-09-10_f32_ref7177/env.json similarity index 100% rename from python/tests/perf/results/2026-09-10_f32_ref7177/env.json rename to python/tests/perf/cf/results/2026-09-10_f32_ref7177/env.json diff --git a/python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv b/python/tests/perf/cf/results/2026-09-10_f32_ref7177/probes.csv similarity index 100% rename from python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv rename to python/tests/perf/cf/results/2026-09-10_f32_ref7177/probes.csv diff --git a/python/tests/perf/run_probes.py b/python/tests/perf/cf/run_probes.py similarity index 93% rename from python/tests/perf/run_probes.py rename to python/tests/perf/cf/run_probes.py index 7f6e4709..f8a819e6 100644 --- a/python/tests/perf/run_probes.py +++ b/python/tests/perf/cf/run_probes.py @@ -20,7 +20,8 @@ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -import common # noqa: E402 +import cf_common # noqa: E402 (puts ../ and the build dir on sys.path) +from cf_common import common # noqa: E402 import gpu_span # noqa: E402 HERE = Path(__file__).resolve().parent @@ -68,7 +69,7 @@ def run_one(cdim, cap, streams, label, n_frames, batch, outdir) -> dict | None: r = gpu_span.analyze(sq, n_frames) r.update(label=label, cluster_dim=cdim, cap=cap, n_streams=streams, - batch=batch, device_ped_type=common.device_ped_type()) + batch=batch, device_ped_type=cf_common.device_ped_type()) print(f" kernel {r['kernel_us_per_frame']:5.2f} us (duty {r['kernel_duty_pct']:4.1f}%) " f"H2D {r['H2D_us_per_frame']:5.2f} ({r['H2D_duty_pct']:4.1f}%) " f"D2H {r['D2H_us_per_frame']:5.2f} ({r['D2H_duty_pct']:4.1f}%)") @@ -87,10 +88,10 @@ def main() -> int: ap.add_argument("--cap", type=int, default=None, help="override every cap") args = ap.parse_args() - common.assert_build_fresh() + common.assert_build_fresh(cf_common.HEADERS) common.assert_idle_gpu() - env = common.capture_env() - outdir = common.results_dir(args.tag) + env = common.capture_env(device_ped_type=cf_common.device_ped_type()) + outdir = cf_common.results_dir(args.tag) common.write_env(outdir / "env.json", env) print(f"build: DEVICE_PED_TYPE={env['device_ped_type']} git={env['git_rev']}" f"{' (dirty)' if env['git_dirty'] else ''}") diff --git a/python/tests/perf/common.py b/python/tests/perf/common.py index 4526f6b3..0a3cdc39 100644 --- a/python/tests/perf/common.py +++ b/python/tests/perf/common.py @@ -1,5 +1,8 @@ """Shared plumbing for the nsys probes: paths, guards, environment stamp. +Used by every probe folder (cf/, ...). Settings specific to one class, such as +the headers to check or extra env.json fields, are passed in by that folder. + Two guards are enforced here because both have produced wrong numbers before: a stale extension (header edited after the build) and a busy GPU (another process leaves per-op averages intact but destroys duty cycles). @@ -24,9 +27,6 @@ DATA_DIR = Path(os.environ.get( DATA_FILE = DATA_DIR / "Cu_factor_10_data_master_0.json" PEDESTAL_FILE = DATA_DIR / "Cu_factor_10_pedestal_master_0.json" -N_PEDESTAL_FRAMES = 1000 -N_SIGMA = 5 - NSYS = os.environ.get("NSYS", "/opt/nvidia/nsight-systems/2024.5.1/bin/nsys") @@ -38,25 +38,13 @@ def _sh(cmd: str, default: str = "unknown") -> str: return default -def device_ped_type() -> str: - """DEVICE_PED_TYPE from the kernel header (the float/double build axis).""" - hdr = REPO / "include/aare/cf/clusterfinder_kernel.cuh" - for line in hdr.read_text().splitlines()[:40]: - if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"): - return line.split("=")[1].split(";")[0].strip() - return "unknown" - - -def assert_build_fresh() -> None: - """Abort if a CUDA header is newer than the installed extension.""" +def assert_build_fresh(headers: list[str]) -> None: + """Abort if any header (repo-relative path) is newer than the extension.""" so = list((BUILD / "aare").glob("_aare_cuda*.so")) if not so: raise RuntimeError(f"no _aare_cuda*.so under {BUILD / 'aare'}") built = max(f.stat().st_mtime for f in so) - headers = ["clusterfinder_kernel.cuh", "clusterfinder_algo.cuh", - "ClusterFinderCUDA.hpp"] - stale = [h for h in headers - if (REPO / "include/aare/cf" / h).stat().st_mtime > built] + stale = [h for h in headers if (REPO / h).stat().st_mtime > built] if stale: raise RuntimeError(f"{', '.join(stale)} newer than the extension: rebuild first") @@ -71,29 +59,32 @@ def assert_idle_gpu(max_pct: int = 5) -> None: raise SystemExit(f"GPU is {val}% busy: numbers would not be quotable. Aborting.") -def capture_env() -> dict: +def capture_env(**extra) -> dict: + """Machine, build and git state; `extra` adds probe-specific fields.""" import aare - return { + env = { "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), "host": _sh("hostname"), "git_rev": _sh(f"git -C {REPO} rev-parse --short HEAD"), "git_branch": _sh(f"git -C {REPO} rev-parse --abbrev-ref HEAD"), "git_dirty": bool(_sh(f"git -C {REPO} status --porcelain", default="")), "aare_version": getattr(aare, "__version__", "unknown"), - "device_ped_type": device_ped_type(), + } + env.update(extra) + env.update({ "gpu": _sh("nvidia-smi --query-gpu=name --format=csv,noheader"), "driver": _sh("nvidia-smi --query-gpu=driver_version --format=csv,noheader"), "nvcc": _sh("nvcc --version | tail -1"), "python": sys.version.split()[0], - } + }) + return env -def results_dir(tag: str = "") -> Path: - """perf/results/_[_tag]/""" - short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk") - name = f"{time.strftime('%Y-%m-%d')}_{short}" + (f"_{tag}" if tag else "") - d = HERE / "results" / name +def results_dir(probe_dir: Path, suffix: str = "", tag: str = "") -> Path: + """/results/[_suffix][_tag]/, created if missing.""" + name = time.strftime("%Y-%m-%d") + "".join(f"_{s}" for s in (suffix, tag) if s) + d = probe_dir / "results" / name d.mkdir(parents=True, exist_ok=True) return d diff --git a/python/tests/perf/gpu_span.py b/python/tests/perf/gpu_span.py index 8a4d9ded..d66cdea3 100644 --- a/python/tests/perf/gpu_span.py +++ b/python/tests/perf/gpu_span.py @@ -1,12 +1,15 @@ """Per-engine busy time and duty cycle from an nsys SQLite export. nsys stats --force-export=true --report cuda_gpu_sum .nsys-rep - python gpu_span.py .sqlite + python gpu_span.py .sqlite [kernel_name] Per engine (kernel / H2D / D2H): sum of op durations, busy = union of the intervals (overlaps not double-counted), duty = busy / processing window. -The window is first kernel start to last D2H end, so the pedestal-upload -copies that precede any kernel are excluded. +The window is first kernel start to last kernel or D2H end, so the +pedestal-upload copies that precede any kernel are excluded. + +`kernel_name` (optional) keeps only kernels whose short name contains it, for +a probe whose per-frame kernel shares the trace with one-off kernels. Roofline = tallest engine's busy time per frame. PCIe is full-duplex and the engines overlap, so the floor is the max, never the sum. @@ -14,12 +17,15 @@ engines overlap, so the floor is the max, never the sum. import sqlite3 import sys -Q = {"kernel": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_KERNEL ORDER BY start", - "H2D": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=1 ORDER BY start", +KERNELS = ("SELECT k.start, k.end, s.value FROM CUPTI_ACTIVITY_KIND_KERNEL k " + "JOIN StringIds s ON k.shortName = s.id ORDER BY k.start") +Q = {"H2D": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=1 ORDER BY start", "D2H": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=2 ORDER BY start"} def union(rows): + if not rows: + return 0 busy, cs, ce = 0, *rows[0] for s, e in rows[1:]: if s > ce: @@ -30,11 +36,16 @@ def union(rows): return busy + ce - cs -def analyze(sqlite_path, n_frames) -> dict: +def analyze(sqlite_path, n_frames, kernel_name=None) -> dict: db = sqlite3.connect(str(sqlite_path)) - ops = {k: db.execute(q).fetchall() for k, q in Q.items()} + kernels = [(s, e) for s, e, name in db.execute(KERNELS) + if kernel_name is None or kernel_name in name] + if not kernels: + raise RuntimeError(f"no kernel matching {kernel_name!r} in {sqlite_path}") + ops = {"kernel": kernels} + ops.update({k: db.execute(q).fetchall() for k, q in Q.items()}) lo = ops["kernel"][0][0] - hi = max(ops["kernel"][-1][1], ops["D2H"][-1][1]) + hi = max([ops["kernel"][-1][1]] + [r[1] for r in ops["D2H"][-1:]]) span = hi - lo out = {"n_frames": n_frames, "window_ms": span / 1e6, @@ -45,7 +56,7 @@ def analyze(sqlite_path, n_frames) -> dict: out[f"{k}_n"] = len(rows) out[f"{k}_sum_ms"] = tot / 1e6 out[f"{k}_busy_ms"] = busy / 1e6 - out[f"{k}_overlap"] = tot / busy + out[f"{k}_overlap"] = tot / busy if busy else 0.0 out[f"{k}_duty_pct"] = 100 * busy / span out[f"{k}_us_per_frame"] = busy / 1e3 / n_frames tallest = max(("kernel", "H2D", "D2H"), key=lambda k: out[f"{k}_us_per_frame"]) @@ -57,7 +68,8 @@ def analyze(sqlite_path, n_frames) -> dict: if __name__ == "__main__": N = int(sys.argv[2]) if len(sys.argv) > 2 else 2000 - r = analyze(sys.argv[1], N) + name = sys.argv[3] if len(sys.argv) > 3 else None + r = analyze(sys.argv[1], N, name) print(f"processing window: {r['window_ms']:.1f} ms / {N} frames = " f"{r['window_us_per_frame']:.1f} us/frame") for k in ("kernel", "H2D", "D2H"):