perf: move ClusterFinderCUDA probes into perf/cf/
Build on RHEL8 / build (push) Successful in 4m1s
Build on RHEL9 / build (push) Successful in 4m15s
Run tests using data on local RHEL8 / build (push) Failing after 4m56s

Make room for probes of other CUDA classes. common.py and gpu_span.py
stay in perf/ and become shared; the cluster-finder driver, traced
process, README and results move to perf/cf/.

- common.py: assert_build_fresh() takes the header list, capture_env()
  takes extra fields, results_dir() takes the probe folder.
- cf/cf_common.py: pedestal frames, n_sigma, rebuild-guard headers,
  DEVICE_PED_TYPE and the <date>_<f32|f64> results naming.
- gpu_span.py: optional kernel-name filter; tolerates an engine with no
  ops. Output without a filter is unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kferjaouiandClaude Opus 5.5 committed 2026-10-27 12:22:18 +01:00
1 parent e969478241
commit b7b786f423
11 files changed
+200 -140

No files matched your search

+18 -27
View File
@@ -1,5 +1,8 @@
"""Shared plumbing for the nsys probes: paths, guards, environment stamp.
Used by every probe folder (cf/, ...). Settings specific to one class, such as
the headers to check or extra env.json fields, are passed in by that folder.
Two guards are enforced here because both have produced wrong numbers before:
a stale extension (header edited after the build) and a busy GPU (another
process leaves per-op averages intact but destroys duty cycles).
@@ -24,9 +27,6 @@ DATA_DIR = Path(os.environ.get(
DATA_FILE = DATA_DIR / "Cu_factor_10_data_master_0.json"
PEDESTAL_FILE = DATA_DIR / "Cu_factor_10_pedestal_master_0.json"
N_PEDESTAL_FRAMES = 1000
N_SIGMA = 5
NSYS = os.environ.get("NSYS", "/opt/nvidia/nsight-systems/2024.5.1/bin/nsys")
@@ -38,25 +38,13 @@ def _sh(cmd: str, default: str = "unknown") -> str:
return default
def device_ped_type() -> str:
"""DEVICE_PED_TYPE from the kernel header (the float/double build axis)."""
hdr = REPO / "include/aare/cf/clusterfinder_kernel.cuh"
for line in hdr.read_text().splitlines()[:40]:
if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"):
return line.split("=")[1].split(";")[0].strip()
return "unknown"
def assert_build_fresh() -> None:
"""Abort if a CUDA header is newer than the installed extension."""
def assert_build_fresh(headers: list[str]) -> None:
"""Abort if any header (repo-relative path) is newer than the extension."""
so = list((BUILD / "aare").glob("_aare_cuda*.so"))
if not so:
raise RuntimeError(f"no _aare_cuda*.so under {BUILD / 'aare'}")
built = max(f.stat().st_mtime for f in so)
headers = ["clusterfinder_kernel.cuh", "clusterfinder_algo.cuh",
"ClusterFinderCUDA.hpp"]
stale = [h for h in headers
if (REPO / "include/aare/cf" / h).stat().st_mtime > built]
stale = [h for h in headers if (REPO / h).stat().st_mtime > built]
if stale:
raise RuntimeError(f"{', '.join(stale)} newer than the extension: rebuild first")
@@ -71,29 +59,32 @@ def assert_idle_gpu(max_pct: int = 5) -> None:
raise SystemExit(f"GPU is {val}% busy: numbers would not be quotable. Aborting.")
def capture_env() -> dict:
def capture_env(**extra) -> dict:
"""Machine, build and git state; `extra` adds probe-specific fields."""
import aare
return {
env = {
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
"host": _sh("hostname"),
"git_rev": _sh(f"git -C {REPO} rev-parse --short HEAD"),
"git_branch": _sh(f"git -C {REPO} rev-parse --abbrev-ref HEAD"),
"git_dirty": bool(_sh(f"git -C {REPO} status --porcelain", default="")),
"aare_version": getattr(aare, "__version__", "unknown"),
"device_ped_type": device_ped_type(),
}
env.update(extra)
env.update({
"gpu": _sh("nvidia-smi --query-gpu=name --format=csv,noheader"),
"driver": _sh("nvidia-smi --query-gpu=driver_version --format=csv,noheader"),
"nvcc": _sh("nvcc --version | tail -1"),
"python": sys.version.split()[0],
}
})
return env
def results_dir(tag: str = "") -> Path:
"""perf/results/<date>_<f32|f64>[_tag]/"""
short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk")
name = f"{time.strftime('%Y-%m-%d')}_{short}" + (f"_{tag}" if tag else "")
d = HERE / "results" / name
def results_dir(probe_dir: Path, suffix: str = "", tag: str = "") -> Path:
"""<probe_dir>/results/<date>[_suffix][_tag]/, created if missing."""
name = time.strftime("%Y-%m-%d") + "".join(f"_{s}" for s in (suffix, tag) if s)
d = probe_dir / "results" / name
d.mkdir(parents=True, exist_ok=True)
return d