mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2026-10-31 13:41:27 +01:00
kferjaoui
da3f563872
Per-engine GPU time, duty cycle and roofline on the campaign configs (3x3 cap 3000, 9x9 cap 1700, 1 and 4 streams, 20k frames).
103 lines
3.6 KiB
Python
103 lines
3.6 KiB
Python
"""Shared plumbing for the nsys probes: paths, guards, environment stamp.
|
|
|
|
Two guards are enforced here because both have produced wrong numbers before:
|
|
a stale extension (header edited after the build) and a busy GPU (another
|
|
process leaves per-op averages intact but destroys duty cycles).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = Path(os.environ.get("AARE_REPO", HERE.parents[2]))
|
|
BUILD = Path(os.environ.get("AARE_BUILD", REPO / "build"))
|
|
sys.path.insert(0, str(BUILD))
|
|
|
|
DATA_DIR = Path(os.environ.get(
|
|
"AARE_PERF_DATA",
|
|
"/mnt/sls_det_storage/moench_data/2603_MaxIVBeamtime/2026032408/process/xrf/"))
|
|
DATA_FILE = DATA_DIR / "Cu_factor_10_data_master_0.json"
|
|
PEDESTAL_FILE = DATA_DIR / "Cu_factor_10_pedestal_master_0.json"
|
|
|
|
N_PEDESTAL_FRAMES = 1000
|
|
N_SIGMA = 5
|
|
|
|
NSYS = os.environ.get("NSYS", "/opt/nvidia/nsight-systems/2024.5.1/bin/nsys")
|
|
|
|
|
|
def _sh(cmd: str, default: str = "unknown") -> str:
|
|
try:
|
|
return subprocess.run(cmd, shell=True, capture_output=True, text=True,
|
|
timeout=30).stdout.strip() or default
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def device_ped_type() -> str:
|
|
"""DEVICE_PED_TYPE from the kernel header (the float/double build axis)."""
|
|
hdr = REPO / "include/aare/clusterfinder_kernel.cuh"
|
|
for line in hdr.read_text().splitlines()[:40]:
|
|
if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"):
|
|
return line.split("=")[1].split(";")[0].strip()
|
|
return "unknown"
|
|
|
|
|
|
def assert_build_fresh() -> None:
|
|
"""Abort if a CUDA header is newer than the installed extension."""
|
|
so = list((BUILD / "aare").glob("_aare_cuda*.so"))
|
|
if not so:
|
|
raise RuntimeError(f"no _aare_cuda*.so under {BUILD / 'aare'}")
|
|
built = max(f.stat().st_mtime for f in so)
|
|
headers = ["clusterfinder_kernel.cuh", "clusterfinder_algo.cuh",
|
|
"ClusterFinderCUDA.hpp"]
|
|
stale = [h for h in headers
|
|
if (REPO / "include/aare" / h).stat().st_mtime > built]
|
|
if stale:
|
|
raise RuntimeError(f"{', '.join(stale)} newer than the extension: rebuild first")
|
|
|
|
|
|
def assert_idle_gpu(max_pct: int = 5) -> None:
|
|
pct = _sh("nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader")
|
|
try:
|
|
val = int(pct.split()[0])
|
|
except Exception:
|
|
return
|
|
if val > max_pct:
|
|
raise SystemExit(f"GPU is {val}% busy: numbers would not be quotable. Aborting.")
|
|
|
|
|
|
def capture_env() -> dict:
|
|
import aare
|
|
|
|
return {
|
|
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
"host": _sh("hostname"),
|
|
"git_rev": _sh(f"git -C {REPO} rev-parse --short HEAD"),
|
|
"git_branch": _sh(f"git -C {REPO} rev-parse --abbrev-ref HEAD"),
|
|
"git_dirty": bool(_sh(f"git -C {REPO} status --porcelain", default="")),
|
|
"aare_version": getattr(aare, "__version__", "unknown"),
|
|
"device_ped_type": device_ped_type(),
|
|
"gpu": _sh("nvidia-smi --query-gpu=name --format=csv,noheader"),
|
|
"driver": _sh("nvidia-smi --query-gpu=driver_version --format=csv,noheader"),
|
|
"nvcc": _sh("nvcc --version | tail -1"),
|
|
"python": sys.version.split()[0],
|
|
}
|
|
|
|
|
|
def results_dir(tag: str = "") -> Path:
|
|
"""perf/results/<date>_<f32|f64>[_tag]/"""
|
|
short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk")
|
|
name = f"{time.strftime('%Y-%m-%d')}_{short}" + (f"_{tag}" if tag else "")
|
|
d = HERE / "results" / name
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d
|
|
|
|
|
|
def write_env(path: Path, env: dict) -> None:
|
|
path.write_text(json.dumps(env, indent=2) + "\n")
|