diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/env.json b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/env.json new file mode 100644 index 00000000..55075a2a --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-29 13:46:59", + "host": "gpu104", + "git_rev": "a550343", + "git_branch": "feature/cuda_clusterfinder", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/throughput.csv b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/throughput.csv new file mode 100644 index 00000000..570c8984 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput/throughput.csv @@ -0,0 +1,17 @@ +label,cluster_dim,cap,n_streams,path,batch,n_frames,repeats,wall_s,fps,us_per_frame,clusters_per_frame,device_ped_type +3x3_s1,3,3000,1,copy,2000,20000,3,0.9566467289987486,20906.359049523457,47.83233644993743,2323.9411,float +3x3_s1,3,3000,1,view,2000,20000,3,0.900972492992878,22198.235967852237,45.0486246496439,2323.9411,float +3x3_s4,3,3000,4,copy,2000,20000,3,0.5476258129929192,36521.28794056426,27.381290649645962,2323.93595,float +3x3_s4,3,3000,4,view,2000,20000,3,0.3638208779739216,54972.10636007971,18.19104389869608,2323.93595,float +5x5_s1,5,2350,1,copy,2000,20000,3,1.5809259899833705,12650.813590717411,79.04629949916853,2065.34305,float +5x5_s1,5,2350,1,view,2000,20000,3,1.00852486200165,19830.943939552853,50.4262431000825,2065.34305,float +5x5_s4,5,2350,4,copy,2000,20000,3,1.0555958909972105,18946.64442195413,52.779794549860526,2065.3578,float +5x5_s4,5,2350,4,view,2000,20000,3,0.5609194769931491,35655.74172466162,28.045973849657457,2065.35785,float +7x7_s1,7,2170,1,copy,2000,20000,3,1.5937661220086738,12548.892666129312,79.68830610043369,1746.6247,float +7x7_s1,7,2170,1,view,2000,20000,3,1.20658721998916,16575.676974416885,60.329360999458004,1746.6247,float +7x7_s4,7,2170,4,copy,2000,20000,3,1.7966621039959136,11131.75368674971,89.83310519979568,1746.6394,float +7x7_s4,7,2170,4,view,2000,20000,3,0.4557413999864366,43884.536275605475,22.78706999932183,1746.64215,float +9x9_s1,9,1700,1,copy,2000,20000,3,1.9693570459785406,10155.598773132748,98.46785229892703,1422.96435,float +9x9_s1,9,1700,1,view,2000,20000,3,1.3747952050180174,14547.621294429733,68.73976025090087,1422.96435,float +9x9_s4,9,1700,4,copy,2000,20000,3,2.299068950000219,8699.173637223057,114.95344750001095,1422.9265,float +9x9_s4,9,1700,4,view,2000,20000,3,0.5560760640073568,35966.30262390759,27.803803200367838,1422.9765,float diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/env.json b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/env.json new file mode 100644 index 00000000..4e7af0d9 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-29 13:53:35", + "host": "gpu104", + "git_rev": "a550343", + "git_branch": "feature/cuda_clusterfinder", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/throughput.csv b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/throughput.csv new file mode 100644 index 00000000..ffbba9c4 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_throughput_5x5/throughput.csv @@ -0,0 +1,5 @@ +label,cluster_dim,cap,n_streams,path,batch,n_frames,repeats,wall_s,fps,us_per_frame,clusters_per_frame,device_ped_type +5x5_s1,5,2350,1,copy,2000,20000,5,1.3398772029904649,14926.741014297508,66.99386014952324,2065.34305,float +5x5_s1,5,2350,1,view,2000,20000,5,1.0176306089852005,19653.496881294046,50.881530449260026,2065.34305,float +5x5_s4,5,2350,4,copy,2000,20000,5,0.9175176069838926,21797.946816241434,45.87588034919463,2065.35785,float +5x5_s4,5,2350,4,view,2000,20000,5,0.4006308230163995,49921.271282667425,20.031541150819976,2065.35785,float diff --git a/python/tests/perf/cf/results/A100/README.md b/python/tests/perf/cf/results/A100/README.md index 8c09b560..4505678f 100644 --- a/python/tests/perf/cf/results/A100/README.md +++ b/python/tests/perf/cf/results/A100/README.md @@ -12,6 +12,8 @@ folder name says `evtrace-on`. Driver and git state of each run are in its | `2026-10-28_f64_lround` | f64 | `lround` | 20 000 / 2000 | 3x3, 9x9 at s1, s4 | | `2026-10-29_f32_round_b2000_f20K` | f32 | `round` | 20 000 / 2000 | 3x3 to 9x9 at s1, s4, s8 | | `2026-10-29_f32_round_b3000_f50K` | f32 | `round` | 50 000 / 3000 | 3x3 to 9x9 at s1, s4, s8 | +| `2026-10-29_f32_round_throughput` | f32 | `round` | 20 000 / 2000 | unprofiled wall clock (`throughput.py`), 3x3 to 9x9 at s1, s4 | +| `2026-10-29_f32_round_throughput_5x5` | f32 | `round` | 20 000 / 2000 | 5x5 rerun, 5 passes (first run disturbed by other jobs on the node) | `lround(float)` is emulated in software on the device and made the f32 9x9 kernel 1.8x slower than f64. It was replaced by `round` in 104e284. The diff --git a/python/tests/perf/cf/throughput.py b/python/tests/perf/cf/throughput.py new file mode 100644 index 00000000..9cec7ff4 --- /dev/null +++ b/python/tests/perf/cf/throughput.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Unprofiled end-to-end throughput: wall-clock FPS, no nsys. + + python throughput.py # 3x3..9x9 at 1 and 4 streams + python throughput.py --streams 4 --repeats 5 --tag mytag + +The nsys probes give per-engine busy time, i.e. the roofline. Under the +profiler the wall clock is inflated, so the achieved rate is measured here. +Two host paths over the same frames: + + copy cf.find_clusters_batched(): one owned ClusterVector per frame, the way + the notebook and most callers consume the output. + view find_cluster_views_batched_iter(): zero-copy BatchViews; the host only + reads the cluster counts. This is the GPU pipeline's own rate. + +The input is pinned once and the output slots are reserved before timing, as +in nsys_kernel_probe.py. Each path gets one untimed pass (clock ramp, first +touch), then the median of --repeats timed passes. Output: one row per +(size, streams, path) in results/__/throughput.csv. +""" +from __future__ import annotations + +import argparse +import csv +import statistics +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import cf_common # noqa: E402 (puts ../ and the build dir on sys.path) +from cf_common import common # noqa: E402 + +from aare import File, ClusterFinderCUDA, find_cluster_views_batched_iter # noqa: E402 + +# (cluster_dim, cap): the caps of run_probes.py, so the D2H bytes match. +SIZES = [(3, 3000), (5, 2350), (7, 2170), (9, 1700)] + + +def one_pass(cf, data, batch, path) -> tuple[float, int]: + n = 0 + t0 = time.perf_counter() + for start in range(0, len(data), batch): + chunk = data[start:start + batch] + if path == "copy": + for cv in cf.find_clusters_batched(chunk, first_frame=start): + n += cv.size + else: + for v in find_cluster_views_batched_iter(cf, chunk, first_frame=start): + n += v.total_clusters + return time.perf_counter() - t0, n + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--frames", type=int, default=20_000) + ap.add_argument("--batch", type=int, default=2000) + ap.add_argument("--sizes", type=int, nargs="+", default=[c for c, _ in SIZES], + help="cluster dims to run, e.g. --sizes 5") + ap.add_argument("--streams", type=int, nargs="+", default=[1, 4]) + ap.add_argument("--paths", nargs="+", choices=["copy", "view"], + default=["copy", "view"]) + ap.add_argument("--repeats", type=int, default=3) + ap.add_argument("--tag", default="throughput") + args = ap.parse_args() + + common.assert_build_fresh(cf_common.HEADERS) + common.assert_idle_gpu() + env = common.capture_env(device_ped_type=cf_common.device_ped_type()) + outdir = cf_common.results_dir(args.tag) + common.write_env(outdir / "env.json", env) + print(f"build: DEVICE_PED_TYPE={env['device_ped_type']} git={env['git_rev']}" + f"{' (dirty)' if env['git_dirty'] else ''}") + print(f"out: {outdir}") + + f = File(common.DATA_FILE) + ped = File(common.PEDESTAL_FILE).read_n(cf_common.N_PEDESTAL_FRAMES) + data = f.read_n(args.frames) + + rows = [] + print(f"\n{'config':<10} {'path':<5} {'FPS':>10} {'us/frame':>9} {'clusters/frame':>15}") + for cdim, cap in SIZES: + if cdim not in args.sizes: + continue + for ns in args.streams: + cf = ClusterFinderCUDA((f.rows, f.cols), cluster_size=(cdim, cdim), + n_sigma=cf_common.N_SIGMA, + max_clusters_per_frame=cap, n_streams=ns) + for frame in ped: + cf.push_pedestal_frame(frame.copy()) + cf.register_input_buffer(data) + cf.reserve_output_slots(cf.chunk_size_for(min(args.frames, args.batch))) + for path in args.paths: + one_pass(cf, data, args.batch, path) + runs = [one_pass(cf, data, args.batch, path) for _ in range(args.repeats)] + t = statistics.median(r[0] for r in runs) + row = dict(label=f"{cdim}x{cdim}_s{ns}", cluster_dim=cdim, cap=cap, + n_streams=ns, path=path, batch=args.batch, + n_frames=args.frames, repeats=args.repeats, wall_s=t, + fps=args.frames / t, us_per_frame=t * 1e6 / args.frames, + clusters_per_frame=runs[-1][1] / args.frames, + device_ped_type=env["device_ped_type"]) + rows.append(row) + print(f"{row['label']:<10} {path:<5} {row['fps']:10,.0f} " + f"{row['us_per_frame']:9.2f} {row['clusters_per_frame']:15.2f}") + cf.unregister_input_buffer() + del cf + + out = outdir / "throughput.csv" + with out.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + print(f"\n-> {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())