From 3d798861784522ea69ac1c584f9efc1cd0db6cd9 Mon Sep 17 00:00:00 2001 From: Ferjaoui Date: Thu, 29 Oct 2026 12:04:55 +0100 Subject: [PATCH] perf: A100 probe results; disable nsys CUDA event trace when supported - common.nsys_profile_flags(): shared nsys profile flags, adding --cuda-event-trace=false on nsys 2025.1+ (on by default there, it widens the host gaps between steps; kernel times are unchanged). nsys 2024.5 rejects the flag, so it is detected from the help text. - cf/run_probes.py: 3x3, 5x5, 7x7, 9x9 at 1, 4 and 8 streams. - cf/results/A100/: Merlin7 A100 runs, f32 and f64 with lround, and f32 with round (104e284) at 20k/2000 and 50k/3000. README lists them. --- .../A100/2026-10-28_f32_lround/env.json | 13 ++++++++++ .../A100/2026-10-28_f32_lround/probes.csv | 5 ++++ .../2026-10-28_f32_lround_evtrace-on/env.json | 13 ++++++++++ .../probes.csv | 5 ++++ .../A100/2026-10-28_f64_lround/env.json | 13 ++++++++++ .../A100/2026-10-28_f64_lround/probes.csv | 5 ++++ .../2026-10-29_f32_round_b2000_f20K/env.json | 13 ++++++++++ .../probes.csv | 13 ++++++++++ .../2026-10-29_f32_round_b3000_f50K/env.json | 13 ++++++++++ .../probes.csv | 13 ++++++++++ python/tests/perf/cf/results/A100/README.md | 24 +++++++++++++++++++ python/tests/perf/cf/run_probes.py | 19 +++++++++++---- python/tests/perf/common.py | 13 ++++++++++ 13 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f32_lround/env.json create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f32_lround/probes.csv create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/env.json create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/probes.csv create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f64_lround/env.json create mode 100644 python/tests/perf/cf/results/A100/2026-10-28_f64_lround/probes.csv create mode 100644 python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/env.json create mode 100644 python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/probes.csv create mode 100644 python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/env.json create mode 100644 python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/probes.csv create mode 100644 python/tests/perf/cf/results/A100/README.md diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/env.json b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/env.json new file mode 100644 index 00000000..ecca324f --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-28 15:49:25", + "host": "gpu102", + "git_rev": "b4d302d", + "git_branch": "feature/cuda_pixelhistogram", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/probes.csv b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/probes.csv new file mode 100644 index 00000000..50bd0804 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround/probes.csv @@ -0,0 +1,5 @@ +n_frames,window_ms,window_us_per_frame,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,643.891349,32.19456745,20000,192.988136,190.334593,1.0139414646500964,29.56004818135862,9.51672965,19999,349.946562,349.946562,1.0,54.34869757816858,17.497328099999997,20000,162.802494,162.802494,1.0,25.284156131751352,8.1401247,H2D,17.497328099999997,57151.5830465567,3x3_s4,3,3000,4,2000,float +20000,1146.409624,57.3204812,20000,160.19295,160.19295,1.0,13.973447766520145,8.0096475,19999,279.906056,279.906056,1.0,24.415885050176446,13.9953028,20000,118.383043,118.383043,1.0,10.326417409768709,5.91915215,H2D,13.9953028,71452.54477809512,3x3_s1_uncontended,3,3000,1,2000,float +20000,1991.737902,99.5868951,20000,686.534959,618.29859,1.1103615148143877,31.043170357863684,30.9149295,19999,489.906258,489.906258,1.0,24.596923998286197,24.4953129,20000,553.600056,553.600056,1.0,27.79482458229587,27.6800028,kernel,30.9149295,32346.831002800765,9x9_s4,9,1700,4,2000,float +20000,2453.211627,122.66058134999999,20000,641.327303,641.327303,1.0,26.1423554307979,32.066365149999996,19999,284.587043,284.587043,1.0,11.60059082827753,14.22935215,20000,477.774029,477.774029,1.0,19.475451026793948,23.88870145,kernel,32.066365149999996,31185.324414607065,9x9_s1_uncontended,9,1700,1,2000,float diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/env.json b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/env.json new file mode 100644 index 00000000..9d7057a5 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-28 14:53:53", + "host": "gpu105", + "git_rev": "b4d302d", + "git_branch": "feature/cuda_pixelhistogram", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/probes.csv b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/probes.csv new file mode 100644 index 00000000..bf9b3bfe --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f32_lround_evtrace-on/probes.csv @@ -0,0 +1,5 @@ +n_frames,window_ms,window_us_per_frame,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,712.152215,35.60761075,20000,192.405024,189.918432,1.013092947186927,26.668235806863283,9.4959216,19999,335.94684,335.94684,1.0,47.1734599603822,16.797342,20000,159.543909,159.543909,1.0,22.403062946311273,7.977195450000001,H2D,16.797342,59533.22853103783,3x3_s4,3,3000,4,2000,float +20000,1151.363903,57.568195149999994,20000,159.331526,159.331526,1.0,13.838502803921932,7.966576300000001,19999,279.076937,279.076937,1.0,24.238812444339764,13.95384685,20000,118.133377,118.133377,1.0,10.260298824046076,5.90666885,H2D,13.95384685,71664.82553160601,3x3_s1_uncontended,3,3000,1,2000,float +20000,2376.000656,118.8000328,20000,680.402812,616.187901,1.1042131968118603,25.93382705699051,30.80939505,19999,488.931373,488.931373,1.0,20.577914057612954,24.44656865,20000,554.867913,554.867913,1.0,23.353020193778935,27.743395649999997,kernel,30.80939505,32457.63178332838,9x9_s4,9,1700,4,2000,float +20000,2499.957377,124.99786884999999,20000,642.043636,642.043636,1.0,25.68218330067953,32.102181800000004,19999,282.576058,282.576058,1.0,11.303235031114692,14.1288029,20000,478.262973,478.262973,1.0,19.130845085604033,23.91314865,kernel,32.102181800000004,31150.530709411156,9x9_s1_uncontended,9,1700,1,2000,float diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/env.json b/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/env.json new file mode 100644 index 00000000..6f676fd3 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-28 15:32:33", + "host": "gpu102", + "git_rev": "b4d302d", + "git_branch": "feature/cuda_pixelhistogram", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "double", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/probes.csv b/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/probes.csv new file mode 100644 index 00000000..0da620e8 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-28_f64_lround/probes.csv @@ -0,0 +1,5 @@ +n_frames,window_ms,window_us_per_frame,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,681.275896,34.0637948,20000,205.953165,203.624901,1.0114340829071784,29.88875757905869,10.181245050000001,19999,345.21322,345.21322,1.0,50.671574031440564,17.260661,20000,163.783599,163.783599,1.0,24.040715363867797,8.18917995,H2D,17.260661,57935.20885440019,3x3_s4,3,3000,4,2000,double +20000,1170.35509,58.5177545,20000,158.332121,158.332121,1.0,13.528554056188195,7.91660605,19999,280.175565,280.175565,1.0,23.939363992512735,14.00877825,20000,118.821603,118.821603,1.0,10.152611289963287,5.94108015,H2D,14.00877825,71383.81250342085,3x3_s1_uncontended,3,3000,1,2000,double +20000,2266.611556,113.3305778,20000,376.488668,365.770028,1.0293043146771994,16.137305354848372,18.2885014,19999,504.512867,504.512867,1.0,22.25846178470644,25.225643350000002,20000,559.423881,559.423881,1.0,24.681065422045346,27.97119405,D2H,27.97119405,35751.06583624734,9x9_s4,9,1700,4,2000,double +20000,2352.22485,117.6112425,20000,349.59613,349.59613,1.0,14.86236020336236,17.4798065,19999,285.855613,285.855613,1.0,12.152563263669288,14.292780650000001,20000,483.90927,483.90927,1.0,20.57240701287549,24.195463500000002,D2H,24.195463500000002,41330.06172830704,9x9_s1_uncontended,9,1700,1,2000,double diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/env.json b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/env.json new file mode 100644 index 00000000..1c2c3c69 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-29 11:19:39", + "host": "gpu105", + "git_rev": "104e284", + "git_branch": "feature/cuda_clusterfinder", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/probes.csv b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/probes.csv new file mode 100644 index 00000000..ff3f40f1 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b2000_f20K/probes.csv @@ -0,0 +1,13 @@ +n_frames,window_ms,window_us_per_frame,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,1173.468708,58.6734354,20000,133.601765,133.601765,1.0,11.385200482056655,6.680088250000001,19999,279.546079,279.546079,1.0,23.822201401215377,13.977303950000001,20000,118.733951,118.733951,1.0,10.118203424645559,5.93669755,H2D,13.977303950000001,71544.55562941378,3x3_s1_uncontended,3,3000,1,2000,float +20000,662.843817,33.14219085,20000,157.677679,155.787634,1.0121321888745034,23.502917279229898,7.7893817,19999,342.1502,342.1502,1.0,51.6185247904334,17.10751,20000,164.807934,164.807934,1.0,24.863765757959843,8.2403967,H2D,17.10751,58453.8603221626,3x3_s4,3,3000,4,2000,float +20000,657.959684,32.8979842,20000,169.854168,168.724024,1.0066981806929878,25.6435201278989,8.436201200000001,19999,349.87012,349.87012,1.0,53.17500882014528,17.493506,20000,170.160589,170.160589,1.0,25.861856453806674,8.50802945,H2D,17.493506,57164.06991257213,3x3_s8,3,3000,8,2000,float +20000,1450.355696,72.5177848,20000,170.110976,170.110976,1.0,11.728914256630741,8.5055488,19999,279.957736,279.957736,1.0,19.30269497145478,13.997886799999998,20000,213.748591,213.748591,1.0,14.737666876443253,10.68742955,H2D,13.997886799999998,71439.35468888062,5x5_s1_uncontended,5,2350,1,2000,float +20000,998.976254,49.9488127,20000,191.741624,190.10828,1.0085916510317172,19.03031020395005,9.505414,19999,393.673137,393.673137,1.0,39.407657131357595,19.68365685,20000,269.665626,269.665626,1.0,26.99419780202303,13.4832813,H2D,19.68365685,50803.56803720647,5x5_s4,5,2350,4,2000,float +20000,995.458631,49.77293155,20000,230.466823,228.149833,1.0101555629891694,22.91906724147937,11.40749165,19999,389.404519,389.404519,1.0,39.11810163409995,19.47022595,20000,277.415209,277.415209,1.0,27.868080135215585,13.870760449999999,H2D,19.47022595,51360.4722702255,5x5_s8,5,2350,8,2000,float +20000,1805.062161,90.25310805000001,20000,217.596819,217.596819,1.0,12.054810283068141,10.87984095,19999,284.393532,284.393532,1.0,15.755331763336432,14.2196766,20000,375.295808,375.295808,1.0,20.79129550818832,18.764790400000003,D2H,18.764790400000003,53291.29602215007,7x7_s1_uncontended,7,2170,1,2000,float +20000,1455.77395,72.7886975,20000,238.06754,234.794902,1.0139382838899968,16.12852750937053,11.7397451,19999,466.709503,466.709503,1.0,32.059201430277,23.33547515,20000,447.260819,447.260819,1.0,30.723232751898053,22.363040950000002,H2D,23.33547515,42853.209269235726,7x7_s4,7,2170,4,2000,float +20000,1534.937701,76.74688504999999,20000,254.608264,251.535528,1.0122159124972596,16.38734443985098,12.5767764,19999,487.785527,487.785527,1.0,31.778848528002897,24.38927635,20000,457.385111,457.385111,1.0,29.798285018474505,22.86925555,H2D,24.38927635,41001.626520173486,7x7_s8,7,2170,8,2000,float +20000,2146.615315,107.33076575,20000,293.12465,293.12465,1.0,13.65520165405137,14.656232500000002,19999,284.039721,284.039721,1.0,13.23198055167141,14.20198605,20000,484.764387,484.764387,1.0,22.582732155714634,24.238219349999998,D2H,24.238219349999998,41257.15612850909,9x9_s1_uncontended,9,1700,1,2000,float +20000,1924.272895,96.21364475,20000,313.452178,306.702235,1.0220081311112714,15.938603916156081,15.33511175,19999,522.802982,522.802982,1.0,27.16885860412226,26.140149100000002,20000,561.998593,561.998593,1.0,29.205763613897393,28.09992965,D2H,28.09992965,35587.27770693903,9x9_s4,9,1700,4,2000,float +20000,2020.810003,101.04050015,20000,344.108027,335.796507,1.0247516571100008,16.6169262078816,16.789825349999997,19999,533.548848,533.548848,1.0,26.402722037594742,26.6774424,20000,572.622922,572.622922,1.0,28.33630678539352,28.631146100000002,D2H,28.631146100000002,34926.99860869349,9x9_s8,9,1700,8,2000,float diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/env.json b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/env.json new file mode 100644 index 00000000..6bbe2104 --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/env.json @@ -0,0 +1,13 @@ +{ + "timestamp": "2026-10-29 11:10:41", + "host": "gpu105", + "git_rev": "104e284", + "git_branch": "feature/cuda_clusterfinder", + "git_dirty": true, + "aare_version": "2026.9.30", + "device_ped_type": "float", + "gpu": "NVIDIA A100-SXM4-80GB", + "driver": "610.57.04", + "nvcc": "Build cuda_12.9.r12.9/compiler.36037853_0", + "python": "3.14.10" +} diff --git a/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/probes.csv b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/probes.csv new file mode 100644 index 00000000..a59cb60c --- /dev/null +++ b/python/tests/perf/cf/results/A100/2026-10-29_f32_round_b3000_f50K/probes.csv @@ -0,0 +1,13 @@ +n_frames,window_ms,window_us_per_frame,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +50000,2793.77305,55.875460999999994,50000,323.196107,323.196107,1.0,11.568445296585562,6.46392214,49999,697.283022,697.283022,1.0,24.958470481344218,13.94566044,50000,293.479098,293.479098,1.0,10.504758001012286,5.86958196,H2D,13.94566044,71706.894363477,3x3_s1_uncontended,3,3000,1,3000,float +50000,1571.355703,31.42711406,50000,386.12095,382.211638,1.0102281343929145,24.323686691071245,7.6442327599999995,49999,872.765101,872.765101,1.0,55.542172872363324,17.45530202,50000,409.563912,409.563912,1.0,26.06436666237116,8.19127824,H2D,17.45530202,57289.18347297665,3x3_s4,3,3000,4,3000,float +50000,1503.190827,30.06381654,50000,410.304025,408.101596,1.005396766446363,27.149021180129914,8.16203192,49999,879.916105,879.916105,1.0,58.536553656071504,17.5983221,50000,414.203543,414.203543,1.0,27.55495413889989,8.28407086,H2D,17.5983221,56823.59910891732,3x3_s8,3,3000,8,3000,float +50000,3315.016749,66.30033498,50000,419.817667,419.817667,1.0,12.664119031273106,8.396353340000001,49999,701.427246,701.427246,1.0,21.159086035133633,14.028544920000002,50000,535.556489,535.556489,1.0,16.15546857075623,10.711129779999998,H2D,14.028544920000002,71283.2304207356,5x5_s1_uncontended,5,2350,1,3000,float +50000,2377.743978,47.55487956,50000,464.366056,461.233672,1.006791316831699,19.397953533582665,9.22467344,49999,1018.072896,1018.072896,1.0,42.816758465994944,20.36145792,50000,671.385408,671.385408,1.0,28.236236290028362,13.427708160000002,H2D,20.36145792,49112.39676102722,5x5_s4,5,2350,4,3000,float +50000,2297.384508,45.94769016,50000,559.001887,553.741472,1.009499767068196,24.103125535658048,11.074829439999998,49999,1040.452858,1040.452858,1.0,45.288581618658675,20.809057160000002,50000,702.175827,702.175827,1.0,30.564140419458248,14.04351654,H2D,20.809057160000002,48055.997554864705,5x5_s8,5,2350,8,3000,float +50000,4105.69513,82.1139026,50000,540.216808,540.216808,1.0,13.157742864361193,10.80433616,49999,712.202522,712.202522,1.0,17.346697683322628,14.24405044,50000,941.949467,941.949467,1.0,22.942508812143586,18.838989339999998,D2H,18.838989339999998,53081.40378192921,7x7_s1_uncontended,7,2170,1,3000,float +50000,3444.082992,68.88165984,50000,586.370154,576.931884,1.0163594182636646,16.75139319639252,11.538637679999999,49999,1197.256295,1197.256295,1.0,34.76270164746367,23.945125899999997,50000,1122.678549,1122.678549,1.0,32.597314048697,22.453570980000002,H2D,23.945125899999997,41762.152522238364,7x7_s4,7,2170,4,3000,float +50000,3419.960725,68.3992145,50000,631.268088,624.620283,1.0106429540969613,18.26396070674174,12.492405660000001,49999,1268.406306,1268.406306,1.0,37.08832960355122,25.368126120000003,50000,1149.292305,1149.292305,1.0,33.60542407983355,22.9858461,H2D,25.368126120000003,39419.545427583194,7x7_s8,7,2170,8,3000,float +50000,4813.511408,96.27022816,50000,724.144584,724.144584,1.0,15.043998499649968,14.48289168,49999,711.818092,711.818092,1.0,14.787917419640195,14.236361839999999,50000,1218.843676,1218.843676,1.0,25.321300246100922,24.37687352,D2H,24.37687352,41022.487940446925,9x9_s1_uncontended,9,1700,1,3000,float +50000,4491.355637,89.82711274,50000,777.528372,760.738041,1.0220711073918807,16.937826849715574,15.214760819999999,49999,1330.854104,1330.854104,1.0,29.63145677078789,26.61708208,50000,1407.727498,1407.727498,1.0,31.343042318962105,28.154549959999997,D2H,28.154549959999997,35518.237777578746,9x9_s4,9,1700,4,3000,float +50000,4422.456404,88.44912808000001,50000,820.82323,800.129222,1.0258633323605821,18.09241627065681,16.00258444,49999,1341.74831,1341.74831,1.0,30.33943554053857,26.8349662,50000,1416.045933,1416.045933,1.0,32.01944357708585,28.32091866,D2H,28.32091866,35309.58907107712,9x9_s8,9,1700,8,3000,float diff --git a/python/tests/perf/cf/results/A100/README.md b/python/tests/perf/cf/results/A100/README.md new file mode 100644 index 00000000..8c09b560 --- /dev/null +++ b/python/tests/perf/cf/results/A100/README.md @@ -0,0 +1,24 @@ +# A100 probe results (Merlin7) + +NVIDIA A100 on Merlin7 (gmerlin7, a100 partitions), CUDA 12.9.1, gcc 13.3.0, +nsys 2025.1.3. CUDA event trace is off (`--cuda-event-trace=false`) unless the +folder name says `evtrace-on`. Driver and git state of each run are in its +`env.json`. + +| folder | precision | copy-out rounding | frames / batch | configs | +|---|---|---|---|---| +| `2026-10-28_f32_lround_evtrace-on` | f32 | `lround` | 20 000 / 2000 | 3x3, 9x9 at s1, s4 | +| `2026-10-28_f32_lround` | f32 | `lround` | 20 000 / 2000 | 3x3, 9x9 at s1, s4 | +| `2026-10-28_f64_lround` | f64 | `lround` | 20 000 / 2000 | 3x3, 9x9 at s1, s4 | +| `2026-10-29_f32_round_b2000_f20K` | f32 | `round` | 20 000 / 2000 | 3x3 to 9x9 at s1, s4, s8 | +| `2026-10-29_f32_round_b3000_f50K` | f32 | `round` | 50 000 / 3000 | 3x3 to 9x9 at s1, s4, s8 | + +`lround(float)` is emulated in software on the device and made the f32 9x9 +kernel 1.8x slower than f64. It was replaced by `round` in 104e284. The +2026-10-28 runs used local, uncommitted edits to `clusterfinder_kernel.cuh` +(the precision aliases), so `env.json` shows `git_dirty: true` and does not +record the rounding. + +Event trace on and off give the same kernel times; with it on, the idle gaps +between steps are 10-16 % wider. + diff --git a/python/tests/perf/cf/run_probes.py b/python/tests/perf/cf/run_probes.py index f8a819e6..c01897ed 100644 --- a/python/tests/perf/cf/run_probes.py +++ b/python/tests/perf/cf/run_probes.py @@ -33,21 +33,30 @@ HERE = Path(__file__).resolve().parent # The cap sets the D2H bar (the slot is copied whole regardless of fill), so a # probe at a different cap is a different measurement. CONFIGS = [ - (3, 3000, 4, "3x3_s4"), (3, 3000, 1, "3x3_s1_uncontended"), - (9, 1700, 4, "9x9_s4"), + (3, 3000, 4, "3x3_s4"), + (3, 3000, 8, "3x3_s8"), + (5, 2350, 1, "5x5_s1_uncontended"), + (5, 2350, 4, "5x5_s4"), + (5, 2350, 8, "5x5_s8"), + (7, 2170, 1, "7x7_s1_uncontended"), + (7, 2170, 4, "7x7_s4"), + (7, 2170, 8, "7x7_s8"), (9, 1700, 1, "9x9_s1_uncontended"), + (9, 1700, 4, "9x9_s4"), + (9, 1700, 8, "9x9_s8"), ] def run_one(cdim, cap, streams, label, n_frames, batch, outdir) -> dict | None: rep = outdir / f"probe_{label}_cap{cap}" print(f"\n--- {label}: {cdim}x{cdim} cap={cap} streams={streams} N={n_frames} ---") - - prof = [common.NSYS, "profile", "--trace=cuda", "--sample=none", - "--cpuctxsw=none", "--force-overwrite=true", "-o", str(rep), + + prof = [common.NSYS, "profile", *common.nsys_profile_flags(), + "-o", str(rep), sys.executable, str(HERE / "nsys_kernel_probe.py"), str(streams), str(n_frames), str(cdim), str(cap), str(batch)] + p = subprocess.run(prof, capture_output=True, text=True) for line in p.stdout.splitlines(): if line.strip().startswith(("n_streams", "H2D/frame", "wall")): diff --git a/python/tests/perf/common.py b/python/tests/perf/common.py index 0a3cdc39..6c61bd78 100644 --- a/python/tests/perf/common.py +++ b/python/tests/perf/common.py @@ -37,6 +37,19 @@ def _sh(cmd: str, default: str = "unknown") -> str: except Exception: return default +def nsys_profile_flags() -> list[str]: + """`nsys profile` flags shared by every probe. + + nsys 2025.1 traces CUDA events by default, which widens the host gaps + between steps (kernel times are unchanged). Older nsys rejects + --cuda-event-trace, so it is added only when the help text lists it. + """ + flags = ["--trace=cuda", "--sample=none", "--cpuctxsw=none", + "--force-overwrite=true"] + if "--cuda-event-trace" in _sh(f"{NSYS} profile --help 2>&1", default=""): + flags.append("--cuda-event-trace=false") + return flags + def assert_build_fresh(headers: list[str]) -> None: """Abort if any header (repo-relative path) is newer than the extension."""