From da3f56387271e9761b51615416c729f6d94a6831 Mon Sep 17 00:00:00 2001 From: kferjaoui Date: Thu, 10 Sep 2026 14:09:20 +0200 Subject: [PATCH] Add nsys probe scripts for ClusterFinderCUDA Per-engine GPU time, duty cycle and roofline on the campaign configs (3x3 cap 3000, 9x9 cap 1700, 1 and 4 streams, 20k frames). --- python/tests/ClusterFinderCUDA.ipynb | 57 ++++++-- python/tests/perf/README.md | 102 ++++++++++++++ python/tests/perf/common.py | 102 ++++++++++++++ python/tests/perf/gpu_span.py | 68 +++++++++ python/tests/perf/nsys_kernel_probe.py | 47 +++++++ .../perf/results/2026-09-10_f32/env.json | 14 ++ .../perf/results/2026-09-10_f32/probes.csv | 5 + .../results/2026-09-10_f32_ref7177/env.json | 14 ++ .../results/2026-09-10_f32_ref7177/probes.csv | 5 + python/tests/perf/run_probes.py | 132 ++++++++++++++++++ 10 files changed, 535 insertions(+), 11 deletions(-) create mode 100644 python/tests/perf/README.md create mode 100644 python/tests/perf/common.py create mode 100644 python/tests/perf/gpu_span.py create mode 100644 python/tests/perf/nsys_kernel_probe.py create mode 100644 python/tests/perf/results/2026-09-10_f32/env.json create mode 100644 python/tests/perf/results/2026-09-10_f32/probes.csv create mode 100644 python/tests/perf/results/2026-09-10_f32_ref7177/env.json create mode 100644 python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv create mode 100644 python/tests/perf/run_probes.py diff --git a/python/tests/ClusterFinderCUDA.ipynb b/python/tests/ClusterFinderCUDA.ipynb index 6f291d78..e65679b9 100644 --- a/python/tests/ClusterFinderCUDA.ipynb +++ b/python/tests/ClusterFinderCUDA.ipynb @@ -141,7 +141,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Pedestal (1000 frames): 1.23s\n" + "Pedestal (1000 frames): 1.34s\n" ] } ], @@ -175,7 +175,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Reading 100000 frames: 33.56s (2,980 FPS, 32.00 GB at 0.95 GB/s)\n" + "Reading 100000 frames: 82.39s (1,214 FPS, 32.00 GB at 0.39 GB/s)\n" ] } ], @@ -207,14 +207,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████████████████████████████████████████████| 100000/100000 [00:23<00:00, 4249.96it/s]\n" + "100%|██████████████████████████████████████████████████| 100000/100000 [00:23<00:00, 4191.14it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU MT (24 threads): 23.53s 4,249 FPS 233,099,227 clusters (2331.0/frame)\n" + "CPU MT (24 threads): 23.86s 4,190 FPS 233,099,227 clusters (2331.0/frame)\n" ] } ], @@ -309,10 +309,10 @@ "output_type": "stream", "text": [ " s FPS us/frame clusters\n", - "CPU MT (24 thr) 23.53 4,249 235.3 233,099,227\n", - "CUDA batched 3.53 28,325 35.3 233,093,554\n", + "CPU MT (24 thr) 23.86 4,190 238.6 233,099,227\n", + "CUDA batched 3.34 29,926 33.4 233,093,554\n", "\n", - "speedup: 6.67x\n" + "speedup: 7.14x\n" ] } ], @@ -413,10 +413,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "sets", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU 233,099,227 clusters\n", + "CUDA 233,093,554 clusters net -5,673 (0.0024%)\n", + "\n", + "exact-position mismatches CPU-only 135,742 CUDA-only 130,069\n", + " of which shifted by 1 px 104,347 104,198\n", + " genuine add / drop 31,395 25,871\n", + "\n", + "frames touched by a genuine difference: 41,569 of 100,000\n", + "[40.6s]\n" + ] + } + ], "source": [ "def cluster_keys(vectors):\n", " \"\"\"One int64 key per cluster: frame * rows*cols + y * cols + x.\n", @@ -501,10 +517,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "drift", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": + "iVBORw0KGgoAAAANSUhEUgAAA3kAAAFeCAYAAAA4xtZyAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbExJREFUeJzt3Qd4U1X/B/BvS9l7770REQEZArK38LpQkKk4APmzRUBlyVIBeUEBURQVFAfoCwoKspQpewiKInvvvZv/8z3X2yZpWtI2bZP0+3metMnNyc3Jzcm993fPCnE4HA6IiIiIiIhIUAhN6gyIiIiIiIiI7yjIExERERERCSIK8kRERERERIKIgjwREREREZEgoiBPREREREQkiCjIExERERERCSIK8kRERERERIKIgjwREREREZEgoiBPREREREQkiCjIk6Cyfft2PPPMMyhatCjSpEmDDBkyoFKlSnjrrbdw9uzZiHR169Y1t4QyZcoUzJw5M8HWH4h8tU1CQkIwbNiwiMdcJ5ft37/fJd1rr72GQoUKISwsDFmyZDHLbt68ia5duyJv3rxIkSIFKlasGO/8SOBzL1Peunr1qnndihUrojwXXbkUV59++inatGmD0qVLIzQ0FEWKFIl2E12+fBm9e/dGvnz5zP6dv985c+Z4TLt582Y0bNjQHAP4+3/sscfwzz//eL35Y/P6yZMno0yZMkidOrU59gwfPhy3bt2Kku7kyZPo3LkzcuTIgXTp0qFGjRpYunSp13nytE9LTn799VezjQ8cOBCxLLp9OssRt3VCiu9vnK/j6z3doivXnuzcuROtW7dGzpw5zfbhZ+/evbtLGi6L7r34W3L23HPPoXz58qaMpU2bFqVKlcLLL7+M06dPu6SbMWMG8ufPjytXrsTp80vCC0uE9xBJFB988IHZsfFkgTukcuXKmQPtxo0bMW3aNKxduxbffvttogU0PJAn9EEmkCTUNmnRooX5bnmQt/3vf//DqFGj8Oqrr6JZs2bmwEdTp07F+++/b07KKleubE7gROKKQR5P6Mn9opGncilRffbZZzh+/DiqVq2K8PBwj8GRjYHWhg0bMHbsWHPi+fnnn6Nt27bmdU8//XREuj/++MN8Hzzh/+qrr3D9+nUMGTIEtWvXxtatW83JcExi83ruZ15//XUMHDgQjRs3NvljMHbkyBFMnz49It2NGzfQoEEDnD9/Hv/973+RK1cuvPfee2jatCl+/vln1KlTJ8Y8RbdPSy4cDocJ8J9//nkULlw4Ynl0+3Qe6zNlyoRA8H//938u5ZdKlizp1WuXL19u9jUsmzzP4TH24MGD2LJli0s6bg+WQWdM99RTT+HRRx91Wc6g7YUXXkCJEiVMAMhzKJa9hQsXmvWmSpXKpOvUqRPefPNNcxHd3g+Kn3GIBIE1a9Y4UqRI4WjatKnj+vXrUZ6/ceOG43//+1/E4zp16phbQrnnnnt8vv7w8HDH1atXHYHKV9uEu62hQ4fGmGbkyJEm3YkTJ1yWP/fcc460adM6fOnKlSs+XZ8kPm/KlCenTp2K82uTi5s3bzpu3boV7fN37tyJuN+iRQtH4cKFPab74YcfzLb+/PPPXZY3atTIkS9fPsft27cjlrVu3dqRI0cOx4ULFyKW7d+/35EyZUrHgAED7ppnb19/+vRpR5o0aRwvvPCCy+tHjRrlCAkJcfz+++8Ry9577z2Tfx6rbNwu5cqVc1StWvWueYpun+ZJIB8norNw4ULz+f/4448E36d76+OPPzZ52rdvX5xez9fx9W+//Xacjz158+Y1vxueH8TWsGHDzPv//PPPd007ZcoUk3bp0qUuy8eNG+fInDmzjoN+SkGeBIWHH37YERYW5jh48KBX6d2DvOXLl5sdGP972glzZ27bu3ev46mnnjI711SpUjly5crlqF+/vmPLli3meZ6k8DXON+cTF5449OvXz1GkSBFz0sATlF69ejkuX77s8t583UsvveSYOnWqo0yZMiYt70eHwe3w4cNN2tSpUzuyZcvmqFu3rmP16tURaa5du+YYOHCgy3t3797dce7cuSjv7enElZ+jU6dOUQ5yy5Ytc3Tt2tWRPXt2876PPvqo48iRIy6vi2mbeMLtxAM415c+fXpHkyZNHH/++WeUvLkfaD29F9O7L3P+XnmA5EnYfffdZ07asmTJ4nj88cfNd+2MZYbB6sqVKx01atQwJxcsC3H5Xj/99FPzXXEdFSpUcCxYsCDKNti9e7ejTZs2poyxrBUsWNDRoUMHlwsZx44dMyeZ+fPnN+/L9+fBO6YTa9ucOXPMSXKePHnM52Z+XnnllSh5pnXr1pnfGb8Plq9ixYqZzxeb/NrfgzcnS/weefLC7VKxYsWI/Nnbia/h43Tp0jkeeOABx4YNG6J8V54uKrD8upc99zJ18uRJR7du3Rxly5Y1ZS9nzpyOevXqOX755Zco+wb3m/378PSZ7PLz22+/OWrVqmW++6JFizrGjBnjEuzQzp07zXfDNAw4+Dv9/vvvPe6nPPn111/NfilDhgxmHSyvfL1t69atZl0ffvhhtCfUzhfG9uzZ42jbtq3ZFvxuue3fffddl9fZ+1GW7b59+5rfAIMdlgtvxBTkcV/Az+Jerhn08T3t/Ryf5+d98cUXo6yjcePGjpIlS8aYh9i8ftasWea9165d65Lu6NGjZjmDPVvDhg0dpUuXjrLO0aNHm7SHDx+ONk/R7dPs57jd5s6da34n/G3yN0z8fmrXrm2+M/5Oypcv73jzzTdN4O3MLpcMQFlO+Fvjej/66CPzPMvN/fffb7YL17Fo0aIoefSmfLCMv/HGG45SpUqZ92BwcO+99zomTpzouJuWLVua37mzmPbp7scqu2yyvAwePNgcvzNmzOho0KBBlMBx8eLFjlatWpl9Krdn8eLFzT6WF3X8KcibOXOmef2KFSti/Voe87iNuB/3JkD8+uuvzXvx2OeMxx/+xmfMmBHrPEjCU3NNCXh37tzBsmXLTFONggULJvj7NW/e3LwnmyiwfwTbqa9Zs8Y0w7GbRTzxxBPInDmzaaJIdtMaNu9is5zDhw9j8ODBqFChAn7//XfTFGjHjh2m2Q7byNu+++470w+Bz+fJk8c08fHk9u3bpgkP07JJS/369c2ydevWmSYZDz74oGnu8sgjj5g+IIMGDTLNO9iHcejQoaZZGW9xbQLENvxsMsLmU4cOHTLNZdu3b2++l7ttE0/svHK78rM/8MADWL16tfmMd8P3YjMo9hf48ccfzXsWKFDANIt64403TPMWO1/Fixc3/1988UXTv6Jnz56m+Qn7b44YMcJst23btiF37twR6z927Jj5bAMGDMDo0aNNP6LYfq8//PCDadbF92DzIpYlNpn5888/UaxYMZOG71urVi3T/Ibp2HyH7z1//nzTD4Xbz27mxjzwvfh5+D2OHDnS9Pf4+OOPY9xWf/31lynPLDPp06c3zdT4+X/77beIbUQ//fQTWrZsibJly2LChAmm3HP9ixcvjkjjTX5ji+tkWWUTNX6PbBLEJntcxnLM7c/t+sorr+Dhhx/Gvn37TB+S+LL77/K3wd8d+4KxXLEJH9+X/9kMk+WL5apLly7mN0B3awrI76xdu3bo16+fWT/Xy8/DfmYdO3Y0abjdWJ74nbA5Gn/3X3zxBXr06OFV/leuXIlGjRqZcsjfAbc9f3f8DrkeNtG67777cP/995sywvw742+B78myQbt27TK/BX7v48ePN9uEZYK/F+7/+Dmc8fOwvxmbj7FsRrffig32O2L5Y380Z/yM9vPM4969e3Ht2rWI5e5plyxZYppfuvdDssXm9XxPuvfee13SsWzwd2A/b+eP+1xP6yTuL9i/KTb7NOf+g7t37zbNRNknkOXG/ixsBshlbGLH3xOb3fF3/tFHH0Upl+zPzv0a183mj88++6zZn3/zzTdmv8b35W+b+2b2T2SZjU354H6OfViZz4ceesg0zWVe7GNndLj/4D6UzRqdcV8X3T49OvwcNWvWxIcffoiLFy+afQd/F9x+7NNnbzeWX/6m+Zm5r+N+j/s37s9TpkwJX2LzY+aLZZtjCPA7aNWq1V1f98svv5j/PB9h3rjf5nfPfRK/B/v78YTbk30beaxwPjY54zkEm3iyiTKbJPM9uO2c8btmf1Qe01hexM8kQiApkqCOHz9urjCxBsFbca3JY/McPr7blcfomibyin1oaGiUWodvvvnGrJdX0G18zCudZ8+evevn4ZVzpv/ggw+iTfPjjz+aNG+99ZbL8i+//NIsnz59epxr8ljL4IzvweW8yheX5pq8UszX//e//3VZzivjd6vJc64xcr/yyryzZsYZr8Iz7fjx412WHzp0yFy5dm6exfx7arIS2+81d+7cjosXL7qUYb6e67GxFoY1iqxVig5rG1i7ceDAgShNaPg+zs3F7oZXc1mLwSu1fO22bdsinuOVbN5YExwdb/Ib25o8bn/nGg679olX4Z2byX733Xdm+fz5831Sk+eOTQG5bXjVn7XU3jTXjK4mj8vWr1/vkpZN9lhTbXv55ZejNPcjpvGmJq969eqmNvXSpUsun4G1MAUKFIi4cj9p0iSzPtaQ27i/Ye0Fa6Wd35evc26+SD169DA1MvY+yt6PPvTQQ464iKkmjzVoztvIvdaMNWLEGj0+/uKLL6KtNeNrohOb1z///PNmW3nC2irW/NlYy+6pdpC1Z56aobqLbp/G7cWuCs7foSesRWMZ5rGC6Z2PK3a53LhxY8SyM2fOmHT8DTq3yrB/gyw7sS0fbAnA2sbY4u+F78mWB+487dNjqslr3ry5S7qvvvrKY22s+36R+1j32u341uSxHLEMMQ+seZ89e7b57d7tWO6+P+B+l8cptqiZNm2aaVFTokSJGJtQsgUKv9/oapDt46J943ZzPmY5a9eunTmmif/R6JoisZAtWzZzpfDtt982V/bYCZmd/r31/fffm1Gr2KGfV8nsW5MmTczVNPdR+lgjlzVr1ojHvGLn/Dr7vRctWmSuLMd0Jc2+0uk+8AlH5eLVv9iM8ubO/aqjfXXaeRS02OCVWWKNhzP3zum+wO+E2561c87bllcoWdvh/p3w++D3Ep/vtV69esiYMWPEY9YUsrbD3l6sGWRtzJNPPhljzRDfl+viFVvn97VrPLmOmPBqPLcpPyuvYvMKtT0ABK9s0549e8yVbdb2RFf74W1+Y4vb07l2gzU5xJo0jk7ovjyu5c0T1kLxqjo/M6+wc9vwN2Jvl7jitmbtq/vvxTnv3JYsTxw8yhkHGbkbDpqwfv16U3PuPLAQv98OHTqY2mbWGNu/L9byOY96y5o+Xr1nrQ6x1oqfmzXN3ObO5Yw1fXyeLQacPf7440gI0dU4eHouNmnj816JmafosPxwIBp3PD5x35w9e/aI3zdri3kc4e/avfaRrWGcj3XcJ/E36Fwj5P5bi035YLlnbSIHSGNNH2vSvHH06FHz3xc1wt4cqzgKKkfsZMsg+7dvD/YSl9+/8zbhzbquZG1zDs7DYzBrybgvZu0ca9g5kA/TxsQ+/rNmni0weCxgqxTW+P7999+mZU10LRXYSog1ftHVHrN2mq1NuC/iQEEsS2wdwH29O34v3GZ3y68kPgV5EvDs4ajZVCuh8UDMAxpP3tn0hCeBPKlls5RLly7d9fUnTpwwTSR50HC+8YSfO373IYrdR+ZjgOn8OjadoVOnTpkDMZtGRefMmTPmgOV+Es7PxBNPPh9XPIlwZjfNY7OnuLDz6r5e5tPX+J1w2zPQcv9eeHJyt+8kLt+r++eyt5m9vc6dO2dOxJybZEWX9wULFkR533vuucc87/6+ztgEkc3HGBCwyQ4DUR7U582bZ56388KyRTHlxdv8xhZPNJ3Zo7pFt5wnlL7ACzjdunVDtWrVMHfuXFMOuG14UhTXMu3td2+Xf+cmwjZPyzx9FyxznsqpfbJu/9a5HXnSy2kM+P0RAz6ejNtliGl58sbme+7lzG7O6c1vJL643Tzto+ymtXaZsLdvdGm5v4tp+oHYvJ5pWeY8nfgyrXM59Tb/ceFpe7OZPn/fHOWTJ+lsys8yzGaf5F6OPb0/f1d3+63FpnywGe+4cePM74kXorhNOOIoR2+MiZ3X6C4y+fJYxcCJo6RyP8hmkzzesxmkHajG9vfPpp7u2yWmi298nkEbtyub03vzWXg+4sy+uMhmvJ7MmjXLXMixm5h7wgu/VapUMc1qeX7DJsM8VnAkU3f8XrjP8dX+V3xHffIk4PEKJQ8UrM3iVeq4nGjaBw/3IYY9nSTzih6vlBGvhnKIbfYzYL8BXv2/W0DKPkPu/SGcn4/p6i5P6J3zaJ+0MXBbtWqVOUBFF+jxgMCDMU/anQM97pzZH4P93pwPfO7bguITCMaGnVe+n/NBmfn0NW5zbmd7DiZ37ss8XXGP7fd6NzyxYrlmeb5b3nklmv1sPImpTwZrdnmFnMGd8/Dt7v1j7LISU168za/z78x5u8YUjMYV3+vChQtRlnvzXjwJYm0h+8M58+ZCji+wzDOAd+dN+WdNM/cB7NcXXY2Ic3lkjd3XX39t+pqxTxUDAefPzfXZtYAvvfSSx/dkny9f1UpFhzULrGXkfsG5Xx77SBFrPu0LYfwt2sudcZk9LHx0YvN6uy8el/OCgPP3xHJm58lOG906nfMfF562N2tqWKvLYMV5ygH2r/Kl2JQPfm99+/Y1N+5n2C+MfdEYlLDvn3PtvDO7vDrPdZtQ2HeStY282MEpAmysGYsL7oP5m3LGaZ5iYtf0xXTRlrjvj2k+vehez/MXXjBiP2ZvMeDj+txrgO3vhftzTUnkf1STJ0GBVwi5Y+QcOgy23LGDNwOk6NgT8LI2xhkHjYgJm8iwEzkP4M5XzdyvzNu4U2XTN57EcafpfotpImDi+zint0/ieVWUV9FimmycgbB9AuuMNRU8GbCft7eH+7ZgUMDan7iKbpt4wmYnNHv2bJfl0TU/iQ9+Jyw7vOLt6TtxH1QhunXE53t1x5NMBl48+Y4pKOH78qSEJ6ae3jemIM8+MXQPYt2v1LKMc/0MYD0F/rHJb3S/s5h+m3HF9+IJiXOeedGAg/ncDbeN+3ZhnjnQgy9rrKPDbcnvlQNaOPNmgmRegWfAwZN753zxAhB/+7wI5ty0j7UWbLLFAVh4YwDj3CyUJ978PbK5Fk8qPZUzT7WTvsbmgNz/cH/l7JNPPjHl3A6yGEhwIA1+fuegnDVbbAbOgXtiEpvXs2aX28t9v2tPks0BSpzzz0FGWBtiY8DK74R5j+m3Gheeft/cz3E+WV+Ka/lgbSibFDMwZJAQ04TidhNR7mMTmrf7RW+x5tN9ezg31fd0rvLll1+awJYXFGLCMsX88gK3Mz7md129evUor2GtKfdlDGDdBzGKCWsfuQ/xlCc2+3dvWi7+QTV5EhQ4EhavPrOtP/sVsKkVmxtxh8mDD9u980opD96esBlgw4YNMWbMGHNlklc+2UzDbrpm486RI9yxDT1HD+QOnMEPl7MNvY2BAU/IuLPmaIk8EeAyjmLIkxQ2gejTp485KHLHyRMIjlTIEfecrwh7iydlPEFjPwL2t+FBl+vlCQUPkG3atDHt6XnFlKOJsS8ER8myR9dkHwBeibXxPkfT4oiNPOHkyea7775rRhqLq+i2iSc88eQ2YnMZBqA8MHJ0TU6c7GvcDpz4lTUaPADyfXmizJoQ1o4yjyxPMUmI79UezY2vY9niwZW1O7zwwBMOniiwuS5rYDiyHZvU8Aoxg32eMHHiWtYsR1ezzdewrLPMsAywmRCDal7FdscmXvzt8KSBn481Pvxs7FdjB+Le5JfNt1jrx/59zDtPMnhCzKv4vsYyzPdlX0te/GGAxybW3kyQzOCZo/Zxu7D88zfF/LJGwrnfCT8T9xWcqJoXSfjZeHIW26DeU3liUM2LN3xfXnXnBQ4GCd5c4ed+jL937gf69+9v9lMcXZOBI2vDnGt+WAvDflr8/rhtGMS4/87Z3I/fLZv/8bfAz8cAiLUbDNCdR2KNLe5b7GCWNWBs+sjRHIknjvbJI7cFPxPfn/svli9+Fo42yUDJHhmROAorWybwe2RZtCcz53fD36Iz1tjyBNauPYnN6/l98yIf95W8b0+GzpYdbArnfOLL/tL8HfHYwdEU2Y+J3wnLFmu0fI3bit87jw3cj/Iz8BjJ5ry+5m354D6Ex2Huz9lCgP3gJk6caH5DMU3+zX0YjxlsMsn9XELiSJG8qMXvnWWC3ys/A/ezvsYaTZ6j8BjEcxDuB9nslbWtPJ47l2m2uODvmfskli87rwySWY64L+JvhBe2WCZ5TGcfaXd2KyT3EXWd+3nzQgCbcfN7Yf54XOT3xN+cexNPHufYnDW69UkSS+qRX0R8iSN/cUStQoUKmbl6OOoW5/cZMmSIy6h/nkbe40iQTzzxhJkHjKNatm/f3ow25jy6Jiei7dy5s5kDiOvmyIac4+ydd95xmYyXE+dyZDXOw+M+JxznIHvttdfMnEnMoz1XUJ8+fcwoi+7zqXmLIx/yc3IUOq6XI2xxxEPnyXeZhnMoMT8c7Y2jFHI+MPd58jh5PEfr4jxnHF2N24rbNrrRNd1HlfQ0WmlM28ST8+fPO5599lkzchjneOKcYZzPyNeja9o4J1S1atXM8/zMHE2yY8eOLiPO2fNJeRLf79V929KuXbvMxMz8LrlOlmuWP+d58vgZe/bsaeZb43fK8lu5cmXHq6++6nG+O2f2vFjcvpzfinORbd68OcrckPZoa82aNTOfy547ip8ttvnlHHEPPvig2c6ch4rfFedqi26ePHeetl9080198sknZq47jvDHESw5kqw3o2uy/Pfv39/kj6+tVKmSGcHT02s5kTD3Mdwm3s6T587TejlPHudW4/vzO+3SpYv5PO4jn95tnjy7PHPUPk9zMdpznNmj6C1ZssRjGn4O/h7t+RhZXvg9cpJu998959TyVnRzWHoatZSjhbKsc15Hli/uez2Ngkn83XI0VJbtTJkyOR555BHH33//HSUdfytcX1xfTxwFmKNp2mWe+Xafi464H+A+hd8nv1d+J9Ft79iMrunpd0L8vu25P/m9cdRWe+Ri531zdOUytr/Bu5UPjmDMZZz30d5WLNc8NtzN66+/7siaNavLviQuo2u6l01Pc+FyP8bjDY9VfE/u0zgHrzfHntjg3HJVq1Y15YHz/PK9OGLmTz/95PG75Htx9ExnPO8YO3asGU0zpmM6Xb161ey/Yxr9lnNa8jyI24/lxp6flGWHI66640jTzNemTZvitA0kYYXwT1IHmiIiIhIz1jiz9oq1kvYAGBJ3rG1iTQ1rKaLrTyb+gf1JWZPOQYI4MElyw9pY/vY5GIsvBqDxZYsJNtdkSxvxP2quKSIi4mfYTJP9tNhMjX3R2IyKEzizKZYCPN/gcPXsj8jmvOLf+FtgM2YOMsUmr3drshxs2B+UzYL9KcBjH0l2v4hPc21JWKrJExER8TPsV8f+ihyxlP0A2WeJ/WF69eqVIKNXigRCzSv7jrJ/I+ewk6QPPFmzyBYG4p8U5ImIiIiIiASR5FXfLSIiIiIiEuQU5ImIiIiIiAQRBXkiIiIiIiJBRKNrJgBODsnhfjk5pTrIi4iIiIiIL3D2Ow5ExFFnYxppVkFeAmCAp5GfREREREQkIRw6dAgFChSI9nkFeQmANXj2xs+UKRP8oWbx3LlzyJo1a7KbW0Z8Q2VIVH4kqWj/IypDkpTC/ew8+uLFi6YyyY43oqMgLwHYTTQZ4PlLkMd5lpgXfyicEnhUhkTlR7T/kUClY5gEY/m5W5cw/8mpiIiIiIiIxJuCPBERERERkSCiIE9ERERERCSIKMgTEREREREJIgryREREREREgoiCvBhMmTIFRYsWRZo0aVC5cmX8+uuviffNiIiIiIiIxIGCvGh8+eWX6N27N1599VVs2bIFtWvXRrNmzXDw4MG4bGcREREREZFEoSAvGhMmTECXLl3w3HPPoWzZspg4caKZeHDq1KmJ882IiIiIiIjEgYI8D27evIlNmzahcePGLsv5eM2aNQg4l44j5cFVwOENwMVjwLkDwPGdwI5vgBuXkzp3IiIiIiLiQ2G+XFmwOH36NO7cuYPcuXO7LOfj48ePR0l/48YNc7NdvHjR/A8PDze3pOb4ZwUyz+8WY5rwJz4GyrQEQlMkWr4kcLAcOxwOvyjPEnhUfkTlR7QPkkAV7mfnQN7mQ0FeDEJCQlwe8wt2X0ZjxozB8OHDoyyfM2cO0qZNiyR36yqQtT9ShN9EyO3r/CQIuXYGSJkeITesgBTffAfgOzhSZ8TtXPfBEZY6qXMtfoRl//bt2wgLC/P4GxBR+RHtf8Rf6RgmwVR+rl275lW6EAdzLlGaa6ZLlw5ff/01Hn300YjlvXr1wtatW7Fy5cq71uSx/965c+eQKVMmv4j4mZesWbMiNNStha4jHFj7LkJ/Huq6OGcZ4MxeOLqtAbKXSNwMi9+JsQyJqPyI9j/ix3QMk2AqP4wzmJcLFy7EGGeoJs+DVKlSmSkTlixZ4hLk8fF//vOfKOlTp05tbu5YEPyhMBCvPHjOTyhQq7d1Y7z/y9vA798i5OQu63XvPQA8vxzIXylJ8i3+I/oyJKLyI9r/iH/TMUyCpfx4mwcFedHo27cvOnTogCpVqqBGjRqYPn26mT6ha9euCFqsgq4zwLrduQXMehzYtxL4oB6QNivQfh6Q734rnYiIiIiI+CUFedF46qmncObMGYwYMQLHjh1D+fLlsXDhQhQuXBjJQoqUQKf5wDfPAjvnAtfOWcEeFakNdJzPSwlJnUsREREREXGjs/QYdO/eHfv37zf97TilwkMPPYRk54mPgKHngQ4cmOVf+38FRmQFVv/XauIpIiIiIiJ+QzV5cndsnlm8HjDsAnDjEjCmgLV8yRDrZkuX/d8mnRW1VUVEREREkohq8iR2Ume0gr18HgZiuXoGmF4HeLMocPumtqyIiIiISBJQTZ7EzQvLgRuXgY0zgPQ5gR3fAHuXWs9dOwuMzAm8sMIaqEVERERERBKNgjyJu9QZgJq9rPsVn7b+Xz4JjCtp3Z9eNzJty/8C93fUYC0iIiIiIglMQZ74VoZc1kAtw7O4Ll/Qy7rZGgwF7u8AZMipb0DEG/tXAzObR11epQvQfJwuoIiIiEgEBXmSMAO1sN8eR948vgN4v3bUNEuHWzdKlQFoPxcoWA24dRX4ZyVQ+EEgTWbrec7Zd+oPK4DMmEffmAQ/lvmxhazfw92wyTRvVHcQUKpp5OBH4eHWSLi2Ag8AJRoCoWHAQ/2tZfydau5LERGRoBLicGgMfF+7ePEiMmfOjAsXLiBTpkxIauHh4Th79iyyZcuG0KSc2+7Obetkcvd84OvO8VsXT2RbTQbCUkcGgxL8ZSg54O/kjeyJ/76dFgBFE2aaGJUfUfmRpKR9kART+fE2zlBNniSeFP8Wt3setW43rwJn9wJftAUuHIrduvb8GNn3L2024NmfgJylgIvHgAllXNMWqws0HgV8/hTQ7msgdzkffSARH/vtA2DhvzVs7tp8AZTx0FyTI9l+2R7466fo18vBka6civm9P2lp/e//t5pRi4iIBDjV5CUA1eTF0ZXTLJJA+mhqMY5uAdZNtZqw7V4An7vvaWDb55GPK3UCHn4HCAkFhmcFUqYFem0H0mYBUqREcuJvV7GCwt8/Azu/BTLmBn4dH/X56i8BTUfHfr23rgN/LbaaZBapBaSJ5irfpRNAqnTAjq+B7/tEv7523wDF6wOhKRBXKj8SHyo/El8qQxJM5cfbOENBXhJu/ORaOH3q8ilgXImoy7OXBLIWtk6kM+YDLh317fuyZpA1hDyB3jDDqpnMex9w7RyQLhvwTRdg5zeur3n9TGRtZoCJVxlii3BHOHB8O5D7XmsbLBpo9a+s1RsBYduXwLcvAH12AZnzA9cvAse2Ace2Ag88Z10A8Mata8C+X4HPW8ecrtW7QKUOSFTOI+N6q/wTQLO3or8wkxz2QZLgVH5EZUiSUrifHcMU5AXAxk+uhTPBsNnazctWkBUdzu039zngsenArMeAwxuQ6Oq8AtQbjEDiVRni9uf8iLacZYFTu2P3Ri//ExkwhN+JV+1RnDEo/eN7qwlkbFTuDDR7GwhLFTl4CmvT2A/1nxXAp/+J+fWPTgfubZ30o2Se/AP4eRiwZ1HsXle6OdDmc4+DuCSbfZAkCJUfURmSpBTuZ8cwBXkBsPGTa+EMWDz5Z1PRRQOALbO8e82LvwC/vO25eWn2EkD3dVbTz+1fA/Oes5bf8xiQszSwYoz1uP5rwEMvR75uRhPg0Dpg0BEg/DawfBTw23TP75/rHqD7GiR4GeKIqJ+2QoJ6YQWQt6IVRDAA5O3aWe9HXOVIk9ymGz4Eitez+m/W7geUbAhcPQts/tQK+hngJYZuawOjfyjL/Y5vgB/68QFw42LM6TkgUoWnrEGR/qV9kMSHyo/El8qQBFP5UZAXABs/uRbOoMW+UCnTWM3x8t1vTRbvfrL8XXfXfn+JjYPUMDCKSb1XgToDIvN86k+EXzqOs5nLI1um9Aid9qBVa9NoRGRt24gYak+fXwbkr2wFWSd2WDV8rO3aOc96bcr0wMaPgD9/iP/nazsH+KINfIZNctvNjWxmu/g1YM1kK+Dkd8zaum9fBHbO9W59DYZYgWUwObje2i6Hf4v6XM8tQLZi2gdJvOgYJvGlMiTBVH4U5AXAxk+uhTPZWz4GWDn27puhQhtg+xzvN1fhmkCpJkC1rlbfL46WyH5wiWno+fjNucbpA1hb+scPQKHqwKR/53tLbE9+BpSLZc0kA2L2yVw1ASjzMJCtuFXDenQzcH97q99mMGMQPyZ/1Ln9SjVFeJsvtA+SeBQtHcMkflSGJJjKj4K8ANj4ybVwilNQ8NcSayCPrEWtkQ49Of2XqU0zE8tzhFHWVrHZ3LopVjPDhkOj36ScpuLQeuCzR1yXDzxoTbZtazEeuHjU8yiP3hp4KPqRHH3hzF5gciXrftOxVjB79Qww7wVg71LXtFkKA+cPRA4O0mSUNfhLitSug4TwOzj7D7D9K2ugk8wFEi7/yQlreN1qd69W6oo0tbohNFuRJMuWBCYdw0RlSLQPiqQgLwkpyJOAdfsGsGs+sP8Xq48a9d6J8POHEDLrUTi6rkFojuKR6RkgXThi1SLqAoK4X2D430vA7/Oibhc2gS3RwAqyVW7kLhTkSXypDEkwlR8FeQGw8ZNr4ZTAozIkcXb7Jhxzn0PI7v9FnyZLIeCl37yfikKSFe1/RGVItA+KfZyhM34REUk4YangaD0Tp3vshaPG/3lOc/4gMCoP8NsHwLl/m9mKiIhInAXmzMwiIhJwHI1GIKTJyMgFnFR+/v8Bu76zHi/sH/lclS7AwxMSP5MiIiJBQDV5IiKSNDhQz5OfAMMuAAWruT63cQYwLLPVb09ERERiRUGeiIgkvS6LrWDvhZXWFBS24VmAa+eTMmciIiIBR0GeiIj4j3wVgTazgeeWRS57szAwpqAZxEVERETuTkGeiIj4nwKVgcHHIh/fuAiMzKnmmyIiIl5QkCciIv4pVTqrCef/bXZtvrlldlLmSkRExO8pyBMREf+WvTjQfX3k4/91ByZVSsociYiI+DUFeSIi4v9ylQFe3hv5+Oxe4M0iwO0bSZkrERERv6QgT0REAkP6HFbzzdr/zqd37RwwMpcmUBcREXGjIE9ERAJLg9etQVnS5bAe/7cCcP1CUudKRETEbyjIExGRwByUpf9fkY/HFgLeq56UORIREfEbCvJERCQwhYYCQ88DabJYj0/tBrZ9mdS5EhERSXIK8kREJHCFhFgDslTqZD3+9gXgxO9JnSsREZEkpSBPREQCW4owoNUkoFg96/HUB4G9y5I6VyIiIkkmYIK8UaNG4cEHH0S6dOmQJcu/TXPcHDx4EC1btkT69OmRI0cO9OzZEzdv3nRJs2PHDtSpUwdp06ZF/vz5MWLECDgcDpc0K1euROXKlZEmTRoUK1YM06ZNS9DPJiIiPtDh28j7nz0KHNuuzSoiIslSwAR5DNZat26Nbt26eXz+zp07aNGiBa5cuYJVq1Zhzpw5mDt3Lvr16xeR5uLFi2jUqBHy5cuHDRs2YPLkyRg3bhwmTJgQkWbfvn1o3rw5ateujS1btmDw4MEmWOS6RETEz5tuvn468vH7tYGD65IyRyIiIkkizNuErPHyJHPmzChdujQaN26MUHaCTyDDhw83/2fOnOnx+cWLF2PXrl04dOiQCeJo/Pjx6Ny5s6kFzJQpE2bPno3r16+bdaROnRrly5fHnj17TJDXt29fhISEmFq7QoUKYeLEiWYdZcuWxcaNG00w+PjjjyfY5xMRER9IkdKaS2/ivcD5g8BHTYC6g4C6A7V5RUQk2fA6yPv2W6dmME7Onz+PI0eO4J577sFPP/2EXLlyISmsXbvWBG12gEdNmjTBjRs3sGnTJtSrV8+kYVNNBnjOaQYNGoT9+/ejaNGiJg0DVmdMM2PGDNy6dQspU6aM8t58D96cawwpPDzc3JIa88Amqf6QFwlMKkMScOWn5zaEvFkIITcuASvGILx4QyB/pcR7f/EZ7X9EZUiSUrifnUd7mw+vgzw2XYzOsWPH8PTTT5umjR9++CGSwvHjx5E7d26XZVmzZkWqVKnMc3aaIkWKuKSxX8PnGOR5Wg8f3759G6dPn0bevHmjvPeYMWMiahqdnTt3zrzOHwrDpUuXTAFNyNpWCV4qQxKQ5ef5rcj6SR2kuHQYoTMa4HS3XUCKyIt8Ehi0/xGVIdE+KBKPpz4N8mLCwGfkyJHo0KFDrF43bNgwj8GRM/adq1KlilfrY3NLdzypcF7unsYedCW2aZyxJpDNPZ1r8goWLGiCTDYT9YcDJPPO/CjIE5UhSVb7oJ6bgFHWhbscU8sh/PWzVt89CRg6honKkGgfFCksLCzxgjziSJUnT56M1Wt69OiBNm3axJjGveYtOnny5MH69euj1KSxiaVdM8c0dq2ezc7z3dJwg2bPnt3je7P5p3MTUBtPZvwlqOIJlj/lRwKPypAEZPkJTQO8tAF47wHr4RvZrD57ElC0/xGVIdE+yOLtcdRnQd62bdu8DshsnOaAN1+oUaOGGWCFTUftJpUcjIXBF6dDsNOwSSlH6mQzTjsN+/HZeWeaBQsWuKybaVib6Kk/noiI+LmcpYAO3wGfPWI9HpYZePUEkDJNUudMREQkQXh9SZVNED3dOJrlvHnz0Lt3b7Rt2zZhcvnvHHhbt241/zldAu/zdvnyZfM8B0spV66caTLK/oNLly5F//798fzzz0c0mWS/QQZ9HHFz586dZjCZ0aNHR4ysSV27dsWBAwfMst27d+Ojjz4yg65wXSIiEqCK1wPqvx75+JOHkzI3IiIiCSrE4T4TeAxVg9H1SePyF1980Uw7kFC1XQzMPvnkkyjLly9fjrp165r7DAC7d++OZcuWmcnOGdRx6gPnppScDP2ll17Cb7/9ZvqHMKgbMmSIy2fjZOh9+vTB77//bmr5XnnlFZPOWwx+ObXEhQsX/KZP3tmzZ5EtWzY11xSVIUne+6A/FwFfOHUTKNUUePrLpMyRBFL5kYCkMiTBVH68jTO8DvIY+HjClZcsWRIZMmSIe26DjII8CTb+toOTwOJ35efAGuDjZpGP6wwE6g1KyhxJIJUfCTgqQxJM5cfbOMPrPnmcX05ERCTgFX7QGnxl13zgqw7AyrHA6v8Crx7TyJsiIhIUvA5Hr169apo5chRNTnjOppCcN05ERCQglWsFpMli3b99DRieBfhjYVLnSkREJPGCvKFDh2LmzJlo0aKFmfZgyZIl6NatW/xzICIiklQGHgCe/jry8Zy2wKp39H2IiEhA87q5JkfQ5CiT9rx27du3R82aNc1IlylSpEjIPIqIiCScUo2B104CI3NZj38eBqRIBdR4SVtdRESCuyaPUyXUrl074nHVqlXNBOFHjx5NqLyJiIgkjrDUVj+90i2sxz8NBua9oK0vIiLBHeSxxs6eQNzGIO/27dsJkS8REZHE12Z25P3tXwJn/9G3ICIiwdtckzMtcK465znnrl+/buaPS58+vUuzThERkYDEOVNZo/dOeeDCIWDS/dZjERGRYAzyOnbsGGUydPbLExERCTp9dgLDMlv3l48G6g1O6hyJiIj4PsjjyJoiIiLJRuuZwNedgZVvAnUHaQ49EREJzj5527dvx7Vr1zzOocfnOCO8iIhIULjnUSD032uhnEPvogYaExGRIAvyPvvsMzz77LNRBl8h9tPjc59//rmv8yciIpJ0Xvwl8v6EslYTzts39Y2IiEhwBHmcI69///4e58TjsgEDBmD69Om+zp+IiEjSyX0P8MRHrstG5gTC7yRVjkRERHwX5P3555+oXr16tM8/8MAD2L17t7erExERCQzlH7dG2CxeP3LZiGzA9YtJmSsREZH4B3lXrlzBxYvRH9AuXbpk+uaJiIgEpQ7fAq/sj3w8tmBS5kZERCT+QV7JkiWxZs2aaJ9ftWqVSSMiIhK00mYF+u2JfLx7QVLmRkREJH5B3tNPP43XXnvNjKLpbtu2bRgyZIhJIyIiEtQy5gYeetm6/2V74OiWpM6RiIhI3ObJ69OnDxYtWoTKlSujYcOGKFOmjJkcnf3wfv75Z9SsWdOkERERCXr1XwN+edu6P72u1WdPREQk0GryUqZMicWLF2PUqFE4duyYGUlz2rRp5j6X8TmmERERSRaGno+8z6kVREREAq0mjxjEcaoE3kRERJK1kBDgsQ+Bec9FBnqq0RMRkUCqyRMRERE3FVoDDYZGPv5CfdNFRCTpKcgTERGJj9p9gaZjrft//gDs+1XbU0REkpSCPBERkfiq3g0o1cy6/8nDwNWz2qYiIpJkFOSJiIj4QtsvIu+/VRQ4/Ze2q4iIBF6Qt3r1aty4ccN3uREREQnkgVheOxX5+N0qwJp3kzJHIiKSTMUryGvWrBmOHDniu9yIiIgEsrBUrlMrLH4V+F5zyIqISAAFeQ6Hw3c5ERERCZYaPeepFDZ+BOxdnpQ5EhGRZEZ98kRERBICA737/p1S4bNHgDu3tJ1FRMT/g7z3338fuXPn9l1uREREgsmjUyPvL+yflDkREZFkJF5B3tNPP4306dP7LjciIiLBpvdO6/+mmeznkNS5ERGRZEDNNUVERBJSloJAmYet+0te17YWEZEEpyBPREQkodV5xfq/ZjLwVSdtbxERSVABEeTt378fXbp0QdGiRZE2bVoUL14cQ4cOxc2bN13SHTx4EC1btjRNSHPkyIGePXtGSbNjxw7UqVPHrCd//vwYMWJElFFCV65cicqVKyNNmjQoVqwYpk2bliifU0REglTeCkC1btb9Xd8BU2smdY5ERCSI+SzIu3PnDr777jskhD/++APh4eFmoJfff/8d77zzjgm8Bg8e7PL+LVq0wJUrV7Bq1SrMmTMHc+fORb9+/SLSXLx4EY0aNUK+fPmwYcMGTJ48GePGjcOECRMi0uzbtw/NmzdH7dq1sWXLFvMeDBa5LhERkThrNhYoXt+6f2InsG2ONqaIiCSIEEc8J7tjAPbRRx/hk08+wblz56LUnCWUt99+G1OnTsU///xjHi9atAgPP/wwDh06ZII4YqDXuXNnnDx5EpkyZTLpBw0ahBMnTiB16tQmzdixY02wd/jwYYSEhOCVV17B/PnzsXv37oj36tq1K7Zt24a1a9d6lTcGk5kzZ8aFCxfM+yY1Bshnz55FtmzZEBoaEJW34mdUhkTlx4d+Hgasese63309kKuMCpj2P5KAdAyTYCo/3sYZccopa8sY2NWsWRP33HMPNm/ejFGjRuHo0aNILPxg3Ng2BmDly5ePCPCoSZMmuHHjBjZt2hSRhk017QDPTsN8s0monaZx48Yu78U0GzduxK1bmuNIRETiqeEwIG1W6/6UasAR6xglIiLiK2GxScwA6MMPP8RXX32FkiVLol27dli/fj0mTZqEcuXKIbHs3bvX1L6NHz8+Ytnx48ejzNmXNWtWpEqVyjxnpylSpIhLGvs1fI59/jyth49v376N06dPI2/evFHyw0CSN+cI2478eUtqzAMrbP0hLxKYVIZE5cfH+uxC6Oh/jycf1Ed4vz1A+pwqaNr/SALQMUyCqfx4mw+vgzwGcVevXjVz4zGws4O6gQMHxjmTw4YNw/Dhw2NMw75zVapUiXjMWremTZuidevWeO6551zSsrmlO34pzsvd09itVWObxtmYMWM8fg42X2Vw6A+F4dKlS+Zz+EM1swQelSFR+UkAL/2NHO+VMHdDx5fCmWfWwpE+lwqb9j/iYzqGSTCVH+bFp0He33//jTZt2qBevXooW7YsfKFHjx5mnTFxrnljgMf3r1GjBqZPn+6SLk+ePCb4dA+y2MTSrpljGrtWz8b+enS3NGFhYciePbvHPLKfX9++fV1q8goWLGhqEv2lTx4DVObHHwqnBB6VIVH5SaDf1qCjCB1jdTPI/nENhPf/G0jn+ViTXGn/IypDon1QJMYkPg3yOOrkzJkz0a1bN1y7dg1t27Y1zTWjq93yBqc54M0bR44cMQEepzb4+OOPowQrDPzYL/DYsWMRTSoXL15s+t/xNXYajpbJwWHYjNNOw358djDJNAsWLHBZN9OwNjFlypQe88b3cO7nZ2Me/SWo4vfkT/mRwKMyJCo/CSB1emDoeWB4FvMwdJxVs4fXTgFh1nFKtP+R+NMxTIKl/HibB69zyjnlXn31VVOj99lnn5naLg68wuaIDP727NmDhMIavLp165raMU55cOrUKfP+zjVuHCyFTUg7dOhgpj5YunQp+vfvj+effz6iNo1NTRmMccTNnTt34ttvv8Xo0aNNLZwdrHIkzQMHDphlHGGTA8zMmDHDrEtERMTnePwZcs512cicwO3EGa1aRESCT5zC0fr162PWrFkm+Hr33XexbNkylClTBhUqVPB9Dv+tSWNwyfcpUKCAqamzb7YUKVLghx9+MBOYM/h88skn8cgjj5ig0MbhRpcsWWKmS2DNXPfu3U0w59zUkoOvLFy4ECtWrEDFihXxxhtvmIFlHn/88QT5bCIiIuCV2dfPAO2+idwYy0Zow4iISNLMk2fbunWrqfViQJTcaZ48CTb+NkeMBBaVn1i6eQUY/e90QM8sAgo/iORM5UdUhkT7oNjHGbGaQoG4QtaGcV45NnEsVqwYGjRoYGq9FOCJiIjEU6r0QGhKIPwW8HEzYOBBIE1mbVYREfFarII8NtHkiJj2PHA2RpPTpk3DU089FZvViYiIiCdDTgPD/g3sxhYCXjsJhEUd4EtERMQTr9tdbd68Gc8884zp58aBTTjCJufN27hxI1q2bGkGPGGTTREREfGBV52m8xmp+fNERCQBgrzJkyebAI8jad53331mlEoOclKpUiV8+umnaNWqlZprioiI+ErKtNZUCjbW7PmmG72IiAQ5r4O81atX48UXX4z2eU49sGrVKl/lS0RERDhX3n/ei9wO3/fWNhEREd8FeZwuoVSpUtE+z+c4YbmIiIj40P3tgV7brfubZgIn/9DmFRER3wR57H/H5pnRYfPN69eve7s6ERER8VbWwkDFdtb9KdWAWzreioiIj0bX/Omnn8xImp6cP38+NqsSERGR2HhkCrB1tnV/VG7g6a+BUo21DUVEJH5BXqdOnWJ8nvPmiYiISAIZdAQYk9+6/3lroGwr4KnPtLlFRCRuzTXDw8Pvertz5463qxMREZHYSp3BmjPPtns+sPg1bUcREYlbkCciIiJ+gJOiD7sQ+XjNZOC3D5IyRyIiEqjNNTkXnifso1e6dGmUKVPGl/kSERGRmDDQ+2cl8GkrYGF/IH9lIH8lbTMREfE+yOvVq5fH5ZcvXzZNNZs3b47PP/8cGTNm1GYVERFJDMXqANVfAta9B3xQD3h2MVComra9iEgy53VzzXPnznm83bhxA+vWrcPBgwcxfPjwhM2tiIiIuGo6OvL+R42BYZkBh0NbSUQkGYt3n7zQ0FA88MADGD9+PBYsWOCbXImIiIj3hpxzffxVB209EZFkzGcDr5QoUQKHDx/21epERETEW6GhVh+99nOtx7sXWDV6fyzUNhQRSYZ8FuTt3bsXBQoU8NXqREREJLZKNASqvhD5eE5bK9jb+rm2pYhIMhLvIM/hcGDz5s3o168fWrZs6ZtciYiISNw0fxvo7FaD9103K9gTEZFkwevRNbNmzYqQkBCPo2tyEvSmTZti2LBhvs6fiIiIxFaRmlbzzSungbeLRy5/tyrQ4zdtTxGRIOd1kDdx4kSPyzNlymTmyCtbtqwv8yUiIiLxlT6HFez9tQSY/QRw+k9g73KgeD1tWxGRIOZ1kNepU6eEzYmIiIgkjJKNrDn0OMXCZ48AHb4FijwEhIRag7aIiEhQ8XrPfvToUfTv3x8XL16M8tyFCxfw8ssv48SJE77On4iIiPgCJ0mv3t26/9mjwBvZgRFZgZtXtX1FRJJrkDdhwgQT4LF5prvMmTPj0qVLJo2IiIj4qaZjgM4/uC77/EkFeiIiyTXI+/HHH9GxY8don+dz33//va/yJSIiIgmhSC2rn97AQ9bj/b8Co/Nao29ePqVtLiKSnIK8ffv2oVChQtE+zzny9u/f76t8iYiISEJKkwloOcl12bgSwJUz2u4iIsklyEubNm2MQRyfYxoREREJEJU7WbV6rd6NXPZ2MSA8PClzJSIiiRXkVatWDZ999lm0z3/66aeoWrVqfPMjIiIiia1SB+C1k5GPOSDLrvn6HkREgj3I48iaH3/8sfnvPIom7/fr1w8zZ840z4mIiEgACksNtP4k8vFXHax+ejcuJWWuREQkIefJq1evHt577z306tUL77zzjhllMyQkxEyfkDJlSkyePBn169ePSx5ERETEH9zzCFDyKDA6X+SyMQWs/4/PAO59IsmyJiIiCRDk0YsvvoiHH34YX331Ff7++284HA6UKlUKTzzxhBl4RURERAJcqvRWP71Lx4HxpSOXz+0CXD4B1HgpKXMnIiK+DvIof/786NOnT2xfJiIiIoEkYx4r2Au/A4zIZi37abD1X4GeiEhw9MkTERGRZCg0BTD0PFCsbmSgx7565w8mdc5ERCTQg7xWrVqZefrSpEmDvHnzokOHDjh69KhLmoMHD6Jly5ZInz49cuTIgZ49e+LmzZsuaXbs2IE6deqY6R5YKzlixAjT7NTZypUrUblyZfNexYoVw7Rp0xLlM4qIiPilkBCg4/9cl028V4GeiIifCpggjwO/sC/gn3/+iblz52Lv3r2mL6Dtzp07aNGiBa5cuYJVq1Zhzpw5Jh1H/rRdvHgRjRo1Qr58+bBhwwYzWMy4ceMwYcIEl0nfmzdvjtq1a2PLli0YPHiwCRa5LhERkWSNzTdfWOEa6G37MilzJCIiHoQ43KuxAsT8+fPxyCOP4MaNG2Z0z0WLFplBYQ4dOmSCOGKg17lzZ5w8edKMBjp16lQMGjTITPuQOnVqk2bs2LEm2Dt8+LAZLfSVV14x6969e3fEe3Xt2hXbtm3D2rVrvcobg8nMmTObkUf5vkktPDwcZ8+eRbZs2RAaGjBxvfgRlSFR+REXPHUYniXyce8dQJZC2v+IX9IxTIKp/HgbZ8Qpp+fPn8eHH35oAiZ+aNq8eTOOHDmCxMD3nD17Nh588EET4BEDsPLly0cEeNSkSRMTBG7atCkiDZtq2gGenYbNPvfv3x+RpnHjxi7vxzQbN27ErVu3EuXziYiI+H3zTdbqOdfo/Tw8KXMkIiLxGV1z+/btaNiwoYkgGRg9//zzJrL99ttvceDAAXz66adIKKxle/fdd3H16lVUr14d33//fcRzx48fR+7cuV3SZ82aFalSpTLP2WmKFCniksZ+DZ8rWrSox/Xw8e3bt3H69GnTH9AdA0nenCNsO/LnLakxD6yw9Ye8SGBSGRKVH/FoyDmEjshq3V81wdzCh5zT/kf8io5hEkzlx9t8xDrI69u3r2kC+dZbbyFjxowRy5s1a4ann346VusaNmwYhg+P+cof+85VqVLF3H/55ZfRpUsXE0zydR07djSBHptZkv3fGb8U5+XuaezWqrFN42zMmDEeP8e5c+dMcOgPheHSpUvmc/hDNbMEHpUhUfmRaHXbhcxzn0LKkzvMQwZ9F5tNwc3iTbT/Eb+gY5gEU/lhXhIkyGPQ9f7770dZzpEq7Rozb/Xo0QNt2rSJMY1zzRtHzOSNE7CXLVsWBQsWxLp161CjRg3kyZMH69evjxJksYmlXTPHNO55ZH89uluasLAwZM+e3WMe2WyVwa9zTR7zxppEf+mTxwCV+fGHwimBR2VIVH4kRl1/QfjadxG65HXzMNOi7ta+4/UzQEj8jjva/0h8qQxJMJUfxiRepYvtijmtgN0c0RlHvcyZM2es1mUHbXFh167ZzSQZ6I0aNQrHjh2LaFK5ePFi0/+O0yHYaThaJqdVYDNOOw378dnBJNMsWLDA5b2YhrWJdv8/d3wP535+NhYEfygMxMLpT/mRwKMyJCo/EqOaPYHq3YA3Io/roW9kd+27p/2PJBEdwyRYyo+3eYh1Tv/zn/+YueXsQUj4oTk/3cCBA/H4448jIfz222+mL97WrVtNU83ly5ebpqHFixc3QRlxsJRy5cqZ+fM49cHSpUvRv39/02fQrk3jaxiMsbnpzp07TT/C0aNHm1o4uykmR9Lke3AZR9j86KOPMGPGDLMuERERiUGKlFZQ13JS5DJOnH7Du+ZFIiLiG7EO8jiv3KlTp5ArVy5cu3bNjFZZokQJ0z+PNWkJgROXz5s3Dw0aNEDp0qXx7LPPmpE0OWm5XYOWIkUK/PDDD6amsWbNmnjyySfNFAvMr42DxSxZssRMl8Caue7du5tgzrmpJQdfWbhwIVasWIGKFSvijTfewKRJkxIsgBUREQk6lTsBzy+PfDymAPBL5PFYRET8dJ68ZcuWmWkT2E61UqVKZsRNsWiePAk2/jZHjAQWlZ9kbOPHwPe9XZfVGQjUG+T1KlR+JL5UhiSYyo+3cUas++Rx2gT2X6tfv765iYiIiHhU5Rnr5hzsrRxr3Yaet+bbExERn4t1OFqsWDHUqlXLjLBpT4QuIiIiEi0Gegzqmr0VuWx4Fqu/Hm9LhnJENW1AEZGkCvI2btxoBjsZOXKkGZWSA7F8/fXXLpOBi4iIiLhgrV21F1376tlWT7SCvpO7tdFERJIiyGP/u7ffftuMqLlo0SIzAMuLL75o/nNAFBEREZFo5a9kjcD52kngUbd5d6dUB9ZM1sYTEYmnOPce5JQD9erVwwcffICff/7ZNOP85JNP4psfERERSQ7CUgP3tbECvv5/RS5f/Brwg6YtEhFJkiDv0KFDeOutt8w0Aw888ADSp09v5rITERERiZUMuaxgL1sx6/GGD6y+esd3akOKiCRGkDd9+nQzNx7nk2PNHeej27t3L1atWoVu3brFJQ8iIiIiQM8tQLO3I7fEtJrA30u1ZUREEjrI4+TgVatWNQOw/P777xg8eLCZUkFEREQk3qq9APT9A8iU3zpR+fwJ4NY1bVgRkViI9Tx5HHCF/fFEREREEkSmvEDfXVaTTQA53i+P8EemAhWf1gYXEfFVkLd9+3aUL1/ezPK+Y8eOGNNWqFDBm1WKiIiIxIwjcI7MZe6GftcNSJsNKN1UW01ExBdBHgdXOX78uJkmgfdZk+dwmrTUfsz/d+7c8WaVIiIiInc5S0mN8CHngHElEXr1NPDFU0C67MCAf7TlRETiG+Tt27cPOXPmjLgvIiIikljOPrse2fb9D6E/9AWunrGacQ49b02wLiIicQvyChcuHHH/wIEDePDBBxEW5vrS27dvY82aNS5pRURERHyi8jNAngrAjIbW4+FZrP+cekFEROI3uiYnQD979myU5RcuXDDPiYiIiCSIgg8Aj053XTa9rkbfFBGJb5Bn971zd+bMGTMhuoiIiEiCue8pa0CWSh2tx0e3AKPyaIOLiMRlCoXHHnvM/GeA17lzZ6ROnTriOQ62whE42YxTREREJEGFpQZaTQaqvgBMq2UtYz+9HKWB55YAaaypF0REkiuva/IyZ85sbqzJy5gxY8Rj3vLkyYMXXngBs2bNStjcioiIiNjy3Av03BK5PU7/CYwtBPz+nbaRiCRrXtfkffzxx+Z/kSJF0L9/fzXNFBERkaSXrZg10uaWz4D5/2ct+7oT8DWAPruAzPmTOociIv7fJ2/o0KEK8ERERMR/cKwA9tFjsFeweuTyd8oB4Zq/V0SSH69r8px98803+Oqrr3Dw4EHcvHnT5bnNmzf7Km8iIiIisQv2uvwE3LoOjMptLRuRDRhyFgi/bfXlExFJBmJdkzdp0iQ888wzyJUrF7Zs2YKqVasie/bs+Oeff9CsWbOEyaWIiIiIt1KmAbquinzMQG9kLmtwFt6O79C2FJGgFusgb8qUKZg+fTreffddpEqVCgMGDMCSJUvQs2dPM1eeiIiIiF8MyvLcUs/PcUTOuc8ldo5ERPw3yGMTTXuqhLRp0+LSpUvmfocOHfDFF1/4PociIiIicVGgitVUc8A+axCWTgsin9vxNTDvRW1XEQlKsQ7yOF0CJz6nwoULY926deb+vn37zPQKIiIiIn4jNAWQLps1ymbRh4BhF4D7O1jPbZ8DrHonqXMoIpL0QV79+vWxYIF1JaxLly7o06cPGjVqhKeeegqPPvqo73MoIiIi4kv/eRdoMtq6//MwYOvnGoVTRJL36JrsjxceHm7ud+3aFdmyZcOqVavQsmVL81hERETE79V4CTi5C9gyC/ium3ULSwv03g5kyJXUuRMRSdwgLzQ01NxsTz75pLmJiIiIBJT/vAcUrw9886z1+PY1YFxJoEhtoM1sIE3mpM6hiEjCBXnbt2/3eoUVKlSIW05EREREElv5x4ESDYGxhSKX7f/VetzoDaBmT30nIhKcQV7FihUREhJy14FVmObOnTu+ypuIiIhIwmONHQdkuXkF+HW8daMlr1u3wrWsmr20WazlnGw9NAxIEesGUSIiicKrvRNHzhQREREJaqnSAw2GWLc57YA/vreWH1gFvFk4avrq3YGGw4Cw1ImeVRGReAd5nCpBREREJNlgzd3J3cCU6tGnWTfFunGkzvvaAmmzsllTYuZSRMSjWLcz+PTTT2N8vmPHjrFdpYiIiIj/yVXWasbJ7ioXDgH/rAAunwAy5AHm94hM99Ng60ZMLyISaPPk9erVy+XWvXt3dO7cGS+88AJ69+6NhHbjxo2IPoJbt251ee7gwYNmKof06dMjR44c6NmzJ27evOmSZseOHahTpw7Spk2L/PnzY8SIEVH6Gq5cuRKVK1dGmjRpUKxYMUybNi3BP5eIiIj4KdbOZSkEVOoIPPQyUKmDFcwNOQsUrumadtNMYFhmYM27SZVbEZHYB3nnzp1zuV2+fBl//vknatWqhS+++CLBN+mAAQOQL1++KMs54EuLFi1w5coVM2/fnDlzMHfuXPTr1y8izcWLF83E7Xz9hg0bMHnyZIwbNw4TJkxw6X/YvHlz1K5dG1u2bMHgwYNNsMh1iYiIiEQITQE8s9AK+DovtJYt6GX9X/yqFexdPasNJiKJzifDQpUsWRJjx45F+/bt8ccffyChLFq0CIsXLzYBF+874/Jdu3bh0KFDEUHg+PHjTS3jqFGjkClTJsyePRvXr1/HzJkzkTp1apQvXx579uwxQV7fvn1N7SBr7QoVKoSJEyeadZQtWxYbN240weDjjz+eYJ9NREREAliRmtace3uXuS5/q6jr4weeBxoOBVJnTNTsiUjy4rOxf1OkSIGjR48ioZw4cQLPP/88vvvuO6RLly7K82vXrjVBm3MtX5MmTUzzzk2bNqFevXomDZtqMsBzTjNo0CDs378fRYsWNWkaN27ssm6mmTFjBm7duoWUKVNGeW++B2/ONYYUHh5ubkmNeWCTVH/IiwQmlSFR+RHtf7zQzqnVz5VTCB1fKmqaDR+YW/hLG4HsxVWwEoGOYRJM5cfbfMQ6yJs/f77LY37oY8eO4d1330XNmm7t0n2E78Eaua5du6JKlSomIHN3/Phx5M6d22VZ1qxZkSpVKvOcnaZIkSIuaezX8DkGeZ7Ww8e3b9/G6dOnkTdv3ijvPWbMGAwfPjzKcjZn5ev8oTBcunTJbMfQ0Fi30BVRGRLtgyTJBO4xLAXQfQ+yzm6MG6VaIsX5f5D6rx8ing19rwqul/oPLjeO7DIiCSNwy5D4g3A/Kz/MS4IEeY888ojLYzZxzJkzJ+rXr2+aR8bGsGHDPAZHzth3bs2aNaZ2jDVuMWFe3PELcV7unsYedCW2aZwxX2zuaWNeCxYsaIJMNhP1h8LJvDM//lA4JfCoDInKj2j/E0e9tiCNvS/lH56XTHsQIaf+QJo9/0OqErWBKl1UwBKQjmESTOUnLCwsYYI8X1ZV9ujRA23atIkxDWveRo4ciXXr1rk0syTW6rVr1w6ffPIJ8uTJg/Xr10epSWMTS7tmjmnsWj3byZMnzf+7peEGzZ49u8c8Ml/ueSMWBH8oDMTC6U/5kcCjMiQqP6L9j4+8tB6Y+zyw4yuELuwP8NZ+LlCioQpZAtExTIKl/HibB5/1yYsLTnPA291MmjTJBHo29v1jP7kvv/wS1apVM8tq1KhhBlhh01G7SSUHY2HwxekQ7DQcLZPTKrAZp52G/fjsZpxMs2DBApf3ZxoGlJ7644mIiIjE2uMfAGGpgC2zrMeznAZ3e+0UEBrGszltWBGJk1gHeWy6+M0332D58uWmhsu9Zm/evHnwNY526SxDhgzmf/HixVGgQAFzn4OllCtXDh06dMDbb7+Ns2fPon///mawFrvJ5NNPP22ah7J/H4O9v/76C6NHj8aQIUMimmKy3x/7F7L5JV/LgVg46EpiTA8hIiIiych/3gNSZwbWvee6fGTOyPtDzinYE5HEmQydgRTnk2OwlTlzZpdbUuHonj/88IOZwJwDwDz55JOm/yCnPrAxf0uWLMHhw4dNzRwncmcw59yfjoOvLFy4ECtWrDCTrr/xxhumJlHTJ4iIiIjPNR1tzbPXdbXn50dkBbbN0YYXkVgJcdijingpW7ZsmDVrlpkwXDzjwCsMKC9cuOA3A6+wZpPfnT+0JZbAozIkKj+i/U8iOv03cPEI8GmrqM+9sh9Im1UFMhZ0DJNgKj/exhmxzilXWqxYsfjmT0REREQ8yVECKFYHeP00EJLC9bk3iwDDMgP7ftW2ExHfBXn2tAfXrl2L7UtFRERExFspUgJDzwKDDgODj7o+98nDVrB3bJv1+OYV6/+pPcBPr1o3Ps/b+YPa5iLJTKwHXmndurUZhCRXrlxmREr3ESc3b97sy/yJiIiIJG+pM1r/h54HTv8FzO0CHN9uLXv/obu/fuK9QLO3gDwVgMI1EjavIhKYQR5Hpty0aRPat29v5paLboJwEREREfEhnnPlLAV0/RU4dwD4b4WY04elBW7/2/Jq0YDI5aWaAY9MAdJk0cidIkEq1kEeR7D86aefUKtWrYTJkYiIiIjELGtha1TOVe8ABasDee8Dlo8Crp4BHp0Wme78IeDDhsDl45HL9iwC3irqur7yjwNPfKStLpJcg7yCBQv6xYiRIiIiIslerT6Rm6DJqKibI0tBoP+f1v0/fwS+eMrzJts517pR0zeBB/5tEnrzKvDb+8DuBUDxBlYgmDaL62uvngUc4UC67NbjvcuAlW8BtfsBJRtZNZDuOLj70S1ArnLAravAF22tUUNbjAcy50/2X6tIok+hwJq8yZMnY9q0aaZPnkSlKRQk2Pjb8MESWFR+ROXHD4WHAzu+AtLnAK5fAL55NvbryJgPuOQ2IIwvsFbSHlCmUkeg1eR4rU77IAmm8uNtnBHrmjz2xbt69SqKFy+OdOnSRRl4hRtBRERERPwYT1bvaxP5uHQL4I/vgR/6AdfPe7eOhAjwyA7waPOn1s326gkgZZqEeV+RIBLrIG/ixIkJkxMRERERSRoMnO59wrpdOQ2kTAekShc13bKRwC9vRz4u0Qh48hMgVXqrdvDmZWs0ULuJ5pFNwA/9raafbFpqv7bzQqBITeDUn1ZTz1xlreUndgFTYxgBdFTuyCalrInkIDKVOljNP8NSA+my+W6biCSn5ppyd2quKcHG35oqSGBR+RGVH4njzgM4tgWY3xM4sTN2ry1WFyj6EHBva4RnKqBjmATNMSzBmmsePBjzhJqFChWK7SpFRERERFzxhDp/ZaDb6shlO76x5gm8m39WWLelIxByX1ug9khtXUlWYh3kcbCVmObGu3PnTnzzJCIiIiISld2kNDpbZgP/6+6yKGTbF8j611KEZMgOPLfMczNUkeQe5G3ZssXl8a1bt8yyCRMmYNQoD0P3ioiIiIgkhvvbWTfbuNJmjsAUV08CvI3OG/U1/5liNe/U1A2SnIO8++67L8qyKlWqIF++fHj77bfx2GOP+SpvIiIiIiJx1/9POMaXRUhMI4G61fyZQWf6/wWkzqAtL8knyItOqVKlsGHDBl+tTkREREQk3hx9fscZ54EzLp0APnsEOLnL8ws4OfuYfydkb/WuNXqnszu3gTeyA41GADV76RuS4AjyOKKLMw7OeezYMQwbNgwlS5b0Zd5ERERERHwrY26g+1rXZRxsnvPzXT4JfN46cvn8HtbNkyVDrBs92BOo84pq/yRwg7wsWbJEGXiFgV7BggUxZ84cX+ZNRERERCTh8dw2X0Xr/msngS/aAnuXev/6NZOsGw05Z40MKhJIQd6yZctcgjxWe+fMmRMlSpRAWJjPWn+KiIiIiCQ+TqreYV7k4zN7gTSZgXTZgdN7gL9/Bip1BFKmB94qAly/4Pr6EVmBXtuBS8etxwUeUNAniS7WUVndunUTJiciIiIiIv4me/HI+zlLWzfbQKf5o4/vAKbVsu7/t4LrOrqtBXKXS+icikSIdV3ymDFj8NFHH0VZzmVvvvlmbFcnIiIiIhL48twLdF/n+bmpNYDRBYDw8MTOlSRTsQ7y3n//fZQpUybK8nvuuQfTpk3zVb5ERERERAJLrrLAsAuRt6HnI5+7eclqyvm/aAZyEUnK5prHjx9H3rxRJ5JkvzyOsikiIiIiIv8O6MJg788fgS+esjbJls+sm61iO6DBECBjHm0ySbqaPI6iuXr16ijLuYwToouIiIiIiJPSTa1RN4vVi7pZts4GxpcGhmUGFr8O3LquTSeJX5P33HPPoXfv3rh16xbq169vli1duhQDBgxAv3794p8jEREREZFgw2kVOn4H3L4JnPnb6qcX3VQMrP0TScwgj8Hc2bNn0b17d9y8edMsS5MmDV555RUMGjQoPnkREREREQluYamskTadA7nLp4BxJSIfs1YvXQ6gxwYgXbYkyaYksyCPc+RxFM3XX38du3fvRtq0aVGyZEmkTp06YXIoIiIiIhLMMuS0gr49i4HPW1vLrp4G3ioaNW3H+UCKVECh6lafPxEP4jx7eYYMGfDAAw/E9eUiIiIiIuKsVGPg5b3A0S3A7Cc8b5tPW0Xez1II6L3DmqMvVzkgNIW2p8QvyBMRERERER9LnwMo2ciq2du7DPjsUaBUU+DWNWDfSte05w9aTTs9GbDPaup5nc1CQ4A0mfRVJSMK8kRERERE/FHx+tEPwrJkCLD6v9G/1r2pZ7O3gGov+jZ/4rcU5ImIiIiIBJpGI6wbORzA7gXAT4OBC4c8p180wLpRrb5A7b5A6oyJl19JVAryREREREQCGQdgKdfKutnu3AZS/Huqv3wMsHJs5HOrJli36t2BpmMSP7/if5OhJ5UiRYqYkT2dbwMHDnRJc/DgQbRs2RLp06dHjhw50LNnz4hpHmw7duxAnTp1zKig+fPnx4gRI+Dg1Q8nK1euROXKlc3UEMWKFcO0adMS5TOKiIiIiPiEHeBRvUHWZOwZ87qmWTfF6tNn396rro0fJAKqJo8B2fPPP+8ywqftzp07aNGiBXLmzIlVq1bhzJkz6NSpkwngJk+ebNJcvHgRjRo1Qr169bBhwwbs2bMHnTt3NkGhPZH7vn370Lx5c/M+s2bNwurVq82cgFzv448/ngSfWkRERETEB5Ox9/sj8vE75aM27Ty123Ugl1dPACnTaNMHoIAK8jJmzIg8efJ4fG7x4sXYtWsXDh06hHz58pll48ePN0HcqFGjkClTJsyePRvXr1/HzJkzzbx+5cuXN4HehAkT0LdvX1M7yFq7QoUKYeLEiWYdZcuWxcaNGzFu3DgFeSIiIiISHPrstP7fuASc3A3MaBQ1zajcQLn/AFVfAIrUAu7cAlKkTPSsSpAHeZyE/Y033kDBggXRunVrvPzyy0iVKpV5bu3atSZoswM8atKkCW7cuIFNmzaZ2jumYVNN54nbmWbQoEHYv38/ihYtatI0btzY5X2ZZsaMGbh16xZSplTBFhEREZEgwcFXCla1RvE8vhOY9Rhw+UTk87v+Z93iolpXoNmbkY9v37Amctck7gkuYIK8Xr16oVKlSsiaNSt+++03E5ixaeWHH35onj9+/Dhy587t8hqmZRDI5+w07NvnzH4Nn2OQ52k9fHz79m2cPn0aefO6tWXmBZAbN8zNxmahFB4ebm5JjXlgs1V/yIsEJpUhUfkR7X8kUOkYFgucUL3vv00679xEKGvy4mP9NOvm/p10W2cFe+cPIGTlWDge7AmUbg5/FO5n59He5iNJg7xhw4Zh+PDhMaZh37kqVaqgT58+EcsqVKhgArgnnnjC1O5lz57dLGdzS3f8UpyXu6exB12JbRpnY8aM8fg5zp07Z4JDfygMly5dMp8jlO2xRVSGRPsgCRA6honKUBLqsdf8C718HKEXDuJ27vuQbsNkpNs0FbfyVkF4+ly4nasCQq6fR+i1M7hZuC5SXDyA9GveinG1oVNdB3gJ+bKd+X8nYz6cb7MQDj+a2iHcz86jmRe/D/J69OiBNm3axJjGvebNVr26VTj+/vtvE+Sxr9769eujBFlsYmnXzDGNXatnO3nypPl/tzRhYWERwaQ71iqyT59zTR6blDIQZV9AfyicDFCZH38onBJ4VIZE5Ue0/5FApWOYD2TLBqCcdb/FaIS3GI0UHMATgHNHJqsTFRDecJB15/oFhCx7A7h6Bo5HpgKHNyL005bRvk2KS0eR/YOKrt8fRwVNQuF+dh7NmMSrdEhCnOaAt7jYsmWL+W83n6xRo4YZYOXYsWMRyzgYC/vfcToEO83gwYPNtAp2Xz6mYT8+O5hkmgULFri8F9OwNjG6/nh8D+d+fjYWBH8oDMTC6U/5kcCjMiQqP6L9jwQqHcOSSLqswMMTrO+Af4o9ZPX9c7dwAPDb+x5XEToiK/DyXiCcreNCgIzxbEIa4OXH2zwkfU69wMFQ3nnnHWzdutX0w/vqq6/w4osvolWrVmYkTOJgKeXKlUOHDh1MALh06VL079/fTIVg16Y9/fTTJhjjiJs7d+7Et99+i9GjR0eMrEldu3bFgQMHzLLdu3fjo48+MoOucF0iIiIiIuJjzd+ygj/e+v8N1B3k2kfv7eLA+NLA+FLAjCba/MEy8AoDsy+//NL0e+MAJ4ULFzbB24ABAyLSpEiRAj/88IOZ065mzZpmsnMGdZz6wJY5c2YsWbIEL730kqmZY7UrgznnppYcfGXhwoWmD+B7771navkmTZqk6RNERERERBJahpxA3YHW/atngbeKuj5/aJ01l9+Qs0AoG4yKJyEOe1QR8Rn2yWNAeeHCBb/pk3f27Flky5bNL6qZJfCoDInKj2j/I4FKx7Ag8uMgYN2UqMsHHgTSOE3iHsTlx9s4I+lzKiIiIiIicjdNxwDP/hR1+dhCwJHN2n5OFOSJiIiIiEhgKFQd4IibXVcDKdNFLv+gHvBmUSD8TlLmzm8oyBMRERERkcDBZpN5ygOvHgMGHQYKPGAtv3YWGJENWDIUyZ2CPBERERERCUycOP25n4FuayOXrZ5oDc5y+yaSKwV5IiIiIiIS2HKXA149DhSsFrlsZE5g+1dIjhTkiYiIiIhI4EuZFuiy2HVwlnnPA9c9TMAe5BTkiYiIiIhIcA3OMuyC6+ibGz9GcqIgT0REREREgk+H7yLvf98beCMnkgsFeSIiIiIiEnyK17Nq9Gr0sB7fuQksGojkQEGeiIiIiIgEryajgGpdrfvrpwLfPBv08+kpyBMRERERkeDW7E2gQFXr/s651nx6J3YhWCnIExERERGR4Pfsj66Pp9YAZj8JOBwINgryREREREQk+IWmsProvX46ctlfPwHDsyDYKMgTEREREZHkI0VKYMg5IGvRyGW/f4tgoiBPRERERESSl9BQoNdWoPMP1uOvOwPn9iNYKMgTEREREZHkqUgtICSFdf+/9wFXzyIYKMgTEREREZHka6hTYPeWUxPOAKYgT0REREREkrfXz0Tef68aAp2CPBERERERSd5ShAEdvrPun/oDOPE7ApmCPBERERERkeL1gNr9rO0w9UFgRpOA3SYK8kRERERERKjBkMipFQ6tA/5ciECkIE9ERERERMTGqRUaDjN3Q79sh5SH1yLQKMgTERERERFxVrN3xN3M37VHoFGQJyIiIiIi4iwkBBh6Ho6MeXG5znAEmrCkzoCIiIiIiIjfCQmBo88uXD97FukQWFSTJyIiIiIiEkQU5ImIiIiIiAQRBXkiIiIiIiJBREGeiIiIiIhIEFGQJyIiIiIiEkQU5ImIiIiIiAQRBXkiIiIiIiJBREGeiIiIiIhIEFGQJyIiIiIiEkTCkjoDwcjhcJj/Fy9ehD8IDw/HpUuXEBYWhtBQxfWiMiTaB0ng0DFMVIZE+6BIdnxhxxvRUZCXABhQUcGCBRNi9SIiIiIikszjjcyZM0f7fIjjbmGgxOmq49GjR5ExY0aEhIT4RcTPgPPQoUPIlClTUmdHApDKkKj8iPY/Eqh0DJNgKj8M3Rjg5cuXL8YWeqrJSwDc4AUKFIC/YcH0h8IpgUtlSFR+RPsfCVQ6hkmwlJ+YavBs6qAlIiIiIiISRBTkiYiIiIiIBBEFeclA6tSpMXToUPNfRGVItA+SQKJjmKgMifZBsaeBV0RERERERIKIavJERERERESCiII8ERERERGRIKIgT0REREREJIgoyEsGpkyZgqJFiyJNmjSoXLkyfv3116TOkiSwMWPG4IEHHkDGjBmRK1cuPPLII/jzzz+jTKY5bNgwM5lm2rRpUbduXfz+++8uaW7cuIH/+7//Q44cOZA+fXq0atUKhw8fdklz7tw5dOjQwczZwhvvnz9/3iXNwYMH0bJlS7MOrqtnz564efNmAm4B8XV5CgkJQe/evSOWqfxITI4cOYL27dsje/bsSJcuHSpWrIhNmzap/Mhd3b59G6+99po5b+GxqVixYhgxYgTCw8NVfsSjX375xZxj8HyGx6rvvvvO5Xl/O17t2LEDderUMXnJnz+/Kd/Mo885JKjNmTPHkTJlSscHH3zg2LVrl6NXr16O9OnTOw4cOJDUWZME1KRJE8fHH3/s2Llzp2Pr1q2OFi1aOAoVKuS4fPlyRJqxY8c6MmbM6Jg7d65jx44djqeeesqRN29ex8WLFyPSdO3a1ZE/f37HkiVLHJs3b3bUq1fPcd999zlu374dkaZp06aO8uXLO9asWWNuvP/www9HPM+0XMbXch1cV758+Rw9evRQGQgAv/32m6NIkSKOChUqmP2HTeVHonP27FlH4cKFHZ07d3asX7/esW/fPsfPP//s+Pvvv1V+5K5GjhzpyJ49u+P77783Zefrr792ZMiQwTFx4kSVH/Fo4cKFjldffdWczzC0+fbbb12e96fj1YULFxy5c+d2tGnTxuSFeWLexo0b5/NvV0FekKtataopuM7KlCnjGDhwYJLlSRLfyZMnzY5v5cqV5nF4eLgjT548Zsdnu379uiNz5syOadOmmcfnz583Fwh4ocB25MgRR2hoqOPHH380j3nhgOtdt25dRJq1a9eaZX/88UfEzpev4WttX3zxhSN16tRmZyf+69KlS46SJUuaA1WdOnUigjyVH4nJK6+84qhVq1a0z6v8SEx4UfLZZ591WfbYY4852rdvr/Ijd+Ue5IX72fnOlClTzHszD7YxY8aYYJB59SU11wxirB5m85jGjRu7LOfjNWvWJFm+JPFduHDB/M+WLZv5v2/fPhw/ftylbHAuKjYfsMsGy86tW7dc0rCpQ/ny5SPSrF271jRZqFatWkSa6tWrm2XOafgavtbWpEkT0zTCufmW+J+XXnoJLVq0QMOGDV2Wq/xITObPn48qVaqgdevWprn4/fffjw8++EDlR7xSq1YtLF26FHv27DGPt23bhlWrVqF58+ba/0is7fOz8x2m4Xs7z13NNEePHsX+/ft9+g2H+XRt4ldOnz6NO3fuIHfu3C7L+ZgFXpIHXtjq27evOXBy50P29++pbBw4cCAiTapUqZA1a9YoaezX8z9P4txxmXMa9/fhOrlulUP/NWfOHGzevBkbNmyI8pzKj8Tkn3/+wdSpU81+Z/Dgwfjtt99MvxSe1HTs2FHlR2L0yiuvmAuTZcqUQYoUKcx5zKhRo9C2bVvtfyTWjvvZ+Q7/FylSJMr72M+xL6qvKMhLBtgJ1f2k332ZBK8ePXpg+/bt5kqoL8qGexpP6eOSRvzHoUOH0KtXLyxevNgM2BQdlR/xhANksCZv9OjR5jFr8jjIAQM/BnkqPxKTL7/8ErNmzcLnn3+Oe+65B1u3bjWDPrF2pFOnTio/EichfnS+4ykv0b02PtRcM4hxVB9eBXOvLTl58mSUKw0SnDhSFJtOLV++HAUKFIhYnidPHvM/prLBNGzyy9GkYkpz4sSJKO976tQplzTu78N1smmEyqF/YrMSfs8cjTcsLMzcVq5ciUmTJpn7zlcdnan8COXNmxflypVz2Rhly5Y1o86R9j8Sk5dffhkDBw5EmzZtcO+995oRDPv06WNG+VX5kdjK42fnO57S8H3I1+dECvKCGKuHeZK2ZMkSl+V8/OCDDyZZviTh8aoQa/DmzZuHZcuWRan+52PuaJzLBndwPJG3ywbLTsqUKV3SHDt2DDt37oxIU6NGDdOshs2xbOvXrzfLnNPwNXytjTVEbLrF9xD/06BBAzPEM6+g2zfWzLRr187c55DmKj8SnZo1a0aZsoX9qwoXLmzua/8jMbl69SpCQ11PT3nB2p5CQeVHYqOon53vMA2nfHCeVoFpWFPt3owz3nw6jIv47RQKM2bMMCMD9e7d20yhsH///qTOmiSgbt26mdGbVqxY4Th27FjE7erVqxFpONIU08ybN88M49u2bVuPQwoXKFDADH/O4YDr16/vcUhhDq/PUaZ4u/feez0OKdygQQOzDq6L69QUCoHFeXRNUvmRmKbdCAsLc4waNcrx119/OWbPnu1Ily6dY9asWSo/cledOnUyQ9nbUyjwGJUjRw7HgAEDVH4k2pGgt2zZYm4MbSZMmGDu29OF+dPxiiN5cgoF5oF5YZ4yZcqkKRQkbt577z0zZ1GqVKkclSpVihhGX4IXd3Kebpw7z8aheocOHWqGFubwvg899JDZ4Ti7du2a2Tlly5bNkTZtWrMzO3jwoEuaM2fOONq1a2fmeeGN98+dO+eShjtaDovNdXBdXKfz8MESeEGeyo/EZMGCBeZkh/sWTtszffp0l+dVfiQ6PPHmvoZzu6ZJk8ZRrFgxMwfajRs3VH7Eo+XLl3s85+nUqZNf7m+2b9/uqF27tskL8zRs2DCfT59AIfzj27pBERERERERSSrqkyciIiIiIhJEFOSJiIiIiIgEEQV5IiIiIiIiQURBnoiIiIiISBBRkCciIiIiIhJEFOSJiIiIiIgEEQV5IiIiIiIiQURBnoiIiIiISBBRkCciIiIiIhJEFOSJiIjEUufOnRESEhLl9vfff2tbiohIkgtL6gyIiIgEoqZNm+Ljjz92WZYzZ06Xxzdv3kSqVKkSOWciIpLcqSZPREQkDlKnTo08efK43Bo0aIAePXqgb9++yJEjBxo1amTSTpgwAffeey/Sp0+PggULonv37rh8+XLEumbOnIksWbLg+++/R+nSpZEuXTo88cQTuHLlCj755BMUKVIEWbNmxf/93//hzp07LkHkgAEDkD9/frPuatWqYcWKFfo+RUSSOdXkiYiI+BCDsm7dumH16tVwOBxmWWhoKCZNmmSCtX379pkgj8HZlClTIl539epVk2bOnDm4dOkSHnvsMXNj8Ldw4UL8888/ePzxx1GrVi089dRT5jXPPPMM9u/fb16TL18+fPvtt6aGcceOHShZsqS+VxGRZCrEYR+BRERExOs+ebNmzUKaNGkiljVr1gynTp3ChQsXsGXLlhhf//XXX5tA8PTp0xE1eQzY2KevePHiZlnXrl3x2Wef4cSJE8iQIYNZxgCOgeK0adOwd+9eE8gdPnzYBHi2hg0bomrVqhg9erS+TRGRZEo1eSIiInFQr149TJ06NeIxm0u2bdsWVapUiZJ2+fLlJujatWsXLl68iNu3b+P69eumOSZfR2yiaQd4lDt3bhPQ2QGevezkyZPm/ubNm01NYalSpVze68aNG8iePbu+UxGRZExBnoiISBwwOCtRooTH5c4OHDiA5s2bm5q5N954A9myZcOqVavQpUsX3Lp1KyJdypQpXV7H0To9LQsPDzf3+T9FihTYtGmT+e/MOTAUEZHkR0GeiIhIAtq4caOpuRs/frzpm0dfffVVvNd7//33m0FYWLNXu3ZtH+RURESChUbXFBERSUBsgskgb/LkyWbwFPazY5+6+GIzzXbt2qFjx46YN2+eGdBlw4YNePPNN81ALSIiknwpyBMREUlAFStWNFMoMPgqX748Zs+ejTFjxvhk3Zynj0Fev379zNQLrVq1wvr16800DSIiknxpdE0REREREZEgopo8ERERERGRIKIgT0REREREJIgoyBMREREREQkiCvJERERERESCiII8ERERERGRIKIgT0REREREJIgoyBMREREREQkiCvJERERERESCiII8ERERERGRIKIgT0REREREJIgoyBMREREREQkiCvJEREREREQQPP4fDMtW8Qx8sIoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first half: -2,032 second half: -3,641\n" + ] + } + ], "source": [ "cnt_cpu = np.bincount(fr_cpu, minlength=N)\n", "cnt_cuda = np.bincount(fr_cuda, minlength=N)\n", diff --git a/python/tests/perf/README.md b/python/tests/perf/README.md new file mode 100644 index 00000000..c20c9ef1 --- /dev/null +++ b/python/tests/perf/README.md @@ -0,0 +1,102 @@ +# ClusterFinderCUDA nsys probes + +Per-engine GPU time per frame (kernel, H2D, D2H) for the CUDA cluster finder, +traced with Nsight Systems on real MOENCH data. The tallest engine is the +roofline: the engines overlap and PCIe is full-duplex, so throughput is bound by +the max, not the sum. + +These are *not* throughput numbers. nsys inflates wall clock several-fold by +tracing every CUDA call. Use them to answer "what does the kernel cost" and +"which engine binds", and measure sustained FPS separately without a profiler. + +## Files + +| file | role | +|---|---| +| `run_probes.py` | driver: runs each config under nsys, exports SQLite, writes `probes.csv` | +| `nsys_kernel_probe.py` | the traced process: pedestal, one batched pass, exit | +| `gpu_span.py` | reads the SQLite export, computes busy time, duty cycle, roofline | +| `common.py` | paths, guards (fresh build, idle GPU), `env.json` stamp | + +## Requirements + +- A built tree with the CUDA extension: `build/aare/_aare_cuda*.so`. Run the + scripts with a Python of the same minor version as the build, with numpy and + matplotlib installed (`aare/__init__.py` imports both). On pc-moench-04 the + build is 3.11 and the `py` conda environment works: + `/home/ferjao_k/.conda/envs/py/bin/python`. +- Nsight Systems. Default path `/opt/nvidia/nsight-systems/2024.5.1/bin/nsys`, + override with `NSYS=...`. +- The MOENCH dataset under `/mnt/sls_det_storage/.../2026032408/process/xrf/` + (override with `AARE_PERF_DATA=`). 1000 pedestal frames, 20 000 data + frames, 5 sigma. +- An idle GPU. The driver aborts above 5 % utilisation: a competing process + leaves per-op averages intact but ruins duty cycles and rooflines. + +Paths are resolved relative to this folder. `AARE_REPO` and `AARE_BUILD` +override the repo root and build directory. + +## Run + +```bash +cd python/tests/perf +python run_probes.py # full sweep: 4 configs, 20k frames each (~5 min) +python run_probes.py --frames 2000 # smoke test only; clocks do not ramp, kernel reads ~7 % slow +python run_probes.py --only 3x3_s1_uncontended 9x9_s1_uncontended --tag mychange +``` + +Configs, matching the campaign in `docs/ClusterFinderCUDA_benchmark_results.md` +on branch `bench/cuda_cf`: + +| label | cluster | cap | streams | batch | frames | +|---|---|---|---|---|---| +| `3x3_s4` | 3x3 | 3000 | 4 | 2000 | 20 000 | +| `3x3_s1_uncontended` | 3x3 | 3000 | 1 | 2000 | 20 000 | +| `9x9_s4` | 9x9 | 1700 | 4 | 2000 | 20 000 | +| `9x9_s1_uncontended` | 9x9 | 1700 | 1 | 2000 | 20 000 | + +Keep the cap: at 9x9 the D2H slot is `4 + cap * sizeof(cluster)` and is copied +whole, so the cap sets the D2H bar. + +Output goes to `results/_[_tag]/`: `env.json` (git rev, dirty +flag, driver, pedestal type), one `.nsys-rep` + `.sqlite` per config, and +`probes.csv`. Re-running one config replaces its own row and keeps the others. + +To inspect one trace by hand: + +```bash +nsys stats --report cuda_gpu_sum results//probe_9x9_s1_uncontended_cap1700.nsys-rep +python gpu_span.py results//probe_9x9_s1_uncontended_cap1700.sqlite 20000 +``` + +`probes.csv` and `env.json` are small and can be committed with a result; +the `.nsys-rep` and `.sqlite` traces are tens of MB each and are gitignored. + +## Reading the numbers + +- **Quote kernel changes from the `s1_uncontended` rows.** With one stream + nothing overlaps, so the kernel column is the kernel alone. The 1-stream + kernel and transfer numbers reproduce across sessions to about 1 %. +- **4-stream transfer columns drift by up to 20 % between sessions** on an + identical build (H2D and D2H fight for the PCIe link). Only compare s4 + rows taken in the same session, and read them for the bottleneck, not for + a percentage. +- A kernel below the tallest transfer bar is invisible end to end. Further + kernel work only pays once it is the tallest bar. + +## Reference, 2026-09-10, RTX 4090, driver 595.71.05, float pedestal + +Kernel us/frame. "before" is commit 7177f00 (pre-refactor kernel, rebuilt and +re-probed the same day, matching its August numbers): `results/2026-09-10_f32_ref7177/`. +"after" is the kernel with the cooperative tile load, size-dependent unroll +and two-phase load: `results/2026-09-10_f32/`. + +| config | before | after | roofline after | binds | +|---|--:|--:|--:|---| +| 3x3, 1 stream | 4.32 | 4.65 | 13.13 | H2D | +| 3x3, 4 streams | 5.00 | 5.64 | 16.35 | H2D | +| 9x9, 1 stream | 23.80 | 15.29 | 22.00 | D2H | +| 9x9, 4 streams | 24.72 | 15.59 | 25.39 | D2H | + +Both sizes are transfer-bound; sustained throughput is unchanged (3x3 about +61 400 FPS, 9x9 about 39 600 FPS on the zero-copy path). diff --git a/python/tests/perf/common.py b/python/tests/perf/common.py new file mode 100644 index 00000000..ff87a6b1 --- /dev/null +++ b/python/tests/perf/common.py @@ -0,0 +1,102 @@ +"""Shared plumbing for the nsys probes: paths, guards, environment stamp. + +Two guards are enforced here because both have produced wrong numbers before: +a stale extension (header edited after the build) and a busy GPU (another +process leaves per-op averages intact but destroys duty cycles). +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = Path(os.environ.get("AARE_REPO", HERE.parents[2])) +BUILD = Path(os.environ.get("AARE_BUILD", REPO / "build")) +sys.path.insert(0, str(BUILD)) + +DATA_DIR = Path(os.environ.get( + "AARE_PERF_DATA", + "/mnt/sls_det_storage/moench_data/2603_MaxIVBeamtime/2026032408/process/xrf/")) +DATA_FILE = DATA_DIR / "Cu_factor_10_data_master_0.json" +PEDESTAL_FILE = DATA_DIR / "Cu_factor_10_pedestal_master_0.json" + +N_PEDESTAL_FRAMES = 1000 +N_SIGMA = 5 + +NSYS = os.environ.get("NSYS", "/opt/nvidia/nsight-systems/2024.5.1/bin/nsys") + + +def _sh(cmd: str, default: str = "unknown") -> str: + try: + return subprocess.run(cmd, shell=True, capture_output=True, text=True, + timeout=30).stdout.strip() or default + except Exception: + return default + + +def device_ped_type() -> str: + """DEVICE_PED_TYPE from the kernel header (the float/double build axis).""" + hdr = REPO / "include/aare/clusterfinder_kernel.cuh" + for line in hdr.read_text().splitlines()[:40]: + if "using DEVICE_PED_TYPE" in line and not line.strip().startswith("//"): + return line.split("=")[1].split(";")[0].strip() + return "unknown" + + +def assert_build_fresh() -> None: + """Abort if a CUDA header is newer than the installed extension.""" + so = list((BUILD / "aare").glob("_aare_cuda*.so")) + if not so: + raise RuntimeError(f"no _aare_cuda*.so under {BUILD / 'aare'}") + built = max(f.stat().st_mtime for f in so) + headers = ["clusterfinder_kernel.cuh", "clusterfinder_algo.cuh", + "ClusterFinderCUDA.hpp"] + stale = [h for h in headers + if (REPO / "include/aare" / h).stat().st_mtime > built] + if stale: + raise RuntimeError(f"{', '.join(stale)} newer than the extension: rebuild first") + + +def assert_idle_gpu(max_pct: int = 5) -> None: + pct = _sh("nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader") + try: + val = int(pct.split()[0]) + except Exception: + return + if val > max_pct: + raise SystemExit(f"GPU is {val}% busy: numbers would not be quotable. Aborting.") + + +def capture_env() -> dict: + import aare + + return { + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "host": _sh("hostname"), + "git_rev": _sh(f"git -C {REPO} rev-parse --short HEAD"), + "git_branch": _sh(f"git -C {REPO} rev-parse --abbrev-ref HEAD"), + "git_dirty": bool(_sh(f"git -C {REPO} status --porcelain", default="")), + "aare_version": getattr(aare, "__version__", "unknown"), + "device_ped_type": device_ped_type(), + "gpu": _sh("nvidia-smi --query-gpu=name --format=csv,noheader"), + "driver": _sh("nvidia-smi --query-gpu=driver_version --format=csv,noheader"), + "nvcc": _sh("nvcc --version | tail -1"), + "python": sys.version.split()[0], + } + + +def results_dir(tag: str = "") -> Path: + """perf/results/_[_tag]/""" + short = {"float": "f32", "double": "f64"}.get(device_ped_type(), "unk") + name = f"{time.strftime('%Y-%m-%d')}_{short}" + (f"_{tag}" if tag else "") + d = HERE / "results" / name + d.mkdir(parents=True, exist_ok=True) + return d + + +def write_env(path: Path, env: dict) -> None: + path.write_text(json.dumps(env, indent=2) + "\n") diff --git a/python/tests/perf/gpu_span.py b/python/tests/perf/gpu_span.py new file mode 100644 index 00000000..8a4d9ded --- /dev/null +++ b/python/tests/perf/gpu_span.py @@ -0,0 +1,68 @@ +"""Per-engine busy time and duty cycle from an nsys SQLite export. + + nsys stats --force-export=true --report cuda_gpu_sum .nsys-rep + python gpu_span.py .sqlite + +Per engine (kernel / H2D / D2H): sum of op durations, busy = union of the +intervals (overlaps not double-counted), duty = busy / processing window. +The window is first kernel start to last D2H end, so the pedestal-upload +copies that precede any kernel are excluded. + +Roofline = tallest engine's busy time per frame. PCIe is full-duplex and the +engines overlap, so the floor is the max, never the sum. +""" +import sqlite3 +import sys + +Q = {"kernel": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_KERNEL ORDER BY start", + "H2D": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=1 ORDER BY start", + "D2H": "SELECT start,end FROM CUPTI_ACTIVITY_KIND_MEMCPY WHERE copyKind=2 ORDER BY start"} + + +def union(rows): + busy, cs, ce = 0, *rows[0] + for s, e in rows[1:]: + if s > ce: + busy += ce - cs + cs, ce = s, e + else: + ce = max(ce, e) + return busy + ce - cs + + +def analyze(sqlite_path, n_frames) -> dict: + db = sqlite3.connect(str(sqlite_path)) + ops = {k: db.execute(q).fetchall() for k, q in Q.items()} + lo = ops["kernel"][0][0] + hi = max(ops["kernel"][-1][1], ops["D2H"][-1][1]) + span = hi - lo + + out = {"n_frames": n_frames, "window_ms": span / 1e6, + "window_us_per_frame": span / 1e3 / n_frames} + for k, rows in ops.items(): + rows = [r for r in rows if r[1] > lo] + tot, busy = sum(e - s for s, e in rows), union(rows) + out[f"{k}_n"] = len(rows) + out[f"{k}_sum_ms"] = tot / 1e6 + out[f"{k}_busy_ms"] = busy / 1e6 + out[f"{k}_overlap"] = tot / busy + out[f"{k}_duty_pct"] = 100 * busy / span + out[f"{k}_us_per_frame"] = busy / 1e3 / n_frames + tallest = max(("kernel", "H2D", "D2H"), key=lambda k: out[f"{k}_us_per_frame"]) + out["bottleneck"] = tallest + out["roofline_us_per_frame"] = out[f"{tallest}_us_per_frame"] + out["roofline_fps"] = 1e6 / out["roofline_us_per_frame"] + return out + + +if __name__ == "__main__": + N = int(sys.argv[2]) if len(sys.argv) > 2 else 2000 + r = analyze(sys.argv[1], N) + print(f"processing window: {r['window_ms']:.1f} ms / {N} frames = " + f"{r['window_us_per_frame']:.1f} us/frame") + for k in ("kernel", "H2D", "D2H"): + print(f" {k:6s} sum={r[k + '_sum_ms']:7.2f} ms busy={r[k + '_busy_ms']:7.2f} ms " + f"overlap={r[k + '_overlap']:4.2f}x duty={r[k + '_duty_pct']:5.1f}% " + f"per-frame={r[k + '_us_per_frame']:5.2f} us") + print(f" roofline = {r['bottleneck']} at {r['roofline_us_per_frame']:.2f} us/frame " + f"= {r['roofline_fps']:,.0f} FPS") diff --git a/python/tests/perf/nsys_kernel_probe.py b/python/tests/perf/nsys_kernel_probe.py new file mode 100644 index 00000000..7d02aa09 --- /dev/null +++ b/python/tests/perf/nsys_kernel_probe.py @@ -0,0 +1,47 @@ +"""The process nsys traces: train the pedestal, run one batched pass, exit. + + nsys_kernel_probe.py + +Run by run_probes.py; run directly only under `nsys profile --trace=cuda`. +The wall time printed is profiler-inflated and is not a throughput number. +""" +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 (puts the build dir on sys.path) + +from aare import File, ClusterFinderCUDA # noqa: E402 + +n_streams, N, cdim, cap, batch = (int(a) for a in sys.argv[1:6]) + +f = File(common.DATA_FILE) +pd = File(common.PEDESTAL_FILE) + +cf = ClusterFinderCUDA((f.rows, f.cols), (cdim, cdim), n_sigma=common.N_SIGMA, + max_clusters_per_frame=cap, n_streams=n_streams) +for _ in range(common.N_PEDESTAL_FRAMES): + cf.push_pedestal_frame(pd.read_frame().copy()) + +data = f.read_n(N) +cf.register_input_buffer(data) +# Pin the output slots without processing frames: a warm-up pass would advance +# the device pedestal and make this run incomparable with the next. +cf.reserve_output_slots(cf.chunk_size_for(min(N, batch))) + +t0 = time.perf_counter() +n = 0 +for start in range(0, N, batch): + for cv in cf.find_clusters_batched(data[start:min(start + batch, N)], + first_frame=start): + n += cv.size +t = time.perf_counter() - t0 +cf.unregister_input_buffer() + +h2d = f.rows * f.cols * 2 +slot = 2 + 2 + cdim * cdim * 4 # x, y (uint16) + data (int32) +print(f"n_streams={n_streams} N={N} cluster={cdim}x{cdim} cap={cap} batch={batch}") +print(f" H2D/frame={h2d:,} B D2H/frame={cap * slot:,} B " + f"({slot} B/slot, {100 * n / N / cap:.0f}% filled)") +print(f" wall={t:.3f}s ({N / t:.0f} FPS, profiler-inflated) clusters/frame={n / N:.2f}") diff --git a/python/tests/perf/results/2026-09-10_f32/env.json b/python/tests/perf/results/2026-09-10_f32/env.json new file mode 100644 index 00000000..a1093126 --- /dev/null +++ b/python/tests/perf/results/2026-09-10_f32/env.json @@ -0,0 +1,14 @@ +{ + "timestamp": "2026-09-10 13:37:02", + "host": "pc-moench-04.psi.ch", + "git_rev": "272b483", + "git_branch": "testing/cuda_cf_api", + "git_dirty": true, + "aare_version": "2026.7.2", + "device_ped_type": "float", + "gpu": "NVIDIA GeForce RTX 4090", + "driver": "595.71.05", + "gpu_busy_pct": "0 %", + "nvcc": "Build cuda_12.4.r12.4/compiler.34097967_0", + "python": "3.11.15" +} diff --git a/python/tests/perf/results/2026-09-10_f32/probes.csv b/python/tests/perf/results/2026-09-10_f32/probes.csv new file mode 100644 index 00000000..ecd247f9 --- /dev/null +++ b/python/tests/perf/results/2026-09-10_f32/probes.csv @@ -0,0 +1,5 @@ +n_frames,window_ms,window_us_per_frame,window_fps,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,467.758071,23.38790355,42757.145712617756,20000,112.903327,112.777756,1.0011134376534323,24.110274732170254,5.6388878,19999,326.931385,326.931385,1.0,69.89326433236467,16.34656925,20000,152.729782,152.729782,1.0,32.65144771815172,7.6364891,H2D,16.34656925,61174.91595369468,3x3_s4,3,3000,4,2000,float +20000,861.486383,43.07431915,23215.689063305832,20000,92.96108,92.96108,1.0,10.790777641345493,4.648054,19999,262.616382,262.616382,1.0,30.484101337211733,13.130819099999998,20000,105.867334,105.867334,1.0,12.288915540525728,5.2933667,H2D,13.130819099999998,76156.71135093166,3x3_s1_uncontended,3,3000,1,2000,float +20000,1502.075647,75.10378235,13314.908633226778,20000,312.858126,311.891228,1.0031001128380566,20.76401602162451,15.5945614,19999,414.382371,414.382371,1.0,27.58731704542441,20.719118549999997,20000,507.805309,507.805309,1.0,33.806906464012464,25.39026545,D2H,25.39026545,39385.17310775103,9x9_s4,9,1700,4,2000,float +20000,1652.407506,82.6203753,12103.551894661994,20000,305.830789,305.830789,1.0,18.508194128234614,15.29153945,19999,264.610496,264.610496,1.0,16.01363435104125,13.2305248,20000,440.039995,440.039995,1.0,26.630234576046522,22.00199975,D2H,22.00199975,45450.41411519878,9x9_s1_uncontended,9,1700,1,2000,float diff --git a/python/tests/perf/results/2026-09-10_f32_ref7177/env.json b/python/tests/perf/results/2026-09-10_f32_ref7177/env.json new file mode 100644 index 00000000..70894656 --- /dev/null +++ b/python/tests/perf/results/2026-09-10_f32_ref7177/env.json @@ -0,0 +1,14 @@ +{ + "timestamp": "2026-09-10 12:31:37", + "host": "pc-moench-04.psi.ch", + "git_rev": "7177f00", + "git_branch": "HEAD", + "git_dirty": true, + "aare_version": "2026.7.2", + "device_ped_type": "float", + "gpu": "NVIDIA GeForce RTX 4090", + "driver": "595.71.05", + "gpu_busy_pct": "0 %", + "nvcc": "Build cuda_12.4.r12.4/compiler.34097967_0", + "python": "3.11.15" +} diff --git a/python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv b/python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv new file mode 100644 index 00000000..b71d6bef --- /dev/null +++ b/python/tests/perf/results/2026-09-10_f32_ref7177/probes.csv @@ -0,0 +1,5 @@ +n_frames,window_ms,window_us_per_frame,window_fps,kernel_n,kernel_sum_ms,kernel_busy_ms,kernel_overlap,kernel_duty_pct,kernel_us_per_frame,H2D_n,H2D_sum_ms,H2D_busy_ms,H2D_overlap,H2D_duty_pct,H2D_us_per_frame,D2H_n,D2H_sum_ms,D2H_busy_ms,D2H_overlap,D2H_duty_pct,D2H_us_per_frame,bottleneck,roofline_us_per_frame,roofline_fps,label,cluster_dim,cap,n_streams,batch,device_ped_type +20000,657.208045,32.86040225,30431.763810803626,20000,100.861444,100.083645,1.0077714895375762,15.228609229821586,5.00418225,19999,316.797856,316.797856,1.0,48.20358764780489,15.839892800000001,20000,144.76269,144.76269,1.0,22.02691995348292,7.2381345,H2D,15.839892800000001,63131.740386525846,3x3_s4,3,3000,4,2000,float +20000,1151.676025,57.58380124999999,17365.99492031624,20000,86.388419,86.388419,1.0,7.501104227640755,4.31942095,19999,262.127428,262.127428,1.0,22.760517915617807,13.1063714,20000,105.9432,105.9432,1.0,9.199045365210237,5.29716,H2D,13.1063714,76298.76870420443,3x3_s1_uncontended,3,3000,1,2000,float +20000,1845.553098,92.2776549,10836.85970437465,20000,511.593396,494.436337,1.0347002388701865,26.790686084069524,24.72181685,19999,342.849695,342.849695,1.0,18.577070222013194,17.14248475,20000,495.751951,495.751951,1.0,26.86197170578508,24.78759755,D2H,24.78759755,40342.75600863949,9x9_s4,9,1700,4,2000,float +20000,2824.450123,141.22250615000002,7081.0243158965495,20000,475.987403,475.987403,1.0,16.852391873517252,23.799370149999998,19999,262.138573,262.138573,1.0,9.281048047737114,13.10692865,20000,439.610736,439.610736,1.0,15.564471555725893,21.9805368,kernel,23.799370149999998,42017.918696894594,9x9_s1_uncontended,9,1700,1,2000,float diff --git a/python/tests/perf/run_probes.py b/python/tests/perf/run_probes.py new file mode 100644 index 00000000..17a07d85 --- /dev/null +++ b/python/tests/perf/run_probes.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""nsys probe sweep: per-engine GPU time per frame, duty cycles, roofline. + + python3.11 run_probes.py # full sweep, 20k frames + python3.11 run_probes.py --frames 2000 # quick check (clocks may not ramp) + python3.11 run_probes.py --only 9x9_s1_uncontended --tag mytag + +Per config: an .nsys-rep + .sqlite in results/_[_tag]/ and one +row in probes.csv (re-running a config replaces its own row only). + +20 000 frames, not fewer: over a short run the GPU clocks never fully ramp and +the kernel reads ~10 % slow. +""" +from __future__ import annotations + +import argparse +import csv +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +import gpu_span # noqa: E402 + +HERE = Path(__file__).resolve().parent + +# (cluster_dim, cap, n_streams, label) +# s4 = the production configuration. +# s1 = uncontended control: one stream, so H2D and D2H never coexist and the +# kernel number is the kernel alone. Quote kernel changes from s1. +# The cap sets the D2H bar (the slot is copied whole regardless of fill), so a +# probe at a different cap is a different measurement. +CONFIGS = [ + (3, 3000, 4, "3x3_s4"), + (3, 3000, 1, "3x3_s1_uncontended"), + (9, 1700, 4, "9x9_s4"), + (9, 1700, 1, "9x9_s1_uncontended"), +] + + +def run_one(cdim, cap, streams, label, n_frames, batch, outdir) -> dict | None: + rep = outdir / f"probe_{label}_cap{cap}" + print(f"\n--- {label}: {cdim}x{cdim} cap={cap} streams={streams} N={n_frames} ---") + + prof = [common.NSYS, "profile", "--trace=cuda", "--sample=none", + "--cpuctxsw=none", "--force-overwrite=true", "-o", str(rep), + sys.executable, str(HERE / "nsys_kernel_probe.py"), + str(streams), str(n_frames), str(cdim), str(cap), str(batch)] + p = subprocess.run(prof, capture_output=True, text=True) + for line in p.stdout.splitlines(): + if line.strip().startswith(("n_streams", "H2D/frame", "wall")): + print(" ", line.strip()) + if not rep.with_suffix(".nsys-rep").exists(): + print(f" FAILED: {p.stderr.strip()[-400:]}") + return None + + subprocess.run([common.NSYS, "stats", "--force-export=true", "--report", + "cuda_gpu_sum", str(rep.with_suffix(".nsys-rep"))], + capture_output=True, text=True) + sq = rep.with_suffix(".sqlite") + if not sq.exists(): + print(" FAILED: no sqlite export") + return None + + r = gpu_span.analyze(sq, n_frames) + r.update(label=label, cluster_dim=cdim, cap=cap, n_streams=streams, + batch=batch, device_ped_type=common.device_ped_type()) + print(f" kernel {r['kernel_us_per_frame']:5.2f} us (duty {r['kernel_duty_pct']:4.1f}%) " + f"H2D {r['H2D_us_per_frame']:5.2f} ({r['H2D_duty_pct']:4.1f}%) " + f"D2H {r['D2H_us_per_frame']:5.2f} ({r['D2H_duty_pct']:4.1f}%)") + print(f" -> roofline: {r['bottleneck']}-bound at " + f"{r['roofline_us_per_frame']:.2f} us/frame = {r['roofline_fps']:,.0f} FPS") + return r + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--frames", type=int, default=20_000) + ap.add_argument("--batch", type=int, default=2000) + ap.add_argument("--tag", default="") + ap.add_argument("--only", nargs="+", default=None, help="subset of labels") + ap.add_argument("--cap", type=int, default=None, help="override every cap") + args = ap.parse_args() + + common.assert_build_fresh() + common.assert_idle_gpu() + env = common.capture_env() + outdir = common.results_dir(args.tag) + common.write_env(outdir / "env.json", env) + print(f"build: DEVICE_PED_TYPE={env['device_ped_type']} git={env['git_rev']}" + f"{' (dirty)' if env['git_dirty'] else ''}") + print(f"out: {outdir}") + + rows = [] + for cdim, cap, streams, label in CONFIGS: + if args.only and label not in args.only: + continue + r = run_one(cdim, args.cap or cap, streams, label, args.frames, args.batch, outdir) + if r: + rows.append(r) + + if not rows: + return 1 + out = outdir / "probes.csv" + + def key(r): + return (str(r["label"]), str(r["cap"]), str(r["n_streams"])) + + prior = list(csv.DictReader(out.open())) if out.exists() else [] + fresh = {key(r) for r in rows} + merged = [r for r in prior if key(r) not in fresh] + rows + with out.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(merged) + + print(f"\n=== rooflines ({env['device_ped_type']} build, us/frame) ===") + print(f"{'config':<22} {'kernel':>8} {'H2D':>8} {'D2H':>8} " + f"{'bottleneck':<10} {'roofline':>9} {'FPS':>10}") + for r in rows: + print(f"{r['label']:<22} {r['kernel_us_per_frame']:8.2f} " + f"{r['H2D_us_per_frame']:8.2f} {r['D2H_us_per_frame']:8.2f} " + f"{r['bottleneck']:<10} {r['roofline_us_per_frame']:9.2f} " + f"{r['roofline_fps']:10,.0f}") + print(f"\n-> {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())