diff --git a/slurm-eff-tool.py b/slurm-eff-tool.py index cf44f53..aa1f5f0 100755 --- a/slurm-eff-tool.py +++ b/slurm-eff-tool.py @@ -17,7 +17,21 @@ Waste definitions: (req_mem - (n_cpus * default_mem_per_cpu )) [GB*hours] waste_total = waste_CPU + waste_Mem/default_mem_per_cpu -2006 D. Feichtinger + +Copyright 2026 Paul Scherrer Institut (PSI) +Author: Derek Feichtinger + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. """ from __future__ import annotations @@ -1520,85 +1534,95 @@ plot_cmd = 'plot "{tmpfile.name}" \\ def parse_args(argv: list[str]) -> argparse.Namespace: p = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description="Display seff-style CPU, memory and walltime efficiency values from sacct data.", - epilog="""-S/-E are ignored when reading data from a cached file. + description="Analyze efficiency related metrics from slurm sacct based data.", + epilog="""-S/-E currently are ignored when reading data back from a cached file. Examples: - # first get an overview (-U/--aggr-user) and write a cachefile - slurm-eff-tool -B sacct.cache -U - slurm-eff-tool.py -B sacct.cache --start 2026-05-01 --end 2026-05-22 -U - slurm-eff-tool.py -B sacct.cache --start 2026-05-01 --end now -U + # first get an overview (-U/-U) and write a cache file + slurm-eff-tool -B cache_file -U + slurm-eff-tool.py -B cache_file --start 2026-05-01 --end 2026-05-22 -U + slurm-eff-tool.py -B cache_file --start 2026-05-01 --end now -U - # now you can read the cachefile for later runs and e.g. sort based on waste_Mem - slurm-eff-tool -L sacct.cache -U -s=-waste_mem + # now you can read the cache file for later runs and e.g. sort based on waste_Mem + slurm-eff-tool -L cache_file -U -s=-waste_mem - # only list a specific user's summary lines - slurm-eff-tool.py -L sacct.cache -U -u dfeich - # list that user's single jobs - slurm-eff-tool.py -L sacct.cache -u dfeich + # only list a specific user's aggregated lines + slurm-eff-tool.py -L cache_file -U -u feichtinger + # list that user's single jobs without any aggregation + slurm-eff-tool.py -L cache_file -u feichtinger - # supports sorting, also by multiple keys - slurm-eff-tool.py -L sacct.cache --aggr-user -s-waste_total - slurm-eff-tool.py -L sacct.cache --aggr-user -s cpu,-mem,time + # supports sorting, also by multiple keys (note that -s-usedmem means sorting in + # negative order of the usedmem metric) + slurm-eff-tool.py -L cache_file -U -s-waste_total + slurm-eff-tool.py -L cache_file -U -s cpu,-mem,time - # you can cluster jobs by multiple regexps applying to the job names - slurm-eff-tool.py -L sacct.cache -u dfeich -R '^vasp','^gromacs' + # you can further cluster jobs by multiple regexps applying to the job names + slurm-eff-tool.py -L cache_file -u feichtinger -R '^vasp','^gromacs' - # supports flexibel output formatting by defining columns to print - slurm-eff-tool.py -L sacct.cache -o username,Y + # supports flexibel output formatting by allowing you to define the + # columns to print with the -o option + slurm-eff-tool.py -L cache_file -o username,mem_eff,usedmem + # There's a bunch of preset formats that can be selected with the -p option + slurm-eff-tool.py -L cache_file -p time # only print rows that evaluate to true based on arithmetic expressions - slurm-eff-tool.py -L sacct.cache -U --expr "(waste_Mem > 2000 and Mem_Eff < 20) and MaxRSS_max/AllocMem < 0.5" - slurm-eff-tool.py -L sacct.cache -U --expr 'username=="feichtinger"' - slurm-eff-tool.py -L sacct.cache -U --expr 'partition in ["standard","short"]' + slurm-eff-tool.py -L cache_file -U --expr "(waste_Mem > 2000 and Mem_Eff < 20) and MaxRSS_max/AllocMem < 0.5" + slurm-eff-tool.py -L cache_file -U --expr 'username=="feichtinger"' + slurm-eff-tool.py -L cache_file -U --expr 'partition in ["standard","short"]' + # generate histograms for selected metrics (see above for available metrics) + slurm-eff-tool -L cache_file -U -s=-waste_mem -H usedmem,walltime """ ) - p.add_argument("--avg-function", choices=["median","mean"], default="median", - help="averaging function to use for reporting aggregate rows",) - p.add_argument("-S", "--start", help="sacct start time, passed to sacct -S", + # Options affecting sacct gathering + p.add_argument("-S", "--start", help="sacct start time, passed to sacct -S [default: now - 24 hours]", default="now - 24 hours") - p.add_argument("-E", "--end", help="sacct end time, passed to sacct -E", + p.add_argument("-E", "--end", help="sacct end time, passed to sacct -E [default: now]", default="now") - p.add_argument("-u", "--user", help="comma separated list of users (affects sacct DB search or filters later from cache file)") - p.add_argument("-U", "--aggr-user", action="store_true", help="aggregate jobs by user, CPUs, nodes, ReqMem, and timelimit") + p.add_argument("-u", "--user", help="filter based on comma separated list of users (affects sacct DB search as well as later filtering from cache file)") + p.add_argument("--state", "--job-state", dest="state", + default=None, + help="state filter passed to sacct, e.g. COMPLETED,FAILED,TIMEOUT [default: get all states]") + # Aggregation related options + p.add_argument("-U", "--aggr_user", action="store_true", help="aggregate jobs by user, CPUs, nodes, ReqMem, and timelimit") p.add_argument( "-R", "--aggr-regexp", action="append", - help="aggregate jobs based on a regexp applied to the jobnames. Option may be repeated multiple times.", + help="additionally aggregate jobs based on a regexp applied to the jobnames. Option may be repeated multiple times.", ) - p.add_argument("--state", "--job-state", dest="state", - default=None, - help="state filter passed to sacct, e.g. COMPLETED,FAILED,TIMEOUT") - - p.add_argument("-B", "--write-binary-cache", help="write a binary cache file in gzipped msgpack format") - p.add_argument("-L", "--load-binary-cache", help="load a binary cache file in gzipped msgpack format") - p.add_argument("-O", "--output-raw", help="write raw sacct output cache to this file (text format, large).") - p.add_argument("-F", "--from-raw", help="read raw sacct output cache from this file.") - p.add_argument("-i", "--info", help="show metadata information for the given binary cache file") - p.add_argument("-H", "--histo", help=f"print histogram of the given metric (Supported columns: {', '.join(HISTO_COLUMNS)}).", default=None) - + p.add_argument("--avg-function", choices=["median","mean"], default="median", + help="averaging function to use for reporting aggregate rows",) p.add_argument("--dflt-mpcpu", help=f"Default memory/CPU ratio of cluster [{default_mempercpu_gb} GB/cpu]. Used in memory waste calculation.") p.add_argument("--sdev", action="store_true", help="after each efficiency average, add sdev, max, and min columns") - p.add_argument("--json", action="store_true", help="emit JSON instead of an ASCII table") - p.add_argument("-s", "--sort", help="comma-separated numeric sort columns or short aliases; prefix with - for descending.") + + # Output related options fmthelp = ", ".join([f"{k}:{ALIASES[k]}" for k in ALIASES]) p.add_argument( "-o", "--format", help= f"String of comma separated column names or short aliases defining the output format [{fmthelp}]", ) - p.add_argument("--expr", help="filter using an arithmetic expression using column names as variables (case insensitive). Don't use for state (use --state flag instead).", - default=None) p.add_argument("-p", "--preset", help=f"use one of several preset output column formats [{','.join(PRESET_COLUMNS.keys())}]", default=None) + p.add_argument("--json", action="store_true", help="emit JSON instead of an ASCII table") + p.add_argument("-s", "--sort", help="comma-separated numeric sort columns or short aliases; prefix with - for descending.") + p.add_argument("--expr", help="filter using an arithmetic expression using column names as variables (case insensitive). Don't use for state (use --state flag instead).", + default=None) + p.add_argument("-H", "--histo", help=f"print histogram of the given metric (Supported columns: {', '.join(HISTO_COLUMNS)}).", default=None) + # Options for writing/reading cache files + p.add_argument("-B", "--write-binary-cache", help="write a binary cache file in gzipped msgpack format") + p.add_argument("-L", "--load-binary-cache", help="load a binary cache file in gzipped msgpack format") + p.add_argument("-O", "--output-raw", help="write a raw sacct output (ASCII) to this cache file (large).") + p.add_argument("-F", "--from-raw", help="read raw sacct output cache from this ASCII file (that was produced by -O option).") + p.add_argument("-i", "--info", help="show metadata information for the given binary cache file") + args = p.parse_args(argv) if args.aggr_user and args.aggr_regexp: - die("choose only one aggregation mode: --aggr-user or --aggr-regexp") + die("choose only one aggregation mode: -U or --aggr-regexp") return args