add README

2026-06-29 10:46:16 +02:00
parent 4cdebd5838
commit 561c116a8d
1 changed files with 100 additions and 0 deletions
@@ -0,0 +1,100 @@
+* slurm-eff-tool - Slurm Efficiency Tool
+
+  2026 D. Feichtinger
+
+  
+  *WORK IN PROGRESS*
+  
+  A tool to produce statistics about Slurm jobs over a given time period.
+
+  Job data are retrieved through ~sacct~ and can be saved in a compressed
+  cache file that can be re-used for different queries and output options.
+
+  The tool offers a number of useful ways to aggregate jobs. Most of the
+  displayed values then are averages over the aggregations (except columns
+  marked with _min/_max)
+
+  
+  Usage information:
+  #+begin_src bash :results output source :lang bash :exports results
+    ./slurm-eff-tool.py --help
+  #+end_src
+
+  #+RESULTS:
+  #+begin_example
+  usage: slurm-eff-tool.py [-h] [-S START] [-E END] [-u USER] [--state STATE]
+                           [-O OUTPUT_RAW] [-F FROM_RAW] [-B WRITE_BINARY_CACHE]
+                           [-L LOAD_BINARY_CACHE] [-i INFO] [-U]
+                           [-R AGGR_REGEXP] [--deflt-mpcpu DEFLT_MPCPU] [--sdev]
+                           [--json] [-s SORT] [-o FORMAT] [--expr EXPR]
+                           [-p PRESET]
+
+  Display seff-style CPU, memory and walltime efficiency values from sacct data.
+
+  options:
+    -h, --help            show this help message and exit
+    -S, --start START     sacct start time, passed to sacct -S
+    -E, --end END         sacct end time, passed to sacct -E
+    -u, --user USER       restrict to one user; passed as sacct -u unless
+                          reading from cache
+    --state, --job-state STATE
+                          sacct state filter, e.g. COMPLETED,FAILED,TIMEOUT
+    -O, --output-raw OUTPUT_RAW
+                          write raw sacct output cache to this file
+    -F, --from-raw FROM_RAW
+                          read raw sacct output cache from this file instead of
+                          running sacct
+    -B, --write-binary-cache WRITE_BINARY_CACHE
+                          write a binary cache file in msgpack format
+    -L, --load-binary-cache LOAD_BINARY_CACHE
+                          load a binary cache file in msgpack format
+    -i, --info INFO       show information for the given binary cache file
+    -U, --aggr-user       aggregate jobs by user, CPUs, nodes, ReqMem, and
+                          timelimit
+    -R, --aggr-regexp AGGR_REGEXP
+                          aggregate jobs matching regexp by regexp, CPUs, nodes,
+                          ReqMem, and timelimit; may be repeated
+    --deflt-mpcpu DEFLT_MPCPU
+                          Default memory to CPU ratio of cluster (2.0 GB/cpu)
+    --sdev                after each efficiency average, add sdev, max, and min
+                          columns
+    --json                emit JSON instead of an ASCII table
+    -s, --sort SORT       comma-separated numeric sort columns or aliases;
+                          prefix with - for descending
+    -o, --format FORMAT   String of comma separated column names or short names
+                          defining the output format: u:username, i:JobID,
+                          c:CPUs, N:Nodes, m:ReqMem, n:UsedMem, p:MemPerCPU,
+                          l:ReqWalltime, C:Count, e:CPU_Eff, M:Mem_Eff,
+                          t:Time_Eff, j:jobname, X:waste_CPU, Y:waste_Mem
+    --expr EXPR           arithmetic expression using column names
+    -p, --preset PRESET   use one of several preset output column formats
+                          (default,all,eff)
+
+  -S/-E are ignored when reading data from a cached file.
+
+    Examples:
+    # first get an overview (-U/--aggr-user) and write a cachefile 
+    slurm-eff-tool -B sacct.cache -U
+    slurm-eff-tool.py -B sacct.cache --start 2026-05-01 --end 2026-05-22 -U
+    slurm-eff-tool.py -B sacct.cache --start 2026-05-01 --end now -U
+
+    # now you can read the cachefile for later runs and e.g. sort based on waste_Mem
+    slurm-eff-tool -L sacct.cache -U -s=-waste_mem
+
+    # only list a specific user's summary lines
+    slurm-eff-tool.py -L sacct.cache -U -u dfeich
+    # list that user's single jobs
+    slurm-eff-tool.py -L sacct.cache -u dfeich
+
+    # supports multiple sort keys
+    slurm-eff-tool.py -L sacct.cache --aggr-user --sdev -s cpu,-mem,time
+
+    # you can cluster jobs by Regexps applying to the job names
+    slurm-eff-tool.py -L sacct.cache -u dfeich -R '^vasp','^gromacs'
+
+    # supports flexibel output formatting
+    slurm-eff-tool.py -L sacct.cache -o username,Y
+
+    # only print rows that evaluate to true based on arithmetic expressions
+    slurm-eff-tool.py -L sacct.cache -U --expr "(waste_Mem > 2000 and Mem_Eff < 20) and MaxRSS_max/AllocMem < 0.5"
+  #+end_example