introduce MaxRSS_max, Walltime_max. Reorder columns

slurm-eff-tool.py

@@ -58,18 +58,21 @@ SACCT_FIELDS = [
     "MaxRSS",
 ]
 
+# Default columns to print in output, and their order
 DEFAULT_COLUMNS = [
     "username",
     "JobID",
+    "Count",
     "CPUs",
     "Nodes",
+    "CPU_Efficiency",
     "ReqMem",
     "MemPerCPU",
     "MaxRSS_max",
-    "ReqWalltime",
-    "Count",
-    "CPU_Efficiency",
     "Memory_Efficiency",
+    "ReqWalltime",
+    "Walltime",
+    "Walltime_max",
     "Time_Efficiency",
     "jobname",
 ]
@@ -98,8 +101,10 @@ NUMERIC_COLUMNS = {
     "ReqWalltime",
     "Count",
     "CPU_Efficiency",
+    "MaxRSS_max",
     "Memory_Efficiency",
     "Time_Efficiency",
+    "Walltime_max",
 }
 
 
@@ -138,6 +143,8 @@ class OutputRow:
     Time_Efficiency: float | None
     jobname: str
     maxrss_max: float | None
+    walltime: float | None
+    walltime_max: float | None
     _cpu_eff_values: list[float] = field(default_factory=list, repr=False)
     _mem_eff_values: list[float] = field(default_factory=list, repr=False)
     _time_eff_values: list[float] = field(default_factory=list, repr=False)
@@ -152,6 +159,8 @@ class OutputRow:
             "MaxRSS_max": self.maxrss_max,
             "MemPerCPU": self.MemPerCPU,
             "ReqWalltime": self.ReqWalltime,
+            "Walltime": self.walltime,
+            "Walltime_max": self.walltime_max,
             "Count": self.Count,
             "CPU_Efficiency": self.CPU_Efficiency,
             "Memory_Efficiency": self.Memory_Efficiency,
@@ -470,6 +479,7 @@ def aggregate_records(records: list[JobRecord], args: argparse.Namespace) -> lis
 
 def make_single_row(rec: JobRecord) -> OutputRow:
     """Returns an OutputRow based on a single slurm job record."""
+    walltime=rec.elapsed_sec / 3600
     return OutputRow(
         username=rec.username,
         JobID=rec.raw.get("JobIDRaw", rec.raw.get("JobID", "")),
@@ -484,6 +494,8 @@ def make_single_row(rec: JobRecord) -> OutputRow:
         Time_Efficiency=rec.time_eff,
         jobname=rec.jobname,
         maxrss_max=rec.maxrss_bytes / (1024**3) if rec.maxrss_bytes is not None else None,
+        walltime=walltime,
+        walltime_max= walltime,
         _cpu_eff_values=[rec.cpu_eff] if rec.cpu_eff is not None else [],
         _mem_eff_values=[rec.mem_eff] if rec.mem_eff is not None else [],
         _time_eff_values=[rec.time_eff] if rec.time_eff is not None else [],
@@ -496,6 +508,9 @@ def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) ->
     cpu_eff_vals = [r.cpu_eff for r in records if r.cpu_eff is not None]
     mem_eff_vals = [r.mem_eff for r in records if r.mem_eff is not None]
     time_eff_vals = [r.time_eff for r in records if r.time_eff is not None]
+    walltime=mean_or_none([r.elapsed_sec for r in records if r.elapsed_sec is not None])
+    if walltime is not None:
+        walltime /= 3600
     return OutputRow(
         username=username,
         JobID="",
@@ -510,6 +525,8 @@ def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) ->
         Time_Efficiency=mean_or_none(time_eff_vals),
         jobname=jobname,
         maxrss_max=max([r.maxrss_bytes for r in records if r.maxrss_bytes is not None]) / (1024**3) or None,
+        walltime=walltime,
+        walltime_max=max([r.elapsed_sec for r in records if r.elapsed_sec is not None]) / 3600,
         _cpu_eff_values=cpu_eff_vals,
         _mem_eff_values=mem_eff_vals,
         _time_eff_values=time_eff_vals,
@@ -580,7 +597,7 @@ def format_gb_value(value: float | None) -> str:
     return f"{value:.2f}G"
 
 
-def format_reqwall_hours(value: float | None) -> str:
+def format_secs_to_hours(value: float | None) -> str:
     if value is None:
         return "NA"
     return f"{value:.2f}h"
@@ -589,8 +606,8 @@ def format_reqwall_hours(value: float | None) -> str:
 def format_value(value: Any, column: str | None = None) -> str:
     if column in ["ReqMem", "MemPerCPU", "MaxRSS_max"]:
         return format_gb_value(value)
-    if column == "ReqWalltime":
-        return format_reqwall_hours(value)
+    if column in ["ReqWalltime", "Walltime", "Walltime_max"]:
+        return format_secs_to_hours(value)
 
     if value is None:
         return "NA"
@@ -710,7 +727,7 @@ def main(argv: list[str] | None = None) -> int:
     if args.aggr_user or args.aggr_regexp:
         output_columns = [c for c in DEFAULT_COLUMNS if c != "JobID"]
     else:
-        output_columns = DEFAULT_COLUMNS
+        output_columns = [c for c in DEFAULT_COLUMNS if c != "Count"]
 
     if args.json:
         print(json.dumps([r.as_dict(sdev=args.sdev) for r in out_rows], indent=2))