diff --git a/slurm-eff-tool.py b/slurm-eff-tool.py index 069581a..36496f7 100755 --- a/slurm-eff-tool.py +++ b/slurm-eff-tool.py @@ -42,10 +42,10 @@ from typing import Any, Iterable, Mapping import msgpack import gzip -VERSION=0.21 +VERSION=0.22 # Default GB per CPUs for the cluster -default_mempercpu_gb = 2.0 +default_mempercpu_gb = 2.2 SACCT_FIELDS = [ "JobIDRaw", @@ -64,6 +64,7 @@ SACCT_FIELDS = [ "ReqTRes", "AllocTRES", "TRESUsageInTot", + "PlannedCPURAW" ] ALL_COLUMNS = [ @@ -82,6 +83,8 @@ ALL_COLUMNS = [ "MaxRSS_max", "MemPerCPU", "Mem_Eff", + "Planned_Time", + "Elig_Qtime", "waste_Mem", "ReqWalltime", "Walltime", @@ -134,6 +137,8 @@ PRESET_COLUMNS = { "Walltime", "Walltime_max", "Time_Eff", + # "Planned_Time", + "Elig_Qtime", "jobname",], "cpu": [ "username", @@ -208,6 +213,7 @@ NUMERIC_COLUMNS = { "AllocMem", "UsedMem", "MemPerCPU", + "Elig_Qtime", "ReqWalltime", "Count", "CPU_Eff", @@ -361,9 +367,11 @@ class JobRecord: mem_per_cpu_gb: float | None mem_alloc_tres: float | None mem_used_tres: float | None + planned_sec: float reqwall_hours: float | None reqmem_bytes_total: float | None elapsed_sec: int + elig_qtime_sec: float timelimit_sec: int totalcpu_sec: float maxrss_bytes: float | None @@ -437,6 +445,8 @@ class OutputRow: UsedMem: float | None MemPerCPU: float | None ReqWalltime: float | None + Planned_Time: float | None + Elig_Qtime: float | None Count: int CPU_Eff: float | None Mem_Eff: float | None @@ -468,6 +478,8 @@ class OutputRow: "ReqWalltime": self.ReqWalltime, "Walltime": self.walltime, "Walltime_max": self.walltime_max, + "Elig_Qtime": self.Elig_Qtime, + "Planned_Time": self.Planned_Time, "Count": self.Count, "CPU_Eff": self.CPU_Eff, "Mem_Eff": self.Mem_Eff, @@ -780,6 +792,8 @@ def build_job_records(rows: list[dict[str, str]]) -> list[JobRecord]: continue nodes = int(float(top.get("NNodes") or 0)) elapsed = int(float(top.get("ElapsedRaw") or 0)) + planned_sec = float(top.get("PlannedCPURAW") or 0) + elig_qtime_sec = planned_sec / alloc_cpus # sacct TimelimitRaw is minutes, while ElapsedRaw and CPUTimeRAW are seconds. timelimit_raw_minutes = float(top.get("TimelimitRaw") or 0) @@ -838,9 +852,11 @@ def build_job_records(rows: list[dict[str, str]]) -> list[JobRecord]: mem_used_tres=mem_used_tres_gb, mem_per_cpu_gb=mem_per_cpu_gb, mem_alloc_tres=mem_alloc_tres_gb, + planned_sec=planned_sec, reqwall_hours=reqwall_hours, reqmem_bytes_total=reqmem_total, elapsed_sec=elapsed, + elig_qtime_sec=elig_qtime_sec, timelimit_sec=timelimit_seconds, totalcpu_sec=totalcpu, maxrss_bytes=maxrss, @@ -923,6 +939,8 @@ def make_single_row(rec: JobRecord, dflt_mpcpu: float) -> OutputRow: """Returns an OutputRow based on a single slurm job record.""" walltime = rec.elapsed_sec / 3600 + # TODO elig_qtime_sec = rec. + waste_mem = 0 if rec.mem_eff is not None and rec.reqmem_gb is not None: waste_mem = max(0,walltime * (100-rec.mem_eff)/100 \ @@ -954,6 +972,8 @@ def make_single_row(rec: JobRecord, dflt_mpcpu: float) -> OutputRow: UsedMem=rec.mem_used_tres, MemPerCPU=rec.mem_per_cpu_gb, ReqWalltime=rec.reqwall_hours, + Planned_Time=rec.planned_sec/3600, + Elig_Qtime=rec.elig_qtime_sec/3600, Count=1, CPU_Eff=rec.cpu_eff, Mem_Eff=rec.mem_eff, @@ -983,6 +1003,11 @@ def make_aggregate_row(records: list[JobRecord], username: str, if walltime is not None: walltime /= 3600 + planned_sec = mean_or_none([r.planned_sec for r in records + if r.planned_sec is not None]) + elig_qtime_sec = mean_or_none([r.elig_qtime_sec for r in records + if r.elig_qtime_sec is not None]) + # We average over all jobs' allocated memory. Some jobs could have received # different allocations, even though all of them had the same user required # Memory. Maybe should generate a warning. Most of the time, what the user @@ -1023,6 +1048,8 @@ def make_aggregate_row(records: list[JobRecord], username: str, UsedMem=used_mem, MemPerCPU=first.mem_per_cpu_gb, ReqWalltime=first.reqwall_hours, + Planned_Time=planned_sec/3600, + Elig_Qtime=elig_qtime_sec/3600, Count=count, CPU_Eff=cpu_efficiency, Mem_Eff=memory_efficiency, @@ -1096,28 +1123,15 @@ def columns_for_sdev(base_cols: list[str]) -> list[str]: return out - - -def format_gb_value(value: float | None) -> str: - if value is None: - return "NA" - return f"{value:.2f}G" - - -def format_secs_to_hours(value: float | None) -> str: - if value is None: - return "NA" - return f"{value:.2f}h" - - def format_value(value: Any, column: str | None = None) -> str: - if column in ["ReqMem", "UsedMem","MemPerCPU", "MaxRSS_max", "AllocMem"]: - return format_gb_value(value) - if column in ["ReqWalltime", "Walltime", "Walltime_max"]: - return format_secs_to_hours(value) - if value is None: return "NA" + if column in ["ReqMem", "UsedMem","MemPerCPU", "MaxRSS_max", "AllocMem"]: + return f"{value:.2f}G" + if column in ["ReqWalltime", "Walltime", "Walltime_max", + "Elig_Qtime", "Planned_Time"]: + return f"{value:.2f}h" + if isinstance(value, float): return f"{value:.2f}" return str(value) @@ -1233,6 +1247,14 @@ def main(argv: list[str] | None = None) -> int: show_cache_info(args.info) sys.exit(0) + if args.write_binary_cache and os.path.exists(args.write_binary_cache): + sys.stderr.write(f"WARNING, file exists: {args.write_binary_cache}. Balking out...\n") + sys.exit(1) + if args.output_raw and os.path.exists(args.output_raw): + sys.stderr.write(f"WARNING, file exists: {args.output_raw}. Balking out...\n") + sys.exit(1) + + output_columns = PRESET_COLUMNS["default"] if args.preset: try: