From bb7a6607ce1ebf7e2ec4f2b0a96e3d271cd20cd9 Mon Sep 17 00:00:00 2001 From: Derek Feichtinger Date: Tue, 2 Jun 2026 15:22:15 +0200 Subject: [PATCH] some more helpful naming --- slurm-eff-tool.py | 53 ++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/slurm-eff-tool.py b/slurm-eff-tool.py index 19012f3..fb44503 100755 --- a/slurm-eff-tool.py +++ b/slurm-eff-tool.py @@ -136,9 +136,9 @@ class OutputRow: Memory_Efficiency: float | None Time_Efficiency: float | None jobname: str - _cpu_values: list[float] = field(default_factory=list, repr=False) - _mem_values: list[float] = field(default_factory=list, repr=False) - _time_values: list[float] = field(default_factory=list, repr=False) + _cpu_eff_values: list[float] = field(default_factory=list, repr=False) + _mem_eff_values: list[float] = field(default_factory=list, repr=False) + _time_eff_values: list[float] = field(default_factory=list, repr=False) def as_dict(self, sdev: bool = False) -> dict[str, Any]: d: dict[str, Any] = { @@ -157,9 +157,9 @@ class OutputRow: } if sdev: for col, vals in [ - ("CPU_Efficiency", self._cpu_values), - ("Memory_Efficiency", self._mem_values), - ("Time_Efficiency", self._time_values), + ("CPU_Efficiency", self._cpu_eff_values), + ("Memory_Efficiency", self._mem_eff_values), + ("Time_Efficiency", self._time_eff_values), ]: d[f"{col}_sdev"] = stdev_or_none(vals) d[f"{col}_max"] = max(vals) if vals else None @@ -172,7 +172,7 @@ def die(msg: str, code: int = 2) -> None: raise SystemExit(code) -def parse_duration_to_seconds(value: str) -> float: +def slurm_duration_to_seconds(value: str) -> float: """Parse Slurm-ish CPU time strings such as 01:02:03, 2-01:02:03, 5:12.345.""" value = (value or "").strip() if not value or value in {"Unknown", "UNLIMITED", "Partition_Limit"}: @@ -392,7 +392,7 @@ def build_job_records(rows: list[dict[str, str]], only_user: str | None = None) timelimit_seconds = int(round(timelimit_raw_minutes * 60)) cputime_raw = float(top.get("CPUTimeRAW") or 0) - totalcpu = parse_duration_to_seconds(top.get("TotalCPU", "")) + totalcpu = slurm_duration_to_seconds(top.get("TotalCPU", "")) reqmem = top.get("ReqMem", "") reqmem_total = reqmem_total_bytes(reqmem, cpus, nodes) @@ -433,6 +433,7 @@ def build_job_records(rows: list[dict[str, str]], only_user: str | None = None) def aggregate_records(records: list[JobRecord], args: argparse.Namespace) -> list[OutputRow]: + """Aggregate records according to given grouping instructions.""" if args.aggr_regexp: compiled = [(pat, re.compile(pat)) for pat in args.aggr_regexp] buckets: dict[tuple[Any, ...], list[JobRecord]] = defaultdict(list) @@ -465,6 +466,7 @@ def aggregate_records(records: list[JobRecord], args: argparse.Namespace) -> lis def make_single_row(rec: JobRecord) -> OutputRow: + """Returns an OutputRow based on a single slurm job record.""" return OutputRow( username=rec.username, JobID=rec.raw.get("JobIDRaw", rec.raw.get("JobID", "")), @@ -478,17 +480,18 @@ def make_single_row(rec: JobRecord) -> OutputRow: Memory_Efficiency=rec.mem_eff, Time_Efficiency=rec.time_eff, jobname=rec.jobname, - _cpu_values=[rec.cpu_eff] if rec.cpu_eff is not None else [], - _mem_values=[rec.mem_eff] if rec.mem_eff is not None else [], - _time_values=[rec.time_eff] if rec.time_eff is not None else [], + _cpu_eff_values=[rec.cpu_eff] if rec.cpu_eff is not None else [], + _mem_eff_values=[rec.mem_eff] if rec.mem_eff is not None else [], + _time_eff_values=[rec.time_eff] if rec.time_eff is not None else [], ) def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) -> OutputRow: + """Returns an OutputRow based on the given list of job records.""" first = records[0] - cpu_vals = [r.cpu_eff for r in records if r.cpu_eff is not None] - mem_vals = [r.mem_eff for r in records if r.mem_eff is not None] - time_vals = [r.time_eff for r in records if r.time_eff is not None] + cpu_eff_vals = [r.cpu_eff for r in records if r.cpu_eff is not None] + mem_eff_vals = [r.mem_eff for r in records if r.mem_eff is not None] + time_eff_vals = [r.time_eff for r in records if r.time_eff is not None] return OutputRow( username=username, JobID="", @@ -498,17 +501,18 @@ def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) -> MemPerCPU=first.mem_per_cpu_gb, ReqWalltime=first.reqwall_hours, Count=len(records), - CPU_Efficiency=mean_or_none(cpu_vals), - Memory_Efficiency=mean_or_none(mem_vals), - Time_Efficiency=mean_or_none(time_vals), + CPU_Efficiency=mean_or_none(cpu_eff_vals), + Memory_Efficiency=mean_or_none(mem_eff_vals), + Time_Efficiency=mean_or_none(time_eff_vals), jobname=jobname, - _cpu_values=cpu_vals, - _mem_values=mem_vals, - _time_values=time_vals, + Testcolumn=10, + _cpu_eff_values=cpu_eff_vals, + _mem_eff_values=mem_eff_vals, + _time_eff_values=time_eff_vals, ) - -def resolve_column(name: str) -> str: +def resolve_column_name(name: str) -> str: + """Returns canonicalized full column name, accepts one letter column codes""" n = name.strip() reverse = {v.lower(): v for v in DEFAULT_COLUMNS} reverse.update({v.lower(): v for v in NUMERIC_COLUMNS}) @@ -533,7 +537,7 @@ def sort_rows(rows: list[OutputRow], spec: str | None) -> list[OutputRow]: desc = t.startswith("-") asc = t.startswith("+") name = t[1:] if (desc or asc) else t - col = resolve_column(name) + col = resolve_column_name(name) parsed.append((col, desc)) sorted_rows = rows @@ -554,6 +558,7 @@ def sort_rows(rows: list[OutputRow], spec: str | None) -> list[OutputRow]: return sorted_rows +# adds sdev columns behind their respective avg column def columns_for_sdev(base_cols: list[str]) -> list[str]: out: list[str] = [] for col in base_cols: @@ -618,7 +623,7 @@ def print_custom_format(rows: list[OutputRow], fmt: str, sdev: bool) -> None: def repl(match: re.Match[str]) -> str: widthspec, alias = match.groups() - col = resolve_column(alias) + col = resolve_column_name(alias) val = d.get(col) text = format_value(val, col) if widthspec: