HPCE/slurm-efficiency-tools
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

some more helpful naming

Browse Source
This commit is contained in:
feichtinger
2026-06-02 15:22:15 +02:00
parent b0935e83dc
commit bb7a6607ce
1 changed files with 29 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
slurm-eff-tool.py
+29 -24
View File
@@ -136,9 +136,9 @@ class OutputRow:
Memory_Efficiency: float | None
Time_Efficiency: float | None
jobname: str
_cpu_values: list[float] = field(default_factory=list, repr=False)
_mem_values: list[float] = field(default_factory=list, repr=False)
_time_values: list[float] = field(default_factory=list, repr=False)
_cpu_eff_values: list[float] = field(default_factory=list, repr=False)
_mem_eff_values: list[float] = field(default_factory=list, repr=False)
_time_eff_values: list[float] = field(default_factory=list, repr=False)
def as_dict(self, sdev: bool = False) -> dict[str, Any]:
d: dict[str, Any] = {
@@ -157,9 +157,9 @@ class OutputRow:
}
if sdev:
for col, vals in [
("CPU_Efficiency", self._cpu_values),
("Memory_Efficiency", self._mem_values),
("Time_Efficiency", self._time_values),
("CPU_Efficiency", self._cpu_eff_values),
("Memory_Efficiency", self._mem_eff_values),
("Time_Efficiency", self._time_eff_values),
]:
d[f"{col}_sdev"] = stdev_or_none(vals)
d[f"{col}_max"] = max(vals) if vals else None
@@ -172,7 +172,7 @@ def die(msg: str, code: int = 2) -> None:
raise SystemExit(code)
def parse_duration_to_seconds(value: str) -> float:
def slurm_duration_to_seconds(value: str) -> float:
"""Parse Slurm-ish CPU time strings such as 01:02:03, 2-01:02:03, 5:12.345."""
value = (value or "").strip()
if not value or value in {"Unknown", "UNLIMITED", "Partition_Limit"}:
@@ -392,7 +392,7 @@ def build_job_records(rows: list[dict[str, str]], only_user: str | None = None)
timelimit_seconds = int(round(timelimit_raw_minutes * 60))
cputime_raw = float(top.get("CPUTimeRAW") or 0)
totalcpu = parse_duration_to_seconds(top.get("TotalCPU", ""))
totalcpu = slurm_duration_to_seconds(top.get("TotalCPU", ""))
reqmem = top.get("ReqMem", "")
reqmem_total = reqmem_total_bytes(reqmem, cpus, nodes)
@@ -433,6 +433,7 @@ def build_job_records(rows: list[dict[str, str]], only_user: str | None = None)
def aggregate_records(records: list[JobRecord], args: argparse.Namespace) -> list[OutputRow]:
"""Aggregate records according to given grouping instructions."""
if args.aggr_regexp:
compiled = [(pat, re.compile(pat)) for pat in args.aggr_regexp]
buckets: dict[tuple[Any, ...], list[JobRecord]] = defaultdict(list)
@@ -465,6 +466,7 @@ def aggregate_records(records: list[JobRecord], args: argparse.Namespace) -> lis
def make_single_row(rec: JobRecord) -> OutputRow:
"""Returns an OutputRow based on a single slurm job record."""
return OutputRow(
username=rec.username,
JobID=rec.raw.get("JobIDRaw", rec.raw.get("JobID", "")),
@@ -478,17 +480,18 @@ def make_single_row(rec: JobRecord) -> OutputRow:
Memory_Efficiency=rec.mem_eff,
Time_Efficiency=rec.time_eff,
jobname=rec.jobname,
_cpu_values=[rec.cpu_eff] if rec.cpu_eff is not None else [],
_mem_values=[rec.mem_eff] if rec.mem_eff is not None else [],
_time_values=[rec.time_eff] if rec.time_eff is not None else [],
_cpu_eff_values=[rec.cpu_eff] if rec.cpu_eff is not None else [],
_mem_eff_values=[rec.mem_eff] if rec.mem_eff is not None else [],
_time_eff_values=[rec.time_eff] if rec.time_eff is not None else [],
)
def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) -> OutputRow:
"""Returns an OutputRow based on the given list of job records."""
first = records[0]
cpu_vals = [r.cpu_eff for r in records if r.cpu_eff is not None]
mem_vals = [r.mem_eff for r in records if r.mem_eff is not None]
time_vals = [r.time_eff for r in records if r.time_eff is not None]
cpu_eff_vals = [r.cpu_eff for r in records if r.cpu_eff is not None]
mem_eff_vals = [r.mem_eff for r in records if r.mem_eff is not None]
time_eff_vals = [r.time_eff for r in records if r.time_eff is not None]
return OutputRow(
username=username,
JobID="",
@@ -498,17 +501,18 @@ def make_aggregate_row(records: list[JobRecord], username: str, jobname: str) ->
MemPerCPU=first.mem_per_cpu_gb,
ReqWalltime=first.reqwall_hours,
Count=len(records),
CPU_Efficiency=mean_or_none(cpu_vals),
Memory_Efficiency=mean_or_none(mem_vals),
Time_Efficiency=mean_or_none(time_vals),
CPU_Efficiency=mean_or_none(cpu_eff_vals),
Memory_Efficiency=mean_or_none(mem_eff_vals),
Time_Efficiency=mean_or_none(time_eff_vals),
jobname=jobname,
_cpu_values=cpu_vals,
_mem_values=mem_vals,
_time_values=time_vals,
Testcolumn=10,
_cpu_eff_values=cpu_eff_vals,
_mem_eff_values=mem_eff_vals,
_time_eff_values=time_eff_vals,
)
def resolve_column(name: str) -> str:
def resolve_column_name(name: str) -> str:
"""Returns canonicalized full column name, accepts one letter column codes"""
n = name.strip()
reverse = {v.lower(): v for v in DEFAULT_COLUMNS}
reverse.update({v.lower(): v for v in NUMERIC_COLUMNS})
@@ -533,7 +537,7 @@ def sort_rows(rows: list[OutputRow], spec: str | None) -> list[OutputRow]:
desc = t.startswith("-")
asc = t.startswith("+")
name = t[1:] if (desc or asc) else t
col = resolve_column(name)
col = resolve_column_name(name)
parsed.append((col, desc))
sorted_rows = rows
@@ -554,6 +558,7 @@ def sort_rows(rows: list[OutputRow], spec: str | None) -> list[OutputRow]:
return sorted_rows
# adds sdev columns behind their respective avg column
def columns_for_sdev(base_cols: list[str]) -> list[str]:
out: list[str] = []
for col in base_cols:
@@ -618,7 +623,7 @@ def print_custom_format(rows: list[OutputRow], fmt: str, sdev: bool) -> None:
def repl(match: re.Match[str]) -> str:
widthspec, alias = match.groups()
col = resolve_column(alias)
col = resolve_column_name(alias)
val = d.get(col)
text = format_value(val, col)
if widthspec: