feat!: submit Slurm jobs through REST and track execution

This commit is contained in:
wakonig_k authored and Klaus Wakonig committed 2026-10-28 18:05:33 +01:00
1 parent 790ca5b396
commit 4d17ce9d54
20 files changed
+2246 -871

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
SLURM_REST_URL=https://ra-slurmapi-001.psi.ch
SLURM_REST_API_VERSION=v0.0.44
SLURM_JWT=
# Optional: only set if required by your Slurm authentication configuration.
# SLURM_USER=your_username
+5
View File
@@ -0,0 +1,5 @@
#!/bin/bash
for second in {1..10}; do
printf 'stdout-test %02d/10\n' "$second"
sleep 1
done
+2 -2
View File
@@ -14,9 +14,9 @@ classifiers = [
]
dependencies = [
"hiredis~=3.0",
"pydantic~=2.12.0",
"redis~=6.2",
"redis[hiredis]>=6.2, <9.0",
"python-dotenv~=1.2",
]
[project.optional-dependencies]
+13 -3
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
import json
from typing import Annotated, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias
from pydantic import BaseModel, Field, TypeAdapter
@@ -49,8 +49,18 @@ SlurmMessage: TypeAlias = Annotated[
SLURM_MESSAGE_ADAPTER = TypeAdapter(SlurmMessage)
def parse_slurm_message(payload: str | dict) -> SlurmMessage:
"""Parse a JSON payload into the appropriate slurm message model."""
def parse_slurm_message(payload: str | dict[str, Any]) -> SlurmMessage:
"""Validate a JSON or dictionary payload as a discriminated Slurm message.
Args:
payload (str | dict[str, Any]): JSON-compatible request body or incoming message payload.
Returns:
SlurmMessage: Heartbeat, status, or log message selected by msg_type.
Raises:
ValueError: The JSON or message schema is invalid.
"""
if isinstance(payload, str):
payload = json.loads(payload)
return SLURM_MESSAGE_ADAPTER.validate_python(payload)
+149 -95
View File
@@ -44,97 +44,133 @@ class RegisteredCallback:
class JobFuture:
"""
A future-like object that represents a Slurm job. It allows users to check the status of the job,
retrieve the job's result, and register callbacks for events (e.g., 'event', 'logs', 'heartbeat').
wait for completion, and register callbacks for status, log, and heartbeat events.
"""
def __init__(
self,
job_id: str,
manager: SlurmManager,
slurm_job_id: str,
registry: dict | None = None,
name: str | None = None,
):
self, job_id: str, manager: SlurmManager, slurm_job_id: str, name: str | None = None
) -> None:
"""Create a future for one job tracked by its submitting manager.
Args:
job_id (str): Opaque manager identifier used in Redis topic names.
manager (SlurmManager): Manager that owns the job and handles subscriptions
and cancellation.
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
name (str | None): Display-name prefix; None uses Job, followed by the
scheduler ID.
"""
self._job_id = job_id
self._manager = manager
self._slurm_job_id, _, cluster = slurm_job_id.strip().partition(";")
self._slurm_cluster = cluster or None
self._registry = registry if registry is not None else {}
self._future = Future()
self._slurm_job_id = slurm_job_id.strip()
self._future: Future[None] = Future()
if name is None:
name = "Job"
self.name = f"{name}-{self._slurm_job_id}"
self._status: str = JobStatus.PENDING.value
self._shutdown_event = threading.Event()
self._lock = threading.RLock()
self._callback_registry: dict[str, list[RegisteredCallback]] = {}
#####################
## Public API
#####################
################################################
# Public Properties #
################################################
@property
def slurm_job_id(self) -> str:
"""The numeric SLURM job ID, without an optional cluster suffix."""
"""Return the numeric scheduler job ID on the configured REST endpoint.
Returns:
str: Scheduler job ID.
"""
return self._slurm_job_id
@property
def slurm_cluster(self) -> str | None:
"""The cluster returned by sbatch, or None for the default cluster."""
return self._slurm_cluster
@property
def status(self) -> str | None:
"""The current status of the job."""
"""Return the most recently accepted lifecycle status.
Returns:
str | None: Pending, running, finished, error, or cancelled status.
"""
return self._status
def cancel(self):
"""Cancel the job."""
self._manager.cancel_job(
slurm_job_id=self.slurm_job_id, job_id=self._job_id, slurm_cluster=self.slurm_cluster
)
################################################
# Public Job Lifecycle #
################################################
def done(self) -> bool:
"""Check whether the job reached any terminal state.
Returns:
bool: True for a finished, failed, or cancelled job.
"""
return self._future.done()
def success(self) -> bool:
"""Check whether the job reached a successful terminal state.
Returns:
bool: True only when the job finished without an exception.
"""
return self._future.done() and self._future.exception() is None
def exception(self) -> Exception | None:
"""If the job has finished with an error, return the exception. Otherwise, return None."""
"""Return the recorded terminal exception without waiting.
Returns:
Exception | None: Failure or cancellation exception, or None if pending or
successful.
"""
if self._future.done():
return self._future.exception()
return None
def success(self) -> bool:
"""Return True if the job finished successfully, False if it finished with an error or is not done yet."""
return self._future.done() and self._future.exception() is None
def done(self) -> bool:
"""Return True if the job has finished (either successfully or with an error), False otherwise."""
return self._future.done()
def wait(self, timeout: float | None = None) -> None:
"""
Wait for the job to finish, with an optional timeout in seconds. Raises TimeoutError if the job
does not finish within the specified timeout.
"""Wait for the job and propagate its failure or cancellation.
Args:
timeout: Maximum time to wait for the job to finish, in seconds. If None, wait indefinitely.
timeout (float | None): Maximum wait in seconds; None waits indefinitely
without cancelling the job.
Raises:
TimeoutError: The job does not reach a terminal state before the timeout.
Exception: The job fails or is cancelled.
Notes:
A timeout does not cancel the job. Callback execution may still be pending
when this method returns; manager shutdown drains accepted callbacks.
"""
self._future.result(timeout=timeout)
def cancel(self) -> None:
"""Request cancellation through the owning manager.
Raises:
SlurmRestError: Slurm rejects cancellation or the request fails.
"""
self._manager.cancel_job(slurm_job_id=self.slurm_job_id, job_id=self._job_id)
################################################
# Public Event Subscriptions #
################################################
def listen(
self,
event_type: Literal["status", "log", "heartbeat"],
callback: Callable[[SlurmMessage], None],
) -> str:
"""
Register a callback to listen for events of the specified type ('status', 'log', or 'heartbeat').
Returns a callback ID that can be used to unregister.
"""Register an asynchronous callback for a job event.
Args:
event_type: The type of event to listen for ('status', 'log', or 'heartbeat').
callback: A function that takes a SlurmMessage object as input. The exact message object
depends on the event type. Please check event_messages.py for the specific
message class based on the event.
event_type (Literal['status', 'log', 'heartbeat']): Event to receive: log,
status, or heartbeat.
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
Slurm message.
Returns:
A unique callback ID that can be used to unregister the callback later.
str: Callback ID accepted by unlisten().
Raises:
RuntimeError: The job has finished or is no longer tracked.
"""
cb_id = self._manager._add_subscription(
job_id=self._job_id, key=event_type, callback=callback
@@ -148,61 +184,53 @@ class JobFuture:
return cb.id
def unlisten(self, callback_id: str) -> None:
"""
Unregister a previously registered callback using its callback ID.
"""Remove a previously registered callback by ID.
Args:
callback_id: The unique ID of the callback to unregister, as returned by listen()
callback_id (str): Subscription ID, or IDs where accepted, returned during
registration.
"""
# Manager cleanup may acquire this future's lock while holding its own.
# Remove local ownership before acquiring the manager lock.
if self._remove_callback(callback_id):
self._manager.remove_subscription_by_id(callback_id=callback_id)
################################################
# Internal Callback Management #
################################################
def _remove_callback(self, callback_id: str) -> bool:
"""Remove local callback ownership without acquiring the manager lock.
Args:
callback_id (str): ID of the callback to remove.
Returns:
bool: True if a registered callback was removed.
"""
removed = False
with self._lock:
for event, callbacks in self._callback_registry.items():
for cb in callbacks:
if cb.id == callback_id:
callbacks.remove(cb)
if not callbacks:
self._callback_registry.pop(event, None)
removed = True
break
if removed:
break
# Manager cleanup may acquire this future's lock while holding its own.
# Release the future lock before acquiring the manager lock.
if removed:
self._manager.remove_subscription_by_id(callback_id=callback_id)
if cb.id != callback_id:
continue
callbacks.remove(cb)
if not callbacks:
self._callback_registry.pop(event)
return True
return False
###################
## Internal Methods
## Not meant to be called by users
###################
def _set_finished(self):
self._shutdown_event.set()
if self._future.done():
logger.warning(
"Job %s is already marked as done. Ignoring _set_finished call.", self.name
)
return
self._future.set_result(None)
def _set_exception(self, status, exc_info=None):
if self._future.done():
logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name)
return
self._shutdown_event.set()
# Here you could log the exception info or store it for later retrieval
if exc_info:
info = f"{self.name} failed with status {status}. Exception info: {exc_info}"
else:
info = f"{self.name} failed with status {status}."
# You could log the info here, e.g., using logging.error(info)
self._future.set_exception(Exception(info))
def _raise_if_failed(self) -> None:
if self._future.done() and self._future.exception() is not None:
raise self._future.exception()
################################################
# Internal State Updates #
################################################
def _update_status(self, status: str, exc_info: str | None = None) -> None:
"""Apply a lifecycle update under the future lock, ignoring late updates.
Args:
status (str): Lifecycle status; a terminal state completes the underlying
future.
exc_info (str | None): Optional worker-provided failure details.
"""
with self._lock:
if self._future.done():
return
@@ -211,3 +239,29 @@ class JobFuture:
self._set_finished()
elif status in (JobStatus.CANCELLED, JobStatus.ERROR):
self._set_exception(status, exc_info)
def _set_finished(self) -> None:
"""Complete the underlying future successfully unless it is already done."""
if self._future.done():
logger.warning(
"Job %s is already marked as done. Ignoring _set_finished call.", self.name
)
return
self._future.set_result(None)
def _set_exception(self, status: str, exc_info: str | None = None) -> None:
"""Record a terminal job failure with optional worker details.
Args:
status (str): Lifecycle status; a terminal state completes the underlying
future.
exc_info (str | None): Optional worker-provided failure details.
"""
if self._future.done():
logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name)
return
if exc_info:
info = f"{self.name} failed with status {status}. Exception info: {exc_info}"
else:
info = f"{self.name} failed with status {status}."
self._future.set_exception(Exception(info))
+104
View File
@@ -0,0 +1,104 @@
"""Validated user-facing Slurm job options."""
from typing import Annotated, Any, Literal
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
class JobOptions(BaseModel):
"""Resources and output settings for one job.
An omitted partition uses the cluster default; an omitted time limit uses
the selected partition's default. Time limits
are wall-clock minutes, independently of JobFuture.wait()'s timeout in seconds.
Output paths refer to the compute node and may use Slurm's %j job-ID pattern.
"""
model_config = ConfigDict(extra="forbid", frozen=True)
name: NonEmptyString | None = Field(
default=None, description="Job name shown in Slurm. Omit to use the scheduler default."
)
partition: Literal["hour", "day", "week", "gpu"] | None = Field(
default=None,
description=(
"RA partition: hour (default/max 1 hour), day (default 4 hours, max 24 hours), "
"week (default 2 days, max 8 days), or gpu (default 1 hour, max 7 days). "
"Omit to use the cluster default, currently hour."
),
)
account: NonEmptyString | None = Field(
default=None,
description=(
"Slurm accounting/project account; omit to use your default account association. "
"The account must also be allowed by the selected partition: hour/day allow "
"user and proprietary; gpu allows user and gpu-csaxs; week denies gpu-csaxs "
"and tomcat. These partition rules do not establish your account membership."
),
)
qos: NonEmptyString | None = Field(
default=None,
description=(
"Quality of Service controlling scheduling priority and resource limits. "
"Must be allowed by both your association and the selected partition. "
"Omit to use your association's default job QoS. The gpu partition allows "
"gpu-default, gpu-day, gpu-hour, and gpu-csaxs, and additionally applies "
"partition QoS gpu-default. Partition QoS is separate from requested job QoS."
),
)
tasks: int = Field(
default=1,
ge=1,
strict=True,
description=(
"Number of Slurm tasks requested for the allocation; defaults to one. "
"The worker executes the workload script once, without launching MPI ranks. "
"This does not specify CPUs per task."
),
)
time_limit_minutes: int | None = Field(
default=None,
ge=1,
strict=True,
description=(
"Maximum job wall-clock runtime in minutes. Slurm terminates the job at this limit. "
"Omit to use the selected partition's default: hour 60, day 240, week 2880, "
"gpu 60 minutes. Partition maxima are respectively 60, 1440, 11520, and "
"10080 minutes; account/QoS limits may impose additional restrictions. "
"Separate from JobFuture.wait(timeout), which is measured in seconds."
),
)
standard_output: NonEmptyString | None = Field(
default=None,
description=(
"Compute-node file path for stdout, in addition to Redis forwarding. "
"Supports Slurm patterns such as %j for the job ID. Relative paths use the job's "
"working directory; the parent directory must exist. Omit for Slurm's default output."
),
)
standard_error: NonEmptyString | None = Field(
default=None,
description=(
"Compute-node file path for stderr, in addition to Redis forwarding. "
"Supports %j for the job ID. Relative paths use the job's working directory; "
"the parent directory must exist. Omit to use Slurm's default stderr handling."
),
)
################################################
# Public Serialization #
################################################
def to_rest(self) -> dict[str, Any]:
"""Serialize validated options into slurmrestd job-description fields.
Returns:
dict[str, Any]: REST options with unset values omitted and runtime expressed
as an optional integer.
"""
options = self.model_dump(exclude={"time_limit_minutes"}, exclude_none=True)
if self.time_limit_minutes is not None:
options["time_limit"] = {"set": True, "number": self.time_limit_minutes}
return options
Whitespace-only changes.
@@ -1,72 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
module load redis
fifo=$1
hash_name=$2
exec {fifo_fd}<"$fifo"
redis_host="bec-slurm.psi.ch"
status="finished"
publish_payload() {
local topic=$1
local payload=$2
redis-cli -h "$redis_host" PUBLISH "$topic" "$payload"
}
timestamp=$(date +%s)
publish_payload \
"info/$hash_name/status" \
"$(jq -cn --arg status "running" '{msg_type:"status", status:$status}')"
heartbeat() {
while :; do
local timestamp
timestamp=$(date +%s)
publish_payload \
"info/$hash_name/heartbeat" \
"$(jq -cn --argjson timestamp "$timestamp" '{msg_type:"heartbeat", timestamp:$timestamp}')" \
|| true
sleep 1
done
}
heartbeat &
hb_pid=$!
cleanup() {
if kill -0 "$hb_pid" 2>/dev/null; then
kill "$hb_pid" 2>/dev/null || true
wait "$hb_pid" 2>/dev/null || true
fi
}
trap cleanup EXIT
while IFS= read -r line <&"$fifo_fd"; do
case "$line" in
stop)
status="finished"
break
;;
error:*)
status="error:${line#error:}"
break
;;
log:*)
msg="${line#log:}"
publish_payload \
"info/$hash_name/log" \
"$(jq -cn --arg log "$msg" '{msg_type:"log", log:$log}')"
;;
*)
:
;;
esac
done
publish_payload \
"info/$hash_name/status" \
"$(jq -cn --arg status "$status" '{msg_type:"status", status:$status}')"
-51
View File
@@ -1,51 +0,0 @@
#!/usr/bin/env bash
set -o pipefail
env_path=$1
script_path=$2
hash_name=$3
have_out_stream=$4
have_out_file=$5
redis_script=$6
fifo=$(mktemp -u)
mkfifo "$fifo"
"$redis_script" "$fifo" "$hash_name" >/dev/null &
worker_pid=$!
trap 'rm -f "$fifo"' EXIT
exec {fifo_fd}>"$fifo"
source "$env_path"
{
stdbuf -o0 -e0 "$script_path" 2>&1
} | while IFS= read -r line; do
if [ "$have_out_stream" -eq 1 ]; then
printf 'log:%s\n' "$line" >&"$fifo_fd"
fi
if [ "$have_out_file" -eq 1 ]; then
printf '%s\n' "$line"
fi
done
exit_code=${PIPESTATUS[0]}
if [ "$exit_code" -ne 0 ]; then
printf 'error:%s\n' "$exit_code" >&"$fifo_fd"
fi
printf 'stop\n' >&"$fifo_fd"
exec {fifo_fd}>&-
wait $worker_pid
exit $exit_code
+612
View File
@@ -0,0 +1,612 @@
"""Own job futures, Redis subscriptions, and callback dispatch."""
from __future__ import annotations
import threading
import time
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from functools import partial, wraps
from logging import getLogger
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict
from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage
from slurm_manager.job_future import JobFuture, JobStatus
from slurm_manager.redis_sub_client import RedisSubClient
logger = getLogger(__name__)
@dataclass(frozen=True, slots=True)
class TopicInfo:
"""Helper for topic specification within Redis bec-slurm pub/sub interface."""
job_id: str
prefix: str = "info"
################################################
# Public Properties #
################################################
@property
def log(self) -> str:
"""Return the channel carrying stdout and stderr log messages.
Returns:
str: Job log channel.
"""
return self.topic("log")
@property
def event(self) -> str:
"""Return the channel carrying lifecycle status messages.
Returns:
str: Job status channel.
"""
return self.topic("status")
@property
def heartbeat(self) -> str:
"""Return the channel carrying worker heartbeats.
Returns:
str: Job heartbeat channel.
"""
return self.topic("heartbeat")
################################################
# Public Topic Lookup #
################################################
def topic(self, v: str) -> str:
"""Build a Redis topic name for a job event.
Args:
v (str): Event suffix appended to this job's Redis topic prefix.
Returns:
str: Redis channel in prefix/job_id/event form.
"""
return f"{self.prefix}/{self.job_id}/{v}"
class SubscriptionInfo(BaseModel):
"""Helper for slurm job information and future tracking."""
model_config = ConfigDict(arbitrary_types_allowed=True)
topic: str
job_id: str
slurm_job_id: str | None = None
callback: Callable[[SlurmMessage], None] | None = None
callback_id: str | None = None
last_heartbeat: float | None = None
heartbeat_received: bool = False
last_scheduler_check: float | None = None
HEARTBEAT_TIMEOUT = 10 # seconds
class JobTracker:
"""Coordinate job state and callback ownership under one lock."""
def __init__(self, sub_client: RedisSubClient) -> None:
"""Create tracking state and the user callback executor.
Args:
sub_client (RedisSubClient): Transport used for job subscriptions; the
tracker closes it during shutdown.
"""
self._sub_client = sub_client
self._future_registry: dict[str, JobFuture] = {}
self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict)
self._topic_by_subscription_id: dict[str, str] = {}
self._active_subscriptions: dict[str, set[str]] = defaultdict(set)
self._lock = threading.RLock()
self._executor = ThreadPoolExecutor(
max_workers=8, thread_name_prefix="SlurmManagerCallback"
)
################################################
# Public Job Tracking #
################################################
def track(self, future: JobFuture) -> None:
"""Register a submitted job and attach internal monitoring callbacks.
Args:
future (JobFuture): Submitted job whose state and subscriptions are owned.
Notes:
The caller must remove tracking and cancel the held job if setup fails.
"""
with self._lock:
self._future_registry[future._job_id] = future
self._subscribe_to_internal_topics(future, TopicInfo(future._job_id))
def get_future(self, job_id: str) -> JobFuture | None:
"""Look up a tracked future under the state lock.
Args:
job_id (str): Opaque manager job identifier.
Returns:
JobFuture | None: Tracked future, or None after cleanup.
"""
with self._lock:
return self._future_registry.get(job_id)
def wait_until_ready(self, job_id: str) -> None:
"""Wait for Redis to acknowledge all subscriptions installed for a job.
Args:
job_id (str): Opaque manager job identifier.
Raises:
TimeoutError: Redis does not acknowledge the subscriptions in time.
"""
with self._lock:
topics = list(self._active_subscriptions.get(job_id, set()))
self._sub_client.wait_until_subscribed(topics)
def begin_scheduler_check(self, job_id: str) -> JobFuture | None:
"""Record a scheduler check for an active job before performing network I/O.
Args:
job_id (str): Opaque manager job identifier.
Returns:
JobFuture | None: Active future, or None if it has completed or was removed.
"""
with self._lock:
future = self._future_registry.get(job_id)
if future is None or future.done():
return None
topic = TopicInfo(job_id).heartbeat
for info in self._subscriptions_by_topic.get(topic, {}).values():
if info.last_heartbeat is not None:
info.last_scheduler_check = time.time()
return future
def apply_status(self, message: StatusMessage, future: JobFuture) -> None:
"""Apply a worker status and clean up terminal job subscriptions.
Args:
message (StatusMessage): Raw Redis payload or parsed worker message, as
specified by the annotation.
future (JobFuture): Tracked future whose lifecycle or subscriptions are
being managed.
Notes:
Statuses use status[:detail]. Unknown statuses fail the job and follow the
same cleanup path as an explicit error.
"""
info = message.status.split(":", 1)
status = info[0]
try: # Catch irregular formatted status messages to avoid crashing the callback loop.
status = JobStatus(status)
except ValueError:
logger.warning("Received unknown job status '%s' for job %s", status, future.name)
future._update_status(
"error",
f"Received unknown job status '{status}' from message: {message.model_dump()}",
)
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
return
exc_info = info[1] if len(info) > 1 else None
future._update_status(status, exc_info)
# Clean up subscriptions for finished job
if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]:
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
def get_missing_heartbeats(self) -> list[SubscriptionInfo]:
"""Select stale heartbeat subscriptions whose scheduler check is due.
Returns:
list[SubscriptionInfo]: Copies of internal heartbeat subscriptions needing
a Slurm query. Mutating a copy does not change tracking state.
"""
missing = []
now = time.time()
with self._lock:
for job_id, topics in self._active_subscriptions.items():
topic = TopicInfo(job_id=job_id).heartbeat
if topic not in topics:
continue
for sub_info in self._subscriptions_by_topic.get(topic, {}).values():
if not sub_info.last_heartbeat:
continue
if now - sub_info.last_heartbeat <= HEARTBEAT_TIMEOUT:
continue
last_check = sub_info.last_scheduler_check
if last_check is not None and now - last_check <= HEARTBEAT_TIMEOUT:
continue
missing.append(sub_info.model_copy())
return missing
################################################
# Public Event Subscriptions #
################################################
def listen(
self,
event_type: Literal["heartbeat", "log", "status"],
callback: Callable[[SlurmMessage], None],
job_id: str | None = None,
) -> list[str]:
"""Attach a callback to one tracked job or all currently tracked jobs.
Args:
event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log,
status, or heartbeat.
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
Slurm message.
job_id (str | None): Opaque manager identifier used in Redis topic names.
Returns:
list[str]: Subscription IDs accepted by unlisten().
Raises:
KeyError: The explicitly requested job ID is not tracked.
RuntimeError: A selected job has already completed.
Notes:
With job_id=None, registration applies to the current jobs, not future
submissions. Callbacks run in the tracker executor.
"""
with self._lock:
job_futures = (
[self._future_registry[job_id]] if job_id else list(self._future_registry.values())
)
callback_ids = []
for future in job_futures:
callback_ids.append(future.listen(event_type=event_type, callback=callback))
return callback_ids
def add_subscription(
self,
job_id: str,
key: Literal["heartbeat", "log", "status"],
callback: Callable[[SlurmMessage], None],
callback_kwargs: dict[str, Any] | None = None,
) -> str:
"""Atomically register a Redis callback and its manager ownership.
Args:
job_id (str): Opaque manager identifier used in Redis topic names.
key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or
heartbeat.
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
Slurm message.
callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed
to the user callback.
Returns:
str: Unique callback subscription ID.
Raises:
RuntimeError: The job is absent or finishes during registration.
"""
topic_str = TopicInfo(job_id=job_id).topic(key)
@wraps(callback)
def wrapped_callback(
message: SlurmMessage,
executor: ThreadPoolExecutor,
job_id: str,
slurm_job_id: str,
callback_kwargs: dict[str, Any] | None = None,
) -> None:
"""Enrich a parsed worker message and dispatch its registered handler.
Args:
message (SlurmMessage): Raw Redis payload or parsed worker message, as
specified by the annotation.
executor (ThreadPoolExecutor): Callback executor that runs user code
away from the listener thread.
job_id (str): Opaque manager identifier used in Redis topic names.
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
callback_kwargs (dict[str, Any] | None): Additional keyword arguments
passed to the user callback.
"""
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": job_id,
"slurm_job_id": slurm_job_id,
}
)
executor.submit(callback, message, **(callback_kwargs or {}))
# Completion removes subscriptions under this same lock. Keep ownership
# registration atomic with cleanup, and undo Redis registration if the
# future became terminal while subscribe() was in progress.
with self._lock:
future = self._future_registry.get(job_id)
if future is None or future.done():
raise RuntimeError(f"Job {job_id} is no longer active")
cb_id = self._sub_client.subscribe(
topic_str,
partial(
wrapped_callback,
executor=self._executor,
callback_kwargs=callback_kwargs,
job_id=job_id,
slurm_job_id=future.slurm_job_id,
),
)
if self._future_registry.get(job_id) is not future or future.done():
self._sub_client.unsubscribe(cb_id)
raise RuntimeError(f"Job {job_id} finished during subscription registration")
sub_info = SubscriptionInfo(
job_id=job_id,
topic=topic_str,
slurm_job_id=future.slurm_job_id,
callback=wrapped_callback,
last_heartbeat=None,
callback_id=cb_id,
heartbeat_received=False,
)
self._register_subscription_for_job(sub_info=sub_info)
return cb_id
def remove_subscription_by_id(self, callback_id: str | list[str]) -> None:
"""Remove the selected callbacks while preserving other topic listeners.
Args:
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
returned during registration.
"""
if isinstance(callback_id, str):
callback_id = [callback_id]
with self._lock:
for cb_id in callback_id:
topic = self._topic_by_subscription_id.pop(cb_id, None)
if topic is None:
continue
callbacks = self._subscriptions_by_topic.get(topic, {})
sub_info = callbacks.pop(cb_id, None)
self._sub_client.unsubscribe(cb_id)
# Other callbacks, including heartbeat monitoring, still own this topic.
if callbacks:
continue
self._subscriptions_by_topic.pop(topic, None)
if sub_info is None:
continue
topics = self._active_subscriptions.get(sub_info.job_id)
if topics is not None:
topics.discard(topic)
def remove_subscription_by_topic(self, topic: str) -> None:
"""Remove all callbacks for one topic and unsubscribe it from Redis.
Args:
topic (str): Redis channel whose callback registrations are selected.
"""
with self._lock:
callbacks = self._subscriptions_by_topic.pop(topic, {})
for callback_id, sub_info in callbacks.items():
self._topic_by_subscription_id.pop(callback_id, None)
job_id = sub_info.job_id
self._active_subscriptions[job_id].discard(topic)
# Remove the topic subscription from Redis
self._sub_client.unsubscribe_topic(topic)
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
"""Remove job subscriptions and discard the tracked future.
Args:
job_id (str): Opaque manager identifier used in Redis topic names.
skip_event_callback (bool): If true, remove monitoring without synthesizing
a failure update.
Notes:
Unless skip_event_callback is true, an unfinished future is marked failed.
This only removes local monitoring; it does not cancel the Slurm allocation.
"""
with self._lock:
topics = self._active_subscriptions.pop(job_id, set())
for topic in topics:
callbacks = self._subscriptions_by_topic.pop(topic, {})
for callback_id in callbacks:
self._topic_by_subscription_id.pop(callback_id, None)
self._sub_client.unsubscribe_topic(topic)
future = self._future_registry.get(job_id)
if future is None:
return
# This callback will trigger a cleanup.
if not skip_event_callback:
self.apply_status(
StatusMessage(
status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}",
metadata={
"msg_received": datetime.now().isoformat(),
"job_id": future._job_id,
"slurm_job_id": future.slurm_job_id,
},
),
future,
)
self._remove_job_future(future._job_id)
################################################
# Public Lifecycle #
################################################
def shutdown(self, timeout: float | None = 1.0) -> None:
"""Stop Redis delivery and drain accepted user callbacks.
Args:
timeout (float | None): Redis listener join timeout in seconds; None waits
indefinitely. Callback draining has no timeout.
Notes:
Stop scheduler monitoring before calling this method. Remote jobs are
not cancelled and tracked futures are not marked complete.
"""
with self._lock:
self._subscriptions_by_topic.clear()
self._topic_by_subscription_id.clear()
self._sub_client.shutdown(timeout=timeout)
self._executor.shutdown(wait=True, cancel_futures=False)
################################################
# Internal Subscriptions #
################################################
def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None:
"""Attach the manager status and heartbeat callbacks before release.
Args:
future (JobFuture): Tracked future whose lifecycle or subscriptions are
being managed.
topic_info (TopicInfo): Redis topic names for the tracked job.
"""
self._subscribe_to_heartbeat(topic_info, future)
self._subscribe_to_event(topic_info, future)
def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None:
"""Register the internal heartbeat callback and its initial timestamp.
Args:
topic_info (TopicInfo): Redis topic names for the tracked job.
future (JobFuture): Tracked future whose lifecycle or subscriptions are
being managed.
"""
sub_info = SubscriptionInfo(
job_id=topic_info.job_id,
topic=topic_info.heartbeat,
slurm_job_id=future.slurm_job_id,
callback=None,
last_heartbeat=None,
callback_id=None,
heartbeat_received=False,
)
topic = topic_info.heartbeat
def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
"""Enrich a parsed worker message and dispatch its registered handler.
Args:
message (HeartBeatMessage): Raw Redis payload or parsed worker message,
as specified by the annotation.
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
bookkeeping.
"""
try:
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": sub_info.job_id,
"slurm_job_id": sub_info.slurm_job_id,
}
)
self._heartbeat_callback(message, sub_info)
except Exception:
logger.error(
"Error in callback for topic %s: %s", message.msg_type, message.timestamp
)
cb_id = self._sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info))
sub_info.callback_id = cb_id
sub_info.last_heartbeat = time.time()
self._register_subscription_for_job(sub_info=sub_info)
def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None:
"""Register the internal callback for terminal and running statuses.
Args:
topic_info (TopicInfo): Redis topic names for the tracked job.
future (JobFuture): Tracked future whose lifecycle or subscriptions are
being managed.
"""
sub_info = SubscriptionInfo(
job_id=topic_info.job_id,
topic=topic_info.event,
slurm_job_id=future.slurm_job_id,
callback=None,
last_heartbeat=None,
callback_id=None,
heartbeat_received=False,
)
event_topic = topic_info.event
def wrapped_callback(message: StatusMessage, future: JobFuture) -> None:
"""Enrich a parsed worker message and dispatch its registered handler.
Args:
message (StatusMessage): Raw Redis payload or parsed worker message, as
specified by the annotation.
future (JobFuture): Tracked future whose lifecycle or subscriptions are
being managed.
"""
try:
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": future._job_id,
"slurm_job_id": future.slurm_job_id,
}
)
self.apply_status(message, future=future)
except Exception:
logger.error("Error in callback for topic %s: %s", message.msg_type, message.status)
cb_id = self._sub_client.subscribe(event_topic, partial(wrapped_callback, future=future))
sub_info.callback_id = cb_id
self._register_subscription_for_job(sub_info=sub_info)
def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None:
"""Index a subscription by callback ID, topic, and job.
Args:
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
bookkeeping.
Raises:
ValueError: The subscription does not contain a callback ID.
"""
with self._lock:
if sub_info.callback_id is None:
raise ValueError("SubscriptionInfo.callback_id must be set before registration")
self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info
self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic
self._active_subscriptions[sub_info.job_id].add(sub_info.topic)
def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
"""Record heartbeat receipt for the monitored subscription.
Args:
message (HeartBeatMessage): Raw Redis payload or parsed worker message, as
specified by the annotation.
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
bookkeeping.
"""
with self._lock:
sub_info.heartbeat_received = True
sub_info.last_heartbeat = time.time()
def _remove_job_future(self, job_id: str) -> None:
"""Remove a future from the manager registry under its lock.
Args:
job_id (str): Opaque manager identifier used in Redis topic names.
"""
with self._lock:
self._future_registry.pop(job_id, None)
+104 -11
View File
@@ -52,6 +52,18 @@ class RedisSubClient:
redis_cls: type[Redis] = Redis,
**redis_kwargs: Any,
) -> None:
"""Create Redis resources without starting the listener until subscribed.
Args:
bootstrap (str | list[str]): Redis host:port or a nonempty address list;
only the first entry is used.
redis_cls (type[Redis]): Redis client implementation, injectable for tests.
**redis_kwargs (Any): Additional Redis constructor options such as socket
timeouts.
Raises:
ValueError: The bootstrap list is empty or its address cannot be parsed.
"""
if isinstance(bootstrap, list):
if not bootstrap:
raise ValueError("bootstrap list must not be empty")
@@ -70,14 +82,34 @@ class RedisSubClient:
self._stop_listener = threading.Event()
self._listener_thread: threading.Thread | None = None
################################################
# Public Subscriptions #
################################################
@property
def registered_topics(self) -> set[str]:
"""Returns all currently subscribed topics."""
"""Return a snapshot of topics with active callbacks.
Returns:
set[str]: Currently registered Redis channel names.
"""
with self._lock:
return set(self._subscriptions_by_topic.keys())
def subscribe(self, topic: str, callback: MessageCallback) -> str:
"""Subscribes callback to a topic and returns a subscription id."""
"""Register a callback and start the listener if needed.
Args:
topic (str): Redis channel whose callback registrations are selected.
callback (MessageCallback): Callable receiving the parsed Slurm message.
Returns:
str: Unique subscription ID.
Notes:
The callback runs on the Redis listener thread and must remain fast and
nonblocking.
"""
sub_id = str(uuid4())
subscription = Subscription(id=sub_id, topic=topic, callback=callback)
@@ -95,10 +127,13 @@ class RedisSubClient:
return sub_id
def unsubscribe(self, subscription_id: str) -> bool:
"""
Unregisters one callback by subscription id.
"""Remove one callback and unsubscribe its topic if it has no owners.
Returns True if a callback was removed, False otherwise.
Args:
subscription_id (str): Unique callback ID returned by subscribe().
Returns:
bool: True if the callback was registered; False if it was unknown.
"""
with self._lock:
topic = self._topic_by_subscription_id.pop(subscription_id, None)
@@ -120,7 +155,14 @@ class RedisSubClient:
return True
def unsubscribe_topic(self, topic: str) -> int:
"""Unregisters all callbacks for a topic and returns removed count."""
"""Remove every callback registered for one topic.
Args:
topic (str): Redis channel whose callback registrations are selected.
Returns:
int: Number of removed callbacks.
"""
with self._lock:
callbacks = self._subscriptions_by_topic.pop(topic, None)
if not callbacks: # No callbacks for this topic, nothing happens
@@ -132,7 +174,19 @@ class RedisSubClient:
return len(callbacks)
def wait_until_subscribed(self, topics: list[str], timeout: float = 5.0) -> None:
"""Wait for Redis to acknowledge each subscription before releasing a job."""
"""Wait for Redis subscription acknowledgements before releasing a job.
Args:
topics (list[str]): Already registered channels whose acknowledgements must
be received.
timeout (float): Shared acknowledgement deadline in seconds across all
requested topics.
Raises:
KeyError: A requested topic is not registered.
TimeoutError: The shared deadline expires before all topics are
acknowledged.
"""
deadline = time.monotonic() + timeout
with self._lock:
ready = [(topic, self._subscription_ready[topic]) for topic in topics]
@@ -140,8 +194,21 @@ class RedisSubClient:
if not event.wait(max(0, deadline - time.monotonic())):
raise TimeoutError(f"Redis did not acknowledge subscription to {topic}")
################################################
# Public Lifecycle #
################################################
def shutdown(self, timeout: float | None = 1.0) -> None:
"""Stops worker threads and closes Redis resources."""
"""Stop the listener and close Redis resources.
Args:
timeout (float | None): Listener join timeout in seconds; None waits
indefinitely.
Notes:
A listener that misses the join timeout is logged before resources are
closed.
"""
self._stop_listener.set()
if self._listener_thread is not None:
@@ -157,8 +224,12 @@ class RedisSubClient:
self._pubsub.close()
self._redis.close()
################################################
# Internal Listener #
################################################
def _ensure_threads_started(self) -> None:
"""Makes sure the listener thread is running to receive messages from Redis."""
"""Start a listener if no listener thread is currently alive."""
if self._listener_thread is None or not self._listener_thread.is_alive():
self._stop_listener.clear()
self._listener_thread = threading.Thread(
@@ -167,7 +238,12 @@ class RedisSubClient:
self._listener_thread.start()
def _listener_loop(self) -> None:
"""Listener thread loop that polls for messages and dispatches them to callbacks."""
"""Poll Redis, acknowledge subscriptions, and dispatch parsed messages.
Notes:
Redis errors are logged and polling resumes after a short delay. Callback
exceptions are logged without stopping other callbacks.
"""
while not self._stop_listener.is_set():
try:
with self._lock:
@@ -212,9 +288,26 @@ class RedisSubClient:
format_exc,
)
################################################
# Internal Message Parsing #
################################################
@staticmethod
def _normalize_message(message: dict[str, Any]) -> tuple[str, SlurmMessage]:
"""Converts raw Redis message dict to normalized Message object."""
"""Decode a Redis pub/sub payload into the corresponding message model.
Args:
message (dict[str, Any]): Raw Redis payload or parsed worker message, as
specified by the annotation.
Returns:
tuple[str, SlurmMessage]: Pair containing the channel name and validated
Slurm message.
Raises:
TypeError: The payload is neither JSON text nor a dictionary.
ValueError: The JSON or message schema is invalid.
"""
channel = message.get("channel")
topic = channel.decode() if isinstance(channel, bytes) else str(channel)
+342 -488
View File
@@ -1,277 +1,195 @@
"""Minimal Slurm manager draft for sbatch + JobFuture creation."""
"""Submit and monitor remote Slurm jobs through slurmrestd and Redis."""
from __future__ import annotations
import os
import subprocess
import json
import shlex
import threading
import time
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from functools import partial, wraps
from logging import getLogger
from pathlib import Path
from typing import Any, Literal
from uuid import uuid4
from pydantic import BaseModel, ConfigDict
from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage
from slurm_manager.event_messages import SlurmMessage, StatusMessage
from slurm_manager.job_future import JobFuture, JobStatus
from slurm_manager.job_options import JobOptions
from slurm_manager.job_tracker import JobTracker, SubscriptionInfo
from slurm_manager.redis_sub_client import RedisSubClient
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
logger = getLogger(__name__)
_STOP = object()
def _job_exit_code(exit_code: Any) -> int | str | None:
"""Decode a v0.0.44 process exit status without treating unset codes as success.
def _cluster_args(cluster: str | None) -> list[str]:
"""Route Slurm CLI operations to the cluster that accepted the job."""
return ["--clusters", cluster] if cluster else []
Args:
exit_code (Any): Integer or v0.0.44 structured process exit status.
@dataclass(frozen=True, slots=True)
class TopicInfo:
"""Helper for topic specification within Redis bec-slurm pub/sub interface."""
job_id: str
prefix: str = "info"
def topic(self, v: str) -> str:
"""Constructs a new topic string for a given key."""
return f"{self.prefix}/{self.job_id}/{v}"
@property
def log(self) -> str:
"""Topic for log messages."""
return self.topic("log")
@property
def event(self) -> str:
"""Topic for event messages."""
return self.topic("status")
@property
def heartbeat(self) -> str:
"""Topic for heartbeat messages."""
return self.topic("heartbeat")
class SubscriptionInfo(BaseModel):
"""Helper for slurm job information and future tracking."""
model_config = ConfigDict(arbitrary_types_allowed=True)
topic: str
job_id: str
slurm_job_id: str | None = None
callback: Callable[[SlurmMessage], None] | None = None
callback_id: str | None = None
last_heartbeat: float | None = None
heartbeat_received: bool = False
last_scheduler_check: float | None = None
HEARTBEAT_TIMEOUT = 10 # seconds
Returns:
int | str | None: Exit code, a signal description, or None when no code is
available.
"""
if not isinstance(exit_code, dict):
return exit_code
code = exit_code.get("return_code")
if isinstance(code, dict):
code = code.get("number") if code.get("set") else None
signal = exit_code.get("signal", {}).get("id", {})
if isinstance(signal, dict):
signal = signal.get("number", 0) if signal.get("set") else 0
return code if not signal else f"signal {signal}"
class SlurmManager:
"""Small manager that submits wrapper jobs and returns JobFuture handles."""
"""Submit Slurm jobs and coordinate tracking and scheduler monitoring."""
def __init__(self, redis_host: str = "localhost", redis_port: int = 6379) -> None:
def __init__(
self,
redis_host: str = "localhost",
redis_port: int = 6379,
*,
env_file: str | Path = ".env",
rest_client: SlurmRestClient | None = None,
worker_path: str = "/usr/local/bin/slurm-worker",
) -> None:
"""Create REST/Redis clients and start background job monitoring.
Args:
redis_host (str): Redis hostname reachable by the submitter and compute
nodes.
redis_port (int): Redis TCP port shared by the submitter and worker.
env_file (str | Path): Local credential dotenv path. Environment variables
take precedence.
rest_client (SlurmRestClient | None): Optional preconfigured REST client;
None creates one from env_file.
worker_path (str): Absolute path to the Go executable on compute nodes.
Raises:
ValueError: The worker path or REST connection settings are invalid.
"""
if not worker_path.startswith("/"):
raise ValueError("Worker path must be an absolute compute-node path")
self.worker_path = worker_path
self._redis_address = f"{redis_host}:{redis_port}"
self.rest_client = rest_client if rest_client is not None else SlurmRestClient(env_file)
self.sub_client = RedisSubClient(bootstrap=f"{redis_host}:{redis_port}")
self._wrapper_path = Path(__file__).resolve().parent / "job_submission" / "wrapper.sh"
self._redis_script_path = (
Path(__file__).resolve().parent / "job_submission" / "redis_worker.sh"
)
# This is a job_id mapping to job futures
self._future_registry: dict[str, JobFuture] = {}
# This subscription id sub_info mapping by topics, used for callback handling
self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict)
# This is a topic mapping to subscriptions
self._topic_by_subscription_id: dict[str, str] = {}
# This is a job_id mapping to topics
self._active_subscriptions: dict[str, set[str]] = defaultdict(set)
# Threading
self._lock = threading.RLock()
self._job_tracker = JobTracker(self.sub_client)
self._stop_heartbeat_thread = threading.Event()
self.executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="SlurmManagerCallback")
# Heartbeat monitoring thread
self._heartbeat_monitoring_thread = threading.Thread(
target=self._heartbeat_monitoring_loop, name="SlurmManagerHeartbeatMonitor", daemon=True
)
self._start_threads()
def _start_threads(self):
"""Starts the heartbeat monitoring thread."""
self._heartbeat_monitoring_thread.start()
################################################
# Public Job Management #
################################################
def _heartbeat_monitoring_loop(self) -> None:
"""Internal loop to check for missing heartbeats. If a heartbeat for a job is missing for
more than HEARTBEAT_TIMEOUT seconds, we will trigger a check of the slurm job status,
and potentially trigger a cleanup of the job future and subscriptions if the job is no longer active.
def submit_job(
self,
script_path: str | Path,
env_path: str | Path | None = None,
*,
working_directory: str,
job_options: JobOptions | None = None,
callbacks: dict[Literal["log", "status", "heartbeat"], Callable[[SlurmMessage], None]]
| None = None,
) -> JobFuture:
"""Upload local Bash files and return a monitored, released Slurm job.
Args:
script_path (str | Path): Local Bash workload file whose contents are
uploaded.
env_path (str | Path | None): Optional local Bash activation file for
workload-specific environment setup; None skips activation commands.
working_directory (str): Absolute, existing compute-node directory in which
the job runs.
job_options (JobOptions | None): Validated resources and Slurm output
settings; None uses JobOptions defaults.
callbacks (dict[str, Callable] | None): Callbacks keyed by log, status, or
heartbeat, acknowledged before job release.
Returns:
JobFuture: Handle for querying, waiting, cancelling, and registering
callbacks.
Raises:
FileNotFoundError: A local workload or activation file does not exist.
ValueError: The remote working directory or callback event names are
invalid.
ValidationError: Resource or output options fail JobOptions validation.
SlurmRestError: Submission or release fails. A known held job is cancelled
on setup failure.
TimeoutError: Redis does not acknowledge the monitoring subscriptions.
Notes:
The credential dotenv file stays local. Callbacks are registered before
release; only the working directory, worker path, and output paths refer to
compute nodes.
"""
while not self._stop_heartbeat_thread.wait(timeout=2):
missing_heartbeats = []
# Check heartbeats for active jobs
with self._lock:
for _, topics in self._active_subscriptions.items():
for topic in topics:
infos = self._subscriptions_by_topic.get(topic, {})
for sub_info in infos.values():
if sub_info and sub_info.last_heartbeat:
now = time.time()
if now - sub_info.last_heartbeat > HEARTBEAT_TIMEOUT and (
sub_info.last_scheduler_check is None
or now - sub_info.last_scheduler_check > HEARTBEAT_TIMEOUT
):
missing_heartbeats.append(sub_info)
for sub_info in missing_heartbeats:
self._check_slurm_job(sub_info)
def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None:
"""Resolve missing heartbeats using Slurm, never assuming silence means failure."""
future = self._future_registry.get(sub_info.job_id)
if future is None or future.done():
return
sub_info.last_scheduler_check = time.time()
try:
queued = self._run_subprocess(
[
"squeue",
*_cluster_args(future.slurm_cluster),
"--noheader",
"--jobs",
future.slurm_job_id,
"--format=%T",
]
)
if queued.returncode == 0 and queued.stdout.strip():
# Pending, configuring, and running jobs may legitimately have no heartbeat.
return
accounting = self._run_subprocess(
[
"sacct",
*_cluster_args(future.slurm_cluster),
"--noheader",
"--parsable2",
"--allocations",
"--jobs",
future.slurm_job_id,
"--format=State,ExitCode",
]
)
except (OSError, subprocess.SubprocessError):
logger.exception("Could not query Slurm job %s", future.slurm_job_id)
return
if accounting.returncode != 0 or not accounting.stdout.strip():
# Accounting may be disabled, delayed, or temporarily unavailable.
return
fields = accounting.stdout.strip().splitlines()[0].split("|")
state = fields[0].split()[0].rstrip("+")
exit_code = fields[1].strip() if len(fields) > 1 else ""
if state == "COMPLETED" and exit_code == "0:0":
status = JobStatus.FINISHED.value
elif state == "CANCELLED":
status = JobStatus.CANCELLED.value
elif state in {
"COMPLETED",
"FAILED",
"TIMEOUT",
"OUT_OF_MEMORY",
"NODE_FAIL",
"BOOT_FAIL",
"DEADLINE",
"PREEMPTED",
"REVOKED",
}:
status = f"error:Slurm state {state}, exit code {exit_code or 'unknown'}"
else:
return
self._event_callback(StatusMessage(status=status), future)
def submit_job(self, script_path: str, env_path: str) -> JobFuture:
"""Create a temporary script from command and submit via sbatch wrapper."""
# Check that the wrapper script and redis worker script exist before submitting the job
if not self._wrapper_path.exists():
raise FileNotFoundError(f"wrapper script not found: {self._wrapper_path}")
if not self._redis_script_path.exists():
raise FileNotFoundError(f"redis worker script not found: {self._redis_script_path}")
env_path = os.path.abspath(env_path)
script_path = os.path.abspath(script_path)
job_id = str(uuid4()) # unique job_id for hash-key in Redis.
sbatch_cmd = [
"sbatch",
"--parsable",
"--hold",
os.path.abspath(self._wrapper_path),
env_path,
if not working_directory.startswith("/"):
raise ValueError("Working directory must be an absolute compute-node path")
if callbacks and set(callbacks) - {"log", "status", "heartbeat"}:
raise ValueError("Callbacks must use log, status, or heartbeat keys")
job_id = str(uuid4())
script = self._build_job_script(
script_path,
env_path,
job_id,
"1",
"1",
os.path.abspath(self._redis_script_path),
]
logger.debug("Submitting job with command: %s", " ".join(sbatch_cmd))
result = self._run_subprocess(sbatch_cmd)
if result.returncode != 0:
stderr = (result.stderr or "").strip()
raise RuntimeError(f"sbatch failed: {stderr}")
slurm_job_id = result.stdout.strip() # ID of job from SLURM scheduler
logger.debug("SBatch job submission id: %s", slurm_job_id)
job_future = JobFuture(
job_id=job_id, manager=self, registry=self._future_registry, slurm_job_id=slurm_job_id
worker_path=self.worker_path,
redis_address=self._redis_address,
)
job = JobOptions.model_validate(job_options or {}).to_rest()
job.update(
{
"current_working_directory": working_directory,
"environment": [
"PATH=/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:"
"/usr/sbin:/usr/lpp/mmfs/bin:"
],
"hold": True,
}
)
result = self.rest_client.submit(script, job)
slurm_job_id = str(result["job_id"])
job_future = JobFuture(job_id=job_id, manager=self, slurm_job_id=slurm_job_id)
slurm_job_id = job_future.slurm_job_id
cluster_args = _cluster_args(job_future.slurm_cluster)
topic_info = TopicInfo(job_id=job_id)
with self._lock:
self._future_registry[job_id] = job_future
try:
self._subscribe_to_internal_topics(future=job_future, topic_info=topic_info)
self.sub_client.wait_until_subscribed([topic_info.heartbeat, topic_info.event])
released = self._run_subprocess(["scontrol", *cluster_args, "release", slurm_job_id])
if released.returncode != 0:
raise RuntimeError(f"scontrol release failed: {(released.stderr or '').strip()}")
self._job_tracker.track(job_future)
for event, callback in (callbacks or {}).items():
job_future.listen(event, callback)
self._job_tracker.wait_until_ready(job_id)
self.rest_client.release(slurm_job_id)
except Exception as exc:
# A setup failure must not leave a held allocation behind.
try:
cancelled = self._run_subprocess(["scancel", *cluster_args, slurm_job_id])
if cancelled.returncode != 0:
logger.error("Could not cancel held job %s: %s", slurm_job_id, cancelled.stderr)
except (OSError, subprocess.SubprocessError):
logger.exception("Could not cancel held job %s", slurm_job_id)
self._event_callback(
self.rest_client.cancel(slurm_job_id)
except (SlurmRestError, ValueError):
logger.error("Could not cancel held job %s", slurm_job_id)
self._job_tracker.apply_status(
StatusMessage(status=f"error:Submission setup failed: {exc}"), job_future
)
raise
return job_future
def cancel_job(
self, slurm_job_id: str, job_id: str, *, slurm_cluster: str | None = None
) -> None:
"""Cancel a job on its recorded cluster via scancel."""
job_future = self._future_registry.get(job_id)
slurm_job_id, _, cluster = slurm_job_id.partition(";")
cluster = slurm_cluster or cluster or (job_future.slurm_cluster if job_future else None)
result = self._run_subprocess(["scancel", *_cluster_args(cluster), slurm_job_id])
if result.returncode != 0:
stderr = (result.stderr or "").strip()
raise RuntimeError(f"scancel failed: {stderr}")
def cancel_job(self, slurm_job_id: str, job_id: str) -> None:
"""Request cancellation through REST and mark the tracked job cancelled.
Args:
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
job_id (str): Opaque manager identifier used in Redis topic names.
Raises:
SlurmRestError: Slurm rejects cancellation or the request fails.
"""
job_future = self._job_tracker.get_future(job_id)
self.rest_client.cancel(slurm_job_id)
if job_future:
self._event_callback(
self._job_tracker.apply_status(
StatusMessage(
status="cancelled",
metadata={
@@ -283,127 +201,107 @@ class SlurmManager:
future=job_future,
)
################################################
# Public Event Subscriptions #
################################################
def listen(
self,
event_type: Literal["heartbeat", "log", "status"],
callback: Callable[[SlurmMessage], None],
job_id: str | None = None,
) -> list[str]:
"""
Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status').
If the job_id is specified, the callback will only be registered for the specific job. If job_id is None,
the callback will be registered for all jobs.
Returns a callback ID that can be used to unregister.
"""Attach a callback to one tracked job or all currently tracked jobs.
Args:
event_type: The type of event to listen for ('heartbeat', 'log', or 'status').
callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for any job.
The SlurmMessage object contains two fields, 'topic', 'data'
event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log,
status, or heartbeat.
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
Slurm message.
job_id (str | None): Opaque manager identifier used in Redis topic names.
Returns:
A list of unique callback IDS will be returned which can be used to unregister the callbacks later.
"""
job_futures: list[JobFuture] = (
[self._future_registry[job_id]] if job_id else list(self._future_registry.values())
)
callback_ids = []
for future in job_futures:
callback_ids.append(future.listen(event_type=event_type, callback=callback))
list[str]: Subscription IDs accepted by unlisten().
return callback_ids
Raises:
KeyError: The explicitly requested job ID is not tracked.
RuntimeError: A selected job has already completed.
Notes:
With job_id=None, registration applies to the current jobs, not future
submissions. Callbacks run in the tracker executor.
"""
return self._job_tracker.listen(event_type, callback, job_id)
def unlisten(self, callback_id: str | list[str]) -> None:
"""Remove one or more callback subscriptions without cancelling the job.
Args:
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
returned during registration.
"""
self.remove_subscription_by_id(callback_id)
def remove_subscription_by_id(self, callback_id: str | list[str]) -> None:
"""Remove a subscription by its callback ID."""
if isinstance(callback_id, str):
callback_id = [callback_id]
with self._lock:
for cb_id in callback_id:
topic = self._topic_by_subscription_id.pop(cb_id, None)
if topic is not None:
callbacks = self._subscriptions_by_topic.get(topic, {})
sub_info = callbacks.pop(cb_id, None)
self.sub_client.unsubscribe(cb_id)
# Other callbacks, including heartbeat monitoring, still own this topic.
if not callbacks:
self._subscriptions_by_topic.pop(topic, None)
if sub_info:
topics = self._active_subscriptions.get(sub_info.job_id)
if topics is not None:
topics.discard(topic)
def remove_subscription_by_topic(self, topic: str) -> None:
"""Remove a subscription by its topic."""
with self._lock:
callbacks = self._subscriptions_by_topic.pop(topic, {})
for callback_id, sub_info in callbacks.items():
self._topic_by_subscription_id.pop(callback_id, None)
job_id = sub_info.job_id
self._active_subscriptions[job_id].discard(topic)
# Remove the topic subscription from Redis
self.sub_client.unsubscribe_topic(topic)
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
"""
Remove all subscriptions for a specific job. If skip_event_callback is False, this will also trigger
the event callback with an error status which will update the JobFuture status.
This can be used to trigger a cleanup of the Job and its subscriptions.
If it finished in success, it should be called with skip_event_callback=True to avoid overwriting
the finished status with an error status.
"""Remove the selected callbacks while preserving other topic listeners.
Args:
job_id: The ID of the job for which to remove subscriptions.
skip_event_callback: If True, the event callback will not be triggered with an error status.
This should be used when the job is finishing successfully to avoid overwriting the
finished status with an error status.
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
returned during registration.
"""
with self._lock:
topics = self._active_subscriptions.pop(job_id, set())
for topic in topics:
callbacks = self._subscriptions_by_topic.pop(topic, {})
for callback_id in callbacks.keys():
self._topic_by_subscription_id.pop(callback_id, None)
self.sub_client.unsubscribe_topic(topic)
future = self._future_registry.get(job_id)
if future:
# This callback will trigger a cleanup
if not skip_event_callback:
self._event_callback(
StatusMessage(
status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}",
metadata={
"msg_received": datetime.now().isoformat(),
"job_id": future._job_id,
"slurm_job_id": future.slurm_job_id,
},
),
future,
)
self._remove_job_future(future._job_id)
self._job_tracker.remove_subscription_by_id(callback_id)
def remove_subscription_by_topic(self, topic: str) -> None:
"""Remove all callbacks for one topic and unsubscribe it from Redis.
Args:
topic (str): Redis channel whose callback registrations are selected.
"""
self._job_tracker.remove_subscription_by_topic(topic)
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
"""Remove job subscriptions and discard the tracked future.
Args:
job_id (str): Opaque manager identifier used in Redis topic names.
skip_event_callback (bool): If true, remove monitoring without synthesizing
a failure update.
Notes:
Unless skip_event_callback is true, an unfinished future is marked failed.
This only removes local monitoring; it does not cancel the Slurm allocation.
"""
self._job_tracker.remove_subscriptions_for_job(job_id, skip_event_callback)
################################################
# Public Lifecycle #
################################################
def shutdown(self, timeout: float | None = 1.0) -> None:
"""Shuts down the manager, including the heartbeat monitoring thread and Redis subscriptions."""
"""Stop monitoring, close Redis, and drain accepted callback work.
Args:
timeout (float | None): Thread-join timeout in seconds; None waits
indefinitely.
Raises:
TimeoutError: The heartbeat monitor is still performing a Slurm query.
Notes:
Remote jobs are not cancelled. The timeout applies to thread joins; running
and queued user callbacks are drained without a callback execution deadline.
"""
self._stop_heartbeat_thread.set()
self._heartbeat_monitoring_thread.join(timeout)
if self._heartbeat_monitoring_thread.is_alive():
raise TimeoutError(
"Heartbeat monitor did not stop; retry after its Slurm query finishes"
)
with self._lock:
self._subscriptions_by_topic.clear()
self._topic_by_subscription_id.clear()
self._job_tracker.shutdown(timeout)
self.sub_client.shutdown(timeout=timeout)
self.executor.shutdown(wait=True, cancel_futures=True)
###################
### Internal Methods
###################
################################################
# Internal Callback Registration #
################################################
def _add_subscription(
self,
@@ -412,182 +310,138 @@ class SlurmManager:
callback: Callable[[SlurmMessage], None],
callback_kwargs: dict[str, Any] | None = None,
) -> str:
"""
Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status') for a specific job.
Callbacks receive a SlurmMessage object, depending on the type of event they listen to. The callback will be executed
in the background and errors will be logged but not raised to not jeopardize the main loop of the manager.
The callback will be executed with additional keyword arguments passed via callback_kwargs dictionary.
"""Atomically register a Redis callback and its manager ownership.
Args:
job_id: The ID of the job for which to register the callback.
key: The type of event to listen for ('heartbeat', 'log', or 'status').
callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for the job.
callback_kwargs: A dictionary of additional keyword arguments to pass to the callback when it is executed
job_id (str): Opaque manager identifier used in Redis topic names.
key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or
heartbeat.
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
Slurm message.
callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed
to the user callback.
Returns:
str: Unique callback subscription ID.
Raises:
RuntimeError: The job is absent or finishes during registration.
"""
topic_str = TopicInfo(job_id=job_id).topic(key)
return self._job_tracker.add_subscription(job_id, key, callback, callback_kwargs)
@wraps(callback)
def wrapped_callback(
message: SlurmMessage,
executor: ThreadPoolExecutor,
job_id: str,
slurm_job_id: str,
callback_kwargs: dict[str, Any] | None = None,
) -> None:
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": job_id,
"slurm_job_id": slurm_job_id,
}
)
executor.submit(callback, message, **(callback_kwargs or {}))
################################################
# Internal Script Packaging #
################################################
# Completion removes subscriptions under this same lock. Keep ownership
# registration atomic with cleanup, and undo Redis registration if the
# future became terminal while subscribe() was in progress.
with self._lock:
future = self._future_registry.get(job_id)
if future is None or future.done():
raise RuntimeError(f"Job {job_id} is no longer active")
cb_id = self.sub_client.subscribe(
topic_str,
partial(
wrapped_callback,
executor=self.executor,
callback_kwargs=callback_kwargs,
job_id=job_id,
slurm_job_id=future.slurm_job_id,
),
)
if self._future_registry.get(job_id) is not future or future.done():
self.sub_client.unsubscribe(cb_id)
raise RuntimeError(f"Job {job_id} finished during subscription registration")
sub_info = SubscriptionInfo(
job_id=job_id,
topic=topic_str,
slurm_job_id=future.slurm_job_id,
callback=wrapped_callback,
last_heartbeat=None,
callback_id=cb_id,
heartbeat_received=False,
)
self._register_subscription_for_job(sub_info=sub_info)
return cb_id
def _remove_job_future(self, job_id: str) -> None:
"""Remove a job future from the registry."""
with self._lock:
self._future_registry.pop(job_id, None)
def _run_subprocess(self, cmd: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=30)
def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None:
self._subscribe_to_heartbeat(topic_info, future)
self._subscribe_to_event(topic_info, future)
def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None:
sub_info = SubscriptionInfo(
job_id=topic_info.job_id,
topic=topic_info.heartbeat,
slurm_job_id=future.slurm_job_id,
callback=None,
last_heartbeat=None,
callback_id=None,
heartbeat_received=False,
)
topic = topic_info.heartbeat
def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
try:
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": sub_info.job_id,
"slurm_job_id": sub_info.slurm_job_id,
}
)
self._heartbeat_callback(message, sub_info)
except Exception:
logger.error(
"Error in callback for topic %s: %s", message.msg_type, message.timestamp
)
cb_id = self.sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info))
sub_info.callback_id = cb_id
sub_info.last_heartbeat = time.time()
self._register_subscription_for_job(sub_info=sub_info)
def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
sub_info.heartbeat_received = True
sub_info.last_heartbeat = time.time()
def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None:
sub_info = SubscriptionInfo(
job_id=topic_info.job_id,
topic=topic_info.event,
slurm_job_id=future.slurm_job_id,
callback=None,
last_heartbeat=None,
callback_id=None,
heartbeat_received=False,
)
event_topic = topic_info.event
def wrapped_callback(message: StatusMessage, future: JobFuture) -> None:
try:
message.metadata.update(
{
"msg_received": datetime.now().isoformat(),
"job_id": future._job_id,
"slurm_job_id": future.slurm_job_id,
}
)
self._event_callback(message, future=future)
except Exception:
logger.error("Error in callback for topic %s: %s", message.msg_type, message.status)
cb_id = self.sub_client.subscribe(event_topic, partial(wrapped_callback, future=future))
sub_info.callback_id = cb_id
self._register_subscription_for_job(sub_info=sub_info)
def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None:
with self._lock:
if sub_info.callback_id is None:
raise ValueError("SubscriptionInfo.callback_id must be set before registration")
self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info
self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic
self._active_subscriptions[sub_info.job_id].add(sub_info.topic)
def _event_callback(self, message: StatusMessage, future: JobFuture) -> None:
"""
Callback for handling event message updates. The method should also be used to trigger
a cleanup of subscriptions and job future when a job is finishing unexpectedly
(e.g. due to missing heartbeat or user requested removal). If finishing unexpectedly,
we recommend using a payload format of 'error:reason' to indicate the error and reason
for the job finishing, which will be reflected in the JobFuture status and exception info.
@staticmethod
def _build_job_script(
script_path: str | Path,
env_path: str | Path | None,
job_id: str,
*,
worker_path: str = "/usr/local/bin/slurm-worker",
redis_address: str = "localhost:6379",
) -> str:
"""Package local Bash file contents for the compute-node Go worker.
Args:
message: The incoming message from the event topic, expected to have a payload format of 'status:optional_info'.
future: The JobFuture associated with the job, which will be updated based on the event message.
script_path (str | Path): Local Bash workload file whose contents are uploaded.
env_path (str | Path | None): Optional local Bash activation file for
workload-specific environment setup; None skips activation commands.
job_id (str): Opaque manager identifier used in Redis topic names.
worker_path (str): Absolute path to the Go executable on compute nodes.
redis_address (str): Redis host:port passed to the worker; no credentials are
embedded.
Returns:
str: Batch script invoking the configured worker with JSON on standard input.
Raises:
FileNotFoundError: A local script file is missing.
ValueError: The worker path is not an absolute compute-node path.
Notes:
The payload contains only the selected workload and activation scripts, not the
credential dotenv file. Staging and cleanup belong to the worker.
"""
info = message.status.split(":", 1)
status = info[0]
try: # Catch irregular formatted status messages to avoid crashing the callback loop.
status = JobStatus(status)
except ValueError:
logger.warning("Received unknown job status '%s' for job %s", status, future.name)
future._update_status(
"error",
f"Received unknown job status '{status}' from message: {message.model_dump()}",
)
if not worker_path.startswith("/"):
raise ValueError("Worker path must be an absolute compute-node path")
payload = json.dumps(
{
"script": Path(script_path).read_text(),
"activation": Path(env_path).read_text() if env_path is not None else "",
}
)
delimiter = f"SLURM_PAYLOAD_{uuid4().hex}"
command = shlex.join([worker_path, "--job-id", job_id, "--redis-address", redis_address])
return f"#!/bin/bash\nexec {command} <<'{delimiter}'\n{payload}\n{delimiter}\n"
################################################
# Internal Monitoring #
################################################
def _start_threads(self) -> None:
"""Start the manager heartbeat-monitor thread."""
self._heartbeat_monitoring_thread.start()
def _heartbeat_monitoring_loop(self) -> None:
"""Check missing heartbeats until shutdown and query Slurm when necessary."""
while not self._stop_heartbeat_thread.wait(timeout=2):
for sub_info in self._job_tracker.get_missing_heartbeats():
self._check_slurm_job(sub_info)
def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None:
"""Resolve a missed heartbeat from controller or accounting records.
Args:
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
bookkeeping.
Notes:
Transport failures, absent records, and incomplete accounting leave the job
unresolved for a later check.
"""
future = self._job_tracker.begin_scheduler_check(sub_info.job_id)
if future is None or future.done():
return
exc_info = info[1] if len(info) > 1 else None
future._update_status(status, exc_info)
# Clean up subscriptions for finished job
if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]:
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
try:
record = self.rest_client.get_job(future.slurm_job_id)
if record is None:
record = self.rest_client.get_job(future.slurm_job_id, accounting=True)
except (SlurmRestError, ValueError):
logger.warning("Could not query Slurm job %s", future.slurm_job_id)
return
if not record:
return
state = record.get("job_state", record.get("state", []))
if isinstance(state, dict):
state = state.get("current", [])
states = [state] if isinstance(state, str) else (state or [])
terminal = {
"COMPLETED",
"CANCELLED",
"FAILED",
"TIMEOUT",
"OUT_OF_MEMORY",
"NODE_FAIL",
"BOOT_FAIL",
"DEADLINE",
"PREEMPTED",
"REVOKED",
}
state = next((value for value in states if value in terminal), None)
if state is None:
return
exit_code = _job_exit_code(record.get("exit_code"))
if state == "COMPLETED" and exit_code == 0:
status = JobStatus.FINISHED.value
elif state == "CANCELLED":
status = JobStatus.CANCELLED.value
elif state == "COMPLETED" and exit_code is None:
# A partial accounting record is not proof of failure.
return
else:
status = f"error:Slurm state {state}, exit code {exit_code}"
self._job_tracker.apply_status(StatusMessage(status=status), future)
+296
View File
@@ -0,0 +1,296 @@
"""Small authenticated client for the slurmrestd v0.0.44 API."""
from __future__ import annotations
import json
import os
import re
from http import HTTPStatus
from http.client import HTTPException
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
from dotenv import dotenv_values
INVALID_JOB_ID = 2017
def _missing_job(result: dict[str, Any]) -> bool:
"""Check whether every reported Slurm error means an unknown job ID.
Args:
result (dict[str, Any]): Decoded Slurm REST response envelope.
Returns:
bool: True only for a nonempty list of invalid-job-ID errors.
"""
errors = result.get("errors")
return (
isinstance(errors, list)
and bool(errors)
and all(
isinstance(error, dict) and error.get("error_number") == INVALID_JOB_ID
for error in errors
)
)
class SlurmRestError(RuntimeError):
"""A transport, HTTP, or Slurm application error."""
class _NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *_args: Any, **_kwargs: Any) -> None:
# Never forward the authentication headers to a redirect target.
"""Reject redirects so authentication headers cannot reach another endpoint.
Args:
*_args (Any): Positional redirect-hook arguments supplied by urllib.
**_kwargs (Any): Keyword redirect-hook arguments supplied by urllib.
Returns:
None: Redirect rejection, preventing urllib from following the response.
"""
return None
class SlurmRestClient:
"""Load connection settings and a JWT from a local dotenv file or environment.
Environment variables override the file. The file is read again for every
request so replacing an expired token does not require restarting a manager.
"""
def __init__(self, env_file: str | Path = ".env", timeout: float = 30) -> None:
"""Load and validate slurmrestd connection settings.
Args:
env_file (str | Path): Local credential dotenv path. Environment variables
take precedence.
timeout (float): Positive HTTP request timeout in seconds.
Raises:
ValueError: The URL, API version, token, or timeout is invalid.
"""
self.env_file = Path(env_file).resolve()
if timeout <= 0:
raise ValueError("REST timeout must be positive")
self.timeout = timeout
settings = self._settings()
self.url = settings.get("SLURM_REST_URL", "").rstrip("/")
parsed = urlsplit(self.url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise ValueError("Set SLURM_REST_URL to the slurmrestd HTTP(S) URL")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("SLURM_REST_URL must not contain credentials, query, or fragment")
self.api_version = settings.get("SLURM_REST_API_VERSION") or "v0.0.44"
if not re.fullmatch(r"v\d+\.\d+\.\d+", self.api_version):
raise ValueError("SLURM_REST_API_VERSION must look like v0.0.44")
self._headers(settings)
self._opener = build_opener(_NoRedirect())
################################################
# Public Job Operations #
################################################
def submit(self, script: str, job: dict[str, Any]) -> dict[str, Any]:
"""Submit a batch script and its compute-node job settings.
Args:
script (str): Complete batch script text sent directly to slurmrestd.
job (dict[str, Any]): REST job-description fields, including compute-node
paths and resources.
Returns:
dict[str, Any]: Submission response containing a numeric job_id.
Raises:
SlurmRestError: Submission fails or no numeric job ID is returned.
"""
result = self.request(
"POST", f"slurm/{self.api_version}/job/submit", payload={"script": script, "job": job}
)
if not str(result.get("job_id", "")).isdigit():
raise SlurmRestError("Slurm REST submission did not return a job ID")
return result
def release(self, job_id: str) -> None:
"""Release a job that was submitted with hold enabled.
Args:
job_id (str): Numeric scheduler job ID on this REST endpoint.
Raises:
SlurmRestError: The job update fails.
"""
self.request("POST", self._job_path(job_id), payload={"hold": False})
def cancel(self, job_id: str) -> None:
"""Cancel a job on the cluster served by this REST endpoint.
Args:
job_id (str): Numeric scheduler job ID on this REST endpoint.
Raises:
SlurmRestError: Cancellation fails.
"""
self.request("DELETE", self._job_path(job_id))
def get_job(self, job_id: str, *, accounting: bool = False) -> dict[str, Any] | None:
"""Fetch a controller or accounting job record.
Args:
job_id (str): Numeric scheduler job ID on this REST endpoint.
accounting (bool): If true, query slurmdb instead of the controller.
Returns:
dict[str, Any] | None: First returned job record, or None when the job is
unavailable.
Raises:
SlurmRestError: The query fails or its records are malformed.
"""
path = self._job_path(job_id, accounting=accounting)
result = self.request("GET", path, missing_ok=True)
jobs = result.get("jobs") or []
if not isinstance(jobs, list) or any(not isinstance(job, dict) for job in jobs):
raise SlurmRestError("Slurm REST returned invalid job records")
return jobs[0] if jobs else None
################################################
# Public HTTP Requests #
################################################
def request(
self,
method: str,
path: str,
*,
payload: dict[str, Any] | None = None,
missing_ok: bool = False,
) -> dict[str, Any]:
"""Send one authenticated JSON request and validate the response envelope.
Args:
method (str): HTTP method such as GET, POST, or DELETE.
path (str): Relative REST endpoint appended to the configured service URL.
payload (dict[str, Any] | None): JSON-compatible request body or incoming
message payload.
missing_ok (bool): If true, treat missing-job responses as an empty result.
Returns:
dict[str, Any]: Decoded response object; an empty object for an allowed
missing job.
Raises:
SlurmRestError: HTTP, transport, JSON, or Slurm application validation
fails.
ValueError: The current token is missing.
Notes:
Requests are not automatically retried. Authentication headers and response
bodies are not included in raised transport errors.
"""
headers = self._headers(self._settings())
data = None if payload is None else json.dumps(payload).encode()
if data is not None:
headers["Content-Type"] = "application/json"
request = Request(f"{self.url}/{path}", data=data, headers=headers, method=method)
try:
with self._opener.open(request, timeout=self.timeout) as response:
raw = response.read()
except HTTPError as exc:
try:
# slurmrestd may use HTTP 500 for a purged controller job. Check
# only its numeric error code; never include the body in logs.
error_body = json.loads(exc.read()) if missing_ok else {}
except (ValueError, UnicodeDecodeError, OSError, HTTPException):
error_body = {}
finally:
exc.close()
if missing_ok and (
exc.code == HTTPStatus.NOT_FOUND
or (
exc.code in {HTTPStatus.BAD_REQUEST, HTTPStatus.INTERNAL_SERVER_ERROR}
and isinstance(error_body, dict)
and _missing_job(error_body)
)
):
return {}
raise SlurmRestError(f"Slurm REST {method} failed (HTTP {exc.code})") from None
except (URLError, OSError, TimeoutError, HTTPException):
raise SlurmRestError(f"Slurm REST {method} failed (connection or timeout)") from None
try:
result = json.loads(raw) if raw else {}
except (ValueError, UnicodeDecodeError):
raise SlurmRestError("Slurm REST returned invalid JSON") from None
if not isinstance(result, dict):
raise SlurmRestError("Slurm REST returned an invalid response")
errors = result.get("errors") or []
if errors:
if missing_ok and _missing_job(result):
return {}
raise SlurmRestError("Slurm REST reported an application error")
return result
################################################
# Internal Authentication #
################################################
def _settings(self) -> dict[str, str]:
"""Read local dotenv values and apply environment-variable overrides.
Returns:
dict[str, str]: Connection and authentication settings without modifying
os.environ.
"""
values = dotenv_values(self.env_file, interpolate=False)
keys = ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER")
return {key: os.environ.get(key, values.get(key) or "") for key in keys}
@staticmethod
def _headers(settings: dict[str, str]) -> dict[str, str]:
"""Build authentication headers from the current settings.
Args:
settings (dict[str, str]): Connection settings containing the JWT and
optional user name.
Returns:
dict[str, str]: JWT, optional user-name, and response-format headers.
Raises:
ValueError: The token is empty.
"""
token = settings.get("SLURM_JWT", "")
if not token:
raise ValueError("Set SLURM_JWT in the local .env file or environment")
headers = {"X-SLURM-USER-TOKEN": token, "Accept": "application/json"}
if settings.get("SLURM_USER"):
headers["X-SLURM-USER-NAME"] = settings["SLURM_USER"]
return headers
################################################
# Internal Endpoints #
################################################
def _job_path(self, job_id: str, *, accounting: bool = False) -> str:
"""Build a versioned controller or accounting job endpoint.
Args:
job_id (str): Numeric scheduler job ID on this REST endpoint.
accounting (bool): If true, query slurmdb instead of the controller.
Returns:
str: Relative REST endpoint path.
Raises:
ValueError: The scheduler job ID is not numeric.
"""
if not str(job_id).isdigit():
raise ValueError("Slurm job ID must be numeric")
service = "slurmdb" if accounting else "slurm"
return f"{service}/{self.api_version}/job/{job_id}"
View File
Whitespace-only changes.
-26
View File
@@ -1,26 +0,0 @@
SLURM_JOB_STATE_CODES = {
"BF": "BOOT_FAIL", # Job terminated due to launch failure, typically due to a hardware failure (e.g. unable to boot the node or block and the job can not be requeued).
"CA": "CANCELLED", # Job was explicitly cancelled by the user or system administrator. The job may or may not have been initiated.
"CD": "COMPLETED", # Job has terminated all processes on all nodes with an exit code of zero.
"CF": "CONFIGURING", # Job has been allocated resources, but are waiting for them to become ready for use (e.g. booting).
"CG": "COMPLETING", # Job is in the process of completing. Some processes on some nodes may still be active.
"DL": "DEADLINE", # Job terminated on deadline.
"F": "FAILED", # Job terminated with non-zero exit code or other failure condition.
"NF": "NODE_FAIL", # Job terminated due to failure of one or more allocated nodes.
"OOM": "OUT_OF_MEMORY", # Job experienced out of memory error.
"PD": "PENDING", # Job is awaiting resource allocation.
"PR": "PREEMPTED", # Job terminated due to preemption.
"R": "RUNNING", # Job currently has an allocation.
"RD": "RESV_DEL_HOLD", # Job is being held after requested reservation was deleted.
"RF": "REQUEUE_FED", # Job is being requeued by a federation.
"RH": "REQUEUE_HOLD", # Held job is being requeued.
"RQ": "REQUEUED", # Completing job is being requeued.
"RS": "RESIZING", # Job is about to change size.
"RV": "REVOKED", # Sibling was removed from cluster due to other cluster starting the job.
"SI": "SIGNALING", # Job is being signaled.
"SE": "SPECIAL_EXIT", # The job was requeued in a special state. This state can be set by users, typically in EpilogSlurmctld, if the job has terminated with a particular exit value.
"SO": "STAGE_OUT", # Job is staging out files.
"ST": "STOPPED", # Job has an allocation, but execution has been stopped with SIGSTOP signal. CPUS have been retained by this job.
"S": "SUSPENDED", # Job has an allocation, but execution has been suspended and CPUs have been released for other jobs.
"TO": "TIMEOUT", # Job terminated upon reaching its time limit
}
+51
View File
@@ -0,0 +1,51 @@
"""Reject invalid resource requests before sending them to Slurm."""
import pytest
from pydantic import ValidationError
from slurm_manager.job_options import JobOptions
@pytest.mark.parametrize(
"field,value",
[
("tasks", 0),
("tasks", -1),
("tasks", True),
("tasks", 1.5),
("time_limit_minutes", 0),
("time_limit_minutes", -1),
("time_limit_minutes", "60"),
("partition", " "),
("hold", False),
("time_limit", {"set": True, "number": 1}),
],
)
def test_invalid_or_internal_options_are_rejected(field, value):
with pytest.raises(ValidationError):
JobOptions(**{field: value})
def test_unset_options_preserve_slurm_defaults():
assert JobOptions().to_rest() == {"tasks": 1}
def test_runtime_uses_minutes_and_slurm_optional_number_shape():
options = JobOptions(partition="hour", tasks=2, time_limit_minutes=60)
assert options.to_rest() == {
"partition": "hour",
"tasks": 2,
"time_limit": {"set": True, "number": 60},
}
assert options.time_limit_minutes == 60
@pytest.mark.parametrize("partition", ["hour", "day", "week", "gpu"])
def test_supported_ra_partitions(partition):
assert JobOptions(partition=partition).to_rest()["partition"] == partition
@pytest.mark.parametrize("partition", ["admin", "jupyter", "gpu-large", "unknown"])
def test_unsupported_partitions_are_rejected(partition):
with pytest.raises(ValidationError):
JobOptions(partition=partition)
+48
View File
@@ -0,0 +1,48 @@
"""Verify local scripts and configuration reach the worker without shell expansion."""
import json
import subprocess
import sys
import pytest
from slurm_manager.slurm_manager import SlurmManager
def test_local_scripts_are_passed_to_worker_as_json(tmp_path):
workload = tmp_path / "work load.sh"
activation = tmp_path / "activate.sh"
workload.write_text('#!/bin/bash\nprintf "$HOME and $value"\n')
activation.write_text("export TEST_VALUE='literal $value and spaces'\n")
worker = tmp_path / "worker $(touch must-not-exist)"
worker.write_text(
f"#!{sys.executable}\nimport json,sys\n"
"print(json.dumps({'args':sys.argv[1:], 'payload':json.load(sys.stdin)}))\n"
)
worker.chmod(0o700)
script = SlurmManager._build_job_script(
workload, activation, "job-id", worker_path=str(worker), redis_address="redis.example:6380"
)
result = subprocess.run(
["bash"], input=script, capture_output=True, text=True, check=True, cwd=tmp_path
)
received = json.loads(result.stdout)
assert received["payload"] == {
"script": workload.read_text(),
"activation": activation.read_text(),
}
assert received["args"] == ["--job-id", "job-id", "--redis-address", "redis.example:6380"]
assert str(workload) not in script
assert not (tmp_path / "must-not-exist").exists()
def test_missing_local_file_fails_before_packaging(tmp_path):
with pytest.raises(FileNotFoundError):
SlurmManager._build_job_script(tmp_path / "missing.sh", tmp_path / "activate.sh", "job-id")
def test_worker_path_must_be_absolute(tmp_path):
with pytest.raises(ValueError, match="absolute"):
SlurmManager._build_job_script(
tmp_path / "unused.sh", tmp_path / "unused-env.sh", "job-id", worker_path="worker"
)
+68
View File
@@ -0,0 +1,68 @@
"""Exercise tracking independently of REST submission and scheduler monitoring."""
from functools import partial
from unittest import mock
import fakeredis
import pytest
from slurm_manager.event_messages import StatusMessage
from slurm_manager.job_future import JobFuture
from slurm_manager.job_tracker import JobTracker, TopicInfo
from slurm_manager.redis_sub_client import RedisSubClient
@pytest.fixture
def tracker():
client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
tracker = JobTracker(client)
try:
yield tracker
finally:
tracker.shutdown(timeout=3)
def test_heartbeat_snapshots_cannot_mutate_tracking_state(tracker):
future = JobFuture("first", manager=None, slurm_job_id="123")
with mock.patch("slurm_manager.job_tracker.time.time", return_value=80):
tracker.track(future)
with mock.patch("slurm_manager.job_tracker.time.time", return_value=100):
snapshot = tracker.get_missing_heartbeats()[0]
snapshot.last_heartbeat = 0
assert tracker.get_missing_heartbeats()[0].last_heartbeat == 80
assert tracker.begin_scheduler_check("first") is future
assert tracker.get_missing_heartbeats() == []
def test_completion_preserves_other_jobs_and_their_redis_subscriptions(tracker):
first = JobFuture("first", manager=None, slurm_job_id="123")
second = JobFuture("second", manager=None, slurm_job_id="124")
tracker.track(first)
tracker.track(second)
tracker.wait_until_ready("second")
tracker.apply_status(StatusMessage(status="finished"), first)
assert first.success()
assert tracker.get_future("first") is None
assert tracker.get_future("second") is second
assert not second.done()
assert tracker._sub_client.registered_topics == {
TopicInfo("second").heartbeat,
TopicInfo("second").event,
}
tracker._sub_client._redis.publish(
TopicInfo("second").event, '{"msg_type":"status","status":"finished"}'
)
second.wait(timeout=1)
assert second.success()
def test_scheduler_check_skips_a_job_removed_after_snapshot(tracker):
future = JobFuture("first", manager=None, slurm_job_id="123")
tracker.track(future)
tracker.apply_status(StatusMessage(status="finished"), future)
assert tracker.begin_scheduler_check("first") is None
assert tracker.begin_scheduler_check("unknown") is None
+234 -123
View File
@@ -2,10 +2,10 @@
from __future__ import annotations
import subprocess
import threading
import time
from functools import partial
from pathlib import Path
from unittest import mock
import fakeredis
@@ -13,23 +13,22 @@ import pytest
from slurm_manager.event_messages import HeartBeatMessage, StatusMessage
from slurm_manager.job_future import JobFuture
from slurm_manager.job_options import JobOptions
from slurm_manager.job_tracker import TopicInfo
from slurm_manager.redis_sub_client import RedisSubClient
from slurm_manager.slurm_manager import SlurmManager, TopicInfo
from slurm_manager.slurm_manager import SlurmManager
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
@pytest.fixture
def manager():
client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
rest = mock.Mock(spec=SlurmRestClient)
rest.submit.return_value = {"job_id": 123}
rest.get_job.return_value = {"job_state": ["PENDING"]}
with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=client):
manager = SlurmManager()
manager = SlurmManager(rest_client=rest)
def slurm(command):
output = (
"123\n" if command[0] == "sbatch" else "PENDING\n" if command[0] == "squeue" else ""
)
return subprocess.CompletedProcess(command, 0, output, "")
manager._run_subprocess = mock.Mock(side_effect=slurm)
try:
yield manager
finally:
@@ -40,13 +39,20 @@ def manager():
def submit(manager):
# The workload and activation environment belong to the remote node. They
# deliberately do not exist in the submitter's Python environment/filesystem.
return manager.submit_job("/remote/node/workload.sh", "/remote/node/activate.sh")
return manager.submit_job(
Path(__file__).parent.parent / "examples/print_for_10s.sh",
working_directory="/remote/node/work",
)
def heartbeat_info(manager, future):
return next(iter(manager._subscriptions_by_topic[TopicInfo(future._job_id).heartbeat].values()))
return next(
iter(
manager._job_tracker._subscriptions_by_topic[
TopicInfo(future._job_id).heartbeat
].values()
)
)
def test_wait_propagates_failure_arriving_during_wait():
@@ -79,46 +85,34 @@ def test_terminal_state_ignores_late_status_updates():
def test_job_is_held_until_redis_acknowledges_subscriptions(manager):
commands = []
def slurm(command):
commands.append(command)
if command[0] == "sbatch":
assert "--hold" in command
assert "/remote/node/activate.sh" in command
assert "/remote/node/workload.sh" in command
assert command[-1].endswith("redis_worker.sh")
return subprocess.CompletedProcess(command, 0, "123\n", "")
assert command == ["scontrol", "release", "123"]
future = next(iter(manager._future_registry.values()))
def release(job_id):
assert job_id == "123"
script, job = manager.rest_client.submit.call_args.args
assert job["hold"] is True
assert job["current_working_directory"] == "/remote/node/work"
assert '"activation": ""' in script
assert "stdout-test %02d/10" in script
assert "/usr/local/bin/slurm-worker" in script
future = next(iter(manager._job_tracker._future_registry.values()))
topics = TopicInfo(future._job_id)
assert manager.sub_client._subscription_ready[topics.event].is_set()
assert manager.sub_client._subscription_ready[topics.heartbeat].is_set()
# Simulate a remote job completing before scontrol returns.
manager.sub_client._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}')
future.wait(timeout=1)
return subprocess.CompletedProcess(command, 0, "", "")
manager._run_subprocess.side_effect = slurm
manager.rest_client.release.side_effect = release
future = submit(manager)
assert future.success()
assert [command[0] for command in commands] == ["sbatch", "scontrol"]
assert not manager.sub_client.registered_topics
@pytest.mark.parametrize("failure", ["subscription", "acknowledgment", "release"])
def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failure):
original = manager._run_subprocess.side_effect
def slurm(command):
if command[0] == "scontrol":
return subprocess.CompletedProcess(command, 1, "", "release rejected")
return original(command)
manager._run_subprocess.side_effect = slurm
if failure == "release":
manager.rest_client.release.side_effect = SlurmRestError("release rejected")
if failure == "subscription":
patch = mock.patch.object(
manager, "_subscribe_to_event", side_effect=RuntimeError("offline")
manager._job_tracker, "_subscribe_to_event", side_effect=RuntimeError("offline")
)
elif failure == "acknowledgment":
patch = mock.patch.object(
@@ -132,33 +126,64 @@ def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failur
)
with patch, pytest.raises((RuntimeError, TimeoutError)):
submit(manager)
assert mock.call(["scancel", "123"]) in manager._run_subprocess.call_args_list
assert not manager._future_registry
assert not manager._active_subscriptions
manager.rest_client.cancel.assert_called_once_with("123")
assert not manager._job_tracker._future_registry
assert not manager._job_tracker._active_subscriptions
assert not manager.sub_client.registered_topics
@pytest.mark.parametrize(
"last_heartbeat,last_check,expected",
[
(None, None, False),
(0, None, False),
(95, None, False),
(90, None, False),
(89, None, True),
(89, 95, False),
(89, 90, False),
(89, 89, True),
],
)
def test_heartbeat_selection_respects_timeout_and_scheduler_throttle(
manager, last_heartbeat, last_check, expected
):
manager._stop_heartbeat_thread.set()
manager._heartbeat_monitoring_thread.join(timeout=1)
future = submit(manager)
future.listen("heartbeat", lambda message: None)
future.listen("log", lambda message: None)
info = heartbeat_info(manager, future)
info.last_heartbeat = last_heartbeat
info.last_scheduler_check = last_check
with mock.patch("slurm_manager.job_tracker.time.time", return_value=100):
missing = manager._job_tracker.get_missing_heartbeats()
assert missing == ([info] if expected else [])
manager.rest_client.get_job.assert_not_called()
@pytest.mark.parametrize("state", ["PENDING", "CONFIGURING", "RUNNING"])
def test_missing_heartbeat_does_not_fail_active_slurm_job(manager, state):
future = submit(manager)
info = heartbeat_info(manager, future)
info.last_heartbeat = time.time() - 100
manager._run_subprocess.return_value = subprocess.CompletedProcess([], 0, f"{state}\n", "")
manager._run_subprocess.side_effect = None
manager.rest_client.get_job.return_value = {"job_state": [state]}
manager._check_slurm_job(info)
assert not future.done()
assert future._job_id in manager._future_registry
assert TopicInfo(future._job_id).heartbeat in manager._active_subscriptions[future._job_id]
assert manager._run_subprocess.call_args.args[0][0] == "squeue"
assert future._job_id in manager._job_tracker._future_registry
assert (
TopicInfo(future._job_id).heartbeat
in manager._job_tracker._active_subscriptions[future._job_id]
)
manager.rest_client.get_job.assert_called_once_with("123")
@pytest.mark.parametrize("accounting", ["", "PENDING|0:0\n"])
@pytest.mark.parametrize("accounting", [None, {"state": {"current": ["PENDING"]}}])
def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting):
future = submit(manager)
manager._run_subprocess.side_effect = [
subprocess.CompletedProcess([], 0, "", ""),
subprocess.CompletedProcess([], 0, accounting, ""),
]
manager.rest_client.get_job.side_effect = [None, accounting]
manager._check_slurm_job(heartbeat_info(manager, future))
assert not future.done()
@@ -166,28 +191,25 @@ def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting):
@pytest.mark.parametrize(
"accounting,status,success",
[
("COMPLETED|0:0\n", "finished", True),
("FAILED|7:0\n", "error", False),
("CANCELLED by 1000|0:15\n", "cancelled", False),
({"job_state": ["COMPLETED"], "exit_code": 0}, "finished", True),
({"job_state": ["FAILED"], "exit_code": 7}, "error", False),
({"job_state": ["CANCELLED"], "exit_code": 15}, "cancelled", False),
],
)
def test_terminal_accounting_resolves_missed_remote_status(manager, accounting, status, success):
future = submit(manager)
manager._run_subprocess.side_effect = [
subprocess.CompletedProcess([], 0, "", ""),
subprocess.CompletedProcess([], 0, accounting, ""),
]
manager.rest_client.get_job.side_effect = [None, accounting]
manager._check_slurm_job(heartbeat_info(manager, future))
assert future.done()
assert future.status == status
assert future.success() == success
assert not manager._future_registry
assert not manager._job_tracker._future_registry
assert not manager.sub_client.registered_topics
def test_scheduler_error_keeps_job_unresolved(manager):
future = submit(manager)
manager._run_subprocess.side_effect = FileNotFoundError("squeue unavailable")
manager.rest_client.get_job.side_effect = SlurmRestError("unavailable")
manager._check_slurm_job(heartbeat_info(manager, future))
assert not future.done()
@@ -201,7 +223,7 @@ def test_unlisten_removes_only_selected_callback_and_preserves_monitoring(manage
future.unlisten(callback_id)
topic = TopicInfo(future._job_id).heartbeat
assert callback_id not in manager.sub_client._topic_by_subscription_id
assert topic in manager._active_subscriptions[future._job_id]
assert topic in manager._job_tracker._active_subscriptions[future._job_id]
manager.sub_client._redis.publish(
topic, HeartBeatMessage(timestamp=time.time()).model_dump_json()
)
@@ -217,67 +239,36 @@ def test_removing_last_log_callback_unsubscribes_topic(manager):
callback_id = future.listen("log", lambda _: None)
future.unlisten(callback_id)
topic = TopicInfo(future._job_id).log
assert topic not in manager._active_subscriptions[future._job_id]
assert topic not in manager._job_tracker._active_subscriptions[future._job_id]
assert topic not in manager.sub_client.registered_topics
assert TopicInfo(future._job_id).heartbeat in manager.sub_client.registered_topics
def cluster_submission(manager):
original = manager._run_subprocess.side_effect
def slurm(command):
if command[0] == "sbatch":
return subprocess.CompletedProcess(command, 0, "123;cluster_a\n", "")
return original(command)
manager._run_subprocess.side_effect = slurm
def test_cluster_qualified_job_is_released_and_cancelled_on_its_cluster(manager):
cluster_submission(manager)
@pytest.mark.parametrize("code,success", [(0, True), (7, False)])
def test_accounting_structured_exit_code(manager, code, success):
future = submit(manager)
assert future.slurm_job_id == "123"
assert future.slurm_cluster == "cluster_a"
assert mock.call(["scontrol", "--clusters", "cluster_a", "release", "123"]) in (
manager._run_subprocess.call_args_list
)
future.cancel()
assert manager._run_subprocess.call_args == mock.call(
["scancel", "--clusters", "cluster_a", "123"]
)
def test_cluster_qualified_setup_failure_cancels_on_its_cluster(manager):
cluster_submission(manager)
with (
mock.patch.object(
manager.sub_client, "wait_until_subscribed", side_effect=TimeoutError("no ack")
),
pytest.raises(TimeoutError),
):
submit(manager)
assert manager._run_subprocess.call_args == mock.call(
["scancel", "--clusters", "cluster_a", "123"]
)
assert not manager._future_registry
assert not manager.sub_client.registered_topics
def test_cluster_qualified_job_monitoring_queries_its_cluster(manager):
cluster_submission(manager)
future = submit(manager)
manager._run_subprocess.reset_mock()
manager._run_subprocess.side_effect = [
subprocess.CompletedProcess([], 0, "", ""),
subprocess.CompletedProcess([], 0, "COMPLETED|0:0\n", ""),
manager.rest_client.get_job.side_effect = [
None,
{
"state": {"current": ["COMPLETED"]},
"exit_code": {
"return_code": {"set": True, "number": code},
"signal": {"id": {"set": False, "number": 0}},
},
},
]
manager._check_slurm_job(heartbeat_info(manager, future))
commands = [call.args[0] for call in manager._run_subprocess.call_args_list]
assert [command[0] for command in commands] == ["squeue", "sacct"]
for command in commands:
assert command[1:3] == ["--clusters", "cluster_a"]
assert command[command.index("--jobs") + 1] == "123"
assert future.success()
assert future.done()
assert future.success() == success
assert manager.rest_client.get_job.call_args == mock.call("123", accounting=True)
def test_cancel_uses_rest_and_cleans_subscriptions(manager):
future = submit(manager)
future.cancel()
manager.rest_client.cancel.assert_called_once_with("123")
assert future.status == "cancelled"
assert not manager.sub_client.registered_topics
def test_completion_during_subscription_registration_leaves_no_orphan(manager):
@@ -287,7 +278,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
cleaned = threading.Event()
errors = []
original_subscribe = manager.sub_client.subscribe
original_event = manager._event_callback
original_event = manager._job_tracker.apply_status
def subscribe(topic, callback):
callback_id = original_subscribe(topic, callback)
@@ -307,7 +298,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
with (
mock.patch.object(manager.sub_client, "subscribe", side_effect=subscribe),
mock.patch.object(manager, "_event_callback", side_effect=event),
mock.patch.object(manager._job_tracker, "apply_status", side_effect=event),
):
thread = threading.Thread(target=listen)
thread.start()
@@ -323,16 +314,16 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
assert not thread.is_alive()
assert cleaned.wait(1)
assert len(errors) == 1
assert not manager._future_registry
assert not manager._active_subscriptions
assert not manager._topic_by_subscription_id
assert not manager._job_tracker._future_registry
assert not manager._job_tracker._active_subscriptions
assert not manager._job_tracker._topic_by_subscription_id
assert not manager.sub_client.registered_topics
assert not manager.sub_client._topic_by_subscription_id
def test_listen_after_completion_does_not_register_callback(manager):
future = submit(manager)
manager._event_callback(StatusMessage(status="finished"), future)
manager._job_tracker.apply_status(StatusMessage(status="finished"), future)
with pytest.raises(RuntimeError, match="no longer active"):
future.listen("log", lambda _: None)
assert not manager.sub_client.registered_topics
@@ -377,5 +368,125 @@ def test_unlisten_allows_concurrent_job_cleanup(manager):
assert completed_during_removal == [True]
assert future.done()
assert not future._callback_registry
assert not manager._future_registry
assert not manager._job_tracker._future_registry
assert not manager.sub_client.registered_topics
@pytest.mark.parametrize("code", [0, 7])
def test_controller_structured_exit_code(manager, code):
future = submit(manager)
manager.rest_client.get_job.return_value = {
"job_state": ["COMPLETED"],
"exit_code": {
"status": ["SUCCESS" if code == 0 else "ERROR"],
"return_code": {"set": True, "number": code},
"signal": {"id": {"set": False, "number": 0}},
},
}
manager._check_slurm_job(heartbeat_info(manager, future))
assert future.done()
assert future.success() == (code == 0)
def test_local_files_are_uploaded_without_mutating_options(manager, tmp_path):
workload = tmp_path / "work load $(touch should_not_exist).sh"
activation = tmp_path / "it's my activation.sh"
workload.write_text("#!/bin/bash\nprintf 'hello'\n")
activation.write_text("module load redis\n")
options = JobOptions(partition="hour", tasks=2, time_limit_minutes=5)
manager.submit_job(workload, activation, working_directory="/remote/work", job_options=options)
script, job = manager.rest_client.submit.call_args.args
assert "hello" in script
assert "module load redis" in script
assert str(tmp_path) not in script
assert job["hold"] is True
assert options == JobOptions(partition="hour", tasks=2, time_limit_minutes=5)
assert job["tasks"] == 2
assert job["time_limit"] == {"set": True, "number": 5}
def test_submission_failure_does_not_register_job(manager):
manager.rest_client.submit.side_effect = SlurmRestError("submission failed")
with pytest.raises(SlurmRestError, match="submission failed"):
submit(manager)
assert not manager._job_tracker._future_registry
assert not manager.sub_client.registered_topics
manager.rest_client.release.assert_not_called()
manager.rest_client.cancel.assert_not_called()
def test_cancellation_failure_keeps_monitoring(manager):
future = submit(manager)
manager.rest_client.cancel.side_effect = SlurmRestError("cancellation failed")
with pytest.raises(SlurmRestError, match="cancellation failed"):
future.cancel()
assert not future.done()
assert future._job_id in manager._job_tracker._future_registry
def test_relative_remote_paths_are_rejected_before_submission(manager):
with pytest.raises(ValueError, match="absolute"):
manager.submit_job("workload.sh", "activate.sh", working_directory="relative")
manager.rest_client.submit.assert_not_called()
def test_public_callbacks_are_ready_before_release(manager):
received = threading.Event()
def release(_job_id):
future = next(iter(manager._job_tracker._future_registry.values()))
topic = TopicInfo(future._job_id).log
assert manager.sub_client._subscription_ready[topic].is_set()
manager.sub_client._redis.publish(topic, '{"msg_type":"log","log":"first line"}')
assert received.wait(1)
manager.rest_client.release.side_effect = release
manager.submit_job(
"examples/print_for_10s.sh",
working_directory="/remote/work",
callbacks={"log": lambda _: received.set()},
)
def test_unknown_status_cleans_up_the_terminal_future(manager):
future = submit(manager)
future.listen("log", lambda _: None)
manager._job_tracker.apply_status(StatusMessage(status="unexpected-status"), future)
assert future.done()
assert future.status == "error"
assert not manager._job_tracker._future_registry
assert not manager._job_tracker._active_subscriptions
assert not manager._job_tracker._subscriptions_by_topic
assert not manager._job_tracker._topic_by_subscription_id
assert not manager.sub_client.registered_topics
def test_shutdown_drains_all_received_log_callbacks(manager):
future = submit(manager)
release = threading.Event()
received = []
blocking = [manager._job_tracker._executor.submit(release.wait) for _ in range(8)]
future.listen("log", lambda message: received.append(message.log))
topic = TopicInfo(future._job_id)
for number in range(25):
manager.sub_client._redis.publish(topic.log, f'{{"msg_type":"log","log":"{number}"}}')
manager.sub_client._redis.publish(topic.event, '{"msg_type":"status","status":"finished"}')
future.wait(timeout=2)
# Release blocked workers only once shutdown reaches the executor. With the
# old cancel_futures=True, all 25 accepted callbacks would be discarded here.
original = manager._job_tracker._executor.shutdown
def shutdown(*, wait=True, cancel_futures=False):
original(wait=False, cancel_futures=cancel_futures)
release.set()
if wait:
original(wait=True)
try:
with mock.patch.object(manager._job_tracker._executor, "shutdown", side_effect=shutdown):
manager.shutdown(timeout=3)
finally:
release.set()
for task in blocking:
task.result(timeout=1)
assert sorted(received, key=int) == [str(number) for number in range(25)]
+213
View File
@@ -0,0 +1,213 @@
"""REST wire-format, authentication, and error handling tests."""
from __future__ import annotations
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pytest
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
@pytest.fixture
def endpoint(tmp_path, monkeypatch):
for key in ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER"):
monkeypatch.delenv(key, raising=False)
calls = []
replies = []
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.respond()
def do_POST(self):
self.respond()
def do_DELETE(self):
self.respond()
def respond(self):
body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
calls.append(
(self.command, self.path, dict(self.headers), json.loads(body) if body else None)
)
status, payload = replies.pop(0)
self.send_response(status)
if status == 302:
self.send_header("Location", "/redirect-target")
self.end_headers()
self.wfile.write(
payload if isinstance(payload, bytes) else json.dumps(payload).encode()
)
def log_message(self, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever)
thread.start()
env_file = tmp_path / ".env"
env_file.write_text(
f"SLURM_REST_URL=http://127.0.0.1:{server.server_port}\n"
"SLURM_REST_API_VERSION=v0.0.44\nSLURM_JWT=test-secret\n"
)
try:
yield env_file, calls, replies
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
assert not thread.is_alive()
def test_submit_release_cancel_and_accounting_wire_format(endpoint):
env_file, calls, replies = endpoint
replies.extend(
[
(200, {"job_id": 123, "errors": []}),
(200, {"errors": []}),
(200, {"jobs": [{"job_state": ["RUNNING"]}]}),
(200, {"jobs": [{"state": {"current": ["COMPLETED"]}}]}),
(200, {"errors": []}),
]
)
client = SlurmRestClient(env_file)
job = {
"current_working_directory": "/remote/work",
"environment": ["PATH=/usr/bin"],
"hold": True,
}
assert client.submit("#!/bin/bash\necho hello", job)["job_id"] == 123
client.release("123")
assert client.get_job("123")["job_state"] == ["RUNNING"]
assert client.get_job("123", accounting=True)["state"]["current"] == ["COMPLETED"]
client.cancel("123")
assert [(method, path) for method, path, _, _ in calls] == [
("POST", "/slurm/v0.0.44/job/submit"),
("POST", "/slurm/v0.0.44/job/123"),
("GET", "/slurm/v0.0.44/job/123"),
("GET", "/slurmdb/v0.0.44/job/123"),
("DELETE", "/slurm/v0.0.44/job/123"),
]
assert calls[0][3] == {"script": "#!/bin/bash\necho hello", "job": job}
assert calls[1][3] == {"hold": False}
assert calls[0][2]["X-Slurm-User-Token"] == "test-secret"
assert "test-secret" not in json.dumps(calls[0][3])
def test_token_rotation_and_environment_override(endpoint, monkeypatch):
env_file, calls, replies = endpoint
replies.extend([(200, {}), (200, {})])
client = SlurmRestClient(env_file)
env_file.write_text(env_file.read_text().replace("test-secret", "replacement-token"))
client.cancel("1")
monkeypatch.setenv("SLURM_JWT", "environment-token")
monkeypatch.setenv("SLURM_USER", "alice")
client.cancel("2")
assert calls[0][2]["X-Slurm-User-Token"] == "replacement-token"
assert calls[1][2]["X-Slurm-User-Token"] == "environment-token"
assert calls[1][2]["X-Slurm-User-Name"] == "alice"
@pytest.mark.parametrize(
"status,payload",
[
(401, b"test-secret"),
(500, b"test-secret"),
(200, {"errors": [{"description": "test-secret", "error_number": 1}]}),
(200, b"bad JSON test-secret"),
(200, []),
(302, b"redirect"),
],
)
def test_errors_are_raised_without_exposing_credentials(endpoint, status, payload):
env_file, calls, replies = endpoint
replies.append((status, payload))
client = SlurmRestClient(env_file)
with pytest.raises(SlurmRestError) as error:
client.cancel("123")
assert "test-secret" not in str(error.value)
assert len(calls) == 1
@pytest.mark.parametrize(
"reply",
[
(404, {}),
(500, {"errors": [{"error_number": 2017}]}),
(200, {"jobs": []}),
(200, {"errors": [{"error_number": 2017}]}),
],
)
def test_missing_job_is_not_a_terminal_failure(endpoint, reply):
env_file, _, replies = endpoint
replies.append(reply)
assert SlurmRestClient(env_file).get_job("123") is None
def test_missing_token_fails_before_any_request(endpoint):
env_file, calls, _ = endpoint
env_file.write_text(env_file.read_text().replace("test-secret", ""))
with pytest.raises(ValueError, match="SLURM_JWT"):
SlurmRestClient(env_file)
assert calls == []
def test_submission_requires_job_id(endpoint):
env_file, _, replies = endpoint
replies.append((200, {"errors": []}))
with pytest.raises(SlurmRestError, match="job ID"):
SlurmRestClient(env_file).submit("#!/bin/bash", {})
def test_manager_submission_and_completion_over_http(endpoint):
from functools import partial
from unittest import mock
import fakeredis
from slurm_manager.job_tracker import TopicInfo
from slurm_manager.redis_sub_client import RedisSubClient
from slurm_manager.slurm_manager import SlurmManager
env_file, calls, replies = endpoint
replies.extend([(200, {"job_id": 123}), (200, {})])
redis = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=redis):
manager = SlurmManager(env_file=env_file)
try:
future = manager.submit_job("examples/print_for_10s.sh", working_directory="/remote/work")
assert calls[0][3]["job"]["hold"] is True
assert calls[1][3] == {"hold": False}
topics = TopicInfo(future._job_id)
assert redis._subscription_ready[topics.event].is_set()
redis._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}')
future.wait(timeout=1)
assert future.success()
finally:
manager.shutdown(timeout=3)
assert not manager._heartbeat_monitoring_thread.is_alive()
assert not redis._listener_thread.is_alive()
@pytest.mark.parametrize("error_response", [False, True])
def test_truncated_responses_raise_a_transport_error(endpoint, error_response):
from http.client import IncompleteRead
from unittest import mock
from urllib.error import HTTPError
env_file, _, _ = endpoint
client = SlurmRestClient(env_file)
response = mock.MagicMock()
response.read.side_effect = IncompleteRead(b"partial", 100)
if error_response:
error = HTTPError("http://local/job", 500, "Server error", {}, response)
patch = mock.patch.object(client._opener, "open", side_effect=error)
else:
opened = mock.MagicMock()
opened.__enter__.return_value = response
patch = mock.patch.object(client._opener, "open", return_value=opened)
with patch, pytest.raises(SlurmRestError):
client.get_job("123")