mirror of
https://github.com/bec-project/slurm_manager.git
synced 2026-11-18 07:14:23 +01:00
feat!: submit Slurm jobs through REST and track execution
This commit is contained in:
1 parent
790ca5b396
commit
4d17ce9d54
20 files changed
+2246
-871
No files matched your search
@@ -0,0 +1,5 @@
|
||||
SLURM_REST_URL=https://ra-slurmapi-001.psi.ch
|
||||
SLURM_REST_API_VERSION=v0.0.44
|
||||
SLURM_JWT=
|
||||
# Optional: only set if required by your Slurm authentication configuration.
|
||||
# SLURM_USER=your_username
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
for second in {1..10}; do
|
||||
printf 'stdout-test %02d/10\n' "$second"
|
||||
sleep 1
|
||||
done
|
||||
+2
-2
@@ -14,9 +14,9 @@ classifiers = [
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"hiredis~=3.0",
|
||||
"pydantic~=2.12.0",
|
||||
"redis~=6.2",
|
||||
"redis[hiredis]>=6.2, <9.0",
|
||||
"python-dotenv~=1.2",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Annotated, Literal, TypeAlias
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, Field, TypeAdapter
|
||||
|
||||
@@ -49,8 +49,18 @@ SlurmMessage: TypeAlias = Annotated[
|
||||
SLURM_MESSAGE_ADAPTER = TypeAdapter(SlurmMessage)
|
||||
|
||||
|
||||
def parse_slurm_message(payload: str | dict) -> SlurmMessage:
|
||||
"""Parse a JSON payload into the appropriate slurm message model."""
|
||||
def parse_slurm_message(payload: str | dict[str, Any]) -> SlurmMessage:
|
||||
"""Validate a JSON or dictionary payload as a discriminated Slurm message.
|
||||
|
||||
Args:
|
||||
payload (str | dict[str, Any]): JSON-compatible request body or incoming message payload.
|
||||
|
||||
Returns:
|
||||
SlurmMessage: Heartbeat, status, or log message selected by msg_type.
|
||||
|
||||
Raises:
|
||||
ValueError: The JSON or message schema is invalid.
|
||||
"""
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
return SLURM_MESSAGE_ADAPTER.validate_python(payload)
|
||||
+149
-95
@@ -44,97 +44,133 @@ class RegisteredCallback:
|
||||
class JobFuture:
|
||||
"""
|
||||
A future-like object that represents a Slurm job. It allows users to check the status of the job,
|
||||
retrieve the job's result, and register callbacks for events (e.g., 'event', 'logs', 'heartbeat').
|
||||
wait for completion, and register callbacks for status, log, and heartbeat events.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
job_id: str,
|
||||
manager: SlurmManager,
|
||||
slurm_job_id: str,
|
||||
registry: dict | None = None,
|
||||
name: str | None = None,
|
||||
):
|
||||
self, job_id: str, manager: SlurmManager, slurm_job_id: str, name: str | None = None
|
||||
) -> None:
|
||||
"""Create a future for one job tracked by its submitting manager.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
manager (SlurmManager): Manager that owns the job and handles subscriptions
|
||||
and cancellation.
|
||||
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
|
||||
name (str | None): Display-name prefix; None uses Job, followed by the
|
||||
scheduler ID.
|
||||
"""
|
||||
self._job_id = job_id
|
||||
self._manager = manager
|
||||
self._slurm_job_id, _, cluster = slurm_job_id.strip().partition(";")
|
||||
self._slurm_cluster = cluster or None
|
||||
self._registry = registry if registry is not None else {}
|
||||
self._future = Future()
|
||||
self._slurm_job_id = slurm_job_id.strip()
|
||||
self._future: Future[None] = Future()
|
||||
if name is None:
|
||||
name = "Job"
|
||||
self.name = f"{name}-{self._slurm_job_id}"
|
||||
self._status: str = JobStatus.PENDING.value
|
||||
self._shutdown_event = threading.Event()
|
||||
self._lock = threading.RLock()
|
||||
self._callback_registry: dict[str, list[RegisteredCallback]] = {}
|
||||
|
||||
#####################
|
||||
## Public API
|
||||
#####################
|
||||
################################################
|
||||
# Public Properties #
|
||||
################################################
|
||||
|
||||
@property
|
||||
def slurm_job_id(self) -> str:
|
||||
"""The numeric SLURM job ID, without an optional cluster suffix."""
|
||||
"""Return the numeric scheduler job ID on the configured REST endpoint.
|
||||
|
||||
Returns:
|
||||
str: Scheduler job ID.
|
||||
"""
|
||||
return self._slurm_job_id
|
||||
|
||||
@property
|
||||
def slurm_cluster(self) -> str | None:
|
||||
"""The cluster returned by sbatch, or None for the default cluster."""
|
||||
return self._slurm_cluster
|
||||
|
||||
@property
|
||||
def status(self) -> str | None:
|
||||
"""The current status of the job."""
|
||||
"""Return the most recently accepted lifecycle status.
|
||||
|
||||
Returns:
|
||||
str | None: Pending, running, finished, error, or cancelled status.
|
||||
"""
|
||||
return self._status
|
||||
|
||||
def cancel(self):
|
||||
"""Cancel the job."""
|
||||
self._manager.cancel_job(
|
||||
slurm_job_id=self.slurm_job_id, job_id=self._job_id, slurm_cluster=self.slurm_cluster
|
||||
)
|
||||
################################################
|
||||
# Public Job Lifecycle #
|
||||
################################################
|
||||
|
||||
def done(self) -> bool:
|
||||
"""Check whether the job reached any terminal state.
|
||||
|
||||
Returns:
|
||||
bool: True for a finished, failed, or cancelled job.
|
||||
"""
|
||||
return self._future.done()
|
||||
|
||||
def success(self) -> bool:
|
||||
"""Check whether the job reached a successful terminal state.
|
||||
|
||||
Returns:
|
||||
bool: True only when the job finished without an exception.
|
||||
"""
|
||||
return self._future.done() and self._future.exception() is None
|
||||
|
||||
def exception(self) -> Exception | None:
|
||||
"""If the job has finished with an error, return the exception. Otherwise, return None."""
|
||||
"""Return the recorded terminal exception without waiting.
|
||||
|
||||
Returns:
|
||||
Exception | None: Failure or cancellation exception, or None if pending or
|
||||
successful.
|
||||
"""
|
||||
if self._future.done():
|
||||
return self._future.exception()
|
||||
return None
|
||||
|
||||
def success(self) -> bool:
|
||||
"""Return True if the job finished successfully, False if it finished with an error or is not done yet."""
|
||||
return self._future.done() and self._future.exception() is None
|
||||
|
||||
def done(self) -> bool:
|
||||
"""Return True if the job has finished (either successfully or with an error), False otherwise."""
|
||||
return self._future.done()
|
||||
|
||||
def wait(self, timeout: float | None = None) -> None:
|
||||
"""
|
||||
Wait for the job to finish, with an optional timeout in seconds. Raises TimeoutError if the job
|
||||
does not finish within the specified timeout.
|
||||
"""Wait for the job and propagate its failure or cancellation.
|
||||
|
||||
Args:
|
||||
timeout: Maximum time to wait for the job to finish, in seconds. If None, wait indefinitely.
|
||||
timeout (float | None): Maximum wait in seconds; None waits indefinitely
|
||||
without cancelling the job.
|
||||
|
||||
Raises:
|
||||
TimeoutError: The job does not reach a terminal state before the timeout.
|
||||
Exception: The job fails or is cancelled.
|
||||
|
||||
Notes:
|
||||
A timeout does not cancel the job. Callback execution may still be pending
|
||||
when this method returns; manager shutdown drains accepted callbacks.
|
||||
"""
|
||||
self._future.result(timeout=timeout)
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""Request cancellation through the owning manager.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: Slurm rejects cancellation or the request fails.
|
||||
"""
|
||||
self._manager.cancel_job(slurm_job_id=self.slurm_job_id, job_id=self._job_id)
|
||||
|
||||
################################################
|
||||
# Public Event Subscriptions #
|
||||
################################################
|
||||
|
||||
def listen(
|
||||
self,
|
||||
event_type: Literal["status", "log", "heartbeat"],
|
||||
callback: Callable[[SlurmMessage], None],
|
||||
) -> str:
|
||||
"""
|
||||
Register a callback to listen for events of the specified type ('status', 'log', or 'heartbeat').
|
||||
Returns a callback ID that can be used to unregister.
|
||||
"""Register an asynchronous callback for a job event.
|
||||
|
||||
Args:
|
||||
event_type: The type of event to listen for ('status', 'log', or 'heartbeat').
|
||||
callback: A function that takes a SlurmMessage object as input. The exact message object
|
||||
depends on the event type. Please check event_messages.py for the specific
|
||||
message class based on the event.
|
||||
event_type (Literal['status', 'log', 'heartbeat']): Event to receive: log,
|
||||
status, or heartbeat.
|
||||
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
|
||||
Slurm message.
|
||||
|
||||
Returns:
|
||||
A unique callback ID that can be used to unregister the callback later.
|
||||
str: Callback ID accepted by unlisten().
|
||||
|
||||
Raises:
|
||||
RuntimeError: The job has finished or is no longer tracked.
|
||||
"""
|
||||
cb_id = self._manager._add_subscription(
|
||||
job_id=self._job_id, key=event_type, callback=callback
|
||||
@@ -148,61 +184,53 @@ class JobFuture:
|
||||
return cb.id
|
||||
|
||||
def unlisten(self, callback_id: str) -> None:
|
||||
"""
|
||||
Unregister a previously registered callback using its callback ID.
|
||||
"""Remove a previously registered callback by ID.
|
||||
|
||||
Args:
|
||||
callback_id: The unique ID of the callback to unregister, as returned by listen()
|
||||
callback_id (str): Subscription ID, or IDs where accepted, returned during
|
||||
registration.
|
||||
"""
|
||||
# Manager cleanup may acquire this future's lock while holding its own.
|
||||
# Remove local ownership before acquiring the manager lock.
|
||||
if self._remove_callback(callback_id):
|
||||
self._manager.remove_subscription_by_id(callback_id=callback_id)
|
||||
|
||||
################################################
|
||||
# Internal Callback Management #
|
||||
################################################
|
||||
|
||||
def _remove_callback(self, callback_id: str) -> bool:
|
||||
"""Remove local callback ownership without acquiring the manager lock.
|
||||
|
||||
Args:
|
||||
callback_id (str): ID of the callback to remove.
|
||||
|
||||
Returns:
|
||||
bool: True if a registered callback was removed.
|
||||
"""
|
||||
removed = False
|
||||
with self._lock:
|
||||
for event, callbacks in self._callback_registry.items():
|
||||
for cb in callbacks:
|
||||
if cb.id == callback_id:
|
||||
callbacks.remove(cb)
|
||||
if not callbacks:
|
||||
self._callback_registry.pop(event, None)
|
||||
removed = True
|
||||
break
|
||||
if removed:
|
||||
break
|
||||
# Manager cleanup may acquire this future's lock while holding its own.
|
||||
# Release the future lock before acquiring the manager lock.
|
||||
if removed:
|
||||
self._manager.remove_subscription_by_id(callback_id=callback_id)
|
||||
if cb.id != callback_id:
|
||||
continue
|
||||
callbacks.remove(cb)
|
||||
if not callbacks:
|
||||
self._callback_registry.pop(event)
|
||||
return True
|
||||
return False
|
||||
|
||||
###################
|
||||
## Internal Methods
|
||||
## Not meant to be called by users
|
||||
###################
|
||||
|
||||
def _set_finished(self):
|
||||
self._shutdown_event.set()
|
||||
if self._future.done():
|
||||
logger.warning(
|
||||
"Job %s is already marked as done. Ignoring _set_finished call.", self.name
|
||||
)
|
||||
return
|
||||
self._future.set_result(None)
|
||||
|
||||
def _set_exception(self, status, exc_info=None):
|
||||
if self._future.done():
|
||||
logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name)
|
||||
return
|
||||
self._shutdown_event.set()
|
||||
# Here you could log the exception info or store it for later retrieval
|
||||
if exc_info:
|
||||
info = f"{self.name} failed with status {status}. Exception info: {exc_info}"
|
||||
else:
|
||||
info = f"{self.name} failed with status {status}."
|
||||
# You could log the info here, e.g., using logging.error(info)
|
||||
self._future.set_exception(Exception(info))
|
||||
|
||||
def _raise_if_failed(self) -> None:
|
||||
if self._future.done() and self._future.exception() is not None:
|
||||
raise self._future.exception()
|
||||
################################################
|
||||
# Internal State Updates #
|
||||
################################################
|
||||
|
||||
def _update_status(self, status: str, exc_info: str | None = None) -> None:
|
||||
"""Apply a lifecycle update under the future lock, ignoring late updates.
|
||||
|
||||
Args:
|
||||
status (str): Lifecycle status; a terminal state completes the underlying
|
||||
future.
|
||||
exc_info (str | None): Optional worker-provided failure details.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._future.done():
|
||||
return
|
||||
@@ -211,3 +239,29 @@ class JobFuture:
|
||||
self._set_finished()
|
||||
elif status in (JobStatus.CANCELLED, JobStatus.ERROR):
|
||||
self._set_exception(status, exc_info)
|
||||
|
||||
def _set_finished(self) -> None:
|
||||
"""Complete the underlying future successfully unless it is already done."""
|
||||
if self._future.done():
|
||||
logger.warning(
|
||||
"Job %s is already marked as done. Ignoring _set_finished call.", self.name
|
||||
)
|
||||
return
|
||||
self._future.set_result(None)
|
||||
|
||||
def _set_exception(self, status: str, exc_info: str | None = None) -> None:
|
||||
"""Record a terminal job failure with optional worker details.
|
||||
|
||||
Args:
|
||||
status (str): Lifecycle status; a terminal state completes the underlying
|
||||
future.
|
||||
exc_info (str | None): Optional worker-provided failure details.
|
||||
"""
|
||||
if self._future.done():
|
||||
logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name)
|
||||
return
|
||||
if exc_info:
|
||||
info = f"{self.name} failed with status {status}. Exception info: {exc_info}"
|
||||
else:
|
||||
info = f"{self.name} failed with status {status}."
|
||||
self._future.set_exception(Exception(info))
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Validated user-facing Slurm job options."""
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
||||
|
||||
|
||||
class JobOptions(BaseModel):
|
||||
"""Resources and output settings for one job.
|
||||
|
||||
An omitted partition uses the cluster default; an omitted time limit uses
|
||||
the selected partition's default. Time limits
|
||||
are wall-clock minutes, independently of JobFuture.wait()'s timeout in seconds.
|
||||
Output paths refer to the compute node and may use Slurm's %j job-ID pattern.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
name: NonEmptyString | None = Field(
|
||||
default=None, description="Job name shown in Slurm. Omit to use the scheduler default."
|
||||
)
|
||||
partition: Literal["hour", "day", "week", "gpu"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"RA partition: hour (default/max 1 hour), day (default 4 hours, max 24 hours), "
|
||||
"week (default 2 days, max 8 days), or gpu (default 1 hour, max 7 days). "
|
||||
"Omit to use the cluster default, currently hour."
|
||||
),
|
||||
)
|
||||
account: NonEmptyString | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Slurm accounting/project account; omit to use your default account association. "
|
||||
"The account must also be allowed by the selected partition: hour/day allow "
|
||||
"user and proprietary; gpu allows user and gpu-csaxs; week denies gpu-csaxs "
|
||||
"and tomcat. These partition rules do not establish your account membership."
|
||||
),
|
||||
)
|
||||
qos: NonEmptyString | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Quality of Service controlling scheduling priority and resource limits. "
|
||||
"Must be allowed by both your association and the selected partition. "
|
||||
"Omit to use your association's default job QoS. The gpu partition allows "
|
||||
"gpu-default, gpu-day, gpu-hour, and gpu-csaxs, and additionally applies "
|
||||
"partition QoS gpu-default. Partition QoS is separate from requested job QoS."
|
||||
),
|
||||
)
|
||||
tasks: int = Field(
|
||||
default=1,
|
||||
ge=1,
|
||||
strict=True,
|
||||
description=(
|
||||
"Number of Slurm tasks requested for the allocation; defaults to one. "
|
||||
"The worker executes the workload script once, without launching MPI ranks. "
|
||||
"This does not specify CPUs per task."
|
||||
),
|
||||
)
|
||||
time_limit_minutes: int | None = Field(
|
||||
default=None,
|
||||
ge=1,
|
||||
strict=True,
|
||||
description=(
|
||||
"Maximum job wall-clock runtime in minutes. Slurm terminates the job at this limit. "
|
||||
"Omit to use the selected partition's default: hour 60, day 240, week 2880, "
|
||||
"gpu 60 minutes. Partition maxima are respectively 60, 1440, 11520, and "
|
||||
"10080 minutes; account/QoS limits may impose additional restrictions. "
|
||||
"Separate from JobFuture.wait(timeout), which is measured in seconds."
|
||||
),
|
||||
)
|
||||
standard_output: NonEmptyString | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Compute-node file path for stdout, in addition to Redis forwarding. "
|
||||
"Supports Slurm patterns such as %j for the job ID. Relative paths use the job's "
|
||||
"working directory; the parent directory must exist. Omit for Slurm's default output."
|
||||
),
|
||||
)
|
||||
standard_error: NonEmptyString | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Compute-node file path for stderr, in addition to Redis forwarding. "
|
||||
"Supports %j for the job ID. Relative paths use the job's working directory; "
|
||||
"the parent directory must exist. Omit to use Slurm's default stderr handling."
|
||||
),
|
||||
)
|
||||
|
||||
################################################
|
||||
# Public Serialization #
|
||||
################################################
|
||||
|
||||
def to_rest(self) -> dict[str, Any]:
|
||||
"""Serialize validated options into slurmrestd job-description fields.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: REST options with unset values omitted and runtime expressed
|
||||
as an optional integer.
|
||||
"""
|
||||
options = self.model_dump(exclude={"time_limit_minutes"}, exclude_none=True)
|
||||
if self.time_limit_minutes is not None:
|
||||
options["time_limit"] = {"set": True, "number": self.time_limit_minutes}
|
||||
return options
|
||||
Whitespace-only changes.
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
module load redis
|
||||
|
||||
fifo=$1
|
||||
hash_name=$2
|
||||
|
||||
exec {fifo_fd}<"$fifo"
|
||||
|
||||
redis_host="bec-slurm.psi.ch"
|
||||
status="finished"
|
||||
|
||||
publish_payload() {
|
||||
local topic=$1
|
||||
local payload=$2
|
||||
redis-cli -h "$redis_host" PUBLISH "$topic" "$payload"
|
||||
}
|
||||
|
||||
timestamp=$(date +%s)
|
||||
publish_payload \
|
||||
"info/$hash_name/status" \
|
||||
"$(jq -cn --arg status "running" '{msg_type:"status", status:$status}')"
|
||||
|
||||
heartbeat() {
|
||||
while :; do
|
||||
local timestamp
|
||||
timestamp=$(date +%s)
|
||||
publish_payload \
|
||||
"info/$hash_name/heartbeat" \
|
||||
"$(jq -cn --argjson timestamp "$timestamp" '{msg_type:"heartbeat", timestamp:$timestamp}')" \
|
||||
|| true
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
heartbeat &
|
||||
hb_pid=$!
|
||||
|
||||
cleanup() {
|
||||
if kill -0 "$hb_pid" 2>/dev/null; then
|
||||
kill "$hb_pid" 2>/dev/null || true
|
||||
wait "$hb_pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
while IFS= read -r line <&"$fifo_fd"; do
|
||||
case "$line" in
|
||||
stop)
|
||||
status="finished"
|
||||
break
|
||||
;;
|
||||
error:*)
|
||||
status="error:${line#error:}"
|
||||
break
|
||||
;;
|
||||
log:*)
|
||||
msg="${line#log:}"
|
||||
publish_payload \
|
||||
"info/$hash_name/log" \
|
||||
"$(jq -cn --arg log "$msg" '{msg_type:"log", log:$log}')"
|
||||
;;
|
||||
*)
|
||||
:
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
publish_payload \
|
||||
"info/$hash_name/status" \
|
||||
"$(jq -cn --arg status "$status" '{msg_type:"status", status:$status}')"
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -o pipefail
|
||||
|
||||
env_path=$1
|
||||
script_path=$2
|
||||
hash_name=$3
|
||||
have_out_stream=$4
|
||||
have_out_file=$5
|
||||
redis_script=$6
|
||||
|
||||
|
||||
fifo=$(mktemp -u)
|
||||
mkfifo "$fifo"
|
||||
|
||||
"$redis_script" "$fifo" "$hash_name" >/dev/null &
|
||||
worker_pid=$!
|
||||
|
||||
trap 'rm -f "$fifo"' EXIT
|
||||
|
||||
exec {fifo_fd}>"$fifo"
|
||||
|
||||
source "$env_path"
|
||||
|
||||
{
|
||||
stdbuf -o0 -e0 "$script_path" 2>&1
|
||||
} | while IFS= read -r line; do
|
||||
|
||||
if [ "$have_out_stream" -eq 1 ]; then
|
||||
printf 'log:%s\n' "$line" >&"$fifo_fd"
|
||||
fi
|
||||
|
||||
if [ "$have_out_file" -eq 1 ]; then
|
||||
printf '%s\n' "$line"
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
|
||||
exit_code=${PIPESTATUS[0]}
|
||||
|
||||
if [ "$exit_code" -ne 0 ]; then
|
||||
printf 'error:%s\n' "$exit_code" >&"$fifo_fd"
|
||||
fi
|
||||
|
||||
printf 'stop\n' >&"$fifo_fd"
|
||||
exec {fifo_fd}>&-
|
||||
|
||||
wait $worker_pid
|
||||
|
||||
exit $exit_code
|
||||
@@ -0,0 +1,612 @@
|
||||
"""Own job futures, Redis subscriptions, and callback dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import partial, wraps
|
||||
from logging import getLogger
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage
|
||||
from slurm_manager.job_future import JobFuture, JobStatus
|
||||
from slurm_manager.redis_sub_client import RedisSubClient
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TopicInfo:
|
||||
"""Helper for topic specification within Redis bec-slurm pub/sub interface."""
|
||||
|
||||
job_id: str
|
||||
prefix: str = "info"
|
||||
|
||||
################################################
|
||||
# Public Properties #
|
||||
################################################
|
||||
|
||||
@property
|
||||
def log(self) -> str:
|
||||
"""Return the channel carrying stdout and stderr log messages.
|
||||
|
||||
Returns:
|
||||
str: Job log channel.
|
||||
"""
|
||||
return self.topic("log")
|
||||
|
||||
@property
|
||||
def event(self) -> str:
|
||||
"""Return the channel carrying lifecycle status messages.
|
||||
|
||||
Returns:
|
||||
str: Job status channel.
|
||||
"""
|
||||
return self.topic("status")
|
||||
|
||||
@property
|
||||
def heartbeat(self) -> str:
|
||||
"""Return the channel carrying worker heartbeats.
|
||||
|
||||
Returns:
|
||||
str: Job heartbeat channel.
|
||||
"""
|
||||
return self.topic("heartbeat")
|
||||
|
||||
################################################
|
||||
# Public Topic Lookup #
|
||||
################################################
|
||||
|
||||
def topic(self, v: str) -> str:
|
||||
"""Build a Redis topic name for a job event.
|
||||
|
||||
Args:
|
||||
v (str): Event suffix appended to this job's Redis topic prefix.
|
||||
|
||||
Returns:
|
||||
str: Redis channel in prefix/job_id/event form.
|
||||
"""
|
||||
return f"{self.prefix}/{self.job_id}/{v}"
|
||||
|
||||
|
||||
class SubscriptionInfo(BaseModel):
|
||||
"""Helper for slurm job information and future tracking."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
topic: str
|
||||
job_id: str
|
||||
slurm_job_id: str | None = None
|
||||
callback: Callable[[SlurmMessage], None] | None = None
|
||||
callback_id: str | None = None
|
||||
last_heartbeat: float | None = None
|
||||
heartbeat_received: bool = False
|
||||
last_scheduler_check: float | None = None
|
||||
|
||||
|
||||
HEARTBEAT_TIMEOUT = 10 # seconds
|
||||
|
||||
|
||||
class JobTracker:
|
||||
"""Coordinate job state and callback ownership under one lock."""
|
||||
|
||||
def __init__(self, sub_client: RedisSubClient) -> None:
|
||||
"""Create tracking state and the user callback executor.
|
||||
|
||||
Args:
|
||||
sub_client (RedisSubClient): Transport used for job subscriptions; the
|
||||
tracker closes it during shutdown.
|
||||
"""
|
||||
self._sub_client = sub_client
|
||||
self._future_registry: dict[str, JobFuture] = {}
|
||||
self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict)
|
||||
self._topic_by_subscription_id: dict[str, str] = {}
|
||||
self._active_subscriptions: dict[str, set[str]] = defaultdict(set)
|
||||
self._lock = threading.RLock()
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=8, thread_name_prefix="SlurmManagerCallback"
|
||||
)
|
||||
|
||||
################################################
|
||||
# Public Job Tracking #
|
||||
################################################
|
||||
|
||||
def track(self, future: JobFuture) -> None:
|
||||
"""Register a submitted job and attach internal monitoring callbacks.
|
||||
|
||||
Args:
|
||||
future (JobFuture): Submitted job whose state and subscriptions are owned.
|
||||
|
||||
Notes:
|
||||
The caller must remove tracking and cancel the held job if setup fails.
|
||||
"""
|
||||
with self._lock:
|
||||
self._future_registry[future._job_id] = future
|
||||
self._subscribe_to_internal_topics(future, TopicInfo(future._job_id))
|
||||
|
||||
def get_future(self, job_id: str) -> JobFuture | None:
|
||||
"""Look up a tracked future under the state lock.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager job identifier.
|
||||
|
||||
Returns:
|
||||
JobFuture | None: Tracked future, or None after cleanup.
|
||||
"""
|
||||
with self._lock:
|
||||
return self._future_registry.get(job_id)
|
||||
|
||||
def wait_until_ready(self, job_id: str) -> None:
|
||||
"""Wait for Redis to acknowledge all subscriptions installed for a job.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager job identifier.
|
||||
|
||||
Raises:
|
||||
TimeoutError: Redis does not acknowledge the subscriptions in time.
|
||||
"""
|
||||
with self._lock:
|
||||
topics = list(self._active_subscriptions.get(job_id, set()))
|
||||
self._sub_client.wait_until_subscribed(topics)
|
||||
|
||||
def begin_scheduler_check(self, job_id: str) -> JobFuture | None:
|
||||
"""Record a scheduler check for an active job before performing network I/O.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager job identifier.
|
||||
|
||||
Returns:
|
||||
JobFuture | None: Active future, or None if it has completed or was removed.
|
||||
"""
|
||||
with self._lock:
|
||||
future = self._future_registry.get(job_id)
|
||||
if future is None or future.done():
|
||||
return None
|
||||
topic = TopicInfo(job_id).heartbeat
|
||||
for info in self._subscriptions_by_topic.get(topic, {}).values():
|
||||
if info.last_heartbeat is not None:
|
||||
info.last_scheduler_check = time.time()
|
||||
return future
|
||||
|
||||
def apply_status(self, message: StatusMessage, future: JobFuture) -> None:
|
||||
"""Apply a worker status and clean up terminal job subscriptions.
|
||||
|
||||
Args:
|
||||
message (StatusMessage): Raw Redis payload or parsed worker message, as
|
||||
specified by the annotation.
|
||||
future (JobFuture): Tracked future whose lifecycle or subscriptions are
|
||||
being managed.
|
||||
|
||||
Notes:
|
||||
Statuses use status[:detail]. Unknown statuses fail the job and follow the
|
||||
same cleanup path as an explicit error.
|
||||
"""
|
||||
info = message.status.split(":", 1)
|
||||
status = info[0]
|
||||
try: # Catch irregular formatted status messages to avoid crashing the callback loop.
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
logger.warning("Received unknown job status '%s' for job %s", status, future.name)
|
||||
future._update_status(
|
||||
"error",
|
||||
f"Received unknown job status '{status}' from message: {message.model_dump()}",
|
||||
)
|
||||
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
|
||||
return
|
||||
exc_info = info[1] if len(info) > 1 else None
|
||||
future._update_status(status, exc_info)
|
||||
# Clean up subscriptions for finished job
|
||||
if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]:
|
||||
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
|
||||
|
||||
def get_missing_heartbeats(self) -> list[SubscriptionInfo]:
|
||||
"""Select stale heartbeat subscriptions whose scheduler check is due.
|
||||
|
||||
Returns:
|
||||
list[SubscriptionInfo]: Copies of internal heartbeat subscriptions needing
|
||||
a Slurm query. Mutating a copy does not change tracking state.
|
||||
"""
|
||||
missing = []
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
for job_id, topics in self._active_subscriptions.items():
|
||||
topic = TopicInfo(job_id=job_id).heartbeat
|
||||
if topic not in topics:
|
||||
continue
|
||||
for sub_info in self._subscriptions_by_topic.get(topic, {}).values():
|
||||
if not sub_info.last_heartbeat:
|
||||
continue
|
||||
if now - sub_info.last_heartbeat <= HEARTBEAT_TIMEOUT:
|
||||
continue
|
||||
last_check = sub_info.last_scheduler_check
|
||||
if last_check is not None and now - last_check <= HEARTBEAT_TIMEOUT:
|
||||
continue
|
||||
missing.append(sub_info.model_copy())
|
||||
return missing
|
||||
|
||||
################################################
|
||||
# Public Event Subscriptions #
|
||||
################################################
|
||||
|
||||
def listen(
|
||||
self,
|
||||
event_type: Literal["heartbeat", "log", "status"],
|
||||
callback: Callable[[SlurmMessage], None],
|
||||
job_id: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Attach a callback to one tracked job or all currently tracked jobs.
|
||||
|
||||
Args:
|
||||
event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log,
|
||||
status, or heartbeat.
|
||||
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
|
||||
Slurm message.
|
||||
job_id (str | None): Opaque manager identifier used in Redis topic names.
|
||||
|
||||
Returns:
|
||||
list[str]: Subscription IDs accepted by unlisten().
|
||||
|
||||
Raises:
|
||||
KeyError: The explicitly requested job ID is not tracked.
|
||||
RuntimeError: A selected job has already completed.
|
||||
|
||||
Notes:
|
||||
With job_id=None, registration applies to the current jobs, not future
|
||||
submissions. Callbacks run in the tracker executor.
|
||||
"""
|
||||
with self._lock:
|
||||
job_futures = (
|
||||
[self._future_registry[job_id]] if job_id else list(self._future_registry.values())
|
||||
)
|
||||
callback_ids = []
|
||||
for future in job_futures:
|
||||
callback_ids.append(future.listen(event_type=event_type, callback=callback))
|
||||
|
||||
return callback_ids
|
||||
|
||||
def add_subscription(
|
||||
self,
|
||||
job_id: str,
|
||||
key: Literal["heartbeat", "log", "status"],
|
||||
callback: Callable[[SlurmMessage], None],
|
||||
callback_kwargs: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""Atomically register a Redis callback and its manager ownership.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or
|
||||
heartbeat.
|
||||
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
|
||||
Slurm message.
|
||||
callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed
|
||||
to the user callback.
|
||||
|
||||
Returns:
|
||||
str: Unique callback subscription ID.
|
||||
|
||||
Raises:
|
||||
RuntimeError: The job is absent or finishes during registration.
|
||||
"""
|
||||
|
||||
topic_str = TopicInfo(job_id=job_id).topic(key)
|
||||
|
||||
@wraps(callback)
|
||||
def wrapped_callback(
|
||||
message: SlurmMessage,
|
||||
executor: ThreadPoolExecutor,
|
||||
job_id: str,
|
||||
slurm_job_id: str,
|
||||
callback_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Enrich a parsed worker message and dispatch its registered handler.
|
||||
|
||||
Args:
|
||||
message (SlurmMessage): Raw Redis payload or parsed worker message, as
|
||||
specified by the annotation.
|
||||
executor (ThreadPoolExecutor): Callback executor that runs user code
|
||||
away from the listener thread.
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
|
||||
callback_kwargs (dict[str, Any] | None): Additional keyword arguments
|
||||
passed to the user callback.
|
||||
"""
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": job_id,
|
||||
"slurm_job_id": slurm_job_id,
|
||||
}
|
||||
)
|
||||
executor.submit(callback, message, **(callback_kwargs or {}))
|
||||
|
||||
# Completion removes subscriptions under this same lock. Keep ownership
|
||||
# registration atomic with cleanup, and undo Redis registration if the
|
||||
# future became terminal while subscribe() was in progress.
|
||||
with self._lock:
|
||||
future = self._future_registry.get(job_id)
|
||||
if future is None or future.done():
|
||||
raise RuntimeError(f"Job {job_id} is no longer active")
|
||||
cb_id = self._sub_client.subscribe(
|
||||
topic_str,
|
||||
partial(
|
||||
wrapped_callback,
|
||||
executor=self._executor,
|
||||
callback_kwargs=callback_kwargs,
|
||||
job_id=job_id,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
),
|
||||
)
|
||||
|
||||
if self._future_registry.get(job_id) is not future or future.done():
|
||||
self._sub_client.unsubscribe(cb_id)
|
||||
raise RuntimeError(f"Job {job_id} finished during subscription registration")
|
||||
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=job_id,
|
||||
topic=topic_str,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=wrapped_callback,
|
||||
last_heartbeat=None,
|
||||
callback_id=cb_id,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
return cb_id
|
||||
|
||||
def remove_subscription_by_id(self, callback_id: str | list[str]) -> None:
|
||||
"""Remove the selected callbacks while preserving other topic listeners.
|
||||
|
||||
Args:
|
||||
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
|
||||
returned during registration.
|
||||
"""
|
||||
if isinstance(callback_id, str):
|
||||
callback_id = [callback_id]
|
||||
with self._lock:
|
||||
for cb_id in callback_id:
|
||||
topic = self._topic_by_subscription_id.pop(cb_id, None)
|
||||
if topic is None:
|
||||
continue
|
||||
|
||||
callbacks = self._subscriptions_by_topic.get(topic, {})
|
||||
sub_info = callbacks.pop(cb_id, None)
|
||||
self._sub_client.unsubscribe(cb_id)
|
||||
|
||||
# Other callbacks, including heartbeat monitoring, still own this topic.
|
||||
if callbacks:
|
||||
continue
|
||||
|
||||
self._subscriptions_by_topic.pop(topic, None)
|
||||
if sub_info is None:
|
||||
continue
|
||||
|
||||
topics = self._active_subscriptions.get(sub_info.job_id)
|
||||
if topics is not None:
|
||||
topics.discard(topic)
|
||||
|
||||
def remove_subscription_by_topic(self, topic: str) -> None:
|
||||
"""Remove all callbacks for one topic and unsubscribe it from Redis.
|
||||
|
||||
Args:
|
||||
topic (str): Redis channel whose callback registrations are selected.
|
||||
"""
|
||||
with self._lock:
|
||||
callbacks = self._subscriptions_by_topic.pop(topic, {})
|
||||
for callback_id, sub_info in callbacks.items():
|
||||
self._topic_by_subscription_id.pop(callback_id, None)
|
||||
job_id = sub_info.job_id
|
||||
self._active_subscriptions[job_id].discard(topic)
|
||||
|
||||
# Remove the topic subscription from Redis
|
||||
self._sub_client.unsubscribe_topic(topic)
|
||||
|
||||
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
|
||||
"""Remove job subscriptions and discard the tracked future.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
skip_event_callback (bool): If true, remove monitoring without synthesizing
|
||||
a failure update.
|
||||
|
||||
Notes:
|
||||
Unless skip_event_callback is true, an unfinished future is marked failed.
|
||||
This only removes local monitoring; it does not cancel the Slurm allocation.
|
||||
"""
|
||||
with self._lock:
|
||||
topics = self._active_subscriptions.pop(job_id, set())
|
||||
for topic in topics:
|
||||
callbacks = self._subscriptions_by_topic.pop(topic, {})
|
||||
for callback_id in callbacks:
|
||||
self._topic_by_subscription_id.pop(callback_id, None)
|
||||
self._sub_client.unsubscribe_topic(topic)
|
||||
future = self._future_registry.get(job_id)
|
||||
if future is None:
|
||||
return
|
||||
# This callback will trigger a cleanup.
|
||||
if not skip_event_callback:
|
||||
self.apply_status(
|
||||
StatusMessage(
|
||||
status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}",
|
||||
metadata={
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": future._job_id,
|
||||
"slurm_job_id": future.slurm_job_id,
|
||||
},
|
||||
),
|
||||
future,
|
||||
)
|
||||
self._remove_job_future(future._job_id)
|
||||
|
||||
################################################
|
||||
# Public Lifecycle #
|
||||
################################################
|
||||
|
||||
def shutdown(self, timeout: float | None = 1.0) -> None:
|
||||
"""Stop Redis delivery and drain accepted user callbacks.
|
||||
|
||||
Args:
|
||||
timeout (float | None): Redis listener join timeout in seconds; None waits
|
||||
indefinitely. Callback draining has no timeout.
|
||||
|
||||
Notes:
|
||||
Stop scheduler monitoring before calling this method. Remote jobs are
|
||||
not cancelled and tracked futures are not marked complete.
|
||||
"""
|
||||
with self._lock:
|
||||
self._subscriptions_by_topic.clear()
|
||||
self._topic_by_subscription_id.clear()
|
||||
self._sub_client.shutdown(timeout=timeout)
|
||||
self._executor.shutdown(wait=True, cancel_futures=False)
|
||||
|
||||
################################################
|
||||
# Internal Subscriptions #
|
||||
################################################
|
||||
|
||||
def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None:
|
||||
"""Attach the manager status and heartbeat callbacks before release.
|
||||
|
||||
Args:
|
||||
future (JobFuture): Tracked future whose lifecycle or subscriptions are
|
||||
being managed.
|
||||
topic_info (TopicInfo): Redis topic names for the tracked job.
|
||||
"""
|
||||
self._subscribe_to_heartbeat(topic_info, future)
|
||||
self._subscribe_to_event(topic_info, future)
|
||||
|
||||
def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None:
|
||||
"""Register the internal heartbeat callback and its initial timestamp.
|
||||
|
||||
Args:
|
||||
topic_info (TopicInfo): Redis topic names for the tracked job.
|
||||
future (JobFuture): Tracked future whose lifecycle or subscriptions are
|
||||
being managed.
|
||||
"""
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=topic_info.job_id,
|
||||
topic=topic_info.heartbeat,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=None,
|
||||
last_heartbeat=None,
|
||||
callback_id=None,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
topic = topic_info.heartbeat
|
||||
|
||||
def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
|
||||
"""Enrich a parsed worker message and dispatch its registered handler.
|
||||
|
||||
Args:
|
||||
message (HeartBeatMessage): Raw Redis payload or parsed worker message,
|
||||
as specified by the annotation.
|
||||
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
|
||||
bookkeeping.
|
||||
"""
|
||||
try:
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": sub_info.job_id,
|
||||
"slurm_job_id": sub_info.slurm_job_id,
|
||||
}
|
||||
)
|
||||
self._heartbeat_callback(message, sub_info)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Error in callback for topic %s: %s", message.msg_type, message.timestamp
|
||||
)
|
||||
|
||||
cb_id = self._sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info))
|
||||
sub_info.callback_id = cb_id
|
||||
sub_info.last_heartbeat = time.time()
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
|
||||
def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None:
|
||||
"""Register the internal callback for terminal and running statuses.
|
||||
|
||||
Args:
|
||||
topic_info (TopicInfo): Redis topic names for the tracked job.
|
||||
future (JobFuture): Tracked future whose lifecycle or subscriptions are
|
||||
being managed.
|
||||
"""
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=topic_info.job_id,
|
||||
topic=topic_info.event,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=None,
|
||||
last_heartbeat=None,
|
||||
callback_id=None,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
event_topic = topic_info.event
|
||||
|
||||
def wrapped_callback(message: StatusMessage, future: JobFuture) -> None:
|
||||
"""Enrich a parsed worker message and dispatch its registered handler.
|
||||
|
||||
Args:
|
||||
message (StatusMessage): Raw Redis payload or parsed worker message, as
|
||||
specified by the annotation.
|
||||
future (JobFuture): Tracked future whose lifecycle or subscriptions are
|
||||
being managed.
|
||||
"""
|
||||
try:
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": future._job_id,
|
||||
"slurm_job_id": future.slurm_job_id,
|
||||
}
|
||||
)
|
||||
self.apply_status(message, future=future)
|
||||
except Exception:
|
||||
logger.error("Error in callback for topic %s: %s", message.msg_type, message.status)
|
||||
|
||||
cb_id = self._sub_client.subscribe(event_topic, partial(wrapped_callback, future=future))
|
||||
sub_info.callback_id = cb_id
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
|
||||
def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None:
|
||||
"""Index a subscription by callback ID, topic, and job.
|
||||
|
||||
Args:
|
||||
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
|
||||
bookkeeping.
|
||||
|
||||
Raises:
|
||||
ValueError: The subscription does not contain a callback ID.
|
||||
"""
|
||||
with self._lock:
|
||||
if sub_info.callback_id is None:
|
||||
raise ValueError("SubscriptionInfo.callback_id must be set before registration")
|
||||
self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info
|
||||
self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic
|
||||
self._active_subscriptions[sub_info.job_id].add(sub_info.topic)
|
||||
|
||||
def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
|
||||
"""Record heartbeat receipt for the monitored subscription.
|
||||
|
||||
Args:
|
||||
message (HeartBeatMessage): Raw Redis payload or parsed worker message, as
|
||||
specified by the annotation.
|
||||
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
|
||||
bookkeeping.
|
||||
"""
|
||||
with self._lock:
|
||||
sub_info.heartbeat_received = True
|
||||
sub_info.last_heartbeat = time.time()
|
||||
|
||||
def _remove_job_future(self, job_id: str) -> None:
|
||||
"""Remove a future from the manager registry under its lock.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
"""
|
||||
with self._lock:
|
||||
self._future_registry.pop(job_id, None)
|
||||
@@ -52,6 +52,18 @@ class RedisSubClient:
|
||||
redis_cls: type[Redis] = Redis,
|
||||
**redis_kwargs: Any,
|
||||
) -> None:
|
||||
"""Create Redis resources without starting the listener until subscribed.
|
||||
|
||||
Args:
|
||||
bootstrap (str | list[str]): Redis host:port or a nonempty address list;
|
||||
only the first entry is used.
|
||||
redis_cls (type[Redis]): Redis client implementation, injectable for tests.
|
||||
**redis_kwargs (Any): Additional Redis constructor options such as socket
|
||||
timeouts.
|
||||
|
||||
Raises:
|
||||
ValueError: The bootstrap list is empty or its address cannot be parsed.
|
||||
"""
|
||||
if isinstance(bootstrap, list):
|
||||
if not bootstrap:
|
||||
raise ValueError("bootstrap list must not be empty")
|
||||
@@ -70,14 +82,34 @@ class RedisSubClient:
|
||||
self._stop_listener = threading.Event()
|
||||
self._listener_thread: threading.Thread | None = None
|
||||
|
||||
################################################
|
||||
# Public Subscriptions #
|
||||
################################################
|
||||
|
||||
@property
|
||||
def registered_topics(self) -> set[str]:
|
||||
"""Returns all currently subscribed topics."""
|
||||
"""Return a snapshot of topics with active callbacks.
|
||||
|
||||
Returns:
|
||||
set[str]: Currently registered Redis channel names.
|
||||
"""
|
||||
with self._lock:
|
||||
return set(self._subscriptions_by_topic.keys())
|
||||
|
||||
def subscribe(self, topic: str, callback: MessageCallback) -> str:
|
||||
"""Subscribes callback to a topic and returns a subscription id."""
|
||||
"""Register a callback and start the listener if needed.
|
||||
|
||||
Args:
|
||||
topic (str): Redis channel whose callback registrations are selected.
|
||||
callback (MessageCallback): Callable receiving the parsed Slurm message.
|
||||
|
||||
Returns:
|
||||
str: Unique subscription ID.
|
||||
|
||||
Notes:
|
||||
The callback runs on the Redis listener thread and must remain fast and
|
||||
nonblocking.
|
||||
"""
|
||||
sub_id = str(uuid4())
|
||||
subscription = Subscription(id=sub_id, topic=topic, callback=callback)
|
||||
|
||||
@@ -95,10 +127,13 @@ class RedisSubClient:
|
||||
return sub_id
|
||||
|
||||
def unsubscribe(self, subscription_id: str) -> bool:
|
||||
"""
|
||||
Unregisters one callback by subscription id.
|
||||
"""Remove one callback and unsubscribe its topic if it has no owners.
|
||||
|
||||
Returns True if a callback was removed, False otherwise.
|
||||
Args:
|
||||
subscription_id (str): Unique callback ID returned by subscribe().
|
||||
|
||||
Returns:
|
||||
bool: True if the callback was registered; False if it was unknown.
|
||||
"""
|
||||
with self._lock:
|
||||
topic = self._topic_by_subscription_id.pop(subscription_id, None)
|
||||
@@ -120,7 +155,14 @@ class RedisSubClient:
|
||||
return True
|
||||
|
||||
def unsubscribe_topic(self, topic: str) -> int:
|
||||
"""Unregisters all callbacks for a topic and returns removed count."""
|
||||
"""Remove every callback registered for one topic.
|
||||
|
||||
Args:
|
||||
topic (str): Redis channel whose callback registrations are selected.
|
||||
|
||||
Returns:
|
||||
int: Number of removed callbacks.
|
||||
"""
|
||||
with self._lock:
|
||||
callbacks = self._subscriptions_by_topic.pop(topic, None)
|
||||
if not callbacks: # No callbacks for this topic, nothing happens
|
||||
@@ -132,7 +174,19 @@ class RedisSubClient:
|
||||
return len(callbacks)
|
||||
|
||||
def wait_until_subscribed(self, topics: list[str], timeout: float = 5.0) -> None:
|
||||
"""Wait for Redis to acknowledge each subscription before releasing a job."""
|
||||
"""Wait for Redis subscription acknowledgements before releasing a job.
|
||||
|
||||
Args:
|
||||
topics (list[str]): Already registered channels whose acknowledgements must
|
||||
be received.
|
||||
timeout (float): Shared acknowledgement deadline in seconds across all
|
||||
requested topics.
|
||||
|
||||
Raises:
|
||||
KeyError: A requested topic is not registered.
|
||||
TimeoutError: The shared deadline expires before all topics are
|
||||
acknowledged.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
with self._lock:
|
||||
ready = [(topic, self._subscription_ready[topic]) for topic in topics]
|
||||
@@ -140,8 +194,21 @@ class RedisSubClient:
|
||||
if not event.wait(max(0, deadline - time.monotonic())):
|
||||
raise TimeoutError(f"Redis did not acknowledge subscription to {topic}")
|
||||
|
||||
################################################
|
||||
# Public Lifecycle #
|
||||
################################################
|
||||
|
||||
def shutdown(self, timeout: float | None = 1.0) -> None:
|
||||
"""Stops worker threads and closes Redis resources."""
|
||||
"""Stop the listener and close Redis resources.
|
||||
|
||||
Args:
|
||||
timeout (float | None): Listener join timeout in seconds; None waits
|
||||
indefinitely.
|
||||
|
||||
Notes:
|
||||
A listener that misses the join timeout is logged before resources are
|
||||
closed.
|
||||
"""
|
||||
self._stop_listener.set()
|
||||
|
||||
if self._listener_thread is not None:
|
||||
@@ -157,8 +224,12 @@ class RedisSubClient:
|
||||
self._pubsub.close()
|
||||
self._redis.close()
|
||||
|
||||
################################################
|
||||
# Internal Listener #
|
||||
################################################
|
||||
|
||||
def _ensure_threads_started(self) -> None:
|
||||
"""Makes sure the listener thread is running to receive messages from Redis."""
|
||||
"""Start a listener if no listener thread is currently alive."""
|
||||
if self._listener_thread is None or not self._listener_thread.is_alive():
|
||||
self._stop_listener.clear()
|
||||
self._listener_thread = threading.Thread(
|
||||
@@ -167,7 +238,12 @@ class RedisSubClient:
|
||||
self._listener_thread.start()
|
||||
|
||||
def _listener_loop(self) -> None:
|
||||
"""Listener thread loop that polls for messages and dispatches them to callbacks."""
|
||||
"""Poll Redis, acknowledge subscriptions, and dispatch parsed messages.
|
||||
|
||||
Notes:
|
||||
Redis errors are logged and polling resumes after a short delay. Callback
|
||||
exceptions are logged without stopping other callbacks.
|
||||
"""
|
||||
while not self._stop_listener.is_set():
|
||||
try:
|
||||
with self._lock:
|
||||
@@ -212,9 +288,26 @@ class RedisSubClient:
|
||||
format_exc,
|
||||
)
|
||||
|
||||
################################################
|
||||
# Internal Message Parsing #
|
||||
################################################
|
||||
|
||||
@staticmethod
|
||||
def _normalize_message(message: dict[str, Any]) -> tuple[str, SlurmMessage]:
|
||||
"""Converts raw Redis message dict to normalized Message object."""
|
||||
"""Decode a Redis pub/sub payload into the corresponding message model.
|
||||
|
||||
Args:
|
||||
message (dict[str, Any]): Raw Redis payload or parsed worker message, as
|
||||
specified by the annotation.
|
||||
|
||||
Returns:
|
||||
tuple[str, SlurmMessage]: Pair containing the channel name and validated
|
||||
Slurm message.
|
||||
|
||||
Raises:
|
||||
TypeError: The payload is neither JSON text nor a dictionary.
|
||||
ValueError: The JSON or message schema is invalid.
|
||||
"""
|
||||
channel = message.get("channel")
|
||||
topic = channel.decode() if isinstance(channel, bytes) else str(channel)
|
||||
|
||||
|
||||
+342
-488
@@ -1,277 +1,195 @@
|
||||
"""Minimal Slurm manager draft for sbatch + JobFuture creation."""
|
||||
"""Submit and monitor remote Slurm jobs through slurmrestd and Redis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import shlex
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from functools import partial, wraps
|
||||
from logging import getLogger
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage
|
||||
from slurm_manager.event_messages import SlurmMessage, StatusMessage
|
||||
from slurm_manager.job_future import JobFuture, JobStatus
|
||||
from slurm_manager.job_options import JobOptions
|
||||
from slurm_manager.job_tracker import JobTracker, SubscriptionInfo
|
||||
from slurm_manager.redis_sub_client import RedisSubClient
|
||||
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
_STOP = object()
|
||||
|
||||
def _job_exit_code(exit_code: Any) -> int | str | None:
|
||||
"""Decode a v0.0.44 process exit status without treating unset codes as success.
|
||||
|
||||
def _cluster_args(cluster: str | None) -> list[str]:
|
||||
"""Route Slurm CLI operations to the cluster that accepted the job."""
|
||||
return ["--clusters", cluster] if cluster else []
|
||||
Args:
|
||||
exit_code (Any): Integer or v0.0.44 structured process exit status.
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TopicInfo:
|
||||
"""Helper for topic specification within Redis bec-slurm pub/sub interface."""
|
||||
|
||||
job_id: str
|
||||
prefix: str = "info"
|
||||
|
||||
def topic(self, v: str) -> str:
|
||||
"""Constructs a new topic string for a given key."""
|
||||
return f"{self.prefix}/{self.job_id}/{v}"
|
||||
|
||||
@property
|
||||
def log(self) -> str:
|
||||
"""Topic for log messages."""
|
||||
return self.topic("log")
|
||||
|
||||
@property
|
||||
def event(self) -> str:
|
||||
"""Topic for event messages."""
|
||||
return self.topic("status")
|
||||
|
||||
@property
|
||||
def heartbeat(self) -> str:
|
||||
"""Topic for heartbeat messages."""
|
||||
return self.topic("heartbeat")
|
||||
|
||||
|
||||
class SubscriptionInfo(BaseModel):
|
||||
"""Helper for slurm job information and future tracking."""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
topic: str
|
||||
job_id: str
|
||||
slurm_job_id: str | None = None
|
||||
callback: Callable[[SlurmMessage], None] | None = None
|
||||
callback_id: str | None = None
|
||||
last_heartbeat: float | None = None
|
||||
heartbeat_received: bool = False
|
||||
last_scheduler_check: float | None = None
|
||||
|
||||
|
||||
HEARTBEAT_TIMEOUT = 10 # seconds
|
||||
Returns:
|
||||
int | str | None: Exit code, a signal description, or None when no code is
|
||||
available.
|
||||
"""
|
||||
if not isinstance(exit_code, dict):
|
||||
return exit_code
|
||||
code = exit_code.get("return_code")
|
||||
if isinstance(code, dict):
|
||||
code = code.get("number") if code.get("set") else None
|
||||
signal = exit_code.get("signal", {}).get("id", {})
|
||||
if isinstance(signal, dict):
|
||||
signal = signal.get("number", 0) if signal.get("set") else 0
|
||||
return code if not signal else f"signal {signal}"
|
||||
|
||||
|
||||
class SlurmManager:
|
||||
"""Small manager that submits wrapper jobs and returns JobFuture handles."""
|
||||
"""Submit Slurm jobs and coordinate tracking and scheduler monitoring."""
|
||||
|
||||
def __init__(self, redis_host: str = "localhost", redis_port: int = 6379) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
redis_host: str = "localhost",
|
||||
redis_port: int = 6379,
|
||||
*,
|
||||
env_file: str | Path = ".env",
|
||||
rest_client: SlurmRestClient | None = None,
|
||||
worker_path: str = "/usr/local/bin/slurm-worker",
|
||||
) -> None:
|
||||
"""Create REST/Redis clients and start background job monitoring.
|
||||
|
||||
Args:
|
||||
redis_host (str): Redis hostname reachable by the submitter and compute
|
||||
nodes.
|
||||
redis_port (int): Redis TCP port shared by the submitter and worker.
|
||||
env_file (str | Path): Local credential dotenv path. Environment variables
|
||||
take precedence.
|
||||
rest_client (SlurmRestClient | None): Optional preconfigured REST client;
|
||||
None creates one from env_file.
|
||||
worker_path (str): Absolute path to the Go executable on compute nodes.
|
||||
|
||||
Raises:
|
||||
ValueError: The worker path or REST connection settings are invalid.
|
||||
"""
|
||||
if not worker_path.startswith("/"):
|
||||
raise ValueError("Worker path must be an absolute compute-node path")
|
||||
self.worker_path = worker_path
|
||||
self._redis_address = f"{redis_host}:{redis_port}"
|
||||
self.rest_client = rest_client if rest_client is not None else SlurmRestClient(env_file)
|
||||
self.sub_client = RedisSubClient(bootstrap=f"{redis_host}:{redis_port}")
|
||||
self._wrapper_path = Path(__file__).resolve().parent / "job_submission" / "wrapper.sh"
|
||||
self._redis_script_path = (
|
||||
Path(__file__).resolve().parent / "job_submission" / "redis_worker.sh"
|
||||
)
|
||||
|
||||
# This is a job_id mapping to job futures
|
||||
self._future_registry: dict[str, JobFuture] = {}
|
||||
# This subscription id sub_info mapping by topics, used for callback handling
|
||||
self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict)
|
||||
# This is a topic mapping to subscriptions
|
||||
self._topic_by_subscription_id: dict[str, str] = {}
|
||||
# This is a job_id mapping to topics
|
||||
self._active_subscriptions: dict[str, set[str]] = defaultdict(set)
|
||||
|
||||
# Threading
|
||||
self._lock = threading.RLock()
|
||||
self._job_tracker = JobTracker(self.sub_client)
|
||||
self._stop_heartbeat_thread = threading.Event()
|
||||
self.executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="SlurmManagerCallback")
|
||||
# Heartbeat monitoring thread
|
||||
self._heartbeat_monitoring_thread = threading.Thread(
|
||||
target=self._heartbeat_monitoring_loop, name="SlurmManagerHeartbeatMonitor", daemon=True
|
||||
)
|
||||
self._start_threads()
|
||||
|
||||
def _start_threads(self):
|
||||
"""Starts the heartbeat monitoring thread."""
|
||||
self._heartbeat_monitoring_thread.start()
|
||||
################################################
|
||||
# Public Job Management #
|
||||
################################################
|
||||
|
||||
def _heartbeat_monitoring_loop(self) -> None:
|
||||
"""Internal loop to check for missing heartbeats. If a heartbeat for a job is missing for
|
||||
more than HEARTBEAT_TIMEOUT seconds, we will trigger a check of the slurm job status,
|
||||
and potentially trigger a cleanup of the job future and subscriptions if the job is no longer active.
|
||||
def submit_job(
|
||||
self,
|
||||
script_path: str | Path,
|
||||
env_path: str | Path | None = None,
|
||||
*,
|
||||
working_directory: str,
|
||||
job_options: JobOptions | None = None,
|
||||
callbacks: dict[Literal["log", "status", "heartbeat"], Callable[[SlurmMessage], None]]
|
||||
| None = None,
|
||||
) -> JobFuture:
|
||||
"""Upload local Bash files and return a monitored, released Slurm job.
|
||||
|
||||
Args:
|
||||
script_path (str | Path): Local Bash workload file whose contents are
|
||||
uploaded.
|
||||
env_path (str | Path | None): Optional local Bash activation file for
|
||||
workload-specific environment setup; None skips activation commands.
|
||||
working_directory (str): Absolute, existing compute-node directory in which
|
||||
the job runs.
|
||||
job_options (JobOptions | None): Validated resources and Slurm output
|
||||
settings; None uses JobOptions defaults.
|
||||
callbacks (dict[str, Callable] | None): Callbacks keyed by log, status, or
|
||||
heartbeat, acknowledged before job release.
|
||||
|
||||
Returns:
|
||||
JobFuture: Handle for querying, waiting, cancelling, and registering
|
||||
callbacks.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: A local workload or activation file does not exist.
|
||||
ValueError: The remote working directory or callback event names are
|
||||
invalid.
|
||||
ValidationError: Resource or output options fail JobOptions validation.
|
||||
SlurmRestError: Submission or release fails. A known held job is cancelled
|
||||
on setup failure.
|
||||
TimeoutError: Redis does not acknowledge the monitoring subscriptions.
|
||||
|
||||
Notes:
|
||||
The credential dotenv file stays local. Callbacks are registered before
|
||||
release; only the working directory, worker path, and output paths refer to
|
||||
compute nodes.
|
||||
"""
|
||||
|
||||
while not self._stop_heartbeat_thread.wait(timeout=2):
|
||||
missing_heartbeats = []
|
||||
# Check heartbeats for active jobs
|
||||
with self._lock:
|
||||
for _, topics in self._active_subscriptions.items():
|
||||
for topic in topics:
|
||||
infos = self._subscriptions_by_topic.get(topic, {})
|
||||
for sub_info in infos.values():
|
||||
if sub_info and sub_info.last_heartbeat:
|
||||
now = time.time()
|
||||
if now - sub_info.last_heartbeat > HEARTBEAT_TIMEOUT and (
|
||||
sub_info.last_scheduler_check is None
|
||||
or now - sub_info.last_scheduler_check > HEARTBEAT_TIMEOUT
|
||||
):
|
||||
missing_heartbeats.append(sub_info)
|
||||
|
||||
for sub_info in missing_heartbeats:
|
||||
self._check_slurm_job(sub_info)
|
||||
|
||||
def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None:
|
||||
"""Resolve missing heartbeats using Slurm, never assuming silence means failure."""
|
||||
future = self._future_registry.get(sub_info.job_id)
|
||||
if future is None or future.done():
|
||||
return
|
||||
sub_info.last_scheduler_check = time.time()
|
||||
try:
|
||||
queued = self._run_subprocess(
|
||||
[
|
||||
"squeue",
|
||||
*_cluster_args(future.slurm_cluster),
|
||||
"--noheader",
|
||||
"--jobs",
|
||||
future.slurm_job_id,
|
||||
"--format=%T",
|
||||
]
|
||||
)
|
||||
if queued.returncode == 0 and queued.stdout.strip():
|
||||
# Pending, configuring, and running jobs may legitimately have no heartbeat.
|
||||
return
|
||||
accounting = self._run_subprocess(
|
||||
[
|
||||
"sacct",
|
||||
*_cluster_args(future.slurm_cluster),
|
||||
"--noheader",
|
||||
"--parsable2",
|
||||
"--allocations",
|
||||
"--jobs",
|
||||
future.slurm_job_id,
|
||||
"--format=State,ExitCode",
|
||||
]
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
logger.exception("Could not query Slurm job %s", future.slurm_job_id)
|
||||
return
|
||||
if accounting.returncode != 0 or not accounting.stdout.strip():
|
||||
# Accounting may be disabled, delayed, or temporarily unavailable.
|
||||
return
|
||||
fields = accounting.stdout.strip().splitlines()[0].split("|")
|
||||
state = fields[0].split()[0].rstrip("+")
|
||||
exit_code = fields[1].strip() if len(fields) > 1 else ""
|
||||
if state == "COMPLETED" and exit_code == "0:0":
|
||||
status = JobStatus.FINISHED.value
|
||||
elif state == "CANCELLED":
|
||||
status = JobStatus.CANCELLED.value
|
||||
elif state in {
|
||||
"COMPLETED",
|
||||
"FAILED",
|
||||
"TIMEOUT",
|
||||
"OUT_OF_MEMORY",
|
||||
"NODE_FAIL",
|
||||
"BOOT_FAIL",
|
||||
"DEADLINE",
|
||||
"PREEMPTED",
|
||||
"REVOKED",
|
||||
}:
|
||||
status = f"error:Slurm state {state}, exit code {exit_code or 'unknown'}"
|
||||
else:
|
||||
return
|
||||
self._event_callback(StatusMessage(status=status), future)
|
||||
|
||||
def submit_job(self, script_path: str, env_path: str) -> JobFuture:
|
||||
"""Create a temporary script from command and submit via sbatch wrapper."""
|
||||
# Check that the wrapper script and redis worker script exist before submitting the job
|
||||
if not self._wrapper_path.exists():
|
||||
raise FileNotFoundError(f"wrapper script not found: {self._wrapper_path}")
|
||||
if not self._redis_script_path.exists():
|
||||
raise FileNotFoundError(f"redis worker script not found: {self._redis_script_path}")
|
||||
|
||||
env_path = os.path.abspath(env_path)
|
||||
script_path = os.path.abspath(script_path)
|
||||
|
||||
job_id = str(uuid4()) # unique job_id for hash-key in Redis.
|
||||
sbatch_cmd = [
|
||||
"sbatch",
|
||||
"--parsable",
|
||||
"--hold",
|
||||
os.path.abspath(self._wrapper_path),
|
||||
env_path,
|
||||
if not working_directory.startswith("/"):
|
||||
raise ValueError("Working directory must be an absolute compute-node path")
|
||||
if callbacks and set(callbacks) - {"log", "status", "heartbeat"}:
|
||||
raise ValueError("Callbacks must use log, status, or heartbeat keys")
|
||||
job_id = str(uuid4())
|
||||
script = self._build_job_script(
|
||||
script_path,
|
||||
env_path,
|
||||
job_id,
|
||||
"1",
|
||||
"1",
|
||||
os.path.abspath(self._redis_script_path),
|
||||
]
|
||||
logger.debug("Submitting job with command: %s", " ".join(sbatch_cmd))
|
||||
|
||||
result = self._run_subprocess(sbatch_cmd)
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or "").strip()
|
||||
raise RuntimeError(f"sbatch failed: {stderr}")
|
||||
slurm_job_id = result.stdout.strip() # ID of job from SLURM scheduler
|
||||
logger.debug("SBatch job submission id: %s", slurm_job_id)
|
||||
|
||||
job_future = JobFuture(
|
||||
job_id=job_id, manager=self, registry=self._future_registry, slurm_job_id=slurm_job_id
|
||||
worker_path=self.worker_path,
|
||||
redis_address=self._redis_address,
|
||||
)
|
||||
job = JobOptions.model_validate(job_options or {}).to_rest()
|
||||
job.update(
|
||||
{
|
||||
"current_working_directory": working_directory,
|
||||
"environment": [
|
||||
"PATH=/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:"
|
||||
"/usr/sbin:/usr/lpp/mmfs/bin:"
|
||||
],
|
||||
"hold": True,
|
||||
}
|
||||
)
|
||||
result = self.rest_client.submit(script, job)
|
||||
slurm_job_id = str(result["job_id"])
|
||||
job_future = JobFuture(job_id=job_id, manager=self, slurm_job_id=slurm_job_id)
|
||||
slurm_job_id = job_future.slurm_job_id
|
||||
cluster_args = _cluster_args(job_future.slurm_cluster)
|
||||
topic_info = TopicInfo(job_id=job_id)
|
||||
with self._lock:
|
||||
self._future_registry[job_id] = job_future
|
||||
try:
|
||||
self._subscribe_to_internal_topics(future=job_future, topic_info=topic_info)
|
||||
self.sub_client.wait_until_subscribed([topic_info.heartbeat, topic_info.event])
|
||||
released = self._run_subprocess(["scontrol", *cluster_args, "release", slurm_job_id])
|
||||
if released.returncode != 0:
|
||||
raise RuntimeError(f"scontrol release failed: {(released.stderr or '').strip()}")
|
||||
self._job_tracker.track(job_future)
|
||||
for event, callback in (callbacks or {}).items():
|
||||
job_future.listen(event, callback)
|
||||
self._job_tracker.wait_until_ready(job_id)
|
||||
self.rest_client.release(slurm_job_id)
|
||||
except Exception as exc:
|
||||
# A setup failure must not leave a held allocation behind.
|
||||
try:
|
||||
cancelled = self._run_subprocess(["scancel", *cluster_args, slurm_job_id])
|
||||
if cancelled.returncode != 0:
|
||||
logger.error("Could not cancel held job %s: %s", slurm_job_id, cancelled.stderr)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
logger.exception("Could not cancel held job %s", slurm_job_id)
|
||||
self._event_callback(
|
||||
self.rest_client.cancel(slurm_job_id)
|
||||
except (SlurmRestError, ValueError):
|
||||
logger.error("Could not cancel held job %s", slurm_job_id)
|
||||
self._job_tracker.apply_status(
|
||||
StatusMessage(status=f"error:Submission setup failed: {exc}"), job_future
|
||||
)
|
||||
raise
|
||||
return job_future
|
||||
|
||||
def cancel_job(
|
||||
self, slurm_job_id: str, job_id: str, *, slurm_cluster: str | None = None
|
||||
) -> None:
|
||||
"""Cancel a job on its recorded cluster via scancel."""
|
||||
job_future = self._future_registry.get(job_id)
|
||||
slurm_job_id, _, cluster = slurm_job_id.partition(";")
|
||||
cluster = slurm_cluster or cluster or (job_future.slurm_cluster if job_future else None)
|
||||
result = self._run_subprocess(["scancel", *_cluster_args(cluster), slurm_job_id])
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or "").strip()
|
||||
raise RuntimeError(f"scancel failed: {stderr}")
|
||||
def cancel_job(self, slurm_job_id: str, job_id: str) -> None:
|
||||
"""Request cancellation through REST and mark the tracked job cancelled.
|
||||
|
||||
Args:
|
||||
slurm_job_id (str): Numeric scheduler identifier returned by Slurm.
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: Slurm rejects cancellation or the request fails.
|
||||
"""
|
||||
job_future = self._job_tracker.get_future(job_id)
|
||||
self.rest_client.cancel(slurm_job_id)
|
||||
|
||||
if job_future:
|
||||
self._event_callback(
|
||||
self._job_tracker.apply_status(
|
||||
StatusMessage(
|
||||
status="cancelled",
|
||||
metadata={
|
||||
@@ -283,127 +201,107 @@ class SlurmManager:
|
||||
future=job_future,
|
||||
)
|
||||
|
||||
################################################
|
||||
# Public Event Subscriptions #
|
||||
################################################
|
||||
|
||||
def listen(
|
||||
self,
|
||||
event_type: Literal["heartbeat", "log", "status"],
|
||||
callback: Callable[[SlurmMessage], None],
|
||||
job_id: str | None = None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status').
|
||||
If the job_id is specified, the callback will only be registered for the specific job. If job_id is None,
|
||||
the callback will be registered for all jobs.
|
||||
Returns a callback ID that can be used to unregister.
|
||||
"""Attach a callback to one tracked job or all currently tracked jobs.
|
||||
|
||||
Args:
|
||||
event_type: The type of event to listen for ('heartbeat', 'log', or 'status').
|
||||
callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for any job.
|
||||
The SlurmMessage object contains two fields, 'topic', 'data'
|
||||
event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log,
|
||||
status, or heartbeat.
|
||||
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
|
||||
Slurm message.
|
||||
job_id (str | None): Opaque manager identifier used in Redis topic names.
|
||||
|
||||
Returns:
|
||||
A list of unique callback IDS will be returned which can be used to unregister the callbacks later.
|
||||
"""
|
||||
job_futures: list[JobFuture] = (
|
||||
[self._future_registry[job_id]] if job_id else list(self._future_registry.values())
|
||||
)
|
||||
callback_ids = []
|
||||
for future in job_futures:
|
||||
callback_ids.append(future.listen(event_type=event_type, callback=callback))
|
||||
list[str]: Subscription IDs accepted by unlisten().
|
||||
|
||||
return callback_ids
|
||||
Raises:
|
||||
KeyError: The explicitly requested job ID is not tracked.
|
||||
RuntimeError: A selected job has already completed.
|
||||
|
||||
Notes:
|
||||
With job_id=None, registration applies to the current jobs, not future
|
||||
submissions. Callbacks run in the tracker executor.
|
||||
"""
|
||||
return self._job_tracker.listen(event_type, callback, job_id)
|
||||
|
||||
def unlisten(self, callback_id: str | list[str]) -> None:
|
||||
"""Remove one or more callback subscriptions without cancelling the job.
|
||||
|
||||
Args:
|
||||
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
|
||||
returned during registration.
|
||||
"""
|
||||
self.remove_subscription_by_id(callback_id)
|
||||
|
||||
def remove_subscription_by_id(self, callback_id: str | list[str]) -> None:
|
||||
"""Remove a subscription by its callback ID."""
|
||||
if isinstance(callback_id, str):
|
||||
callback_id = [callback_id]
|
||||
with self._lock:
|
||||
for cb_id in callback_id:
|
||||
topic = self._topic_by_subscription_id.pop(cb_id, None)
|
||||
if topic is not None:
|
||||
callbacks = self._subscriptions_by_topic.get(topic, {})
|
||||
sub_info = callbacks.pop(cb_id, None)
|
||||
self.sub_client.unsubscribe(cb_id)
|
||||
|
||||
# Other callbacks, including heartbeat monitoring, still own this topic.
|
||||
if not callbacks:
|
||||
self._subscriptions_by_topic.pop(topic, None)
|
||||
if sub_info:
|
||||
topics = self._active_subscriptions.get(sub_info.job_id)
|
||||
if topics is not None:
|
||||
topics.discard(topic)
|
||||
|
||||
def remove_subscription_by_topic(self, topic: str) -> None:
|
||||
"""Remove a subscription by its topic."""
|
||||
with self._lock:
|
||||
callbacks = self._subscriptions_by_topic.pop(topic, {})
|
||||
for callback_id, sub_info in callbacks.items():
|
||||
self._topic_by_subscription_id.pop(callback_id, None)
|
||||
job_id = sub_info.job_id
|
||||
self._active_subscriptions[job_id].discard(topic)
|
||||
|
||||
# Remove the topic subscription from Redis
|
||||
self.sub_client.unsubscribe_topic(topic)
|
||||
|
||||
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
|
||||
"""
|
||||
Remove all subscriptions for a specific job. If skip_event_callback is False, this will also trigger
|
||||
the event callback with an error status which will update the JobFuture status.
|
||||
This can be used to trigger a cleanup of the Job and its subscriptions.
|
||||
|
||||
If it finished in success, it should be called with skip_event_callback=True to avoid overwriting
|
||||
the finished status with an error status.
|
||||
"""Remove the selected callbacks while preserving other topic listeners.
|
||||
|
||||
Args:
|
||||
job_id: The ID of the job for which to remove subscriptions.
|
||||
skip_event_callback: If True, the event callback will not be triggered with an error status.
|
||||
This should be used when the job is finishing successfully to avoid overwriting the
|
||||
finished status with an error status.
|
||||
callback_id (str | list[str]): Subscription ID, or IDs where accepted,
|
||||
returned during registration.
|
||||
"""
|
||||
with self._lock:
|
||||
topics = self._active_subscriptions.pop(job_id, set())
|
||||
for topic in topics:
|
||||
callbacks = self._subscriptions_by_topic.pop(topic, {})
|
||||
for callback_id in callbacks.keys():
|
||||
self._topic_by_subscription_id.pop(callback_id, None)
|
||||
self.sub_client.unsubscribe_topic(topic)
|
||||
future = self._future_registry.get(job_id)
|
||||
if future:
|
||||
# This callback will trigger a cleanup
|
||||
if not skip_event_callback:
|
||||
self._event_callback(
|
||||
StatusMessage(
|
||||
status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}",
|
||||
metadata={
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": future._job_id,
|
||||
"slurm_job_id": future.slurm_job_id,
|
||||
},
|
||||
),
|
||||
future,
|
||||
)
|
||||
self._remove_job_future(future._job_id)
|
||||
self._job_tracker.remove_subscription_by_id(callback_id)
|
||||
|
||||
def remove_subscription_by_topic(self, topic: str) -> None:
|
||||
"""Remove all callbacks for one topic and unsubscribe it from Redis.
|
||||
|
||||
Args:
|
||||
topic (str): Redis channel whose callback registrations are selected.
|
||||
"""
|
||||
self._job_tracker.remove_subscription_by_topic(topic)
|
||||
|
||||
def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None:
|
||||
"""Remove job subscriptions and discard the tracked future.
|
||||
|
||||
Args:
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
skip_event_callback (bool): If true, remove monitoring without synthesizing
|
||||
a failure update.
|
||||
|
||||
Notes:
|
||||
Unless skip_event_callback is true, an unfinished future is marked failed.
|
||||
This only removes local monitoring; it does not cancel the Slurm allocation.
|
||||
"""
|
||||
self._job_tracker.remove_subscriptions_for_job(job_id, skip_event_callback)
|
||||
|
||||
################################################
|
||||
# Public Lifecycle #
|
||||
################################################
|
||||
|
||||
def shutdown(self, timeout: float | None = 1.0) -> None:
|
||||
"""Shuts down the manager, including the heartbeat monitoring thread and Redis subscriptions."""
|
||||
"""Stop monitoring, close Redis, and drain accepted callback work.
|
||||
|
||||
Args:
|
||||
timeout (float | None): Thread-join timeout in seconds; None waits
|
||||
indefinitely.
|
||||
|
||||
Raises:
|
||||
TimeoutError: The heartbeat monitor is still performing a Slurm query.
|
||||
|
||||
Notes:
|
||||
Remote jobs are not cancelled. The timeout applies to thread joins; running
|
||||
and queued user callbacks are drained without a callback execution deadline.
|
||||
"""
|
||||
self._stop_heartbeat_thread.set()
|
||||
self._heartbeat_monitoring_thread.join(timeout)
|
||||
if self._heartbeat_monitoring_thread.is_alive():
|
||||
raise TimeoutError(
|
||||
"Heartbeat monitor did not stop; retry after its Slurm query finishes"
|
||||
)
|
||||
with self._lock:
|
||||
self._subscriptions_by_topic.clear()
|
||||
self._topic_by_subscription_id.clear()
|
||||
self._job_tracker.shutdown(timeout)
|
||||
|
||||
self.sub_client.shutdown(timeout=timeout)
|
||||
self.executor.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
###################
|
||||
### Internal Methods
|
||||
###################
|
||||
################################################
|
||||
# Internal Callback Registration #
|
||||
################################################
|
||||
|
||||
def _add_subscription(
|
||||
self,
|
||||
@@ -412,182 +310,138 @@ class SlurmManager:
|
||||
callback: Callable[[SlurmMessage], None],
|
||||
callback_kwargs: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status') for a specific job.
|
||||
Callbacks receive a SlurmMessage object, depending on the type of event they listen to. The callback will be executed
|
||||
in the background and errors will be logged but not raised to not jeopardize the main loop of the manager.
|
||||
The callback will be executed with additional keyword arguments passed via callback_kwargs dictionary.
|
||||
"""Atomically register a Redis callback and its manager ownership.
|
||||
|
||||
Args:
|
||||
job_id: The ID of the job for which to register the callback.
|
||||
key: The type of event to listen for ('heartbeat', 'log', or 'status').
|
||||
callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for the job.
|
||||
callback_kwargs: A dictionary of additional keyword arguments to pass to the callback when it is executed
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or
|
||||
heartbeat.
|
||||
callback (Callable[[SlurmMessage], None]): Callable receiving the parsed
|
||||
Slurm message.
|
||||
callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed
|
||||
to the user callback.
|
||||
|
||||
Returns:
|
||||
str: Unique callback subscription ID.
|
||||
|
||||
Raises:
|
||||
RuntimeError: The job is absent or finishes during registration.
|
||||
"""
|
||||
|
||||
topic_str = TopicInfo(job_id=job_id).topic(key)
|
||||
return self._job_tracker.add_subscription(job_id, key, callback, callback_kwargs)
|
||||
|
||||
@wraps(callback)
|
||||
def wrapped_callback(
|
||||
message: SlurmMessage,
|
||||
executor: ThreadPoolExecutor,
|
||||
job_id: str,
|
||||
slurm_job_id: str,
|
||||
callback_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": job_id,
|
||||
"slurm_job_id": slurm_job_id,
|
||||
}
|
||||
)
|
||||
executor.submit(callback, message, **(callback_kwargs or {}))
|
||||
################################################
|
||||
# Internal Script Packaging #
|
||||
################################################
|
||||
|
||||
# Completion removes subscriptions under this same lock. Keep ownership
|
||||
# registration atomic with cleanup, and undo Redis registration if the
|
||||
# future became terminal while subscribe() was in progress.
|
||||
with self._lock:
|
||||
future = self._future_registry.get(job_id)
|
||||
if future is None or future.done():
|
||||
raise RuntimeError(f"Job {job_id} is no longer active")
|
||||
cb_id = self.sub_client.subscribe(
|
||||
topic_str,
|
||||
partial(
|
||||
wrapped_callback,
|
||||
executor=self.executor,
|
||||
callback_kwargs=callback_kwargs,
|
||||
job_id=job_id,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
),
|
||||
)
|
||||
|
||||
if self._future_registry.get(job_id) is not future or future.done():
|
||||
self.sub_client.unsubscribe(cb_id)
|
||||
raise RuntimeError(f"Job {job_id} finished during subscription registration")
|
||||
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=job_id,
|
||||
topic=topic_str,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=wrapped_callback,
|
||||
last_heartbeat=None,
|
||||
callback_id=cb_id,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
return cb_id
|
||||
|
||||
def _remove_job_future(self, job_id: str) -> None:
|
||||
"""Remove a job future from the registry."""
|
||||
with self._lock:
|
||||
self._future_registry.pop(job_id, None)
|
||||
|
||||
def _run_subprocess(self, cmd: list[str]) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=30)
|
||||
|
||||
def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None:
|
||||
self._subscribe_to_heartbeat(topic_info, future)
|
||||
self._subscribe_to_event(topic_info, future)
|
||||
|
||||
def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None:
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=topic_info.job_id,
|
||||
topic=topic_info.heartbeat,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=None,
|
||||
last_heartbeat=None,
|
||||
callback_id=None,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
topic = topic_info.heartbeat
|
||||
|
||||
def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
|
||||
try:
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": sub_info.job_id,
|
||||
"slurm_job_id": sub_info.slurm_job_id,
|
||||
}
|
||||
)
|
||||
self._heartbeat_callback(message, sub_info)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Error in callback for topic %s: %s", message.msg_type, message.timestamp
|
||||
)
|
||||
|
||||
cb_id = self.sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info))
|
||||
sub_info.callback_id = cb_id
|
||||
sub_info.last_heartbeat = time.time()
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
|
||||
def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None:
|
||||
sub_info.heartbeat_received = True
|
||||
sub_info.last_heartbeat = time.time()
|
||||
|
||||
def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None:
|
||||
sub_info = SubscriptionInfo(
|
||||
job_id=topic_info.job_id,
|
||||
topic=topic_info.event,
|
||||
slurm_job_id=future.slurm_job_id,
|
||||
callback=None,
|
||||
last_heartbeat=None,
|
||||
callback_id=None,
|
||||
heartbeat_received=False,
|
||||
)
|
||||
event_topic = topic_info.event
|
||||
|
||||
def wrapped_callback(message: StatusMessage, future: JobFuture) -> None:
|
||||
try:
|
||||
message.metadata.update(
|
||||
{
|
||||
"msg_received": datetime.now().isoformat(),
|
||||
"job_id": future._job_id,
|
||||
"slurm_job_id": future.slurm_job_id,
|
||||
}
|
||||
)
|
||||
self._event_callback(message, future=future)
|
||||
except Exception:
|
||||
logger.error("Error in callback for topic %s: %s", message.msg_type, message.status)
|
||||
|
||||
cb_id = self.sub_client.subscribe(event_topic, partial(wrapped_callback, future=future))
|
||||
sub_info.callback_id = cb_id
|
||||
self._register_subscription_for_job(sub_info=sub_info)
|
||||
|
||||
def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None:
|
||||
with self._lock:
|
||||
if sub_info.callback_id is None:
|
||||
raise ValueError("SubscriptionInfo.callback_id must be set before registration")
|
||||
self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info
|
||||
self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic
|
||||
self._active_subscriptions[sub_info.job_id].add(sub_info.topic)
|
||||
|
||||
def _event_callback(self, message: StatusMessage, future: JobFuture) -> None:
|
||||
"""
|
||||
Callback for handling event message updates. The method should also be used to trigger
|
||||
a cleanup of subscriptions and job future when a job is finishing unexpectedly
|
||||
(e.g. due to missing heartbeat or user requested removal). If finishing unexpectedly,
|
||||
we recommend using a payload format of 'error:reason' to indicate the error and reason
|
||||
for the job finishing, which will be reflected in the JobFuture status and exception info.
|
||||
@staticmethod
|
||||
def _build_job_script(
|
||||
script_path: str | Path,
|
||||
env_path: str | Path | None,
|
||||
job_id: str,
|
||||
*,
|
||||
worker_path: str = "/usr/local/bin/slurm-worker",
|
||||
redis_address: str = "localhost:6379",
|
||||
) -> str:
|
||||
"""Package local Bash file contents for the compute-node Go worker.
|
||||
|
||||
Args:
|
||||
message: The incoming message from the event topic, expected to have a payload format of 'status:optional_info'.
|
||||
future: The JobFuture associated with the job, which will be updated based on the event message.
|
||||
script_path (str | Path): Local Bash workload file whose contents are uploaded.
|
||||
env_path (str | Path | None): Optional local Bash activation file for
|
||||
workload-specific environment setup; None skips activation commands.
|
||||
job_id (str): Opaque manager identifier used in Redis topic names.
|
||||
worker_path (str): Absolute path to the Go executable on compute nodes.
|
||||
redis_address (str): Redis host:port passed to the worker; no credentials are
|
||||
embedded.
|
||||
|
||||
Returns:
|
||||
str: Batch script invoking the configured worker with JSON on standard input.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: A local script file is missing.
|
||||
ValueError: The worker path is not an absolute compute-node path.
|
||||
|
||||
Notes:
|
||||
The payload contains only the selected workload and activation scripts, not the
|
||||
credential dotenv file. Staging and cleanup belong to the worker.
|
||||
"""
|
||||
info = message.status.split(":", 1)
|
||||
status = info[0]
|
||||
try: # Catch irregular formatted status messages to avoid crashing the callback loop.
|
||||
status = JobStatus(status)
|
||||
except ValueError:
|
||||
logger.warning("Received unknown job status '%s' for job %s", status, future.name)
|
||||
future._update_status(
|
||||
"error",
|
||||
f"Received unknown job status '{status}' from message: {message.model_dump()}",
|
||||
)
|
||||
if not worker_path.startswith("/"):
|
||||
raise ValueError("Worker path must be an absolute compute-node path")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"script": Path(script_path).read_text(),
|
||||
"activation": Path(env_path).read_text() if env_path is not None else "",
|
||||
}
|
||||
)
|
||||
delimiter = f"SLURM_PAYLOAD_{uuid4().hex}"
|
||||
command = shlex.join([worker_path, "--job-id", job_id, "--redis-address", redis_address])
|
||||
return f"#!/bin/bash\nexec {command} <<'{delimiter}'\n{payload}\n{delimiter}\n"
|
||||
|
||||
################################################
|
||||
# Internal Monitoring #
|
||||
################################################
|
||||
|
||||
def _start_threads(self) -> None:
|
||||
"""Start the manager heartbeat-monitor thread."""
|
||||
self._heartbeat_monitoring_thread.start()
|
||||
|
||||
def _heartbeat_monitoring_loop(self) -> None:
|
||||
"""Check missing heartbeats until shutdown and query Slurm when necessary."""
|
||||
|
||||
while not self._stop_heartbeat_thread.wait(timeout=2):
|
||||
for sub_info in self._job_tracker.get_missing_heartbeats():
|
||||
self._check_slurm_job(sub_info)
|
||||
|
||||
def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None:
|
||||
"""Resolve a missed heartbeat from controller or accounting records.
|
||||
|
||||
Args:
|
||||
sub_info (SubscriptionInfo): Subscription ownership and heartbeat
|
||||
bookkeeping.
|
||||
|
||||
Notes:
|
||||
Transport failures, absent records, and incomplete accounting leave the job
|
||||
unresolved for a later check.
|
||||
"""
|
||||
future = self._job_tracker.begin_scheduler_check(sub_info.job_id)
|
||||
if future is None or future.done():
|
||||
return
|
||||
exc_info = info[1] if len(info) > 1 else None
|
||||
future._update_status(status, exc_info)
|
||||
# Clean up subscriptions for finished job
|
||||
if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]:
|
||||
self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True)
|
||||
try:
|
||||
record = self.rest_client.get_job(future.slurm_job_id)
|
||||
if record is None:
|
||||
record = self.rest_client.get_job(future.slurm_job_id, accounting=True)
|
||||
except (SlurmRestError, ValueError):
|
||||
logger.warning("Could not query Slurm job %s", future.slurm_job_id)
|
||||
return
|
||||
if not record:
|
||||
return
|
||||
state = record.get("job_state", record.get("state", []))
|
||||
if isinstance(state, dict):
|
||||
state = state.get("current", [])
|
||||
states = [state] if isinstance(state, str) else (state or [])
|
||||
terminal = {
|
||||
"COMPLETED",
|
||||
"CANCELLED",
|
||||
"FAILED",
|
||||
"TIMEOUT",
|
||||
"OUT_OF_MEMORY",
|
||||
"NODE_FAIL",
|
||||
"BOOT_FAIL",
|
||||
"DEADLINE",
|
||||
"PREEMPTED",
|
||||
"REVOKED",
|
||||
}
|
||||
state = next((value for value in states if value in terminal), None)
|
||||
if state is None:
|
||||
return
|
||||
exit_code = _job_exit_code(record.get("exit_code"))
|
||||
if state == "COMPLETED" and exit_code == 0:
|
||||
status = JobStatus.FINISHED.value
|
||||
elif state == "CANCELLED":
|
||||
status = JobStatus.CANCELLED.value
|
||||
elif state == "COMPLETED" and exit_code is None:
|
||||
# A partial accounting record is not proof of failure.
|
||||
return
|
||||
else:
|
||||
status = f"error:Slurm state {state}, exit code {exit_code}"
|
||||
self._job_tracker.apply_status(StatusMessage(status=status), future)
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Small authenticated client for the slurmrestd v0.0.44 API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from http import HTTPStatus
|
||||
from http.client import HTTPException
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
from dotenv import dotenv_values
|
||||
|
||||
INVALID_JOB_ID = 2017
|
||||
|
||||
|
||||
def _missing_job(result: dict[str, Any]) -> bool:
|
||||
"""Check whether every reported Slurm error means an unknown job ID.
|
||||
|
||||
Args:
|
||||
result (dict[str, Any]): Decoded Slurm REST response envelope.
|
||||
|
||||
Returns:
|
||||
bool: True only for a nonempty list of invalid-job-ID errors.
|
||||
"""
|
||||
errors = result.get("errors")
|
||||
return (
|
||||
isinstance(errors, list)
|
||||
and bool(errors)
|
||||
and all(
|
||||
isinstance(error, dict) and error.get("error_number") == INVALID_JOB_ID
|
||||
for error in errors
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class SlurmRestError(RuntimeError):
|
||||
"""A transport, HTTP, or Slurm application error."""
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, *_args: Any, **_kwargs: Any) -> None:
|
||||
# Never forward the authentication headers to a redirect target.
|
||||
"""Reject redirects so authentication headers cannot reach another endpoint.
|
||||
|
||||
Args:
|
||||
*_args (Any): Positional redirect-hook arguments supplied by urllib.
|
||||
**_kwargs (Any): Keyword redirect-hook arguments supplied by urllib.
|
||||
|
||||
Returns:
|
||||
None: Redirect rejection, preventing urllib from following the response.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
class SlurmRestClient:
|
||||
"""Load connection settings and a JWT from a local dotenv file or environment.
|
||||
|
||||
Environment variables override the file. The file is read again for every
|
||||
request so replacing an expired token does not require restarting a manager.
|
||||
"""
|
||||
|
||||
def __init__(self, env_file: str | Path = ".env", timeout: float = 30) -> None:
|
||||
"""Load and validate slurmrestd connection settings.
|
||||
|
||||
Args:
|
||||
env_file (str | Path): Local credential dotenv path. Environment variables
|
||||
take precedence.
|
||||
timeout (float): Positive HTTP request timeout in seconds.
|
||||
|
||||
Raises:
|
||||
ValueError: The URL, API version, token, or timeout is invalid.
|
||||
"""
|
||||
self.env_file = Path(env_file).resolve()
|
||||
if timeout <= 0:
|
||||
raise ValueError("REST timeout must be positive")
|
||||
self.timeout = timeout
|
||||
settings = self._settings()
|
||||
self.url = settings.get("SLURM_REST_URL", "").rstrip("/")
|
||||
parsed = urlsplit(self.url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise ValueError("Set SLURM_REST_URL to the slurmrestd HTTP(S) URL")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("SLURM_REST_URL must not contain credentials, query, or fragment")
|
||||
self.api_version = settings.get("SLURM_REST_API_VERSION") or "v0.0.44"
|
||||
if not re.fullmatch(r"v\d+\.\d+\.\d+", self.api_version):
|
||||
raise ValueError("SLURM_REST_API_VERSION must look like v0.0.44")
|
||||
self._headers(settings)
|
||||
self._opener = build_opener(_NoRedirect())
|
||||
|
||||
################################################
|
||||
# Public Job Operations #
|
||||
################################################
|
||||
|
||||
def submit(self, script: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Submit a batch script and its compute-node job settings.
|
||||
|
||||
Args:
|
||||
script (str): Complete batch script text sent directly to slurmrestd.
|
||||
job (dict[str, Any]): REST job-description fields, including compute-node
|
||||
paths and resources.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Submission response containing a numeric job_id.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: Submission fails or no numeric job ID is returned.
|
||||
"""
|
||||
result = self.request(
|
||||
"POST", f"slurm/{self.api_version}/job/submit", payload={"script": script, "job": job}
|
||||
)
|
||||
if not str(result.get("job_id", "")).isdigit():
|
||||
raise SlurmRestError("Slurm REST submission did not return a job ID")
|
||||
return result
|
||||
|
||||
def release(self, job_id: str) -> None:
|
||||
"""Release a job that was submitted with hold enabled.
|
||||
|
||||
Args:
|
||||
job_id (str): Numeric scheduler job ID on this REST endpoint.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: The job update fails.
|
||||
"""
|
||||
self.request("POST", self._job_path(job_id), payload={"hold": False})
|
||||
|
||||
def cancel(self, job_id: str) -> None:
|
||||
"""Cancel a job on the cluster served by this REST endpoint.
|
||||
|
||||
Args:
|
||||
job_id (str): Numeric scheduler job ID on this REST endpoint.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: Cancellation fails.
|
||||
"""
|
||||
self.request("DELETE", self._job_path(job_id))
|
||||
|
||||
def get_job(self, job_id: str, *, accounting: bool = False) -> dict[str, Any] | None:
|
||||
"""Fetch a controller or accounting job record.
|
||||
|
||||
Args:
|
||||
job_id (str): Numeric scheduler job ID on this REST endpoint.
|
||||
accounting (bool): If true, query slurmdb instead of the controller.
|
||||
|
||||
Returns:
|
||||
dict[str, Any] | None: First returned job record, or None when the job is
|
||||
unavailable.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: The query fails or its records are malformed.
|
||||
"""
|
||||
path = self._job_path(job_id, accounting=accounting)
|
||||
result = self.request("GET", path, missing_ok=True)
|
||||
jobs = result.get("jobs") or []
|
||||
if not isinstance(jobs, list) or any(not isinstance(job, dict) for job in jobs):
|
||||
raise SlurmRestError("Slurm REST returned invalid job records")
|
||||
return jobs[0] if jobs else None
|
||||
|
||||
################################################
|
||||
# Public HTTP Requests #
|
||||
################################################
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
missing_ok: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Send one authenticated JSON request and validate the response envelope.
|
||||
|
||||
Args:
|
||||
method (str): HTTP method such as GET, POST, or DELETE.
|
||||
path (str): Relative REST endpoint appended to the configured service URL.
|
||||
payload (dict[str, Any] | None): JSON-compatible request body or incoming
|
||||
message payload.
|
||||
missing_ok (bool): If true, treat missing-job responses as an empty result.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Decoded response object; an empty object for an allowed
|
||||
missing job.
|
||||
|
||||
Raises:
|
||||
SlurmRestError: HTTP, transport, JSON, or Slurm application validation
|
||||
fails.
|
||||
ValueError: The current token is missing.
|
||||
|
||||
Notes:
|
||||
Requests are not automatically retried. Authentication headers and response
|
||||
bodies are not included in raised transport errors.
|
||||
"""
|
||||
headers = self._headers(self._settings())
|
||||
data = None if payload is None else json.dumps(payload).encode()
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = Request(f"{self.url}/{path}", data=data, headers=headers, method=method)
|
||||
try:
|
||||
with self._opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read()
|
||||
except HTTPError as exc:
|
||||
try:
|
||||
# slurmrestd may use HTTP 500 for a purged controller job. Check
|
||||
# only its numeric error code; never include the body in logs.
|
||||
error_body = json.loads(exc.read()) if missing_ok else {}
|
||||
except (ValueError, UnicodeDecodeError, OSError, HTTPException):
|
||||
error_body = {}
|
||||
finally:
|
||||
exc.close()
|
||||
if missing_ok and (
|
||||
exc.code == HTTPStatus.NOT_FOUND
|
||||
or (
|
||||
exc.code in {HTTPStatus.BAD_REQUEST, HTTPStatus.INTERNAL_SERVER_ERROR}
|
||||
and isinstance(error_body, dict)
|
||||
and _missing_job(error_body)
|
||||
)
|
||||
):
|
||||
return {}
|
||||
raise SlurmRestError(f"Slurm REST {method} failed (HTTP {exc.code})") from None
|
||||
except (URLError, OSError, TimeoutError, HTTPException):
|
||||
raise SlurmRestError(f"Slurm REST {method} failed (connection or timeout)") from None
|
||||
try:
|
||||
result = json.loads(raw) if raw else {}
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
raise SlurmRestError("Slurm REST returned invalid JSON") from None
|
||||
if not isinstance(result, dict):
|
||||
raise SlurmRestError("Slurm REST returned an invalid response")
|
||||
errors = result.get("errors") or []
|
||||
if errors:
|
||||
if missing_ok and _missing_job(result):
|
||||
return {}
|
||||
raise SlurmRestError("Slurm REST reported an application error")
|
||||
return result
|
||||
|
||||
################################################
|
||||
# Internal Authentication #
|
||||
################################################
|
||||
|
||||
def _settings(self) -> dict[str, str]:
|
||||
"""Read local dotenv values and apply environment-variable overrides.
|
||||
|
||||
Returns:
|
||||
dict[str, str]: Connection and authentication settings without modifying
|
||||
os.environ.
|
||||
"""
|
||||
values = dotenv_values(self.env_file, interpolate=False)
|
||||
keys = ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER")
|
||||
return {key: os.environ.get(key, values.get(key) or "") for key in keys}
|
||||
|
||||
@staticmethod
|
||||
def _headers(settings: dict[str, str]) -> dict[str, str]:
|
||||
"""Build authentication headers from the current settings.
|
||||
|
||||
Args:
|
||||
settings (dict[str, str]): Connection settings containing the JWT and
|
||||
optional user name.
|
||||
|
||||
Returns:
|
||||
dict[str, str]: JWT, optional user-name, and response-format headers.
|
||||
|
||||
Raises:
|
||||
ValueError: The token is empty.
|
||||
"""
|
||||
token = settings.get("SLURM_JWT", "")
|
||||
if not token:
|
||||
raise ValueError("Set SLURM_JWT in the local .env file or environment")
|
||||
headers = {"X-SLURM-USER-TOKEN": token, "Accept": "application/json"}
|
||||
if settings.get("SLURM_USER"):
|
||||
headers["X-SLURM-USER-NAME"] = settings["SLURM_USER"]
|
||||
return headers
|
||||
|
||||
################################################
|
||||
# Internal Endpoints #
|
||||
################################################
|
||||
|
||||
def _job_path(self, job_id: str, *, accounting: bool = False) -> str:
|
||||
"""Build a versioned controller or accounting job endpoint.
|
||||
|
||||
Args:
|
||||
job_id (str): Numeric scheduler job ID on this REST endpoint.
|
||||
accounting (bool): If true, query slurmdb instead of the controller.
|
||||
|
||||
Returns:
|
||||
str: Relative REST endpoint path.
|
||||
|
||||
Raises:
|
||||
ValueError: The scheduler job ID is not numeric.
|
||||
"""
|
||||
if not str(job_id).isdigit():
|
||||
raise ValueError("Slurm job ID must be numeric")
|
||||
service = "slurmdb" if accounting else "slurm"
|
||||
return f"{service}/{self.api_version}/job/{job_id}"
|
||||
Whitespace-only changes.
@@ -1,26 +0,0 @@
|
||||
SLURM_JOB_STATE_CODES = {
|
||||
"BF": "BOOT_FAIL", # Job terminated due to launch failure, typically due to a hardware failure (e.g. unable to boot the node or block and the job can not be requeued).
|
||||
"CA": "CANCELLED", # Job was explicitly cancelled by the user or system administrator. The job may or may not have been initiated.
|
||||
"CD": "COMPLETED", # Job has terminated all processes on all nodes with an exit code of zero.
|
||||
"CF": "CONFIGURING", # Job has been allocated resources, but are waiting for them to become ready for use (e.g. booting).
|
||||
"CG": "COMPLETING", # Job is in the process of completing. Some processes on some nodes may still be active.
|
||||
"DL": "DEADLINE", # Job terminated on deadline.
|
||||
"F": "FAILED", # Job terminated with non-zero exit code or other failure condition.
|
||||
"NF": "NODE_FAIL", # Job terminated due to failure of one or more allocated nodes.
|
||||
"OOM": "OUT_OF_MEMORY", # Job experienced out of memory error.
|
||||
"PD": "PENDING", # Job is awaiting resource allocation.
|
||||
"PR": "PREEMPTED", # Job terminated due to preemption.
|
||||
"R": "RUNNING", # Job currently has an allocation.
|
||||
"RD": "RESV_DEL_HOLD", # Job is being held after requested reservation was deleted.
|
||||
"RF": "REQUEUE_FED", # Job is being requeued by a federation.
|
||||
"RH": "REQUEUE_HOLD", # Held job is being requeued.
|
||||
"RQ": "REQUEUED", # Completing job is being requeued.
|
||||
"RS": "RESIZING", # Job is about to change size.
|
||||
"RV": "REVOKED", # Sibling was removed from cluster due to other cluster starting the job.
|
||||
"SI": "SIGNALING", # Job is being signaled.
|
||||
"SE": "SPECIAL_EXIT", # The job was requeued in a special state. This state can be set by users, typically in EpilogSlurmctld, if the job has terminated with a particular exit value.
|
||||
"SO": "STAGE_OUT", # Job is staging out files.
|
||||
"ST": "STOPPED", # Job has an allocation, but execution has been stopped with SIGSTOP signal. CPUS have been retained by this job.
|
||||
"S": "SUSPENDED", # Job has an allocation, but execution has been suspended and CPUs have been released for other jobs.
|
||||
"TO": "TIMEOUT", # Job terminated upon reaching its time limit
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Reject invalid resource requests before sending them to Slurm."""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from slurm_manager.job_options import JobOptions
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("tasks", 0),
|
||||
("tasks", -1),
|
||||
("tasks", True),
|
||||
("tasks", 1.5),
|
||||
("time_limit_minutes", 0),
|
||||
("time_limit_minutes", -1),
|
||||
("time_limit_minutes", "60"),
|
||||
("partition", " "),
|
||||
("hold", False),
|
||||
("time_limit", {"set": True, "number": 1}),
|
||||
],
|
||||
)
|
||||
def test_invalid_or_internal_options_are_rejected(field, value):
|
||||
with pytest.raises(ValidationError):
|
||||
JobOptions(**{field: value})
|
||||
|
||||
|
||||
def test_unset_options_preserve_slurm_defaults():
|
||||
assert JobOptions().to_rest() == {"tasks": 1}
|
||||
|
||||
|
||||
def test_runtime_uses_minutes_and_slurm_optional_number_shape():
|
||||
options = JobOptions(partition="hour", tasks=2, time_limit_minutes=60)
|
||||
assert options.to_rest() == {
|
||||
"partition": "hour",
|
||||
"tasks": 2,
|
||||
"time_limit": {"set": True, "number": 60},
|
||||
}
|
||||
assert options.time_limit_minutes == 60
|
||||
|
||||
|
||||
@pytest.mark.parametrize("partition", ["hour", "day", "week", "gpu"])
|
||||
def test_supported_ra_partitions(partition):
|
||||
assert JobOptions(partition=partition).to_rest()["partition"] == partition
|
||||
|
||||
|
||||
@pytest.mark.parametrize("partition", ["admin", "jupyter", "gpu-large", "unknown"])
|
||||
def test_unsupported_partitions_are_rejected(partition):
|
||||
with pytest.raises(ValidationError):
|
||||
JobOptions(partition=partition)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Verify local scripts and configuration reach the worker without shell expansion."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from slurm_manager.slurm_manager import SlurmManager
|
||||
|
||||
|
||||
def test_local_scripts_are_passed_to_worker_as_json(tmp_path):
|
||||
workload = tmp_path / "work load.sh"
|
||||
activation = tmp_path / "activate.sh"
|
||||
workload.write_text('#!/bin/bash\nprintf "$HOME and $value"\n')
|
||||
activation.write_text("export TEST_VALUE='literal $value and spaces'\n")
|
||||
worker = tmp_path / "worker $(touch must-not-exist)"
|
||||
worker.write_text(
|
||||
f"#!{sys.executable}\nimport json,sys\n"
|
||||
"print(json.dumps({'args':sys.argv[1:], 'payload':json.load(sys.stdin)}))\n"
|
||||
)
|
||||
worker.chmod(0o700)
|
||||
script = SlurmManager._build_job_script(
|
||||
workload, activation, "job-id", worker_path=str(worker), redis_address="redis.example:6380"
|
||||
)
|
||||
result = subprocess.run(
|
||||
["bash"], input=script, capture_output=True, text=True, check=True, cwd=tmp_path
|
||||
)
|
||||
received = json.loads(result.stdout)
|
||||
assert received["payload"] == {
|
||||
"script": workload.read_text(),
|
||||
"activation": activation.read_text(),
|
||||
}
|
||||
assert received["args"] == ["--job-id", "job-id", "--redis-address", "redis.example:6380"]
|
||||
assert str(workload) not in script
|
||||
assert not (tmp_path / "must-not-exist").exists()
|
||||
|
||||
|
||||
def test_missing_local_file_fails_before_packaging(tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
SlurmManager._build_job_script(tmp_path / "missing.sh", tmp_path / "activate.sh", "job-id")
|
||||
|
||||
|
||||
def test_worker_path_must_be_absolute(tmp_path):
|
||||
with pytest.raises(ValueError, match="absolute"):
|
||||
SlurmManager._build_job_script(
|
||||
tmp_path / "unused.sh", tmp_path / "unused-env.sh", "job-id", worker_path="worker"
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Exercise tracking independently of REST submission and scheduler monitoring."""
|
||||
|
||||
from functools import partial
|
||||
from unittest import mock
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
|
||||
from slurm_manager.event_messages import StatusMessage
|
||||
from slurm_manager.job_future import JobFuture
|
||||
from slurm_manager.job_tracker import JobTracker, TopicInfo
|
||||
from slurm_manager.redis_sub_client import RedisSubClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracker():
|
||||
client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
|
||||
tracker = JobTracker(client)
|
||||
try:
|
||||
yield tracker
|
||||
finally:
|
||||
tracker.shutdown(timeout=3)
|
||||
|
||||
|
||||
def test_heartbeat_snapshots_cannot_mutate_tracking_state(tracker):
|
||||
future = JobFuture("first", manager=None, slurm_job_id="123")
|
||||
with mock.patch("slurm_manager.job_tracker.time.time", return_value=80):
|
||||
tracker.track(future)
|
||||
|
||||
with mock.patch("slurm_manager.job_tracker.time.time", return_value=100):
|
||||
snapshot = tracker.get_missing_heartbeats()[0]
|
||||
snapshot.last_heartbeat = 0
|
||||
assert tracker.get_missing_heartbeats()[0].last_heartbeat == 80
|
||||
assert tracker.begin_scheduler_check("first") is future
|
||||
assert tracker.get_missing_heartbeats() == []
|
||||
|
||||
|
||||
def test_completion_preserves_other_jobs_and_their_redis_subscriptions(tracker):
|
||||
first = JobFuture("first", manager=None, slurm_job_id="123")
|
||||
second = JobFuture("second", manager=None, slurm_job_id="124")
|
||||
tracker.track(first)
|
||||
tracker.track(second)
|
||||
tracker.wait_until_ready("second")
|
||||
|
||||
tracker.apply_status(StatusMessage(status="finished"), first)
|
||||
|
||||
assert first.success()
|
||||
assert tracker.get_future("first") is None
|
||||
assert tracker.get_future("second") is second
|
||||
assert not second.done()
|
||||
assert tracker._sub_client.registered_topics == {
|
||||
TopicInfo("second").heartbeat,
|
||||
TopicInfo("second").event,
|
||||
}
|
||||
tracker._sub_client._redis.publish(
|
||||
TopicInfo("second").event, '{"msg_type":"status","status":"finished"}'
|
||||
)
|
||||
second.wait(timeout=1)
|
||||
assert second.success()
|
||||
|
||||
|
||||
def test_scheduler_check_skips_a_job_removed_after_snapshot(tracker):
|
||||
future = JobFuture("first", manager=None, slurm_job_id="123")
|
||||
tracker.track(future)
|
||||
tracker.apply_status(StatusMessage(status="finished"), future)
|
||||
|
||||
assert tracker.begin_scheduler_check("first") is None
|
||||
assert tracker.begin_scheduler_check("unknown") is None
|
||||
+234
-123
@@ -2,10 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import fakeredis
|
||||
@@ -13,23 +13,22 @@ import pytest
|
||||
|
||||
from slurm_manager.event_messages import HeartBeatMessage, StatusMessage
|
||||
from slurm_manager.job_future import JobFuture
|
||||
from slurm_manager.job_options import JobOptions
|
||||
from slurm_manager.job_tracker import TopicInfo
|
||||
from slurm_manager.redis_sub_client import RedisSubClient
|
||||
from slurm_manager.slurm_manager import SlurmManager, TopicInfo
|
||||
from slurm_manager.slurm_manager import SlurmManager
|
||||
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager():
|
||||
client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
|
||||
rest = mock.Mock(spec=SlurmRestClient)
|
||||
rest.submit.return_value = {"job_id": 123}
|
||||
rest.get_job.return_value = {"job_state": ["PENDING"]}
|
||||
with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=client):
|
||||
manager = SlurmManager()
|
||||
manager = SlurmManager(rest_client=rest)
|
||||
|
||||
def slurm(command):
|
||||
output = (
|
||||
"123\n" if command[0] == "sbatch" else "PENDING\n" if command[0] == "squeue" else ""
|
||||
)
|
||||
return subprocess.CompletedProcess(command, 0, output, "")
|
||||
|
||||
manager._run_subprocess = mock.Mock(side_effect=slurm)
|
||||
try:
|
||||
yield manager
|
||||
finally:
|
||||
@@ -40,13 +39,20 @@ def manager():
|
||||
|
||||
|
||||
def submit(manager):
|
||||
# The workload and activation environment belong to the remote node. They
|
||||
# deliberately do not exist in the submitter's Python environment/filesystem.
|
||||
return manager.submit_job("/remote/node/workload.sh", "/remote/node/activate.sh")
|
||||
return manager.submit_job(
|
||||
Path(__file__).parent.parent / "examples/print_for_10s.sh",
|
||||
working_directory="/remote/node/work",
|
||||
)
|
||||
|
||||
|
||||
def heartbeat_info(manager, future):
|
||||
return next(iter(manager._subscriptions_by_topic[TopicInfo(future._job_id).heartbeat].values()))
|
||||
return next(
|
||||
iter(
|
||||
manager._job_tracker._subscriptions_by_topic[
|
||||
TopicInfo(future._job_id).heartbeat
|
||||
].values()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_wait_propagates_failure_arriving_during_wait():
|
||||
@@ -79,46 +85,34 @@ def test_terminal_state_ignores_late_status_updates():
|
||||
|
||||
|
||||
def test_job_is_held_until_redis_acknowledges_subscriptions(manager):
|
||||
commands = []
|
||||
|
||||
def slurm(command):
|
||||
commands.append(command)
|
||||
if command[0] == "sbatch":
|
||||
assert "--hold" in command
|
||||
assert "/remote/node/activate.sh" in command
|
||||
assert "/remote/node/workload.sh" in command
|
||||
assert command[-1].endswith("redis_worker.sh")
|
||||
return subprocess.CompletedProcess(command, 0, "123\n", "")
|
||||
assert command == ["scontrol", "release", "123"]
|
||||
future = next(iter(manager._future_registry.values()))
|
||||
def release(job_id):
|
||||
assert job_id == "123"
|
||||
script, job = manager.rest_client.submit.call_args.args
|
||||
assert job["hold"] is True
|
||||
assert job["current_working_directory"] == "/remote/node/work"
|
||||
assert '"activation": ""' in script
|
||||
assert "stdout-test %02d/10" in script
|
||||
assert "/usr/local/bin/slurm-worker" in script
|
||||
future = next(iter(manager._job_tracker._future_registry.values()))
|
||||
topics = TopicInfo(future._job_id)
|
||||
assert manager.sub_client._subscription_ready[topics.event].is_set()
|
||||
assert manager.sub_client._subscription_ready[topics.heartbeat].is_set()
|
||||
# Simulate a remote job completing before scontrol returns.
|
||||
manager.sub_client._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}')
|
||||
future.wait(timeout=1)
|
||||
return subprocess.CompletedProcess(command, 0, "", "")
|
||||
|
||||
manager._run_subprocess.side_effect = slurm
|
||||
manager.rest_client.release.side_effect = release
|
||||
future = submit(manager)
|
||||
assert future.success()
|
||||
assert [command[0] for command in commands] == ["sbatch", "scontrol"]
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["subscription", "acknowledgment", "release"])
|
||||
def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failure):
|
||||
original = manager._run_subprocess.side_effect
|
||||
|
||||
def slurm(command):
|
||||
if command[0] == "scontrol":
|
||||
return subprocess.CompletedProcess(command, 1, "", "release rejected")
|
||||
return original(command)
|
||||
|
||||
manager._run_subprocess.side_effect = slurm
|
||||
if failure == "release":
|
||||
manager.rest_client.release.side_effect = SlurmRestError("release rejected")
|
||||
if failure == "subscription":
|
||||
patch = mock.patch.object(
|
||||
manager, "_subscribe_to_event", side_effect=RuntimeError("offline")
|
||||
manager._job_tracker, "_subscribe_to_event", side_effect=RuntimeError("offline")
|
||||
)
|
||||
elif failure == "acknowledgment":
|
||||
patch = mock.patch.object(
|
||||
@@ -132,33 +126,64 @@ def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failur
|
||||
)
|
||||
with patch, pytest.raises((RuntimeError, TimeoutError)):
|
||||
submit(manager)
|
||||
assert mock.call(["scancel", "123"]) in manager._run_subprocess.call_args_list
|
||||
assert not manager._future_registry
|
||||
assert not manager._active_subscriptions
|
||||
manager.rest_client.cancel.assert_called_once_with("123")
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager._job_tracker._active_subscriptions
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"last_heartbeat,last_check,expected",
|
||||
[
|
||||
(None, None, False),
|
||||
(0, None, False),
|
||||
(95, None, False),
|
||||
(90, None, False),
|
||||
(89, None, True),
|
||||
(89, 95, False),
|
||||
(89, 90, False),
|
||||
(89, 89, True),
|
||||
],
|
||||
)
|
||||
def test_heartbeat_selection_respects_timeout_and_scheduler_throttle(
|
||||
manager, last_heartbeat, last_check, expected
|
||||
):
|
||||
manager._stop_heartbeat_thread.set()
|
||||
manager._heartbeat_monitoring_thread.join(timeout=1)
|
||||
future = submit(manager)
|
||||
future.listen("heartbeat", lambda message: None)
|
||||
future.listen("log", lambda message: None)
|
||||
info = heartbeat_info(manager, future)
|
||||
info.last_heartbeat = last_heartbeat
|
||||
info.last_scheduler_check = last_check
|
||||
|
||||
with mock.patch("slurm_manager.job_tracker.time.time", return_value=100):
|
||||
missing = manager._job_tracker.get_missing_heartbeats()
|
||||
|
||||
assert missing == ([info] if expected else [])
|
||||
manager.rest_client.get_job.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["PENDING", "CONFIGURING", "RUNNING"])
|
||||
def test_missing_heartbeat_does_not_fail_active_slurm_job(manager, state):
|
||||
future = submit(manager)
|
||||
info = heartbeat_info(manager, future)
|
||||
info.last_heartbeat = time.time() - 100
|
||||
manager._run_subprocess.return_value = subprocess.CompletedProcess([], 0, f"{state}\n", "")
|
||||
manager._run_subprocess.side_effect = None
|
||||
manager.rest_client.get_job.return_value = {"job_state": [state]}
|
||||
manager._check_slurm_job(info)
|
||||
assert not future.done()
|
||||
assert future._job_id in manager._future_registry
|
||||
assert TopicInfo(future._job_id).heartbeat in manager._active_subscriptions[future._job_id]
|
||||
assert manager._run_subprocess.call_args.args[0][0] == "squeue"
|
||||
assert future._job_id in manager._job_tracker._future_registry
|
||||
assert (
|
||||
TopicInfo(future._job_id).heartbeat
|
||||
in manager._job_tracker._active_subscriptions[future._job_id]
|
||||
)
|
||||
manager.rest_client.get_job.assert_called_once_with("123")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accounting", ["", "PENDING|0:0\n"])
|
||||
@pytest.mark.parametrize("accounting", [None, {"state": {"current": ["PENDING"]}}])
|
||||
def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting):
|
||||
future = submit(manager)
|
||||
manager._run_subprocess.side_effect = [
|
||||
subprocess.CompletedProcess([], 0, "", ""),
|
||||
subprocess.CompletedProcess([], 0, accounting, ""),
|
||||
]
|
||||
manager.rest_client.get_job.side_effect = [None, accounting]
|
||||
manager._check_slurm_job(heartbeat_info(manager, future))
|
||||
assert not future.done()
|
||||
|
||||
@@ -166,28 +191,25 @@ def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting):
|
||||
@pytest.mark.parametrize(
|
||||
"accounting,status,success",
|
||||
[
|
||||
("COMPLETED|0:0\n", "finished", True),
|
||||
("FAILED|7:0\n", "error", False),
|
||||
("CANCELLED by 1000|0:15\n", "cancelled", False),
|
||||
({"job_state": ["COMPLETED"], "exit_code": 0}, "finished", True),
|
||||
({"job_state": ["FAILED"], "exit_code": 7}, "error", False),
|
||||
({"job_state": ["CANCELLED"], "exit_code": 15}, "cancelled", False),
|
||||
],
|
||||
)
|
||||
def test_terminal_accounting_resolves_missed_remote_status(manager, accounting, status, success):
|
||||
future = submit(manager)
|
||||
manager._run_subprocess.side_effect = [
|
||||
subprocess.CompletedProcess([], 0, "", ""),
|
||||
subprocess.CompletedProcess([], 0, accounting, ""),
|
||||
]
|
||||
manager.rest_client.get_job.side_effect = [None, accounting]
|
||||
manager._check_slurm_job(heartbeat_info(manager, future))
|
||||
assert future.done()
|
||||
assert future.status == status
|
||||
assert future.success() == success
|
||||
assert not manager._future_registry
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
def test_scheduler_error_keeps_job_unresolved(manager):
|
||||
future = submit(manager)
|
||||
manager._run_subprocess.side_effect = FileNotFoundError("squeue unavailable")
|
||||
manager.rest_client.get_job.side_effect = SlurmRestError("unavailable")
|
||||
manager._check_slurm_job(heartbeat_info(manager, future))
|
||||
assert not future.done()
|
||||
|
||||
@@ -201,7 +223,7 @@ def test_unlisten_removes_only_selected_callback_and_preserves_monitoring(manage
|
||||
future.unlisten(callback_id)
|
||||
topic = TopicInfo(future._job_id).heartbeat
|
||||
assert callback_id not in manager.sub_client._topic_by_subscription_id
|
||||
assert topic in manager._active_subscriptions[future._job_id]
|
||||
assert topic in manager._job_tracker._active_subscriptions[future._job_id]
|
||||
manager.sub_client._redis.publish(
|
||||
topic, HeartBeatMessage(timestamp=time.time()).model_dump_json()
|
||||
)
|
||||
@@ -217,67 +239,36 @@ def test_removing_last_log_callback_unsubscribes_topic(manager):
|
||||
callback_id = future.listen("log", lambda _: None)
|
||||
future.unlisten(callback_id)
|
||||
topic = TopicInfo(future._job_id).log
|
||||
assert topic not in manager._active_subscriptions[future._job_id]
|
||||
assert topic not in manager._job_tracker._active_subscriptions[future._job_id]
|
||||
assert topic not in manager.sub_client.registered_topics
|
||||
assert TopicInfo(future._job_id).heartbeat in manager.sub_client.registered_topics
|
||||
|
||||
|
||||
def cluster_submission(manager):
|
||||
original = manager._run_subprocess.side_effect
|
||||
|
||||
def slurm(command):
|
||||
if command[0] == "sbatch":
|
||||
return subprocess.CompletedProcess(command, 0, "123;cluster_a\n", "")
|
||||
return original(command)
|
||||
|
||||
manager._run_subprocess.side_effect = slurm
|
||||
|
||||
|
||||
def test_cluster_qualified_job_is_released_and_cancelled_on_its_cluster(manager):
|
||||
cluster_submission(manager)
|
||||
@pytest.mark.parametrize("code,success", [(0, True), (7, False)])
|
||||
def test_accounting_structured_exit_code(manager, code, success):
|
||||
future = submit(manager)
|
||||
assert future.slurm_job_id == "123"
|
||||
assert future.slurm_cluster == "cluster_a"
|
||||
assert mock.call(["scontrol", "--clusters", "cluster_a", "release", "123"]) in (
|
||||
manager._run_subprocess.call_args_list
|
||||
)
|
||||
future.cancel()
|
||||
assert manager._run_subprocess.call_args == mock.call(
|
||||
["scancel", "--clusters", "cluster_a", "123"]
|
||||
)
|
||||
|
||||
|
||||
def test_cluster_qualified_setup_failure_cancels_on_its_cluster(manager):
|
||||
cluster_submission(manager)
|
||||
with (
|
||||
mock.patch.object(
|
||||
manager.sub_client, "wait_until_subscribed", side_effect=TimeoutError("no ack")
|
||||
),
|
||||
pytest.raises(TimeoutError),
|
||||
):
|
||||
submit(manager)
|
||||
assert manager._run_subprocess.call_args == mock.call(
|
||||
["scancel", "--clusters", "cluster_a", "123"]
|
||||
)
|
||||
assert not manager._future_registry
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
def test_cluster_qualified_job_monitoring_queries_its_cluster(manager):
|
||||
cluster_submission(manager)
|
||||
future = submit(manager)
|
||||
manager._run_subprocess.reset_mock()
|
||||
manager._run_subprocess.side_effect = [
|
||||
subprocess.CompletedProcess([], 0, "", ""),
|
||||
subprocess.CompletedProcess([], 0, "COMPLETED|0:0\n", ""),
|
||||
manager.rest_client.get_job.side_effect = [
|
||||
None,
|
||||
{
|
||||
"state": {"current": ["COMPLETED"]},
|
||||
"exit_code": {
|
||||
"return_code": {"set": True, "number": code},
|
||||
"signal": {"id": {"set": False, "number": 0}},
|
||||
},
|
||||
},
|
||||
]
|
||||
manager._check_slurm_job(heartbeat_info(manager, future))
|
||||
commands = [call.args[0] for call in manager._run_subprocess.call_args_list]
|
||||
assert [command[0] for command in commands] == ["squeue", "sacct"]
|
||||
for command in commands:
|
||||
assert command[1:3] == ["--clusters", "cluster_a"]
|
||||
assert command[command.index("--jobs") + 1] == "123"
|
||||
assert future.success()
|
||||
assert future.done()
|
||||
assert future.success() == success
|
||||
assert manager.rest_client.get_job.call_args == mock.call("123", accounting=True)
|
||||
|
||||
|
||||
def test_cancel_uses_rest_and_cleans_subscriptions(manager):
|
||||
future = submit(manager)
|
||||
future.cancel()
|
||||
manager.rest_client.cancel.assert_called_once_with("123")
|
||||
assert future.status == "cancelled"
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
def test_completion_during_subscription_registration_leaves_no_orphan(manager):
|
||||
@@ -287,7 +278,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
|
||||
cleaned = threading.Event()
|
||||
errors = []
|
||||
original_subscribe = manager.sub_client.subscribe
|
||||
original_event = manager._event_callback
|
||||
original_event = manager._job_tracker.apply_status
|
||||
|
||||
def subscribe(topic, callback):
|
||||
callback_id = original_subscribe(topic, callback)
|
||||
@@ -307,7 +298,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
|
||||
|
||||
with (
|
||||
mock.patch.object(manager.sub_client, "subscribe", side_effect=subscribe),
|
||||
mock.patch.object(manager, "_event_callback", side_effect=event),
|
||||
mock.patch.object(manager._job_tracker, "apply_status", side_effect=event),
|
||||
):
|
||||
thread = threading.Thread(target=listen)
|
||||
thread.start()
|
||||
@@ -323,16 +314,16 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager):
|
||||
assert not thread.is_alive()
|
||||
assert cleaned.wait(1)
|
||||
assert len(errors) == 1
|
||||
assert not manager._future_registry
|
||||
assert not manager._active_subscriptions
|
||||
assert not manager._topic_by_subscription_id
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager._job_tracker._active_subscriptions
|
||||
assert not manager._job_tracker._topic_by_subscription_id
|
||||
assert not manager.sub_client.registered_topics
|
||||
assert not manager.sub_client._topic_by_subscription_id
|
||||
|
||||
|
||||
def test_listen_after_completion_does_not_register_callback(manager):
|
||||
future = submit(manager)
|
||||
manager._event_callback(StatusMessage(status="finished"), future)
|
||||
manager._job_tracker.apply_status(StatusMessage(status="finished"), future)
|
||||
with pytest.raises(RuntimeError, match="no longer active"):
|
||||
future.listen("log", lambda _: None)
|
||||
assert not manager.sub_client.registered_topics
|
||||
@@ -377,5 +368,125 @@ def test_unlisten_allows_concurrent_job_cleanup(manager):
|
||||
assert completed_during_removal == [True]
|
||||
assert future.done()
|
||||
assert not future._callback_registry
|
||||
assert not manager._future_registry
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [0, 7])
|
||||
def test_controller_structured_exit_code(manager, code):
|
||||
future = submit(manager)
|
||||
manager.rest_client.get_job.return_value = {
|
||||
"job_state": ["COMPLETED"],
|
||||
"exit_code": {
|
||||
"status": ["SUCCESS" if code == 0 else "ERROR"],
|
||||
"return_code": {"set": True, "number": code},
|
||||
"signal": {"id": {"set": False, "number": 0}},
|
||||
},
|
||||
}
|
||||
manager._check_slurm_job(heartbeat_info(manager, future))
|
||||
assert future.done()
|
||||
assert future.success() == (code == 0)
|
||||
|
||||
|
||||
def test_local_files_are_uploaded_without_mutating_options(manager, tmp_path):
|
||||
workload = tmp_path / "work load $(touch should_not_exist).sh"
|
||||
activation = tmp_path / "it's my activation.sh"
|
||||
workload.write_text("#!/bin/bash\nprintf 'hello'\n")
|
||||
activation.write_text("module load redis\n")
|
||||
options = JobOptions(partition="hour", tasks=2, time_limit_minutes=5)
|
||||
manager.submit_job(workload, activation, working_directory="/remote/work", job_options=options)
|
||||
script, job = manager.rest_client.submit.call_args.args
|
||||
assert "hello" in script
|
||||
assert "module load redis" in script
|
||||
assert str(tmp_path) not in script
|
||||
assert job["hold"] is True
|
||||
assert options == JobOptions(partition="hour", tasks=2, time_limit_minutes=5)
|
||||
assert job["tasks"] == 2
|
||||
assert job["time_limit"] == {"set": True, "number": 5}
|
||||
|
||||
|
||||
def test_submission_failure_does_not_register_job(manager):
|
||||
manager.rest_client.submit.side_effect = SlurmRestError("submission failed")
|
||||
with pytest.raises(SlurmRestError, match="submission failed"):
|
||||
submit(manager)
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager.sub_client.registered_topics
|
||||
manager.rest_client.release.assert_not_called()
|
||||
manager.rest_client.cancel.assert_not_called()
|
||||
|
||||
|
||||
def test_cancellation_failure_keeps_monitoring(manager):
|
||||
future = submit(manager)
|
||||
manager.rest_client.cancel.side_effect = SlurmRestError("cancellation failed")
|
||||
with pytest.raises(SlurmRestError, match="cancellation failed"):
|
||||
future.cancel()
|
||||
assert not future.done()
|
||||
assert future._job_id in manager._job_tracker._future_registry
|
||||
|
||||
|
||||
def test_relative_remote_paths_are_rejected_before_submission(manager):
|
||||
with pytest.raises(ValueError, match="absolute"):
|
||||
manager.submit_job("workload.sh", "activate.sh", working_directory="relative")
|
||||
manager.rest_client.submit.assert_not_called()
|
||||
|
||||
|
||||
def test_public_callbacks_are_ready_before_release(manager):
|
||||
received = threading.Event()
|
||||
|
||||
def release(_job_id):
|
||||
future = next(iter(manager._job_tracker._future_registry.values()))
|
||||
topic = TopicInfo(future._job_id).log
|
||||
assert manager.sub_client._subscription_ready[topic].is_set()
|
||||
manager.sub_client._redis.publish(topic, '{"msg_type":"log","log":"first line"}')
|
||||
assert received.wait(1)
|
||||
|
||||
manager.rest_client.release.side_effect = release
|
||||
manager.submit_job(
|
||||
"examples/print_for_10s.sh",
|
||||
working_directory="/remote/work",
|
||||
callbacks={"log": lambda _: received.set()},
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_status_cleans_up_the_terminal_future(manager):
|
||||
future = submit(manager)
|
||||
future.listen("log", lambda _: None)
|
||||
manager._job_tracker.apply_status(StatusMessage(status="unexpected-status"), future)
|
||||
assert future.done()
|
||||
assert future.status == "error"
|
||||
assert not manager._job_tracker._future_registry
|
||||
assert not manager._job_tracker._active_subscriptions
|
||||
assert not manager._job_tracker._subscriptions_by_topic
|
||||
assert not manager._job_tracker._topic_by_subscription_id
|
||||
assert not manager.sub_client.registered_topics
|
||||
|
||||
|
||||
def test_shutdown_drains_all_received_log_callbacks(manager):
|
||||
future = submit(manager)
|
||||
release = threading.Event()
|
||||
received = []
|
||||
blocking = [manager._job_tracker._executor.submit(release.wait) for _ in range(8)]
|
||||
future.listen("log", lambda message: received.append(message.log))
|
||||
topic = TopicInfo(future._job_id)
|
||||
for number in range(25):
|
||||
manager.sub_client._redis.publish(topic.log, f'{{"msg_type":"log","log":"{number}"}}')
|
||||
manager.sub_client._redis.publish(topic.event, '{"msg_type":"status","status":"finished"}')
|
||||
future.wait(timeout=2)
|
||||
# Release blocked workers only once shutdown reaches the executor. With the
|
||||
# old cancel_futures=True, all 25 accepted callbacks would be discarded here.
|
||||
original = manager._job_tracker._executor.shutdown
|
||||
|
||||
def shutdown(*, wait=True, cancel_futures=False):
|
||||
original(wait=False, cancel_futures=cancel_futures)
|
||||
release.set()
|
||||
if wait:
|
||||
original(wait=True)
|
||||
|
||||
try:
|
||||
with mock.patch.object(manager._job_tracker._executor, "shutdown", side_effect=shutdown):
|
||||
manager.shutdown(timeout=3)
|
||||
finally:
|
||||
release.set()
|
||||
for task in blocking:
|
||||
task.result(timeout=1)
|
||||
assert sorted(received, key=int) == [str(number) for number in range(25)]
|
||||
@@ -0,0 +1,213 @@
|
||||
"""REST wire-format, authentication, and error handling tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def endpoint(tmp_path, monkeypatch):
|
||||
for key in ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
calls = []
|
||||
replies = []
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.respond()
|
||||
|
||||
def do_POST(self):
|
||||
self.respond()
|
||||
|
||||
def do_DELETE(self):
|
||||
self.respond()
|
||||
|
||||
def respond(self):
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
calls.append(
|
||||
(self.command, self.path, dict(self.headers), json.loads(body) if body else None)
|
||||
)
|
||||
status, payload = replies.pop(0)
|
||||
self.send_response(status)
|
||||
if status == 302:
|
||||
self.send_header("Location", "/redirect-target")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
payload if isinstance(payload, bytes) else json.dumps(payload).encode()
|
||||
)
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
env_file = tmp_path / ".env"
|
||||
env_file.write_text(
|
||||
f"SLURM_REST_URL=http://127.0.0.1:{server.server_port}\n"
|
||||
"SLURM_REST_API_VERSION=v0.0.44\nSLURM_JWT=test-secret\n"
|
||||
)
|
||||
try:
|
||||
yield env_file, calls, replies
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
assert not thread.is_alive()
|
||||
|
||||
|
||||
def test_submit_release_cancel_and_accounting_wire_format(endpoint):
|
||||
env_file, calls, replies = endpoint
|
||||
replies.extend(
|
||||
[
|
||||
(200, {"job_id": 123, "errors": []}),
|
||||
(200, {"errors": []}),
|
||||
(200, {"jobs": [{"job_state": ["RUNNING"]}]}),
|
||||
(200, {"jobs": [{"state": {"current": ["COMPLETED"]}}]}),
|
||||
(200, {"errors": []}),
|
||||
]
|
||||
)
|
||||
client = SlurmRestClient(env_file)
|
||||
job = {
|
||||
"current_working_directory": "/remote/work",
|
||||
"environment": ["PATH=/usr/bin"],
|
||||
"hold": True,
|
||||
}
|
||||
assert client.submit("#!/bin/bash\necho hello", job)["job_id"] == 123
|
||||
client.release("123")
|
||||
assert client.get_job("123")["job_state"] == ["RUNNING"]
|
||||
assert client.get_job("123", accounting=True)["state"]["current"] == ["COMPLETED"]
|
||||
client.cancel("123")
|
||||
assert [(method, path) for method, path, _, _ in calls] == [
|
||||
("POST", "/slurm/v0.0.44/job/submit"),
|
||||
("POST", "/slurm/v0.0.44/job/123"),
|
||||
("GET", "/slurm/v0.0.44/job/123"),
|
||||
("GET", "/slurmdb/v0.0.44/job/123"),
|
||||
("DELETE", "/slurm/v0.0.44/job/123"),
|
||||
]
|
||||
assert calls[0][3] == {"script": "#!/bin/bash\necho hello", "job": job}
|
||||
assert calls[1][3] == {"hold": False}
|
||||
assert calls[0][2]["X-Slurm-User-Token"] == "test-secret"
|
||||
assert "test-secret" not in json.dumps(calls[0][3])
|
||||
|
||||
|
||||
def test_token_rotation_and_environment_override(endpoint, monkeypatch):
|
||||
env_file, calls, replies = endpoint
|
||||
replies.extend([(200, {}), (200, {})])
|
||||
client = SlurmRestClient(env_file)
|
||||
env_file.write_text(env_file.read_text().replace("test-secret", "replacement-token"))
|
||||
client.cancel("1")
|
||||
monkeypatch.setenv("SLURM_JWT", "environment-token")
|
||||
monkeypatch.setenv("SLURM_USER", "alice")
|
||||
client.cancel("2")
|
||||
assert calls[0][2]["X-Slurm-User-Token"] == "replacement-token"
|
||||
assert calls[1][2]["X-Slurm-User-Token"] == "environment-token"
|
||||
assert calls[1][2]["X-Slurm-User-Name"] == "alice"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status,payload",
|
||||
[
|
||||
(401, b"test-secret"),
|
||||
(500, b"test-secret"),
|
||||
(200, {"errors": [{"description": "test-secret", "error_number": 1}]}),
|
||||
(200, b"bad JSON test-secret"),
|
||||
(200, []),
|
||||
(302, b"redirect"),
|
||||
],
|
||||
)
|
||||
def test_errors_are_raised_without_exposing_credentials(endpoint, status, payload):
|
||||
env_file, calls, replies = endpoint
|
||||
replies.append((status, payload))
|
||||
client = SlurmRestClient(env_file)
|
||||
with pytest.raises(SlurmRestError) as error:
|
||||
client.cancel("123")
|
||||
assert "test-secret" not in str(error.value)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reply",
|
||||
[
|
||||
(404, {}),
|
||||
(500, {"errors": [{"error_number": 2017}]}),
|
||||
(200, {"jobs": []}),
|
||||
(200, {"errors": [{"error_number": 2017}]}),
|
||||
],
|
||||
)
|
||||
def test_missing_job_is_not_a_terminal_failure(endpoint, reply):
|
||||
env_file, _, replies = endpoint
|
||||
replies.append(reply)
|
||||
assert SlurmRestClient(env_file).get_job("123") is None
|
||||
|
||||
|
||||
def test_missing_token_fails_before_any_request(endpoint):
|
||||
env_file, calls, _ = endpoint
|
||||
env_file.write_text(env_file.read_text().replace("test-secret", ""))
|
||||
with pytest.raises(ValueError, match="SLURM_JWT"):
|
||||
SlurmRestClient(env_file)
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_submission_requires_job_id(endpoint):
|
||||
env_file, _, replies = endpoint
|
||||
replies.append((200, {"errors": []}))
|
||||
with pytest.raises(SlurmRestError, match="job ID"):
|
||||
SlurmRestClient(env_file).submit("#!/bin/bash", {})
|
||||
|
||||
|
||||
def test_manager_submission_and_completion_over_http(endpoint):
|
||||
from functools import partial
|
||||
from unittest import mock
|
||||
|
||||
import fakeredis
|
||||
|
||||
from slurm_manager.job_tracker import TopicInfo
|
||||
from slurm_manager.redis_sub_client import RedisSubClient
|
||||
from slurm_manager.slurm_manager import SlurmManager
|
||||
|
||||
env_file, calls, replies = endpoint
|
||||
replies.extend([(200, {"job_id": 123}), (200, {})])
|
||||
redis = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True))
|
||||
with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=redis):
|
||||
manager = SlurmManager(env_file=env_file)
|
||||
try:
|
||||
future = manager.submit_job("examples/print_for_10s.sh", working_directory="/remote/work")
|
||||
assert calls[0][3]["job"]["hold"] is True
|
||||
assert calls[1][3] == {"hold": False}
|
||||
topics = TopicInfo(future._job_id)
|
||||
assert redis._subscription_ready[topics.event].is_set()
|
||||
redis._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}')
|
||||
future.wait(timeout=1)
|
||||
assert future.success()
|
||||
finally:
|
||||
manager.shutdown(timeout=3)
|
||||
assert not manager._heartbeat_monitoring_thread.is_alive()
|
||||
assert not redis._listener_thread.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_response", [False, True])
|
||||
def test_truncated_responses_raise_a_transport_error(endpoint, error_response):
|
||||
from http.client import IncompleteRead
|
||||
from unittest import mock
|
||||
from urllib.error import HTTPError
|
||||
|
||||
env_file, _, _ = endpoint
|
||||
client = SlurmRestClient(env_file)
|
||||
response = mock.MagicMock()
|
||||
response.read.side_effect = IncompleteRead(b"partial", 100)
|
||||
if error_response:
|
||||
error = HTTPError("http://local/job", 500, "Server error", {}, response)
|
||||
patch = mock.patch.object(client._opener, "open", side_effect=error)
|
||||
else:
|
||||
opened = mock.MagicMock()
|
||||
opened.__enter__.return_value = response
|
||||
patch = mock.patch.object(client._opener, "open", return_value=opened)
|
||||
with patch, pytest.raises(SlurmRestError):
|
||||
client.get_job("123")
|
||||
Reference in new issue
Block a user