diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..54bdd76 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +SLURM_REST_URL=https://ra-slurmapi-001.psi.ch +SLURM_REST_API_VERSION=v0.0.44 +SLURM_JWT= +# Optional: only set if required by your Slurm authentication configuration. +# SLURM_USER=your_username diff --git a/examples/print_for_10s.sh b/examples/print_for_10s.sh new file mode 100644 index 0000000..9615bc6 --- /dev/null +++ b/examples/print_for_10s.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for second in {1..10}; do + printf 'stdout-test %02d/10\n' "$second" + sleep 1 +done diff --git a/pyproject.toml b/pyproject.toml index 045b899..25da5ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,9 @@ classifiers = [ ] dependencies = [ - "hiredis~=3.0", "pydantic~=2.12.0", - "redis~=6.2", + "redis[hiredis]>=6.2, <9.0", + "python-dotenv~=1.2", ] [project.optional-dependencies] diff --git a/slurm_manager/event_messages.py b/slurm_manager/event_messages.py index 87ea800..150cb6d 100644 --- a/slurm_manager/event_messages.py +++ b/slurm_manager/event_messages.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias from pydantic import BaseModel, Field, TypeAdapter @@ -49,8 +49,18 @@ SlurmMessage: TypeAlias = Annotated[ SLURM_MESSAGE_ADAPTER = TypeAdapter(SlurmMessage) -def parse_slurm_message(payload: str | dict) -> SlurmMessage: - """Parse a JSON payload into the appropriate slurm message model.""" +def parse_slurm_message(payload: str | dict[str, Any]) -> SlurmMessage: + """Validate a JSON or dictionary payload as a discriminated Slurm message. + + Args: + payload (str | dict[str, Any]): JSON-compatible request body or incoming message payload. + + Returns: + SlurmMessage: Heartbeat, status, or log message selected by msg_type. + + Raises: + ValueError: The JSON or message schema is invalid. + """ if isinstance(payload, str): payload = json.loads(payload) return SLURM_MESSAGE_ADAPTER.validate_python(payload) diff --git a/slurm_manager/job_future.py b/slurm_manager/job_future.py index a75a845..1b97215 100644 --- a/slurm_manager/job_future.py +++ b/slurm_manager/job_future.py @@ -44,97 +44,133 @@ class RegisteredCallback: class JobFuture: """ A future-like object that represents a Slurm job. It allows users to check the status of the job, - retrieve the job's result, and register callbacks for events (e.g., 'event', 'logs', 'heartbeat'). + wait for completion, and register callbacks for status, log, and heartbeat events. """ def __init__( - self, - job_id: str, - manager: SlurmManager, - slurm_job_id: str, - registry: dict | None = None, - name: str | None = None, - ): + self, job_id: str, manager: SlurmManager, slurm_job_id: str, name: str | None = None + ) -> None: + """Create a future for one job tracked by its submitting manager. + + Args: + job_id (str): Opaque manager identifier used in Redis topic names. + manager (SlurmManager): Manager that owns the job and handles subscriptions + and cancellation. + slurm_job_id (str): Numeric scheduler identifier returned by Slurm. + name (str | None): Display-name prefix; None uses Job, followed by the + scheduler ID. + """ self._job_id = job_id self._manager = manager - self._slurm_job_id, _, cluster = slurm_job_id.strip().partition(";") - self._slurm_cluster = cluster or None - self._registry = registry if registry is not None else {} - self._future = Future() + self._slurm_job_id = slurm_job_id.strip() + self._future: Future[None] = Future() if name is None: name = "Job" self.name = f"{name}-{self._slurm_job_id}" self._status: str = JobStatus.PENDING.value - self._shutdown_event = threading.Event() self._lock = threading.RLock() self._callback_registry: dict[str, list[RegisteredCallback]] = {} - ##################### - ## Public API - ##################### + ################################################ + # Public Properties # + ################################################ @property def slurm_job_id(self) -> str: - """The numeric SLURM job ID, without an optional cluster suffix.""" + """Return the numeric scheduler job ID on the configured REST endpoint. + + Returns: + str: Scheduler job ID. + """ return self._slurm_job_id - @property - def slurm_cluster(self) -> str | None: - """The cluster returned by sbatch, or None for the default cluster.""" - return self._slurm_cluster - @property def status(self) -> str | None: - """The current status of the job.""" + """Return the most recently accepted lifecycle status. + + Returns: + str | None: Pending, running, finished, error, or cancelled status. + """ return self._status - def cancel(self): - """Cancel the job.""" - self._manager.cancel_job( - slurm_job_id=self.slurm_job_id, job_id=self._job_id, slurm_cluster=self.slurm_cluster - ) + ################################################ + # Public Job Lifecycle # + ################################################ + + def done(self) -> bool: + """Check whether the job reached any terminal state. + + Returns: + bool: True for a finished, failed, or cancelled job. + """ + return self._future.done() + + def success(self) -> bool: + """Check whether the job reached a successful terminal state. + + Returns: + bool: True only when the job finished without an exception. + """ + return self._future.done() and self._future.exception() is None def exception(self) -> Exception | None: - """If the job has finished with an error, return the exception. Otherwise, return None.""" + """Return the recorded terminal exception without waiting. + + Returns: + Exception | None: Failure or cancellation exception, or None if pending or + successful. + """ if self._future.done(): return self._future.exception() return None - def success(self) -> bool: - """Return True if the job finished successfully, False if it finished with an error or is not done yet.""" - return self._future.done() and self._future.exception() is None - - def done(self) -> bool: - """Return True if the job has finished (either successfully or with an error), False otherwise.""" - return self._future.done() - def wait(self, timeout: float | None = None) -> None: - """ - Wait for the job to finish, with an optional timeout in seconds. Raises TimeoutError if the job - does not finish within the specified timeout. + """Wait for the job and propagate its failure or cancellation. Args: - timeout: Maximum time to wait for the job to finish, in seconds. If None, wait indefinitely. + timeout (float | None): Maximum wait in seconds; None waits indefinitely + without cancelling the job. + + Raises: + TimeoutError: The job does not reach a terminal state before the timeout. + Exception: The job fails or is cancelled. + + Notes: + A timeout does not cancel the job. Callback execution may still be pending + when this method returns; manager shutdown drains accepted callbacks. """ self._future.result(timeout=timeout) + def cancel(self) -> None: + """Request cancellation through the owning manager. + + Raises: + SlurmRestError: Slurm rejects cancellation or the request fails. + """ + self._manager.cancel_job(slurm_job_id=self.slurm_job_id, job_id=self._job_id) + + ################################################ + # Public Event Subscriptions # + ################################################ + def listen( self, event_type: Literal["status", "log", "heartbeat"], callback: Callable[[SlurmMessage], None], ) -> str: - """ - Register a callback to listen for events of the specified type ('status', 'log', or 'heartbeat'). - Returns a callback ID that can be used to unregister. + """Register an asynchronous callback for a job event. Args: - event_type: The type of event to listen for ('status', 'log', or 'heartbeat'). - callback: A function that takes a SlurmMessage object as input. The exact message object - depends on the event type. Please check event_messages.py for the specific - message class based on the event. + event_type (Literal['status', 'log', 'heartbeat']): Event to receive: log, + status, or heartbeat. + callback (Callable[[SlurmMessage], None]): Callable receiving the parsed + Slurm message. Returns: - A unique callback ID that can be used to unregister the callback later. + str: Callback ID accepted by unlisten(). + + Raises: + RuntimeError: The job has finished or is no longer tracked. """ cb_id = self._manager._add_subscription( job_id=self._job_id, key=event_type, callback=callback @@ -148,61 +184,53 @@ class JobFuture: return cb.id def unlisten(self, callback_id: str) -> None: - """ - Unregister a previously registered callback using its callback ID. + """Remove a previously registered callback by ID. Args: - callback_id: The unique ID of the callback to unregister, as returned by listen() + callback_id (str): Subscription ID, or IDs where accepted, returned during + registration. + """ + # Manager cleanup may acquire this future's lock while holding its own. + # Remove local ownership before acquiring the manager lock. + if self._remove_callback(callback_id): + self._manager.remove_subscription_by_id(callback_id=callback_id) + + ################################################ + # Internal Callback Management # + ################################################ + + def _remove_callback(self, callback_id: str) -> bool: + """Remove local callback ownership without acquiring the manager lock. + + Args: + callback_id (str): ID of the callback to remove. + + Returns: + bool: True if a registered callback was removed. """ - removed = False with self._lock: for event, callbacks in self._callback_registry.items(): for cb in callbacks: - if cb.id == callback_id: - callbacks.remove(cb) - if not callbacks: - self._callback_registry.pop(event, None) - removed = True - break - if removed: - break - # Manager cleanup may acquire this future's lock while holding its own. - # Release the future lock before acquiring the manager lock. - if removed: - self._manager.remove_subscription_by_id(callback_id=callback_id) + if cb.id != callback_id: + continue + callbacks.remove(cb) + if not callbacks: + self._callback_registry.pop(event) + return True + return False - ################### - ## Internal Methods - ## Not meant to be called by users - ################### - - def _set_finished(self): - self._shutdown_event.set() - if self._future.done(): - logger.warning( - "Job %s is already marked as done. Ignoring _set_finished call.", self.name - ) - return - self._future.set_result(None) - - def _set_exception(self, status, exc_info=None): - if self._future.done(): - logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name) - return - self._shutdown_event.set() - # Here you could log the exception info or store it for later retrieval - if exc_info: - info = f"{self.name} failed with status {status}. Exception info: {exc_info}" - else: - info = f"{self.name} failed with status {status}." - # You could log the info here, e.g., using logging.error(info) - self._future.set_exception(Exception(info)) - - def _raise_if_failed(self) -> None: - if self._future.done() and self._future.exception() is not None: - raise self._future.exception() + ################################################ + # Internal State Updates # + ################################################ def _update_status(self, status: str, exc_info: str | None = None) -> None: + """Apply a lifecycle update under the future lock, ignoring late updates. + + Args: + status (str): Lifecycle status; a terminal state completes the underlying + future. + exc_info (str | None): Optional worker-provided failure details. + """ with self._lock: if self._future.done(): return @@ -211,3 +239,29 @@ class JobFuture: self._set_finished() elif status in (JobStatus.CANCELLED, JobStatus.ERROR): self._set_exception(status, exc_info) + + def _set_finished(self) -> None: + """Complete the underlying future successfully unless it is already done.""" + if self._future.done(): + logger.warning( + "Job %s is already marked as done. Ignoring _set_finished call.", self.name + ) + return + self._future.set_result(None) + + def _set_exception(self, status: str, exc_info: str | None = None) -> None: + """Record a terminal job failure with optional worker details. + + Args: + status (str): Lifecycle status; a terminal state completes the underlying + future. + exc_info (str | None): Optional worker-provided failure details. + """ + if self._future.done(): + logger.warning("%s is already marked as done. Ignoring _set_exception call.", self.name) + return + if exc_info: + info = f"{self.name} failed with status {status}. Exception info: {exc_info}" + else: + info = f"{self.name} failed with status {status}." + self._future.set_exception(Exception(info)) diff --git a/slurm_manager/job_options.py b/slurm_manager/job_options.py new file mode 100644 index 0000000..0e10531 --- /dev/null +++ b/slurm_manager/job_options.py @@ -0,0 +1,104 @@ +"""Validated user-facing Slurm job options.""" + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints + +NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + +class JobOptions(BaseModel): + """Resources and output settings for one job. + + An omitted partition uses the cluster default; an omitted time limit uses + the selected partition's default. Time limits + are wall-clock minutes, independently of JobFuture.wait()'s timeout in seconds. + Output paths refer to the compute node and may use Slurm's %j job-ID pattern. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + name: NonEmptyString | None = Field( + default=None, description="Job name shown in Slurm. Omit to use the scheduler default." + ) + partition: Literal["hour", "day", "week", "gpu"] | None = Field( + default=None, + description=( + "RA partition: hour (default/max 1 hour), day (default 4 hours, max 24 hours), " + "week (default 2 days, max 8 days), or gpu (default 1 hour, max 7 days). " + "Omit to use the cluster default, currently hour." + ), + ) + account: NonEmptyString | None = Field( + default=None, + description=( + "Slurm accounting/project account; omit to use your default account association. " + "The account must also be allowed by the selected partition: hour/day allow " + "user and proprietary; gpu allows user and gpu-csaxs; week denies gpu-csaxs " + "and tomcat. These partition rules do not establish your account membership." + ), + ) + qos: NonEmptyString | None = Field( + default=None, + description=( + "Quality of Service controlling scheduling priority and resource limits. " + "Must be allowed by both your association and the selected partition. " + "Omit to use your association's default job QoS. The gpu partition allows " + "gpu-default, gpu-day, gpu-hour, and gpu-csaxs, and additionally applies " + "partition QoS gpu-default. Partition QoS is separate from requested job QoS." + ), + ) + tasks: int = Field( + default=1, + ge=1, + strict=True, + description=( + "Number of Slurm tasks requested for the allocation; defaults to one. " + "The worker executes the workload script once, without launching MPI ranks. " + "This does not specify CPUs per task." + ), + ) + time_limit_minutes: int | None = Field( + default=None, + ge=1, + strict=True, + description=( + "Maximum job wall-clock runtime in minutes. Slurm terminates the job at this limit. " + "Omit to use the selected partition's default: hour 60, day 240, week 2880, " + "gpu 60 minutes. Partition maxima are respectively 60, 1440, 11520, and " + "10080 minutes; account/QoS limits may impose additional restrictions. " + "Separate from JobFuture.wait(timeout), which is measured in seconds." + ), + ) + standard_output: NonEmptyString | None = Field( + default=None, + description=( + "Compute-node file path for stdout, in addition to Redis forwarding. " + "Supports Slurm patterns such as %j for the job ID. Relative paths use the job's " + "working directory; the parent directory must exist. Omit for Slurm's default output." + ), + ) + standard_error: NonEmptyString | None = Field( + default=None, + description=( + "Compute-node file path for stderr, in addition to Redis forwarding. " + "Supports %j for the job ID. Relative paths use the job's working directory; " + "the parent directory must exist. Omit to use Slurm's default stderr handling." + ), + ) + + ################################################ + # Public Serialization # + ################################################ + + def to_rest(self) -> dict[str, Any]: + """Serialize validated options into slurmrestd job-description fields. + + Returns: + dict[str, Any]: REST options with unset values omitted and runtime expressed + as an optional integer. + """ + options = self.model_dump(exclude={"time_limit_minutes"}, exclude_none=True) + if self.time_limit_minutes is not None: + options["time_limit"] = {"set": True, "number": self.time_limit_minutes} + return options diff --git a/slurm_manager/job_submission/__init__.py b/slurm_manager/job_submission/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/slurm_manager/job_submission/redis_worker.sh b/slurm_manager/job_submission/redis_worker.sh deleted file mode 100755 index 7c7a9a5..0000000 --- a/slurm_manager/job_submission/redis_worker.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -module load redis - -fifo=$1 -hash_name=$2 - -exec {fifo_fd}<"$fifo" - -redis_host="bec-slurm.psi.ch" -status="finished" - -publish_payload() { - local topic=$1 - local payload=$2 - redis-cli -h "$redis_host" PUBLISH "$topic" "$payload" -} - -timestamp=$(date +%s) -publish_payload \ - "info/$hash_name/status" \ - "$(jq -cn --arg status "running" '{msg_type:"status", status:$status}')" - -heartbeat() { - while :; do - local timestamp - timestamp=$(date +%s) - publish_payload \ - "info/$hash_name/heartbeat" \ - "$(jq -cn --argjson timestamp "$timestamp" '{msg_type:"heartbeat", timestamp:$timestamp}')" \ - || true - sleep 1 - done -} - -heartbeat & -hb_pid=$! - -cleanup() { - if kill -0 "$hb_pid" 2>/dev/null; then - kill "$hb_pid" 2>/dev/null || true - wait "$hb_pid" 2>/dev/null || true - fi -} -trap cleanup EXIT - -while IFS= read -r line <&"$fifo_fd"; do - case "$line" in - stop) - status="finished" - break - ;; - error:*) - status="error:${line#error:}" - break - ;; - log:*) - msg="${line#log:}" - publish_payload \ - "info/$hash_name/log" \ - "$(jq -cn --arg log "$msg" '{msg_type:"log", log:$log}')" - ;; - *) - : - ;; - esac -done - -publish_payload \ - "info/$hash_name/status" \ - "$(jq -cn --arg status "$status" '{msg_type:"status", status:$status}')" diff --git a/slurm_manager/job_submission/wrapper.sh b/slurm_manager/job_submission/wrapper.sh deleted file mode 100755 index fe3ee2b..0000000 --- a/slurm_manager/job_submission/wrapper.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -o pipefail - -env_path=$1 -script_path=$2 -hash_name=$3 -have_out_stream=$4 -have_out_file=$5 -redis_script=$6 - - -fifo=$(mktemp -u) -mkfifo "$fifo" - -"$redis_script" "$fifo" "$hash_name" >/dev/null & -worker_pid=$! - -trap 'rm -f "$fifo"' EXIT - -exec {fifo_fd}>"$fifo" - -source "$env_path" - -{ - stdbuf -o0 -e0 "$script_path" 2>&1 -} | while IFS= read -r line; do - - if [ "$have_out_stream" -eq 1 ]; then - printf 'log:%s\n' "$line" >&"$fifo_fd" - fi - - if [ "$have_out_file" -eq 1 ]; then - printf '%s\n' "$line" - fi - -done - - -exit_code=${PIPESTATUS[0]} - -if [ "$exit_code" -ne 0 ]; then - printf 'error:%s\n' "$exit_code" >&"$fifo_fd" -fi - -printf 'stop\n' >&"$fifo_fd" -exec {fifo_fd}>&- - -wait $worker_pid - -exit $exit_code diff --git a/slurm_manager/job_tracker.py b/slurm_manager/job_tracker.py new file mode 100644 index 0000000..6a58857 --- /dev/null +++ b/slurm_manager/job_tracker.py @@ -0,0 +1,612 @@ +"""Own job futures, Redis subscriptions, and callback dispatch.""" + +from __future__ import annotations + +import threading +import time +from collections import defaultdict +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime +from functools import partial, wraps +from logging import getLogger +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict + +from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage +from slurm_manager.job_future import JobFuture, JobStatus +from slurm_manager.redis_sub_client import RedisSubClient + +logger = getLogger(__name__) + + +@dataclass(frozen=True, slots=True) +class TopicInfo: + """Helper for topic specification within Redis bec-slurm pub/sub interface.""" + + job_id: str + prefix: str = "info" + + ################################################ + # Public Properties # + ################################################ + + @property + def log(self) -> str: + """Return the channel carrying stdout and stderr log messages. + + Returns: + str: Job log channel. + """ + return self.topic("log") + + @property + def event(self) -> str: + """Return the channel carrying lifecycle status messages. + + Returns: + str: Job status channel. + """ + return self.topic("status") + + @property + def heartbeat(self) -> str: + """Return the channel carrying worker heartbeats. + + Returns: + str: Job heartbeat channel. + """ + return self.topic("heartbeat") + + ################################################ + # Public Topic Lookup # + ################################################ + + def topic(self, v: str) -> str: + """Build a Redis topic name for a job event. + + Args: + v (str): Event suffix appended to this job's Redis topic prefix. + + Returns: + str: Redis channel in prefix/job_id/event form. + """ + return f"{self.prefix}/{self.job_id}/{v}" + + +class SubscriptionInfo(BaseModel): + """Helper for slurm job information and future tracking.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + topic: str + job_id: str + slurm_job_id: str | None = None + callback: Callable[[SlurmMessage], None] | None = None + callback_id: str | None = None + last_heartbeat: float | None = None + heartbeat_received: bool = False + last_scheduler_check: float | None = None + + +HEARTBEAT_TIMEOUT = 10 # seconds + + +class JobTracker: + """Coordinate job state and callback ownership under one lock.""" + + def __init__(self, sub_client: RedisSubClient) -> None: + """Create tracking state and the user callback executor. + + Args: + sub_client (RedisSubClient): Transport used for job subscriptions; the + tracker closes it during shutdown. + """ + self._sub_client = sub_client + self._future_registry: dict[str, JobFuture] = {} + self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict) + self._topic_by_subscription_id: dict[str, str] = {} + self._active_subscriptions: dict[str, set[str]] = defaultdict(set) + self._lock = threading.RLock() + self._executor = ThreadPoolExecutor( + max_workers=8, thread_name_prefix="SlurmManagerCallback" + ) + + ################################################ + # Public Job Tracking # + ################################################ + + def track(self, future: JobFuture) -> None: + """Register a submitted job and attach internal monitoring callbacks. + + Args: + future (JobFuture): Submitted job whose state and subscriptions are owned. + + Notes: + The caller must remove tracking and cancel the held job if setup fails. + """ + with self._lock: + self._future_registry[future._job_id] = future + self._subscribe_to_internal_topics(future, TopicInfo(future._job_id)) + + def get_future(self, job_id: str) -> JobFuture | None: + """Look up a tracked future under the state lock. + + Args: + job_id (str): Opaque manager job identifier. + + Returns: + JobFuture | None: Tracked future, or None after cleanup. + """ + with self._lock: + return self._future_registry.get(job_id) + + def wait_until_ready(self, job_id: str) -> None: + """Wait for Redis to acknowledge all subscriptions installed for a job. + + Args: + job_id (str): Opaque manager job identifier. + + Raises: + TimeoutError: Redis does not acknowledge the subscriptions in time. + """ + with self._lock: + topics = list(self._active_subscriptions.get(job_id, set())) + self._sub_client.wait_until_subscribed(topics) + + def begin_scheduler_check(self, job_id: str) -> JobFuture | None: + """Record a scheduler check for an active job before performing network I/O. + + Args: + job_id (str): Opaque manager job identifier. + + Returns: + JobFuture | None: Active future, or None if it has completed or was removed. + """ + with self._lock: + future = self._future_registry.get(job_id) + if future is None or future.done(): + return None + topic = TopicInfo(job_id).heartbeat + for info in self._subscriptions_by_topic.get(topic, {}).values(): + if info.last_heartbeat is not None: + info.last_scheduler_check = time.time() + return future + + def apply_status(self, message: StatusMessage, future: JobFuture) -> None: + """Apply a worker status and clean up terminal job subscriptions. + + Args: + message (StatusMessage): Raw Redis payload or parsed worker message, as + specified by the annotation. + future (JobFuture): Tracked future whose lifecycle or subscriptions are + being managed. + + Notes: + Statuses use status[:detail]. Unknown statuses fail the job and follow the + same cleanup path as an explicit error. + """ + info = message.status.split(":", 1) + status = info[0] + try: # Catch irregular formatted status messages to avoid crashing the callback loop. + status = JobStatus(status) + except ValueError: + logger.warning("Received unknown job status '%s' for job %s", status, future.name) + future._update_status( + "error", + f"Received unknown job status '{status}' from message: {message.model_dump()}", + ) + self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True) + return + exc_info = info[1] if len(info) > 1 else None + future._update_status(status, exc_info) + # Clean up subscriptions for finished job + if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]: + self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True) + + def get_missing_heartbeats(self) -> list[SubscriptionInfo]: + """Select stale heartbeat subscriptions whose scheduler check is due. + + Returns: + list[SubscriptionInfo]: Copies of internal heartbeat subscriptions needing + a Slurm query. Mutating a copy does not change tracking state. + """ + missing = [] + now = time.time() + with self._lock: + for job_id, topics in self._active_subscriptions.items(): + topic = TopicInfo(job_id=job_id).heartbeat + if topic not in topics: + continue + for sub_info in self._subscriptions_by_topic.get(topic, {}).values(): + if not sub_info.last_heartbeat: + continue + if now - sub_info.last_heartbeat <= HEARTBEAT_TIMEOUT: + continue + last_check = sub_info.last_scheduler_check + if last_check is not None and now - last_check <= HEARTBEAT_TIMEOUT: + continue + missing.append(sub_info.model_copy()) + return missing + + ################################################ + # Public Event Subscriptions # + ################################################ + + def listen( + self, + event_type: Literal["heartbeat", "log", "status"], + callback: Callable[[SlurmMessage], None], + job_id: str | None = None, + ) -> list[str]: + """Attach a callback to one tracked job or all currently tracked jobs. + + Args: + event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log, + status, or heartbeat. + callback (Callable[[SlurmMessage], None]): Callable receiving the parsed + Slurm message. + job_id (str | None): Opaque manager identifier used in Redis topic names. + + Returns: + list[str]: Subscription IDs accepted by unlisten(). + + Raises: + KeyError: The explicitly requested job ID is not tracked. + RuntimeError: A selected job has already completed. + + Notes: + With job_id=None, registration applies to the current jobs, not future + submissions. Callbacks run in the tracker executor. + """ + with self._lock: + job_futures = ( + [self._future_registry[job_id]] if job_id else list(self._future_registry.values()) + ) + callback_ids = [] + for future in job_futures: + callback_ids.append(future.listen(event_type=event_type, callback=callback)) + + return callback_ids + + def add_subscription( + self, + job_id: str, + key: Literal["heartbeat", "log", "status"], + callback: Callable[[SlurmMessage], None], + callback_kwargs: dict[str, Any] | None = None, + ) -> str: + """Atomically register a Redis callback and its manager ownership. + + Args: + job_id (str): Opaque manager identifier used in Redis topic names. + key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or + heartbeat. + callback (Callable[[SlurmMessage], None]): Callable receiving the parsed + Slurm message. + callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed + to the user callback. + + Returns: + str: Unique callback subscription ID. + + Raises: + RuntimeError: The job is absent or finishes during registration. + """ + + topic_str = TopicInfo(job_id=job_id).topic(key) + + @wraps(callback) + def wrapped_callback( + message: SlurmMessage, + executor: ThreadPoolExecutor, + job_id: str, + slurm_job_id: str, + callback_kwargs: dict[str, Any] | None = None, + ) -> None: + """Enrich a parsed worker message and dispatch its registered handler. + + Args: + message (SlurmMessage): Raw Redis payload or parsed worker message, as + specified by the annotation. + executor (ThreadPoolExecutor): Callback executor that runs user code + away from the listener thread. + job_id (str): Opaque manager identifier used in Redis topic names. + slurm_job_id (str): Numeric scheduler identifier returned by Slurm. + callback_kwargs (dict[str, Any] | None): Additional keyword arguments + passed to the user callback. + """ + message.metadata.update( + { + "msg_received": datetime.now().isoformat(), + "job_id": job_id, + "slurm_job_id": slurm_job_id, + } + ) + executor.submit(callback, message, **(callback_kwargs or {})) + + # Completion removes subscriptions under this same lock. Keep ownership + # registration atomic with cleanup, and undo Redis registration if the + # future became terminal while subscribe() was in progress. + with self._lock: + future = self._future_registry.get(job_id) + if future is None or future.done(): + raise RuntimeError(f"Job {job_id} is no longer active") + cb_id = self._sub_client.subscribe( + topic_str, + partial( + wrapped_callback, + executor=self._executor, + callback_kwargs=callback_kwargs, + job_id=job_id, + slurm_job_id=future.slurm_job_id, + ), + ) + + if self._future_registry.get(job_id) is not future or future.done(): + self._sub_client.unsubscribe(cb_id) + raise RuntimeError(f"Job {job_id} finished during subscription registration") + + sub_info = SubscriptionInfo( + job_id=job_id, + topic=topic_str, + slurm_job_id=future.slurm_job_id, + callback=wrapped_callback, + last_heartbeat=None, + callback_id=cb_id, + heartbeat_received=False, + ) + self._register_subscription_for_job(sub_info=sub_info) + return cb_id + + def remove_subscription_by_id(self, callback_id: str | list[str]) -> None: + """Remove the selected callbacks while preserving other topic listeners. + + Args: + callback_id (str | list[str]): Subscription ID, or IDs where accepted, + returned during registration. + """ + if isinstance(callback_id, str): + callback_id = [callback_id] + with self._lock: + for cb_id in callback_id: + topic = self._topic_by_subscription_id.pop(cb_id, None) + if topic is None: + continue + + callbacks = self._subscriptions_by_topic.get(topic, {}) + sub_info = callbacks.pop(cb_id, None) + self._sub_client.unsubscribe(cb_id) + + # Other callbacks, including heartbeat monitoring, still own this topic. + if callbacks: + continue + + self._subscriptions_by_topic.pop(topic, None) + if sub_info is None: + continue + + topics = self._active_subscriptions.get(sub_info.job_id) + if topics is not None: + topics.discard(topic) + + def remove_subscription_by_topic(self, topic: str) -> None: + """Remove all callbacks for one topic and unsubscribe it from Redis. + + Args: + topic (str): Redis channel whose callback registrations are selected. + """ + with self._lock: + callbacks = self._subscriptions_by_topic.pop(topic, {}) + for callback_id, sub_info in callbacks.items(): + self._topic_by_subscription_id.pop(callback_id, None) + job_id = sub_info.job_id + self._active_subscriptions[job_id].discard(topic) + + # Remove the topic subscription from Redis + self._sub_client.unsubscribe_topic(topic) + + def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None: + """Remove job subscriptions and discard the tracked future. + + Args: + job_id (str): Opaque manager identifier used in Redis topic names. + skip_event_callback (bool): If true, remove monitoring without synthesizing + a failure update. + + Notes: + Unless skip_event_callback is true, an unfinished future is marked failed. + This only removes local monitoring; it does not cancel the Slurm allocation. + """ + with self._lock: + topics = self._active_subscriptions.pop(job_id, set()) + for topic in topics: + callbacks = self._subscriptions_by_topic.pop(topic, {}) + for callback_id in callbacks: + self._topic_by_subscription_id.pop(callback_id, None) + self._sub_client.unsubscribe_topic(topic) + future = self._future_registry.get(job_id) + if future is None: + return + # This callback will trigger a cleanup. + if not skip_event_callback: + self.apply_status( + StatusMessage( + status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}", + metadata={ + "msg_received": datetime.now().isoformat(), + "job_id": future._job_id, + "slurm_job_id": future.slurm_job_id, + }, + ), + future, + ) + self._remove_job_future(future._job_id) + + ################################################ + # Public Lifecycle # + ################################################ + + def shutdown(self, timeout: float | None = 1.0) -> None: + """Stop Redis delivery and drain accepted user callbacks. + + Args: + timeout (float | None): Redis listener join timeout in seconds; None waits + indefinitely. Callback draining has no timeout. + + Notes: + Stop scheduler monitoring before calling this method. Remote jobs are + not cancelled and tracked futures are not marked complete. + """ + with self._lock: + self._subscriptions_by_topic.clear() + self._topic_by_subscription_id.clear() + self._sub_client.shutdown(timeout=timeout) + self._executor.shutdown(wait=True, cancel_futures=False) + + ################################################ + # Internal Subscriptions # + ################################################ + + def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None: + """Attach the manager status and heartbeat callbacks before release. + + Args: + future (JobFuture): Tracked future whose lifecycle or subscriptions are + being managed. + topic_info (TopicInfo): Redis topic names for the tracked job. + """ + self._subscribe_to_heartbeat(topic_info, future) + self._subscribe_to_event(topic_info, future) + + def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None: + """Register the internal heartbeat callback and its initial timestamp. + + Args: + topic_info (TopicInfo): Redis topic names for the tracked job. + future (JobFuture): Tracked future whose lifecycle or subscriptions are + being managed. + """ + sub_info = SubscriptionInfo( + job_id=topic_info.job_id, + topic=topic_info.heartbeat, + slurm_job_id=future.slurm_job_id, + callback=None, + last_heartbeat=None, + callback_id=None, + heartbeat_received=False, + ) + topic = topic_info.heartbeat + + def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None: + """Enrich a parsed worker message and dispatch its registered handler. + + Args: + message (HeartBeatMessage): Raw Redis payload or parsed worker message, + as specified by the annotation. + sub_info (SubscriptionInfo): Subscription ownership and heartbeat + bookkeeping. + """ + try: + message.metadata.update( + { + "msg_received": datetime.now().isoformat(), + "job_id": sub_info.job_id, + "slurm_job_id": sub_info.slurm_job_id, + } + ) + self._heartbeat_callback(message, sub_info) + except Exception: + logger.error( + "Error in callback for topic %s: %s", message.msg_type, message.timestamp + ) + + cb_id = self._sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info)) + sub_info.callback_id = cb_id + sub_info.last_heartbeat = time.time() + self._register_subscription_for_job(sub_info=sub_info) + + def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None: + """Register the internal callback for terminal and running statuses. + + Args: + topic_info (TopicInfo): Redis topic names for the tracked job. + future (JobFuture): Tracked future whose lifecycle or subscriptions are + being managed. + """ + sub_info = SubscriptionInfo( + job_id=topic_info.job_id, + topic=topic_info.event, + slurm_job_id=future.slurm_job_id, + callback=None, + last_heartbeat=None, + callback_id=None, + heartbeat_received=False, + ) + event_topic = topic_info.event + + def wrapped_callback(message: StatusMessage, future: JobFuture) -> None: + """Enrich a parsed worker message and dispatch its registered handler. + + Args: + message (StatusMessage): Raw Redis payload or parsed worker message, as + specified by the annotation. + future (JobFuture): Tracked future whose lifecycle or subscriptions are + being managed. + """ + try: + message.metadata.update( + { + "msg_received": datetime.now().isoformat(), + "job_id": future._job_id, + "slurm_job_id": future.slurm_job_id, + } + ) + self.apply_status(message, future=future) + except Exception: + logger.error("Error in callback for topic %s: %s", message.msg_type, message.status) + + cb_id = self._sub_client.subscribe(event_topic, partial(wrapped_callback, future=future)) + sub_info.callback_id = cb_id + self._register_subscription_for_job(sub_info=sub_info) + + def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None: + """Index a subscription by callback ID, topic, and job. + + Args: + sub_info (SubscriptionInfo): Subscription ownership and heartbeat + bookkeeping. + + Raises: + ValueError: The subscription does not contain a callback ID. + """ + with self._lock: + if sub_info.callback_id is None: + raise ValueError("SubscriptionInfo.callback_id must be set before registration") + self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info + self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic + self._active_subscriptions[sub_info.job_id].add(sub_info.topic) + + def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None: + """Record heartbeat receipt for the monitored subscription. + + Args: + message (HeartBeatMessage): Raw Redis payload or parsed worker message, as + specified by the annotation. + sub_info (SubscriptionInfo): Subscription ownership and heartbeat + bookkeeping. + """ + with self._lock: + sub_info.heartbeat_received = True + sub_info.last_heartbeat = time.time() + + def _remove_job_future(self, job_id: str) -> None: + """Remove a future from the manager registry under its lock. + + Args: + job_id (str): Opaque manager identifier used in Redis topic names. + """ + with self._lock: + self._future_registry.pop(job_id, None) diff --git a/slurm_manager/redis_sub_client.py b/slurm_manager/redis_sub_client.py index 0ba56a4..0ab2345 100644 --- a/slurm_manager/redis_sub_client.py +++ b/slurm_manager/redis_sub_client.py @@ -52,6 +52,18 @@ class RedisSubClient: redis_cls: type[Redis] = Redis, **redis_kwargs: Any, ) -> None: + """Create Redis resources without starting the listener until subscribed. + + Args: + bootstrap (str | list[str]): Redis host:port or a nonempty address list; + only the first entry is used. + redis_cls (type[Redis]): Redis client implementation, injectable for tests. + **redis_kwargs (Any): Additional Redis constructor options such as socket + timeouts. + + Raises: + ValueError: The bootstrap list is empty or its address cannot be parsed. + """ if isinstance(bootstrap, list): if not bootstrap: raise ValueError("bootstrap list must not be empty") @@ -70,14 +82,34 @@ class RedisSubClient: self._stop_listener = threading.Event() self._listener_thread: threading.Thread | None = None + ################################################ + # Public Subscriptions # + ################################################ + @property def registered_topics(self) -> set[str]: - """Returns all currently subscribed topics.""" + """Return a snapshot of topics with active callbacks. + + Returns: + set[str]: Currently registered Redis channel names. + """ with self._lock: return set(self._subscriptions_by_topic.keys()) def subscribe(self, topic: str, callback: MessageCallback) -> str: - """Subscribes callback to a topic and returns a subscription id.""" + """Register a callback and start the listener if needed. + + Args: + topic (str): Redis channel whose callback registrations are selected. + callback (MessageCallback): Callable receiving the parsed Slurm message. + + Returns: + str: Unique subscription ID. + + Notes: + The callback runs on the Redis listener thread and must remain fast and + nonblocking. + """ sub_id = str(uuid4()) subscription = Subscription(id=sub_id, topic=topic, callback=callback) @@ -95,10 +127,13 @@ class RedisSubClient: return sub_id def unsubscribe(self, subscription_id: str) -> bool: - """ - Unregisters one callback by subscription id. + """Remove one callback and unsubscribe its topic if it has no owners. - Returns True if a callback was removed, False otherwise. + Args: + subscription_id (str): Unique callback ID returned by subscribe(). + + Returns: + bool: True if the callback was registered; False if it was unknown. """ with self._lock: topic = self._topic_by_subscription_id.pop(subscription_id, None) @@ -120,7 +155,14 @@ class RedisSubClient: return True def unsubscribe_topic(self, topic: str) -> int: - """Unregisters all callbacks for a topic and returns removed count.""" + """Remove every callback registered for one topic. + + Args: + topic (str): Redis channel whose callback registrations are selected. + + Returns: + int: Number of removed callbacks. + """ with self._lock: callbacks = self._subscriptions_by_topic.pop(topic, None) if not callbacks: # No callbacks for this topic, nothing happens @@ -132,7 +174,19 @@ class RedisSubClient: return len(callbacks) def wait_until_subscribed(self, topics: list[str], timeout: float = 5.0) -> None: - """Wait for Redis to acknowledge each subscription before releasing a job.""" + """Wait for Redis subscription acknowledgements before releasing a job. + + Args: + topics (list[str]): Already registered channels whose acknowledgements must + be received. + timeout (float): Shared acknowledgement deadline in seconds across all + requested topics. + + Raises: + KeyError: A requested topic is not registered. + TimeoutError: The shared deadline expires before all topics are + acknowledged. + """ deadline = time.monotonic() + timeout with self._lock: ready = [(topic, self._subscription_ready[topic]) for topic in topics] @@ -140,8 +194,21 @@ class RedisSubClient: if not event.wait(max(0, deadline - time.monotonic())): raise TimeoutError(f"Redis did not acknowledge subscription to {topic}") + ################################################ + # Public Lifecycle # + ################################################ + def shutdown(self, timeout: float | None = 1.0) -> None: - """Stops worker threads and closes Redis resources.""" + """Stop the listener and close Redis resources. + + Args: + timeout (float | None): Listener join timeout in seconds; None waits + indefinitely. + + Notes: + A listener that misses the join timeout is logged before resources are + closed. + """ self._stop_listener.set() if self._listener_thread is not None: @@ -157,8 +224,12 @@ class RedisSubClient: self._pubsub.close() self._redis.close() + ################################################ + # Internal Listener # + ################################################ + def _ensure_threads_started(self) -> None: - """Makes sure the listener thread is running to receive messages from Redis.""" + """Start a listener if no listener thread is currently alive.""" if self._listener_thread is None or not self._listener_thread.is_alive(): self._stop_listener.clear() self._listener_thread = threading.Thread( @@ -167,7 +238,12 @@ class RedisSubClient: self._listener_thread.start() def _listener_loop(self) -> None: - """Listener thread loop that polls for messages and dispatches them to callbacks.""" + """Poll Redis, acknowledge subscriptions, and dispatch parsed messages. + + Notes: + Redis errors are logged and polling resumes after a short delay. Callback + exceptions are logged without stopping other callbacks. + """ while not self._stop_listener.is_set(): try: with self._lock: @@ -212,9 +288,26 @@ class RedisSubClient: format_exc, ) + ################################################ + # Internal Message Parsing # + ################################################ + @staticmethod def _normalize_message(message: dict[str, Any]) -> tuple[str, SlurmMessage]: - """Converts raw Redis message dict to normalized Message object.""" + """Decode a Redis pub/sub payload into the corresponding message model. + + Args: + message (dict[str, Any]): Raw Redis payload or parsed worker message, as + specified by the annotation. + + Returns: + tuple[str, SlurmMessage]: Pair containing the channel name and validated + Slurm message. + + Raises: + TypeError: The payload is neither JSON text nor a dictionary. + ValueError: The JSON or message schema is invalid. + """ channel = message.get("channel") topic = channel.decode() if isinstance(channel, bytes) else str(channel) diff --git a/slurm_manager/slurm_manager.py b/slurm_manager/slurm_manager.py index da8b0b0..5a08cd5 100644 --- a/slurm_manager/slurm_manager.py +++ b/slurm_manager/slurm_manager.py @@ -1,277 +1,195 @@ -"""Minimal Slurm manager draft for sbatch + JobFuture creation.""" +"""Submit and monitor remote Slurm jobs through slurmrestd and Redis.""" from __future__ import annotations -import os -import subprocess +import json +import shlex import threading -import time -from collections import defaultdict from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass from datetime import datetime -from functools import partial, wraps from logging import getLogger from pathlib import Path from typing import Any, Literal from uuid import uuid4 -from pydantic import BaseModel, ConfigDict - -from slurm_manager.event_messages import HeartBeatMessage, SlurmMessage, StatusMessage +from slurm_manager.event_messages import SlurmMessage, StatusMessage from slurm_manager.job_future import JobFuture, JobStatus +from slurm_manager.job_options import JobOptions +from slurm_manager.job_tracker import JobTracker, SubscriptionInfo from slurm_manager.redis_sub_client import RedisSubClient +from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError logger = getLogger(__name__) -_STOP = object() +def _job_exit_code(exit_code: Any) -> int | str | None: + """Decode a v0.0.44 process exit status without treating unset codes as success. -def _cluster_args(cluster: str | None) -> list[str]: - """Route Slurm CLI operations to the cluster that accepted the job.""" - return ["--clusters", cluster] if cluster else [] + Args: + exit_code (Any): Integer or v0.0.44 structured process exit status. - -@dataclass(frozen=True, slots=True) -class TopicInfo: - """Helper for topic specification within Redis bec-slurm pub/sub interface.""" - - job_id: str - prefix: str = "info" - - def topic(self, v: str) -> str: - """Constructs a new topic string for a given key.""" - return f"{self.prefix}/{self.job_id}/{v}" - - @property - def log(self) -> str: - """Topic for log messages.""" - return self.topic("log") - - @property - def event(self) -> str: - """Topic for event messages.""" - return self.topic("status") - - @property - def heartbeat(self) -> str: - """Topic for heartbeat messages.""" - return self.topic("heartbeat") - - -class SubscriptionInfo(BaseModel): - """Helper for slurm job information and future tracking.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - topic: str - job_id: str - slurm_job_id: str | None = None - callback: Callable[[SlurmMessage], None] | None = None - callback_id: str | None = None - last_heartbeat: float | None = None - heartbeat_received: bool = False - last_scheduler_check: float | None = None - - -HEARTBEAT_TIMEOUT = 10 # seconds + Returns: + int | str | None: Exit code, a signal description, or None when no code is + available. + """ + if not isinstance(exit_code, dict): + return exit_code + code = exit_code.get("return_code") + if isinstance(code, dict): + code = code.get("number") if code.get("set") else None + signal = exit_code.get("signal", {}).get("id", {}) + if isinstance(signal, dict): + signal = signal.get("number", 0) if signal.get("set") else 0 + return code if not signal else f"signal {signal}" class SlurmManager: - """Small manager that submits wrapper jobs and returns JobFuture handles.""" + """Submit Slurm jobs and coordinate tracking and scheduler monitoring.""" - def __init__(self, redis_host: str = "localhost", redis_port: int = 6379) -> None: + def __init__( + self, + redis_host: str = "localhost", + redis_port: int = 6379, + *, + env_file: str | Path = ".env", + rest_client: SlurmRestClient | None = None, + worker_path: str = "/usr/local/bin/slurm-worker", + ) -> None: + """Create REST/Redis clients and start background job monitoring. + + Args: + redis_host (str): Redis hostname reachable by the submitter and compute + nodes. + redis_port (int): Redis TCP port shared by the submitter and worker. + env_file (str | Path): Local credential dotenv path. Environment variables + take precedence. + rest_client (SlurmRestClient | None): Optional preconfigured REST client; + None creates one from env_file. + worker_path (str): Absolute path to the Go executable on compute nodes. + + Raises: + ValueError: The worker path or REST connection settings are invalid. + """ + if not worker_path.startswith("/"): + raise ValueError("Worker path must be an absolute compute-node path") + self.worker_path = worker_path + self._redis_address = f"{redis_host}:{redis_port}" + self.rest_client = rest_client if rest_client is not None else SlurmRestClient(env_file) self.sub_client = RedisSubClient(bootstrap=f"{redis_host}:{redis_port}") - self._wrapper_path = Path(__file__).resolve().parent / "job_submission" / "wrapper.sh" - self._redis_script_path = ( - Path(__file__).resolve().parent / "job_submission" / "redis_worker.sh" - ) - - # This is a job_id mapping to job futures - self._future_registry: dict[str, JobFuture] = {} - # This subscription id sub_info mapping by topics, used for callback handling - self._subscriptions_by_topic: dict[str, dict[str, SubscriptionInfo]] = defaultdict(dict) - # This is a topic mapping to subscriptions - self._topic_by_subscription_id: dict[str, str] = {} - # This is a job_id mapping to topics - self._active_subscriptions: dict[str, set[str]] = defaultdict(set) - - # Threading - self._lock = threading.RLock() + self._job_tracker = JobTracker(self.sub_client) self._stop_heartbeat_thread = threading.Event() - self.executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="SlurmManagerCallback") # Heartbeat monitoring thread self._heartbeat_monitoring_thread = threading.Thread( target=self._heartbeat_monitoring_loop, name="SlurmManagerHeartbeatMonitor", daemon=True ) self._start_threads() - def _start_threads(self): - """Starts the heartbeat monitoring thread.""" - self._heartbeat_monitoring_thread.start() + ################################################ + # Public Job Management # + ################################################ - def _heartbeat_monitoring_loop(self) -> None: - """Internal loop to check for missing heartbeats. If a heartbeat for a job is missing for - more than HEARTBEAT_TIMEOUT seconds, we will trigger a check of the slurm job status, - and potentially trigger a cleanup of the job future and subscriptions if the job is no longer active. + def submit_job( + self, + script_path: str | Path, + env_path: str | Path | None = None, + *, + working_directory: str, + job_options: JobOptions | None = None, + callbacks: dict[Literal["log", "status", "heartbeat"], Callable[[SlurmMessage], None]] + | None = None, + ) -> JobFuture: + """Upload local Bash files and return a monitored, released Slurm job. + + Args: + script_path (str | Path): Local Bash workload file whose contents are + uploaded. + env_path (str | Path | None): Optional local Bash activation file for + workload-specific environment setup; None skips activation commands. + working_directory (str): Absolute, existing compute-node directory in which + the job runs. + job_options (JobOptions | None): Validated resources and Slurm output + settings; None uses JobOptions defaults. + callbacks (dict[str, Callable] | None): Callbacks keyed by log, status, or + heartbeat, acknowledged before job release. + + Returns: + JobFuture: Handle for querying, waiting, cancelling, and registering + callbacks. + + Raises: + FileNotFoundError: A local workload or activation file does not exist. + ValueError: The remote working directory or callback event names are + invalid. + ValidationError: Resource or output options fail JobOptions validation. + SlurmRestError: Submission or release fails. A known held job is cancelled + on setup failure. + TimeoutError: Redis does not acknowledge the monitoring subscriptions. + + Notes: + The credential dotenv file stays local. Callbacks are registered before + release; only the working directory, worker path, and output paths refer to + compute nodes. """ - - while not self._stop_heartbeat_thread.wait(timeout=2): - missing_heartbeats = [] - # Check heartbeats for active jobs - with self._lock: - for _, topics in self._active_subscriptions.items(): - for topic in topics: - infos = self._subscriptions_by_topic.get(topic, {}) - for sub_info in infos.values(): - if sub_info and sub_info.last_heartbeat: - now = time.time() - if now - sub_info.last_heartbeat > HEARTBEAT_TIMEOUT and ( - sub_info.last_scheduler_check is None - or now - sub_info.last_scheduler_check > HEARTBEAT_TIMEOUT - ): - missing_heartbeats.append(sub_info) - - for sub_info in missing_heartbeats: - self._check_slurm_job(sub_info) - - def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None: - """Resolve missing heartbeats using Slurm, never assuming silence means failure.""" - future = self._future_registry.get(sub_info.job_id) - if future is None or future.done(): - return - sub_info.last_scheduler_check = time.time() - try: - queued = self._run_subprocess( - [ - "squeue", - *_cluster_args(future.slurm_cluster), - "--noheader", - "--jobs", - future.slurm_job_id, - "--format=%T", - ] - ) - if queued.returncode == 0 and queued.stdout.strip(): - # Pending, configuring, and running jobs may legitimately have no heartbeat. - return - accounting = self._run_subprocess( - [ - "sacct", - *_cluster_args(future.slurm_cluster), - "--noheader", - "--parsable2", - "--allocations", - "--jobs", - future.slurm_job_id, - "--format=State,ExitCode", - ] - ) - except (OSError, subprocess.SubprocessError): - logger.exception("Could not query Slurm job %s", future.slurm_job_id) - return - if accounting.returncode != 0 or not accounting.stdout.strip(): - # Accounting may be disabled, delayed, or temporarily unavailable. - return - fields = accounting.stdout.strip().splitlines()[0].split("|") - state = fields[0].split()[0].rstrip("+") - exit_code = fields[1].strip() if len(fields) > 1 else "" - if state == "COMPLETED" and exit_code == "0:0": - status = JobStatus.FINISHED.value - elif state == "CANCELLED": - status = JobStatus.CANCELLED.value - elif state in { - "COMPLETED", - "FAILED", - "TIMEOUT", - "OUT_OF_MEMORY", - "NODE_FAIL", - "BOOT_FAIL", - "DEADLINE", - "PREEMPTED", - "REVOKED", - }: - status = f"error:Slurm state {state}, exit code {exit_code or 'unknown'}" - else: - return - self._event_callback(StatusMessage(status=status), future) - - def submit_job(self, script_path: str, env_path: str) -> JobFuture: - """Create a temporary script from command and submit via sbatch wrapper.""" - # Check that the wrapper script and redis worker script exist before submitting the job - if not self._wrapper_path.exists(): - raise FileNotFoundError(f"wrapper script not found: {self._wrapper_path}") - if not self._redis_script_path.exists(): - raise FileNotFoundError(f"redis worker script not found: {self._redis_script_path}") - - env_path = os.path.abspath(env_path) - script_path = os.path.abspath(script_path) - - job_id = str(uuid4()) # unique job_id for hash-key in Redis. - sbatch_cmd = [ - "sbatch", - "--parsable", - "--hold", - os.path.abspath(self._wrapper_path), - env_path, + if not working_directory.startswith("/"): + raise ValueError("Working directory must be an absolute compute-node path") + if callbacks and set(callbacks) - {"log", "status", "heartbeat"}: + raise ValueError("Callbacks must use log, status, or heartbeat keys") + job_id = str(uuid4()) + script = self._build_job_script( script_path, + env_path, job_id, - "1", - "1", - os.path.abspath(self._redis_script_path), - ] - logger.debug("Submitting job with command: %s", " ".join(sbatch_cmd)) - - result = self._run_subprocess(sbatch_cmd) - if result.returncode != 0: - stderr = (result.stderr or "").strip() - raise RuntimeError(f"sbatch failed: {stderr}") - slurm_job_id = result.stdout.strip() # ID of job from SLURM scheduler - logger.debug("SBatch job submission id: %s", slurm_job_id) - - job_future = JobFuture( - job_id=job_id, manager=self, registry=self._future_registry, slurm_job_id=slurm_job_id + worker_path=self.worker_path, + redis_address=self._redis_address, ) + job = JobOptions.model_validate(job_options or {}).to_rest() + job.update( + { + "current_working_directory": working_directory, + "environment": [ + "PATH=/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:" + "/usr/sbin:/usr/lpp/mmfs/bin:" + ], + "hold": True, + } + ) + result = self.rest_client.submit(script, job) + slurm_job_id = str(result["job_id"]) + job_future = JobFuture(job_id=job_id, manager=self, slurm_job_id=slurm_job_id) slurm_job_id = job_future.slurm_job_id - cluster_args = _cluster_args(job_future.slurm_cluster) - topic_info = TopicInfo(job_id=job_id) - with self._lock: - self._future_registry[job_id] = job_future try: - self._subscribe_to_internal_topics(future=job_future, topic_info=topic_info) - self.sub_client.wait_until_subscribed([topic_info.heartbeat, topic_info.event]) - released = self._run_subprocess(["scontrol", *cluster_args, "release", slurm_job_id]) - if released.returncode != 0: - raise RuntimeError(f"scontrol release failed: {(released.stderr or '').strip()}") + self._job_tracker.track(job_future) + for event, callback in (callbacks or {}).items(): + job_future.listen(event, callback) + self._job_tracker.wait_until_ready(job_id) + self.rest_client.release(slurm_job_id) except Exception as exc: # A setup failure must not leave a held allocation behind. try: - cancelled = self._run_subprocess(["scancel", *cluster_args, slurm_job_id]) - if cancelled.returncode != 0: - logger.error("Could not cancel held job %s: %s", slurm_job_id, cancelled.stderr) - except (OSError, subprocess.SubprocessError): - logger.exception("Could not cancel held job %s", slurm_job_id) - self._event_callback( + self.rest_client.cancel(slurm_job_id) + except (SlurmRestError, ValueError): + logger.error("Could not cancel held job %s", slurm_job_id) + self._job_tracker.apply_status( StatusMessage(status=f"error:Submission setup failed: {exc}"), job_future ) raise return job_future - def cancel_job( - self, slurm_job_id: str, job_id: str, *, slurm_cluster: str | None = None - ) -> None: - """Cancel a job on its recorded cluster via scancel.""" - job_future = self._future_registry.get(job_id) - slurm_job_id, _, cluster = slurm_job_id.partition(";") - cluster = slurm_cluster or cluster or (job_future.slurm_cluster if job_future else None) - result = self._run_subprocess(["scancel", *_cluster_args(cluster), slurm_job_id]) - if result.returncode != 0: - stderr = (result.stderr or "").strip() - raise RuntimeError(f"scancel failed: {stderr}") + def cancel_job(self, slurm_job_id: str, job_id: str) -> None: + """Request cancellation through REST and mark the tracked job cancelled. + + Args: + slurm_job_id (str): Numeric scheduler identifier returned by Slurm. + job_id (str): Opaque manager identifier used in Redis topic names. + + Raises: + SlurmRestError: Slurm rejects cancellation or the request fails. + """ + job_future = self._job_tracker.get_future(job_id) + self.rest_client.cancel(slurm_job_id) if job_future: - self._event_callback( + self._job_tracker.apply_status( StatusMessage( status="cancelled", metadata={ @@ -283,127 +201,107 @@ class SlurmManager: future=job_future, ) + ################################################ + # Public Event Subscriptions # + ################################################ + def listen( self, event_type: Literal["heartbeat", "log", "status"], callback: Callable[[SlurmMessage], None], job_id: str | None = None, ) -> list[str]: - """ - Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status'). - If the job_id is specified, the callback will only be registered for the specific job. If job_id is None, - the callback will be registered for all jobs. - Returns a callback ID that can be used to unregister. + """Attach a callback to one tracked job or all currently tracked jobs. Args: - event_type: The type of event to listen for ('heartbeat', 'log', or 'status'). - callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for any job. - The SlurmMessage object contains two fields, 'topic', 'data' + event_type (Literal['heartbeat', 'log', 'status']): Event to receive: log, + status, or heartbeat. + callback (Callable[[SlurmMessage], None]): Callable receiving the parsed + Slurm message. + job_id (str | None): Opaque manager identifier used in Redis topic names. Returns: - A list of unique callback IDS will be returned which can be used to unregister the callbacks later. - """ - job_futures: list[JobFuture] = ( - [self._future_registry[job_id]] if job_id else list(self._future_registry.values()) - ) - callback_ids = [] - for future in job_futures: - callback_ids.append(future.listen(event_type=event_type, callback=callback)) + list[str]: Subscription IDs accepted by unlisten(). - return callback_ids + Raises: + KeyError: The explicitly requested job ID is not tracked. + RuntimeError: A selected job has already completed. + + Notes: + With job_id=None, registration applies to the current jobs, not future + submissions. Callbacks run in the tracker executor. + """ + return self._job_tracker.listen(event_type, callback, job_id) def unlisten(self, callback_id: str | list[str]) -> None: + """Remove one or more callback subscriptions without cancelling the job. + + Args: + callback_id (str | list[str]): Subscription ID, or IDs where accepted, + returned during registration. + """ self.remove_subscription_by_id(callback_id) def remove_subscription_by_id(self, callback_id: str | list[str]) -> None: - """Remove a subscription by its callback ID.""" - if isinstance(callback_id, str): - callback_id = [callback_id] - with self._lock: - for cb_id in callback_id: - topic = self._topic_by_subscription_id.pop(cb_id, None) - if topic is not None: - callbacks = self._subscriptions_by_topic.get(topic, {}) - sub_info = callbacks.pop(cb_id, None) - self.sub_client.unsubscribe(cb_id) - - # Other callbacks, including heartbeat monitoring, still own this topic. - if not callbacks: - self._subscriptions_by_topic.pop(topic, None) - if sub_info: - topics = self._active_subscriptions.get(sub_info.job_id) - if topics is not None: - topics.discard(topic) - - def remove_subscription_by_topic(self, topic: str) -> None: - """Remove a subscription by its topic.""" - with self._lock: - callbacks = self._subscriptions_by_topic.pop(topic, {}) - for callback_id, sub_info in callbacks.items(): - self._topic_by_subscription_id.pop(callback_id, None) - job_id = sub_info.job_id - self._active_subscriptions[job_id].discard(topic) - - # Remove the topic subscription from Redis - self.sub_client.unsubscribe_topic(topic) - - def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None: - """ - Remove all subscriptions for a specific job. If skip_event_callback is False, this will also trigger - the event callback with an error status which will update the JobFuture status. - This can be used to trigger a cleanup of the Job and its subscriptions. - - If it finished in success, it should be called with skip_event_callback=True to avoid overwriting - the finished status with an error status. + """Remove the selected callbacks while preserving other topic listeners. Args: - job_id: The ID of the job for which to remove subscriptions. - skip_event_callback: If True, the event callback will not be triggered with an error status. - This should be used when the job is finishing successfully to avoid overwriting the - finished status with an error status. + callback_id (str | list[str]): Subscription ID, or IDs where accepted, + returned during registration. """ - with self._lock: - topics = self._active_subscriptions.pop(job_id, set()) - for topic in topics: - callbacks = self._subscriptions_by_topic.pop(topic, {}) - for callback_id in callbacks.keys(): - self._topic_by_subscription_id.pop(callback_id, None) - self.sub_client.unsubscribe_topic(topic) - future = self._future_registry.get(job_id) - if future: - # This callback will trigger a cleanup - if not skip_event_callback: - self._event_callback( - StatusMessage( - status=f"error:Removal requested by user; timestamp {datetime.now().isoformat()}", - metadata={ - "msg_received": datetime.now().isoformat(), - "job_id": future._job_id, - "slurm_job_id": future.slurm_job_id, - }, - ), - future, - ) - self._remove_job_future(future._job_id) + self._job_tracker.remove_subscription_by_id(callback_id) + + def remove_subscription_by_topic(self, topic: str) -> None: + """Remove all callbacks for one topic and unsubscribe it from Redis. + + Args: + topic (str): Redis channel whose callback registrations are selected. + """ + self._job_tracker.remove_subscription_by_topic(topic) + + def remove_subscriptions_for_job(self, job_id: str, skip_event_callback: bool = False) -> None: + """Remove job subscriptions and discard the tracked future. + + Args: + job_id (str): Opaque manager identifier used in Redis topic names. + skip_event_callback (bool): If true, remove monitoring without synthesizing + a failure update. + + Notes: + Unless skip_event_callback is true, an unfinished future is marked failed. + This only removes local monitoring; it does not cancel the Slurm allocation. + """ + self._job_tracker.remove_subscriptions_for_job(job_id, skip_event_callback) + + ################################################ + # Public Lifecycle # + ################################################ def shutdown(self, timeout: float | None = 1.0) -> None: - """Shuts down the manager, including the heartbeat monitoring thread and Redis subscriptions.""" + """Stop monitoring, close Redis, and drain accepted callback work. + + Args: + timeout (float | None): Thread-join timeout in seconds; None waits + indefinitely. + + Raises: + TimeoutError: The heartbeat monitor is still performing a Slurm query. + + Notes: + Remote jobs are not cancelled. The timeout applies to thread joins; running + and queued user callbacks are drained without a callback execution deadline. + """ self._stop_heartbeat_thread.set() self._heartbeat_monitoring_thread.join(timeout) if self._heartbeat_monitoring_thread.is_alive(): raise TimeoutError( "Heartbeat monitor did not stop; retry after its Slurm query finishes" ) - with self._lock: - self._subscriptions_by_topic.clear() - self._topic_by_subscription_id.clear() + self._job_tracker.shutdown(timeout) - self.sub_client.shutdown(timeout=timeout) - self.executor.shutdown(wait=True, cancel_futures=True) - - ################### - ### Internal Methods - ################### + ################################################ + # Internal Callback Registration # + ################################################ def _add_subscription( self, @@ -412,182 +310,138 @@ class SlurmManager: callback: Callable[[SlurmMessage], None], callback_kwargs: dict[str, Any] | None = None, ) -> str: - """ - Register a callback to listen for events of the specified type ('heartbeat', 'log', or 'status') for a specific job. - Callbacks receive a SlurmMessage object, depending on the type of event they listen to. The callback will be executed - in the background and errors will be logged but not raised to not jeopardize the main loop of the manager. - The callback will be executed with additional keyword arguments passed via callback_kwargs dictionary. + """Atomically register a Redis callback and its manager ownership. Args: - job_id: The ID of the job for which to register the callback. - key: The type of event to listen for ('heartbeat', 'log', or 'status'). - callback: A function that takes a SlurmMessage object as input and is called when an event of the specified type is received for the job. - callback_kwargs: A dictionary of additional keyword arguments to pass to the callback when it is executed + job_id (str): Opaque manager identifier used in Redis topic names. + key (Literal['heartbeat', 'log', 'status']): Event key: log, status, or + heartbeat. + callback (Callable[[SlurmMessage], None]): Callable receiving the parsed + Slurm message. + callback_kwargs (dict[str, Any] | None): Additional keyword arguments passed + to the user callback. + + Returns: + str: Unique callback subscription ID. + + Raises: + RuntimeError: The job is absent or finishes during registration. """ - topic_str = TopicInfo(job_id=job_id).topic(key) + return self._job_tracker.add_subscription(job_id, key, callback, callback_kwargs) - @wraps(callback) - def wrapped_callback( - message: SlurmMessage, - executor: ThreadPoolExecutor, - job_id: str, - slurm_job_id: str, - callback_kwargs: dict[str, Any] | None = None, - ) -> None: - message.metadata.update( - { - "msg_received": datetime.now().isoformat(), - "job_id": job_id, - "slurm_job_id": slurm_job_id, - } - ) - executor.submit(callback, message, **(callback_kwargs or {})) + ################################################ + # Internal Script Packaging # + ################################################ - # Completion removes subscriptions under this same lock. Keep ownership - # registration atomic with cleanup, and undo Redis registration if the - # future became terminal while subscribe() was in progress. - with self._lock: - future = self._future_registry.get(job_id) - if future is None or future.done(): - raise RuntimeError(f"Job {job_id} is no longer active") - cb_id = self.sub_client.subscribe( - topic_str, - partial( - wrapped_callback, - executor=self.executor, - callback_kwargs=callback_kwargs, - job_id=job_id, - slurm_job_id=future.slurm_job_id, - ), - ) - - if self._future_registry.get(job_id) is not future or future.done(): - self.sub_client.unsubscribe(cb_id) - raise RuntimeError(f"Job {job_id} finished during subscription registration") - - sub_info = SubscriptionInfo( - job_id=job_id, - topic=topic_str, - slurm_job_id=future.slurm_job_id, - callback=wrapped_callback, - last_heartbeat=None, - callback_id=cb_id, - heartbeat_received=False, - ) - self._register_subscription_for_job(sub_info=sub_info) - return cb_id - - def _remove_job_future(self, job_id: str) -> None: - """Remove a job future from the registry.""" - with self._lock: - self._future_registry.pop(job_id, None) - - def _run_subprocess(self, cmd: list[str]) -> subprocess.CompletedProcess: - return subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=30) - - def _subscribe_to_internal_topics(self, future: JobFuture, topic_info: TopicInfo) -> None: - self._subscribe_to_heartbeat(topic_info, future) - self._subscribe_to_event(topic_info, future) - - def _subscribe_to_heartbeat(self, topic_info: TopicInfo, future: JobFuture) -> None: - sub_info = SubscriptionInfo( - job_id=topic_info.job_id, - topic=topic_info.heartbeat, - slurm_job_id=future.slurm_job_id, - callback=None, - last_heartbeat=None, - callback_id=None, - heartbeat_received=False, - ) - topic = topic_info.heartbeat - - def wrapped_callback(message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None: - try: - message.metadata.update( - { - "msg_received": datetime.now().isoformat(), - "job_id": sub_info.job_id, - "slurm_job_id": sub_info.slurm_job_id, - } - ) - self._heartbeat_callback(message, sub_info) - except Exception: - logger.error( - "Error in callback for topic %s: %s", message.msg_type, message.timestamp - ) - - cb_id = self.sub_client.subscribe(topic, partial(wrapped_callback, sub_info=sub_info)) - sub_info.callback_id = cb_id - sub_info.last_heartbeat = time.time() - self._register_subscription_for_job(sub_info=sub_info) - - def _heartbeat_callback(self, message: HeartBeatMessage, sub_info: SubscriptionInfo) -> None: - sub_info.heartbeat_received = True - sub_info.last_heartbeat = time.time() - - def _subscribe_to_event(self, topic_info: TopicInfo, future: JobFuture) -> None: - sub_info = SubscriptionInfo( - job_id=topic_info.job_id, - topic=topic_info.event, - slurm_job_id=future.slurm_job_id, - callback=None, - last_heartbeat=None, - callback_id=None, - heartbeat_received=False, - ) - event_topic = topic_info.event - - def wrapped_callback(message: StatusMessage, future: JobFuture) -> None: - try: - message.metadata.update( - { - "msg_received": datetime.now().isoformat(), - "job_id": future._job_id, - "slurm_job_id": future.slurm_job_id, - } - ) - self._event_callback(message, future=future) - except Exception: - logger.error("Error in callback for topic %s: %s", message.msg_type, message.status) - - cb_id = self.sub_client.subscribe(event_topic, partial(wrapped_callback, future=future)) - sub_info.callback_id = cb_id - self._register_subscription_for_job(sub_info=sub_info) - - def _register_subscription_for_job(self, sub_info: SubscriptionInfo) -> None: - with self._lock: - if sub_info.callback_id is None: - raise ValueError("SubscriptionInfo.callback_id must be set before registration") - self._subscriptions_by_topic[sub_info.topic][sub_info.callback_id] = sub_info - self._topic_by_subscription_id[sub_info.callback_id] = sub_info.topic - self._active_subscriptions[sub_info.job_id].add(sub_info.topic) - - def _event_callback(self, message: StatusMessage, future: JobFuture) -> None: - """ - Callback for handling event message updates. The method should also be used to trigger - a cleanup of subscriptions and job future when a job is finishing unexpectedly - (e.g. due to missing heartbeat or user requested removal). If finishing unexpectedly, - we recommend using a payload format of 'error:reason' to indicate the error and reason - for the job finishing, which will be reflected in the JobFuture status and exception info. + @staticmethod + def _build_job_script( + script_path: str | Path, + env_path: str | Path | None, + job_id: str, + *, + worker_path: str = "/usr/local/bin/slurm-worker", + redis_address: str = "localhost:6379", + ) -> str: + """Package local Bash file contents for the compute-node Go worker. Args: - message: The incoming message from the event topic, expected to have a payload format of 'status:optional_info'. - future: The JobFuture associated with the job, which will be updated based on the event message. + script_path (str | Path): Local Bash workload file whose contents are uploaded. + env_path (str | Path | None): Optional local Bash activation file for + workload-specific environment setup; None skips activation commands. + job_id (str): Opaque manager identifier used in Redis topic names. + worker_path (str): Absolute path to the Go executable on compute nodes. + redis_address (str): Redis host:port passed to the worker; no credentials are + embedded. + + Returns: + str: Batch script invoking the configured worker with JSON on standard input. + + Raises: + FileNotFoundError: A local script file is missing. + ValueError: The worker path is not an absolute compute-node path. + + Notes: + The payload contains only the selected workload and activation scripts, not the + credential dotenv file. Staging and cleanup belong to the worker. """ - info = message.status.split(":", 1) - status = info[0] - try: # Catch irregular formatted status messages to avoid crashing the callback loop. - status = JobStatus(status) - except ValueError: - logger.warning("Received unknown job status '%s' for job %s", status, future.name) - future._update_status( - "error", - f"Received unknown job status '{status}' from message: {message.model_dump()}", - ) + if not worker_path.startswith("/"): + raise ValueError("Worker path must be an absolute compute-node path") + payload = json.dumps( + { + "script": Path(script_path).read_text(), + "activation": Path(env_path).read_text() if env_path is not None else "", + } + ) + delimiter = f"SLURM_PAYLOAD_{uuid4().hex}" + command = shlex.join([worker_path, "--job-id", job_id, "--redis-address", redis_address]) + return f"#!/bin/bash\nexec {command} <<'{delimiter}'\n{payload}\n{delimiter}\n" + + ################################################ + # Internal Monitoring # + ################################################ + + def _start_threads(self) -> None: + """Start the manager heartbeat-monitor thread.""" + self._heartbeat_monitoring_thread.start() + + def _heartbeat_monitoring_loop(self) -> None: + """Check missing heartbeats until shutdown and query Slurm when necessary.""" + + while not self._stop_heartbeat_thread.wait(timeout=2): + for sub_info in self._job_tracker.get_missing_heartbeats(): + self._check_slurm_job(sub_info) + + def _check_slurm_job(self, sub_info: SubscriptionInfo) -> None: + """Resolve a missed heartbeat from controller or accounting records. + + Args: + sub_info (SubscriptionInfo): Subscription ownership and heartbeat + bookkeeping. + + Notes: + Transport failures, absent records, and incomplete accounting leave the job + unresolved for a later check. + """ + future = self._job_tracker.begin_scheduler_check(sub_info.job_id) + if future is None or future.done(): return - exc_info = info[1] if len(info) > 1 else None - future._update_status(status, exc_info) - # Clean up subscriptions for finished job - if status in [JobStatus.ERROR, JobStatus.FINISHED, JobStatus.CANCELLED]: - self.remove_subscriptions_for_job(future._job_id, skip_event_callback=True) + try: + record = self.rest_client.get_job(future.slurm_job_id) + if record is None: + record = self.rest_client.get_job(future.slurm_job_id, accounting=True) + except (SlurmRestError, ValueError): + logger.warning("Could not query Slurm job %s", future.slurm_job_id) + return + if not record: + return + state = record.get("job_state", record.get("state", [])) + if isinstance(state, dict): + state = state.get("current", []) + states = [state] if isinstance(state, str) else (state or []) + terminal = { + "COMPLETED", + "CANCELLED", + "FAILED", + "TIMEOUT", + "OUT_OF_MEMORY", + "NODE_FAIL", + "BOOT_FAIL", + "DEADLINE", + "PREEMPTED", + "REVOKED", + } + state = next((value for value in states if value in terminal), None) + if state is None: + return + exit_code = _job_exit_code(record.get("exit_code")) + if state == "COMPLETED" and exit_code == 0: + status = JobStatus.FINISHED.value + elif state == "CANCELLED": + status = JobStatus.CANCELLED.value + elif state == "COMPLETED" and exit_code is None: + # A partial accounting record is not proof of failure. + return + else: + status = f"error:Slurm state {state}, exit code {exit_code}" + self._job_tracker.apply_status(StatusMessage(status=status), future) diff --git a/slurm_manager/slurm_rest_client.py b/slurm_manager/slurm_rest_client.py new file mode 100644 index 0000000..de542ca --- /dev/null +++ b/slurm_manager/slurm_rest_client.py @@ -0,0 +1,296 @@ +"""Small authenticated client for the slurmrestd v0.0.44 API.""" + +from __future__ import annotations + +import json +import os +import re +from http import HTTPStatus +from http.client import HTTPException +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from dotenv import dotenv_values + +INVALID_JOB_ID = 2017 + + +def _missing_job(result: dict[str, Any]) -> bool: + """Check whether every reported Slurm error means an unknown job ID. + + Args: + result (dict[str, Any]): Decoded Slurm REST response envelope. + + Returns: + bool: True only for a nonempty list of invalid-job-ID errors. + """ + errors = result.get("errors") + return ( + isinstance(errors, list) + and bool(errors) + and all( + isinstance(error, dict) and error.get("error_number") == INVALID_JOB_ID + for error in errors + ) + ) + + +class SlurmRestError(RuntimeError): + """A transport, HTTP, or Slurm application error.""" + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *_args: Any, **_kwargs: Any) -> None: + # Never forward the authentication headers to a redirect target. + """Reject redirects so authentication headers cannot reach another endpoint. + + Args: + *_args (Any): Positional redirect-hook arguments supplied by urllib. + **_kwargs (Any): Keyword redirect-hook arguments supplied by urllib. + + Returns: + None: Redirect rejection, preventing urllib from following the response. + """ + return None + + +class SlurmRestClient: + """Load connection settings and a JWT from a local dotenv file or environment. + + Environment variables override the file. The file is read again for every + request so replacing an expired token does not require restarting a manager. + """ + + def __init__(self, env_file: str | Path = ".env", timeout: float = 30) -> None: + """Load and validate slurmrestd connection settings. + + Args: + env_file (str | Path): Local credential dotenv path. Environment variables + take precedence. + timeout (float): Positive HTTP request timeout in seconds. + + Raises: + ValueError: The URL, API version, token, or timeout is invalid. + """ + self.env_file = Path(env_file).resolve() + if timeout <= 0: + raise ValueError("REST timeout must be positive") + self.timeout = timeout + settings = self._settings() + self.url = settings.get("SLURM_REST_URL", "").rstrip("/") + parsed = urlsplit(self.url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("Set SLURM_REST_URL to the slurmrestd HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("SLURM_REST_URL must not contain credentials, query, or fragment") + self.api_version = settings.get("SLURM_REST_API_VERSION") or "v0.0.44" + if not re.fullmatch(r"v\d+\.\d+\.\d+", self.api_version): + raise ValueError("SLURM_REST_API_VERSION must look like v0.0.44") + self._headers(settings) + self._opener = build_opener(_NoRedirect()) + + ################################################ + # Public Job Operations # + ################################################ + + def submit(self, script: str, job: dict[str, Any]) -> dict[str, Any]: + """Submit a batch script and its compute-node job settings. + + Args: + script (str): Complete batch script text sent directly to slurmrestd. + job (dict[str, Any]): REST job-description fields, including compute-node + paths and resources. + + Returns: + dict[str, Any]: Submission response containing a numeric job_id. + + Raises: + SlurmRestError: Submission fails or no numeric job ID is returned. + """ + result = self.request( + "POST", f"slurm/{self.api_version}/job/submit", payload={"script": script, "job": job} + ) + if not str(result.get("job_id", "")).isdigit(): + raise SlurmRestError("Slurm REST submission did not return a job ID") + return result + + def release(self, job_id: str) -> None: + """Release a job that was submitted with hold enabled. + + Args: + job_id (str): Numeric scheduler job ID on this REST endpoint. + + Raises: + SlurmRestError: The job update fails. + """ + self.request("POST", self._job_path(job_id), payload={"hold": False}) + + def cancel(self, job_id: str) -> None: + """Cancel a job on the cluster served by this REST endpoint. + + Args: + job_id (str): Numeric scheduler job ID on this REST endpoint. + + Raises: + SlurmRestError: Cancellation fails. + """ + self.request("DELETE", self._job_path(job_id)) + + def get_job(self, job_id: str, *, accounting: bool = False) -> dict[str, Any] | None: + """Fetch a controller or accounting job record. + + Args: + job_id (str): Numeric scheduler job ID on this REST endpoint. + accounting (bool): If true, query slurmdb instead of the controller. + + Returns: + dict[str, Any] | None: First returned job record, or None when the job is + unavailable. + + Raises: + SlurmRestError: The query fails or its records are malformed. + """ + path = self._job_path(job_id, accounting=accounting) + result = self.request("GET", path, missing_ok=True) + jobs = result.get("jobs") or [] + if not isinstance(jobs, list) or any(not isinstance(job, dict) for job in jobs): + raise SlurmRestError("Slurm REST returned invalid job records") + return jobs[0] if jobs else None + + ################################################ + # Public HTTP Requests # + ################################################ + + def request( + self, + method: str, + path: str, + *, + payload: dict[str, Any] | None = None, + missing_ok: bool = False, + ) -> dict[str, Any]: + """Send one authenticated JSON request and validate the response envelope. + + Args: + method (str): HTTP method such as GET, POST, or DELETE. + path (str): Relative REST endpoint appended to the configured service URL. + payload (dict[str, Any] | None): JSON-compatible request body or incoming + message payload. + missing_ok (bool): If true, treat missing-job responses as an empty result. + + Returns: + dict[str, Any]: Decoded response object; an empty object for an allowed + missing job. + + Raises: + SlurmRestError: HTTP, transport, JSON, or Slurm application validation + fails. + ValueError: The current token is missing. + + Notes: + Requests are not automatically retried. Authentication headers and response + bodies are not included in raised transport errors. + """ + headers = self._headers(self._settings()) + data = None if payload is None else json.dumps(payload).encode() + if data is not None: + headers["Content-Type"] = "application/json" + request = Request(f"{self.url}/{path}", data=data, headers=headers, method=method) + try: + with self._opener.open(request, timeout=self.timeout) as response: + raw = response.read() + except HTTPError as exc: + try: + # slurmrestd may use HTTP 500 for a purged controller job. Check + # only its numeric error code; never include the body in logs. + error_body = json.loads(exc.read()) if missing_ok else {} + except (ValueError, UnicodeDecodeError, OSError, HTTPException): + error_body = {} + finally: + exc.close() + if missing_ok and ( + exc.code == HTTPStatus.NOT_FOUND + or ( + exc.code in {HTTPStatus.BAD_REQUEST, HTTPStatus.INTERNAL_SERVER_ERROR} + and isinstance(error_body, dict) + and _missing_job(error_body) + ) + ): + return {} + raise SlurmRestError(f"Slurm REST {method} failed (HTTP {exc.code})") from None + except (URLError, OSError, TimeoutError, HTTPException): + raise SlurmRestError(f"Slurm REST {method} failed (connection or timeout)") from None + try: + result = json.loads(raw) if raw else {} + except (ValueError, UnicodeDecodeError): + raise SlurmRestError("Slurm REST returned invalid JSON") from None + if not isinstance(result, dict): + raise SlurmRestError("Slurm REST returned an invalid response") + errors = result.get("errors") or [] + if errors: + if missing_ok and _missing_job(result): + return {} + raise SlurmRestError("Slurm REST reported an application error") + return result + + ################################################ + # Internal Authentication # + ################################################ + + def _settings(self) -> dict[str, str]: + """Read local dotenv values and apply environment-variable overrides. + + Returns: + dict[str, str]: Connection and authentication settings without modifying + os.environ. + """ + values = dotenv_values(self.env_file, interpolate=False) + keys = ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER") + return {key: os.environ.get(key, values.get(key) or "") for key in keys} + + @staticmethod + def _headers(settings: dict[str, str]) -> dict[str, str]: + """Build authentication headers from the current settings. + + Args: + settings (dict[str, str]): Connection settings containing the JWT and + optional user name. + + Returns: + dict[str, str]: JWT, optional user-name, and response-format headers. + + Raises: + ValueError: The token is empty. + """ + token = settings.get("SLURM_JWT", "") + if not token: + raise ValueError("Set SLURM_JWT in the local .env file or environment") + headers = {"X-SLURM-USER-TOKEN": token, "Accept": "application/json"} + if settings.get("SLURM_USER"): + headers["X-SLURM-USER-NAME"] = settings["SLURM_USER"] + return headers + + ################################################ + # Internal Endpoints # + ################################################ + + def _job_path(self, job_id: str, *, accounting: bool = False) -> str: + """Build a versioned controller or accounting job endpoint. + + Args: + job_id (str): Numeric scheduler job ID on this REST endpoint. + accounting (bool): If true, query slurmdb instead of the controller. + + Returns: + str: Relative REST endpoint path. + + Raises: + ValueError: The scheduler job ID is not numeric. + """ + if not str(job_id).isdigit(): + raise ValueError("Slurm job ID must be numeric") + service = "slurmdb" if accounting else "slurm" + return f"{service}/{self.api_version}/job/{job_id}" diff --git a/slurm_manager/utils/__init__.py b/slurm_manager/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/slurm_manager/utils/utils.py b/slurm_manager/utils/utils.py deleted file mode 100644 index dc08437..0000000 --- a/slurm_manager/utils/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -SLURM_JOB_STATE_CODES = { - "BF": "BOOT_FAIL", # Job terminated due to launch failure, typically due to a hardware failure (e.g. unable to boot the node or block and the job can not be requeued). - "CA": "CANCELLED", # Job was explicitly cancelled by the user or system administrator. The job may or may not have been initiated. - "CD": "COMPLETED", # Job has terminated all processes on all nodes with an exit code of zero. - "CF": "CONFIGURING", # Job has been allocated resources, but are waiting for them to become ready for use (e.g. booting). - "CG": "COMPLETING", # Job is in the process of completing. Some processes on some nodes may still be active. - "DL": "DEADLINE", # Job terminated on deadline. - "F": "FAILED", # Job terminated with non-zero exit code or other failure condition. - "NF": "NODE_FAIL", # Job terminated due to failure of one or more allocated nodes. - "OOM": "OUT_OF_MEMORY", # Job experienced out of memory error. - "PD": "PENDING", # Job is awaiting resource allocation. - "PR": "PREEMPTED", # Job terminated due to preemption. - "R": "RUNNING", # Job currently has an allocation. - "RD": "RESV_DEL_HOLD", # Job is being held after requested reservation was deleted. - "RF": "REQUEUE_FED", # Job is being requeued by a federation. - "RH": "REQUEUE_HOLD", # Held job is being requeued. - "RQ": "REQUEUED", # Completing job is being requeued. - "RS": "RESIZING", # Job is about to change size. - "RV": "REVOKED", # Sibling was removed from cluster due to other cluster starting the job. - "SI": "SIGNALING", # Job is being signaled. - "SE": "SPECIAL_EXIT", # The job was requeued in a special state. This state can be set by users, typically in EpilogSlurmctld, if the job has terminated with a particular exit value. - "SO": "STAGE_OUT", # Job is staging out files. - "ST": "STOPPED", # Job has an allocation, but execution has been stopped with SIGSTOP signal. CPUS have been retained by this job. - "S": "SUSPENDED", # Job has an allocation, but execution has been suspended and CPUs have been released for other jobs. - "TO": "TIMEOUT", # Job terminated upon reaching its time limit -} diff --git a/test/test_job_options.py b/test/test_job_options.py new file mode 100644 index 0000000..549d414 --- /dev/null +++ b/test/test_job_options.py @@ -0,0 +1,51 @@ +"""Reject invalid resource requests before sending them to Slurm.""" + +import pytest +from pydantic import ValidationError + +from slurm_manager.job_options import JobOptions + + +@pytest.mark.parametrize( + "field,value", + [ + ("tasks", 0), + ("tasks", -1), + ("tasks", True), + ("tasks", 1.5), + ("time_limit_minutes", 0), + ("time_limit_minutes", -1), + ("time_limit_minutes", "60"), + ("partition", " "), + ("hold", False), + ("time_limit", {"set": True, "number": 1}), + ], +) +def test_invalid_or_internal_options_are_rejected(field, value): + with pytest.raises(ValidationError): + JobOptions(**{field: value}) + + +def test_unset_options_preserve_slurm_defaults(): + assert JobOptions().to_rest() == {"tasks": 1} + + +def test_runtime_uses_minutes_and_slurm_optional_number_shape(): + options = JobOptions(partition="hour", tasks=2, time_limit_minutes=60) + assert options.to_rest() == { + "partition": "hour", + "tasks": 2, + "time_limit": {"set": True, "number": 60}, + } + assert options.time_limit_minutes == 60 + + +@pytest.mark.parametrize("partition", ["hour", "day", "week", "gpu"]) +def test_supported_ra_partitions(partition): + assert JobOptions(partition=partition).to_rest()["partition"] == partition + + +@pytest.mark.parametrize("partition", ["admin", "jupyter", "gpu-large", "unknown"]) +def test_unsupported_partitions_are_rejected(partition): + with pytest.raises(ValidationError): + JobOptions(partition=partition) diff --git a/test/test_job_script.py b/test/test_job_script.py new file mode 100644 index 0000000..62720b6 --- /dev/null +++ b/test/test_job_script.py @@ -0,0 +1,48 @@ +"""Verify local scripts and configuration reach the worker without shell expansion.""" + +import json +import subprocess +import sys + +import pytest + +from slurm_manager.slurm_manager import SlurmManager + + +def test_local_scripts_are_passed_to_worker_as_json(tmp_path): + workload = tmp_path / "work load.sh" + activation = tmp_path / "activate.sh" + workload.write_text('#!/bin/bash\nprintf "$HOME and $value"\n') + activation.write_text("export TEST_VALUE='literal $value and spaces'\n") + worker = tmp_path / "worker $(touch must-not-exist)" + worker.write_text( + f"#!{sys.executable}\nimport json,sys\n" + "print(json.dumps({'args':sys.argv[1:], 'payload':json.load(sys.stdin)}))\n" + ) + worker.chmod(0o700) + script = SlurmManager._build_job_script( + workload, activation, "job-id", worker_path=str(worker), redis_address="redis.example:6380" + ) + result = subprocess.run( + ["bash"], input=script, capture_output=True, text=True, check=True, cwd=tmp_path + ) + received = json.loads(result.stdout) + assert received["payload"] == { + "script": workload.read_text(), + "activation": activation.read_text(), + } + assert received["args"] == ["--job-id", "job-id", "--redis-address", "redis.example:6380"] + assert str(workload) not in script + assert not (tmp_path / "must-not-exist").exists() + + +def test_missing_local_file_fails_before_packaging(tmp_path): + with pytest.raises(FileNotFoundError): + SlurmManager._build_job_script(tmp_path / "missing.sh", tmp_path / "activate.sh", "job-id") + + +def test_worker_path_must_be_absolute(tmp_path): + with pytest.raises(ValueError, match="absolute"): + SlurmManager._build_job_script( + tmp_path / "unused.sh", tmp_path / "unused-env.sh", "job-id", worker_path="worker" + ) diff --git a/test/test_job_tracker.py b/test/test_job_tracker.py new file mode 100644 index 0000000..8a1d33a --- /dev/null +++ b/test/test_job_tracker.py @@ -0,0 +1,68 @@ +"""Exercise tracking independently of REST submission and scheduler monitoring.""" + +from functools import partial +from unittest import mock + +import fakeredis +import pytest + +from slurm_manager.event_messages import StatusMessage +from slurm_manager.job_future import JobFuture +from slurm_manager.job_tracker import JobTracker, TopicInfo +from slurm_manager.redis_sub_client import RedisSubClient + + +@pytest.fixture +def tracker(): + client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True)) + tracker = JobTracker(client) + try: + yield tracker + finally: + tracker.shutdown(timeout=3) + + +def test_heartbeat_snapshots_cannot_mutate_tracking_state(tracker): + future = JobFuture("first", manager=None, slurm_job_id="123") + with mock.patch("slurm_manager.job_tracker.time.time", return_value=80): + tracker.track(future) + + with mock.patch("slurm_manager.job_tracker.time.time", return_value=100): + snapshot = tracker.get_missing_heartbeats()[0] + snapshot.last_heartbeat = 0 + assert tracker.get_missing_heartbeats()[0].last_heartbeat == 80 + assert tracker.begin_scheduler_check("first") is future + assert tracker.get_missing_heartbeats() == [] + + +def test_completion_preserves_other_jobs_and_their_redis_subscriptions(tracker): + first = JobFuture("first", manager=None, slurm_job_id="123") + second = JobFuture("second", manager=None, slurm_job_id="124") + tracker.track(first) + tracker.track(second) + tracker.wait_until_ready("second") + + tracker.apply_status(StatusMessage(status="finished"), first) + + assert first.success() + assert tracker.get_future("first") is None + assert tracker.get_future("second") is second + assert not second.done() + assert tracker._sub_client.registered_topics == { + TopicInfo("second").heartbeat, + TopicInfo("second").event, + } + tracker._sub_client._redis.publish( + TopicInfo("second").event, '{"msg_type":"status","status":"finished"}' + ) + second.wait(timeout=1) + assert second.success() + + +def test_scheduler_check_skips_a_job_removed_after_snapshot(tracker): + future = JobFuture("first", manager=None, slurm_job_id="123") + tracker.track(future) + tracker.apply_status(StatusMessage(status="finished"), future) + + assert tracker.begin_scheduler_check("first") is None + assert tracker.begin_scheduler_check("unknown") is None diff --git a/test/test_review_regressions.py b/test/test_review_regressions.py index 855261a..bc02081 100644 --- a/test/test_review_regressions.py +++ b/test/test_review_regressions.py @@ -2,10 +2,10 @@ from __future__ import annotations -import subprocess import threading import time from functools import partial +from pathlib import Path from unittest import mock import fakeredis @@ -13,23 +13,22 @@ import pytest from slurm_manager.event_messages import HeartBeatMessage, StatusMessage from slurm_manager.job_future import JobFuture +from slurm_manager.job_options import JobOptions +from slurm_manager.job_tracker import TopicInfo from slurm_manager.redis_sub_client import RedisSubClient -from slurm_manager.slurm_manager import SlurmManager, TopicInfo +from slurm_manager.slurm_manager import SlurmManager +from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError @pytest.fixture def manager(): client = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True)) + rest = mock.Mock(spec=SlurmRestClient) + rest.submit.return_value = {"job_id": 123} + rest.get_job.return_value = {"job_state": ["PENDING"]} with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=client): - manager = SlurmManager() + manager = SlurmManager(rest_client=rest) - def slurm(command): - output = ( - "123\n" if command[0] == "sbatch" else "PENDING\n" if command[0] == "squeue" else "" - ) - return subprocess.CompletedProcess(command, 0, output, "") - - manager._run_subprocess = mock.Mock(side_effect=slurm) try: yield manager finally: @@ -40,13 +39,20 @@ def manager(): def submit(manager): - # The workload and activation environment belong to the remote node. They - # deliberately do not exist in the submitter's Python environment/filesystem. - return manager.submit_job("/remote/node/workload.sh", "/remote/node/activate.sh") + return manager.submit_job( + Path(__file__).parent.parent / "examples/print_for_10s.sh", + working_directory="/remote/node/work", + ) def heartbeat_info(manager, future): - return next(iter(manager._subscriptions_by_topic[TopicInfo(future._job_id).heartbeat].values())) + return next( + iter( + manager._job_tracker._subscriptions_by_topic[ + TopicInfo(future._job_id).heartbeat + ].values() + ) + ) def test_wait_propagates_failure_arriving_during_wait(): @@ -79,46 +85,34 @@ def test_terminal_state_ignores_late_status_updates(): def test_job_is_held_until_redis_acknowledges_subscriptions(manager): - commands = [] - - def slurm(command): - commands.append(command) - if command[0] == "sbatch": - assert "--hold" in command - assert "/remote/node/activate.sh" in command - assert "/remote/node/workload.sh" in command - assert command[-1].endswith("redis_worker.sh") - return subprocess.CompletedProcess(command, 0, "123\n", "") - assert command == ["scontrol", "release", "123"] - future = next(iter(manager._future_registry.values())) + def release(job_id): + assert job_id == "123" + script, job = manager.rest_client.submit.call_args.args + assert job["hold"] is True + assert job["current_working_directory"] == "/remote/node/work" + assert '"activation": ""' in script + assert "stdout-test %02d/10" in script + assert "/usr/local/bin/slurm-worker" in script + future = next(iter(manager._job_tracker._future_registry.values())) topics = TopicInfo(future._job_id) assert manager.sub_client._subscription_ready[topics.event].is_set() assert manager.sub_client._subscription_ready[topics.heartbeat].is_set() - # Simulate a remote job completing before scontrol returns. manager.sub_client._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}') future.wait(timeout=1) - return subprocess.CompletedProcess(command, 0, "", "") - manager._run_subprocess.side_effect = slurm + manager.rest_client.release.side_effect = release future = submit(manager) assert future.success() - assert [command[0] for command in commands] == ["sbatch", "scontrol"] assert not manager.sub_client.registered_topics @pytest.mark.parametrize("failure", ["subscription", "acknowledgment", "release"]) def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failure): - original = manager._run_subprocess.side_effect - - def slurm(command): - if command[0] == "scontrol": - return subprocess.CompletedProcess(command, 1, "", "release rejected") - return original(command) - - manager._run_subprocess.side_effect = slurm + if failure == "release": + manager.rest_client.release.side_effect = SlurmRestError("release rejected") if failure == "subscription": patch = mock.patch.object( - manager, "_subscribe_to_event", side_effect=RuntimeError("offline") + manager._job_tracker, "_subscribe_to_event", side_effect=RuntimeError("offline") ) elif failure == "acknowledgment": patch = mock.patch.object( @@ -132,33 +126,64 @@ def test_setup_failure_cancels_held_job_and_cleans_subscriptions(manager, failur ) with patch, pytest.raises((RuntimeError, TimeoutError)): submit(manager) - assert mock.call(["scancel", "123"]) in manager._run_subprocess.call_args_list - assert not manager._future_registry - assert not manager._active_subscriptions + manager.rest_client.cancel.assert_called_once_with("123") + assert not manager._job_tracker._future_registry + assert not manager._job_tracker._active_subscriptions assert not manager.sub_client.registered_topics +@pytest.mark.parametrize( + "last_heartbeat,last_check,expected", + [ + (None, None, False), + (0, None, False), + (95, None, False), + (90, None, False), + (89, None, True), + (89, 95, False), + (89, 90, False), + (89, 89, True), + ], +) +def test_heartbeat_selection_respects_timeout_and_scheduler_throttle( + manager, last_heartbeat, last_check, expected +): + manager._stop_heartbeat_thread.set() + manager._heartbeat_monitoring_thread.join(timeout=1) + future = submit(manager) + future.listen("heartbeat", lambda message: None) + future.listen("log", lambda message: None) + info = heartbeat_info(manager, future) + info.last_heartbeat = last_heartbeat + info.last_scheduler_check = last_check + + with mock.patch("slurm_manager.job_tracker.time.time", return_value=100): + missing = manager._job_tracker.get_missing_heartbeats() + + assert missing == ([info] if expected else []) + manager.rest_client.get_job.assert_not_called() + + @pytest.mark.parametrize("state", ["PENDING", "CONFIGURING", "RUNNING"]) def test_missing_heartbeat_does_not_fail_active_slurm_job(manager, state): future = submit(manager) info = heartbeat_info(manager, future) info.last_heartbeat = time.time() - 100 - manager._run_subprocess.return_value = subprocess.CompletedProcess([], 0, f"{state}\n", "") - manager._run_subprocess.side_effect = None + manager.rest_client.get_job.return_value = {"job_state": [state]} manager._check_slurm_job(info) assert not future.done() - assert future._job_id in manager._future_registry - assert TopicInfo(future._job_id).heartbeat in manager._active_subscriptions[future._job_id] - assert manager._run_subprocess.call_args.args[0][0] == "squeue" + assert future._job_id in manager._job_tracker._future_registry + assert ( + TopicInfo(future._job_id).heartbeat + in manager._job_tracker._active_subscriptions[future._job_id] + ) + manager.rest_client.get_job.assert_called_once_with("123") -@pytest.mark.parametrize("accounting", ["", "PENDING|0:0\n"]) +@pytest.mark.parametrize("accounting", [None, {"state": {"current": ["PENDING"]}}]) def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting): future = submit(manager) - manager._run_subprocess.side_effect = [ - subprocess.CompletedProcess([], 0, "", ""), - subprocess.CompletedProcess([], 0, accounting, ""), - ] + manager.rest_client.get_job.side_effect = [None, accounting] manager._check_slurm_job(heartbeat_info(manager, future)) assert not future.done() @@ -166,28 +191,25 @@ def test_unknown_or_delayed_accounting_is_not_failure(manager, accounting): @pytest.mark.parametrize( "accounting,status,success", [ - ("COMPLETED|0:0\n", "finished", True), - ("FAILED|7:0\n", "error", False), - ("CANCELLED by 1000|0:15\n", "cancelled", False), + ({"job_state": ["COMPLETED"], "exit_code": 0}, "finished", True), + ({"job_state": ["FAILED"], "exit_code": 7}, "error", False), + ({"job_state": ["CANCELLED"], "exit_code": 15}, "cancelled", False), ], ) def test_terminal_accounting_resolves_missed_remote_status(manager, accounting, status, success): future = submit(manager) - manager._run_subprocess.side_effect = [ - subprocess.CompletedProcess([], 0, "", ""), - subprocess.CompletedProcess([], 0, accounting, ""), - ] + manager.rest_client.get_job.side_effect = [None, accounting] manager._check_slurm_job(heartbeat_info(manager, future)) assert future.done() assert future.status == status assert future.success() == success - assert not manager._future_registry + assert not manager._job_tracker._future_registry assert not manager.sub_client.registered_topics def test_scheduler_error_keeps_job_unresolved(manager): future = submit(manager) - manager._run_subprocess.side_effect = FileNotFoundError("squeue unavailable") + manager.rest_client.get_job.side_effect = SlurmRestError("unavailable") manager._check_slurm_job(heartbeat_info(manager, future)) assert not future.done() @@ -201,7 +223,7 @@ def test_unlisten_removes_only_selected_callback_and_preserves_monitoring(manage future.unlisten(callback_id) topic = TopicInfo(future._job_id).heartbeat assert callback_id not in manager.sub_client._topic_by_subscription_id - assert topic in manager._active_subscriptions[future._job_id] + assert topic in manager._job_tracker._active_subscriptions[future._job_id] manager.sub_client._redis.publish( topic, HeartBeatMessage(timestamp=time.time()).model_dump_json() ) @@ -217,67 +239,36 @@ def test_removing_last_log_callback_unsubscribes_topic(manager): callback_id = future.listen("log", lambda _: None) future.unlisten(callback_id) topic = TopicInfo(future._job_id).log - assert topic not in manager._active_subscriptions[future._job_id] + assert topic not in manager._job_tracker._active_subscriptions[future._job_id] assert topic not in manager.sub_client.registered_topics assert TopicInfo(future._job_id).heartbeat in manager.sub_client.registered_topics -def cluster_submission(manager): - original = manager._run_subprocess.side_effect - - def slurm(command): - if command[0] == "sbatch": - return subprocess.CompletedProcess(command, 0, "123;cluster_a\n", "") - return original(command) - - manager._run_subprocess.side_effect = slurm - - -def test_cluster_qualified_job_is_released_and_cancelled_on_its_cluster(manager): - cluster_submission(manager) +@pytest.mark.parametrize("code,success", [(0, True), (7, False)]) +def test_accounting_structured_exit_code(manager, code, success): future = submit(manager) - assert future.slurm_job_id == "123" - assert future.slurm_cluster == "cluster_a" - assert mock.call(["scontrol", "--clusters", "cluster_a", "release", "123"]) in ( - manager._run_subprocess.call_args_list - ) - future.cancel() - assert manager._run_subprocess.call_args == mock.call( - ["scancel", "--clusters", "cluster_a", "123"] - ) - - -def test_cluster_qualified_setup_failure_cancels_on_its_cluster(manager): - cluster_submission(manager) - with ( - mock.patch.object( - manager.sub_client, "wait_until_subscribed", side_effect=TimeoutError("no ack") - ), - pytest.raises(TimeoutError), - ): - submit(manager) - assert manager._run_subprocess.call_args == mock.call( - ["scancel", "--clusters", "cluster_a", "123"] - ) - assert not manager._future_registry - assert not manager.sub_client.registered_topics - - -def test_cluster_qualified_job_monitoring_queries_its_cluster(manager): - cluster_submission(manager) - future = submit(manager) - manager._run_subprocess.reset_mock() - manager._run_subprocess.side_effect = [ - subprocess.CompletedProcess([], 0, "", ""), - subprocess.CompletedProcess([], 0, "COMPLETED|0:0\n", ""), + manager.rest_client.get_job.side_effect = [ + None, + { + "state": {"current": ["COMPLETED"]}, + "exit_code": { + "return_code": {"set": True, "number": code}, + "signal": {"id": {"set": False, "number": 0}}, + }, + }, ] manager._check_slurm_job(heartbeat_info(manager, future)) - commands = [call.args[0] for call in manager._run_subprocess.call_args_list] - assert [command[0] for command in commands] == ["squeue", "sacct"] - for command in commands: - assert command[1:3] == ["--clusters", "cluster_a"] - assert command[command.index("--jobs") + 1] == "123" - assert future.success() + assert future.done() + assert future.success() == success + assert manager.rest_client.get_job.call_args == mock.call("123", accounting=True) + + +def test_cancel_uses_rest_and_cleans_subscriptions(manager): + future = submit(manager) + future.cancel() + manager.rest_client.cancel.assert_called_once_with("123") + assert future.status == "cancelled" + assert not manager.sub_client.registered_topics def test_completion_during_subscription_registration_leaves_no_orphan(manager): @@ -287,7 +278,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager): cleaned = threading.Event() errors = [] original_subscribe = manager.sub_client.subscribe - original_event = manager._event_callback + original_event = manager._job_tracker.apply_status def subscribe(topic, callback): callback_id = original_subscribe(topic, callback) @@ -307,7 +298,7 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager): with ( mock.patch.object(manager.sub_client, "subscribe", side_effect=subscribe), - mock.patch.object(manager, "_event_callback", side_effect=event), + mock.patch.object(manager._job_tracker, "apply_status", side_effect=event), ): thread = threading.Thread(target=listen) thread.start() @@ -323,16 +314,16 @@ def test_completion_during_subscription_registration_leaves_no_orphan(manager): assert not thread.is_alive() assert cleaned.wait(1) assert len(errors) == 1 - assert not manager._future_registry - assert not manager._active_subscriptions - assert not manager._topic_by_subscription_id + assert not manager._job_tracker._future_registry + assert not manager._job_tracker._active_subscriptions + assert not manager._job_tracker._topic_by_subscription_id assert not manager.sub_client.registered_topics assert not manager.sub_client._topic_by_subscription_id def test_listen_after_completion_does_not_register_callback(manager): future = submit(manager) - manager._event_callback(StatusMessage(status="finished"), future) + manager._job_tracker.apply_status(StatusMessage(status="finished"), future) with pytest.raises(RuntimeError, match="no longer active"): future.listen("log", lambda _: None) assert not manager.sub_client.registered_topics @@ -377,5 +368,125 @@ def test_unlisten_allows_concurrent_job_cleanup(manager): assert completed_during_removal == [True] assert future.done() assert not future._callback_registry - assert not manager._future_registry + assert not manager._job_tracker._future_registry assert not manager.sub_client.registered_topics + + +@pytest.mark.parametrize("code", [0, 7]) +def test_controller_structured_exit_code(manager, code): + future = submit(manager) + manager.rest_client.get_job.return_value = { + "job_state": ["COMPLETED"], + "exit_code": { + "status": ["SUCCESS" if code == 0 else "ERROR"], + "return_code": {"set": True, "number": code}, + "signal": {"id": {"set": False, "number": 0}}, + }, + } + manager._check_slurm_job(heartbeat_info(manager, future)) + assert future.done() + assert future.success() == (code == 0) + + +def test_local_files_are_uploaded_without_mutating_options(manager, tmp_path): + workload = tmp_path / "work load $(touch should_not_exist).sh" + activation = tmp_path / "it's my activation.sh" + workload.write_text("#!/bin/bash\nprintf 'hello'\n") + activation.write_text("module load redis\n") + options = JobOptions(partition="hour", tasks=2, time_limit_minutes=5) + manager.submit_job(workload, activation, working_directory="/remote/work", job_options=options) + script, job = manager.rest_client.submit.call_args.args + assert "hello" in script + assert "module load redis" in script + assert str(tmp_path) not in script + assert job["hold"] is True + assert options == JobOptions(partition="hour", tasks=2, time_limit_minutes=5) + assert job["tasks"] == 2 + assert job["time_limit"] == {"set": True, "number": 5} + + +def test_submission_failure_does_not_register_job(manager): + manager.rest_client.submit.side_effect = SlurmRestError("submission failed") + with pytest.raises(SlurmRestError, match="submission failed"): + submit(manager) + assert not manager._job_tracker._future_registry + assert not manager.sub_client.registered_topics + manager.rest_client.release.assert_not_called() + manager.rest_client.cancel.assert_not_called() + + +def test_cancellation_failure_keeps_monitoring(manager): + future = submit(manager) + manager.rest_client.cancel.side_effect = SlurmRestError("cancellation failed") + with pytest.raises(SlurmRestError, match="cancellation failed"): + future.cancel() + assert not future.done() + assert future._job_id in manager._job_tracker._future_registry + + +def test_relative_remote_paths_are_rejected_before_submission(manager): + with pytest.raises(ValueError, match="absolute"): + manager.submit_job("workload.sh", "activate.sh", working_directory="relative") + manager.rest_client.submit.assert_not_called() + + +def test_public_callbacks_are_ready_before_release(manager): + received = threading.Event() + + def release(_job_id): + future = next(iter(manager._job_tracker._future_registry.values())) + topic = TopicInfo(future._job_id).log + assert manager.sub_client._subscription_ready[topic].is_set() + manager.sub_client._redis.publish(topic, '{"msg_type":"log","log":"first line"}') + assert received.wait(1) + + manager.rest_client.release.side_effect = release + manager.submit_job( + "examples/print_for_10s.sh", + working_directory="/remote/work", + callbacks={"log": lambda _: received.set()}, + ) + + +def test_unknown_status_cleans_up_the_terminal_future(manager): + future = submit(manager) + future.listen("log", lambda _: None) + manager._job_tracker.apply_status(StatusMessage(status="unexpected-status"), future) + assert future.done() + assert future.status == "error" + assert not manager._job_tracker._future_registry + assert not manager._job_tracker._active_subscriptions + assert not manager._job_tracker._subscriptions_by_topic + assert not manager._job_tracker._topic_by_subscription_id + assert not manager.sub_client.registered_topics + + +def test_shutdown_drains_all_received_log_callbacks(manager): + future = submit(manager) + release = threading.Event() + received = [] + blocking = [manager._job_tracker._executor.submit(release.wait) for _ in range(8)] + future.listen("log", lambda message: received.append(message.log)) + topic = TopicInfo(future._job_id) + for number in range(25): + manager.sub_client._redis.publish(topic.log, f'{{"msg_type":"log","log":"{number}"}}') + manager.sub_client._redis.publish(topic.event, '{"msg_type":"status","status":"finished"}') + future.wait(timeout=2) + # Release blocked workers only once shutdown reaches the executor. With the + # old cancel_futures=True, all 25 accepted callbacks would be discarded here. + original = manager._job_tracker._executor.shutdown + + def shutdown(*, wait=True, cancel_futures=False): + original(wait=False, cancel_futures=cancel_futures) + release.set() + if wait: + original(wait=True) + + try: + with mock.patch.object(manager._job_tracker._executor, "shutdown", side_effect=shutdown): + manager.shutdown(timeout=3) + finally: + release.set() + for task in blocking: + task.result(timeout=1) + assert sorted(received, key=int) == [str(number) for number in range(25)] diff --git a/test/test_slurm_rest_client.py b/test/test_slurm_rest_client.py new file mode 100644 index 0000000..f8bf761 --- /dev/null +++ b/test/test_slurm_rest_client.py @@ -0,0 +1,213 @@ +"""REST wire-format, authentication, and error handling tests.""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from slurm_manager.slurm_rest_client import SlurmRestClient, SlurmRestError + + +@pytest.fixture +def endpoint(tmp_path, monkeypatch): + for key in ("SLURM_REST_URL", "SLURM_REST_API_VERSION", "SLURM_JWT", "SLURM_USER"): + monkeypatch.delenv(key, raising=False) + calls = [] + replies = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.respond() + + def do_POST(self): + self.respond() + + def do_DELETE(self): + self.respond() + + def respond(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + calls.append( + (self.command, self.path, dict(self.headers), json.loads(body) if body else None) + ) + status, payload = replies.pop(0) + self.send_response(status) + if status == 302: + self.send_header("Location", "/redirect-target") + self.end_headers() + self.wfile.write( + payload if isinstance(payload, bytes) else json.dumps(payload).encode() + ) + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + env_file = tmp_path / ".env" + env_file.write_text( + f"SLURM_REST_URL=http://127.0.0.1:{server.server_port}\n" + "SLURM_REST_API_VERSION=v0.0.44\nSLURM_JWT=test-secret\n" + ) + try: + yield env_file, calls, replies + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + assert not thread.is_alive() + + +def test_submit_release_cancel_and_accounting_wire_format(endpoint): + env_file, calls, replies = endpoint + replies.extend( + [ + (200, {"job_id": 123, "errors": []}), + (200, {"errors": []}), + (200, {"jobs": [{"job_state": ["RUNNING"]}]}), + (200, {"jobs": [{"state": {"current": ["COMPLETED"]}}]}), + (200, {"errors": []}), + ] + ) + client = SlurmRestClient(env_file) + job = { + "current_working_directory": "/remote/work", + "environment": ["PATH=/usr/bin"], + "hold": True, + } + assert client.submit("#!/bin/bash\necho hello", job)["job_id"] == 123 + client.release("123") + assert client.get_job("123")["job_state"] == ["RUNNING"] + assert client.get_job("123", accounting=True)["state"]["current"] == ["COMPLETED"] + client.cancel("123") + assert [(method, path) for method, path, _, _ in calls] == [ + ("POST", "/slurm/v0.0.44/job/submit"), + ("POST", "/slurm/v0.0.44/job/123"), + ("GET", "/slurm/v0.0.44/job/123"), + ("GET", "/slurmdb/v0.0.44/job/123"), + ("DELETE", "/slurm/v0.0.44/job/123"), + ] + assert calls[0][3] == {"script": "#!/bin/bash\necho hello", "job": job} + assert calls[1][3] == {"hold": False} + assert calls[0][2]["X-Slurm-User-Token"] == "test-secret" + assert "test-secret" not in json.dumps(calls[0][3]) + + +def test_token_rotation_and_environment_override(endpoint, monkeypatch): + env_file, calls, replies = endpoint + replies.extend([(200, {}), (200, {})]) + client = SlurmRestClient(env_file) + env_file.write_text(env_file.read_text().replace("test-secret", "replacement-token")) + client.cancel("1") + monkeypatch.setenv("SLURM_JWT", "environment-token") + monkeypatch.setenv("SLURM_USER", "alice") + client.cancel("2") + assert calls[0][2]["X-Slurm-User-Token"] == "replacement-token" + assert calls[1][2]["X-Slurm-User-Token"] == "environment-token" + assert calls[1][2]["X-Slurm-User-Name"] == "alice" + + +@pytest.mark.parametrize( + "status,payload", + [ + (401, b"test-secret"), + (500, b"test-secret"), + (200, {"errors": [{"description": "test-secret", "error_number": 1}]}), + (200, b"bad JSON test-secret"), + (200, []), + (302, b"redirect"), + ], +) +def test_errors_are_raised_without_exposing_credentials(endpoint, status, payload): + env_file, calls, replies = endpoint + replies.append((status, payload)) + client = SlurmRestClient(env_file) + with pytest.raises(SlurmRestError) as error: + client.cancel("123") + assert "test-secret" not in str(error.value) + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "reply", + [ + (404, {}), + (500, {"errors": [{"error_number": 2017}]}), + (200, {"jobs": []}), + (200, {"errors": [{"error_number": 2017}]}), + ], +) +def test_missing_job_is_not_a_terminal_failure(endpoint, reply): + env_file, _, replies = endpoint + replies.append(reply) + assert SlurmRestClient(env_file).get_job("123") is None + + +def test_missing_token_fails_before_any_request(endpoint): + env_file, calls, _ = endpoint + env_file.write_text(env_file.read_text().replace("test-secret", "")) + with pytest.raises(ValueError, match="SLURM_JWT"): + SlurmRestClient(env_file) + assert calls == [] + + +def test_submission_requires_job_id(endpoint): + env_file, _, replies = endpoint + replies.append((200, {"errors": []})) + with pytest.raises(SlurmRestError, match="job ID"): + SlurmRestClient(env_file).submit("#!/bin/bash", {}) + + +def test_manager_submission_and_completion_over_http(endpoint): + from functools import partial + from unittest import mock + + import fakeredis + + from slurm_manager.job_tracker import TopicInfo + from slurm_manager.redis_sub_client import RedisSubClient + from slurm_manager.slurm_manager import SlurmManager + + env_file, calls, replies = endpoint + replies.extend([(200, {"job_id": 123}), (200, {})]) + redis = RedisSubClient(redis_cls=partial(fakeredis.FakeRedis, decode_responses=True)) + with mock.patch("slurm_manager.slurm_manager.RedisSubClient", return_value=redis): + manager = SlurmManager(env_file=env_file) + try: + future = manager.submit_job("examples/print_for_10s.sh", working_directory="/remote/work") + assert calls[0][3]["job"]["hold"] is True + assert calls[1][3] == {"hold": False} + topics = TopicInfo(future._job_id) + assert redis._subscription_ready[topics.event].is_set() + redis._redis.publish(topics.event, '{"msg_type":"status","status":"finished"}') + future.wait(timeout=1) + assert future.success() + finally: + manager.shutdown(timeout=3) + assert not manager._heartbeat_monitoring_thread.is_alive() + assert not redis._listener_thread.is_alive() + + +@pytest.mark.parametrize("error_response", [False, True]) +def test_truncated_responses_raise_a_transport_error(endpoint, error_response): + from http.client import IncompleteRead + from unittest import mock + from urllib.error import HTTPError + + env_file, _, _ = endpoint + client = SlurmRestClient(env_file) + response = mock.MagicMock() + response.read.side_effect = IncompleteRead(b"partial", 100) + if error_response: + error = HTTPError("http://local/job", 500, "Server error", {}, response) + patch = mock.patch.object(client._opener, "open", side_effect=error) + else: + opened = mock.MagicMock() + opened.__enter__.return_value = response + patch = mock.patch.object(client._opener, "open", return_value=opened) + with patch, pytest.raises(SlurmRestError): + client.get_job("123")