From ee036873cdde5a41602a0e47a612e8ab4263c0fb Mon Sep 17 00:00:00 2001 From: x01dc Date: Tue, 14 Jul 2026 13:00:08 +0200 Subject: [PATCH] fix(bec_widgets): let delete/clear escape a stale "running" queue job Real incident: a job got stuck at status "running" after its process died mid-scan (Ctrl-C during a repeated confirmation-prompt loop, a separate bug fixed alongside this one) with no live process behind it -- and the GUI's delete/clear guard treated "running" as an absolute block, with no way to escape from the GUI at all (the CLI's tomo_queue_delete() has no such guard and already worked, but that's not obvious/discoverable from the GUI). Adds a staleness check using the same tomo_progress heartbeat signal TomoParamsWidget._is_tomo_running() already uses: if a "running" job's heartbeat is missing or older than 120s, it's very likely orphaned, not actively executing. Delete/clear now offer an explicit confirmation ("looks stale -- proceed anyway?") in that case instead of an unconditional refusal; a genuinely fresh heartbeat still blocks outright as before. Co-Authored-By: Claude Sonnet 5 --- .../widgets/tomo_params/tomo_params.py | 79 +++++++++++++++---- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/csaxs_bec/bec_widgets/widgets/tomo_params/tomo_params.py b/csaxs_bec/bec_widgets/widgets/tomo_params/tomo_params.py index 5789368..e21b791 100644 --- a/csaxs_bec/bec_widgets/widgets/tomo_params/tomo_params.py +++ b/csaxs_bec/bec_widgets/widgets/tomo_params/tomo_params.py @@ -1122,6 +1122,16 @@ class TomoQueueDialog(QDialog): _POLL_MS = 2000 + # Same heartbeat-staleness window as TomoParamsWidget._is_tomo_running(). + # A "running" job whose tomo_progress heartbeat is older than this (or + # missing) means the process that was executing it died -- crash, + # Ctrl-C, kernel restart -- before it had a chance to update the job's + # status (see tomo_queue_execute()'s own docstring for this exact + # scenario). Distinguishing that from a genuinely active job is what + # lets delete/clear escape a permanently stuck "running" row instead of + # blocking it forever. + _RUNNING_STALE_AFTER_S = 120 + # Statuses a job must have to be reorderable in sort mode. "running" and # "incomplete" are both hard floors: tomo_queue_execute()'s pick-next # loop always resumes whichever of them it finds first, regardless of @@ -1434,19 +1444,57 @@ class TomoQueueDialog(QDialog): self._table.selectRow(other) self._update_sort_buttons() + def _running_job_is_stale(self) -> bool: + """True if there's no evidence a scan is genuinely active right now + -- tomo_progress's heartbeat is missing or older than + _RUNNING_STALE_AFTER_S. Same signal/window as + TomoParamsWidget._is_tomo_running().""" + prog = self._gv_get("tomo_progress") + if not isinstance(prog, dict): + return True + heartbeat_str = prog.get("heartbeat") + if heartbeat_str is None: + return True + try: + heartbeat = datetime.datetime.fromisoformat(heartbeat_str) + except (ValueError, TypeError): + return True + age_s = (datetime.datetime.now() - heartbeat).total_seconds() + return age_s >= self._RUNNING_STALE_AFTER_S + + def _confirm_stale_running_override(self, noun: str) -> bool: + """A 'running' job is blocking delete/clear, but the heartbeat says + nothing is actually active -- offer to override instead of blocking + forever. Returns True if the operator chose to proceed.""" + reply = QMessageBox.warning( + self, + "Job shows 'running' but looks stale", + f"A job marked 'running' is blocking this {noun}, but no live scan " + "activity has been detected recently (heartbeat is stale or " + "missing). This usually means the session that was running it " + "crashed or was interrupted (e.g. Ctrl-C) before it could update " + "the job's status.\n\n" + f"Proceed with the {noun} anyway?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.Cancel, + ) + return reply == QMessageBox.StandardButton.Yes + def _delete_selected(self) -> None: rows = sorted({idx.row() for idx in self._table.selectedIndexes()}, reverse=True) if not rows: return jobs = self._load_queue() if any(row < len(jobs) and jobs[row].get("status") == "running" for row in rows): - QMessageBox.warning( - self, - "Cannot delete", - "A running job can't be deleted from here. Stop the current " - "measurement first (it becomes 'incomplete'), then delete it.", - ) - return + if not self._running_job_is_stale(): + QMessageBox.warning( + self, + "Cannot delete", + "A running job can't be deleted from here. Stop the current " + "measurement first (it becomes 'incomplete'), then delete it.", + ) + return + if not self._confirm_stale_running_override("delete"): + return reply = QMessageBox.question( self, "Delete jobs", @@ -1464,13 +1512,16 @@ class TomoQueueDialog(QDialog): def _clear_queue(self) -> None: jobs = self._load_queue() if any(job.get("status") == "running" for job in jobs): - QMessageBox.warning( - self, - "Cannot clear", - "A job is currently running. Stop it first (it becomes " - "'incomplete'), then clear the queue.", - ) - return + if not self._running_job_is_stale(): + QMessageBox.warning( + self, + "Cannot clear", + "A job is currently running. Stop it first (it becomes " + "'incomplete'), then clear the queue.", + ) + return + if not self._confirm_stale_running_override("clear"): + return reply = QMessageBox.question( self, "Clear queue",