diff --git a/bec_widgets/widgets/services/bec_status_box/bec_status_box.py b/bec_widgets/widgets/services/bec_status_box/bec_status_box.py index f9e39ece..e252af65 100644 --- a/bec_widgets/widgets/services/bec_status_box/bec_status_box.py +++ b/bec_widgets/widgets/services/bec_status_box/bec_status_box.py @@ -4,7 +4,6 @@ The widget automatically updates the status of all running BEC services, and dis from __future__ import annotations -import time import weakref from dataclasses import dataclass from typing import TYPE_CHECKING @@ -209,7 +208,7 @@ class BECStatusBox(BECWidget, QWidget): self.view.setMinimumSize(280, 220) self.main_layout.addWidget(self.view) self.model.restart_requested.connect(self.confirm_restart) - self._restart_requested_at: float | None = None + self._restart_baseline: dict[str, float | None] | None = None self._restart_timeout = QTimer(self) self._restart_timeout.setSingleShot(True) self._restart_timeout.timeout.connect(self._restart_timed_out) @@ -372,11 +371,11 @@ class BECStatusBox(BECWidget, QWidget): """Request a restart of all BEC server services through SciHub. The progress is shown in the header: a service counts as restarted once it reports - RUNNING with a process start time after the request. + RUNNING from a different process than before the request. """ if not self.model.can_restart: return - self._restart_requested_at = time.time() + self._restart_baseline = {name: self._process_start(name) for name in self.CORE_SERVICES} self.client.connector.send( MessageEndpoints.service_request(), ServiceRequestMessage(action="restart") ) @@ -387,26 +386,30 @@ class BECStatusBox(BECWidget, QWidget): RestartState.RUNNING, f"Restarting BEC server · 0/{len(self.CORE_SERVICES)}" ) + def _process_start(self, service_name: str) -> float | None: + container = self.status_container.get(service_name) + return (container.metrics or {}).get("create_time") if container else None + def _restarted_services(self) -> list[str]: restarted = [] for name in self.CORE_SERVICES: container = self.status_container.get(name) if container is None or container.status not in ("RUNNING", "BUSY"): continue - started = (container.metrics or {}).get("create_time") - if started is not None and started >= self._restart_requested_at: + started = self._process_start(name) + if started is not None and started != self._restart_baseline[name]: restarted.append(name) return restarted def _check_restart_progress(self) -> None: - if self._restart_requested_at is None: + if self._restart_baseline is None: return done = len(self._restarted_services()) total = len(self.CORE_SERVICES) if done < total: self.model.set_restart(RestartState.RUNNING, f"Restarting BEC server · {done}/{total}") return - self._restart_requested_at = None + self._restart_baseline = None self._restart_timeout.stop() self.model.set_restart(RestartState.DONE, "BEC server restarted") self._restart_clear.start(6000) @@ -422,10 +425,10 @@ class BECStatusBox(BECWidget, QWidget): logger.warning(f"Reloading the client after the server restart failed: {exc}") def _restart_timed_out(self) -> None: - if self._restart_requested_at is None: + if self._restart_baseline is None: return missing = [n for n in self.CORE_SERVICES if n not in self._restarted_services()] - self._restart_requested_at = None + self._restart_baseline = None names = ", ".join(friendly_name(n)[0] for n in missing) self.model.set_restart( RestartState.FAILED, f"Restart not finished: {names} did not come back" diff --git a/tests/unit_tests/test_bec_status_box.py b/tests/unit_tests/test_bec_status_box.py index f5cf4c4f..8c8f72a2 100644 --- a/tests/unit_tests/test_bec_status_box.py +++ b/tests/unit_tests/test_bec_status_box.py @@ -268,7 +268,8 @@ def test_restart_confirm_cancel(status_box, send): def test_restart_progress_and_done(qtbot, status_box, send): - status_box.update_service_status(*_all_services()) + old_start = time.time() - 100 + status_box.update_service_status(*_restarted(old_start)) with ( mock.patch( "bec_widgets.widgets.services.bec_status_box.bec_status_box.QMessageBox.warning", @@ -284,7 +285,7 @@ def test_restart_progress_and_done(qtbot, status_box, send): assert model.restart_state == RestartState.RUNNING assert not model.can_restart # old processes are still reporting: nothing counts as restarted yet - status_box.update_service_status(*_restarted(time.time() - 100)) + status_box.update_service_status(*_restarted(old_start)) assert model.restart_text.endswith("0/5") status_box.update_service_status(*_all_services(skip=BECStatusBox.CORE_SERVICES)) assert model.restart_state == RestartState.RUNNING @@ -296,6 +297,16 @@ def test_restart_progress_and_done(qtbot, status_box, send): reload_client.assert_called_once() +def test_restart_is_detected_when_the_server_clock_lags(qtbot, status_box, send): + status_box.update_service_status(*_restarted(time.time() - 100)) + with mock.patch.object(status_box, "_reload_client") as reload_client: + status_box.request_restart() + # the new processes report a start time before the GUI host's request time + status_box.update_service_status(*_restarted(time.time() - 5)) + assert status_box.model.restart_state == RestartState.DONE + qtbot.waitUntil(lambda: reload_client.called) + + def test_restart_timeout(status_box, send): status_box.update_service_status(*_all_services()) with mock.patch.object(status_box.model, "restart_requested"):