mirror of
https://github.com/bec-project/bec_widgets.git
synced 2026-11-20 00:04:16 +01:00
fix(bec_status_box): detect restarted services by process start change
This commit is contained in:
1 parent
0889addc25
commit
4ce119560f
2 files changed
+26
-12
No files matched your search
@@ -4,7 +4,6 @@ The widget automatically updates the status of all running BEC services, and dis
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import weakref
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -209,7 +208,7 @@ class BECStatusBox(BECWidget, QWidget):
|
||||
self.view.setMinimumSize(280, 220)
|
||||
self.main_layout.addWidget(self.view)
|
||||
self.model.restart_requested.connect(self.confirm_restart)
|
||||
self._restart_requested_at: float | None = None
|
||||
self._restart_baseline: dict[str, float | None] | None = None
|
||||
self._restart_timeout = QTimer(self)
|
||||
self._restart_timeout.setSingleShot(True)
|
||||
self._restart_timeout.timeout.connect(self._restart_timed_out)
|
||||
@@ -372,11 +371,11 @@ class BECStatusBox(BECWidget, QWidget):
|
||||
"""Request a restart of all BEC server services through SciHub.
|
||||
|
||||
The progress is shown in the header: a service counts as restarted once it reports
|
||||
RUNNING with a process start time after the request.
|
||||
RUNNING from a different process than before the request.
|
||||
"""
|
||||
if not self.model.can_restart:
|
||||
return
|
||||
self._restart_requested_at = time.time()
|
||||
self._restart_baseline = {name: self._process_start(name) for name in self.CORE_SERVICES}
|
||||
self.client.connector.send(
|
||||
MessageEndpoints.service_request(), ServiceRequestMessage(action="restart")
|
||||
)
|
||||
@@ -387,26 +386,30 @@ class BECStatusBox(BECWidget, QWidget):
|
||||
RestartState.RUNNING, f"Restarting BEC server · 0/{len(self.CORE_SERVICES)}"
|
||||
)
|
||||
|
||||
def _process_start(self, service_name: str) -> float | None:
|
||||
container = self.status_container.get(service_name)
|
||||
return (container.metrics or {}).get("create_time") if container else None
|
||||
|
||||
def _restarted_services(self) -> list[str]:
|
||||
restarted = []
|
||||
for name in self.CORE_SERVICES:
|
||||
container = self.status_container.get(name)
|
||||
if container is None or container.status not in ("RUNNING", "BUSY"):
|
||||
continue
|
||||
started = (container.metrics or {}).get("create_time")
|
||||
if started is not None and started >= self._restart_requested_at:
|
||||
started = self._process_start(name)
|
||||
if started is not None and started != self._restart_baseline[name]:
|
||||
restarted.append(name)
|
||||
return restarted
|
||||
|
||||
def _check_restart_progress(self) -> None:
|
||||
if self._restart_requested_at is None:
|
||||
if self._restart_baseline is None:
|
||||
return
|
||||
done = len(self._restarted_services())
|
||||
total = len(self.CORE_SERVICES)
|
||||
if done < total:
|
||||
self.model.set_restart(RestartState.RUNNING, f"Restarting BEC server · {done}/{total}")
|
||||
return
|
||||
self._restart_requested_at = None
|
||||
self._restart_baseline = None
|
||||
self._restart_timeout.stop()
|
||||
self.model.set_restart(RestartState.DONE, "BEC server restarted")
|
||||
self._restart_clear.start(6000)
|
||||
@@ -422,10 +425,10 @@ class BECStatusBox(BECWidget, QWidget):
|
||||
logger.warning(f"Reloading the client after the server restart failed: {exc}")
|
||||
|
||||
def _restart_timed_out(self) -> None:
|
||||
if self._restart_requested_at is None:
|
||||
if self._restart_baseline is None:
|
||||
return
|
||||
missing = [n for n in self.CORE_SERVICES if n not in self._restarted_services()]
|
||||
self._restart_requested_at = None
|
||||
self._restart_baseline = None
|
||||
names = ", ".join(friendly_name(n)[0] for n in missing)
|
||||
self.model.set_restart(
|
||||
RestartState.FAILED, f"Restart not finished: {names} did not come back"
|
||||
|
||||
@@ -268,7 +268,8 @@ def test_restart_confirm_cancel(status_box, send):
|
||||
|
||||
|
||||
def test_restart_progress_and_done(qtbot, status_box, send):
|
||||
status_box.update_service_status(*_all_services())
|
||||
old_start = time.time() - 100
|
||||
status_box.update_service_status(*_restarted(old_start))
|
||||
with (
|
||||
mock.patch(
|
||||
"bec_widgets.widgets.services.bec_status_box.bec_status_box.QMessageBox.warning",
|
||||
@@ -284,7 +285,7 @@ def test_restart_progress_and_done(qtbot, status_box, send):
|
||||
assert model.restart_state == RestartState.RUNNING
|
||||
assert not model.can_restart
|
||||
# old processes are still reporting: nothing counts as restarted yet
|
||||
status_box.update_service_status(*_restarted(time.time() - 100))
|
||||
status_box.update_service_status(*_restarted(old_start))
|
||||
assert model.restart_text.endswith("0/5")
|
||||
status_box.update_service_status(*_all_services(skip=BECStatusBox.CORE_SERVICES))
|
||||
assert model.restart_state == RestartState.RUNNING
|
||||
@@ -296,6 +297,16 @@ def test_restart_progress_and_done(qtbot, status_box, send):
|
||||
reload_client.assert_called_once()
|
||||
|
||||
|
||||
def test_restart_is_detected_when_the_server_clock_lags(qtbot, status_box, send):
|
||||
status_box.update_service_status(*_restarted(time.time() - 100))
|
||||
with mock.patch.object(status_box, "_reload_client") as reload_client:
|
||||
status_box.request_restart()
|
||||
# the new processes report a start time before the GUI host's request time
|
||||
status_box.update_service_status(*_restarted(time.time() - 5))
|
||||
assert status_box.model.restart_state == RestartState.DONE
|
||||
qtbot.waitUntil(lambda: reload_client.called)
|
||||
|
||||
|
||||
def test_restart_timeout(status_box, send):
|
||||
status_box.update_service_status(*_all_services())
|
||||
with mock.patch.object(status_box.model, "restart_requested"):
|
||||
|
||||
Reference in new issue
Block a user