feat(bec_status_box): restart the bec server from the status box

bec_widgets/widgets/services/bec_status_box/bec_status_box.py

@@ -5,32 +5,47 @@ The widget automatically updates the status of all running BEC services, and dis
 from __future__ import annotations
 
 import os
+import threading
+import time
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from bec_lib.endpoints import MessageEndpoints
 from bec_lib.logger import bec_logger
 from bec_lib.utils.import_utils import lazy_import_from
 from pydantic import BaseModel
 from qtpy.QtCore import QObject, QTimer, Signal
-from qtpy.QtWidgets import QHBoxLayout, QWidget
+from qtpy.QtWidgets import QHBoxLayout, QMessageBox, QWidget
 
 from bec_widgets.utils.bec_widget import BECWidget
 from bec_widgets.utils.compact_popup import CompactPopupWidget
 from bec_widgets.utils.error_popups import SafeSlot
 from bec_widgets.widgets.services.bec_status_box.status_box_widgets import StatusBoxWidgetView
-from bec_widgets.widgets.services.bec_status_box.status_model import ServiceStatusModel, build_rows
+from bec_widgets.widgets.services.bec_status_box.status_model import (
+    ServiceStatusModel,
+    build_rows,
+    friendly_name,
+)
 
 if TYPE_CHECKING:  # pragma: no cover
     from bec_lib.client import BECClient
-    from bec_lib.messages import BECStatus, ServiceMetricMessage, StatusMessage
+    from bec_lib.messages import (
+        BECStatus,
+        ServiceMetricMessage,
+        ServiceRequestMessage,
+        StatusMessage,
+    )
 else:
     # TODO : Put normal imports back when Pydantic gets faster
-    BECStatus = lazy_import_from("bec_lib.messages", ("BECStatus",))
+    BECStatus, ServiceRequestMessage = lazy_import_from(
+        "bec_lib.messages", ("BECStatus", "ServiceRequestMessage")
+    )
 
 logger = bec_logger.logger
 
 QML_FILE = Path(__file__).parent / "qml" / "StatusBox.qml"
+RESTART_TIMEOUT_S = 120
 
 
 @dataclass
@@ -125,6 +140,14 @@ class BECStatusBox(BECWidget, CompactPopupWidget):
         self.layout.setContentsMargins(0, 0, 0, 0)
         self.addWidget(self.view)
         self.model.summaryChanged.connect(self._update_compact_state)
+        self.model.restartRequested.connect(self.confirm_restart)
+        self._restart_requested_at: float | None = None
+        self._restart_timeout = QTimer(self)
+        self._restart_timeout.setSingleShot(True)
+        self._restart_timeout.timeout.connect(self._restart_timed_out)
+        self._restart_clear = QTimer(self)
+        self._restart_clear.setSingleShot(True)
+        self._restart_clear.timeout.connect(lambda: self.model.set_restart(""))
         self.bec_service_status.services_update.connect(self.update_service_status)
         self.bec_core_state.connect(self.update_top_item_status)
         self._update_compact_state()
@@ -261,6 +284,86 @@ class BECStatusBox(BECWidget, CompactPopupWidget):
     def _refresh_model(self) -> None:
         rows = build_rows(self.status_container.values(), self.CORE_SERVICES)
         self.model.set_rows(rows, self.core_state)
+        scihub = self.status_container.get("SciHub")
+        self.model.set_can_restart(scihub is not None and scihub.status != "NOTCONNECTED")
+        self._check_restart_progress()
+
+    # ---- server restart -----------------------------------------------------------------------
+
+    @SafeSlot()
+    def confirm_restart(self) -> None:
+        """Ask the user to confirm, then request a restart of the BEC server."""
+        answer = QMessageBox.warning(
+            self,
+            "Restart BEC server",
+            "Restart all BEC server services?\n\n"
+            "A running scan is aborted, and devices are reloaded. "
+            "Clients stay connected and continue after the restart.",
+            QMessageBox.StandardButton.Cancel | QMessageBox.StandardButton.Yes,
+            QMessageBox.StandardButton.Cancel,
+        )
+        if answer == QMessageBox.StandardButton.Yes:
+            self.request_restart()
+
+    def request_restart(self) -> None:
+        """Request a restart of all BEC server services through SciHub.
+
+        The progress is shown in the header: a service counts as restarted once it reports
+        RUNNING with a process start time after the request.
+        """
+        if not self.model.canRestart:
+            return
+        self._restart_requested_at = time.time()
+        self.client.connector.send(
+            MessageEndpoints.service_request(), ServiceRequestMessage(action="restart")
+        )
+        logger.info("BEC server restart requested from the status box")
+        self._restart_clear.stop()
+        self._restart_timeout.start(int(RESTART_TIMEOUT_S * 1000))
+        self.model.set_restart("running", f"Restarting BEC server · 0/{len(self.CORE_SERVICES)}")
+
+    def _restarted_services(self) -> list[str]:
+        restarted = []
+        for name in self.CORE_SERVICES:
+            container = self.status_container.get(name)
+            if container is None or container.status not in ("RUNNING", "BUSY"):
+                continue
+            started = (container.metrics or {}).get("create_time")
+            if started is not None and started >= self._restart_requested_at:
+                restarted.append(name)
+        return restarted
+
+    def _check_restart_progress(self) -> None:
+        if self._restart_requested_at is None:
+            return
+        done = len(self._restarted_services())
+        total = len(self.CORE_SERVICES)
+        if done < total:
+            self.model.set_restart("running", f"Restarting BEC server · {done}/{total}")
+            return
+        self._restart_requested_at = None
+        self._restart_timeout.stop()
+        self.model.set_restart("done", "BEC server restarted")
+        self._restart_clear.start(6000)
+        threading.Thread(target=self._reload_client, daemon=True, name="status_box_reload").start()
+
+    def _reload_client(self) -> None:
+        """Reload scans and devices in this client after a restart (runs off the GUI thread)."""
+        # pylint: disable=protected-access
+        try:
+            self.client._load_scans()
+            self.client.device_manager._reload_action()
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.warning(f"Reloading the client after the server restart failed: {exc}")
+
+    def _restart_timed_out(self) -> None:
+        if self._restart_requested_at is None:
+            return
+        missing = [n for n in self.CORE_SERVICES if n not in self._restarted_services()]
+        self._restart_requested_at = None
+        names = ", ".join(friendly_name(n)[0] for n in missing)
+        self.model.set_restart("failed", f"Restart not finished: {names} did not come back")
+        self._restart_clear.start(15000)
 
     def _update_compact_state(self) -> None:
         tone = self.model.tone
@@ -284,6 +387,8 @@ class BECStatusBox(BECWidget, CompactPopupWidget):
     def cleanup(self):
         """Cleanup the BECStatusBox widget."""
         self.bec_service_status.cleanup()
+        self._restart_timeout.stop()
+        self._restart_clear.stop()
         if isinstance(self.view, StatusBoxWidgetView):
             self.view.cleanup()
         return super().cleanup()

bec_widgets/widgets/services/bec_status_box/compare_demo.py

@@ -44,6 +44,7 @@ from qtpy.QtWidgets import (
 )
 
 from bec_widgets.utils.colors import apply_theme, get_theme_name
+from bec_widgets.widgets.services.bec_status_box import bec_status_box
 from bec_widgets.widgets.services.bec_status_box.bec_status_box import BECStatusBox
 
 STATES = ["Running", "Busy", "Idle", "Error", "Not connected"]
@@ -135,6 +136,28 @@ class SimulatedServices(QObject):
             )
         return infos, metrics
 
+    def simulate_restart(self, fail: bool = False) -> None:
+        """Mimic SciHub restarting the server: core services drop out, then come back one by one.
+
+        Args:
+            fail (bool): Keep the scan bundler down, so the restart never finishes.
+        """
+        saved = dict(self.core_states)
+        for name in BECStatusBox.CORE_SERVICES:
+            self.core_states[name] = "Not connected"
+        self.emit_update()
+
+        def bring_back(index: int):
+            name = BECStatusBox.CORE_SERVICES[index]
+            if fail and name == "ScanBundler":
+                return
+            self.start_times[name] = time.time()
+            self.core_states[name] = saved[name] if saved[name] != "Not connected" else "Running"
+            self.emit_update()
+
+        for index in range(len(BECStatusBox.CORE_SERVICES)):
+            QTimer.singleShot(2500 + 900 * index, lambda i=index: bring_back(i))
+
     def emit_update(self) -> None:
         """Emit one update, like the real mixin does every second."""
         self._tick += 1
@@ -217,6 +240,9 @@ class ControlPanel(QWidget):
             lambda on: self.flap_timer.start(2000) if on else self.flap_timer.stop()
         )
         grid.addWidget(self.flapping)
+        self.restart_fails = QCheckBox("Server restart fails (Scan bundler stays down)")
+        grid.addWidget(self.restart_fails)
+        grid.addWidget(QLabel("Restart: use the button in either view's header."))
         layout.addWidget(scenarios)
 
         view = QGroupBox("Look")
@@ -308,6 +334,8 @@ class CompareWindow(QMainWindow):
         self.setWindowTitle("BEC status box: QWidget vs QML")
         self.sim = SimulatedServices(self)
         client = mock.MagicMock()
+        client.connector.send.side_effect = self._on_send
+        bec_status_box.RESTART_TIMEOUT_S = 20  # a failed restart shows up quickly in the demo
         self.boxes = {
             "QWidget": TimedStatusBox(
                 client=client, bec_service_status_mixin=self.sim, view="widgets"
@@ -358,6 +386,11 @@ class CompareWindow(QMainWindow):
         self.resize(1250, 780)
         self.sim.emit_update()
 
+    def _on_send(self, endpoint, msg):
+        if getattr(msg, "action", None) == "restart":
+            fail = self.findChild(ControlPanel).restart_fails.isChecked()
+            self.sim.simulate_restart(fail=fail)
+
     def closeEvent(self, event):
         self.sim.timer.stop()
         for box in self.boxes.values():

bec_widgets/widgets/services/bec_status_box/qml/StatusBox.qml

@@ -11,11 +11,18 @@ Rectangle {
     implicitHeight: header.height + list.contentHeight + 16
 
     property string expandedService: ""
+    // while a restart runs or just finished, the header shows its progress instead
+    readonly property string headerTone: statusModel.restartState === "running" ? "progress"
+                                       : statusModel.restartState === "done" ? "success"
+                                       : statusModel.restartState === "failed" ? "emergency"
+                                       : statusModel.tone
+    readonly property color headerColor: toneColor(headerTone)
 
     function toneColor(tone) {
         if (tone === "success") return theme.success
         if (tone === "warning") return theme.warning
         if (tone === "emergency") return theme.emergency
+        if (tone === "progress") return theme.primary
         return theme.muted
     }
     function iconUrl(name, color, filled) {
@@ -29,10 +36,9 @@ Rectangle {
         anchors { left: parent.left; right: parent.right; top: parent.top; margins: 8 }
         height: 56
         radius: 8
-        color: Qt.rgba(root.toneColor(statusModel.tone).r, root.toneColor(statusModel.tone).g,
-                       root.toneColor(statusModel.tone).b, theme.dark ? 0.16 : 0.12)
-        border.color: Qt.rgba(root.toneColor(statusModel.tone).r, root.toneColor(statusModel.tone).g,
-                              root.toneColor(statusModel.tone).b, 0.45)
+        color: Qt.rgba(root.headerColor.r, root.headerColor.g, root.headerColor.b,
+                       theme.dark ? 0.16 : 0.12)
+        border.color: Qt.rgba(root.headerColor.r, root.headerColor.g, root.headerColor.b, 0.45)
 
         RowLayout {
             anchors { fill: parent; leftMargin: 12; rightMargin: 12 }
@@ -41,22 +47,36 @@ Rectangle {
                 id: dot
                 Layout.alignment: Qt.AlignVCenter
                 width: 12; height: 12; radius: 6
-                color: root.toneColor(statusModel.tone)
+                visible: statusModel.restartState !== "running"
+                color: root.headerColor
                 SequentialAnimation on opacity {
-                    running: statusModel.tone === "emergency"
+                    running: root.headerTone === "emergency"
                     loops: Animation.Infinite
                     onRunningChanged: if (!running) dot.opacity = 1
                     NumberAnimation { to: 0.35; duration: 700; easing.type: Easing.InOutQuad }
                     NumberAnimation { to: 1.0; duration: 700; easing.type: Easing.InOutQuad }
                 }
             }
+            Image {
+                id: restartSpinner
+                visible: statusModel.restartState === "running"
+                Layout.preferredWidth: 16; Layout.preferredHeight: 16
+                sourceSize: Qt.size(16, 16)
+                source: root.iconUrl("progress_activity", theme.primary, false)
+                RotationAnimation on rotation {
+                    running: restartSpinner.visible
+                    loops: Animation.Infinite
+                    from: 0; to: 360; duration: 1000
+                }
+            }
             ColumnLayout {
                 Layout.fillWidth: true
                 spacing: 2
                 Text {
                     objectName: "headline"
                     Layout.fillWidth: true
-                    text: statusModel.headline
+                    text: statusModel.restartState !== "" ? statusModel.restartText
+                                                          : statusModel.headline
                     color: theme.text
                     font.pixelSize: 14; font.bold: true
                     elide: Text.ElideRight
@@ -69,6 +89,31 @@ Rectangle {
                     elide: Text.ElideRight
                 }
             }
+            Rectangle {
+                id: restartButton
+                objectName: "restartButton"
+                Layout.preferredWidth: 30; Layout.preferredHeight: 30
+                radius: 6
+                opacity: statusModel.canRestart ? 1.0 : 0.4
+                color: restartArea.containsMouse && statusModel.canRestart
+                       ? Qt.rgba(theme.text.r, theme.text.g, theme.text.b, 0.08) : "transparent"
+                ToolTip.visible: restartArea.containsMouse
+                ToolTip.text: statusModel.canRestart ? "Restart BEC server"
+                              : statusModel.restartState === "running" ? "Restart in progress"
+                              : "Restart needs SciHub to be running"
+                Image {
+                    anchors.centerIn: parent
+                    sourceSize: Qt.size(18, 18)
+                    source: root.iconUrl("restart_alt", theme.text, false)
+                }
+                MouseArea {
+                    id: restartArea
+                    anchors.fill: parent
+                    hoverEnabled: true
+                    cursorShape: statusModel.canRestart ? Qt.PointingHandCursor : Qt.ArrowCursor
+                    onClicked: statusModel.requestRestart()
+                }
+            }
         }
     }
 

bec_widgets/widgets/services/bec_status_box/status_box_widgets.py

@@ -35,6 +35,7 @@ _FALLBACK = {
     "ACCENT_WARNING": "#eda200",
     "ACCENT_EMERGENCY": "#e0534b",
     "ACCENT_SUCCESS": "#59a869",
+    "PRIMARY": "#6089ef",
 }
 
 
@@ -52,6 +53,7 @@ def tone_color(colors: dict[str, QColor], tone: str) -> QColor:
         "success": colors["ACCENT_SUCCESS"],
         "warning": colors["ACCENT_WARNING"],
         "emergency": colors["ACCENT_EMERGENCY"],
+        "progress": colors["PRIMARY"],
     }.get(tone, colors["DISABLED_FG"])
 
 
@@ -119,14 +121,58 @@ class StatusHeader(QFrame):
         text.addWidget(self.headline)
         text.addWidget(self.detail)
         text.addStretch()
+        self.spinner = QLabel(self)
+        self.spinner.setFixedSize(16, 16)
+        self.spinner.setVisible(False)
+        self.restart_button = QToolButton(self)
+        self.restart_button.setObjectName("restartButton")
+        self.restart_button.setAutoRaise(True)
+        self.restart_button.setIconSize(QSize(18, 18))
+        self.restart_button.setCursor(Qt.CursorShape.PointingHandCursor)
         layout.addWidget(self.dot)
+        layout.addWidget(self.spinner)
         layout.addLayout(text, 1)
+        layout.addWidget(self.restart_button)
+        self._angle = 0
+        self._spin = QTimer(self)
+        self._spin.setInterval(80)
+        self._spin.timeout.connect(self._step_spinner)
         self._tone = "neutral"
         self._pulse_up = False
         self._pulse = QTimer(self)
         self._pulse.setInterval(50)
         self._pulse.timeout.connect(self._step_pulse)
 
+    def set_restart(self, state: str, can_restart: bool) -> None:
+        """Show the restart action and spinner for the model's restart state."""
+        running = state == "running"
+        self.restart_button.setEnabled(can_restart)
+        self.restart_button.setToolTip(
+            "Restart BEC server"
+            if can_restart
+            else "Restart in progress" if running else "Restart needs SciHub to be running"
+        )
+        self.spinner.setVisible(running)
+        self.dot.setVisible(not running)
+        if running:
+            self._spin.start()
+        else:
+            self._spin.stop()
+
+    def _step_spinner(self):
+        self._angle = (self._angle + 30) % 360
+        self._paint_spinner()
+
+    def _paint_spinner(self):
+        color = theme_colors()["PRIMARY"]
+        pixmap = material_icon("progress_activity", size=(16, 16), color=color)
+        rotated = pixmap.transformed(
+            QTransform().rotate(self._angle), Qt.TransformationMode.SmoothTransformation
+        )
+        x = (rotated.width() - pixmap.width()) // 2
+        y = (rotated.height() - pixmap.height()) // 2
+        self.spinner.setPixmap(rotated.copy(x, y, pixmap.width(), pixmap.height()))
+
     def set_summary(self, headline: str, detail: str, tone: str) -> None:
         """Show the summary of the model."""
         self.headline.setText(headline)
@@ -152,6 +198,10 @@ class StatusHeader(QFrame):
         accent = tone_color(colors, self._tone)
         alpha = 41 if get_theme_name() != "light" else 31
         self.dot.set_color(accent)
+        self.restart_button.setIcon(
+            material_icon("restart_alt", size=(18, 18), color=colors["FG"], convert_to_pixmap=False)
+        )
+        self._paint_spinner()
         self.setStyleSheet(
             f"#StatusHeader {{ background: {rgba(accent, alpha)}; border: 1px solid "
             f"{rgba(accent, 115)}; border-radius: 8px; }}"
@@ -160,8 +210,9 @@ class StatusHeader(QFrame):
         )
 
     def stop(self) -> None:
-        """Stop the pulse timer."""
+        """Stop the pulse and spinner timers."""
         self._pulse.stop()
+        self._spin.stop()
 
 
 class ServiceRowWidget(QFrame):
@@ -413,12 +464,20 @@ class StatusBoxWidgetView(QWidget):
 
         model.rowsChanged.connect(self.sync)
         model.summaryChanged.connect(self._update_summary)
+        model.restartChanged.connect(self._update_summary)
+        self.header.restart_button.clicked.connect(model.requestRestart)
         self._update_summary()
         self.sync()
         self.apply_theme()
 
     def _update_summary(self):
-        self.header.set_summary(self.model.headline, self.model.detail, self.model.tone)
+        state = self.model.restartState
+        if state:
+            tone = {"running": "progress", "done": "success"}.get(state, "emergency")
+            self.header.set_summary(self.model.restartText, self.model.detail, tone)
+        else:
+            self.header.set_summary(self.model.headline, self.model.detail, self.model.tone)
+        self.header.set_restart(state, self.model.canRestart)
 
     def sync(self) -> None:
         """Bring the row widgets in line with the model, reusing existing widgets."""

bec_widgets/widgets/services/bec_status_box/status_model.py

@@ -260,6 +260,8 @@ class ServiceStatusModel(QAbstractListModel):
 
     summaryChanged = Signal()
     rowsChanged = Signal()
+    restartChanged = Signal()
+    restartRequested = Signal()
 
     ROLE_NAMES = (
         "serviceName",
@@ -284,6 +286,9 @@ class ServiceStatusModel(QAbstractListModel):
         self._headline = "Waiting for BEC services"
         self._detail = ""
         self._tone = "neutral"
+        self._restart_state = ""
+        self._restart_text = ""
+        self._can_restart = False
         self._roles = {Qt.ItemDataRole.UserRole + i: n for i, n in enumerate(self.ROLE_NAMES)}
         self._role_ids = {name: role for role, name in self._roles.items()}
 
@@ -406,6 +411,46 @@ class ServiceStatusModel(QAbstractListModel):
         """The overall tone: success, warning, emergency or neutral."""
         return self._tone
 
+    # ---- server restart -----------------------------------------------------------------------
+
+    def set_restart(self, state: str, text: str = "") -> None:
+        """Set the restart progress shown in the header.
+
+        Args:
+            state (str): "" (none), "running", "done" or "failed".
+            text (str): The line shown while the state is set.
+        """
+        if (state, text) != (self._restart_state, self._restart_text):
+            self._restart_state, self._restart_text = state, text
+            self.restartChanged.emit()
+
+    def set_can_restart(self, can_restart: bool) -> None:
+        """Enable or disable the restart action."""
+        if can_restart != self._can_restart:
+            self._can_restart = can_restart
+            self.restartChanged.emit()
+
+    @Property(str, notify=restartChanged)
+    def restartState(self) -> str:
+        """The restart progress: "", "running", "done" or "failed"."""
+        return self._restart_state
+
+    @Property(str, notify=restartChanged)
+    def restartText(self) -> str:
+        """The restart progress line."""
+        return self._restart_text
+
+    @Property(bool, notify=restartChanged)
+    def canRestart(self) -> bool:
+        """Whether a restart can be requested now (SciHub is up and no restart is running)."""
+        return self._can_restart and self._restart_state != "running"
+
+    @Slot()
+    def requestRestart(self) -> None:
+        """Ask the owner widget to confirm and request a BEC server restart."""
+        if self.canRestart:
+            self.restartRequested.emit()
+
     @Slot(str)
     def copyDetails(self, service_name: str) -> None:
         """Copy the details of a service to the clipboard.

tests/unit_tests/test_bec_status_box.py

@@ -3,9 +3,10 @@ import time
 from unittest import mock
 
 import pytest
+from bec_lib.endpoints import MessageEndpoints
 from bec_lib.messages import BECStatus, ServiceInfo, ServiceMetricMessage, StatusMessage
 from qtpy.QtQuickWidgets import QQuickWidget
-from qtpy.QtWidgets import QApplication
+from qtpy.QtWidgets import QApplication, QMessageBox
 
 from bec_widgets.tests.client_mocks import mocked_client
 from bec_widgets.tests.utils import create_widget
@@ -276,3 +277,72 @@ def test_widget_view_follows_theme(qtbot, mocked_client):
     qtbot.waitUntil(lambda: box.view.styleSheet() != dark)
     apply_theme("dark")
     qtbot.waitUntil(lambda: box.view.styleSheet() == dark)
+
+
+def _restarted(started):
+    info, metrics = _all_services()
+    for name in BECStatusBox.CORE_SERVICES:
+        metrics[name].metrics["create_time"] = started
+    return info, metrics
+
+
+@pytest.fixture
+def send(status_box):
+    with mock.patch.object(status_box.client.connector, "send") as send_mock:
+        yield send_mock
+
+
+def test_restart_needs_scihub(status_box, send):
+    status_box.update_service_status(*_all_services(skip=["SciHub"]))
+    assert not status_box.model.canRestart
+    status_box.model.requestRestart()
+    send.assert_not_called()
+    status_box.update_service_status(*_all_services())
+    assert status_box.model.canRestart
+
+
+def test_restart_confirm_cancel(status_box, send):
+    status_box.update_service_status(*_all_services())
+    with mock.patch(
+        "bec_widgets.widgets.services.bec_status_box.bec_status_box.QMessageBox.warning",
+        return_value=QMessageBox.StandardButton.Cancel,
+    ) as dialog:
+        status_box.model.requestRestart()
+    dialog.assert_called_once()
+    send.assert_not_called()
+
+
+def test_restart_progress_and_done(status_box, send):
+    status_box.update_service_status(*_all_services())
+    with (
+        mock.patch(
+            "bec_widgets.widgets.services.bec_status_box.bec_status_box.QMessageBox.warning",
+            return_value=QMessageBox.StandardButton.Yes,
+        ),
+        mock.patch.object(status_box, "_reload_client") as reload_client,
+    ):
+        status_box.model.requestRestart()
+        endpoint, msg = send.call_args[0]
+        assert endpoint == MessageEndpoints.service_request()
+        assert msg.action == "restart"
+        model = status_box.model
+        assert model.restartState == "running"
+        assert not model.canRestart
+        # old processes are still reporting: nothing counts as restarted yet
+        status_box.update_service_status(*_restarted(time.time() - 100))
+        assert model.restartText.endswith("0/5")
+        status_box.update_service_status(*_all_services(skip=BECStatusBox.CORE_SERVICES))
+        assert model.restartState == "running"
+        status_box.update_service_status(*_restarted(time.time() + 1))
+        assert model.restartState == "done"
+        assert model.canRestart
+    reload_client.assert_called_once()
+
+
+def test_restart_timeout(status_box, send):
+    status_box.update_service_status(*_all_services())
+    with mock.patch.object(status_box.model, "restartRequested"):
+        status_box.request_restart()
+    status_box._restart_timed_out()
+    assert status_box.model.restartState == "failed"
+    assert "Device server" in status_box.model.restartText