fix: add tests and cleanup after exceptions
Docs build and publish / docker (push) Successful in 19s
CI / lint (push) Successful in 45s
CI / test (3.12) (push) Canceled after 52s
CI / test (3.13) (push) Canceled after 48s
CI / test (3.14) (push) Canceled after 47s
CI / test-with-beamline-plugins (pxi_bec) (push) Canceled after 43s
CI / test-with-beamline-plugins (pxii_bec) (push) Canceled after 42s
CI / test-with-beamline-plugins (pxiii_bec) (push) Canceled after 38s
CI / test-with-coverage (push) Canceled after 37s
CI / coverage-analysis (push) Canceled after 0s
Build and Publish / release (push) Successful in 19s
Docs build and publish / docker (push) Successful in 19s
CI / lint (push) Successful in 45s
CI / test (3.12) (push) Canceled after 52s
CI / test (3.13) (push) Canceled after 48s
CI / test (3.14) (push) Canceled after 47s
CI / test-with-beamline-plugins (pxi_bec) (push) Canceled after 43s
CI / test-with-beamline-plugins (pxii_bec) (push) Canceled after 42s
CI / test-with-beamline-plugins (pxiii_bec) (push) Canceled after 38s
CI / test-with-coverage (push) Canceled after 37s
CI / coverage-analysis (push) Canceled after 0s
Build and Publish / release (push) Successful in 19s
This commit was merged in pull request #201.
This commit is contained in:
1 parent
6483d50d1d
commit
1cd8102b69
5 files changed
+91
-11
No files matched your search
@@ -41,6 +41,7 @@ test = [
|
||||
"pytest-qt",
|
||||
"pytest-asyncio",
|
||||
"pytest-timeout",
|
||||
"fakeredis[lua]",
|
||||
"diff-cover",
|
||||
"ruff>=0.15"
|
||||
]
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import grp
|
||||
import logging
|
||||
import os
|
||||
|
||||
@@ -100,8 +100,8 @@ class BeamlineConfig:
|
||||
host = "localhost"
|
||||
else:
|
||||
host = cfg_get("daq.hardware.redis_url", f"{self._bl}-redis.psi.ch")
|
||||
self.hw_lock = RedisLock(cfg.redis, name=f"{bl}:hardware_busy_lock")
|
||||
self.redis = Redis(host=host, port=6379, db=0, decode_responses=True)
|
||||
self.hw_lock = RedisLock(self.redis, name=f"{bl}:hardware_busy_lock")
|
||||
self.simulated_detector = bl is MXBeamline.SIMULATED
|
||||
self._initialize_optional_yaml_defaults()
|
||||
|
||||
@@ -683,7 +683,7 @@ class BeamlineConfig:
|
||||
def reference_tools(self, data: SampleShortInfoList):
|
||||
self.redis.set(f"{self._bl}:reference-tools", data.model_dump_json())
|
||||
|
||||
def listen_changes_reference_tools(self) -> redis.client.PubSub:
|
||||
def listen_changes_reference_tools(self) -> PubSub:
|
||||
self.redis.config_set("notify-keyspace-events", "KEA")
|
||||
pubsub = self.redis.pubsub()
|
||||
pubsub.psubscribe(f"__keyspace@0__:{self._bl}:reference-tools")
|
||||
@@ -711,7 +711,7 @@ class BeamlineConfig:
|
||||
if tmp is None:
|
||||
return BeamMarkCoeffModel()
|
||||
|
||||
data_dict = json.loads(tmp)
|
||||
data_dict = json.loads(tmp) # pyright: ignore[reportArgumentType] # using sync client - remove ignore on upgrade to redis v8
|
||||
return BeamMarkCoeffModel(**data_dict)
|
||||
|
||||
@beam_mark_coeff.setter
|
||||
|
||||
+12
-8
@@ -95,8 +95,10 @@ async def _lock_hw():
|
||||
if not hardware_busy_lock.acquire(blocking=False):
|
||||
raise BeamlineBusyException("Beamline hardware lock is held by another worker")
|
||||
logger.debug("Hardware lock acquired by process")
|
||||
yield
|
||||
hardware_busy_lock.release()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
hardware_busy_lock.release()
|
||||
logger.debug("Hardware lock released by process")
|
||||
|
||||
|
||||
@@ -734,7 +736,7 @@ async def bec_reinitialise_planner_and_position_devices(
|
||||
|
||||
@app.post("/bec/save_current_bs_pos")
|
||||
@needs_hw_lock
|
||||
async def bec_save_current_bs_pos(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def bec_save_current_bs_pos(token: str = Depends(oauth2_scheme)) -> dict[str, Any]:
|
||||
"""
|
||||
Save the current BEC beamstop work position. Staff only.
|
||||
"""
|
||||
@@ -746,7 +748,7 @@ async def bec_save_current_bs_pos(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
|
||||
@app.post("/bec/save_current_collimator_pos")
|
||||
@needs_hw_lock
|
||||
async def bec_save_current_collimator_pos(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def bec_save_current_collimator_pos(token: str = Depends(oauth2_scheme)) -> dict[str, Any]:
|
||||
"""
|
||||
Save the current BEC collimator work position. Staff only.
|
||||
"""
|
||||
@@ -758,7 +760,7 @@ async def bec_save_current_collimator_pos(token: str = Depends(oauth2_scheme)) -
|
||||
|
||||
@app.post("/bec/save_current_aerotech_position")
|
||||
@needs_hw_lock
|
||||
async def bec_save_current_aerotech_position(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def bec_save_current_aerotech_position(token: str = Depends(oauth2_scheme)) -> dict[str, Any]:
|
||||
"""
|
||||
Save the current BEC aerotech work position and reload device config. Staff only.
|
||||
"""
|
||||
@@ -873,7 +875,9 @@ async def local_contact_restart_device(device: str, token: str = Depends(oauth2_
|
||||
|
||||
@app.post("/local_contact/resync/detector_metadata")
|
||||
@needs_hw_lock
|
||||
async def local_contact_resync_detector_metadata(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def local_contact_resync_detector_metadata(
|
||||
token: str = Depends(oauth2_scheme),
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Refresh cached detector metadata and DTZ limits. Staff only.
|
||||
"""
|
||||
@@ -1086,7 +1090,7 @@ async def park_and_dry(token: str = Depends(oauth2_scheme)):
|
||||
|
||||
@app.post("/tell/toggle_blower")
|
||||
@needs_hw_lock
|
||||
async def tell_toggle_blower(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def tell_toggle_blower(token: str = Depends(oauth2_scheme)) -> dict[str, Any]:
|
||||
"""
|
||||
Toggle the TELL blower. Staff only.
|
||||
|
||||
@@ -1180,7 +1184,7 @@ async def manual(s: SampleShortInfo, token: str = Depends(oauth2_scheme)):
|
||||
|
||||
@app.post("/sample/resync")
|
||||
@needs_hw_lock
|
||||
async def sample_resync(token: str = Depends(oauth2_scheme)) -> dict:
|
||||
async def sample_resync(token: str = Depends(oauth2_scheme)) -> dict[str, Any]:
|
||||
"""
|
||||
Manually trigger a resynchronization of the sample information from the changer (TELL).
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
from redis.lock import Lock as RedisLock
|
||||
|
||||
LOCK_NAME = "SIMULATED:hardware_busy_lock"
|
||||
AUTH = {"Authorization": "Bearer fake-token"}
|
||||
|
||||
THREAD_LOCAL = False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_redis():
|
||||
return fakeredis.FakeRedis(decode_responses=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hw_lock(server_module, client, fake_redis, monkeypatch):
|
||||
"""This worker's lock.
|
||||
|
||||
Depends on ``client`` so that TestClient's lifespan (which overwrites
|
||||
``hardware_busy_lock``) has already run by the time we patch.
|
||||
"""
|
||||
lock = RedisLock(fake_redis, name=LOCK_NAME, thread_local=THREAD_LOCAL)
|
||||
monkeypatch.setattr(server_module, "hardware_busy_lock", lock)
|
||||
return lock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def other_worker(fake_redis):
|
||||
"""A second uvicorn worker's view of the same lock in Redis."""
|
||||
return RedisLock(fake_redis, name=LOCK_NAME, thread_local=THREAD_LOCAL)
|
||||
|
||||
|
||||
def test_endpoint_acquires_and_releases_the_lock(client, hw_lock, other_worker):
|
||||
with patch("aare.daq.auth.check_jwt_rw"), patch("aare.daq.server.daq"):
|
||||
response = client.put("/beamline/omega?val=10.5", headers=AUTH)
|
||||
|
||||
assert response.status_code == 200
|
||||
# Released on the way out, so a second worker can take it.
|
||||
assert not hw_lock.owned()
|
||||
assert other_worker.acquire(blocking=False)
|
||||
|
||||
|
||||
def test_endpoint_is_503_while_another_worker_holds_the_lock(client, hw_lock, other_worker):
|
||||
assert other_worker.acquire(blocking=False)
|
||||
|
||||
with patch("aare.daq.auth.check_jwt_rw"), patch("aare.daq.server.daq"):
|
||||
response = client.put("/beamline/omega?val=10.5", headers=AUTH)
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json()["exception_class"] == "BeamlineBusyException"
|
||||
# The rejected request must not have stolen or released the other worker's lock.
|
||||
assert other_worker.owned()
|
||||
assert not hw_lock.owned()
|
||||
|
||||
|
||||
async def test_lock_hw_is_reentrant_for_the_owning_worker(server_module, hw_lock):
|
||||
assert hw_lock.acquire(blocking=False)
|
||||
|
||||
async with server_module._lock_hw():
|
||||
assert hw_lock.owned()
|
||||
|
||||
assert hw_lock.owned(), "re-entrant exit released a lock it did not acquire"
|
||||
|
||||
|
||||
async def test_lock_is_released_when_the_handler_raises(server_module, hw_lock):
|
||||
with pytest.raises(RuntimeError):
|
||||
async with server_module._lock_hw():
|
||||
raise RuntimeError("handler blew up")
|
||||
|
||||
assert not hw_lock.owned(), "hardware lock leaked after an exception"
|
||||
Reference in new issue
Block a user