CI: add ci linting and tests #1

Merged
perl_d merged 3 commits from ci/add_ci_tests into main 2026-07-02 16:09:47 +02:00
25 changed files with 203 additions and 362 deletions
+29
View File
@@ -0,0 +1,29 @@
name: "Install"
description: "Setup system and python environment and install repo"
inputs:
python_version:
required: false
default: "3.12"
description: "Python version to use"
runs:
using: "composite"
steps:
- name: Install Libraries
shell: bash
run: |
sudo apt-get update
sudo apt-get install -y libgl1 libegl1
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: ${{ inputs.python_version }}
- name: Install repo
shell: bash
run: |
pip install uv
uv venv
source .venv/bin/activate
uv pip install -e .[test]
+44
View File
@@ -0,0 +1,44 @@
name: Full CI
on:
push:
pull_request:
workflow_dispatch:
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup
uses: ./.gitea/actions/install
with:
python_version: "3.12"
- name: Format
run: |
source .venv/bin/activate
ruff format --check
test:
runs-on: ubuntu-latest
needs: "lint"
strategy:
matrix:
python-version: ["3.11", "3.12", "3.13"]
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup
uses: ./.gitea/actions/install
with:
python_version: ${{ matrix.python-version }}
- name: Run Pytest with Coverage
run: |
source .venv/bin/activate
pytest --random-order --cov=./ --cov-config=./pyproject.toml --cov-branch --cov-report=xml --no-cov-on-fail ./tests/
+21
View File
@@ -9,7 +9,11 @@ description = "Common components (models, calculations) for Aare packages"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"pyyaml",
"jfjoch-client",
"pydantic",
"numpy",
"scipy",
"opencv-python",
"pillow"
]
@@ -17,6 +21,8 @@ dependencies = [
[project.optional-dependencies]
test = [
"pytest",
"pytest-cov",
"pytest-random-order"
]
[[tool.uv.index]]
@@ -28,3 +34,18 @@ testpaths = ["tests"]
norecursedirs = ["scripts", ".venv", "logs", "docs"]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
[tool.black]
line-length = 100
skip-magic-trailing-comma = true
[tool.isort]
profile = "black"
line_length = 100
multi_line_output = 3
[tool.ruff]
line-length = 100
[tool.ruff.format]
skip-magic-trailing-comma = true
+9 -24
View File
@@ -39,15 +39,15 @@ def get_uvicorn_logging_config() -> dict:
"disable_existing_loggers": False,
"filters": {
"ignore_successful_status_access": {
"()": "aarecommon.logger_config.IgnoreSuccessfulStatusAccessFilter",
},
"()": "aarecommon.logger_config.IgnoreSuccessfulStatusAccessFilter"
}
},
"formatters": {
"default": {
"()": "uvicorn.logging.DefaultFormatter",
"fmt": "%(levelprefix)s %(message)s",
"use_colors": True,
},
}
},
"handlers": {
"default": {
@@ -63,16 +63,8 @@ def get_uvicorn_logging_config() -> dict:
},
},
"loggers": {
"uvicorn": {
"handlers": ["default"],
"level": "DEBUG",
"propagate": False,
},
"uvicorn.access": {
"handlers": ["access"],
"level": "INFO",
"propagate": False,
},
"uvicorn": {"handlers": ["default"], "level": "DEBUG", "propagate": False},
"uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False},
},
}
@@ -90,9 +82,7 @@ def get_config_path(env: str = "dev") -> Path:
# TODO fix logging.
def setup_logger(
name="aareDAQ",
base_dir: str | None = f"~/tmp/mxlogs/",
config_path: str | None = None,
name="aareDAQ", base_dir: str | None = f"~/tmp/mxlogs/", config_path: str | None = None
):
# switch to production mode using: $ APP_ENV=prod python main.py
env = os.getenv("APP_ENV", "dev") # default: dev
@@ -107,9 +97,7 @@ def setup_logger(
# Force base directory to be under the user's home dir
effective_base = base_dir + f"{name}" or f"/tmp/logs/mxlogs/{name}"
effective_base = os.path.abspath(
os.path.expanduser(os.path.expandvars(effective_base))
)
effective_base = os.path.abspath(os.path.expanduser(os.path.expandvars(effective_base)))
# Ensure directories for file handlers exist
handlers = config.get("handlers", {})
@@ -135,8 +123,7 @@ def setup_logger(
def find_existing_formatter(
default_fmt="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
default_datefmt=None,
default_fmt="%(asctime)s [%(levelname)s] %(name)s: %(message)s", default_datefmt=None
):
root = logging.getLogger()
for h in root.handlers:
@@ -160,9 +147,7 @@ if __name__ == "__main__":
os.environ["APP_ENV"] = "dev" # or "prod"
# os.environ["LOG_DIR"] = "/tmp/mxlogs" # optional; matches your setup code
log = setup_logger(
"aareDAQ", base_dir="/tmp/test_logs"
) # use the configured logger name
log = setup_logger("aareDAQ", base_dir="/tmp/test_logs") # use the configured logger name
log.debug("debug smoke test")
log.error("error smoke test")
+7 -23
View File
@@ -77,9 +77,7 @@ def raster_request_log_context(request: RasterGridRequest | None) -> dict[str, A
def rotation_request_log_context(
request: RotationScanRequest | None,
*,
total_time_s: float | None = None,
request: RotationScanRequest | None, *, total_time_s: float | None = None
) -> dict[str, Any]:
if request is None:
return {}
@@ -107,12 +105,8 @@ def geom_log_context(geom, *, prefix: str = "") -> dict[str, Any]:
return {
f"{prefix}omega_deg": getattr(geom, "omega_deg", None),
f"{prefix}beam_x_pxl": getattr(
getattr(geom, "beam_location_pxl", None), "x", None
),
f"{prefix}beam_y_pxl": getattr(
getattr(geom, "beam_location_pxl", None), "y", None
),
f"{prefix}beam_x_pxl": getattr(getattr(geom, "beam_location_pxl", None), "x", None),
f"{prefix}beam_y_pxl": getattr(getattr(geom, "beam_location_pxl", None), "y", None),
f"{prefix}pixel_in_mm": getattr(geom, "pixel_in_mm", None),
f"{prefix}sh_x_mm": getattr(sh_mm, "x", None),
f"{prefix}sh_y_mm": getattr(sh_mm, "y", None),
@@ -121,14 +115,9 @@ def geom_log_context(geom, *, prefix: str = "") -> dict[str, Any]:
def ml_bundle_meta_log_context(
*,
target_point: tuple[float, float] | None = None,
focus: float | None = None,
*, target_point: tuple[float, float] | None = None, focus: float | None = None
) -> dict[str, Any]:
return {
"target_point": target_point,
"focus": focus,
}
return {"target_point": target_point, "focus": focus}
def log_ml_bundle_meta(
@@ -144,8 +133,7 @@ def log_ml_bundle_meta(
logger.debug(
"ML bundle metadata",
extra=merge_log_context(
{"context": context},
ml_bundle_meta_log_context(target_point=target_point, focus=focus),
{"context": context}, ml_bundle_meta_log_context(target_point=target_point, focus=focus)
),
)
@@ -158,8 +146,4 @@ def log_duration(
level: int = logging.INFO,
extra: dict[str, Any] | None = None,
) -> None:
logger.log(
level,
message,
extra=merge_log_context(extra, duration_s=duration_s),
)
logger.log(level, message, extra=merge_log_context(extra, duration_s=duration_s))
+17 -61
View File
@@ -79,10 +79,7 @@ class TransformationInvalidException(AutomationError):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Transformation is not implemented",
*,
critical: bool | None = None,
self, message: str = "Transformation is not implemented", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -95,10 +92,7 @@ class StateTransitionFailed(BeamlineStateException):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Beamline state transition failed",
*,
critical: bool | None = None,
self, message: str = "Beamline state transition failed", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -111,10 +105,7 @@ class MaintenanceStateException(BeamlineStateException):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Beamline is in Maintenance state",
*,
critical: bool | None = None,
self, message: str = "Beamline is in Maintenance state", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -126,9 +117,7 @@ class MaintenanceStateException(BeamlineStateException):
class DataCollectionException(AutomationError):
"""Group parent for data-collection-time exceptions; also raisable directly."""
def __init__(
self, message: str = "Data collection failed", *, critical: bool | None = None
):
def __init__(self, message: str = "Data collection failed", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -137,9 +126,7 @@ class DataCollectionException(AutomationError):
class RasterScanException(DataCollectionException):
def __init__(
self, message: str = "Data collection failed", *, critical: bool | None = None
):
def __init__(self, message: str = "Data collection failed", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -169,10 +156,7 @@ class UnmountingFailed(TellException):
"""Sample failed to unmount. Optionally critical via instance flag."""
def __init__(
self,
message: str = "A sample was not unmounted",
*,
critical: bool | None = None,
self, message: str = "A sample was not unmounted", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -184,9 +168,7 @@ class UnmountingFailed(TellException):
class MountingFailed(TellException):
"""Sample failed to mount. Optionally critical via instance flag."""
def __init__(
self, message: str = "A sample was not mounted", *, critical: bool | None = None
):
def __init__(self, message: str = "A sample was not mounted", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -197,9 +179,7 @@ class MountingFailed(TellException):
class ManualMountException(AareUserError):
"""Custom exception for manual mounting"""
def __init__(
self, message: str = "Manual mounting failed", *, critical: bool | None = None
):
def __init__(self, message: str = "Manual mounting failed", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -212,9 +192,7 @@ class SmartMagnetFaultException(AutomationError):
critical: ClassVar[bool] = True
def __init__(
self, message: str = "Smart magnet fault", *, critical: bool | None = None
):
def __init__(self, message: str = "Smart magnet fault", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -234,10 +212,7 @@ class DoorSafetyError(AutomationError):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Door safety could not be activated",
*,
critical: bool | None = None,
self, message: str = "Door safety could not be activated", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -262,9 +237,7 @@ class TellConnectionException(TellException):
critical: ClassVar[bool] = True
def __init__(
self, message: str = "Lost connection to Tell", *, critical: bool | None = None
):
def __init__(self, message: str = "Lost connection to Tell", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -273,9 +246,7 @@ class TellConnectionException(TellException):
class WarningTellException(TellException):
def __init__(
self, message: str = "Warning error in TELL", *, critical: bool | None = None
):
def __init__(self, message: str = "Warning error in TELL", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -286,9 +257,7 @@ class WarningTellException(TellException):
class CriticalTellException(TellException):
critical: ClassVar[bool] = True
def __init__(
self, message: str = "Critical error in TELL", *, critical: bool | None = None
):
def __init__(self, message: str = "Critical error in TELL", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -298,10 +267,7 @@ class CriticalTellException(TellException):
class AXCFailed(AutomationError):
def __init__(
self,
message: str = "Auto X-ray centering failed",
*,
critical: bool | None = None,
self, message: str = "Auto X-ray centering failed", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
@@ -313,12 +279,7 @@ class AXCFailed(AutomationError):
class BeamlineBusyException(BeamlineStateException):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Beamline is in busy state",
*,
critical: bool | None = None,
):
def __init__(self, message: str = "Beamline is in busy state", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -349,9 +310,7 @@ class SampleException(AareUserError):
AareUserError pending confirmation -- "sample not found" reads as bad
input rather than a runtime failure."""
def __init__(
self, message: str = "Sample not found", *, critical: bool | None = None
):
def __init__(self, message: str = "Sample not found", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
@@ -563,10 +522,7 @@ class MagnetPositionSensorErorr(AutomationError):
critical: ClassVar[bool] = True
def __init__(
self,
message: str = "Magnet position sensor error",
*,
critical: bool | None = None,
self, message: str = "Magnet position sensor error", *, critical: bool | None = None
):
super().__init__(message, critical=critical)
self.message = message
+2 -8
View File
@@ -11,11 +11,7 @@ from PIL import Image
class AareLCInferWrapper:
def __init__(
self,
bl: MXBeamline,
secret: str = "1s3ng@rd",
):
def __init__(self, bl: MXBeamline, secret: str = "1s3ng@rd"):
if bl == MXBeamline.X10SA or bl == MXBeamline.X06DA:
host = cfg_get("daq.hardware.aarelc_url")
if host is None:
@@ -37,9 +33,7 @@ class AareLCInferWrapper:
def get_config(self) -> config.ConfigSnapshotResponse:
return self._api_config.get_config(self.client)
def update_config(
self, patch: RuntimeConfigPatchModel
) -> config.ConfigUpdateResponse:
def update_config(self, patch: RuntimeConfigPatchModel) -> config.ConfigUpdateResponse:
return self._api_config.update_config(self.client, patch)
def get_latest_prediction(self) -> LatestPredictionModel:
+1 -5
View File
@@ -24,11 +24,7 @@ def focus_measure_edges(
return 0.0
if verbose:
mask_sum = mask.sum() if mask is not None else roi.size
print(
f"mask pixels: {mask_sum}, "
f"focus={strong.mean():.2f}"
f"strong_size={strong.size}"
)
print(f"mask pixels: {mask_sum}, focus={strong.mean():.2f}strong_size={strong.size}")
return float(strong.mean()) # higher = sharper
+2 -6
View File
@@ -12,17 +12,13 @@ class Coordinate(BaseModel):
# Overload +
def __add__(self, other: "Coordinate") -> "Coordinate":
if isinstance(other, Coordinate):
return Coordinate(
x=self.x + other.x, y=self.y + other.y, z=self.z + other.z
)
return Coordinate(x=self.x + other.x, y=self.y + other.y, z=self.z + other.z)
return NotImplemented
# Overload -
def __sub__(self, other: "Coordinate") -> "Coordinate":
if isinstance(other, Coordinate):
return Coordinate(
x=self.x - other.x, y=self.y - other.y, z=self.z - other.z
)
return Coordinate(x=self.x - other.x, y=self.y - other.y, z=self.z - other.z)
return NotImplemented
# Overload *
+1 -3
View File
@@ -81,7 +81,5 @@ if __name__ == "__main__":
print(f"geom.max_resolution_angstrom: {geom.max_resolution_angstrom:.2f} Angstrom")
print(f"geom.calc_dtz_mm(3.9): {geom.calc_dtz_mm(3.9):.2f} mm")
print(
f"geom.resolution_angstrom(1244.42): {geom.resolution_angstrom(1244.42):.2f} Angstrom"
)
print(f"geom.resolution_angstrom(1244.42): {geom.resolution_angstrom(1244.42):.2f} Angstrom")
print(f"geom.detector_radius_mm: {geom.detector_radius_mm:.2f} mm")
+25 -75
View File
@@ -16,15 +16,11 @@ def identify_crystal_raster(result, r: RasterGridRequest) -> CenterOfMassModel |
max_image = max(images, key=lambda img: img.spots_low_res)
logger.debug(f"Image with maximum spots_low_res: {max_image}")
logger.debug(f"Maximum spots_low_res value: {max_image.spots_low_res}")
logger.debug(
f"Maximum image found at grid coordiantes {max_image.nx}, {max_image.ny}"
)
logger.debug(f"Maximum image found at grid coordiantes {max_image.nx}, {max_image.ny}")
logger.debug(
f"Maximum image found at umL {max_image.nx * r.grid_size_mm.x}, {max_image.ny * r.grid_size_mm.y}"
)
com = CenterOfMassModel(
n_x=max_image.nx, n_y=max_image.ny, max_image=max_image.number
)
com = CenterOfMassModel(n_x=max_image.nx, n_y=max_image.ny, max_image=max_image.number)
com_mm = com.get_com_mm(r)
grid_mm_x = com_mm.x
grid_mm_y = com_mm.y
@@ -168,16 +164,11 @@ def create_quality_filtered_array(
if min_spots is None:
min_spots = min(
(result.spots for result in scan_results if result.spots is not None),
default=1,
(result.spots for result in scan_results if result.spots is not None), default=1
)
if min_low_res_spots is None:
min_low_res_spots = min(
(
result.spots_low_res
for result in scan_results
if result.spots_low_res is not None
),
(result.spots_low_res for result in scan_results if result.spots_low_res is not None),
default=1,
)
if min_background is None:
@@ -187,8 +178,7 @@ def create_quality_filtered_array(
if min_efficiency is None:
min_efficiency = 1.0
max_spots_low_res = max(
(result.spots_low_res for result in scan_results if result.spots is not None),
default=1,
(result.spots_low_res for result in scan_results if result.spots is not None), default=1
)
for result in scan_results:
@@ -231,22 +221,16 @@ def get_xtal_size(crystal_size, result_array, r: RasterGridRequest):
cols = np.any(object_mask, axis=0)
row_min, row_max = np.where(rows)[0][[0, -1]]
col_min, col_max = np.where(cols)[0][[0, -1]]
logger.info(
f"Largest bbox: width={col_max - col_min}, height={row_max - row_min}"
)
logger.info(f"Largest bbox: width={col_max - col_min}, height={row_max - row_min}")
logger.info(
f"Largest bbox: width={(col_max - col_min) * r.grid_size_mm.x}, y={(row_max - row_min) * r.grid_size_mm.y}"
)
if r.n_x == 1:
logger.debug("calculating z")
crystal_size = CrystalSize(
x=crystal_size.x,
y=crystal_size.y,
z=(col_max - col_min) * r.grid_size_mm.y * 1000,
)
logger.info(
f"Crystal Size: x={crystal_size.x}, y={crystal_size.y}, z={crystal_size.z}"
x=crystal_size.x, y=crystal_size.y, z=(col_max - col_min) * r.grid_size_mm.y * 1000
)
logger.info(f"Crystal Size: x={crystal_size.x}, y={crystal_size.y}, z={crystal_size.z}")
else:
logger.debug("calculating x and y")
@@ -255,9 +239,7 @@ def get_xtal_size(crystal_size, result_array, r: RasterGridRequest):
y=(col_max - col_min) * r.grid_size_mm.y * 1000,
z=crystal_size.z,
)
logger.info(
f"Crystal Size: x={crystal_size.x}, y={crystal_size.y}, z={crystal_size.z}"
)
logger.info(f"Crystal Size: x={crystal_size.x}, y={crystal_size.y}, z={crystal_size.z}")
except ValueError as e:
logger.error(f"error calculating xtal size: {e}")
crystal_size = CrystalSize(x=0, y=0, z=0)
@@ -269,9 +251,7 @@ def get_best_b_factor(result_list: List):
if not result_list:
return None
best_b_factor = min(
(img for img in result_list if img.b is not None),
key=lambda img: img.b,
default=None,
(img for img in result_list if img.b is not None), key=lambda img: img.b, default=None
)
if best_b_factor is None:
return None
@@ -283,9 +263,7 @@ def get_best_res(result_list: List):
if not result_list:
return None
best_res = min(
(img for img in result_list if img.res is not None),
key=lambda img: img.res,
default=None,
(img for img in result_list if img.res is not None), key=lambda img: img.res, default=None
)
if best_res is None:
return None
@@ -308,9 +286,7 @@ def get_result_list_from_com(images, com: CenterOfMassModel):
start_y, end_y = round(cy - 1), round(cy + 1)
logger.info(f"range x {start_x} {end_x}, y {start_y} {end_y}")
result_list = [
img
for img in images
if start_x <= img.nx <= end_x and start_y <= img.ny <= end_y
img for img in images if start_x <= img.nx <= end_x and start_y <= img.ny <= end_y
]
return result_list
@@ -353,10 +329,7 @@ def raster_highest_score(images) -> CenterOfMassModel | None:
return _to_com_model(_max_cell(score_array), images, "Highest score")
def has_sufficient_low_res_spots(
result_array: np.ndarray,
min_spots_low_res: float,
) -> bool:
def has_sufficient_low_res_spots(result_array: np.ndarray, min_spots_low_res: float) -> bool:
if result_array is None or result_array.size == 0:
logger.warning("Empty result_array passed to has_sufficient_low_res_spots")
return False
@@ -365,20 +338,14 @@ def has_sufficient_low_res_spots(
logger.info(f"Max spots_low_res in raster result_array: {max_val}")
if max_val < min_spots_low_res:
logger.info(
"Raster rejected: max spots_low_res "
f"{max_val} < required {min_spots_low_res}"
)
logger.info(f"Raster rejected: max spots_low_res {max_val} < required {min_spots_low_res}")
return False
return True
def compute_crystal_score_array(
scan_results: List,
w_bkg: float = 0.25,
w_low_res: float = 0.55,
w_indexed: float = 0.20,
scan_results: List, w_bkg: float = 0.25, w_low_res: float = 0.55, w_indexed: float = 0.20
) -> np.ndarray:
"""Combine bkg (25%), spots_low_res (55%), and spots_indexed (20%) into a 0100 score.
@@ -395,9 +362,7 @@ def compute_crystal_score_array(
return np.zeros_like(a, dtype=float)
return (a - mn) / (mx - mn) * 100.0
return (
w_bkg * _norm(arr_bkg) + w_low_res * _norm(arr_low) + w_indexed * _norm(arr_idx)
)
return w_bkg * _norm(arr_bkg) + w_low_res * _norm(arr_low) + w_indexed * _norm(arr_idx)
def _max_cell(arr: np.ndarray) -> tuple[int, int]:
@@ -445,12 +410,7 @@ def _draw_panel(
linewidths=1.8,
)
rect = mpatches.Rectangle(
(max_nx - 0.5, max_ny - 0.5),
1,
1,
linewidth=2,
edgecolor="red",
facecolor="none",
(max_nx - 0.5, max_ny - 0.5), 1, 1, linewidth=2, edgecolor="red", facecolor="none"
)
ax.add_patch(rect)
@@ -466,10 +426,7 @@ def _draw_panel(
xtal_size = get_xtal_size(CrystalSize(x=0, y=0, z=0), mask.astype(float), r)
size_line = f"\n{xtal_size.x:.0f}×{xtal_size.y:.0f} µm"
ax.set_title(
f"{label}\nthresh≈{threshold:.1f} | {n_cells} cells{size_line}",
fontsize=8,
)
ax.set_title(f"{label}\nthresh≈{threshold:.1f} | {n_cells} cells{size_line}", fontsize=8)
ax.set_xlabel("nx", fontsize=7)
ax.set_ylabel("ny", fontsize=7)
ax.tick_params(labelsize=6)
@@ -506,9 +463,7 @@ def crystal_mask_otsu_nonzero(arr: np.ndarray) -> tuple[np.ndarray, float]:
return arr > threshold, threshold
def crystal_mask_mean_sigma(
arr: np.ndarray, n_sigma: float = 0.5
) -> tuple[np.ndarray, float]:
def crystal_mask_mean_sigma(arr: np.ndarray, n_sigma: float = 0.5) -> tuple[np.ndarray, float]:
"""Threshold = mean + n_sigma * std of nonzero values.
n_sigma=0.5 keeps cells within ~1/2 std above average signal.
@@ -569,9 +524,7 @@ def crystal_mask_dbscan(
return mask, min_signal
def crystal_mask_kmeans(
arr: np.ndarray, n_clusters: int = 3
) -> tuple[np.ndarray, float]:
def crystal_mask_kmeans(arr: np.ndarray, n_clusters: int = 3) -> tuple[np.ndarray, float]:
"""K-means on signal values (1-D feature).
Splits cells into n_clusters groups; the cluster with the highest
@@ -590,10 +543,7 @@ def crystal_mask_kmeans(
CRYSTAL_METHOD_MAP: dict = {
"corner": ("Corner background", crystal_mask_corner_background),
"otsu": ("Otsu (nonzero)", crystal_mask_otsu_nonzero),
"mean_sigma": (
"Mean + 0.5σ",
lambda arr: crystal_mask_mean_sigma(arr, n_sigma=0.5),
),
"mean_sigma": ("Mean + 0.5σ", lambda arr: crystal_mask_mean_sigma(arr, n_sigma=0.5)),
"percentile": (
"Top-40% signal",
lambda arr: crystal_mask_signal_percentile(arr, percentile=60.0),
@@ -671,9 +621,7 @@ def compare_crystal_methods_scored(
results, w_bkg=w_bkg, w_low_res=w_low_res, w_indexed=w_indexed
)
method_entries = [
(name, label, fn) for name, (label, fn) in CRYSTAL_METHOD_MAP.items()
]
method_entries = [(name, label, fn) for name, (label, fn) in CRYSTAL_METHOD_MAP.items()]
fig, axes = plt.subplots(2, 3, figsize=(15, 9))
for ax, (name, label, fn) in zip(axes.flat, method_entries):
@@ -692,7 +640,9 @@ def compare_crystal_methods_scored(
spine.set_edgecolor("yellow")
spine.set_linewidth(3)
weight_note = f"bkg×{w_bkg:.0%} + spots_low_res×{w_low_res:.0%} + spots_indexed×{w_indexed:.0%}"
weight_note = (
f"bkg×{w_bkg:.0%} + spots_low_res×{w_low_res:.0%} + spots_indexed×{w_indexed:.0%}"
)
fig.suptitle(
f"Crystal region detection on composite score [{weight_note}]\n"
"cyan = crystal outline | red square = max-score cell | score range 0100",
+4 -15
View File
@@ -69,15 +69,11 @@ class _SeedConfig:
SEED_CATALOGUE: dict[int, _SeedConfig] = {
# --- 1: single crystal, centred, compact, strong signal ----------------
1: _SeedConfig(
clusters=[
_ClusterParams(cx=0.50, cy=0.50, ax=0.15, ay=0.15, theta=0.0, peak=120),
]
clusters=[_ClusterParams(cx=0.50, cy=0.50, ax=0.15, ay=0.15, theta=0.0, peak=120)]
),
# --- 2: single crystal, off-centre, slightly elongated -----------------
2: _SeedConfig(
clusters=[
_ClusterParams(cx=0.25, cy=0.70, ax=0.12, ay=0.18, theta=0.35, peak=90),
]
clusters=[_ClusterParams(cx=0.25, cy=0.70, ax=0.12, ay=0.18, theta=0.35, peak=90)]
),
# --- 3: two well-separated crystals ------------------------------------
3: _SeedConfig(
@@ -111,11 +107,7 @@ SEED_CATALOGUE: dict[int, _SeedConfig] = {
def _gaussian_cluster(
ix: np.ndarray,
iy: np.ndarray,
p: _ClusterParams,
n_x: int,
n_y: int,
ix: np.ndarray, iy: np.ndarray, p: _ClusterParams, n_x: int, n_y: int
) -> np.ndarray:
"""Return a 2-D Gaussian signal array for one cluster.
@@ -135,10 +127,7 @@ def _gaussian_cluster(
return p.peak * np.exp(-2.0 * (dx_r**2 + dy_r**2))
def generate_no_beam_scan_result(
request: RasterGridRequest,
seed: int | None = None,
) -> ScanResult:
def generate_no_beam_scan_result(request: RasterGridRequest, seed: int | None = None) -> ScanResult:
"""Build a synthetic ScanResult for no-beam / offline operation.
Parameters
+14 -50
View File
@@ -10,14 +10,7 @@ from aarecommon.math.diffraction_geometry import DiffractionGeometry
from aarecommon.math.sample_geometry import SampleGeometryModel
from aarecommon.models.tell import TellStateModel
from jfjoch_client.models.scan_result import ScanResult
from pydantic import (
AfterValidator,
AliasChoices,
BaseModel,
ConfigDict,
Field,
field_validator,
)
from pydantic import AfterValidator, AliasChoices, BaseModel, ConfigDict, Field, field_validator
class StagePositionEnum(Enum):
@@ -68,31 +61,22 @@ class DataCollectionParameters(BaseModel):
totalangle: Optional[int] = Field( # was totalrange
default=None, validation_alias=AliasChoices("totalangle", "totalrange")
) # Only accept positive integers between 0 and 360
transmission: Optional[int] = (
None # Only accept positive integers between 0 and 100
)
transmission: Optional[int] = None # Only accept positive integers between 0 and 100
targetresolution: Optional[float] = None # Only accept positive float
beamsize: Optional[str] = None
aperture: Optional[int] = None # Optional string field
datacollectiontype: Optional[str] = (
None # Only accept "standard", other types might be added later
)
processingpipeline: Optional[str] = (
"" # Only accept "gopy", "autoproc", "xia2dials"
)
spacegroupnumber: Optional[int] = (
None # Only accept positive integers between 1 and 230
)
processingpipeline: Optional[str] = "" # Only accept "gopy", "autoproc", "xia2dials"
spacegroupnumber: Optional[int] = None # Only accept positive integers between 1 and 230
unitcell: Optional[str] = Field( # was cellparameters
default=None, validation_alias=AliasChoices("unitcell", "cellparameters")
) # Must be a set of six positive floats or integers
rescutkey: Optional[str] = None # Only accept "is" or "cchalf"
rescutvalue: Optional[float] = (
None # Must be a positive float if rescutkey is provided
)
rescutvalue: Optional[float] = None # Must be a positive float if rescutkey is provided
processingresolution: Optional[float] = Field( # was userresolution
default=None,
validation_alias=AliasChoices("processingresolution", "userresolution"),
default=None, validation_alias=AliasChoices("processingresolution", "userresolution")
)
pdbid: Optional[str] = (
"" # Accepts either the format of the protein data bank code or {provided}
@@ -142,9 +126,7 @@ class DataCollectionParameters(BaseModel):
"{protein}",
"{method}",
]
valid_macro_pattern = re.compile(
"|".join(re.escape(macro) for macro in valid_macros)
)
valid_macro_pattern = re.compile("|".join(re.escape(macro) for macro in valid_macros))
# Check if the value contains valid macros
allowed_chars_pattern = "[a-z0-9_.+-/]"
@@ -154,9 +136,7 @@ class DataCollectionParameters(BaseModel):
f"^(({allowed_chars_pattern}+|macro)*/*)*$", re.IGNORECASE
)
if not allowed_path_pattern.match(v_without_macros):
raise ValueError(
f"'{v}' is not valid. Value must be a valid path or macro."
)
raise ValueError(f"'{v}' is not valid. Value must be a valid path or macro.")
return v
@field_validator("unitcell", mode="before")
@@ -347,9 +327,7 @@ class MLOutputModel(BaseModel):
return None
return (m.box.top_x, m.box.top_y, m.box.bottom_x, m.box.bottom_y)
def get_box_tuple_with_conf(
self, key: str
) -> tuple[float, float, float, float, float] | None:
def get_box_tuple_with_conf(self, key: str) -> tuple[float, float, float, float, float] | None:
m = self.get_box_model(key)
if not m or not m.box:
return None
@@ -368,9 +346,7 @@ class MLOutputModel(BaseModel):
keys = self.get_keys_for_class(cls)
return [self.boxes[k] for k in keys]
def get_tuples_for_class(
self, cls: MLBoxType
) -> list[tuple[float, float, float, float]]:
def get_tuples_for_class(self, cls: MLBoxType) -> list[tuple[float, float, float, float]]:
out: list[tuple[float, float, float, float]] = []
for m in self.get_models_for_class(cls):
if m.box:
@@ -384,13 +360,7 @@ class MLOutputModel(BaseModel):
for m in self.get_models_for_class(cls):
if m.box:
out.append(
(
m.box.top_x,
m.box.top_y,
m.box.bottom_x,
m.box.bottom_y,
float(m.conf),
)
(m.box.top_x, m.box.top_y, m.box.bottom_x, m.box.bottom_y, float(m.conf))
)
return out
@@ -465,9 +435,7 @@ class ZoomModeEnum(Enum):
class ZoomModel(BaseModel):
z: dict[float, SampleCameraSettings]
def get_camera_settings(
self, zoom_value: float
) -> SampleCameraSettings:
def get_camera_settings(self, zoom_value: float) -> SampleCameraSettings:
if not self.z:
raise ValueError("No zoom data available")
@@ -495,16 +463,12 @@ class ZoomModel(BaseModel):
t = (zoom_value - lower_zoom) / (upper_zoom - lower_zoom)
interpolated_gain = lower_elem.gain + t * (
upper_elem.gain - lower_elem.gain
)
interpolated_gain = lower_elem.gain + t * (upper_elem.gain - lower_elem.gain)
interpolated_exp = lower_elem.exposure + t * (
upper_elem.exposure - lower_elem.exposure
)
return SampleCameraSettings(
gain=interpolated_gain, exposure=interpolated_exp
)
return SampleCameraSettings(gain=interpolated_gain, exposure=interpolated_exp)
closest = min(self.z.keys(), key=lambda x: abs(x - zoom_value))
elem = self.z[closest]
+1 -3
View File
@@ -59,9 +59,7 @@ class CenterOfMassModel(BaseModel):
x=(self.n_x + 0.5) * r.grid_size_mm.x, y=(self.n_y + 0.5) * r.grid_size_mm.y
)
def get_com_pxl(
self, r: RasterGridRequest, geom: SampleGeometryModel
) -> Coordinate:
def get_com_pxl(self, r: RasterGridRequest, geom: SampleGeometryModel) -> Coordinate:
# defines COM in pixels at beam position, relative to the top left corner of grid
com_mm = self.get_com_mm(r)
return Coordinate(x=com_mm.x / geom.pixel_in_mm, y=com_mm.y / geom.pixel_in_mm)
+5 -18
View File
@@ -47,9 +47,7 @@ class RecurrenceWatcher:
if exception_class is None:
self._streak = 0
return False
if not isinstance(exception_class, type) or not issubclass(
exception_class, self._observes
):
if not isinstance(exception_class, type) or not issubclass(exception_class, self._observes):
return False
self._streak += 1
return self._streak >= self.threshold
@@ -57,9 +55,7 @@ class RecurrenceWatcher:
def maybe_trip(self, exception_class: type | None) -> WatcherTrip | None:
if not self.observe(exception_class):
return None
class_name = (
exception_class.__name__ if isinstance(exception_class, type) else "Unknown"
)
class_name = exception_class.__name__ if isinstance(exception_class, type) else "Unknown"
return WatcherTrip(
name=self.name,
threshold=self.threshold,
@@ -76,16 +72,10 @@ DEFAULT_WATCHERS: tuple[tuple[str, ExceptionType, int], ...] = (
)
def create_default_watchers(
overrides: dict[str, int] | None = None,
) -> list[RecurrenceWatcher]:
def create_default_watchers(overrides: dict[str, int] | None = None) -> list[RecurrenceWatcher]:
thresholds = dict(overrides or {})
return [
RecurrenceWatcher(
name=name,
observes=observes,
threshold=thresholds.get(name, threshold),
)
RecurrenceWatcher(name=name, observes=observes, threshold=thresholds.get(name, threshold))
for name, observes, threshold in DEFAULT_WATCHERS
]
@@ -103,10 +93,7 @@ def redis_key_to_env_var(key: str) -> str:
def load_watcher_threshold_overrides(
*,
beamline: str,
watcher_names: list[str],
get_value: Callable[[str], str | bytes | None],
*, beamline: str, watcher_names: list[str], get_value: Callable[[str], str | bytes | None]
) -> dict[str, int]:
overrides: dict[str, int] = {}
for watcher_name in watcher_names:
+3 -9
View File
@@ -77,9 +77,7 @@ def test_phase3_classes_have_critical_default_true():
SmartMagnetFaultException,
TransformationInvalidException,
):
assert cls.critical is True, (
f"{cls.__name__} should default to critical=True in Phase 3"
)
assert cls.critical is True, f"{cls.__name__} should default to critical=True in Phase 3"
def test_other_core_classes_keep_critical_default_false():
@@ -157,9 +155,7 @@ def test_automation_errors_are_automation_error():
AutoRasterSampleSkipped,
):
exc = cls() if cls is not AutoRasterSampleSkipped else cls("skipped")
assert isinstance(exc, AutomationError), (
f"{cls.__name__} should be AutomationError"
)
assert isinstance(exc, AutomationError), f"{cls.__name__} should be AutomationError"
assert isinstance(exc, AareException), f"{cls.__name__} should be AareException"
@@ -193,9 +189,7 @@ def test_tell_family_membership():
UnmountingFailed,
):
exc = cls()
assert isinstance(exc, TellException), (
f"{cls.__name__} should be in Tell family"
)
assert isinstance(exc, TellException), f"{cls.__name__} should be in Tell family"
assert isinstance(exc, AutomationError)
+2 -6
View File
@@ -104,9 +104,7 @@ def test_coordinate_rotate_invalid_axis():
def test_smargon_coordinate_equality():
s1 = SmargonCoordinate(sh_mm=Coordinate(x=1, y=1, z=1), phi_deg=10, chi_deg=20)
s2 = SmargonCoordinate(
sh_mm=Coordinate(x=1.05, y=0.95, z=1.01), phi_deg=10.05, chi_deg=19.95
)
s2 = SmargonCoordinate(sh_mm=Coordinate(x=1.05, y=0.95, z=1.01), phi_deg=10.05, chi_deg=19.95)
assert s1 == s2
s3 = SmargonCoordinate(sh_mm=Coordinate(x=2, y=1, z=1), phi_deg=10, chi_deg=20)
@@ -115,9 +113,7 @@ def test_smargon_coordinate_equality():
def test_aerotech_coordinate_equality():
a1 = AerotechCoordinate(at_mm=Coordinate(x=1, y=1, z=1), omega_deg=10)
a2 = AerotechCoordinate(
at_mm=Coordinate(x=1.005, y=0.995, z=1.001), omega_deg=10.005
)
a2 = AerotechCoordinate(at_mm=Coordinate(x=1.005, y=0.995, z=1.001), omega_deg=10.005)
assert a1 == a2
a3 = AerotechCoordinate(at_mm=Coordinate(x=1.1, y=1, z=1), omega_deg=10)
+1 -3
View File
@@ -50,9 +50,7 @@ def test_processingpipeline_accepts_unknown_value_after_refactor():
def test_datacollectionparameters_accepts_legacy_aliases():
params = DataCollectionParameters(
totalrange=180,
cellparameters="10 20 30 90 90 120",
userresolution=1.4,
totalrange=180, cellparameters="10 20 30 90 90 120", userresolution=1.4
)
assert params.totalangle == 180
assert params.unitcell == "10 20 30 90 90 120"
+1 -3
View File
@@ -51,9 +51,7 @@ def test_resolution_angstrom(sample_dg):
def test_max_resolution_angstrom(sample_dg):
assert sample_dg.max_resolution_angstrom == sample_dg.resolution_angstrom(
sample_dg.dtz_mm
)
assert sample_dg.max_resolution_angstrom == sample_dg.resolution_angstrom(sample_dg.dtz_mm)
def test_calc_dtz_mm(sample_dg):
+3 -11
View File
@@ -25,23 +25,15 @@ def test_export_error_codes_contains_known_codes():
def test_code_for_exception_class_basic():
assert (
code_for_exception_class("TellCommunicationError") == "TELL_COMMUNICATION_ERROR"
)
assert code_for_exception_class("TellCommunicationError") == "TELL_COMMUNICATION_ERROR"
assert code_for_exception_class("MountingFailed") == "MOUNTING_FAILED"
assert code_for_exception_class("LoopCenteringFailed") == "LOOP_CENTERING_FAILED"
def test_code_for_exception_class_acronyms():
assert code_for_exception_class("AXCFailed") == "AXC_FAILED"
assert (
code_for_exception_class("AareDBCommunicationError")
== "AARE_DB_COMMUNICATION_ERROR"
)
assert (
code_for_exception_class("JFJochCommunicationError")
== "JF_JOCH_COMMUNICATION_ERROR"
)
assert code_for_exception_class("AareDBCommunicationError") == "AARE_DB_COMMUNICATION_ERROR"
assert code_for_exception_class("JFJochCommunicationError") == "JF_JOCH_COMMUNICATION_ERROR"
def test_code_for_each_concrete_exception_is_in_aare_error_code_enum():
+1 -3
View File
@@ -38,9 +38,7 @@ def test_data_collection_exception_default_message():
def test_authentication_exception_properties():
exc = AuthenticationException(
"Failed", status_code=403, code=AuthErrorCode.FORBIDDEN
)
exc = AuthenticationException("Failed", status_code=403, code=AuthErrorCode.FORBIDDEN)
assert exc.status_code == 403
assert exc.code == AuthErrorCode.FORBIDDEN
assert "Failed" in str(exc)
+4 -16
View File
@@ -41,11 +41,7 @@ def mock_raster_results():
@pytest.fixture
def raster_request():
return RasterGridRequest(
exp_time_s=0.1,
n_x=3,
n_y=3,
grid_size_mm=Coordinate(x=0.02, y=0.02),
smargon_top_left=None,
exp_time_s=0.1, n_x=3, n_y=3, grid_size_mm=Coordinate(x=0.02, y=0.02), smargon_top_left=None
)
@@ -62,9 +58,7 @@ def test_identify_crystal_raster(mock_raster_results, raster_request):
def test_rebuild_array_from_scan_results(mock_raster_results):
arr = rebuild_array_from_scan_results(
mock_raster_results, "spots_low_res", array_shape=(3, 3)
)
arr = rebuild_array_from_scan_results(mock_raster_results, "spots_low_res", array_shape=(3, 3))
assert arr.shape == (3, 3)
assert arr[0, 0] == 0.0
assert arr[2, 2] == 8.0
@@ -73,11 +67,7 @@ def test_rebuild_array_from_scan_results(mock_raster_results):
def test_create_quality_filtered_array(mock_raster_results):
# Testing create_quality_filtered_array with a more lenient filter
arr = create_quality_filtered_array(
mock_raster_results,
"spots",
min_low_res_spots=0.0,
min_background=0.0,
array_shape=(3, 3),
mock_raster_results, "spots", min_low_res_spots=0.0, min_background=0.0, array_shape=(3, 3)
)
# i=8 has nx=2, ny=2 -> row=2, col=2
assert arr[2, 2] == 18.0
@@ -163,9 +153,7 @@ def test_compute_crystal_score_array_weights():
r.bkg, r.spots_low_res, r.spots_indexed = 0.0, low, idx
return r
score = compute_crystal_score_array(
[_cell(0, 0, 10.0, 0.0), _cell(1, 1, 0.0, 10.0)]
)
score = compute_crystal_score_array([_cell(0, 0, 10.0, 0.0), _cell(1, 1, 0.0, 10.0)])
assert score[0, 0] > score[1, 0]
assert score[0, 0] == pytest.approx(55.0)
assert score[1, 0] == pytest.approx(20.0)
+2 -8
View File
@@ -18,9 +18,7 @@ def test_sample_short_info_methods():
sample_name="sample1",
run_number=1,
aaredb_params=DataCollectionParameters(
totalangle=180,
processingresolution=1.5,
cloud=True,
totalangle=180, processingresolution=1.5, cloud=True
),
user="group1",
pin=3,
@@ -48,11 +46,7 @@ def test_sample_short_info_methods():
"dewar_name": "dew2",
"sample_name": "sample2",
"run_number": 2,
"aaredb_params": {
"totalrange": 120,
"userresolution": 1.8,
"cloud": "",
},
"aaredb_params": {"totalrange": 120, "userresolution": 1.8, "cloud": ""},
"user": "group2",
"pin": 4,
"location": {"segment": "B", "pos": 5},
+2 -6
View File
@@ -43,9 +43,7 @@ def test_round_trip_conversion_for_multiple_grid_shapes():
for grid_y in range(1, 7):
for grid_x in range(1, number_of_cols + 1):
image_id = grid_to_image_id(
grid_x=grid_x,
grid_y=grid_y,
number_of_cols=number_of_cols,
grid_x=grid_x, grid_y=grid_y, number_of_cols=number_of_cols
)
assert image_id_to_grid(image_id, number_of_cols) == (grid_x, grid_y)
@@ -83,9 +81,7 @@ def test_grid_to_image_id_rejects_zero_grid_y():
def test_grid_to_image_id_rejects_grid_x_larger_than_number_of_cols():
with pytest.raises(
ValueError, match="grid_x cannot be greater than number_of_cols"
):
with pytest.raises(ValueError, match="grid_x cannot be greater than number_of_cols"):
grid_to_image_id(grid_x=6, grid_y=1, number_of_cols=5)
+2 -6
View File
@@ -38,18 +38,14 @@ def test_load_watcher_threshold_overrides_reads_valid_values_only():
}
overrides = load_watcher_threshold_overrides(
beamline="mx",
watcher_names=["tell", "alc", "smargon"],
get_value=values.get,
beamline="mx", watcher_names=["tell", "alc", "smargon"], get_value=values.get
)
assert overrides == {"tell": 7, "alc": 3}
def test_create_default_watchers_applies_overrides():
watchers = {
watcher.name: watcher for watcher in create_default_watchers({"alc": 4})
}
watchers = {watcher.name: watcher for watcher in create_default_watchers({"alc": 4})}
assert watchers["alc"].threshold == 4
assert watchers["tell"].threshold > 0