feat/lamni tomogram timing overview + repeat-on-interruption
Port two flomni tomo_scan() behaviors lamni was missing: an end-of-scan timing overview (total time, time excluding gaps, time lost to gaps, sample/hook info) printed and written to scilog, and automatic repeat of an entire projection when the beamline interlock trips mid-scan. The latter needed two pieces together: enabling bec.builtin_actors.scan_interlock at tomo_scan() start (previously never enabled for lamni), and wrapping _tomo_scan_at_angle in @scan_repeat so the resulting ScanRestart (or any other transient exception) redoes the whole projection instead of aborting the scan. A new LamNIError marks the one non-retryable case (unregistered at_each_angle_hook), replacing a plain ValueError there. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -88,6 +88,9 @@ class DataDrivenLamNI(LamNI):
|
||||
bec = builtins.__dict__.get("bec")
|
||||
scans = builtins.__dict__.get("scans")
|
||||
|
||||
bec.builtin_actors.scan_interlock.trigger_setting = "restart_scan"
|
||||
bec.builtin_actors.scan_interlock.enabled = True
|
||||
|
||||
fname = os.path.expanduser(fname)
|
||||
if not os.path.exists(fname):
|
||||
raise FileNotFoundError(f"Could not find datadriven params file in {fname}.")
|
||||
|
||||
@@ -9,6 +9,7 @@ import numpy as np
|
||||
from bec_lib import bec_logger
|
||||
from bec_lib.alarm_handler import AlarmBase
|
||||
from bec_lib.pdf_writer import PDFWriter
|
||||
from bec_lib.scan_repeat import scan_repeat
|
||||
from typeguard import typechecked
|
||||
|
||||
from csaxs_bec.bec_ipython_client.plugins.omny.omny_general_tools import (
|
||||
@@ -107,6 +108,11 @@ class _ProgressProxy:
|
||||
return self._load()
|
||||
|
||||
|
||||
class LamNIError(Exception):
|
||||
"""A definite, non-transient tomo-scan failure (bad config, unmet
|
||||
precondition, ...) that should never be retried by @scan_repeat."""
|
||||
|
||||
|
||||
class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools):
|
||||
# Ordered set of all global-var-backed tomo scan parameters snapshotted
|
||||
# by tomo_queue_add()/restored by tomo_queue_execute(). Lamni's own
|
||||
@@ -778,7 +784,7 @@ class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools
|
||||
if hook_name:
|
||||
hook = self._at_each_angle_hooks.get(hook_name)
|
||||
if hook is None:
|
||||
raise ValueError(
|
||||
raise LamNIError(
|
||||
f"at_each_angle_hook '{hook_name}' is not registered in this "
|
||||
"session. Hooks are session-only and do not survive a kernel "
|
||||
f"restart -- call register_at_each_angle_hook({hook_name!r}, "
|
||||
@@ -891,6 +897,20 @@ class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools
|
||||
self.progress["angle"] = angle
|
||||
self._tomo_scan_at_angle(angle, subtomo_number)
|
||||
|
||||
@staticmethod
|
||||
def _retry_unless_lamni_error(exc: Exception, attempt: int) -> bool:
|
||||
"""scan_repeat() exc_handler: retry any exception except LamNIError.
|
||||
|
||||
LamNIError marks a definite, non-transient failure (bad config,
|
||||
unmet precondition, ...) -- e.g. _at_each_angle() raises it when a
|
||||
job's at_each_angle_hook name isn't registered in this session.
|
||||
Retrying that up to max_repeats times just burns 10 attempts before
|
||||
surfacing a much less informative TooManyScanRestarts, instead of
|
||||
the actual, actionable error message immediately.
|
||||
"""
|
||||
return not isinstance(exc, LamNIError)
|
||||
|
||||
@scan_repeat(max_repeats=10, default=True, exc_handler=_retry_unless_lamni_error)
|
||||
def _tomo_scan_at_angle(self, angle, subtomo_number):
|
||||
successful = False
|
||||
error_caught = False
|
||||
@@ -1013,6 +1033,10 @@ class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools
|
||||
|
||||
bec = builtins.__dict__.get("bec")
|
||||
scans = builtins.__dict__.get("scans")
|
||||
|
||||
bec.builtin_actors.scan_interlock.trigger_setting = "restart_scan"
|
||||
bec.builtin_actors.scan_interlock.enabled = True
|
||||
|
||||
self._current_special_angles = self.special_angles.copy()
|
||||
|
||||
if (
|
||||
@@ -1147,10 +1171,37 @@ class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools
|
||||
self.progress["projection"] = self.progress["total_projections"]
|
||||
self.progress["subtomo_projection"] = self.progress["subtomo_total_projections"]
|
||||
self._print_progress()
|
||||
print(
|
||||
"Total measurement time lost to detected gaps:"
|
||||
f" {self._format_duration(self.progress.get('accumulated_idle_time', 0.0))}"
|
||||
self.OMNYTools.printgreenbold("Tomoscan finished")
|
||||
|
||||
idle_s = self.progress.get("accumulated_idle_time", 0.0)
|
||||
start_str = self.progress.get("tomo_start_time")
|
||||
elapsed_s = None
|
||||
if start_str is not None:
|
||||
try:
|
||||
elapsed_s = (
|
||||
datetime.datetime.now() - datetime.datetime.fromisoformat(start_str)
|
||||
).total_seconds()
|
||||
except (ValueError, TypeError):
|
||||
elapsed_s = None
|
||||
timing_lines = [
|
||||
"Tomoscan finished.",
|
||||
f"Measurement ID: {self.tomo_id}",
|
||||
f"Sample: {self.sample_name}",
|
||||
]
|
||||
if self.at_each_angle_hook:
|
||||
timing_lines.append(f"At-each-angle hook: {self.at_each_angle_hook}")
|
||||
if elapsed_s is not None:
|
||||
timing_lines.append(f"Total measurement time: {self._format_duration(elapsed_s)}")
|
||||
timing_lines.append(
|
||||
f"Total measurement time excluding detected gaps: {self._format_duration(elapsed_s - idle_s)}"
|
||||
)
|
||||
timing_lines.append(
|
||||
f"Total measurement time lost to detected gaps: {self._format_duration(idle_s)}"
|
||||
)
|
||||
for line in timing_lines[3:]:
|
||||
print(line)
|
||||
timing_content = "\n".join(timing_lines)
|
||||
self.write_to_scilog(timing_content, ["tomoscan"])
|
||||
|
||||
def tomo_scan_resume(self) -> None:
|
||||
"""Resume a tomo_scan() that crashed or was interrupted, picking up
|
||||
|
||||
@@ -166,8 +166,14 @@ def make_lamni_for_tomo_scan(tomo_angle_stepsize: float, active_account: str) ->
|
||||
obj._print_progress = lambda: None
|
||||
obj._format_duration = lambda seconds: "0s"
|
||||
obj.lamnigui_show_progress = lambda: None
|
||||
obj.at_each_angle_hook = None
|
||||
obj.OMNYTools = types.SimpleNamespace(printgreenbold=lambda msg: None)
|
||||
builtins.__dict__["bec"] = types.SimpleNamespace(
|
||||
active_account=active_account, queue=types.SimpleNamespace(next_scan_number=1)
|
||||
active_account=active_account,
|
||||
queue=types.SimpleNamespace(next_scan_number=1),
|
||||
builtin_actors=types.SimpleNamespace(
|
||||
scan_interlock=types.SimpleNamespace(trigger_setting=None, enabled=False)
|
||||
),
|
||||
)
|
||||
builtins.__dict__["scans"] = _FakeScans()
|
||||
return obj
|
||||
|
||||
@@ -11,7 +11,7 @@ import types
|
||||
|
||||
import csaxs_bec.bec_ipython_client.plugins.LamNI.lamni as lamni_module
|
||||
import csaxs_bec.bec_ipython_client.plugins.OMNY_shared.tomo_queue_mixin as tomo_queue_mixin
|
||||
from csaxs_bec.bec_ipython_client.plugins.LamNI.lamni import LamNI
|
||||
from csaxs_bec.bec_ipython_client.plugins.LamNI.lamni import LamNI, LamNIError
|
||||
|
||||
|
||||
class _FakeRtx:
|
||||
@@ -200,7 +200,7 @@ def test_at_each_angle_raises_for_unregistered_hook_name():
|
||||
lamni.at_each_angle_hook = "missing_hook"
|
||||
try:
|
||||
lamni._at_each_angle(0.0)
|
||||
assert False, "expected ValueError for an unregistered hook name"
|
||||
except ValueError as exc:
|
||||
assert False, "expected LamNIError for an unregistered hook name"
|
||||
except LamNIError as exc:
|
||||
assert "missing_hook" in str(exc)
|
||||
assert "not registered" in str(exc)
|
||||
|
||||
Reference in New Issue
Block a user