From a201f1791d2231ec3d4e4c66272e6f4b7317fa1e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 12 Nov 2026 19:22:45 +0100 Subject: [PATCH 1/4] receiver: stills live counts a frame as received before it takes a slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live status's images_received, the frame rate and the end latency of a stills collection ran from StillsGPUAnalysis::Hold, which only sees the frames that got a slot of the image buffer: a frame dropped at a full buffer was not counted, so after a loss the "N / M ยท Hz" header and the latency figure ran from the last frame rugnux kept, not the last frame the detector sent. The receiver now reports the frame to the live status in OnImage, before the slot is taken, as the rotation receiver does; the drop itself is unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- receiver/JFJochReceiverRugnuxStills.cpp | 1 + receiver/StillsGPUAnalysis.cpp | 1 - tests/StillsGPUAnalysisTest.cpp | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/receiver/JFJochReceiverRugnuxStills.cpp b/receiver/JFJochReceiverRugnuxStills.cpp index 511335781..b82734d14 100644 --- a/receiver/JFJochReceiverRugnuxStills.cpp +++ b/receiver/JFJochReceiverRugnuxStills.cpp @@ -66,6 +66,7 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std: logger.Error("Dropping image {} - the collection has {} images", message.number, experiment.GetImageNum()); return; } + live_status.ImageReceived(received); CountImage(message); const size_t size = message.image.GetCompressedSize(); diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index 65721f92c..cd8e5328f 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -67,7 +67,6 @@ void StillsGPUAnalysis::Publish() { } void StillsGPUAnalysis::Hold(const std::shared_ptr &image) { - status.ImageReceived(std::chrono::steady_clock::now()); queue.PutBlocking(image); { std::unique_lock ul(prepass_mutex); diff --git a/tests/StillsGPUAnalysisTest.cpp b/tests/StillsGPUAnalysisTest.cpp index 23c44e234..bacdfa9b2 100644 --- a/tests/StillsGPUAnalysisTest.cpp +++ b/tests/StillsGPUAnalysisTest.cpp @@ -114,8 +114,10 @@ TEST_CASE("StillsGPUAnalysis_FailingFrameDoesNotStopAnalysis", "[StillsGPUAnalys }; StillsGPUAnalysis gpu(x, mask, Start(x), nullptr, 1, 100, logger, sink, status); // The frames carry no image, so their analysis fails too - they are passed on all the same. - for (int i = 0; i < 3; i++) + for (int i = 0; i < 3; i++) { + status.ImageReceived(std::chrono::steady_clock::now()); // as the receiver does before it holds a frame gpu.Hold(Frame(buffer, i)); + } REQUIRE_NOTHROW(gpu.Finish()); CHECK(sunk == 3); // The outcome waits for everything to be written, and comes after the latency. From c9704d1c5cb158b9d92f8e6e1b85bb63de79e14e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 12 Nov 2026 19:22:45 +0100 Subject: [PATCH 2/4] broker: a /start without detect_ice_rings leaves rugnux live its own default A rotation sweep run live had ice-ring detection off unless /start asked for it: the dataset settings held the flag as a plain bool (default false), the start message always carried it, and MakeRugnuxLiveSetup took it from there as the file's value - where a bare `rugnux` on the same images defaults it on for a rotation sweep. The flag is now optional through the dataset settings and the start message (which was already std::optional, as is the OpenAPI model, so neither the CBOR stream nor the API schema changes): left out of /start, the start message carries no detect_ice_rings, rugnux live decides as the CLI would (on for rotation, off for stills - unchanged there), and the master file carries no detect_ice_rings key, so an offline `rugnux` on the written file takes the same default. Given on /start, true or false, it is honoured everywhere as before. The broker's own spot finding (SpotUtils) keeps treating an unset flag as off. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- broker/OpenAPIConvert.cpp | 3 ++- broker/gen/model/Dataset_settings.h | 2 +- broker/jfjoch_api.yaml | 5 ++++- common/DatasetSettings.cpp | 6 ++++-- common/DatasetSettings.h | 7 +++++-- common/DiffractionExperiment.cpp | 2 +- docs/CBOR.md | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index a9f7800c8..6527d4b84 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -730,7 +730,8 @@ DatasetSettings Convert(const org::openapitools::server::model::Dataset_settings ret.SpotFindingEnable(input.isSpotFinding()); ret.MaxSpotCount(input.getMaxSpotCount()); - ret.DetectIceRings(input.isDetectIceRings()); + if (input.detectIceRingsIsSet()) + ret.DetectIceRings(input.isDetectIceRings()); if (input.xrayFluorescenceSpectrumIsSet()) { auto fl = input.getXrayFluorescenceSpectrum(); diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index 601df8418..6a91d8d51 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -325,7 +325,7 @@ public: bool maxSpotCountIsSet() const; void unsetMax_spot_count(); /// - /// Flag spots as ice rings and reduce their effect on indexing + /// Flag spots as ice rings and reduce their effect on indexing. Left out, the broker's own spot finding treats no spot as an ice ring, rugnux live decides as a bare `rugnux` would (on for a rotation sweep, off for stills), and the master file carries no `detect_ice_rings`. /// bool isDetectIceRings() const; void setDetectIceRings(bool const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 5201be5e1..788c23fd3 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -671,7 +671,10 @@ components: description: Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected detect_ice_rings: type: boolean - description: Flag spots as ice rings and reduce their effect on indexing + description: | + Flag spots as ice rings and reduce their effect on indexing. Left out, the broker's own + spot finding treats no spot as an ice ring, rugnux live decides as a bare `rugnux` would (on + for a rotation sweep, off for stills), and the master file carries no `detect_ice_rings`. async_start: type: boolean default: false diff --git a/common/DatasetSettings.cpp b/common/DatasetSettings.cpp index aa82553a6..601b5114b 100644 --- a/common/DatasetSettings.cpp +++ b/common/DatasetSettings.cpp @@ -30,8 +30,6 @@ DatasetSettings::DatasetSettings() { poni_rot_3_rad = 0.0f; max_spot_count = std::min(MAX_SPOT_COUNT, 250); - - detect_ice_rings = false; } DatasetSettings &DatasetSettings::ImagesPerTrigger(int64_t input) { @@ -542,6 +540,10 @@ DatasetSettings & DatasetSettings::DetectIceRings(bool input) { } bool DatasetSettings::IsDetectIceRings() const { + return detect_ice_rings.value_or(false); +} + +std::optional DatasetSettings::GetDetectIceRings() const { return detect_ice_rings; } diff --git a/common/DatasetSettings.h b/common/DatasetSettings.h index 1d3cd785a..208c521ca 100644 --- a/common/DatasetSettings.h +++ b/common/DatasetSettings.h @@ -75,7 +75,9 @@ class DatasetSettings { bool spot_finding_enable; int64_t max_spot_count; - bool detect_ice_rings; // Enable special handling of ice rings + // Special handling of ice rings; unset where the user did not ask either way, so that the analysis takes its own + // default (rugnux: on for a rotation sweep, off for stills) and the master file carries no detect_ice_rings + std::optional detect_ice_rings; XrayFluorescenceSpectrum fluorescence_spectrum; std::optional smargon_position; @@ -186,7 +188,8 @@ public: bool IsSpotFindingEnabled() const; int64_t GetMaxSpotCount() const; - bool IsDetectIceRings() const; + bool IsDetectIceRings() const; // false where unset + std::optional GetDetectIceRings() const; const XrayFluorescenceSpectrum &GetFluorescenceSpectrum() const; std::optional GetSmargonPosition() const; }; diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index e97974bdc..d31d4926e 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -773,7 +773,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const { message.poni_rot2 = dataset.GetPoniRot2_rad(); message.poni_rot3 = dataset.GetPoniRot3_rad(); - message.detect_ice_rings = dataset.IsDetectIceRings(); + message.detect_ice_rings = dataset.GetDetectIceRings(); message.channels = {"default"}; diff --git a/docs/CBOR.md b/docs/CBOR.md index 9b9d3167b..ccee96418 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -120,7 +120,7 @@ There are minor differences at the moment: | - xfel_pulse_id | uint64 | Pulse IDs are recorded for images | | | - ring_current_mA | float | Ring current at the start of the measurement | | | - sample_temperature_K | float | Sample temperature \[K\] | | -| - detect_ice_rings | bool | Ice ring detection feature is enabled | | +| - detect_ice_rings | bool | Ice ring detection feature is enabled; absent where the dataset settings left it unset (rugnux then decides) | | | - indexing_algorithm | string | Indexing algorithm used on-the-fly; allowed values: ffbidx, fft, fftw, none | | | - geom_refinement_algorithm | string | Post-indexing detector geometry refinement algorithm; allowed values: none, beam_center | | | - poni_rot1 | float | Tilt of the detector rot1 according to PyFAI PONI convention \[rad\] | | From d2e4e4d8789edf275f817539c86b1a207d52b445 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 12 Nov 2026 19:22:45 +0100 Subject: [PATCH 3/4] Logger: the version line once per service name Every Logger construction printed the version line, and a rugnux run makes a logger per FFT indexer of its first-pass ladder: about 180 version lines in a 20 s live run. The line now goes out the first time a service name is seen. The set is a function-local static, as jfjoch_writer's logger is a global made before this file's own namespace-scope statics. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- common/Logger.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/Logger.cpp b/common/Logger.cpp index e48564d7a..cccee71f4 100644 --- a/common/Logger.cpp +++ b/common/Logger.cpp @@ -7,6 +7,7 @@ #include "spdlog/fmt/bin_to_hex.h" #include +#include #include "Logger.h" #include "GitInfo.h" @@ -64,6 +65,15 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) { spdlog_logger = std::make_shared(service_name, std::begin(sinks), std::end(sinks)); if (const auto it = service_levels.find(service_name); it != service_levels.end()) spdlog_logger->set_level(it->second); + { + // The version line goes out once per service: a run makes many loggers of one service (every FFT indexer + // is one). Function-local, as a static Logger (jfjoch_writer's) is made before this file's own statics. + static std::mutex version_mutex; + static std::set version_logged; + std::unique_lock ul(version_mutex); + if (!version_logged.insert(service_name).second) + return; + } if (jfjoch_git_sha1().empty()) spdlog_logger->info("Version {} {}", jfjoch_version(), variant); else From 8d4e64703b3e9183627b950ae5bd26b7a7c99d57 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Thu, 12 Nov 2026 19:22:45 +0100 Subject: [PATCH 4/4] receiver: the run's lattice takes the place of the probe's answer The lattice probe's "no lattice on the first 60 deg" stayed on the dashboard while the main rotation run indexed the sweep. When the run reports its lattice (the first pass's "lattice ..." note), the probe banner becomes "lattice on the sweep: ..." and a probe answer that comes later is not shown. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- broker/gen/model/Rugnux_live_status.h | 4 ++-- broker/jfjoch_api.yaml | 4 ++-- receiver/JFJochReceiverRugnuxRotation.cpp | 7 ++++++- receiver/RugnuxLiveStatus.cpp | 11 +++++++++++ receiver/RugnuxLiveStatus.h | 3 +++ tests/RugnuxLiveStatusTest.cpp | 19 +++++++++++++++++++ 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/broker/gen/model/Rugnux_live_status.h b/broker/gen/model/Rugnux_live_status.h index 1113d4d96..06f4edded 100644 --- a/broker/gen/model/Rugnux_live_status.h +++ b/broker/gen/model/Rugnux_live_status.h @@ -162,12 +162,12 @@ public: bool isaIsSet() const; void unsetIsa(); /// - /// Answer of the early lattice probe on the first 60 degrees; empty until it answers + /// Answer of the early lattice probe on the first 60 degrees; empty until it answers, and the run's own lattice once the run has indexed /// std::string getProbe() const; void setProbe(std::string const& value); /// - /// When the probe answered, since the first image + /// When the probe answered (or the run indexed), since the first image /// double getProbeS() const; void setProbeS(double const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 788c23fd3..2a2237417 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -3348,11 +3348,11 @@ components: description: ISa of the error model; absent until the run is over, or where it was not determined probe: type: string - description: Answer of the early lattice probe on the first 60 degrees; empty until it answers + description: Answer of the early lattice probe on the first 60 degrees; empty until it answers, and the run's own lattice once the run has indexed probe_s: type: number format: double - description: When the probe answered, since the first image + description: When the probe answered (or the run indexed), since the first image start_to_first_image_s: type: number format: double diff --git a/receiver/JFJochReceiverRugnuxRotation.cpp b/receiver/JFJochReceiverRugnuxRotation.cpp index c6e00a3ea..b7d40a13a 100644 --- a/receiver/JFJochReceiverRugnuxRotation.cpp +++ b/receiver/JFJochReceiverRugnuxRotation.cpp @@ -74,7 +74,12 @@ namespace { } void OnProgress(uint64_t done, uint64_t total) override { status.Progress(done, total); } void OnStep(const std::string &title) override { status.Step(title); } - void OnNote(const std::string &text) override { status.Note(text); } + void OnNote(const std::string &text) override { + status.Note(text); + // The run has its lattice (RugnuxFirstPassAccept): the probe's answer on the first degrees is overtaken + if (text.starts_with("lattice ")) + status.Lattice("lattice on the sweep: " + text.substr(8)); + } }; std::string StateName(RugnuxLiveState state) { diff --git a/receiver/RugnuxLiveStatus.cpp b/receiver/RugnuxLiveStatus.cpp index f94d69e14..7304a1149 100644 --- a/receiver/RugnuxLiveStatus.cpp +++ b/receiver/RugnuxLiveStatus.cpp @@ -86,6 +86,7 @@ void RugnuxLiveStatus::Start(int64_t images_expected, std::optional expe snapshot.expected_collection_s = expected_collection_s; start = start_received; last_image.reset(); + lattice = false; } void RugnuxLiveStatus::ImageReceived(std::chrono::steady_clock::time_point received) { @@ -226,10 +227,19 @@ void RugnuxLiveStatus::Cut(const std::string &reason) { void RugnuxLiveStatus::Probe(const std::string &text) { std::unique_lock ul(status_mutex); + if (lattice) + return; snapshot.probe = text; snapshot.probe_s = Now(); } +void RugnuxLiveStatus::Lattice(const std::string &text) { + std::unique_lock ul(status_mutex); + snapshot.probe = text; + snapshot.probe_s = Now(); + lattice = true; +} + void RugnuxLiveStatus::Warning(const std::string &text) { std::unique_lock ul(status_mutex); snapshot.warnings.push_back(text); @@ -246,6 +256,7 @@ void RugnuxLiveStatus::Clear() { std::unique_lock ul(status_mutex); snapshot = RugnuxLiveSnapshot{}; last_image.reset(); + lattice = false; } RugnuxLiveSnapshot RugnuxLiveStatus::Get() const { diff --git a/receiver/RugnuxLiveStatus.h b/receiver/RugnuxLiveStatus.h index 90cee0a16..902de6e90 100644 --- a/receiver/RugnuxLiveStatus.h +++ b/receiver/RugnuxLiveStatus.h @@ -181,6 +181,7 @@ class RugnuxLiveStatus { // Kept since the START message, shifted to the first image by Get(). std::chrono::steady_clock::time_point start; std::optional last_image; + bool lattice = false; // the run has its lattice: the probe's answer is overtaken double Since(std::chrono::steady_clock::time_point t) const; double Now() const; void CloseStep(); @@ -204,7 +205,9 @@ public: // The collection is over: last_image_s is set, and the last image's time is returned (none if no image came). std::optional LastImage(); void Cut(const std::string &reason); + // The probe's answer, unless the run has its lattice already; Lattice: the run's, in the probe's place void Probe(const std::string &text); + void Lattice(const std::string &text); void Warning(const std::string &text); void Structure(std::shared_ptr structure, std::shared_ptr refined_structure = nullptr); diff --git a/tests/RugnuxLiveStatusTest.cpp b/tests/RugnuxLiveStatusTest.cpp index bd3b33483..57a6a1471 100644 --- a/tests/RugnuxLiveStatusTest.cpp +++ b/tests/RugnuxLiveStatusTest.cpp @@ -54,6 +54,25 @@ TEST_CASE("RugnuxLiveStatus_BeforeFirstImage", "[RugnuxLive]") { CHECK(!status.LastImage()); } +TEST_CASE("RugnuxLiveStatus_RunLatticeOvertakesProbe", "[RugnuxLive]") { + RugnuxLiveStatus status; + status.Start(10, 1.0, std::chrono::steady_clock::now()); + + // The probe answers first, the run indexes later: the banner is the run's + status.Probe("no lattice on the first 60 deg"); + CHECK(status.Get().probe == "no lattice on the first 60 deg"); + status.Lattice("lattice on the sweep: monoclinic P: 35.00 29.00 63.00 90.0 105.0 90.0"); + CHECK(status.Get().probe.starts_with("lattice on the sweep")); + // The run indexed before the probe answered: the probe's answer is not shown + status.Probe("no lattice on the first 60 deg"); + CHECK(status.Get().probe.starts_with("lattice on the sweep")); + + // The next collection starts afresh + status.Start(10, 1.0, std::chrono::steady_clock::now()); + status.Probe("no lattice on the first 60 deg"); + CHECK(status.Get().probe == "no lattice on the first 60 deg"); +} + TEST_CASE("RugnuxLiveStatus_Clear", "[RugnuxLive]") { RugnuxLiveStatus status; status.Start(10, 1.0, std::chrono::steady_clock::now() - 2s);