Merge broker-rugnux-live into rc176: rugnux in jfjoch_broker as a live thread during a DECTRIS rotation collection (off by default), the rugnux library without file I/O (results in ProcessResult::outputs, the CLI writes them through JFJochRugnuxOutput), END carrying the merged table, map coefficients and placed model

Branch reviewed at e16db07ff (review: owner's decisions accepted - _maps.mtz instead of .ccp4 in the
viewer, -o "" computes everything incl. MR/validation, Ctrl+C/error paths may leave fewer files, the
writer links gemmi/zlib) and merged at 55184eb09 with its review fixes (END size guard, frame drop as
in Lite, outputs released before re-runs, pool priority, docs, frontend gating).

Conflicts: rugnux/Rugnux.h (kept rc176's host_memory_budget_bytes, output_prefix removed as on the
branch), rugnux/RugnuxPipeline.h (MtzTable future + rc176's GPUMerge pointer), MolecularReplacement.h/.cpp
(rc176's GPU cost model, packed transform and split functions with the branch's RugnuxModel signature;
search_model with rc176's other_budgets, result.model_name). tests/RugnuxLargeTest.cpp: rc176's
host-budget test writes its files through WriteRugnuxOutputs now that the library writes none.
Generated code (broker/gen, frontend client, Python client docs, Redoc) regenerated from the merged
jfjoch_api.yaml with update_version.sh (version unchanged).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

broker/JFJochBrokerHttp.cpp

@@ -16,6 +16,7 @@
 #include "../preview/JFJochTIFF.h"
 #include "OpenAPIConvert.h"
 #include "../image_pusher/LiveCollection.h"
+#include "../writer/RugnuxModelFile.h"
 #include <base64/Base64.h>
 #include "gen/model/Error_message.h"
 
@@ -152,7 +153,10 @@ static const std::set<std::string> PROTECTED_PATHS{
         "/image_buffer/image.tiff",
         "/preview/plot",
         "/preview/plot.bin",
-        "/live/events"
+        "/live/events",
+        "/config/rugnux",
+        "/config/rugnux/model",
+        "/rugnux/status"
 };
 
 std::optional<std::string> JFJochBrokerHttp::AuthorizationHeader(const httplib::Request &req) {
@@ -251,6 +255,11 @@ void JFJochBrokerHttp::register_routes(httplib::Server &server) {
     server.Post("/config/image_format/raw", bind_noarg(&JFJochBrokerHttp::config_image_format_raw_post));
     server.Get("/config/indexing", bind_noarg(&JFJochBrokerHttp::config_indexing_get));
     server.Put("/config/indexing", bind_json(&JFJochBrokerHttp::config_indexing_put, Indexing_settings{}));
+    server.Get("/config/rugnux", bind_noarg(&JFJochBrokerHttp::config_rugnux_get));
+    server.Put("/config/rugnux", bind_json(&JFJochBrokerHttp::config_rugnux_put, Rugnux_settings{}));
+    server.Put("/config/rugnux/model", bind_req(&JFJochBrokerHttp::config_rugnux_model_put));
+    server.Delete("/config/rugnux/model", bind_noarg(&JFJochBrokerHttp::config_rugnux_model_delete));
+    server.Get("/rugnux/status", bind_noarg(&JFJochBrokerHttp::rugnux_status_get));
     server.Get("/config/bragg_integration", bind_noarg(&JFJochBrokerHttp::config_bragg_integration_get));
     server.Put("/config/bragg_integration", bind_json(&JFJochBrokerHttp::config_bragg_integration_put, Bragg_integration_settings{}));
     server.Get("/config/instrument", bind_noarg(&JFJochBrokerHttp::config_instrument_get));
@@ -963,6 +972,60 @@ void JFJochBrokerHttp::config_indexing_put(const Indexing_settings &indexingSett
     response.status = 200;
 }
 
+void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) {
+    const RugnuxLiveSettings s = state_machine.GetRugnuxLive();
+    Rugnux_status ret;
+    ret.setEnable(s.enable);
+    ret.setReservedCpus(s.reserved_cpus);
+    ret.setDeadlineS(s.deadline_s);
+    ret.setModelLoaded(s.model != nullptr);
+    if (s.model)
+        ret.setModelSizeBytes(s.model_size_bytes);
+    ProcessOutput(ret, response);
+}
+
+void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings,
+                                         httplib::Response &response) {
+    state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(),
+                                rugnuxSettings.getDeadlineS());
+    response.status = 200;
+}
+
+void JFJochBrokerHttp::config_rugnux_model_put(const httplib::Request &request,
+                                               httplib::Response &response) {
+    if (request.body.empty()) {
+        response.status = 400;
+        response.set_content("Request body cannot be empty", "text/plain");
+        return;
+    }
+    if (request.body.size() > 64 * 1024 * 1024) {
+        response.status = 413;
+        response.set_content("Model cannot be larger than 64 MiB", "text/plain");
+        return;
+    }
+    std::shared_ptr<const RugnuxModel> model;
+    try {
+        model = std::make_shared<const RugnuxModel>(ParseRugnuxModel(request.body, "model"));
+    } catch (const std::exception &e) {
+        response.status = 400;
+        response.set_content(std::string("Not an atomic model: ") + e.what(), "text/plain");
+        return;
+    }
+    state_machine.SetRugnuxModel(std::move(model), static_cast<int64_t>(request.body.size()));
+    logger.Info("Rugnux model uploaded ({} bytes)", request.body.size());
+    response.status = 200;
+}
+
+void JFJochBrokerHttp::config_rugnux_model_delete(httplib::Response &response) {
+    state_machine.SetRugnuxModel(nullptr, 0);
+    logger.Info("Rugnux model removed");
+    response.status = 200;
+}
+
+void JFJochBrokerHttp::rugnux_status_get(httplib::Response &response) {
+    ProcessOutput(Convert(state_machine.GetRugnuxStatus()), response);
+}
+
 void JFJochBrokerHttp::config_bragg_integration_get(httplib::Response &response) {
     ProcessOutput(Convert(state_machine.GetBraggIntegrationSettings()), response);
 }

broker/JFJochBrokerHttp.h

@@ -34,6 +34,8 @@
 #include "gen/model/Jfjoch_statistics.h"
 #include "gen/model/Plots.h"
 #include "gen/model/Roi_definitions.h"
+#include "gen/model/Rugnux_settings.h"
+#include "gen/model/Rugnux_status.h"
 #include "gen/model/Scan_result.h"
 #include "gen/model/Spot_finding_settings.h"
 #include "gen/model/Zeromq_metadata_settings.h"
@@ -198,6 +200,12 @@ class JFJochBrokerHttp {
     void config_indexing_get(httplib::Response &response);
     void config_indexing_put(const org::openapitools::server::model::Indexing_settings &indexingSettings,
                              httplib::Response &response);
+    void config_rugnux_get(httplib::Response &response);
+    void config_rugnux_put(const org::openapitools::server::model::Rugnux_settings &rugnuxSettings,
+                           httplib::Response &response);
+    void config_rugnux_model_put(const httplib::Request &request, httplib::Response &response);
+    void config_rugnux_model_delete(httplib::Response &response);
+    void rugnux_status_get(httplib::Response &response);
     void config_bragg_integration_get(httplib::Response &response);
     void config_bragg_integration_put(const org::openapitools::server::model::Bragg_integration_settings &braggIntegrationSettings,
                                        httplib::Response &response);

broker/JFJochBrokerParser.cpp

@@ -276,6 +276,15 @@ void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settin
     }
 }
 
+void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochServices &services) {
+    if (input.rugnuxIsSet())
+        services.SetRugnuxLive(RugnuxLiveSettings{
+            .enable = input.getRugnux().isEnable(),
+            .reserved_cpus = input.getRugnux().getReservedCpus(),
+            .deadline_s = input.getRugnux().getDeadlineS()
+        });
+}
+
 SpotFindingSettings ParseSpotFindingSettings(const org::openapitools::server::model::Jfjoch_settings &input) {
     if (input.spotFindingIsSet())
         return Convert(input.getSpotFinding());

broker/JFJochBrokerParser.h

@@ -7,6 +7,7 @@
 #include "../acquisition_device/AcquisitionDeviceGroup.h"
 #include "../image_pusher/ImagePusher.h"
 #include "../receiver/JFJochReceiverService.h"
+#include "JFJochServices.h"
 #include "gen/model/Jfjoch_settings.h"
 
 DetectorGeometryModular ParseStandardDetectorGeometry(const org::openapitools::server::model::Detector &j);
@@ -20,4 +21,5 @@ std::unique_ptr<ImagePusher> ParseImagePusher(const org::openapitools::server::m
 
 void ParseAcquisitionDeviceGroup(const org::openapitools::server::model::Jfjoch_settings &input, AcquisitionDeviceGroup &aq_devices);
 void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochReceiverService &service);
+void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochServices &services);
 SpotFindingSettings ParseSpotFindingSettings(const org::openapitools::server::model::Jfjoch_settings &input);

broker/JFJochServices.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 #include "JFJochServices.h"
+
 #include "../common/JFJochException.h"
 #include "../detector_control/SLSDetectorWrapper.h"
 #include "../detector_control/DectrisDetectorWrapper.h"
@@ -370,6 +371,29 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) {
         receiver->Indexing(input);
 }
 
+void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) {
+    if (receiver)
+        receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s);
+    rugnux = input;
+}
+
+RugnuxLiveSettings JFJochServices::GetRugnuxLive() const {
+    return rugnux;
+}
+
+RugnuxLiveSnapshot JFJochServices::GetRugnuxStatus() const {
+    if (receiver)
+        return receiver->GetRugnuxStatus();
+    return {};
+}
+
+void JFJochServices::SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes) {
+    RugnuxLiveSettings s = rugnux;
+    s.model = std::move(model);
+    s.model_size_bytes = s.model ? size_bytes : 0;
+    SetRugnuxLive(s);
+}
+
 LiveCollection *JFJochServices::GetLiveCollection() {
     return receiver ? receiver->GetLiveCollection() : nullptr;
 }

broker/JFJochServices.h

@@ -14,6 +14,14 @@ struct JFJochServicesOutput {
     JFJochReceiverOutput receiver_output;
 };
 
+struct RugnuxLiveSettings {
+    bool enable = false;
+    int64_t reserved_cpus = 4;
+    double deadline_s = 0;   // the analysis is cut this long after the last image (0 = never)
+    std::shared_ptr<const RugnuxModel> model;   // null = no model
+    int64_t model_size_bytes = 0;                // as uploaded
+};
+
 class JFJochServices {
     mutable std::shared_timed_mutex detector_mutex;
 
@@ -22,6 +30,8 @@ class JFJochServices {
     std::shared_ptr<ZMQImagePuller> image_puller;
 
     std::atomic<bool> cannot_stop_detector = false;
+
+    RugnuxLiveSettings rugnux;
     Logger &logger;
 public:
     explicit JFJochServices(Logger &in_logger);
@@ -76,6 +86,12 @@ public:
 
     void SetupIndexing(const IndexingSettings& input);
 
+    void SetRugnuxLive(const RugnuxLiveSettings &input);
+    RugnuxLiveSettings GetRugnuxLive() const;
+    // The model is kept in memory, parsed, for every following collection; nullptr removes it
+    void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes);
+    RugnuxLiveSnapshot GetRugnuxStatus() const;
+
     ImagePusherStatus GetImagePusherStatus() const;
     LiveCollection *GetLiveCollection();
 };

broker/JFJochStateMachine.cpp

@@ -556,6 +556,10 @@ void JFJochStateMachine::MeasurementThread() {
                 SetState(JFJochState::Idle,
                          "Stream receiver (writer or downstream analysis) cannot cope with data; reduce frame rate",
                          BrokerStatus::MessageSeverity::Warning);
+            else if (!tmp_output.receiver_output.analysis_warning.empty())
+                SetState(JFJochState::Idle,
+                         "Data collection without problems, but " + tmp_output.receiver_output.analysis_warning,
+                         BrokerStatus::MessageSeverity::Warning);
             else
                 SetState(JFJochState::Idle,
                          "Data collection without problems",
@@ -1209,6 +1213,35 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const {
     return experiment.GetIndexingSettings();
 }
 
+void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s) {
+    std::unique_lock ul(m);
+    if (IsRunning())
+        throw WrongDAQStateException("Cannot change rugnux settings during data collection");
+    std::unique_lock ul2(rugnux_mutex);
+    RugnuxLiveSettings s = services.GetRugnuxLive();
+    s.enable = enable;
+    s.reserved_cpus = reserved_cpus;
+    s.deadline_s = deadline_s;
+    services.SetRugnuxLive(s);
+}
+
+void JFJochStateMachine::SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes) {
+    std::unique_lock ul(m);
+    if (IsRunning())
+        throw WrongDAQStateException("Cannot change rugnux model during data collection");
+    std::unique_lock ul2(rugnux_mutex);
+    services.SetRugnuxModel(std::move(model), size_bytes);
+}
+
+RugnuxLiveSettings JFJochStateMachine::GetRugnuxLive() const {
+    std::unique_lock ul(rugnux_mutex);
+    return services.GetRugnuxLive();
+}
+
+RugnuxLiveSnapshot JFJochStateMachine::GetRugnuxStatus() const {
+    return services.GetRugnuxStatus();
+}
+
 void JFJochStateMachine::SetIndexingSettings(const IndexingSettings &input) {
     std::unique_lock ul(m);
     if (IsRunning())

broker/JFJochStateMachine.h

@@ -90,6 +90,7 @@ class JFJochStateMachine {
     mutable std::mutex experiment_file_writer_settings_mutex;
     mutable std::mutex experiment_indexing_settings_mutex;
     mutable std::mutex experiment_dark_mask_settings_mutex;
+    mutable std::mutex rugnux_mutex;
     DiffractionExperiment experiment;
 
     // mutex m is protecting:
@@ -247,6 +248,11 @@ public:
     void SetIndexingSettings(const IndexingSettings &input);
     IndexingSettings GetIndexingSettings() const;
 
+    void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s);
+    void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes);   // null = remove the model
+    RugnuxLiveSettings GetRugnuxLive() const;
+    RugnuxLiveSnapshot GetRugnuxStatus() const;
+
     void SetBraggIntegrationSettings(const BraggIntegrationSettings &input);
     BraggIntegrationSettings GetBraggIntegrationSettings() const;
     PixelMaskStatistics GetPixelMaskStatistics() const;

broker/OpenAPIConvert.cpp

@@ -1221,6 +1221,60 @@ DarkMaskSettings Convert(const org::openapitools::server::model::Dark_mask_setti
     return ret;
 }
 
+org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSnapshot& input) {
+    org::openapitools::server::model::Rugnux_live_status ret;
+    switch (input.state) {
+        case RugnuxLiveState::Idle: ret.setState("Idle"); break;
+        case RugnuxLiveState::Running: ret.setState("Running"); break;
+        case RugnuxLiveState::Done: ret.setState("Done"); break;
+        case RugnuxLiveState::Failed: ret.setState("Failed"); break;
+        case RugnuxLiveState::Cancelled: ret.setState("Cancelled"); break;
+        case RugnuxLiveState::Skipped: ret.setState("Skipped"); break;
+    }
+    ret.setMessage(input.message);
+    ret.setImagesExpected(input.images_expected);
+    ret.setImagesReceived(input.images_received);
+    ret.setElapsedS(input.elapsed_s);
+    ret.setProbe(input.probe);
+    if (input.probe_s)
+        ret.setProbeS(input.probe_s.value());
+    if (input.step_progress)
+        ret.setStepProgress(input.step_progress.value());
+
+    std::vector<org::openapitools::server::model::Rugnux_live_step> steps;
+    for (const auto &s: input.steps) {
+        org::openapitools::server::model::Rugnux_live_step step;
+        step.setTitle(s.title);
+        step.setStartedS(s.started_s);
+        if (s.duration_s)
+            step.setDurationS(s.duration_s.value());
+        step.setNotes(s.notes);
+        steps.push_back(step);
+    }
+    ret.setSteps(steps);
+
+    if (input.last_image_s)
+        ret.setLastImageS(input.last_image_s.value());
+    ret.setCut(input.cut);
+    if (input.cut_s)
+        ret.setCutS(input.cut_s.value());
+    if (input.last_image_to_written_s)
+        ret.setLastImageToWrittenS(input.last_image_to_written_s.value());
+    ret.setVerdict(input.verdict);
+    ret.setVerdictText(input.verdict_text);
+
+    std::vector<org::openapitools::server::model::Rugnux_summary_row> summary;
+    for (const auto &[label, value]: input.summary) {
+        org::openapitools::server::model::Rugnux_summary_row row;
+        row.setLabel(label);
+        row.setValue(value);
+        summary.push_back(row);
+    }
+    ret.setSummary(summary);
+    ret.setWarnings(input.warnings);
+    return ret;
+}
+
 org::openapitools::server::model::Image_pusher_status Convert(const ImagePusherStatus& input) {
     org::openapitools::server::model::Image_pusher_status ret;
     ret.setAddr(input.address);

broker/OpenAPIConvert.h

@@ -30,6 +30,7 @@
 #include "gen/model/Indexing_settings.h"
 #include "gen/model/Bragg_integration_settings.h"
 #include "gen/model/Scan_result.h"
+#include "gen/model/Rugnux_live_status.h"
 
 #include "../common/JFJochMessages.h"
 #include "../common/DatasetSettings.h"
@@ -41,6 +42,7 @@
 #include "../common/InstrumentMetadata.h"
 #include "../common/ScanResult.h"
 #include "../image_pusher/ImagePusher.h"
+#include "../receiver/RugnuxLiveStatus.h"
 
 SpotFindingSettings Convert(const org::openapitools::server::model::Spot_finding_settings &input);
 org::openapitools::server::model::Spot_finding_settings Convert(const SpotFindingSettings &input);
@@ -99,4 +101,5 @@ org::openapitools::server::model::Dark_mask_settings Convert(const DarkMaskSetti
 DarkMaskSettings Convert(const org::openapitools::server::model::Dark_mask_settings& input);
 
 org::openapitools::server::model::Image_pusher_status Convert(const ImagePusherStatus& input);
+org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSnapshot& input);
 

broker/gen/model/Jfjoch_settings.cpp

@@ -44,6 +44,7 @@ Jfjoch_settings::Jfjoch_settings()
     m_Zeromq_previewIsSet = false;
     m_Zeromq_metadataIsSet = false;
     m_Dark_maskIsSet = false;
+    m_RugnuxIsSet = false;
     
 }
 
@@ -141,7 +142,7 @@ bool Jfjoch_settings::validate(std::stringstream& msg, const std::string& pathPr
         }
 
     }
-                                
+                                    
     return success;
 }
 
@@ -211,7 +212,10 @@ bool Jfjoch_settings::operator==(const Jfjoch_settings& rhs) const
     ((!zeromqMetadataIsSet() && !rhs.zeromqMetadataIsSet()) || (zeromqMetadataIsSet() && rhs.zeromqMetadataIsSet() && getZeromqMetadata() == rhs.getZeromqMetadata())) &&
     
     
-    ((!darkMaskIsSet() && !rhs.darkMaskIsSet()) || (darkMaskIsSet() && rhs.darkMaskIsSet() && getDarkMask() == rhs.getDarkMask()))
+    ((!darkMaskIsSet() && !rhs.darkMaskIsSet()) || (darkMaskIsSet() && rhs.darkMaskIsSet() && getDarkMask() == rhs.getDarkMask())) &&
+    
+    
+    ((!rugnuxIsSet() && !rhs.rugnuxIsSet()) || (rugnuxIsSet() && rhs.rugnuxIsSet() && getRugnux() == rhs.getRugnux()))
     
     ;
 }
@@ -263,6 +267,8 @@ void to_json(nlohmann::json& j, const Jfjoch_settings& o)
         j["zeromq_metadata"] = o.m_Zeromq_metadata;
     if(o.darkMaskIsSet())
         j["dark_mask"] = o.m_Dark_mask;
+    if(o.rugnuxIsSet())
+        j["rugnux"] = o.m_Rugnux;
     
 }
 
@@ -361,6 +367,11 @@ void from_json(const nlohmann::json& j, Jfjoch_settings& o)
         j.at("dark_mask").get_to(o.m_Dark_mask);
         o.m_Dark_maskIsSet = true;
     } 
+    if(j.find("rugnux") != j.end())
+    {
+        j.at("rugnux").get_to(o.m_Rugnux);
+        o.m_RugnuxIsSet = true;
+    } 
     
 }
 
@@ -694,6 +705,23 @@ void Jfjoch_settings::unsetDark_mask()
 {
     m_Dark_maskIsSet = false;
 }
+org::openapitools::server::model::Rugnux_settings Jfjoch_settings::getRugnux() const
+{
+    return m_Rugnux;
+}
+void Jfjoch_settings::setRugnux(org::openapitools::server::model::Rugnux_settings const& value)
+{
+    m_Rugnux = value;
+    m_RugnuxIsSet = true;
+}
+bool Jfjoch_settings::rugnuxIsSet() const
+{
+    return m_RugnuxIsSet;
+}
+void Jfjoch_settings::unsetRugnux()
+{
+    m_RugnuxIsSet = false;
+}
 
 
 } // namespace org::openapitools::server::model

broker/gen/model/Jfjoch_settings.h

@@ -31,6 +31,7 @@
 #include "Image_format_settings.h"
 #include "Zeromq_metadata_settings.h"
 #include "Detector_settings.h"
+#include "Rugnux_settings.h"
 #include <vector>
 #include "Indexing_settings.h"
 #include "Detector.h"
@@ -216,6 +217,13 @@ public:
     void setDarkMask(org::openapitools::server::model::Dark_mask_settings const& value);
     bool darkMaskIsSet() const;
     void unsetDark_mask();
+    /// <summary>
+    /// 
+    /// </summary>
+    org::openapitools::server::model::Rugnux_settings getRugnux() const;
+    void setRugnux(org::openapitools::server::model::Rugnux_settings const& value);
+    bool rugnuxIsSet() const;
+    void unsetRugnux();
 
     friend  void to_json(nlohmann::json& j, const Jfjoch_settings& o);
     friend  void from_json(const nlohmann::json& j, Jfjoch_settings& o);
@@ -262,6 +270,8 @@ protected:
     bool m_Zeromq_metadataIsSet;
     org::openapitools::server::model::Dark_mask_settings m_Dark_mask;
     bool m_Dark_maskIsSet;
+    org::openapitools::server::model::Rugnux_settings m_Rugnux;
+    bool m_RugnuxIsSet;
     
 };
 

broker/gen/model/Rugnux_live_status.cpp

@@ -0,0 +1,484 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+
+
+#include "Rugnux_live_status.h"
+#include "Helpers.h"
+
+#include <sstream>
+
+namespace org::openapitools::server::model
+{
+
+Rugnux_live_status::Rugnux_live_status()
+{
+    m_State = "";
+    m_Probe = "";
+    m_Probe_s = 0.0;
+    m_Probe_sIsSet = false;
+    m_Message = "";
+    m_Images_expected = 0L;
+    m_Images_received = 0L;
+    m_Elapsed_s = 0.0;
+    m_Step_progress = 0.0f;
+    m_Step_progressIsSet = false;
+    m_Last_image_s = 0.0;
+    m_Last_image_sIsSet = false;
+    m_Cut = "";
+    m_CutIsSet = false;
+    m_Cut_s = 0.0;
+    m_Cut_sIsSet = false;
+    m_Last_image_to_written_s = 0.0;
+    m_Last_image_to_written_sIsSet = false;
+    m_Verdict = "";
+    m_Verdict_text = "";
+    
+}
+
+void Rugnux_live_status::validate() const
+{
+    std::stringstream msg;
+    if (!validate(msg))
+    {
+        throw org::openapitools::server::helpers::ValidationException(msg.str());
+    }
+}
+
+bool Rugnux_live_status::validate(std::stringstream& msg) const
+{
+    return validate(msg, "");
+}
+
+bool Rugnux_live_status::validate(std::stringstream& msg, const std::string& pathPrefix) const
+{
+    bool success = true;
+    const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_status" : pathPrefix;
+
+                                     
+    if (stepProgressIsSet())
+    {
+        const float& value = m_Step_progress;
+        const std::string currentValuePath = _pathPrefix + ".stepProgress";
+                
+        
+        if (value < static_cast<float>(0.0))
+        {
+            success = false;
+            msg << currentValuePath << ": must be greater than or equal to 0.0;";
+        }
+        if (value > static_cast<float>(1.0))
+        {
+            success = false;
+            msg << currentValuePath << ": must be less than or equal to 1.0;";
+        }
+
+    }
+         
+    
+    /* Steps */ {
+        const std::vector<org::openapitools::server::model::Rugnux_live_step>& value = m_Steps;
+        const std::string currentValuePath = _pathPrefix + ".steps";
+                
+        
+        { // Recursive validation of array elements
+            const std::string oldValuePath = currentValuePath;
+            int i = 0;
+            for (const org::openapitools::server::model::Rugnux_live_step& value : value)
+            { 
+                const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]";
+                        
+        success = value.validate(msg, currentValuePath + ".steps") && success;
+ 
+                i++;
+            }
+        }
+
+    }
+                                 
+    
+    /* Summary */ {
+        const std::vector<org::openapitools::server::model::Rugnux_summary_row>& value = m_Summary;
+        const std::string currentValuePath = _pathPrefix + ".summary";
+                
+        
+        { // Recursive validation of array elements
+            const std::string oldValuePath = currentValuePath;
+            int i = 0;
+            for (const org::openapitools::server::model::Rugnux_summary_row& value : value)
+            { 
+                const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]";
+                        
+        success = value.validate(msg, currentValuePath + ".summary") && success;
+ 
+                i++;
+            }
+        }
+
+    }
+         
+    
+    /* Warnings */ {
+        const std::vector<std::string>& value = m_Warnings;
+        const std::string currentValuePath = _pathPrefix + ".warnings";
+                
+        
+        { // Recursive validation of array elements
+            const std::string oldValuePath = currentValuePath;
+            int i = 0;
+            for (const std::string& value : value)
+            { 
+                const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]";
+                        
+        
+ 
+                i++;
+            }
+        }
+
+    }
+    
+    return success;
+}
+
+bool Rugnux_live_status::operator==(const Rugnux_live_status& rhs) const
+{
+    return
+    
+    
+    (getState() == rhs.getState())
+     &&
+    
+    (getProbe() == rhs.getProbe())
+     &&
+    
+    
+    ((!probeSIsSet() && !rhs.probeSIsSet()) || (probeSIsSet() && rhs.probeSIsSet() && getProbeS() == rhs.getProbeS())) &&
+    
+    (getMessage() == rhs.getMessage())
+     &&
+    
+    (getImagesExpected() == rhs.getImagesExpected())
+     &&
+    
+    (getImagesReceived() == rhs.getImagesReceived())
+     &&
+    
+    (getElapsedS() == rhs.getElapsedS())
+     &&
+    
+    
+    ((!stepProgressIsSet() && !rhs.stepProgressIsSet()) || (stepProgressIsSet() && rhs.stepProgressIsSet() && getStepProgress() == rhs.getStepProgress())) &&
+    
+    (getSteps() == rhs.getSteps())
+     &&
+    
+    
+    ((!lastImageSIsSet() && !rhs.lastImageSIsSet()) || (lastImageSIsSet() && rhs.lastImageSIsSet() && getLastImageS() == rhs.getLastImageS())) &&
+    
+    
+    ((!cutIsSet() && !rhs.cutIsSet()) || (cutIsSet() && rhs.cutIsSet() && getCut() == rhs.getCut())) &&
+    
+    
+    ((!cutSIsSet() && !rhs.cutSIsSet()) || (cutSIsSet() && rhs.cutSIsSet() && getCutS() == rhs.getCutS())) &&
+    
+    
+    ((!lastImageToWrittenSIsSet() && !rhs.lastImageToWrittenSIsSet()) || (lastImageToWrittenSIsSet() && rhs.lastImageToWrittenSIsSet() && getLastImageToWrittenS() == rhs.getLastImageToWrittenS())) &&
+    
+    (getVerdict() == rhs.getVerdict())
+     &&
+    
+    (getVerdictText() == rhs.getVerdictText())
+     &&
+    
+    (getSummary() == rhs.getSummary())
+     &&
+    
+    (getWarnings() == rhs.getWarnings())
+    
+    
+    ;
+}
+
+bool Rugnux_live_status::operator!=(const Rugnux_live_status& rhs) const
+{
+    return !(*this == rhs);
+}
+
+void to_json(nlohmann::json& j, const Rugnux_live_status& o)
+{
+    j = nlohmann::json::object();
+    j["state"] = o.m_State;
+    j["probe"] = o.m_Probe;
+    if(o.probeSIsSet())
+        j["probe_s"] = o.m_Probe_s;
+    j["message"] = o.m_Message;
+    j["images_expected"] = o.m_Images_expected;
+    j["images_received"] = o.m_Images_received;
+    j["elapsed_s"] = o.m_Elapsed_s;
+    if(o.stepProgressIsSet())
+        j["step_progress"] = o.m_Step_progress;
+    j["steps"] = o.m_Steps;
+    if(o.lastImageSIsSet())
+        j["last_image_s"] = o.m_Last_image_s;
+    if(o.cutIsSet())
+        j["cut"] = o.m_Cut;
+    if(o.cutSIsSet())
+        j["cut_s"] = o.m_Cut_s;
+    if(o.lastImageToWrittenSIsSet())
+        j["last_image_to_written_s"] = o.m_Last_image_to_written_s;
+    j["verdict"] = o.m_Verdict;
+    j["verdict_text"] = o.m_Verdict_text;
+    j["summary"] = o.m_Summary;
+    j["warnings"] = o.m_Warnings;
+    
+}
+
+void from_json(const nlohmann::json& j, Rugnux_live_status& o)
+{
+    j.at("state").get_to(o.m_State);
+    j.at("probe").get_to(o.m_Probe);
+    if(j.find("probe_s") != j.end())
+    {
+        j.at("probe_s").get_to(o.m_Probe_s);
+        o.m_Probe_sIsSet = true;
+    } 
+    j.at("message").get_to(o.m_Message);
+    j.at("images_expected").get_to(o.m_Images_expected);
+    j.at("images_received").get_to(o.m_Images_received);
+    j.at("elapsed_s").get_to(o.m_Elapsed_s);
+    if(j.find("step_progress") != j.end())
+    {
+        j.at("step_progress").get_to(o.m_Step_progress);
+        o.m_Step_progressIsSet = true;
+    } 
+    j.at("steps").get_to(o.m_Steps);
+    if(j.find("last_image_s") != j.end())
+    {
+        j.at("last_image_s").get_to(o.m_Last_image_s);
+        o.m_Last_image_sIsSet = true;
+    } 
+    if(j.find("cut") != j.end())
+    {
+        j.at("cut").get_to(o.m_Cut);
+        o.m_CutIsSet = true;
+    } 
+    if(j.find("cut_s") != j.end())
+    {
+        j.at("cut_s").get_to(o.m_Cut_s);
+        o.m_Cut_sIsSet = true;
+    } 
+    if(j.find("last_image_to_written_s") != j.end())
+    {
+        j.at("last_image_to_written_s").get_to(o.m_Last_image_to_written_s);
+        o.m_Last_image_to_written_sIsSet = true;
+    } 
+    j.at("verdict").get_to(o.m_Verdict);
+    j.at("verdict_text").get_to(o.m_Verdict_text);
+    j.at("summary").get_to(o.m_Summary);
+    j.at("warnings").get_to(o.m_Warnings);
+    
+}
+
+std::string Rugnux_live_status::getState() const
+{
+    return m_State;
+}
+void Rugnux_live_status::setState(std::string const& value)
+{
+    m_State = value;
+}
+std::string Rugnux_live_status::getProbe() const
+{
+    return m_Probe;
+}
+void Rugnux_live_status::setProbe(std::string const& value)
+{
+    m_Probe = value;
+}
+double Rugnux_live_status::getProbeS() const
+{
+    return m_Probe_s;
+}
+void Rugnux_live_status::setProbeS(double const value)
+{
+    m_Probe_s = value;
+    m_Probe_sIsSet = true;
+}
+bool Rugnux_live_status::probeSIsSet() const
+{
+    return m_Probe_sIsSet;
+}
+void Rugnux_live_status::unsetProbe_s()
+{
+    m_Probe_sIsSet = false;
+}
+std::string Rugnux_live_status::getMessage() const
+{
+    return m_Message;
+}
+void Rugnux_live_status::setMessage(std::string const& value)
+{
+    m_Message = value;
+}
+int64_t Rugnux_live_status::getImagesExpected() const
+{
+    return m_Images_expected;
+}
+void Rugnux_live_status::setImagesExpected(int64_t const value)
+{
+    m_Images_expected = value;
+}
+int64_t Rugnux_live_status::getImagesReceived() const
+{
+    return m_Images_received;
+}
+void Rugnux_live_status::setImagesReceived(int64_t const value)
+{
+    m_Images_received = value;
+}
+double Rugnux_live_status::getElapsedS() const
+{
+    return m_Elapsed_s;
+}
+void Rugnux_live_status::setElapsedS(double const value)
+{
+    m_Elapsed_s = value;
+}
+float Rugnux_live_status::getStepProgress() const
+{
+    return m_Step_progress;
+}
+void Rugnux_live_status::setStepProgress(float const value)
+{
+    m_Step_progress = value;
+    m_Step_progressIsSet = true;
+}
+bool Rugnux_live_status::stepProgressIsSet() const
+{
+    return m_Step_progressIsSet;
+}
+void Rugnux_live_status::unsetStep_progress()
+{
+    m_Step_progressIsSet = false;
+}
+std::vector<org::openapitools::server::model::Rugnux_live_step> Rugnux_live_status::getSteps() const
+{
+    return m_Steps;
+}
+void Rugnux_live_status::setSteps(std::vector<org::openapitools::server::model::Rugnux_live_step> const& value)
+{
+    m_Steps = value;
+}
+double Rugnux_live_status::getLastImageS() const
+{
+    return m_Last_image_s;
+}
+void Rugnux_live_status::setLastImageS(double const value)
+{
+    m_Last_image_s = value;
+    m_Last_image_sIsSet = true;
+}
+bool Rugnux_live_status::lastImageSIsSet() const
+{
+    return m_Last_image_sIsSet;
+}
+void Rugnux_live_status::unsetLast_image_s()
+{
+    m_Last_image_sIsSet = false;
+}
+std::string Rugnux_live_status::getCut() const
+{
+    return m_Cut;
+}
+void Rugnux_live_status::setCut(std::string const& value)
+{
+    m_Cut = value;
+    m_CutIsSet = true;
+}
+bool Rugnux_live_status::cutIsSet() const
+{
+    return m_CutIsSet;
+}
+void Rugnux_live_status::unsetCut()
+{
+    m_CutIsSet = false;
+}
+double Rugnux_live_status::getCutS() const
+{
+    return m_Cut_s;
+}
+void Rugnux_live_status::setCutS(double const value)
+{
+    m_Cut_s = value;
+    m_Cut_sIsSet = true;
+}
+bool Rugnux_live_status::cutSIsSet() const
+{
+    return m_Cut_sIsSet;
+}
+void Rugnux_live_status::unsetCut_s()
+{
+    m_Cut_sIsSet = false;
+}
+double Rugnux_live_status::getLastImageToWrittenS() const
+{
+    return m_Last_image_to_written_s;
+}
+void Rugnux_live_status::setLastImageToWrittenS(double const value)
+{
+    m_Last_image_to_written_s = value;
+    m_Last_image_to_written_sIsSet = true;
+}
+bool Rugnux_live_status::lastImageToWrittenSIsSet() const
+{
+    return m_Last_image_to_written_sIsSet;
+}
+void Rugnux_live_status::unsetLast_image_to_written_s()
+{
+    m_Last_image_to_written_sIsSet = false;
+}
+std::string Rugnux_live_status::getVerdict() const
+{
+    return m_Verdict;
+}
+void Rugnux_live_status::setVerdict(std::string const& value)
+{
+    m_Verdict = value;
+}
+std::string Rugnux_live_status::getVerdictText() const
+{
+    return m_Verdict_text;
+}
+void Rugnux_live_status::setVerdictText(std::string const& value)
+{
+    m_Verdict_text = value;
+}
+std::vector<org::openapitools::server::model::Rugnux_summary_row> Rugnux_live_status::getSummary() const
+{
+    return m_Summary;
+}
+void Rugnux_live_status::setSummary(std::vector<org::openapitools::server::model::Rugnux_summary_row> const& value)
+{
+    m_Summary = value;
+}
+std::vector<std::string> Rugnux_live_status::getWarnings() const
+{
+    return m_Warnings;
+}
+void Rugnux_live_status::setWarnings(std::vector<std::string> const& value)
+{
+    m_Warnings = value;
+}
+
+
+} // namespace org::openapitools::server::model
+

broker/gen/model/Rugnux_live_status.h

@@ -0,0 +1,204 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * Rugnux_live_status.h
+ *
+ * Live rugnux processing of the last collection; kept until the next collection starts
+ */
+
+#ifndef Rugnux_live_status_H_
+#define Rugnux_live_status_H_
+
+
+#include "Rugnux_summary_row.h"
+#include <string>
+#include "Rugnux_live_step.h"
+#include <vector>
+#include <nlohmann/json.hpp>
+
+namespace org::openapitools::server::model
+{
+
+/// <summary>
+/// Live rugnux processing of the last collection; kept until the next collection starts
+/// </summary>
+class  Rugnux_live_status
+{
+public:
+    Rugnux_live_status();
+    virtual ~Rugnux_live_status() = default;
+
+
+    /// <summary>
+    /// Validate the current data in the model. Throws a ValidationException on failure.
+    /// </summary>
+    void validate() const;
+
+    /// <summary>
+    /// Validate the current data in the model. Returns false on error and writes an error
+    /// message into the given stringstream.
+    /// </summary>
+    bool validate(std::stringstream& msg) const;
+
+    /// <summary>
+    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
+    /// Not meant to be called outside that case.
+    /// </summary>
+    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
+
+    bool operator==(const Rugnux_live_status& rhs) const;
+    bool operator!=(const Rugnux_live_status& rhs) const;
+
+    /////////////////////////////////////////////
+    /// Rugnux_live_status members
+
+    /// <summary>
+    /// 
+    /// </summary>
+    std::string getState() const;
+    void setState(std::string const& value);
+    /// <summary>
+    /// Answer of the early lattice probe on the first 60 degrees; empty until it answers
+    /// </summary>
+    std::string getProbe() const;
+    void setProbe(std::string const& value);
+    /// <summary>
+    /// When the probe answered, since the collection started
+    /// </summary>
+    double getProbeS() const;
+    void setProbeS(double const value);
+    bool probeSIsSet() const;
+    void unsetProbe_s();
+    /// <summary>
+    /// Why the run failed or was skipped
+    /// </summary>
+    std::string getMessage() const;
+    void setMessage(std::string const& value);
+    /// <summary>
+    /// 
+    /// </summary>
+    int64_t getImagesExpected() const;
+    void setImagesExpected(int64_t const value);
+    /// <summary>
+    /// 
+    /// </summary>
+    int64_t getImagesReceived() const;
+    void setImagesReceived(int64_t const value);
+    /// <summary>
+    /// 
+    /// </summary>
+    double getElapsedS() const;
+    void setElapsedS(double const value);
+    /// <summary>
+    /// Fraction of the current pass&#39;s images
+    /// </summary>
+    float getStepProgress() const;
+    void setStepProgress(float const value);
+    bool stepProgressIsSet() const;
+    void unsetStep_progress();
+    /// <summary>
+    /// 
+    /// </summary>
+    std::vector<org::openapitools::server::model::Rugnux_live_step> getSteps() const;
+    void setSteps(std::vector<org::openapitools::server::model::Rugnux_live_step> const& value);
+    /// <summary>
+    /// When the last image arrived (and the cutoff timer started), since the collection started
+    /// </summary>
+    double getLastImageS() const;
+    void setLastImageS(double const value);
+    bool lastImageSIsSet() const;
+    void unsetLast_image_s();
+    /// <summary>
+    /// Empty, or why the analysis was cut: &#x60;deadline&#x60; (&#x60;deadline_s&#x60; after the last image) or &#x60;cancel&#x60; (&#x60;POST /cancel&#x60;). The end message then carried what was established at the cut. 
+    /// </summary>
+    std::string getCut() const;
+    void setCut(std::string const& value);
+    bool cutIsSet() const;
+    void unsetCut();
+    /// <summary>
+    /// When the analysis was cut, since the collection started
+    /// </summary>
+    double getCutS() const;
+    void setCutS(double const value);
+    bool cutSIsSet() const;
+    void unsetCut_s();
+    /// <summary>
+    /// From the last image received to all results written
+    /// </summary>
+    double getLastImageToWrittenS() const;
+    void setLastImageToWrittenS(double const value);
+    bool lastImageToWrittenSIsSet() const;
+    void unsetLast_image_to_written_s();
+    /// <summary>
+    /// OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over
+    /// </summary>
+    std::string getVerdict() const;
+    void setVerdict(std::string const& value);
+    /// <summary>
+    /// 
+    /// </summary>
+    std::string getVerdictText() const;
+    void setVerdictText(std::string const& value);
+    /// <summary>
+    /// 
+    /// </summary>
+    std::vector<org::openapitools::server::model::Rugnux_summary_row> getSummary() const;
+    void setSummary(std::vector<org::openapitools::server::model::Rugnux_summary_row> const& value);
+    /// <summary>
+    /// 
+    /// </summary>
+    std::vector<std::string> getWarnings() const;
+    void setWarnings(std::vector<std::string> const& value);
+
+    friend  void to_json(nlohmann::json& j, const Rugnux_live_status& o);
+    friend  void from_json(const nlohmann::json& j, Rugnux_live_status& o);
+protected:
+    std::string m_State;
+
+    std::string m_Probe;
+
+    double m_Probe_s;
+    bool m_Probe_sIsSet;
+    std::string m_Message;
+
+    int64_t m_Images_expected;
+
+    int64_t m_Images_received;
+
+    double m_Elapsed_s;
+
+    float m_Step_progress;
+    bool m_Step_progressIsSet;
+    std::vector<org::openapitools::server::model::Rugnux_live_step> m_Steps;
+
+    double m_Last_image_s;
+    bool m_Last_image_sIsSet;
+    std::string m_Cut;
+    bool m_CutIsSet;
+    double m_Cut_s;
+    bool m_Cut_sIsSet;
+    double m_Last_image_to_written_s;
+    bool m_Last_image_to_written_sIsSet;
+    std::string m_Verdict;
+
+    std::string m_Verdict_text;
+
+    std::vector<org::openapitools::server::model::Rugnux_summary_row> m_Summary;
+
+    std::vector<std::string> m_Warnings;
+
+    
+};
+
+} // namespace org::openapitools::server::model
+
+#endif /* Rugnux_live_status_H_ */

broker/gen/model/Rugnux_live_step.cpp

@@ -0,0 +1,168 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+
+
+#include "Rugnux_live_step.h"
+#include "Helpers.h"
+
+#include <sstream>
+
+namespace org::openapitools::server::model
+{
+
+Rugnux_live_step::Rugnux_live_step()
+{
+    m_Title = "";
+    m_Started_s = 0.0;
+    m_Duration_s = 0.0;
+    m_Duration_sIsSet = false;
+    
+}
+
+void Rugnux_live_step::validate() const
+{
+    std::stringstream msg;
+    if (!validate(msg))
+    {
+        throw org::openapitools::server::helpers::ValidationException(msg.str());
+    }
+}
+
+bool Rugnux_live_step::validate(std::stringstream& msg) const
+{
+    return validate(msg, "");
+}
+
+bool Rugnux_live_step::validate(std::stringstream& msg, const std::string& pathPrefix) const
+{
+    bool success = true;
+    const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_step" : pathPrefix;
+
+                     
+    
+    /* Notes */ {
+        const std::vector<std::string>& value = m_Notes;
+        const std::string currentValuePath = _pathPrefix + ".notes";
+                
+        
+        { // Recursive validation of array elements
+            const std::string oldValuePath = currentValuePath;
+            int i = 0;
+            for (const std::string& value : value)
+            { 
+                const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]";
+                        
+        
+ 
+                i++;
+            }
+        }
+
+    }
+    
+    return success;
+}
+
+bool Rugnux_live_step::operator==(const Rugnux_live_step& rhs) const
+{
+    return
+    
+    
+    (getTitle() == rhs.getTitle())
+     &&
+    
+    (getStartedS() == rhs.getStartedS())
+     &&
+    
+    
+    ((!durationSIsSet() && !rhs.durationSIsSet()) || (durationSIsSet() && rhs.durationSIsSet() && getDurationS() == rhs.getDurationS())) &&
+    
+    (getNotes() == rhs.getNotes())
+    
+    
+    ;
+}
+
+bool Rugnux_live_step::operator!=(const Rugnux_live_step& rhs) const
+{
+    return !(*this == rhs);
+}
+
+void to_json(nlohmann::json& j, const Rugnux_live_step& o)
+{
+    j = nlohmann::json::object();
+    j["title"] = o.m_Title;
+    j["started_s"] = o.m_Started_s;
+    if(o.durationSIsSet())
+        j["duration_s"] = o.m_Duration_s;
+    j["notes"] = o.m_Notes;
+    
+}
+
+void from_json(const nlohmann::json& j, Rugnux_live_step& o)
+{
+    j.at("title").get_to(o.m_Title);
+    j.at("started_s").get_to(o.m_Started_s);
+    if(j.find("duration_s") != j.end())
+    {
+        j.at("duration_s").get_to(o.m_Duration_s);
+        o.m_Duration_sIsSet = true;
+    } 
+    j.at("notes").get_to(o.m_Notes);
+    
+}
+
+std::string Rugnux_live_step::getTitle() const
+{
+    return m_Title;
+}
+void Rugnux_live_step::setTitle(std::string const& value)
+{
+    m_Title = value;
+}
+double Rugnux_live_step::getStartedS() const
+{
+    return m_Started_s;
+}
+void Rugnux_live_step::setStartedS(double const value)
+{
+    m_Started_s = value;
+}
+double Rugnux_live_step::getDurationS() const
+{
+    return m_Duration_s;
+}
+void Rugnux_live_step::setDurationS(double const value)
+{
+    m_Duration_s = value;
+    m_Duration_sIsSet = true;
+}
+bool Rugnux_live_step::durationSIsSet() const
+{
+    return m_Duration_sIsSet;
+}
+void Rugnux_live_step::unsetDuration_s()
+{
+    m_Duration_sIsSet = false;
+}
+std::vector<std::string> Rugnux_live_step::getNotes() const
+{
+    return m_Notes;
+}
+void Rugnux_live_step::setNotes(std::vector<std::string> const& value)
+{
+    m_Notes = value;
+}
+
+
+} // namespace org::openapitools::server::model
+

broker/gen/model/Rugnux_live_step.h

@@ -0,0 +1,101 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * Rugnux_live_step.h
+ *
+ * 
+ */
+
+#ifndef Rugnux_live_step_H_
+#define Rugnux_live_step_H_
+
+
+#include <string>
+#include <vector>
+#include <nlohmann/json.hpp>
+
+namespace org::openapitools::server::model
+{
+
+/// <summary>
+/// 
+/// </summary>
+class  Rugnux_live_step
+{
+public:
+    Rugnux_live_step();
+    virtual ~Rugnux_live_step() = default;
+
+
+    /// <summary>
+    /// Validate the current data in the model. Throws a ValidationException on failure.
+    /// </summary>
+    void validate() const;
+
+    /// <summary>
+    /// Validate the current data in the model. Returns false on error and writes an error
+    /// message into the given stringstream.
+    /// </summary>
+    bool validate(std::stringstream& msg) const;
+
+    /// <summary>
+    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
+    /// Not meant to be called outside that case.
+    /// </summary>
+    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
+
+    bool operator==(const Rugnux_live_step& rhs) const;
+    bool operator!=(const Rugnux_live_step& rhs) const;
+
+    /////////////////////////////////////////////
+    /// Rugnux_live_step members
+
+    /// <summary>
+    /// 
+    /// </summary>
+    std::string getTitle() const;
+    void setTitle(std::string const& value);
+    /// <summary>
+    /// Start of the step, since the collection started
+    /// </summary>
+    double getStartedS() const;
+    void setStartedS(double const value);
+    /// <summary>
+    /// Set once the next step starts or the run ends
+    /// </summary>
+    double getDurationS() const;
+    void setDurationS(double const value);
+    bool durationSIsSet() const;
+    void unsetDuration_s();
+    /// <summary>
+    /// What the step found (e.g. lattice, space group)
+    /// </summary>
+    std::vector<std::string> getNotes() const;
+    void setNotes(std::vector<std::string> const& value);
+
+    friend  void to_json(nlohmann::json& j, const Rugnux_live_step& o);
+    friend  void from_json(const nlohmann::json& j, Rugnux_live_step& o);
+protected:
+    std::string m_Title;
+
+    double m_Started_s;
+
+    double m_Duration_s;
+    bool m_Duration_sIsSet;
+    std::vector<std::string> m_Notes;
+
+    
+};
+
+} // namespace org::openapitools::server::model
+
+#endif /* Rugnux_live_step_H_ */

broker/gen/model/Rugnux_settings.cpp

@@ -0,0 +1,192 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+
+
+#include "Rugnux_settings.h"
+#include "Helpers.h"
+
+#include <sstream>
+
+namespace org::openapitools::server::model
+{
+
+Rugnux_settings::Rugnux_settings()
+{
+    m_Enable = false;
+    m_EnableIsSet = false;
+    m_Reserved_cpus = 4L;
+    m_Reserved_cpusIsSet = false;
+    m_Deadline_s = 0.0;
+    m_Deadline_sIsSet = false;
+    
+}
+
+void Rugnux_settings::validate() const
+{
+    std::stringstream msg;
+    if (!validate(msg))
+    {
+        throw org::openapitools::server::helpers::ValidationException(msg.str());
+    }
+}
+
+bool Rugnux_settings::validate(std::stringstream& msg) const
+{
+    return validate(msg, "");
+}
+
+bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPrefix) const
+{
+    bool success = true;
+    const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_settings" : pathPrefix;
+
+             
+    if (reservedCpusIsSet())
+    {
+        const int64_t& value = m_Reserved_cpus;
+        const std::string currentValuePath = _pathPrefix + ".reservedCpus";
+                
+        
+        if (value < 0ll)
+        {
+            success = false;
+            msg << currentValuePath << ": must be greater than or equal to 0;";
+        }
+
+    }
+         
+    if (deadlineSIsSet())
+    {
+        const double& value = m_Deadline_s;
+        const std::string currentValuePath = _pathPrefix + ".deadlineS";
+                
+        
+        if (value < 0.0)
+        {
+            success = false;
+            msg << currentValuePath << ": must be greater than or equal to 0.0;";
+        }
+
+    }
+    
+    return success;
+}
+
+bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const
+{
+    return
+    
+    
+    
+    ((!enableIsSet() && !rhs.enableIsSet()) || (enableIsSet() && rhs.enableIsSet() && isEnable() == rhs.isEnable())) &&
+    
+    
+    ((!reservedCpusIsSet() && !rhs.reservedCpusIsSet()) || (reservedCpusIsSet() && rhs.reservedCpusIsSet() && getReservedCpus() == rhs.getReservedCpus())) &&
+    
+    
+    ((!deadlineSIsSet() && !rhs.deadlineSIsSet()) || (deadlineSIsSet() && rhs.deadlineSIsSet() && getDeadlineS() == rhs.getDeadlineS()))
+    
+    ;
+}
+
+bool Rugnux_settings::operator!=(const Rugnux_settings& rhs) const
+{
+    return !(*this == rhs);
+}
+
+void to_json(nlohmann::json& j, const Rugnux_settings& o)
+{
+    j = nlohmann::json::object();
+    if(o.enableIsSet())
+        j["enable"] = o.m_Enable;
+    if(o.reservedCpusIsSet())
+        j["reserved_cpus"] = o.m_Reserved_cpus;
+    if(o.deadlineSIsSet())
+        j["deadline_s"] = o.m_Deadline_s;
+    
+}
+
+void from_json(const nlohmann::json& j, Rugnux_settings& o)
+{
+    if(j.find("enable") != j.end())
+    {
+        j.at("enable").get_to(o.m_Enable);
+        o.m_EnableIsSet = true;
+    } 
+    if(j.find("reserved_cpus") != j.end())
+    {
+        j.at("reserved_cpus").get_to(o.m_Reserved_cpus);
+        o.m_Reserved_cpusIsSet = true;
+    } 
+    if(j.find("deadline_s") != j.end())
+    {
+        j.at("deadline_s").get_to(o.m_Deadline_s);
+        o.m_Deadline_sIsSet = true;
+    } 
+    
+}
+
+bool Rugnux_settings::isEnable() const
+{
+    return m_Enable;
+}
+void Rugnux_settings::setEnable(bool const value)
+{
+    m_Enable = value;
+    m_EnableIsSet = true;
+}
+bool Rugnux_settings::enableIsSet() const
+{
+    return m_EnableIsSet;
+}
+void Rugnux_settings::unsetEnable()
+{
+    m_EnableIsSet = false;
+}
+int64_t Rugnux_settings::getReservedCpus() const
+{
+    return m_Reserved_cpus;
+}
+void Rugnux_settings::setReservedCpus(int64_t const value)
+{
+    m_Reserved_cpus = value;
+    m_Reserved_cpusIsSet = true;
+}
+bool Rugnux_settings::reservedCpusIsSet() const
+{
+    return m_Reserved_cpusIsSet;
+}
+void Rugnux_settings::unsetReserved_cpus()
+{
+    m_Reserved_cpusIsSet = false;
+}
+double Rugnux_settings::getDeadlineS() const
+{
+    return m_Deadline_s;
+}
+void Rugnux_settings::setDeadlineS(double const value)
+{
+    m_Deadline_s = value;
+    m_Deadline_sIsSet = true;
+}
+bool Rugnux_settings::deadlineSIsSet() const
+{
+    return m_Deadline_sIsSet;
+}
+void Rugnux_settings::unsetDeadline_s()
+{
+    m_Deadline_sIsSet = false;
+}
+
+
+} // namespace org::openapitools::server::model
+

broker/gen/model/Rugnux_settings.h

@@ -0,0 +1,96 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * Rugnux_settings.h
+ *
+ * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. 
+ */
+
+#ifndef Rugnux_settings_H_
+#define Rugnux_settings_H_
+
+
+#include <nlohmann/json.hpp>
+
+namespace org::openapitools::server::model
+{
+
+/// <summary>
+/// Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. 
+/// </summary>
+class  Rugnux_settings
+{
+public:
+    Rugnux_settings();
+    virtual ~Rugnux_settings() = default;
+
+
+    /// <summary>
+    /// Validate the current data in the model. Throws a ValidationException on failure.
+    /// </summary>
+    void validate() const;
+
+    /// <summary>
+    /// Validate the current data in the model. Returns false on error and writes an error
+    /// message into the given stringstream.
+    /// </summary>
+    bool validate(std::stringstream& msg) const;
+
+    /// <summary>
+    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
+    /// Not meant to be called outside that case.
+    /// </summary>
+    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
+
+    bool operator==(const Rugnux_settings& rhs) const;
+    bool operator!=(const Rugnux_settings& rhs) const;
+
+    /////////////////////////////////////////////
+    /// Rugnux_settings members
+
+    /// <summary>
+    /// Process DECTRIS rotation collections with rugnux live
+    /// </summary>
+    bool isEnable() const;
+    void setEnable(bool const value);
+    bool enableIsSet() const;
+    void unsetEnable();
+    /// <summary>
+    /// CPU threads left to the receiver and the image pusher while rugnux runs
+    /// </summary>
+    int64_t getReservedCpus() const;
+    void setReservedCpus(int64_t const value);
+    bool reservedCpusIsSet() const;
+    void unsetReserved_cpus();
+    /// <summary>
+    /// Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as &#x60;POST /cancel&#x60; after the last image does). 0 means no cutoff. 
+    /// </summary>
+    double getDeadlineS() const;
+    void setDeadlineS(double const value);
+    bool deadlineSIsSet() const;
+    void unsetDeadline_s();
+
+    friend  void to_json(nlohmann::json& j, const Rugnux_settings& o);
+    friend  void from_json(const nlohmann::json& j, Rugnux_settings& o);
+protected:
+    bool m_Enable;
+    bool m_EnableIsSet;
+    int64_t m_Reserved_cpus;
+    bool m_Reserved_cpusIsSet;
+    double m_Deadline_s;
+    bool m_Deadline_sIsSet;
+    
+};
+
+} // namespace org::openapitools::server::model
+
+#endif /* Rugnux_settings_H_ */

broker/gen/model/Rugnux_status.cpp

@@ -0,0 +1,177 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+
+
+#include "Rugnux_status.h"
+#include "Helpers.h"
+
+#include <sstream>
+
+namespace org::openapitools::server::model
+{
+
+Rugnux_status::Rugnux_status()
+{
+    m_Enable = false;
+    m_Reserved_cpus = 0L;
+    m_Deadline_s = 0.0;
+    m_Deadline_sIsSet = false;
+    m_Model_loaded = false;
+    m_Model_size_bytes = 0L;
+    m_Model_size_bytesIsSet = false;
+    
+}
+
+void Rugnux_status::validate() const
+{
+    std::stringstream msg;
+    if (!validate(msg))
+    {
+        throw org::openapitools::server::helpers::ValidationException(msg.str());
+    }
+}
+
+bool Rugnux_status::validate(std::stringstream& msg) const
+{
+    return validate(msg, "");
+}
+
+bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPrefix) const
+{
+    bool success = true;
+    const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix;
+
+                        
+    return success;
+}
+
+bool Rugnux_status::operator==(const Rugnux_status& rhs) const
+{
+    return
+    
+    
+    (isEnable() == rhs.isEnable())
+     &&
+    
+    (getReservedCpus() == rhs.getReservedCpus())
+     &&
+    
+    
+    ((!deadlineSIsSet() && !rhs.deadlineSIsSet()) || (deadlineSIsSet() && rhs.deadlineSIsSet() && getDeadlineS() == rhs.getDeadlineS())) &&
+    
+    (isModelLoaded() == rhs.isModelLoaded())
+     &&
+    
+    
+    ((!modelSizeBytesIsSet() && !rhs.modelSizeBytesIsSet()) || (modelSizeBytesIsSet() && rhs.modelSizeBytesIsSet() && getModelSizeBytes() == rhs.getModelSizeBytes()))
+    
+    ;
+}
+
+bool Rugnux_status::operator!=(const Rugnux_status& rhs) const
+{
+    return !(*this == rhs);
+}
+
+void to_json(nlohmann::json& j, const Rugnux_status& o)
+{
+    j = nlohmann::json::object();
+    j["enable"] = o.m_Enable;
+    j["reserved_cpus"] = o.m_Reserved_cpus;
+    if(o.deadlineSIsSet())
+        j["deadline_s"] = o.m_Deadline_s;
+    j["model_loaded"] = o.m_Model_loaded;
+    if(o.modelSizeBytesIsSet())
+        j["model_size_bytes"] = o.m_Model_size_bytes;
+    
+}
+
+void from_json(const nlohmann::json& j, Rugnux_status& o)
+{
+    j.at("enable").get_to(o.m_Enable);
+    j.at("reserved_cpus").get_to(o.m_Reserved_cpus);
+    if(j.find("deadline_s") != j.end())
+    {
+        j.at("deadline_s").get_to(o.m_Deadline_s);
+        o.m_Deadline_sIsSet = true;
+    } 
+    j.at("model_loaded").get_to(o.m_Model_loaded);
+    if(j.find("model_size_bytes") != j.end())
+    {
+        j.at("model_size_bytes").get_to(o.m_Model_size_bytes);
+        o.m_Model_size_bytesIsSet = true;
+    } 
+    
+}
+
+bool Rugnux_status::isEnable() const
+{
+    return m_Enable;
+}
+void Rugnux_status::setEnable(bool const value)
+{
+    m_Enable = value;
+}
+int64_t Rugnux_status::getReservedCpus() const
+{
+    return m_Reserved_cpus;
+}
+void Rugnux_status::setReservedCpus(int64_t const value)
+{
+    m_Reserved_cpus = value;
+}
+double Rugnux_status::getDeadlineS() const
+{
+    return m_Deadline_s;
+}
+void Rugnux_status::setDeadlineS(double const value)
+{
+    m_Deadline_s = value;
+    m_Deadline_sIsSet = true;
+}
+bool Rugnux_status::deadlineSIsSet() const
+{
+    return m_Deadline_sIsSet;
+}
+void Rugnux_status::unsetDeadline_s()
+{
+    m_Deadline_sIsSet = false;
+}
+bool Rugnux_status::isModelLoaded() const
+{
+    return m_Model_loaded;
+}
+void Rugnux_status::setModelLoaded(bool const value)
+{
+    m_Model_loaded = value;
+}
+int64_t Rugnux_status::getModelSizeBytes() const
+{
+    return m_Model_size_bytes;
+}
+void Rugnux_status::setModelSizeBytes(int64_t const value)
+{
+    m_Model_size_bytes = value;
+    m_Model_size_bytesIsSet = true;
+}
+bool Rugnux_status::modelSizeBytesIsSet() const
+{
+    return m_Model_size_bytesIsSet;
+}
+void Rugnux_status::unsetModel_size_bytes()
+{
+    m_Model_size_bytesIsSet = false;
+}
+
+
+} // namespace org::openapitools::server::model
+

broker/gen/model/Rugnux_status.h

@@ -0,0 +1,108 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * Rugnux_status.h
+ *
+ * 
+ */
+
+#ifndef Rugnux_status_H_
+#define Rugnux_status_H_
+
+
+#include <nlohmann/json.hpp>
+
+namespace org::openapitools::server::model
+{
+
+/// <summary>
+/// 
+/// </summary>
+class  Rugnux_status
+{
+public:
+    Rugnux_status();
+    virtual ~Rugnux_status() = default;
+
+
+    /// <summary>
+    /// Validate the current data in the model. Throws a ValidationException on failure.
+    /// </summary>
+    void validate() const;
+
+    /// <summary>
+    /// Validate the current data in the model. Returns false on error and writes an error
+    /// message into the given stringstream.
+    /// </summary>
+    bool validate(std::stringstream& msg) const;
+
+    /// <summary>
+    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
+    /// Not meant to be called outside that case.
+    /// </summary>
+    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
+
+    bool operator==(const Rugnux_status& rhs) const;
+    bool operator!=(const Rugnux_status& rhs) const;
+
+    /////////////////////////////////////////////
+    /// Rugnux_status members
+
+    /// <summary>
+    /// Rugnux live mode is enabled
+    /// </summary>
+    bool isEnable() const;
+    void setEnable(bool const value);
+    /// <summary>
+    /// CPU threads left to the receiver and the image pusher while rugnux runs
+    /// </summary>
+    int64_t getReservedCpus() const;
+    void setReservedCpus(int64_t const value);
+    /// <summary>
+    /// Seconds after the last image at which the analysis is cut; 0 means no cutoff
+    /// </summary>
+    double getDeadlineS() const;
+    void setDeadlineS(double const value);
+    bool deadlineSIsSet() const;
+    void unsetDeadline_s();
+    /// <summary>
+    /// A model was uploaded with &#x60;PUT /config/rugnux/model&#x60;
+    /// </summary>
+    bool isModelLoaded() const;
+    void setModelLoaded(bool const value);
+    /// <summary>
+    /// Size of the uploaded model file
+    /// </summary>
+    int64_t getModelSizeBytes() const;
+    void setModelSizeBytes(int64_t const value);
+    bool modelSizeBytesIsSet() const;
+    void unsetModel_size_bytes();
+
+    friend  void to_json(nlohmann::json& j, const Rugnux_status& o);
+    friend  void from_json(const nlohmann::json& j, Rugnux_status& o);
+protected:
+    bool m_Enable;
+
+    int64_t m_Reserved_cpus;
+
+    double m_Deadline_s;
+    bool m_Deadline_sIsSet;
+    bool m_Model_loaded;
+
+    int64_t m_Model_size_bytes;
+    bool m_Model_size_bytesIsSet;
+    
+};
+
+} // namespace org::openapitools::server::model
+
+#endif /* Rugnux_status_H_ */

broker/gen/model/Rugnux_summary_row.cpp

@@ -0,0 +1,105 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+
+
+#include "Rugnux_summary_row.h"
+#include "Helpers.h"
+
+#include <sstream>
+
+namespace org::openapitools::server::model
+{
+
+Rugnux_summary_row::Rugnux_summary_row()
+{
+    m_Label = "";
+    m_Value = "";
+    
+}
+
+void Rugnux_summary_row::validate() const
+{
+    std::stringstream msg;
+    if (!validate(msg))
+    {
+        throw org::openapitools::server::helpers::ValidationException(msg.str());
+    }
+}
+
+bool Rugnux_summary_row::validate(std::stringstream& msg) const
+{
+    return validate(msg, "");
+}
+
+bool Rugnux_summary_row::validate(std::stringstream& msg, const std::string& pathPrefix) const
+{
+    bool success = true;
+    const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_summary_row" : pathPrefix;
+
+            
+    return success;
+}
+
+bool Rugnux_summary_row::operator==(const Rugnux_summary_row& rhs) const
+{
+    return
+    
+    
+    (getLabel() == rhs.getLabel())
+     &&
+    
+    (getValue() == rhs.getValue())
+    
+    
+    ;
+}
+
+bool Rugnux_summary_row::operator!=(const Rugnux_summary_row& rhs) const
+{
+    return !(*this == rhs);
+}
+
+void to_json(nlohmann::json& j, const Rugnux_summary_row& o)
+{
+    j = nlohmann::json::object();
+    j["label"] = o.m_Label;
+    j["value"] = o.m_Value;
+    
+}
+
+void from_json(const nlohmann::json& j, Rugnux_summary_row& o)
+{
+    j.at("label").get_to(o.m_Label);
+    j.at("value").get_to(o.m_Value);
+    
+}
+
+std::string Rugnux_summary_row::getLabel() const
+{
+    return m_Label;
+}
+void Rugnux_summary_row::setLabel(std::string const& value)
+{
+    m_Label = value;
+}
+std::string Rugnux_summary_row::getValue() const
+{
+    return m_Value;
+}
+void Rugnux_summary_row::setValue(std::string const& value)
+{
+    m_Value = value;
+}
+
+
+} // namespace org::openapitools::server::model
+

broker/gen/model/Rugnux_summary_row.h

@@ -0,0 +1,84 @@
+/**
+* Jungfraujoch
+* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates.  # License Clarification  While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing.  This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL.  This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. 
+*
+* The version of the OpenAPI document: 1.0.0-rc.176
+* Contact: filip.leonarski@psi.ch
+*
+* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+* https://openapi-generator.tech
+* Do not edit the class manually.
+*/
+/*
+ * Rugnux_summary_row.h
+ *
+ * 
+ */
+
+#ifndef Rugnux_summary_row_H_
+#define Rugnux_summary_row_H_
+
+
+#include <string>
+#include <nlohmann/json.hpp>
+
+namespace org::openapitools::server::model
+{
+
+/// <summary>
+/// 
+/// </summary>
+class  Rugnux_summary_row
+{
+public:
+    Rugnux_summary_row();
+    virtual ~Rugnux_summary_row() = default;
+
+
+    /// <summary>
+    /// Validate the current data in the model. Throws a ValidationException on failure.
+    /// </summary>
+    void validate() const;
+
+    /// <summary>
+    /// Validate the current data in the model. Returns false on error and writes an error
+    /// message into the given stringstream.
+    /// </summary>
+    bool validate(std::stringstream& msg) const;
+
+    /// <summary>
+    /// Helper overload for validate. Used when one model stores another model and calls it's validate.
+    /// Not meant to be called outside that case.
+    /// </summary>
+    bool validate(std::stringstream& msg, const std::string& pathPrefix) const;
+
+    bool operator==(const Rugnux_summary_row& rhs) const;
+    bool operator!=(const Rugnux_summary_row& rhs) const;
+
+    /////////////////////////////////////////////
+    /// Rugnux_summary_row members
+
+    /// <summary>
+    /// 
+    /// </summary>
+    std::string getLabel() const;
+    void setLabel(std::string const& value);
+    /// <summary>
+    /// 
+    /// </summary>
+    std::string getValue() const;
+    void setValue(std::string const& value);
+
+    friend  void to_json(nlohmann::json& j, const Rugnux_summary_row& o);
+    friend  void from_json(const nlohmann::json& j, Rugnux_summary_row& o);
+protected:
+    std::string m_Label;
+
+    std::string m_Value;
+
+    
+};
+
+} // namespace org::openapitools::server::model
+
+#endif /* Rugnux_summary_row_H_ */

broker/jfjoch_api.yaml

@@ -2684,6 +2684,167 @@ components:
           $ref: '#/components/schemas/zeromq_metadata_settings'
         dark_mask:
           $ref: '#/components/schemas/dark_mask_settings'
+        rugnux:
+          $ref: '#/components/schemas/rugnux_settings'
+    rugnux_settings:
+      type: object
+      description: |
+        Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux
+        inside the broker while they arrive, and the merged result is sent to the writer.
+      properties:
+        enable:
+          type: boolean
+          default: false
+          description: Process DECTRIS rotation collections with rugnux live
+        reserved_cpus:
+          type: integer
+          format: int64
+          minimum: 0
+          default: 4
+          description: CPU threads left to the receiver and the image pusher while rugnux runs
+        deadline_s:
+          type: number
+          format: double
+          minimum: 0.0
+          default: 0.0
+          description: |
+            Fixed cutoff: this long after the last image the analysis is cut, and the collection ends
+            with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff.
+    rugnux_status:
+      type: object
+      required:
+        - enable
+        - reserved_cpus
+        - model_loaded
+      properties:
+        enable:
+          type: boolean
+          description: Rugnux live mode is enabled
+        reserved_cpus:
+          type: integer
+          format: int64
+          description: CPU threads left to the receiver and the image pusher while rugnux runs
+        deadline_s:
+          type: number
+          format: double
+          description: Seconds after the last image at which the analysis is cut; 0 means no cutoff
+        model_loaded:
+          type: boolean
+          description: A model was uploaded with `PUT /config/rugnux/model`
+        model_size_bytes:
+          type: integer
+          format: int64
+          description: Size of the uploaded model file
+    rugnux_live_step:
+      type: object
+      required:
+        - title
+        - started_s
+        - notes
+      properties:
+        title:
+          type: string
+        started_s:
+          type: number
+          format: double
+          description: Start of the step, since the collection started
+        duration_s:
+          type: number
+          format: double
+          description: Set once the next step starts or the run ends
+        notes:
+          type: array
+          description: What the step found (e.g. lattice, space group)
+          items:
+            type: string
+    rugnux_summary_row:
+      type: object
+      required:
+        - label
+        - value
+      properties:
+        label:
+          type: string
+        value:
+          type: string
+    rugnux_live_status:
+      type: object
+      description: Live rugnux processing of the last collection; kept until the next collection starts
+      required:
+        - state
+        - message
+        - images_expected
+        - images_received
+        - elapsed_s
+        - steps
+        - verdict
+        - verdict_text
+        - summary
+        - warnings
+        - probe
+      properties:
+        state:
+          type: string
+          enum: [Idle, Running, Done, Failed, Cancelled, Skipped]
+        probe:
+          type: string
+          description: Answer of the early lattice probe on the first 60 degrees; empty until it answers
+        probe_s:
+          type: number
+          format: double
+          description: When the probe answered, since the collection started
+        message:
+          type: string
+          description: Why the run failed or was skipped
+        images_expected:
+          type: integer
+          format: int64
+        images_received:
+          type: integer
+          format: int64
+        elapsed_s:
+          type: number
+          format: double
+        step_progress:
+          type: number
+          format: float
+          minimum: 0.0
+          maximum: 1.0
+          description: Fraction of the current pass's images
+        steps:
+          type: array
+          items:
+            $ref: '#/components/schemas/rugnux_live_step'
+        last_image_s:
+          type: number
+          format: double
+          description: When the last image arrived (and the cutoff timer started), since the collection started
+        cut:
+          type: string
+          description: |
+            Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel`
+            (`POST /cancel`). The end message then carried what was established at the cut.
+        cut_s:
+          type: number
+          format: double
+          description: When the analysis was cut, since the collection started
+        last_image_to_written_s:
+          type: number
+          format: double
+          description: From the last image received to all results written
+        verdict:
+          type: string
+          description: OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over
+        verdict_text:
+          type: string
+        summary:
+          type: array
+          items:
+            $ref: '#/components/schemas/rugnux_summary_row'
+        warnings:
+          type: array
+          items:
+            type: string
     zeromq_metadata_settings:
       type: object
       required:
@@ -2895,6 +3056,11 @@ paths:
         Any frame that is currently being processed by CPU will be finished and sent to writer.
         Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state.
         
+        With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection
+        or after its last image alike: the collection ends with what the analysis has established
+        (see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not
+        marked as cancelled.
+        
         If data collection is not running, the command has no effect.
       responses:
         "200":
@@ -3331,6 +3497,133 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/detector_list'
+  /config/rugnux:
+    put:
+      summary: Change rugnux live mode settings
+      description: |
+        This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+      security:
+        - bearerAuth: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/rugnux_settings'
+      responses:
+        "401":
+          description: The current dataset is protected and no valid bearer token was presented.
+          content:
+            text/plain:
+              schema:
+                type: string
+        "200":
+          description: Everything OK
+        "400":
+          description: Input parsing or validation error
+          content:
+            text/plain:
+              schema:
+                type: string
+                description: Exception error
+        "500":
+          description: Error within Jungfraujoch code - see output message.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_message'
+    get:
+      summary: Get rugnux live mode settings and model status
+      description: Can be done anytime
+      security:
+        - bearerAuth: []
+      responses:
+        "401":
+          description: The current dataset is protected and no valid bearer token was presented.
+          content:
+            text/plain:
+              schema:
+                type: string
+        "200":
+          description: Everything OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/rugnux_status'
+  /config/rugnux/model:
+    put:
+      summary: Upload the model used by rugnux live mode
+      description: |
+        This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+        The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement
+        and validation of every following rugnux live collection, until it is removed with
+        `DELETE /config/rugnux/model` or replaced by another upload.
+      security:
+        - bearerAuth: []
+      requestBody:
+        content:
+          application/octet-stream:
+            schema:
+              type: string
+              format: binary
+      responses:
+        "401":
+          description: The current dataset is protected and no valid bearer token was presented.
+          content:
+            text/plain:
+              schema:
+                type: string
+        "200":
+          description: All good
+        "400":
+          description: Empty body, or not a model that can be read (PDB or mmCIF, optionally gzipped)
+        "413":
+          description: Model larger than 64 MiB
+        "500":
+          description: Error within Jungfraujoch code - see output message.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_message'
+    delete:
+      summary: Remove the model used by rugnux live mode
+      description: |
+        This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+      security:
+        - bearerAuth: []
+      responses:
+        "401":
+          description: The current dataset is protected and no valid bearer token was presented.
+          content:
+            text/plain:
+              schema:
+                type: string
+        "200":
+          description: All good
+        "500":
+          description: Error within Jungfraujoch code - see output message.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/error_message'
+  /rugnux/status:
+    get:
+      summary: Get live rugnux processing status of the last collection
+      description: Can be done anytime
+      security:
+        - bearerAuth: []
+      responses:
+        "401":
+          description: The current dataset is protected and no valid bearer token was presented.
+          content:
+            text/plain:
+              schema:
+                type: string
+        "200":
+          description: Everything OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/rugnux_live_status'
   /config/zeromq_preview:
     put:
       summary: Set ZeroMQ preview settings

broker/jfjoch_broker.cpp

@@ -116,6 +116,7 @@ int main(int argc, char **argv) {
 
     if (receiver)
         broker.Services().Receiver(receiver.get());
+    ParseRugnuxSettings(settings, broker.Services());
 
     httplib::Server server;
     httpServer = &server;

common/CMakeLists.txt

@@ -162,6 +162,7 @@ ADD_LIBRARY(JFJochCommon STATIC
         ScalingSettings.h
         UnitCell.cpp
         UnitCell.h
+        MtzTable.h
         CorrelationCoefficient.cpp
         CorrelationCoefficient.h
         JFJochReceiverPlots.cpp

common/JFJochMessages.h

@@ -16,6 +16,7 @@
 #include "CompressedImage.h"
 #include "SpotToSave.h"
 #include "UnitCell.h"
+#include "MtzTable.h"
 #include "GoniometerAxis.h"
 #include "DetectorTransformation.h"
 #include "GridScanSettings.h"
@@ -368,6 +369,30 @@ struct StartMessage {
     std::optional<SmargonPosition> smargon_position;
 };
 
+// The atomic model as placed against the data, one vector per atom field in file order, for
+// WritePlacedModel to write <prefix>_model.cif. A new residue starts wherever the model, chain or
+// residue key changes. unit_cell and space_group_name are what WritePlacedModel is given (the data's);
+// model_space_group_name is the group the coordinates were built in.
+struct PlacedModelMessage {
+    std::string name;
+    UnitCell unit_cell;
+    std::string space_group_name;
+    std::string model_space_group_name;
+    std::vector<int32_t> model_number;
+    std::vector<std::string> chain;
+    std::vector<std::string> residue_name;
+    std::vector<int32_t> seq_num;
+    std::vector<uint8_t> icode;            // ' ' where there is none
+    std::vector<uint8_t> het_flag;         // 'A', 'H', or 0 where unknown
+    std::vector<std::string> atom_name;
+    std::vector<std::string> element;
+    std::vector<uint8_t> altloc;           // 0 where there is none
+    std::vector<int32_t> charge;
+    std::vector<double> x, y, z;
+    std::vector<float> b_iso, occupancy;
+    std::vector<float> aniso;              // 6 per atom (u11 u22 u33 u12 u13 u23), empty when none has one
+};
+
 struct EndMessage {
     // The sample transformation chain, base first (see DetectorTransformation). Optional: when it is
     // empty the writer builds the same chain itself from the start message, so a producer that does
@@ -481,6 +506,12 @@ struct EndMessage {
     // this says what was DONE about it. Offline (rugnux) only; the broker does not merge.
     std::vector<uint8_t> frame_disposition;
     std::vector<std::string> frame_disposition_codes;
+
+    // The merged result of an analysis run inside the broker, for the writer to put beside the master
+    // file as <file_prefix>.mtz, _maps.mtz and _model.cif. Absent where nothing was merged.
+    std::optional<MtzTable> merged;             // MergedMtzTable (image_analysis/WriteReflections.h)
+    std::optional<MtzTable> map_coefficients;   // ModelMaps::coefficients (ModelValidation.h)
+    std::optional<PlacedModelMessage> placed_model;
 };
 
 struct MetadataMessage {

common/MtzTable.h

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "UnitCell.h"
+
+// What an MTZ file holds - its header, its columns and its batch headers - with no file behind it: the
+// merged reflections, the map coefficients and the unmerged observations rugnux makes, for whatever
+// writes them (WriteMtz, writer/WriteRugnuxFiles.h) or carries them (the END message). Every dataset
+// has the table's cell; a dataset's id is its position in `datasets`.
+struct MtzTable {
+    struct Dataset {
+        std::string project_name;
+        std::string crystal_name;
+        std::string dataset_name;
+        float wavelength_A = 0.0f;   // 0 = not set
+    };
+    struct Column {
+        std::string label;
+        char type = 'R';             // the MTZ column type: H, J, Q, F, I, ...
+        int32_t dataset_id = 0;
+        std::vector<float> values;   // one per row
+    };
+    // The header of one image of unmerged data: the integer and real arrays as the MTZ format lays them
+    // out (29 and 156 long), and the names of its goniostat axes.
+    struct Batch {
+        int32_t number = 0;
+        std::string title;
+        std::vector<int32_t> ints;
+        std::vector<float> floats;
+        std::vector<std::string> axes;
+    };
+
+    std::string title;
+    UnitCell unit_cell{};
+    std::string space_group_name;    // extended Hermann-Mauguin (xhm), which keeps the setting
+    std::vector<std::string> history;
+    std::array<int32_t, 5> sort_order{};
+    std::vector<Dataset> datasets;
+    std::vector<Column> columns;
+    std::vector<Batch> batches;
+};

common/ParallelFor.h

@@ -16,6 +16,12 @@
 
 #include "ThreadAffinity.h"
 
+#ifdef __linux__
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#endif
+
 // Two shapes of "run this over a range on several threads", used by the analysis code. Both take the
 // worker count from the caller rather than asking the hardware, so a run that was told how many
 // threads to use keeps to it.
@@ -48,6 +54,20 @@ namespace parallel_detail {
 
         size_t WorkerCount() const { return workers.size(); }
 
+        // The scheduling priority of every pool thread (Linux; elsewhere nothing). The pool is made once
+        // per process, so a thread that lowers its own priority does not lower the work it hands to the
+        // pool: the broker running rugnux beside its receiving threads sets it here as well. Raising
+        // it back needs the privilege to do so; returns false where that was refused.
+        bool SetNice(int nice) {
+            bool ok = true;
+#ifdef __linux__
+            std::lock_guard lock(m);
+            for (const int tid: tids)
+                ok = (setpriority(PRIO_PROCESS, static_cast<id_t>(tid), nice) == 0) && ok;
+#endif
+            return ok;
+        }
+
         void Submit(std::function<void()> job) {
             {
                 std::lock_guard lock(m);
@@ -66,6 +86,13 @@ namespace parallel_detail {
                         // It inherited whatever CPUs the thread that first used the pool was kept to.
                         RestoreThreadAffinity();
                         InWorkerFlag() = true;
+#ifdef __linux__
+                        {
+                            std::lock_guard lock(m);
+                            tids.push_back(static_cast<int>(syscall(SYS_gettid)));
+                        }
+                        cv.notify_all();
+#endif
                         Loop();
                     });
             } catch (...) {
@@ -75,6 +102,11 @@ namespace parallel_detail {
                 StopAndJoin();
                 throw;
             }
+#ifdef __linux__
+            // Every worker is in tids once the pool exists, so a SetNice right after reaches all of them.
+            std::unique_lock lock(m);
+            cv.wait(lock, [this] { return tids.size() == workers.size(); });
+#endif
         }
 
         ~WorkerPool() { StopAndJoin(); }
@@ -106,6 +138,7 @@ namespace parallel_detail {
         std::condition_variable cv;
         std::deque<std::function<void()> > queue;
         std::vector<std::thread> workers;
+        std::vector<int> tids;   // the workers' kernel thread ids, for SetNice
         bool stop = false;
 
         // A function-local thread_local rather than an inline static member: Apple's linker rejects the

docs/CBOR.md

@@ -342,9 +342,25 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s
 | pixel_sum                        | Array(int64)             | Per-image sum of all valid pixels, excluding error/saturated pixels                                                               |                           |
 | image_scale_mosaicity            | Array(float)             | Scaling result: Image scale mosaicity \[deg\]                                                                                     |                           |
 | image_scale_cc                   | Array(float)             | Scaling result: Image scale CC                                                                                                    |                           |
+| merged                           | object (optional)        | Merged reflections of an in-broker analysis run, written by the writer as `<file_prefix>.mtz`. An MTZ table (layout below) | |
+| map_coefficients                 | object (optional)        | Map coefficients of an in-broker analysis run, written as `<file_prefix>_maps.mtz`. An MTZ table (layout below) | |
+| placed_model                     | object (optional)        | Atomic model placed against the data, written as `<file_prefix>_model.cif` and `<file_prefix>_model.pdb`. `name` (string), `unit_cell` (object, as above), `space_group_name` (the data's, gemmi extended H-M), `model_space_group_name` (the coordinates'), then one array per atom: `model_number`, `seq_num`, `charge` (int32), `chain`, `residue_name`, `atom_name`, `element` (Array(string)), `icode` (uint8, space where none), `het_flag` (uint8, `A`, `H` or 0 where unknown), `altloc` (uint8, 0 where none), `x`, `y`, `z` (double, Angstrom), `b_iso`, `occupancy` (float), `aniso` (float, 6 per atom: U11 U22 U33 U12 U13 U23; empty where no atom has one) | |
 
 End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted.
 
+An MTZ table (`merged`, `map_coefficients`) carries an MTZ file's content as the writer writes it, column by column:
+
+| Field name       | Type          | Description                                                                                          |
+|------------------|---------------|------------------------------------------------------------------------------------------------------|
+| title            | string        | MTZ title                                                                                            |
+| unit_cell        | object        | a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]                                             |
+| space_group_name | string        | Space group, gemmi extended Hermann-Mauguin (keeps the setting)                                      |
+| history          | Array(string) | MTZ HISTORY records                                                                                  |
+| sort_order       | Array(int32)  | MTZ sort order, 5 entries                                                                            |
+| datasets         | Array(object) | One per MTZ dataset: `project_name`, `crystal_name`, `dataset_name` (string), `wavelength` (float, Angstrom; 0 = not set) |
+| columns          | Array(object) | One per MTZ column: `label` (string), `type` (string, one character: the MTZ column type H, J, F, Q, I, ...), `dataset_id` (int), `values` (float typed array, one per row) |
+| batches          | Array(object) | One per image of unmerged data (empty for merged data): `number` (int), `title` (string), `ints` (int32), `floats` (float) - the MTZ batch header arrays - and `axes` (Array(string), goniostat axis names) |
+
 ## Calibration message
 
 | Field name   | Type                 | Description                                                                                                                       | Present in DECTRIS format |

docs/JFJOCH_BROKER.md

@@ -170,6 +170,49 @@ The writers report each closed data file to the broker (TCP protocol version 5,
 for `FILE_CLOSED`); the in-process HDF5 writer reports to it directly. The ZeroMQ writer has no way
 back to the broker, so with it the route answers 409.
 
+## Rugnux live mode
+
+With rugnux live mode on, rotation collections from a DECTRIS detector are processed by rugnux inside
+the broker while they arrive. It is off by default; the configuration file switches it on with
+`"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads
+left to the receiver and the image pusher.
+
+Rugnux reads the images back from the broker's image buffer, so the buffer must hold the whole
+collection - one slot per image. Where it holds fewer, the collection is only forwarded to the writer,
+as with the mode off; the live status reports it as skipped and the collection's statistics carry a
+warning saying how many images the buffer holds against how many the collection has.
+
+The results - the merged data, the map coefficients and, with a model, the placed model - travel to
+the writer in the end message, which writes them beside the master file as `<file_prefix>.mtz`,
+`<file_prefix>_maps.mtz`, `<file_prefix>_model.cif` and `<file_prefix>_model.pdb`. An end message
+that would be too large for the pushers' buffer (256 MiB) goes without them, with an error in the log
+and a warning in the live status; the collection itself is not affected.
+
+The same settings are read with `GET /config/rugnux` and changed with `PUT /config/rugnux`. A model
+for molecular replacement and validation (PDB or mmCIF, optionally gzipped, at most 64 MiB) is
+uploaded as the raw file with `PUT /config/rugnux/model` (`application/octet-stream`; a larger body
+gets `413`, an empty one or one that cannot be read as a model `400`). The broker parses it at upload
+and keeps it in memory - nothing is written to disk - for every following collection until
+`DELETE /config/rugnux/model` removes it or another upload replaces it; a broker restart drops it.
+`GET /config/rugnux` reports whether a model is loaded and its size, never its contents. Changes are
+refused during a collection, and while the current dataset was started with tokens, all of these
+routes need its bearer token.
+
+The end message, and with it the end of the collection, waits for the analysis. `deadline_s` (default
+0, no cutoff) bounds that wait: this long after the last image the analysis is cut. `POST /cancel` is
+the other way to cut it: while images still arrive it cancels the collection as with the mode off,
+and the analysis with it; after the last image it ends only the analysis. A cut analysis gives what it
+has established: a pass that is past its images still merges and makes its outputs, the canonical pass
+cut among its images merges those it has read, only the placed model is left out, and
+any other pass cut short gives way to the last one that ran to its end. The end message goes out at
+once with that, and the live status records the cut (`deadline` or `cancel`) and when.
+
+`GET /rugnux/status` reports the live run of the last collection, kept until the next one starts: its
+state, the images received, the step it is on and how far into it, the steps so far with what each
+found, and once it is over the verdict, the summary, warnings and the time from the last image to
+every result written. The web frontend shows it on the dashboard. It needs the dataset's bearer token
+like the routes above.
+
 ## Setting up a local test for Jungfraujoch
 For development, it is possible to set up a local installation of Jungfraujoch. 
 This will work without FPGA installed in the computer and allows testing the Jungfraujoch software layer, including

docs/python_client/README.md

@@ -126,6 +126,10 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**config_mask_tiff_get**](docs/DefaultApi.md#config_mask_tiff_get) | **GET** /config/mask.tiff | Get mask of the detector (TIFF)
 *DefaultApi* | [**config_roi_get**](docs/DefaultApi.md#config_roi_get) | **GET** /config/roi | Get ROI definitions
 *DefaultApi* | [**config_roi_put**](docs/DefaultApi.md#config_roi_put) | **PUT** /config/roi | Upload ROI definitions
+*DefaultApi* | [**config_rugnux_get**](docs/DefaultApi.md#config_rugnux_get) | **GET** /config/rugnux | Get rugnux live mode settings and model status
+*DefaultApi* | [**config_rugnux_model_delete**](docs/DefaultApi.md#config_rugnux_model_delete) | **DELETE** /config/rugnux/model | Remove the model used by rugnux live mode
+*DefaultApi* | [**config_rugnux_model_put**](docs/DefaultApi.md#config_rugnux_model_put) | **PUT** /config/rugnux/model | Upload the model used by rugnux live mode
+*DefaultApi* | [**config_rugnux_put**](docs/DefaultApi.md#config_rugnux_put) | **PUT** /config/rugnux | Change rugnux live mode settings
 *DefaultApi* | [**config_select_detector_get**](docs/DefaultApi.md#config_select_detector_get) | **GET** /config/select_detector | List available detectors
 *DefaultApi* | [**config_select_detector_put**](docs/DefaultApi.md#config_select_detector_put) | **PUT** /config/select_detector | Select detector
 *DefaultApi* | [**config_spot_finding_get**](docs/DefaultApi.md#config_spot_finding_get) | **GET** /config/spot_finding | Get data processing configuration
@@ -155,6 +159,7 @@ Class | Method | HTTP request | Description
 *DefaultApi* | [**preview_plot_bin_get**](docs/DefaultApi.md#preview_plot_bin_get) | **GET** /preview/plot.bin | Generate 1D plot from Jungfraujoch and send in raw binary format. Data are provided as (32-bit) float binary array. This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. 
 *DefaultApi* | [**preview_plot_get**](docs/DefaultApi.md#preview_plot_get) | **GET** /preview/plot | Generate 1D plot from Jungfraujoch
 *DefaultApi* | [**result_scan_get**](docs/DefaultApi.md#result_scan_get) | **GET** /result/scan | Get full scan result
+*DefaultApi* | [**rugnux_status_get**](docs/DefaultApi.md#rugnux_status_get) | **GET** /rugnux/status | Get live rugnux processing status of the last collection
 *DefaultApi* | [**start_post**](docs/DefaultApi.md#start_post) | **POST** /start | Start detector
 *DefaultApi* | [**statistics_calibration_get**](docs/DefaultApi.md#statistics_calibration_get) | **GET** /statistics/calibration | Get calibration statistics
 *DefaultApi* | [**statistics_data_collection_get**](docs/DefaultApi.md#statistics_data_collection_get) | **GET** /statistics/data_collection | Get data collection statistics
@@ -224,6 +229,11 @@ Class | Method | HTTP request | Description
  - [RoiCircleList](docs/RoiCircleList.md)
  - [RoiDefinitions](docs/RoiDefinitions.md)
  - [RotationAxis](docs/RotationAxis.md)
+ - [RugnuxLiveStatus](docs/RugnuxLiveStatus.md)
+ - [RugnuxLiveStep](docs/RugnuxLiveStep.md)
+ - [RugnuxSettings](docs/RugnuxSettings.md)
+ - [RugnuxStatus](docs/RugnuxStatus.md)
+ - [RugnuxSummaryRow](docs/RugnuxSummaryRow.md)
  - [ScanResult](docs/ScanResult.md)
  - [ScanResultImagesInner](docs/ScanResultImagesInner.md)
  - [SpotFindingSettings](docs/SpotFindingSettings.md)

docs/python_client/docs/DefaultApi.md

@@ -29,6 +29,10 @@ Method | HTTP request | Description
 [**config_mask_tiff_get**](DefaultApi.md#config_mask_tiff_get) | **GET** /config/mask.tiff | Get mask of the detector (TIFF)
 [**config_roi_get**](DefaultApi.md#config_roi_get) | **GET** /config/roi | Get ROI definitions
 [**config_roi_put**](DefaultApi.md#config_roi_put) | **PUT** /config/roi | Upload ROI definitions
+[**config_rugnux_get**](DefaultApi.md#config_rugnux_get) | **GET** /config/rugnux | Get rugnux live mode settings and model status
+[**config_rugnux_model_delete**](DefaultApi.md#config_rugnux_model_delete) | **DELETE** /config/rugnux/model | Remove the model used by rugnux live mode
+[**config_rugnux_model_put**](DefaultApi.md#config_rugnux_model_put) | **PUT** /config/rugnux/model | Upload the model used by rugnux live mode
+[**config_rugnux_put**](DefaultApi.md#config_rugnux_put) | **PUT** /config/rugnux | Change rugnux live mode settings
 [**config_select_detector_get**](DefaultApi.md#config_select_detector_get) | **GET** /config/select_detector | List available detectors
 [**config_select_detector_put**](DefaultApi.md#config_select_detector_put) | **PUT** /config/select_detector | Select detector
 [**config_spot_finding_get**](DefaultApi.md#config_spot_finding_get) | **GET** /config/spot_finding | Get data processing configuration
@@ -58,6 +62,7 @@ Method | HTTP request | Description
 [**preview_plot_bin_get**](DefaultApi.md#preview_plot_bin_get) | **GET** /preview/plot.bin | Generate 1D plot from Jungfraujoch and send in raw binary format. Data are provided as (32-bit) float binary array. This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. 
 [**preview_plot_get**](DefaultApi.md#preview_plot_get) | **GET** /preview/plot | Generate 1D plot from Jungfraujoch
 [**result_scan_get**](DefaultApi.md#result_scan_get) | **GET** /result/scan | Get full scan result
+[**rugnux_status_get**](DefaultApi.md#rugnux_status_get) | **GET** /rugnux/status | Get live rugnux processing status of the last collection
 [**start_post**](DefaultApi.md#start_post) | **POST** /start | Start detector
 [**statistics_calibration_get**](DefaultApi.md#statistics_calibration_get) | **GET** /statistics/calibration | Get calibration statistics
 [**statistics_data_collection_get**](DefaultApi.md#statistics_data_collection_get) | **GET** /statistics/data_collection | Get data collection statistics
@@ -80,6 +85,11 @@ Command will inform FPGA network card to stop pedestal or data collection at the
 Any frame that is currently being processed by CPU will be finished and sent to writer.
 Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state.
 
+With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection
+or after its last image alike: the collection ends with what the analysis has established
+(see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not
+marked as cancelled.
+
 If data collection is not running, the command has no effect.
 
 
@@ -1730,6 +1740,318 @@ No authorization required
 
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
+# **config_rugnux_get**
+> RugnuxStatus config_rugnux_get()
+
+Get rugnux live mode settings and model status
+
+Can be done anytime
+
+### Example
+
+* Bearer Authentication (bearerAuth):
+
+```python
+import jfjoch_client
+from jfjoch_client.models.rugnux_status import RugnuxStatus
+from jfjoch_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:5232
+# See configuration.py for a list of all supported configuration parameters.
+configuration = jfjoch_client.Configuration(
+    host = "http://localhost:5232"
+)
+
+# The client must configure the authentication and authorization parameters
+# in accordance with the API server security policy.
+# Examples for each auth method are provided below, use the example that
+# satisfies your auth use case.
+
+# Configure Bearer authorization: bearerAuth
+configuration = jfjoch_client.Configuration(
+    access_token = os.environ["BEARER_TOKEN"]
+)
+
+# Enter a context with an instance of the API client
+with jfjoch_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = jfjoch_client.DefaultApi(api_client)
+
+    try:
+        # Get rugnux live mode settings and model status
+        api_response = api_instance.config_rugnux_get()
+        print("The response of DefaultApi->config_rugnux_get:\n")
+        pprint(api_response)
+    except Exception as e:
+        print("Exception when calling DefaultApi->config_rugnux_get: %s\n" % e)
+```
+
+
+
+### Parameters
+
+This endpoint does not need any parameter.
+
+### Return type
+
+[**RugnuxStatus**](RugnuxStatus.md)
+
+### Authorization
+
+[bearerAuth](../README.md#bearerAuth)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: text/plain, application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**401** | The current dataset is protected and no valid bearer token was presented. |  -  |
+**200** | Everything OK |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
+# **config_rugnux_model_delete**
+> config_rugnux_model_delete()
+
+Remove the model used by rugnux live mode
+
+This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+
+
+### Example
+
+* Bearer Authentication (bearerAuth):
+
+```python
+import jfjoch_client
+from jfjoch_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:5232
+# See configuration.py for a list of all supported configuration parameters.
+configuration = jfjoch_client.Configuration(
+    host = "http://localhost:5232"
+)
+
+# The client must configure the authentication and authorization parameters
+# in accordance with the API server security policy.
+# Examples for each auth method are provided below, use the example that
+# satisfies your auth use case.
+
+# Configure Bearer authorization: bearerAuth
+configuration = jfjoch_client.Configuration(
+    access_token = os.environ["BEARER_TOKEN"]
+)
+
+# Enter a context with an instance of the API client
+with jfjoch_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = jfjoch_client.DefaultApi(api_client)
+
+    try:
+        # Remove the model used by rugnux live mode
+        api_instance.config_rugnux_model_delete()
+    except Exception as e:
+        print("Exception when calling DefaultApi->config_rugnux_model_delete: %s\n" % e)
+```
+
+
+
+### Parameters
+
+This endpoint does not need any parameter.
+
+### Return type
+
+void (empty response body)
+
+### Authorization
+
+[bearerAuth](../README.md#bearerAuth)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: text/plain, application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**401** | The current dataset is protected and no valid bearer token was presented. |  -  |
+**200** | All good |  -  |
+**500** | Error within Jungfraujoch code - see output message. |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
+# **config_rugnux_model_put**
+> config_rugnux_model_put(body=body)
+
+Upload the model used by rugnux live mode
+
+This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement
+and validation of every following rugnux live collection, until it is removed with
+`DELETE /config/rugnux/model` or replaced by another upload.
+
+
+### Example
+
+* Bearer Authentication (bearerAuth):
+
+```python
+import jfjoch_client
+from jfjoch_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:5232
+# See configuration.py for a list of all supported configuration parameters.
+configuration = jfjoch_client.Configuration(
+    host = "http://localhost:5232"
+)
+
+# The client must configure the authentication and authorization parameters
+# in accordance with the API server security policy.
+# Examples for each auth method are provided below, use the example that
+# satisfies your auth use case.
+
+# Configure Bearer authorization: bearerAuth
+configuration = jfjoch_client.Configuration(
+    access_token = os.environ["BEARER_TOKEN"]
+)
+
+# Enter a context with an instance of the API client
+with jfjoch_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = jfjoch_client.DefaultApi(api_client)
+    body = None # bytearray |  (optional)
+
+    try:
+        # Upload the model used by rugnux live mode
+        api_instance.config_rugnux_model_put(body=body)
+    except Exception as e:
+        print("Exception when calling DefaultApi->config_rugnux_model_put: %s\n" % e)
+```
+
+
+
+### Parameters
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **body** | **bytearray**|  | [optional] 
+
+### Return type
+
+void (empty response body)
+
+### Authorization
+
+[bearerAuth](../README.md#bearerAuth)
+
+### HTTP request headers
+
+ - **Content-Type**: application/octet-stream
+ - **Accept**: text/plain, application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**401** | The current dataset is protected and no valid bearer token was presented. |  -  |
+**200** | All good |  -  |
+**400** | Empty body, or not a model that can be read (PDB or mmCIF, optionally gzipped) |  -  |
+**413** | Model larger than 64 MiB |  -  |
+**500** | Error within Jungfraujoch code - see output message. |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
+# **config_rugnux_put**
+> config_rugnux_put(rugnux_settings=rugnux_settings)
+
+Change rugnux live mode settings
+
+This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+
+
+### Example
+
+* Bearer Authentication (bearerAuth):
+
+```python
+import jfjoch_client
+from jfjoch_client.models.rugnux_settings import RugnuxSettings
+from jfjoch_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:5232
+# See configuration.py for a list of all supported configuration parameters.
+configuration = jfjoch_client.Configuration(
+    host = "http://localhost:5232"
+)
+
+# The client must configure the authentication and authorization parameters
+# in accordance with the API server security policy.
+# Examples for each auth method are provided below, use the example that
+# satisfies your auth use case.
+
+# Configure Bearer authorization: bearerAuth
+configuration = jfjoch_client.Configuration(
+    access_token = os.environ["BEARER_TOKEN"]
+)
+
+# Enter a context with an instance of the API client
+with jfjoch_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = jfjoch_client.DefaultApi(api_client)
+    rugnux_settings = jfjoch_client.RugnuxSettings() # RugnuxSettings |  (optional)
+
+    try:
+        # Change rugnux live mode settings
+        api_instance.config_rugnux_put(rugnux_settings=rugnux_settings)
+    except Exception as e:
+        print("Exception when calling DefaultApi->config_rugnux_put: %s\n" % e)
+```
+
+
+
+### Parameters
+
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+ **rugnux_settings** | [**RugnuxSettings**](RugnuxSettings.md)|  | [optional] 
+
+### Return type
+
+void (empty response body)
+
+### Authorization
+
+[bearerAuth](../README.md#bearerAuth)
+
+### HTTP request headers
+
+ - **Content-Type**: application/json
+ - **Accept**: text/plain, application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**401** | The current dataset is protected and no valid bearer token was presented. |  -  |
+**200** | Everything OK |  -  |
+**400** | Input parsing or validation error |  -  |
+**500** | Error within Jungfraujoch code - see output message. |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **config_select_detector_get**
 > DetectorList config_select_detector_get()
 
@@ -3793,6 +4115,81 @@ This endpoint does not need any parameter.
 
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
+# **rugnux_status_get**
+> RugnuxLiveStatus rugnux_status_get()
+
+Get live rugnux processing status of the last collection
+
+Can be done anytime
+
+### Example
+
+* Bearer Authentication (bearerAuth):
+
+```python
+import jfjoch_client
+from jfjoch_client.models.rugnux_live_status import RugnuxLiveStatus
+from jfjoch_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:5232
+# See configuration.py for a list of all supported configuration parameters.
+configuration = jfjoch_client.Configuration(
+    host = "http://localhost:5232"
+)
+
+# The client must configure the authentication and authorization parameters
+# in accordance with the API server security policy.
+# Examples for each auth method are provided below, use the example that
+# satisfies your auth use case.
+
+# Configure Bearer authorization: bearerAuth
+configuration = jfjoch_client.Configuration(
+    access_token = os.environ["BEARER_TOKEN"]
+)
+
+# Enter a context with an instance of the API client
+with jfjoch_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = jfjoch_client.DefaultApi(api_client)
+
+    try:
+        # Get live rugnux processing status of the last collection
+        api_response = api_instance.rugnux_status_get()
+        print("The response of DefaultApi->rugnux_status_get:\n")
+        pprint(api_response)
+    except Exception as e:
+        print("Exception when calling DefaultApi->rugnux_status_get: %s\n" % e)
+```
+
+
+
+### Parameters
+
+This endpoint does not need any parameter.
+
+### Return type
+
+[**RugnuxLiveStatus**](RugnuxLiveStatus.md)
+
+### Authorization
+
+[bearerAuth](../README.md#bearerAuth)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: text/plain, application/json
+
+### HTTP response details
+
+| Status code | Description | Response headers |
+|-------------|-------------|------------------|
+**401** | The current dataset is protected and no valid bearer token was presented. |  -  |
+**200** | Everything OK |  -  |
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **start_post**
 > start_post(dataset_settings=dataset_settings)
 

docs/python_client/docs/JfjochSettings.md

@@ -27,6 +27,7 @@ Name | Type | Description | Notes
 **zeromq_preview** | [**ZeromqPreviewSettings**](ZeromqPreviewSettings.md) |  | [optional] 
 **zeromq_metadata** | [**ZeromqMetadataSettings**](ZeromqMetadataSettings.md) |  | [optional] 
 **dark_mask** | [**DarkMaskSettings**](DarkMaskSettings.md) |  | [optional] 
+**rugnux** | [**RugnuxSettings**](RugnuxSettings.md) |  | [optional] 
 
 ## Example
 

docs/python_client/docs/RugnuxLiveStatus.md

@@ -0,0 +1,46 @@
+# RugnuxLiveStatus
+
+Live rugnux processing of the last collection; kept until the next collection starts
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**state** | **str** |  | 
+**probe** | **str** | Answer of the early lattice probe on the first 60 degrees; empty until it answers | 
+**probe_s** | **float** | When the probe answered, since the collection started | [optional] 
+**message** | **str** | Why the run failed or was skipped | 
+**images_expected** | **int** |  | 
+**images_received** | **int** |  | 
+**elapsed_s** | **float** |  | 
+**step_progress** | **float** | Fraction of the current pass's images | [optional] 
+**steps** | [**List[RugnuxLiveStep]**](RugnuxLiveStep.md) |  | 
+**last_image_s** | **float** | When the last image arrived (and the cutoff timer started), since the collection started | [optional] 
+**cut** | **str** | Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel` (`POST /cancel`). The end message then carried what was established at the cut.  | [optional] 
+**cut_s** | **float** | When the analysis was cut, since the collection started | [optional] 
+**last_image_to_written_s** | **float** | From the last image received to all results written | [optional] 
+**verdict** | **str** | OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over | 
+**verdict_text** | **str** |  | 
+**summary** | [**List[RugnuxSummaryRow]**](RugnuxSummaryRow.md) |  | 
+**warnings** | **List[str]** |  | 
+
+## Example
+
+```python
+from jfjoch_client.models.rugnux_live_status import RugnuxLiveStatus
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of RugnuxLiveStatus from a JSON string
+rugnux_live_status_instance = RugnuxLiveStatus.from_json(json)
+# print the JSON string representation of the object
+print(RugnuxLiveStatus.to_json())
+
+# convert the object into a dict
+rugnux_live_status_dict = rugnux_live_status_instance.to_dict()
+# create an instance of RugnuxLiveStatus from a dict
+rugnux_live_status_from_dict = RugnuxLiveStatus.from_dict(rugnux_live_status_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

docs/python_client/docs/RugnuxLiveStep.md

@@ -0,0 +1,32 @@
+# RugnuxLiveStep
+
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**title** | **str** |  | 
+**started_s** | **float** | Start of the step, since the collection started | 
+**duration_s** | **float** | Set once the next step starts or the run ends | [optional] 
+**notes** | **List[str]** | What the step found (e.g. lattice, space group) | 
+
+## Example
+
+```python
+from jfjoch_client.models.rugnux_live_step import RugnuxLiveStep
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of RugnuxLiveStep from a JSON string
+rugnux_live_step_instance = RugnuxLiveStep.from_json(json)
+# print the JSON string representation of the object
+print(RugnuxLiveStep.to_json())
+
+# convert the object into a dict
+rugnux_live_step_dict = rugnux_live_step_instance.to_dict()
+# create an instance of RugnuxLiveStep from a dict
+rugnux_live_step_from_dict = RugnuxLiveStep.from_dict(rugnux_live_step_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

docs/python_client/docs/RugnuxSettings.md

@@ -0,0 +1,32 @@
+# RugnuxSettings
+
+Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. 
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**enable** | **bool** | Process DECTRIS rotation collections with rugnux live | [optional] [default to False]
+**reserved_cpus** | **int** | CPU threads left to the receiver and the image pusher while rugnux runs | [optional] [default to 4]
+**deadline_s** | **float** | Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff.  | [optional] [default to 0.0]
+
+## Example
+
+```python
+from jfjoch_client.models.rugnux_settings import RugnuxSettings
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of RugnuxSettings from a JSON string
+rugnux_settings_instance = RugnuxSettings.from_json(json)
+# print the JSON string representation of the object
+print(RugnuxSettings.to_json())
+
+# convert the object into a dict
+rugnux_settings_dict = rugnux_settings_instance.to_dict()
+# create an instance of RugnuxSettings from a dict
+rugnux_settings_from_dict = RugnuxSettings.from_dict(rugnux_settings_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

docs/python_client/docs/RugnuxStatus.md

@@ -0,0 +1,33 @@
+# RugnuxStatus
+
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**enable** | **bool** | Rugnux live mode is enabled | 
+**reserved_cpus** | **int** | CPU threads left to the receiver and the image pusher while rugnux runs | 
+**deadline_s** | **float** | Seconds after the last image at which the analysis is cut; 0 means no cutoff | [optional] 
+**model_loaded** | **bool** | A model was uploaded with `PUT /config/rugnux/model` | 
+**model_size_bytes** | **int** | Size of the uploaded model file | [optional] 
+
+## Example
+
+```python
+from jfjoch_client.models.rugnux_status import RugnuxStatus
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of RugnuxStatus from a JSON string
+rugnux_status_instance = RugnuxStatus.from_json(json)
+# print the JSON string representation of the object
+print(RugnuxStatus.to_json())
+
+# convert the object into a dict
+rugnux_status_dict = rugnux_status_instance.to_dict()
+# create an instance of RugnuxStatus from a dict
+rugnux_status_from_dict = RugnuxStatus.from_dict(rugnux_status_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

docs/python_client/docs/RugnuxSummaryRow.md

@@ -0,0 +1,30 @@
+# RugnuxSummaryRow
+
+
+## Properties
+
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**label** | **str** |  | 
+**value** | **str** |  | 
+
+## Example
+
+```python
+from jfjoch_client.models.rugnux_summary_row import RugnuxSummaryRow
+
+# TODO update the JSON string below
+json = "{}"
+# create an instance of RugnuxSummaryRow from a JSON string
+rugnux_summary_row_instance = RugnuxSummaryRow.from_json(json)
+# print the JSON string representation of the object
+print(RugnuxSummaryRow.to_json())
+
+# convert the object into a dict
+rugnux_summary_row_dict = rugnux_summary_row_instance.to_dict()
+# create an instance of RugnuxSummaryRow from a dict
+rugnux_summary_row_from_dict = RugnuxSummaryRow.from_dict(rugnux_summary_row_dict)
+```
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

frame_serialize/CBORStream2Deserializer.cpp

@@ -320,6 +320,19 @@ namespace {
         memcpy(v.data(), ptr, len);
     }
 
+    void GetCBORDoubleArray(CborValue &value, std::vector<double> &v) {
+        if (GetCBORTag(value) != TagDoubleLE)
+            throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag");
+
+        auto [ptr, len] = GetCBORByteString(value);
+
+        if (len % sizeof(double))
+            throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch");
+
+        v.resize(len / sizeof(double));
+        memcpy(v.data(), ptr, len);
+    }
+
     void GetCBORStringArray(CborValue &value, std::vector<std::string> &v) {
         if (!cbor_value_is_array(&value))
             throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected");
@@ -1490,6 +1503,125 @@ namespace {
         cborErr(cbor_value_leave_container(&value, &map_value));
     }
 
+    // A key this build does not know: a tagged item takes two advances (see ProcessEndMessageElement).
+    void SkipCBORValue(CborValue &value) {
+        if (cbor_value_is_tag(&value))
+            cbor_value_advance(&value);
+        cbor_value_advance(&value);
+    }
+
+    MtzTable::Dataset ProcessMtzDatasetElement(CborValue &value) {
+        MtzTable::Dataset d;
+        CborValue map_value;
+        cborErr(cbor_value_enter_container(&value, &map_value));
+        while (!cbor_value_at_end(&map_value)) {
+            auto key = GetCBORString(map_value);
+            if (key == "project_name") d.project_name = GetCBORString(map_value);
+            else if (key == "crystal_name") d.crystal_name = GetCBORString(map_value);
+            else if (key == "dataset_name") d.dataset_name = GetCBORString(map_value);
+            else if (key == "wavelength") d.wavelength_A = GetCBORFloat(map_value);
+            else SkipCBORValue(map_value);
+        }
+        cborErr(cbor_value_leave_container(&value, &map_value));
+        return d;
+    }
+
+    MtzTable::Column ProcessMtzColumnElement(CborValue &value) {
+        MtzTable::Column c;
+        CborValue map_value;
+        cborErr(cbor_value_enter_container(&value, &map_value));
+        while (!cbor_value_at_end(&map_value)) {
+            auto key = GetCBORString(map_value);
+            if (key == "label") c.label = GetCBORString(map_value);
+            else if (key == "type") c.type = GetCBORString(map_value).at(0);
+            else if (key == "dataset_id") c.dataset_id = static_cast<int32_t>(GetCBORInt(map_value));
+            else if (key == "values") GetCBORFloatArray(map_value, c.values);
+            else SkipCBORValue(map_value);
+        }
+        cborErr(cbor_value_leave_container(&value, &map_value));
+        return c;
+    }
+
+    MtzTable::Batch ProcessMtzBatchElement(CborValue &value) {
+        MtzTable::Batch b;
+        CborValue map_value;
+        cborErr(cbor_value_enter_container(&value, &map_value));
+        while (!cbor_value_at_end(&map_value)) {
+            auto key = GetCBORString(map_value);
+            if (key == "number") b.number = static_cast<int32_t>(GetCBORInt(map_value));
+            else if (key == "title") b.title = GetCBORString(map_value);
+            else if (key == "ints") GetCBORInt32Array(map_value, b.ints);
+            else if (key == "floats") GetCBORFloatArray(map_value, b.floats);
+            else if (key == "axes") GetCBORStringArray(map_value, b.axes);
+            else SkipCBORValue(map_value);
+        }
+        cborErr(cbor_value_leave_container(&value, &map_value));
+        return b;
+    }
+
+    MtzTable ProcessMtzTableElement(CborValue &value) {
+        MtzTable t;
+        CborValue map_value;
+        cborErr(cbor_value_enter_container(&value, &map_value));
+        while (!cbor_value_at_end(&map_value)) {
+            auto key = GetCBORString(map_value);
+            if (key == "title") t.title = GetCBORString(map_value);
+            else if (key == "unit_cell") t.unit_cell = ProcessUnitCellElement(map_value);
+            else if (key == "space_group_name") t.space_group_name = GetCBORString(map_value);
+            else if (key == "history") GetCBORStringArray(map_value, t.history);
+            else if (key == "sort_order") {
+                std::vector<int32_t> v;
+                GetCBORInt32Array(map_value, v);
+                if (v.size() != t.sort_order.size())
+                    throw JFJochException(JFJochExceptionCategory::CBORError, "MTZ sort order of wrong length");
+                std::copy(v.begin(), v.end(), t.sort_order.begin());
+            } else if (key == "datasets" || key == "columns" || key == "batches") {
+                CborValue array_value;
+                cborErr(cbor_value_enter_container(&map_value, &array_value));
+                while (!cbor_value_at_end(&array_value)) {
+                    if (key == "datasets") t.datasets.push_back(ProcessMtzDatasetElement(array_value));
+                    else if (key == "columns") t.columns.push_back(ProcessMtzColumnElement(array_value));
+                    else t.batches.push_back(ProcessMtzBatchElement(array_value));
+                }
+                cborErr(cbor_value_leave_container(&map_value, &array_value));
+            } else SkipCBORValue(map_value);
+        }
+        cborErr(cbor_value_leave_container(&value, &map_value));
+        return t;
+    }
+
+    PlacedModelMessage ProcessPlacedModelElement(CborValue &value) {
+        PlacedModelMessage m;
+        CborValue map_value;
+        cborErr(cbor_value_enter_container(&value, &map_value));
+        while (!cbor_value_at_end(&map_value)) {
+            auto key = GetCBORString(map_value);
+            if (key == "name") m.name = GetCBORString(map_value);
+            else if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value);
+            else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value);
+            else if (key == "model_space_group_name") m.model_space_group_name = GetCBORString(map_value);
+            else if (key == "model_number") GetCBORInt32Array(map_value, m.model_number);
+            else if (key == "chain") GetCBORStringArray(map_value, m.chain);
+            else if (key == "residue_name") GetCBORStringArray(map_value, m.residue_name);
+            else if (key == "seq_num") GetCBORInt32Array(map_value, m.seq_num);
+            else if (key == "icode") GetCBORUInt8Array(map_value, m.icode);
+            else if (key == "het_flag") GetCBORUInt8Array(map_value, m.het_flag);
+            else if (key == "atom_name") GetCBORStringArray(map_value, m.atom_name);
+            else if (key == "element") GetCBORStringArray(map_value, m.element);
+            else if (key == "altloc") GetCBORUInt8Array(map_value, m.altloc);
+            else if (key == "charge") GetCBORInt32Array(map_value, m.charge);
+            else if (key == "x") GetCBORDoubleArray(map_value, m.x);
+            else if (key == "y") GetCBORDoubleArray(map_value, m.y);
+            else if (key == "z") GetCBORDoubleArray(map_value, m.z);
+            else if (key == "b_iso") GetCBORFloatArray(map_value, m.b_iso);
+            else if (key == "occupancy") GetCBORFloatArray(map_value, m.occupancy);
+            else if (key == "aniso") GetCBORFloatArray(map_value, m.aniso);
+            else SkipCBORValue(map_value);
+        }
+        cborErr(cbor_value_leave_container(&value, &map_value));
+        return m;
+    }
+
     bool ProcessEndMessageElement(EndMessage &message, CborValue &value) {
         if (cbor_value_at_end(&value))
             return false;
@@ -1586,6 +1718,12 @@ namespace {
                 message.space_group_number = GetCBORUInt(value);
             else if (key == "pixel_sum")
                 GetCBORInt64Array(value, message.pixel_sum);
+            else if (key == "merged")
+                message.merged = ProcessMtzTableElement(value);
+            else if (key == "map_coefficients")
+                message.map_coefficients = ProcessMtzTableElement(value);
+            else if (key == "placed_model")
+                message.placed_model = ProcessPlacedModelElement(value);
             else if (key == "rotation_lattice") {
                 std::vector<float> tmp;
                 GetCBORFloatArray(value, tmp);

frame_serialize/CBORStream2Serializer.cpp

@@ -515,6 +515,88 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const UnitCell &val)
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 }
 
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<double>& v) {
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encode_tag(&encoder, TagDoubleLE));
+    cborErr(cbor_encode_byte_string(&encoder, reinterpret_cast<const uint8_t *>(v.data()), v.size() * sizeof(double)));
+}
+
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MtzTable &t) {
+    CborEncoder mapEncoder, arrayEncoder, elementEncoder;
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength));
+    CBOR_ENC(mapEncoder, "title", t.title);
+    CBOR_ENC(mapEncoder, "unit_cell", t.unit_cell);
+    CBOR_ENC(mapEncoder, "space_group_name", t.space_group_name);
+    CBOR_ENC(mapEncoder, "history", t.history);
+    CBOR_ENC(mapEncoder, "sort_order", std::vector<int32_t>(t.sort_order.begin(), t.sort_order.end()));
+
+    cborErr(cbor_encode_text_stringz(&mapEncoder, "datasets"));
+    cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.datasets.size()));
+    for (const auto &d : t.datasets) {
+        cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 4));
+        CBOR_ENC(elementEncoder, "project_name", d.project_name);
+        CBOR_ENC(elementEncoder, "crystal_name", d.crystal_name);
+        CBOR_ENC(elementEncoder, "dataset_name", d.dataset_name);
+        CBOR_ENC(elementEncoder, "wavelength", d.wavelength_A);
+        cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder));
+    }
+    cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder));
+
+    cborErr(cbor_encode_text_stringz(&mapEncoder, "columns"));
+    cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.columns.size()));
+    for (const auto &c : t.columns) {
+        cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 4));
+        CBOR_ENC(elementEncoder, "label", c.label);
+        CBOR_ENC(elementEncoder, "type", std::string(1, c.type));
+        CBOR_ENC(elementEncoder, "dataset_id", static_cast<int64_t>(c.dataset_id));
+        CBOR_ENC(elementEncoder, "values", c.values);
+        cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder));
+    }
+    cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder));
+
+    cborErr(cbor_encode_text_stringz(&mapEncoder, "batches"));
+    cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.batches.size()));
+    for (const auto &b : t.batches) {
+        cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 5));
+        CBOR_ENC(elementEncoder, "number", static_cast<int64_t>(b.number));
+        CBOR_ENC(elementEncoder, "title", b.title);
+        CBOR_ENC(elementEncoder, "ints", b.ints);
+        CBOR_ENC(elementEncoder, "floats", b.floats);
+        CBOR_ENC(elementEncoder, "axes", b.axes);
+        cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder));
+    }
+    cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder));
+    cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
+}
+
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const PlacedModelMessage &m) {
+    CborEncoder mapEncoder;
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength));
+    CBOR_ENC(mapEncoder, "name", m.name);
+    CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell);
+    CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name);
+    CBOR_ENC(mapEncoder, "model_space_group_name", m.model_space_group_name);
+    CBOR_ENC(mapEncoder, "model_number", m.model_number);
+    CBOR_ENC(mapEncoder, "chain", m.chain);
+    CBOR_ENC(mapEncoder, "residue_name", m.residue_name);
+    CBOR_ENC(mapEncoder, "seq_num", m.seq_num);
+    CBOR_ENC(mapEncoder, "icode", m.icode);
+    CBOR_ENC(mapEncoder, "het_flag", m.het_flag);
+    CBOR_ENC(mapEncoder, "atom_name", m.atom_name);
+    CBOR_ENC(mapEncoder, "element", m.element);
+    CBOR_ENC(mapEncoder, "altloc", m.altloc);
+    CBOR_ENC(mapEncoder, "charge", m.charge);
+    CBOR_ENC(mapEncoder, "x", m.x);
+    CBOR_ENC(mapEncoder, "y", m.y);
+    CBOR_ENC(mapEncoder, "z", m.z);
+    CBOR_ENC(mapEncoder, "b_iso", m.b_iso);
+    CBOR_ENC(mapEncoder, "occupancy", m.occupancy);
+    CBOR_ENC(mapEncoder, "aniso", m.aniso);
+    cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
+}
+
 // Add encoder for LatticeMessage
 inline void CBOR_ENC(CborEncoder &encoder, const char* key, const LatticeMessage &val) {
     CborEncoder mapEncoder;
@@ -857,6 +939,9 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
     // should take; the number is beside it for a reader written before the name existed.
     CBOR_ENC(mapEncoder, "space_group_name", message.space_group_name);
     CBOR_ENC(mapEncoder, "space_group_number", message.space_group_number);
+    CBOR_ENC(mapEncoder, "merged", message.merged);
+    CBOR_ENC(mapEncoder, "map_coefficients", message.map_coefficients);
+    CBOR_ENC(mapEncoder, "placed_model", message.placed_model);
 
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 

frontend/src/App.tsx

@@ -24,6 +24,7 @@ import MenuBookIcon from "@mui/icons-material/MenuBook";
 import HistoryIcon from "@mui/icons-material/History";
 import CodeIcon from "@mui/icons-material/Code";
 import GavelIcon from "@mui/icons-material/Gavel";
+import AutoGraphIcon from "@mui/icons-material/AutoGraph";
 import _ from "lodash";
 import Markdown from "react-markdown";
 import remarkGfm from "remark-gfm";
@@ -32,6 +33,8 @@ import StatusBar from "./components/StatusBar";
 import ErrorMessage from "./components/ErrorMessage";
 import DataProcessingPlots from "./components/DataProcessingPlots";
 import MeasurementStatistics from "./components/MeasurementStatistics";
+import RugnuxStatus from "./components/RugnuxStatus";
+import RugnuxPage from "./components/RugnuxPage";
 import DataProcessingSettings from "./components/DataProcessingSettings";
 import PreviewImage from "./components/PreviewImage";
 import DataCollection from "./components/DataCollection";
@@ -168,7 +171,10 @@ function App() {
                         <DataProcessingPlots height={550} type={plot_type.BKG_ESTIMATE}/>
                     </Grid>
                     <Grid item xs={12} lg={4}>
-                        <MeasurementStatistics s={s.measurement ?? {}}/>
+                        <Stack spacing={3}>
+                            <MeasurementStatistics s={s.measurement ?? {}}/>
+                            <RugnuxStatus/>
+                        </Stack>
                     </Grid>
                     <Grid item xs={12} lg={8}>
                         <DataProcessingPlots height={800} type={plot_type.INDEXING_RATE}/>
@@ -188,6 +194,9 @@ function App() {
             { id: 'data-collection', label: 'Data collection', icon: <VideocamIcon/>, render: () => (
                 <DataCollection frame_time_us={s.detector_settings?.frame_time_us ?? 500}/>
             )},
+            { id: 'rugnux', label: 'Rugnux live', icon: <AutoGraphIcon/>, render: () => (
+                <RugnuxPage measuring={isMeasuring || s.broker?.state === 'Busy'}/>
+            )},
         ]},
         { subheader: 'Expert', items: [
             { id: 'detector', label: 'Detector', icon: <PhotoCameraIcon/>, render: () => (

frontend/src/client/@tanstack/react-query.gen.ts

@@ -3,8 +3,8 @@
 import { type DefaultError, queryOptions, type UseMutationOptions } from '@tanstack/react-query';
 
 import { client } from '../client.gen';
-import { getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from '../sdk.gen';
-import type { getConfigAzimIntData, getConfigAzimIntResponse, getConfigBraggIntegrationData, getConfigBraggIntegrationResponse, getConfigDarkMaskData, getConfigDarkMaskResponse, getConfigDetectorData, getConfigDetectorResponse, getConfigFileWriterData, getConfigFileWriterResponse, getConfigImageFormatData, getConfigImageFormatResponse, getConfigIndexingData, getConfigIndexingResponse, getConfigInstrumentData, getConfigInstrumentResponse, getConfigMaskData, getConfigMaskResponse, getConfigMaskTiffData, getConfigMaskTiffResponse, getConfigRoiData, getConfigRoiResponse, getConfigSelectDetectorData, getConfigSelectDetectorResponse, getConfigSpotFindingData, getConfigSpotFindingResponse, getConfigUserMaskData, getConfigUserMaskResponse, getConfigUserMaskTiffData, getConfigUserMaskTiffResponse, getConfigZeromqMetadataData, getConfigZeromqMetadataResponse, getConfigZeromqPreviewData, getConfigZeromqPreviewResponse, getDetectorStatusData, getDetectorStatusError, getDetectorStatusResponse, getFpgaStatusData, getFpgaStatusResponse, getImageBufferImageCborData, getImageBufferImageCborError, getImageBufferImageCborResponse, getImageBufferImageJpegData, getImageBufferImageJpegError, getImageBufferImageJpegResponse, getImageBufferImageTiffData, getImageBufferImageTiffError, getImageBufferImageTiffResponse, getImageBufferStartCborData, getImageBufferStartCborError, getImageBufferStartCborResponse, getImageBufferStatusData, getImageBufferStatusError, getImageBufferStatusResponse, getImagePusherStatusData, getImagePusherStatusError, getImagePusherStatusResponse, getPreviewPedestalTiffData, getPreviewPedestalTiffResponse, getPreviewPlotBinData, getPreviewPlotBinError, getPreviewPlotBinResponse, getPreviewPlotData, getPreviewPlotError, getPreviewPlotResponse, getResultScanData, getResultScanError, getResultScanResponse, getStatisticsCalibrationData, getStatisticsCalibrationResponse, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionError, getStatisticsDataCollectionResponse, getStatisticsResponse, getStatusData, getStatusResponse, getVersionData, getVersionResponse, getXfelEventCodeData, getXfelEventCodeResponse, getXfelPulseIdData, getXfelPulseIdResponse, postCancelData, postConfigImageFormatConversionData, postConfigImageFormatConversionError, postConfigImageFormatRawData, postConfigImageFormatRawError, postDeactivateData, postDeactivateError, postImageBufferClearData, postImageBufferClearError, postInitializeData, postInitializeError, postPedestalData, postPedestalError, postStartData, postStartError, postTriggerData, postWaitTillDoneData, postWaitTillDoneError, postWaitUntilRunningData, postWaitUntilRunningError, putConfigAzimIntData, putConfigAzimIntError, putConfigBraggIntegrationData, putConfigBraggIntegrationError, putConfigDarkMaskData, putConfigDarkMaskError, putConfigDetectorData, putConfigDetectorError, putConfigFileWriterData, putConfigFileWriterError, putConfigImageFormatData, putConfigImageFormatError, putConfigIndexingData, putConfigIndexingError, putConfigInstrumentData, putConfigInstrumentError, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageError, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffError, putConfigRoiData, putConfigRoiError, putConfigSelectDetectorData, putConfigSelectDetectorError, putConfigSpotFindingData, putConfigSpotFindingError, putConfigUserMaskData, putConfigUserMaskError, putConfigUserMaskTiffData, putConfigUserMaskTiffError, putConfigZeromqMetadataData, putConfigZeromqMetadataError, putConfigZeromqPreviewData, putConfigZeromqPreviewError } from '../types.gen';
+import { deleteConfigRugnuxModel, getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigRugnux, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getRugnuxStatus, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigRugnux, putConfigRugnuxModel, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from '../sdk.gen';
+import type { deleteConfigRugnuxModelData, deleteConfigRugnuxModelError, getConfigAzimIntData, getConfigAzimIntResponse, getConfigBraggIntegrationData, getConfigBraggIntegrationResponse, getConfigDarkMaskData, getConfigDarkMaskResponse, getConfigDetectorData, getConfigDetectorResponse, getConfigFileWriterData, getConfigFileWriterResponse, getConfigImageFormatData, getConfigImageFormatResponse, getConfigIndexingData, getConfigIndexingResponse, getConfigInstrumentData, getConfigInstrumentResponse, getConfigMaskData, getConfigMaskResponse, getConfigMaskTiffData, getConfigMaskTiffResponse, getConfigRoiData, getConfigRoiResponse, getConfigRugnuxData, getConfigRugnuxError, getConfigRugnuxResponse, getConfigSelectDetectorData, getConfigSelectDetectorResponse, getConfigSpotFindingData, getConfigSpotFindingResponse, getConfigUserMaskData, getConfigUserMaskResponse, getConfigUserMaskTiffData, getConfigUserMaskTiffResponse, getConfigZeromqMetadataData, getConfigZeromqMetadataResponse, getConfigZeromqPreviewData, getConfigZeromqPreviewResponse, getDetectorStatusData, getDetectorStatusError, getDetectorStatusResponse, getFpgaStatusData, getFpgaStatusResponse, getImageBufferImageCborData, getImageBufferImageCborError, getImageBufferImageCborResponse, getImageBufferImageJpegData, getImageBufferImageJpegError, getImageBufferImageJpegResponse, getImageBufferImageTiffData, getImageBufferImageTiffError, getImageBufferImageTiffResponse, getImageBufferStartCborData, getImageBufferStartCborError, getImageBufferStartCborResponse, getImageBufferStatusData, getImageBufferStatusError, getImageBufferStatusResponse, getImagePusherStatusData, getImagePusherStatusError, getImagePusherStatusResponse, getPreviewPedestalTiffData, getPreviewPedestalTiffResponse, getPreviewPlotBinData, getPreviewPlotBinError, getPreviewPlotBinResponse, getPreviewPlotData, getPreviewPlotError, getPreviewPlotResponse, getResultScanData, getResultScanError, getResultScanResponse, getRugnuxStatusData, getRugnuxStatusError, getRugnuxStatusResponse, getStatisticsCalibrationData, getStatisticsCalibrationResponse, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionError, getStatisticsDataCollectionResponse, getStatisticsResponse, getStatusData, getStatusResponse, getVersionData, getVersionResponse, getXfelEventCodeData, getXfelEventCodeResponse, getXfelPulseIdData, getXfelPulseIdResponse, postCancelData, postConfigImageFormatConversionData, postConfigImageFormatConversionError, postConfigImageFormatRawData, postConfigImageFormatRawError, postDeactivateData, postDeactivateError, postImageBufferClearData, postImageBufferClearError, postInitializeData, postInitializeError, postPedestalData, postPedestalError, postStartData, postStartError, postTriggerData, postWaitTillDoneData, postWaitTillDoneError, postWaitUntilRunningData, postWaitUntilRunningError, putConfigAzimIntData, putConfigAzimIntError, putConfigBraggIntegrationData, putConfigBraggIntegrationError, putConfigDarkMaskData, putConfigDarkMaskError, putConfigDetectorData, putConfigDetectorError, putConfigFileWriterData, putConfigFileWriterError, putConfigImageFormatData, putConfigImageFormatError, putConfigIndexingData, putConfigIndexingError, putConfigInstrumentData, putConfigInstrumentError, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageError, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffError, putConfigRoiData, putConfigRoiError, putConfigRugnuxData, putConfigRugnuxError, putConfigRugnuxModelData, putConfigRugnuxModelError, putConfigSelectDetectorData, putConfigSelectDetectorError, putConfigSpotFindingData, putConfigSpotFindingError, putConfigUserMaskData, putConfigUserMaskError, putConfigUserMaskTiffData, putConfigUserMaskTiffError, putConfigZeromqMetadataData, putConfigZeromqMetadataError, putConfigZeromqPreviewData, putConfigZeromqPreviewError } from '../types.gen';
 
 /**
  * Initialize detector and data acquisition
@@ -156,6 +156,11 @@ export const postTriggerMutation = (options?: Partial<Options<postTriggerData>>)
  * Any frame that is currently being processed by CPU will be finished and sent to writer.
  * Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state.
  *
+ * With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection
+ * or after its last image alike: the collection ends with what the analysis has established
+ * (see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not
+ * marked as cancelled.
+ *
  * If data collection is not running, the command has no effect.
  *
  */
@@ -675,6 +680,109 @@ export const putConfigSelectDetectorMutation = (options?: Partial<Options<putCon
     return mutationOptions;
 };
 
+export const getConfigRugnuxQueryKey = (options?: Options<getConfigRugnuxData>) => createQueryKey('getConfigRugnux', options);
+
+/**
+ * Get rugnux live mode settings and model status
+ *
+ * Can be done anytime
+ */
+export const getConfigRugnuxOptions = (options?: Options<getConfigRugnuxData>) => queryOptions<getConfigRugnuxResponse, getConfigRugnuxError, getConfigRugnuxResponse, ReturnType<typeof getConfigRugnuxQueryKey>>({
+    queryFn: async ({ queryKey, signal }) => {
+        const { data } = await getConfigRugnux({
+            ...options,
+            ...queryKey[0],
+            signal,
+            throwOnError: true
+        });
+        return data;
+    },
+    queryKey: getConfigRugnuxQueryKey(options)
+});
+
+/**
+ * Change rugnux live mode settings
+ *
+ * This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+ *
+ */
+export const putConfigRugnuxMutation = (options?: Partial<Options<putConfigRugnuxData>>): UseMutationOptions<unknown, putConfigRugnuxError, Options<putConfigRugnuxData>> => {
+    const mutationOptions: UseMutationOptions<unknown, putConfigRugnuxError, Options<putConfigRugnuxData>> = {
+        mutationFn: async (fnOptions) => {
+            const { data } = await putConfigRugnux({
+                ...options,
+                ...fnOptions,
+                throwOnError: true
+            });
+            return data;
+        }
+    };
+    return mutationOptions;
+};
+
+/**
+ * Remove the model used by rugnux live mode
+ *
+ * This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+ *
+ */
+export const deleteConfigRugnuxModelMutation = (options?: Partial<Options<deleteConfigRugnuxModelData>>): UseMutationOptions<unknown, deleteConfigRugnuxModelError, Options<deleteConfigRugnuxModelData>> => {
+    const mutationOptions: UseMutationOptions<unknown, deleteConfigRugnuxModelError, Options<deleteConfigRugnuxModelData>> = {
+        mutationFn: async (fnOptions) => {
+            const { data } = await deleteConfigRugnuxModel({
+                ...options,
+                ...fnOptions,
+                throwOnError: true
+            });
+            return data;
+        }
+    };
+    return mutationOptions;
+};
+
+/**
+ * Upload the model used by rugnux live mode
+ *
+ * This can only be done when detector is `Idle`, `Error` or `Inactive` states.
+ * The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement
+ * and validation of every following rugnux live collection, until it is removed with
+ * `DELETE /config/rugnux/model` or replaced by another upload.
+ *
+ */
+export const putConfigRugnuxModelMutation = (options?: Partial<Options<putConfigRugnuxModelData>>): UseMutationOptions<unknown, putConfigRugnuxModelError, Options<putConfigRugnuxModelData>> => {
+    const mutationOptions: UseMutationOptions<unknown, putConfigRugnuxModelError, Options<putConfigRugnuxModelData>> = {
+        mutationFn: async (fnOptions) => {
+            const { data } = await putConfigRugnuxModel({
+                ...options,
+                ...fnOptions,
+                throwOnError: true
+            });
+            return data;
+        }
+    };
+    return mutationOptions;
+};
+
+export const getRugnuxStatusQueryKey = (options?: Options<getRugnuxStatusData>) => createQueryKey('getRugnuxStatus', options);
+
+/**
+ * Get live rugnux processing status of the last collection
+ *
+ * Can be done anytime
+ */
+export const getRugnuxStatusOptions = (options?: Options<getRugnuxStatusData>) => queryOptions<getRugnuxStatusResponse, getRugnuxStatusError, getRugnuxStatusResponse, ReturnType<typeof getRugnuxStatusQueryKey>>({
+    queryFn: async ({ queryKey, signal }) => {
+        const { data } = await getRugnuxStatus({
+            ...options,
+            ...queryKey[0],
+            signal,
+            throwOnError: true
+        });
+        return data;
+    },
+    queryKey: getRugnuxStatusQueryKey(options)
+});
+
 export const getConfigZeromqPreviewQueryKey = (options?: Options<getConfigZeromqPreviewData>) => createQueryKey('getConfigZeromqPreview', options);
 
 /**

frontend/src/client/types.gen.ts

@@ -1873,6 +1873,125 @@ export type jfjoch_settings = {
     zeromq_preview?: zeromq_preview_settings;
     zeromq_metadata?: zeromq_metadata_settings;
     dark_mask?: dark_mask_settings;
+    rugnux?: rugnux_settings;
+};
+
+/**
+ * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux
+ * inside the broker while they arrive, and the merged result is sent to the writer.
+ *
+ */
+export type rugnux_settings = {
+    /**
+     * Process DECTRIS rotation collections with rugnux live
+     */
+    enable?: boolean;
+    /**
+     * CPU threads left to the receiver and the image pusher while rugnux runs
+     */
+    reserved_cpus?: number;
+    /**
+     * Fixed cutoff: this long after the last image the analysis is cut, and the collection ends
+     * with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff.
+     *
+     */
+    deadline_s?: number;
+};
+
+export type rugnux_status = {
+    /**
+     * Rugnux live mode is enabled
+     */
+    enable: boolean;
+    /**
+     * CPU threads left to the receiver and the image pusher while rugnux runs
+     */
+    reserved_cpus: number;
+    /**
+     * Seconds after the last image at which the analysis is cut; 0 means no cutoff
+     */
+    deadline_s?: number;
+    /**
+     * A model was uploaded with `PUT /config/rugnux/model`
+     */
+    model_loaded: boolean;
+    /**
+     * Size of the uploaded model file
+     */
+    model_size_bytes?: number;
+};
+
+export type rugnux_live_step = {
+    title: string;
+    /**
+     * Start of the step, since the collection started
+     */
+    started_s: number;
+    /**
+     * Set once the next step starts or the run ends
+     */
+    duration_s?: number;
+    /**
+     * What the step found (e.g. lattice, space group)
+     */
+    notes: Array<string>;
+};
+
+export type rugnux_summary_row = {
+    label: string;
+    value: string;
+};
+
+/**
+ * Live rugnux processing of the last collection; kept until the next collection starts
+ */
+export type rugnux_live_status = {
+    state: 'Idle' | 'Running' | 'Done' | 'Failed' | 'Cancelled' | 'Skipped';
+    /**
+     * Answer of the early lattice probe on the first 60 degrees; empty until it answers
+     */
+    probe: string;
+    /**
+     * When the probe answered, since the collection started
+     */
+    probe_s?: number;
+    /**
+     * Why the run failed or was skipped
+     */
+    message: string;
+    images_expected: number;
+    images_received: number;
+    elapsed_s: number;
+    /**
+     * Fraction of the current pass's images
+     */
+    step_progress?: number;
+    steps: Array<rugnux_live_step>;
+    /**
+     * When the last image arrived (and the cutoff timer started), since the collection started
+     */
+    last_image_s?: number;
+    /**
+     * Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel`
+     * (`POST /cancel`). The end message then carried what was established at the cut.
+     *
+     */
+    cut?: string;
+    /**
+     * When the analysis was cut, since the collection started
+     */
+    cut_s?: number;
+    /**
+     * From the last image received to all results written
+     */
+    last_image_to_written_s?: number;
+    /**
+     * OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over
+     */
+    verdict: string;
+    verdict_text: string;
+    summary: Array<rugnux_summary_row>;
+    warnings: Array<string>;
 };
 
 export type zeromq_metadata_settings = {
@@ -3060,6 +3179,149 @@ export type putConfigSelectDetectorResponses = {
     200: unknown;
 };
 
+export type getConfigRugnuxData = {
+    body?: never;
+    path?: never;
+    query?: never;
+    url: '/config/rugnux';
+};
+
+export type getConfigRugnuxErrors = {
+    /**
+     * The current dataset is protected and no valid bearer token was presented.
+     */
+    401: string;
+};
+
+export type getConfigRugnuxError = getConfigRugnuxErrors[keyof getConfigRugnuxErrors];
+
+export type getConfigRugnuxResponses = {
+    /**
+     * Everything OK
+     */
+    200: rugnux_status;
+};
+
+export type getConfigRugnuxResponse = getConfigRugnuxResponses[keyof getConfigRugnuxResponses];
+
+export type putConfigRugnuxData = {
+    body?: rugnux_settings;
+    path?: never;
+    query?: never;
+    url: '/config/rugnux';
+};
+
+export type putConfigRugnuxErrors = {
+    /**
+     * Exception error
+     */
+    400: string;
+    /**
+     * The current dataset is protected and no valid bearer token was presented.
+     */
+    401: string;
+    /**
+     * Error within Jungfraujoch code - see output message.
+     */
+    500: error_message;
+};
+
+export type putConfigRugnuxError = putConfigRugnuxErrors[keyof putConfigRugnuxErrors];
+
+export type putConfigRugnuxResponses = {
+    /**
+     * Everything OK
+     */
+    200: unknown;
+};
+
+export type deleteConfigRugnuxModelData = {
+    body?: never;
+    path?: never;
+    query?: never;
+    url: '/config/rugnux/model';
+};
+
+export type deleteConfigRugnuxModelErrors = {
+    /**
+     * The current dataset is protected and no valid bearer token was presented.
+     */
+    401: string;
+    /**
+     * Error within Jungfraujoch code - see output message.
+     */
+    500: error_message;
+};
+
+export type deleteConfigRugnuxModelError = deleteConfigRugnuxModelErrors[keyof deleteConfigRugnuxModelErrors];
+
+export type deleteConfigRugnuxModelResponses = {
+    /**
+     * All good
+     */
+    200: unknown;
+};
+
+export type putConfigRugnuxModelData = {
+    body?: Blob | File;
+    path?: never;
+    query?: never;
+    url: '/config/rugnux/model';
+};
+
+export type putConfigRugnuxModelErrors = {
+    /**
+     * Empty body, or not a model that can be read (PDB or mmCIF, optionally gzipped)
+     */
+    400: unknown;
+    /**
+     * The current dataset is protected and no valid bearer token was presented.
+     */
+    401: string;
+    /**
+     * Model larger than 64 MiB
+     */
+    413: unknown;
+    /**
+     * Error within Jungfraujoch code - see output message.
+     */
+    500: error_message;
+};
+
+export type putConfigRugnuxModelError = putConfigRugnuxModelErrors[keyof putConfigRugnuxModelErrors];
+
+export type putConfigRugnuxModelResponses = {
+    /**
+     * All good
+     */
+    200: unknown;
+};
+
+export type getRugnuxStatusData = {
+    body?: never;
+    path?: never;
+    query?: never;
+    url: '/rugnux/status';
+};
+
+export type getRugnuxStatusErrors = {
+    /**
+     * The current dataset is protected and no valid bearer token was presented.
+     */
+    401: string;
+};
+
+export type getRugnuxStatusError = getRugnuxStatusErrors[keyof getRugnuxStatusErrors];
+
+export type getRugnuxStatusResponses = {
+    /**
+     * Everything OK
+     */
+    200: rugnux_live_status;
+};
+
+export type getRugnuxStatusResponse = getRugnuxStatusResponses[keyof getRugnuxStatusResponses];
+
 export type getConfigZeromqPreviewData = {
     body?: never;
     path?: never;

frontend/src/client/zod.gen.ts

@@ -792,6 +792,67 @@ export const zBraggIntegrationSettings = z.object({
     max_hkl: z.int().gte(1).lte(511).optional().default(100)
 });
 
+/**
+ * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux
+ * inside the broker while they arrive, and the merged result is sent to the writer.
+ *
+ */
+export const zRugnuxSettings = z.object({
+    enable: z.boolean().optional().default(false),
+    reserved_cpus: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(4)),
+    deadline_s: z.number().gte(0).optional().default(0)
+});
+
+export const zRugnuxStatus = z.object({
+    enable: z.boolean(),
+    reserved_cpus: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }),
+    deadline_s: z.number().optional(),
+    model_loaded: z.boolean(),
+    model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional()
+});
+
+export const zRugnuxLiveStep = z.object({
+    title: z.string(),
+    started_s: z.number(),
+    duration_s: z.number().optional(),
+    notes: z.array(z.string())
+});
+
+export const zRugnuxSummaryRow = z.object({
+    label: z.string(),
+    value: z.string()
+});
+
+/**
+ * Live rugnux processing of the last collection; kept until the next collection starts
+ */
+export const zRugnuxLiveStatus = z.object({
+    state: z.enum([
+        'Idle',
+        'Running',
+        'Done',
+        'Failed',
+        'Cancelled',
+        'Skipped'
+    ]),
+    probe: z.string(),
+    probe_s: z.number().optional(),
+    message: z.string(),
+    images_expected: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }),
+    images_received: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }),
+    elapsed_s: z.number(),
+    step_progress: z.number().gte(0).lte(1).optional(),
+    steps: z.array(zRugnuxLiveStep),
+    last_image_s: z.number().optional(),
+    cut: z.string().optional(),
+    cut_s: z.number().optional(),
+    last_image_to_written_s: z.number().optional(),
+    verdict: z.string(),
+    verdict_text: z.string(),
+    summary: z.array(zRugnuxSummaryRow),
+    warnings: z.array(z.string())
+});
+
 export const zZeromqMetadataSettings = z.object({
     enabled: z.boolean().default(true),
     period_ms: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).default(BigInt(1000)),
@@ -857,7 +918,8 @@ export const zJfjochSettings = z.object({
     image_pusher: zImagePusherType,
     zeromq_preview: zZeromqPreviewSettings.optional(),
     zeromq_metadata: zZeromqMetadataSettings.optional(),
-    dark_mask: zDarkMaskSettings.optional()
+    dark_mask: zDarkMaskSettings.optional(),
+    rugnux: zRugnuxSettings.optional()
 });
 
 export const zDatasetSettingsWritable = z.object({
@@ -1186,6 +1248,20 @@ export const zGetConfigSelectDetectorResponse = zDetectorList;
 
 export const zPutConfigSelectDetectorBody = zDetectorSelection;
 
+/**
+ * Everything OK
+ */
+export const zGetConfigRugnuxResponse = zRugnuxStatus;
+
+export const zPutConfigRugnuxBody = zRugnuxSettings;
+
+export const zPutConfigRugnuxModelBody = z.string();
+
+/**
+ * Everything OK
+ */
+export const zGetRugnuxStatusResponse = zRugnuxLiveStatus;
+
 /**
  * Everything OK
  */

frontend/src/components/RugnuxPage.tsx

@@ -0,0 +1,171 @@
+import {Alert, Box, Chip, LinearProgress, Paper, Stack, Typography} from "@mui/material";
+import {indigo, lime, deepOrange, grey} from "@mui/material/colors";
+import {useQuery} from "@tanstack/react-query";
+import {getConfigRugnuxOptions, getRugnuxStatusOptions} from "../client/@tanstack/react-query.gen";
+import {rugnux_live_status} from "../client";
+import {useTokenRequired} from "../auth";
+import EmptyState from "./EmptyState";
+import RugnuxTimeline from "./RugnuxTimeline";
+import RugnuxSettings from "./RugnuxSettings";
+import {CutAlert, lastImageTime, ProbeAlert, StepTable, VerdictAlert} from "./RugnuxParts";
+import BoltIcon from "@mui/icons-material/Bolt";
+
+const STATE_COLOUR: Record<string, { bg: string, fg: string }> = {
+    Running: {bg: indigo[500], fg: "#fff"},
+    Done: {bg: lime[600], fg: "rgba(0,0,0,.87)"},
+    Failed: {bg: deepOrange[700], fg: "#fff"},
+    Cancelled: {bg: grey[500], fg: "#fff"},
+    Skipped: {bg: grey[400], fg: "rgba(0,0,0,.87)"},
+};
+
+// The summary rows rugnux writes; the ones a crystallographer looks at first get a stronger tile.
+const KEY_ROW = /cell|space group|resolution|cc1\/2|r-?free|model/i;
+
+function Section({title, children}: { title: string, children: React.ReactNode }) {
+    return (
+        <Paper sx={{p: 3, height: "100%"}}>
+            <Typography sx={{fontSize: "0.8rem", fontWeight: 700, letterSpacing: ".08em", textTransform: "uppercase",
+                color: "primary.dark", mb: 2}}>
+                {title}
+            </Typography>
+            {children}
+        </Paper>
+    );
+}
+
+function Header({s}: { s: rugnux_live_status }) {
+    const colour = STATE_COLOUR[s.state] ?? STATE_COLOUR.Skipped;
+    const latency = s.last_image_to_written_s;
+    const lastImage = lastImageTime(s);
+    return (
+        <Paper sx={{p: 3, display: "flex", alignItems: "center", gap: 3, flexWrap: "wrap",
+            background: `linear-gradient(90deg, ${indigo[50]} 0%, #fff 70%)`}}>
+            <Box sx={{flexGrow: 1, minWidth: 280}}>
+                <Stack direction="row" spacing={2} alignItems="center">
+                    <Typography variant="h4" sx={{fontWeight: 700, color: "primary.dark"}}>Rugnux live</Typography>
+                    <Chip label={s.state} sx={{bgcolor: colour.bg, color: colour.fg, fontWeight: 700,
+                        fontSize: "1rem", height: 34, px: 1}}/>
+                </Stack>
+                <Typography sx={{mt: 1, color: "text.secondary", fontSize: "1.1rem"}}>
+                    {s.images_received} / {s.images_expected} images
+                    {lastImage !== undefined && ` · collected in ${lastImage.toFixed(1)} s`}
+                    {` · ${s.elapsed_s.toFixed(1)} s since the first image`}
+                </Typography>
+                {s.message && <Typography sx={{mt: 0.5, color: "error.main"}}>{s.message}</Typography>}
+            </Box>
+            <Box sx={{textAlign: "right"}}>
+                <Typography sx={{fontSize: "0.85rem", fontWeight: 700, letterSpacing: ".08em",
+                    textTransform: "uppercase", color: "text.secondary"}}>
+                    {latency !== undefined ? "Last image → all results written" : "Processing for"}
+                </Typography>
+                <Typography sx={{fontSize: "4.5rem", lineHeight: 1.05, fontWeight: 800,
+                    color: latency === undefined ? indigo[300] : s.state === 'Done' ? lime[900] : indigo[700]}}>
+                    {(latency ?? s.elapsed_s).toFixed(1)} s
+                </Typography>
+            </Box>
+        </Paper>
+    );
+}
+
+function CurrentStep({s}: { s: rugnux_live_status }) {
+    const current = s.state === 'Running' && s.steps.length > 0 ? s.steps[s.steps.length - 1] : undefined;
+    if (!current)
+        return null;
+    return (
+        <Box sx={{mb: 2}}>
+            <Stack direction="row" justifyContent="space-between" alignItems="baseline">
+                <Typography sx={{fontWeight: 700, fontSize: "1.25rem"}}>{current.title}</Typography>
+                {s.step_progress !== undefined &&
+                    <Typography sx={{fontWeight: 700, color: "primary.main"}}>{(s.step_progress * 100).toFixed(0)} %</Typography>}
+            </Stack>
+            <LinearProgress variant={s.step_progress !== undefined ? "determinate" : "indeterminate"}
+                            value={(s.step_progress ?? 0) * 100} sx={{height: 10, borderRadius: 5, mt: 1}}/>
+        </Box>
+    );
+}
+
+function SummaryTiles({s}: { s: rugnux_live_status }) {
+    if (s.summary.length === 0)
+        return null;
+    return (
+        <Box sx={{display: "grid", gridTemplateColumns: {xs: "1fr", sm: "1fr 1fr"}, gap: 1.5, mt: 2}}>
+            {s.summary.map((row, i) => {
+                const key = KEY_ROW.test(row.label) || KEY_ROW.test(row.value);
+                const short = row.value.length <= 24;
+                return (
+                    <Box key={i} sx={{p: 1.5, borderRadius: 2, gridColumn: key ? "auto" : "1 / -1",
+                        bgcolor: key ? indigo[50] : grey[50],
+                        border: "1px solid", borderColor: key ? indigo[100] : grey[200]}}>
+                        <Typography sx={{fontSize: "0.75rem", fontWeight: 700, letterSpacing: ".06em",
+                            textTransform: "uppercase", color: key ? "primary.main" : "text.secondary"}}>
+                            {row.label}
+                        </Typography>
+                        <Typography sx={{fontSize: key && short ? "1.2rem" : "0.95rem", fontWeight: key ? 700 : 400,
+                            color: key ? "primary.dark" : "text.primary", wordBreak: "break-word"}}>
+                            {row.value}
+                        </Typography>
+                    </Box>
+                );
+            })}
+        </Box>
+    );
+}
+
+function RugnuxPage({measuring}: { measuring: boolean }) {
+    const {data: settings} = useQuery({...getConfigRugnuxOptions(), refetchInterval: 2000});
+    const enabled = settings?.enable === true;
+    const {data: s, isError} = useQuery({
+        ...getRugnuxStatusOptions(),
+        refetchInterval: 1000,
+        enabled,
+    });
+    const tokenRequired = useTokenRequired();
+
+    let live;
+    if (settings !== undefined && !enabled)
+        live = <Paper><EmptyState icon={<BoltIcon fontSize="inherit"/>}
+                                  text="Rugnux live mode is off - enable it below"/></Paper>;
+    else if (isError && tokenRequired)
+        live = <Paper><EmptyState icon={<BoltIcon fontSize="inherit"/>}
+                                  text="Dataset token required (key button in the top bar)"/></Paper>;
+    else if (isError || s === undefined || s.state === 'Idle')
+        live = <Paper><EmptyState icon={<BoltIcon fontSize="inherit"/>}
+                                  text="No rugnux run yet - start a rotation collection with live processing enabled"/></Paper>;
+    else
+        live = (
+            <Stack spacing={2}>
+                <Header s={s}/>
+                <CutAlert s={s}/>
+                <ProbeAlert s={s} large/>
+                <Section title="Timeline">
+                    <RugnuxTimeline s={s} deadline_s={settings?.deadline_s ?? 0}/>
+                </Section>
+                <Box sx={{display: "grid", gridTemplateColumns: {xs: "1fr", lg: "1fr 1fr"}, gap: 2}}>
+                    <Section title="Processing">
+                        <CurrentStep s={s}/>
+                        <StepTable steps={s.steps}/>
+                    </Section>
+                    <Section title="Result">
+                        {s.verdict
+                            ? <VerdictAlert s={s}/>
+                            : <Typography color="text.secondary">
+                                {s.state === 'Running' ? "Results appear when the run is over." : "No result."}
+                              </Typography>}
+                        <SummaryTiles s={s}/>
+                        <Stack spacing={1} sx={{mt: s.warnings.length > 0 ? 2 : 0}}>
+                            {s.warnings.map((w, i) => <Alert key={i} severity="warning">{w}</Alert>)}
+                        </Stack>
+                    </Section>
+                </Box>
+            </Stack>
+        );
+
+    return (
+        <Stack spacing={2}>
+            {live}
+            <RugnuxSettings measuring={measuring}/>
+        </Stack>
+    );
+}
+
+export default RugnuxPage;

frontend/src/components/RugnuxParts.tsx

@@ -0,0 +1,88 @@
+import {Alert, AlertTitle, Box, Paper, Table, TableBody, TableCell, TableContainer, TableRow, Typography} from "@mui/material";
+import {rugnux_live_status, rugnux_live_step} from "../client";
+
+// Pieces of the rugnux live status shared by the dashboard card and the Rugnux page.
+
+export const verdictSeverity = (verdict: string) => {
+    switch (verdict) {
+        case "OK":
+            return "success";
+        case "WARNINGS":
+            return "warning";
+        default:
+            return "error";
+    }
+};
+
+// When the last image arrived, in seconds since the collection started. Older brokers do not report
+// it; there it follows from the latency (every result written `latency` after the last image).
+export const lastImageTime = (s: rugnux_live_status) =>
+    s.last_image_s ?? (s.last_image_to_written_s !== undefined ? s.elapsed_s - s.last_image_to_written_s : undefined);
+
+export function CutAlert({s}: { s: rugnux_live_status }) {
+    if (!s.cut)
+        return null;
+    return (
+        <Alert severity="warning" variant="filled" sx={{width: "100%", textAlign: "left", fontSize: "1.05rem"}}>
+            Analysis cut ({s.cut}){s.cut_s !== undefined && ` at ${s.cut_s.toFixed(1)} s`} - results as established
+        </Alert>
+    );
+}
+
+export function ProbeAlert({s, large}: { s: rugnux_live_status, large?: boolean }) {
+    if (!s.probe)
+        return null;
+    const noLattice = s.probe.startsWith("no lattice");
+    return (
+        <Alert severity={noLattice ? "warning" : "info"} variant={large ? "filled" : "standard"}
+               sx={{width: "100%", textAlign: "left", ...(large && {fontSize: "1.15rem", alignItems: "center"})}}>
+            {large && <AlertTitle sx={{mb: 0, fontSize: "0.8rem", textTransform: "uppercase", letterSpacing: ".06em"}}>
+                Lattice probe{s.probe_s !== undefined && ` · ${s.probe_s.toFixed(1)} s after the first image`}
+            </AlertTitle>}
+            {s.probe}{!large && s.probe_s !== undefined && ` (at ${s.probe_s.toFixed(1)} s)`}
+        </Alert>
+    );
+}
+
+export function VerdictAlert({s}: { s: rugnux_live_status }) {
+    if (!s.verdict)
+        return null;
+    return (
+        <Alert severity={verdictSeverity(s.verdict)} sx={{width: "100%", textAlign: "left"}}>
+            <AlertTitle sx={{fontWeight: 700}}>{s.verdict}</AlertTitle>
+            {s.verdict_text}
+        </Alert>
+    );
+}
+
+const stepDuration = (step: rugnux_live_step) =>
+    step.duration_s !== undefined ? step.duration_s.toFixed(1) + " s" : "running";
+
+export function StepTable({steps}: { steps: rugnux_live_step[] }) {
+    if (steps.length === 0)
+        return null;
+    return (
+        <TableContainer component={Paper} variant="outlined">
+            <Table size="small">
+                <TableBody>
+                    {steps.map((step, i) => (
+                        <TableRow key={i} sx={step.duration_s === undefined ? {bgcolor: "action.hover"} : undefined}>
+                            <TableCell sx={{verticalAlign: "top", color: "text.secondary", width: 72, whiteSpace: "nowrap"}}>
+                                {step.started_s.toFixed(1)} s
+                            </TableCell>
+                            <TableCell sx={{verticalAlign: "top"}}>
+                                <Typography variant="body2" sx={{fontWeight: 600}}>{step.title}</Typography>
+                                {step.notes.map((note, j) =>
+                                    <Box key={j} sx={{color: "text.secondary", fontSize: "0.85rem"}}>{note}</Box>)}
+                            </TableCell>
+                            <TableCell align="right" sx={{verticalAlign: "top", whiteSpace: "nowrap",
+                                fontWeight: step.duration_s === undefined ? 700 : 400}}>
+                                {stepDuration(step)}
+                            </TableCell>
+                        </TableRow>
+                    ))}
+                </TableBody>
+            </Table>
+        </TableContainer>
+    );
+}

frontend/src/components/RugnuxSettings.tsx

@@ -0,0 +1,94 @@
+import {ChangeEvent, useEffect, useState} from 'react';
+
+import {Button, FormControlLabel, Stack, Switch, Typography} from "@mui/material";
+import UploadFileIcon from "@mui/icons-material/UploadFile";
+import DeleteIcon from "@mui/icons-material/Delete";
+import {useQuery, useQueryClient} from "@tanstack/react-query";
+import {
+    deleteConfigRugnuxModelMutation, getConfigRugnuxOptions, getConfigRugnuxQueryKey,
+    putConfigRugnuxModelMutation, putConfigRugnuxMutation
+} from "../client/@tanstack/react-query.gen";
+import NumberTextField from "./NumberTextField";
+import SettingsPanel, {UploadSnackbarView} from "./SettingsPanel";
+import {useUpload} from "./useUpload";
+
+const sizeText = (bytes?: number) =>
+    bytes === undefined ? "" : bytes >= 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)} MiB` : `${(bytes / 1024).toFixed(0)} KiB`;
+
+// Rugnux live mode (enable, reserved CPUs) and the model used for MR, as two panels side by side.
+function RugnuxSettings({measuring}: { measuring: boolean }) {
+    const queryClient = useQueryClient();
+    const {data: s} = useQuery({...getConfigRugnuxOptions(), refetchInterval: 2000});
+    const refresh = {onSuccess: () => queryClient.invalidateQueries({queryKey: getConfigRugnuxQueryKey()})};
+
+    const settings = useUpload({...putConfigRugnuxMutation(), ...refresh});
+    const upload = useUpload({...putConfigRugnuxModelMutation(), ...refresh});
+    const remove = useUpload({...deleteConfigRugnuxModelMutation(), ...refresh});
+
+    const [enable, setEnable] = useState(false);
+    const [cpus, setCpus] = useState(4);
+    const [cpusError, setCpusError] = useState(false);
+    const [deadline, setDeadline] = useState(0);
+    const [deadlineError, setDeadlineError] = useState(false);
+    const [counter, setCounter] = useState(0);
+    const [file, setFile] = useState<File>();
+
+    useEffect(() => {
+        if (s !== undefined) {
+            setEnable(s.enable);
+            setCpus(s.reserved_cpus);
+            setDeadline(s.deadline_s ?? 0);
+            setCounter(c => c + 1);
+        }
+    }, [s?.enable, s?.reserved_cpus, s?.deadline_s]);
+
+    const dirty = s !== undefined && (enable !== s.enable || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0));
+
+    return (
+        <Stack direction={{xs: "column", md: "row"}} spacing={2} alignItems="stretch">
+            <SettingsPanel title="Live processing" dirty={dirty} spacing={2}
+                           onUpload={() => settings.submit({enable, reserved_cpus: cpus, deadline_s: deadline})}
+                           uploadDisabled={measuring || cpusError || deadlineError || settings.pending}
+                           snackbar={settings.snackbar}>
+                <FormControlLabel label="Process DECTRIS rotation collections with rugnux"
+                                  control={<Switch checked={enable} disabled={measuring}
+                                                   onChange={e => setEnable(e.target.checked)}/>}/>
+                <NumberTextField label="Reserved CPU threads" start_val={cpus} min={0} max={512}
+                                 counter={counter} disabled={measuring}
+                                 callback={(v, err) => { setCpus(v); setCpusError(err); }}/>
+                <NumberTextField label="Cut the analysis this long after the last image (0 = never)" units="s"
+                                 start_val={deadline} min={0} float decimals={1} counter={counter} disabled={measuring}
+                                 sx={{width: "100%"}} callback={(v, err) => { setDeadline(v); setDeadlineError(err); }}/>
+            </SettingsPanel>
+
+            <SettingsPanel title="Model for molecular replacement" spacing={2}>
+                <Typography sx={{fontWeight: 600}}>
+                    {s?.model_loaded ? `Model loaded · ${sizeText(s.model_size_bytes)}` : "No model loaded"}
+                </Typography>
+                <Typography variant="body2" color="text.secondary">
+                    PDB or mmCIF, optionally gzipped, up to 64 MiB. Kept until removed.
+                </Typography>
+                <Stack direction="row" spacing={1} alignItems="center">
+                    <Button component="label" variant="outlined" startIcon={<UploadFileIcon/>} disabled={measuring}>
+                        {file ? file.name : "Choose file"}
+                        <input hidden type="file" accept=".pdb,.cif,.ent,.gz"
+                               onChange={(e: ChangeEvent<HTMLInputElement>) => setFile(e.target.files?.[0])}/>
+                    </Button>
+                    <Button variant="contained" disableElevation disabled={!file || measuring || upload.pending}
+                            onClick={() => file && upload.submit(file)}>
+                        Upload
+                    </Button>
+                    <Button color="error" startIcon={<DeleteIcon/>}
+                            disabled={!s?.model_loaded || measuring || remove.pending}
+                            onClick={() => remove.submit(undefined)}>
+                        Remove
+                    </Button>
+                </Stack>
+                <UploadSnackbarView snackbar={upload.snackbar}/>
+                <UploadSnackbarView snackbar={remove.snackbar}/>
+            </SettingsPanel>
+        </Stack>
+    );
+}
+
+export default RugnuxSettings;

frontend/src/components/RugnuxStatus.tsx

@@ -0,0 +1,68 @@
+import {memo} from 'react';
+
+import {Alert, Box, LinearProgress, Typography} from "@mui/material";
+import {useQuery} from "@tanstack/react-query";
+import {getConfigRugnuxOptions, getRugnuxStatusOptions} from "../client/@tanstack/react-query.gen";
+import {useTokenRequired} from "../auth";
+import SettingsPanel from "./SettingsPanel";
+import {CutAlert, ProbeAlert, StepTable, VerdictAlert} from "./RugnuxParts";
+
+// Live rugnux processing of the last collection, as a dashboard card: the step it is on, the steps
+// behind it with what they found, and once it is over the verdict. Nothing while rugnux has not run,
+// and nothing - not even the status polling - while the mode is off (the settings are read once, on mount).
+function RugnuxStatus() {
+    const {data: settings} = useQuery(getConfigRugnuxOptions());
+    const enabled = settings?.enable === true;
+    const {data: s, isError} = useQuery({
+        ...getRugnuxStatusOptions(),
+        refetchInterval: 1000,
+        enabled,
+    });
+    const tokenRequired = useTokenRequired();
+
+    if (!enabled)
+        return null;
+    if (isError && tokenRequired)
+        return (
+            <SettingsPanel title="Rugnux">
+                <Typography>Dataset token required (key button in the top bar)</Typography>
+            </SettingsPanel>
+        );
+    if (isError || s === undefined || s.state === 'Idle')
+        return null;
+
+    const current = s.state === 'Running' && s.steps.length > 0 ? s.steps[s.steps.length - 1] : undefined;
+
+    return (
+        <SettingsPanel title="Rugnux" spacing={2}>
+            <Typography variant="h6">
+                {s.state} · {s.images_received} / {s.images_expected} images · {s.elapsed_s.toFixed(0)} s
+            </Typography>
+            {s.message && <Typography color="text.secondary">{s.message}</Typography>}
+            <CutAlert s={s}/>
+            <ProbeAlert s={s}/>
+
+            {current && (
+                <Box sx={{width: "100%"}}>
+                    <Typography>{current.title}</Typography>
+                    <LinearProgress variant={s.step_progress !== undefined ? "determinate" : "indeterminate"}
+                                    value={(s.step_progress ?? 0) * 100}/>
+                </Box>
+            )}
+
+            {s.last_image_to_written_s !== undefined && (
+                <Typography variant="h5">
+                    Last image → all written: {s.last_image_to_written_s.toFixed(1)} s
+                </Typography>
+            )}
+
+            <VerdictAlert s={s}/>
+            <StepTable steps={s.steps}/>
+            {s.warnings.map((w, i) => (
+                <Alert key={i} severity="warning" sx={{width: "100%", textAlign: "left"}}>{w}</Alert>
+            ))}
+        </SettingsPanel>
+    );
+}
+
+export default memo(RugnuxStatus);

frontend/src/components/RugnuxTimeline.tsx

@@ -0,0 +1,174 @@
+import {Box, Tooltip, Typography} from "@mui/material";
+import {keyframes} from "@mui/system";
+import {indigo, lime, lightBlue, orange, grey, deepOrange} from "@mui/material/colors";
+import {rugnux_live_status} from "../client";
+import {lastImageTime} from "./RugnuxParts";
+
+// The overlap of collection and processing on one time axis (seconds since the first image): the
+// detector lane fills while images arrive, the rugnux lane shows its steps where they ran, and the
+// gap between "last image" and "all written" is the headline latency.
+
+const stripes = keyframes`
+    from { background-position: 0 0; }
+    to { background-position: 40px 0; }
+`;
+
+const runningSx = {
+    backgroundImage: "linear-gradient(135deg, rgba(255,255,255,.28) 25%, transparent 25%, transparent 50%, "
+        + "rgba(255,255,255,.28) 50%, rgba(255,255,255,.28) 75%, transparent 75%, transparent)",
+    backgroundSize: "40px 40px",
+    animation: `${stripes} 1s linear infinite`,
+};
+
+const STEP_COLOURS = [indigo[500], indigo[300], indigo[700], lightBlue[700], indigo[400], lightBlue[900]];
+
+const LABEL_WIDTH = 120;
+const LANE_HEIGHT = 64;
+const AXIS_HEIGHT = 28;
+
+function niceStep(span: number) {
+    for (const s of [1, 2, 5, 10, 20, 30, 60, 120])
+        if (span / s <= 12)
+            return s;
+    return 300;
+}
+
+function Marker({x, colour, label, top, dashed}: { x: number, colour: string, label: string, top: number, dashed?: boolean }) {
+    return (
+        <Box sx={{position: "absolute", left: `${x}%`, top, bottom: AXIS_HEIGHT, pointerEvents: "none"}}>
+            <Box sx={{position: "absolute", top: 0, bottom: 0, left: -1.5, width: 0, zIndex: 1,
+                borderLeft: `3px ${dashed ? "dashed" : "solid"} ${colour}`}}/>
+            <Box sx={{position: "absolute", top: 0, left: 0, transform: "translate(-50%, -100%)", zIndex: 4,
+                bgcolor: colour, color: "#fff", px: 1, py: 0.25, borderRadius: 1, whiteSpace: "nowrap",
+                fontSize: "0.85rem", fontWeight: 700, boxShadow: 1}}>
+                {label}
+            </Box>
+        </Box>
+    );
+}
+
+// deadline_s: the configured cutoff after the last image, 0 = none
+function RugnuxTimeline({s, deadline_s}: { s: rugnux_live_status, deadline_s: number }) {
+    const lastImage = lastImageTime(s);
+    const collecting = lastImage === undefined && s.state === 'Running';
+    const collectionEnd = lastImage ?? s.elapsed_s;
+    const rate = collectionEnd > 0 ? s.images_received / collectionEnd : 0;
+    const expectedEnd = rate > 0 ? s.images_expected / rate : 0;
+
+    const deadline = lastImage !== undefined && deadline_s > 0 ? lastImage + deadline_s : undefined;
+    const span = Math.max(s.elapsed_s, expectedEnd, deadline ?? 0, 10) * 1.06;
+    const pct = (t: number) => 100 * Math.min(Math.max(t, 0), span) / span;
+    const tick = niceStep(span);
+    const ticks = Array.from({length: Math.floor(span / tick) + 1}, (_, i) => i * tick);
+
+    const done = s.state !== 'Running';
+    const latency = s.last_image_to_written_s;
+
+    return (
+        <Box sx={{display: "flex", pt: 6, pb: latency !== undefined ? 6 : 1}}>
+            <Box sx={{width: LABEL_WIDTH, flexShrink: 0}}>
+                {["Detector", "Rugnux"].map((l, i) => (
+                    <Box key={l} sx={{height: LANE_HEIGHT, mb: i === 0 ? 1.5 : 0, display: "flex", alignItems: "center"}}>
+                        <Typography sx={{fontWeight: 700, fontSize: "1.1rem", color: "primary.dark"}}>{l}</Typography>
+                    </Box>
+                ))}
+            </Box>
+
+            <Box sx={{position: "relative", flexGrow: 1, minWidth: 0}}>
+                {/* grid lines */}
+                {ticks.map(t => (
+                    <Box key={t} sx={{position: "absolute", left: `${pct(t)}%`, top: 0,
+                        height: 2 * LANE_HEIGHT + 12, borderLeft: `1px solid ${grey[200]}`}}/>
+                ))}
+
+                {/* detector lane */}
+                <Box sx={{position: "relative", height: LANE_HEIGHT, mb: 1.5, bgcolor: grey[100], borderRadius: 2}}>
+                    <Box sx={{position: "absolute", left: 0, top: 0, bottom: 0, width: `${pct(collectionEnd)}%`,
+                        bgcolor: lime[600], borderRadius: 2, display: "flex", alignItems: "center", px: 2,
+                        overflow: "hidden", ...(collecting && runningSx)}}>
+                        <Typography sx={{position: "relative", zIndex: 2, fontWeight: 700, fontSize: "1.1rem",
+                            color: "rgba(0,0,0,.85)", whiteSpace: "nowrap", bgcolor: "rgba(255,255,255,.55)",
+                            px: 1, borderRadius: 1}}>
+                            {s.images_received} / {s.images_expected} images{rate > 0 && ` · ${rate.toFixed(0)} Hz`}
+                        </Typography>
+                    </Box>
+                    {collecting && expectedEnd > collectionEnd && (
+                        <Box sx={{position: "absolute", left: `${pct(collectionEnd)}%`, top: 0, bottom: 0,
+                            width: `${pct(expectedEnd) - pct(collectionEnd)}%`,
+                            border: `2px dashed ${lime[700]}`, borderLeft: "none", borderRadius: "0 8px 8px 0"}}/>
+                    )}
+                </Box>
+
+                {/* rugnux lane */}
+                <Box sx={{position: "relative", height: LANE_HEIGHT, bgcolor: grey[100], borderRadius: 2}}>
+                    {s.steps.map((step, i) => {
+                        const open = step.duration_s === undefined;
+                        const end = open ? s.elapsed_s : step.started_s + step.duration_s!;
+                        const width = Math.max(pct(end) - pct(step.started_s), 0.4);
+                        return (
+                            <Tooltip key={i} arrow title={
+                                <Box>
+                                    <b>{step.title}</b> · {open ? "running" : `${step.duration_s!.toFixed(1)} s`}
+                                    {step.notes.map((n, j) => <div key={j}>{n}</div>)}
+                                </Box>}>
+                                <Box sx={{position: "absolute", left: `${pct(step.started_s)}%`, width: `${width}%`,
+                                    top: 0, bottom: 0, bgcolor: STEP_COLOURS[i % STEP_COLOURS.length],
+                                    borderRight: "2px solid #fff", display: "flex", alignItems: "center",
+                                    justifyContent: "center", overflow: "hidden", px: 0.5,
+                                    borderRadius: i === 0 ? "8px 0 0 8px" : 0,
+                                    ...(open && !done && runningSx),
+                                    ...(i === s.steps.length - 1 && {borderTopRightRadius: 8, borderBottomRightRadius: 8})}}>
+                                    <Typography sx={{position: "relative", zIndex: 2, color: "#fff", fontWeight: 600, fontSize: "0.9rem",
+                                        whiteSpace: "nowrap", overflow: "hidden", textOverflow: "ellipsis"}}>
+                                        {step.title}
+                                    </Typography>
+                                </Box>
+                            </Tooltip>
+                        );
+                    })}
+                </Box>
+
+                {/* axis */}
+                <Box sx={{position: "relative", height: AXIS_HEIGHT}}>
+                    {ticks.map(t => (
+                        <Typography key={t} sx={{position: "absolute", left: `${pct(t)}%`, transform: "translateX(-50%)",
+                            fontSize: "0.85rem", color: "text.secondary"}}>
+                            {t} s
+                        </Typography>
+                    ))}
+                </Box>
+
+                {/* markers */}
+                {s.probe_s !== undefined && (
+                    <Marker x={pct(s.probe_s)} top={-8} label={`probe ${s.probe_s.toFixed(1)} s`}
+                            colour={s.probe.startsWith("no lattice") ? orange[800] : lightBlue[800]}/>
+                )}
+                {lastImage !== undefined && (
+                    <Marker x={pct(lastImage)} top={-8} dashed colour={grey[800]} label="last image"/>
+                )}
+                {deadline !== undefined && s.cut !== 'deadline' && (
+                    <Marker x={pct(deadline)} top={-8} dashed colour={deepOrange[700]} label={`deadline ${deadline.toFixed(1)} s`}/>
+                )}
+                {s.cut && s.cut_s !== undefined && (
+                    <Marker x={pct(s.cut_s)} top={-8} colour={deepOrange[900]} label={`${s.cut} cut ${s.cut_s.toFixed(1)} s`}/>
+                )}
+                {!done && <Marker x={pct(s.elapsed_s)} top={-8} colour="#d32f2f" label="now"/>}
+
+                {/* latency bracket */}
+                {latency !== undefined && lastImage !== undefined && (
+                    <Box sx={{position: "absolute", left: `${pct(lastImage)}%`, width: `${pct(lastImage + latency) - pct(lastImage)}%`,
+                        top: 2 * LANE_HEIGHT + 12 + AXIS_HEIGHT, height: 14,
+                        borderBottom: `4px solid ${lime[700]}`, borderLeft: `4px solid ${lime[700]}`,
+                        borderRight: `4px solid ${lime[700]}`, borderRadius: "0 0 8px 8px"}}>
+                        <Typography sx={{position: "absolute", right: 0, top: "100%", mt: 0.5, fontWeight: 800,
+                            fontSize: "1.4rem", color: lime[900], whiteSpace: "nowrap"}}>
+                            +{latency.toFixed(1)} s → all written
+                        </Typography>
+                    </Box>
+                )}
+            </Box>
+        </Box>
+    );
+}
+
+export default RugnuxTimeline;

image_analysis/CMakeLists.txt

@@ -131,4 +131,4 @@ ADD_SUBDIRECTORY(azint)
 ADD_SUBDIRECTORY(roi)
 ADD_SUBDIRECTORY(structure_refinement)
 
-TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochROIIntegration JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochStructureRefinement gemmi)
+TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochROIIntegration JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochStructureRefinement JFJochRugnuxFiles gemmi)

image_analysis/WriteReflections.cpp

@@ -7,6 +7,7 @@
 #include "scale_merge/TwinningAnalysis.h"
 #include "bragg_integration/SystematicAbsence.h"
 #include "../common/ParallelFor.h"
+#include "../writer/WriteRugnuxFiles.h"
 
 #include <algorithm>
 #include <charconv>
@@ -134,16 +135,16 @@ std::string GitDescription() {
 // (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each
 // continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is
 // never cut just after a space: the space starts the next one, behind its '+'.
-void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) {
-    const auto add = [&mtz](const std::string &label, const std::string &value) {
+void AddProvenanceHistory(std::vector<std::string> &history, const OutputProvenance &provenance) {
+    const auto add = [&history](const std::string &label, const std::string &value) {
         if (value.empty())
             return;
         std::string rest = label + value;
-        while (!rest.empty() && mtz.history.size() < 30) {
+        while (!rest.empty() && history.size() < 30) {
             size_t n = std::min<size_t>(80, rest.size());
             while (n > 1 && n < rest.size() && rest[n - 1] == ' ')
                 --n;
-            mtz.history.push_back(rest.substr(0, n));
+            history.push_back(rest.substr(0, n));
             rest = n < rest.size() ? "+" + rest.substr(n) : std::string();
         }
     };
@@ -619,81 +620,58 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
     out.close();
 }
 
-void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
-                         const UnitCell &unitCell,
-                         const DiffractionExperiment &experiment,
-                         const OutputProvenance &provenance,
-                         const std::string &filename) {
-    gemmi::Mtz mtz;
-
-    // Optional but recommended metadata
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
-
+MtzTable MergedMtzTable(const std::vector<MergedReflection> &reflections,
+                        const UnitCell &unitCell,
+                        const DiffractionExperiment &experiment,
+                        const OutputProvenance &provenance) {
+    MtzTable m;
+    m.unit_cell = unitCell;
+    m.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
     // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention).
-    mtz.title = "Rugnux merged reflections";
-    mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-
-    // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the
-    // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is
-    // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that
-    // takes f'/f'' from the file would phase on.
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset& ds = mtz.add_dataset("native");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
-
-    const int dataset_id = ds.id;
+    m.title = "Rugnux merged reflections";
+    m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
+    AddProvenanceHistory(m.history, provenance);
+    // HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the reserved base dataset
+    // as far as CCP4's mtzlib is concerned, and the wavelength on it is dropped - mtzinfo then reports the
+    // Cu K-alpha default 1.54187 A, which is what a program that takes f'/f'' from the file would phase on.
+    m.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                  {"native", experiment.GetSampleName(), "native", experiment.GetWavelength_A()}};
 
     // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for
     // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly.
     bool has_anom = true;
     const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
+    std::vector<std::pair<const char *, char>> columns{{"H", 'H'}, {"K", 'H'}, {"L", 'H'},
+                                                       {"IMEAN", 'J'}, {"SIGIMEAN", 'Q'}};
+    if (has_anom)
+        columns.insert(columns.end(), {{"I(+)", 'K'}, {"SIGI(+)", 'M'}, {"I(-)", 'K'}, {"SIGI(-)", 'M'}});
+    columns.insert(columns.end(), {{"F", 'F'}, {"SIGF", 'Q'}});   // F: the French-Wilson amplitude
+    if (has_anom)
+        columns.insert(columns.end(), {{"F(+)", 'G'}, {"SIGF(+)", 'L'}, {"F(-)", 'G'}, {"SIGF(-)", 'L'}});
+    columns.push_back({"FreeR_flag", 'I'});
+    for (const auto &[label, type] : columns)
+        m.columns.push_back({label, type, type == 'H' ? 0 : 1, {}});
+    for (const auto &row : out_rows) {
+        std::vector<float> v{static_cast<float>(row.h), static_cast<float>(row.k), static_cast<float>(row.l),
+                             row.Imean, row.sImean};
+        if (has_anom)
+            v.insert(v.end(), {row.Ip, row.sIp, row.Im, row.sIm});
+        v.insert(v.end(), {row.Fmean, row.sFmean});
+        if (has_anom)
+            v.insert(v.end(), {row.Fp, row.sFp, row.Fm, row.sFm});
+        v.push_back(static_cast<float>(row.rfree));
+        for (size_t c = 0; c < v.size(); c++)
+            m.columns[c].values.push_back(v[c]);
+    }
+    return m;
+}
 
-    mtz.add_column("IMEAN", 'J', dataset_id, -1, false);
-    mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("I(+)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false);
-        mtz.add_column("I(-)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false);
-    }
-    mtz.add_column("F", 'F', dataset_id, -1, false);       // French-Wilson amplitude
-    mtz.add_column("SIGF", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("F(+)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false);
-        mtz.add_column("F(-)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false);
-    }
-    mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false);
-
-    mtz.nreflections = static_cast<int>(out_rows.size());
-    mtz.data.reserve(out_rows.size() * 16);
-    for (const auto& row : out_rows) {
-        mtz.data.push_back(static_cast<float>(row.h));
-        mtz.data.push_back(static_cast<float>(row.k));
-        mtz.data.push_back(static_cast<float>(row.l));
-        mtz.data.push_back(row.Imean);
-        mtz.data.push_back(row.sImean);
-        if (has_anom) {
-            mtz.data.push_back(row.Ip);
-            mtz.data.push_back(row.sIp);
-            mtz.data.push_back(row.Im);
-            mtz.data.push_back(row.sIm);
-        }
-        mtz.data.push_back(row.Fmean);
-        mtz.data.push_back(row.sFmean);
-        if (has_anom) {
-            mtz.data.push_back(row.Fp);
-            mtz.data.push_back(row.sFp);
-            mtz.data.push_back(row.Fm);
-            mtz.data.push_back(row.sFm);
-        }
-        mtz.data.push_back(static_cast<float>(row.rfree));
-    }
-    mtz.write_to_file(filename);
+void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
+                         const UnitCell &unitCell,
+                         const DiffractionExperiment &experiment,
+                         const OutputProvenance &provenance,
+                         const std::string &filename) {
+    WriteMtz(MergedMtzTable(reflections, unitCell, experiment, provenance), filename);
 }
 
 namespace {
@@ -946,22 +924,22 @@ std::vector<std::vector<Reflection>> SumRockingEvents(const std::vector<Integrat
 
 } // namespace
 
-gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
-                       const UnitCell &unitCell,
-                       const DiffractionExperiment &experiment,
-                       bool sum_partials,
-                       const OutputProvenance &provenance,
-                       size_t nthreads) {
-    gemmi::Mtz mtz;
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
+MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
+                          const UnitCell &unitCell,
+                          const DiffractionExperiment &experiment,
+                          bool sum_partials,
+                          const OutputProvenance &provenance,
+                          size_t nthreads) {
+    const gemmi::SpaceGroup *spacegroup = &experiment.GetSpaceGroupOrP1();
+    MtzTable mtz;
+    mtz.space_group_name = spacegroup->xhm();
+    mtz.unit_cell = unitCell;
     mtz.title = "Rugnux unmerged observations";
     mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
+    AddProvenanceHistory(mtz.history, provenance);
+    mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                    {"unmerged", experiment.GetSampleName(), "unmerged", experiment.GetWavelength_A()}};
+    const int dataset_id = 1;
 
     // Every data column belongs to HKL_base and only the batches carry a dataset id, as in a
     // POINTLESS file. Down to FLAG this is POINTLESS's own column set, plus QE and FLIGHT; the four after it
@@ -981,22 +959,11 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // rises above 1 toward the detector edge where a longer crossing makes the sensor more
     // efficient. DIALS stores the un-normalised absorbed fraction instead; the two differ by the
     // per-dataset constant QE(0), which is an overall scale.
-    mtz.add_column("M/ISYM", 'Y', 0, -1, false);
-    mtz.add_column("BATCH", 'B', 0, -1, false);
-    mtz.add_column("I", 'J', 0, -1, false);
-    mtz.add_column("SIGI", 'Q', 0, -1, false);
-    mtz.add_column("FRACTIONCALC", 'R', 0, -1, false);
-    mtz.add_column("XDET", 'R', 0, -1, false);
-    mtz.add_column("YDET", 'R', 0, -1, false);
-    mtz.add_column("ROT", 'R', 0, -1, false);
-    mtz.add_column("LP", 'R', 0, -1, false);
-    mtz.add_column("QE", 'R', 0, -1, false);
-    mtz.add_column("FLIGHT", 'R', 0, -1, false);
-    mtz.add_column("FLAG", 'I', 0, -1, false);
-    mtz.add_column("DELPHI", 'R', 0, -1, false);
-    mtz.add_column("ZETA", 'R', 0, -1, false);
-    mtz.add_column("BGMEAN", 'R', 0, -1, false);
-    mtz.add_column("BGVAR", 'R', 0, -1, false);
+    for (const auto &[label, type] : std::vector<std::pair<const char *, char>>{
+             {"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'},
+             {"FRACTIONCALC", 'R'}, {"XDET", 'R'}, {"YDET", 'R'}, {"ROT", 'R'}, {"LP", 'R'}, {"QE", 'R'},
+             {"FLIGHT", 'R'}, {"FLAG", 'I'}, {"DELPHI", 'R'}, {"ZETA", 'R'}, {"BGMEAN", 'R'}, {"BGVAR", 'R'}})
+        mtz.columns.push_back({label, type, 0, {}});
 
     const auto gon = experiment.GetGoniometer();
     // A scanning goniometer means rugnux integrated a rocking curve image by image, so each
@@ -1025,7 +992,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // None of the three is a scale, and a program that reads this file has no way to recompute them. The
     // partiality is NOT divided out - that IS a scale, FRACTIONCALC carries it, and every program
     // this file is for wants to handle it its own way.
-    gemmi::UnmergedHklMover hkl_mover(mtz.spacegroup);
+    gemmi::UnmergedHklMover hkl_mover(spacegroup);
     // The batch headers are written for every image the observations span, not only for the images
     // that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that
     // indexed nothing leaves a hole that cuts the sweep in two: measured on a 360 deg sweep with 67
@@ -1040,7 +1007,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // roughly halve AIMLESS's <I/sigma>. Screw and glide absences ARE written: they are the evidence
     // the space group was chosen on, so a program reading this file can check that choice, and
     // removing them turns its test into an assumption. XDS and DIALS draw the line in the same place.
-    const char centering = mtz.spacegroup ? mtz.spacegroup->hm[0] : 'P';
+    const char centering = spacegroup->hm[0];
 
     // On stills only the crystals the merge takes are written: one whose cell is not the run's was indexed
     // in another lattice (on few spots, typically an axis multiple), so its indices do not name the
@@ -1080,13 +1047,14 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
                     add_row(r);
     }
 
-    // Every row has its own slot, so they are filled on several workers.
+    // Every row has its own slot in each column, so they are filled on several workers.
     const size_t w = mtz.columns.size();
-    mtz.data.resize(rows.size() * w);
+    for (auto &c : mtz.columns)
+        c.values.resize(rows.size());
     ParallelChunks(static_cast<int>(rows.size()), ThreadsForWork(rows.size(), nthreads), [&](int lo, int hi) {
+        std::vector<float> row(w);
         for (int i = lo; i < hi; ++i) {
             const Reflection &r = *rows[i];
-            float *row = &mtz.data[static_cast<size_t>(i) * w];
             std::array<int, 3> hkl{r.h, r.k, r.l};
             const int isym = hkl_mover.move_to_asu(hkl);
             row[0] = static_cast<float>(hkl[0]);
@@ -1113,9 +1081,10 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
             row[16] = r.zeta;
             row[17] = r.bkg;
             row[18] = r.var_bkg;
+            for (size_t c = 0; c < w; c++)
+                mtz.columns[c].values[i] = row[c];
         }
     });
-    mtz.nreflections = static_cast<int>(mtz.data.size() / mtz.columns.size());
 
     // The batch header's orientation matrix is the crystal at rotation angle zero - each batch's own
     // PHISTT is applied on top of it - but the lattice stored with an outcome is the crystal as it
@@ -1146,7 +1115,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     gemmi::Mtz::Batch batch;
     batch.title = "Rugnux";
     batch.axes.emplace_back("PHI");
-    batch.set_dataset_id(ds.id);
+    batch.set_dataset_id(dataset_id);
     batch.ints[12] = 1;   // ncryst
     batch.ints[14] = partials ? 1 : 2;   // ldtype: oscillation data (2D spots) / area detector (3D)
     batch.ints[15] = 1;   // jsaxs: the goniostat scan axis
@@ -1183,38 +1152,41 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
         batch.number = number;
         batch.floats[36] = phi_start_deg(static_cast<float>(number - 1));   // phistt
         batch.floats[37] = batch.floats[36] + wedge_deg;                    // phiend
-        mtz.batches.push_back(batch);
+        mtz.batches.push_back({batch.number, batch.title, {batch.ints.begin(), batch.ints.end()}, batch.floats,
+                               batch.axes});
     }
 
     // By H K L M/ISYM BATCH, the order POINTLESS leaves an unmerged file in. This is Mtz::sort(5) -
     // a stable sort on those five columns - done on several workers: rows equal in all five keep the
     // order they were added in, so with the row number as the last key the order is total and the
     // parallel sort returns the same rows in the same order.
-    if (!mtz.has_data())
-        mtz.sort(5);   // no rows: its own "No data." failure, as before
+    if (rows.empty())
+        gemmi::fail("No data.");   // as gemmi's own sort has it
     {
-        std::vector<int> order(mtz.nreflections);
+        std::vector<int> order(rows.size());
         std::iota(order.begin(), order.end(), 0);
         ParallelSort(order.begin(), order.end(), nthreads, [&](int i, int j) {
-            const float *a = &mtz.data[static_cast<size_t>(i) * w], *b = &mtz.data[static_cast<size_t>(j) * w];
-            for (int n = 0; n < 5; ++n)
-                if (a[n] != b[n])
-                    return a[n] < b[n];
+            for (int n = 0; n < 5; ++n) {
+                const float a = mtz.columns[n].values[i], b = mtz.columns[n].values[j];
+                if (a != b)
+                    return a < b;
+            }
             return i < j;
         });
-        std::vector<float> sorted(mtz.data.size());
-        ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
-            for (int i = lo; i < hi; ++i)
-                std::memcpy(&sorted[static_cast<size_t>(i) * w], &mtz.data[static_cast<size_t>(order[i]) * w],
-                            w * sizeof(float));
-        });
-        mtz.data.swap(sorted);
-        mtz.sort_order = {{1, 2, 3, 4, 5}};
+        for (auto &c : mtz.columns) {
+            std::vector<float> sorted(order.size());
+            ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
+                for (int i = lo; i < hi; ++i)
+                    sorted[i] = c.values[order[i]];
+            });
+            c.values.swap(sorted);
+        }
+        mtz.sort_order = {1, 2, 3, 4, 5};
     }
     return mtz;
 }
 
-void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector<IntegrationOutcome> &outcomes) {
+void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes) {
     // The reflecting range of the same image the orientation matrix is taken from.
     std::optional<float> mosaicity_deg;
     for (const auto &outcome : outcomes) {
@@ -1234,9 +1206,9 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
                                  const OutputProvenance &provenance,
                                  const std::string &filename,
                                  size_t nthreads) {
-    gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
+    MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
     SetUnmergedMtzMosaicity(mtz, outcomes);
-    mtz.write_to_file(filename);
+    WriteMtz(mtz, filename);
 }
 
 void WriteReflections(const std::vector<MergedReflection> &reflections,

image_analysis/WriteReflections.h

@@ -9,10 +9,9 @@
 #include "../common/Reflection.h"
 #include "../common/UnitCell.h"
 #include "../common/DiffractionExperiment.h"
+#include "../common/MtzTable.h"
 #include "IntegrationOutcome.h"
 
-#include <gemmi/mtz.hpp>
-
 struct MergeStatistics;
 struct TwinningAnalysisResult;
 
@@ -49,6 +48,15 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
                            const std::string &filename,
                            size_t nthreads);
 
+// The merged MTZ: H K L IMEAN SIGIMEAN [I(+) SIGI(+) I(-) SIGI(-)] F SIGF [F(+) SIGF(+) F(-) SIGF(-)]
+// FreeR_flag, one row per unique reflection in the CCP4 anomalous layout, the bracketed columns only where
+// the merge kept a Bijvoet split; with the HISTORY records saying where it came from. WriteMtzReflections
+// writes it (WriteMtz, writer/WriteRugnuxFiles.h).
+MtzTable MergedMtzTable(const std::vector<MergedReflection> &reflections,
+                        const UnitCell &unitCell,
+                        const DiffractionExperiment &experiment,
+                        const OutputProvenance &provenance);
+
 void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
                          const UnitCell &unitCell,
                          const DiffractionExperiment &experiment,
@@ -89,17 +97,16 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
                                  const std::string &filename,
                                  size_t nthreads = 1);
 
-// The same file in two steps, for a caller that builds it while the outcomes' per-image scale fields
-// are still being written: UnmergedMtz reads everything but mosaicity_deg, which
-// SetUnmergedMtzMosaicity then copies into the batch headers. The Mtz points at the experiment's space
-// group, so the experiment has to outlive it.
-gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
-                       const UnitCell &unitCell,
-                       const DiffractionExperiment &experiment,
-                       bool sum_partials,
-                       const OutputProvenance &provenance,
-                       size_t nthreads = 1);
-void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector<IntegrationOutcome> &outcomes);
+// The same file as its table, in two steps, for a caller that builds it while the outcomes' per-image
+// scale fields are still being written: UnmergedMtzTable reads everything but mosaicity_deg, which
+// SetUnmergedMtzMosaicity then copies into the batch headers.
+MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
+                          const UnitCell &unitCell,
+                          const DiffractionExperiment &experiment,
+                          bool sum_partials,
+                          const OutputProvenance &provenance,
+                          size_t nthreads = 1);
+void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes);
 
 void WriteReflections(const std::vector<MergedReflection> &reflections,
                          const UnitCell &unitCell,

image_analysis/scale_merge/RotationScaleMerge.cpp

@@ -12,7 +12,7 @@
 #include <array>
 #include <cmath>
 #include <cstdint>
-#include <fstream>
+#include <sstream>
 #include <future>
 #include <limits>
 #include <numeric>
@@ -303,10 +303,10 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
                                        std::optional<UnitCell> reference_cell,
                                        int scaling_iterations,
                                        size_t nthreads, Logger &logger,
-                                       std::string observation_dump_path)
+                                       std::function<void(const std::string &)> observation_dump)
     : x(experiment), partials_out(partial_outcomes), reference_cell(std::move(reference_cell)),
       nthreads(nthreads == 0 ? AvailableCpus() : nthreads), logger(&logger),
-      observation_dump_path(std::move(observation_dump_path)) {
+      observation_dump(std::move(observation_dump)) {
     const auto s = x.GetScalingSettings();
     min_partiality = s.GetMinPartiality();
     d_min_limit = s.GetHighResolutionLimit_A();
@@ -401,7 +401,7 @@ void RotationScaleMerge::IngestHost() {
     // only the ingest arrays depends on whether the device pipeline will be active.
     gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
     gpu_active_ = gpu_->Available() && !scale_on_cpu_;
-    resident_ingest = gpu_active_ && observation_dump_path.empty();
+    resident_ingest = gpu_active_ && !observation_dump;
 #endif
 
     frame_cell_ok.assign(n_frames, 1);
@@ -4440,7 +4440,7 @@ void RotationScaleMerge::Combine() {
     // Combine one raw-hkl run into fulls, appended to `out`. Independent per run (the events of one hkl
     // touch no shared state), so runs parallelise cleanly. Returns the number of usable partials seen.
     // `dump` (serial path only) writes each emitted full for the diagnostic observation dump.
-    auto process_rawrun = [&](int r, std::vector<Obs> &out, std::ofstream *dump, int64_t &n_overloaded) -> size_t {
+    auto process_rawrun = [&](int r, std::vector<Obs> &out, std::ostream *dump, int64_t &n_overloaded) -> size_t {
         prefetch_run(r + COMBINE_PREFETCH_RUNS);
         const int lo = rawrun_start[r], hi = rawrun_start[r] + rawrun_count[r];
         std::vector<int32_t> ev;   // usable perm-indices of this raw hkl, in image-number order
@@ -4583,11 +4583,10 @@ void RotationScaleMerge::Combine() {
     size_t n_used = 0;
     std::vector<std::vector<Obs>> part;   // the fulls of consecutive runs, in run order
 
-    if (!observation_dump_path.empty() || nthreads <= 1) {
+    if (observation_dump || nthreads <= 1) {
         // Serial (diagnostic dump needs a single writer).
-        std::ofstream dump;
-        if (!observation_dump_path.empty()) {
-            dump.open(observation_dump_path);
+        std::ostringstream dump;
+        if (observation_dump) {
             // Columns are enough to rescale/merge the combined fulls in an external program: peak_frame
             // is a per-full batch (ZD) and px/py the predicted detector position (XD/YD).
             dump << "# h k l I sigma d n_frames captured_fraction peak_frame px py\n";
@@ -4595,7 +4594,9 @@ void RotationScaleMerge::Combine() {
         part.resize(1);
         part[0].reserve(n_run);
         for (int r = 0; r < n_run; ++r)
-            n_used += process_rawrun(r, part[0], dump.is_open() ? &dump : nullptr, fulls_dropped_overloaded);
+            n_used += process_rawrun(r, part[0], observation_dump ? &dump : nullptr, fulls_dropped_overloaded);
+        if (observation_dump)
+            observation_dump(dump.str());
     } else {
         // Parallel over contiguous rawrun chunks; the per-thread fulls are taken in run order below, so
         // the result is deterministic.
@@ -6876,7 +6877,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
     // scale-fulls loop below. Only the diagnostic dump (serial, one writer) falls back to CPU. The
     // decay/absorption correction surfaces run on the downloaded fulls (px/py is carried through the
     // combine) and their corrected corr is re-uploaded before the resident merge - so they stay on the GPU.
-    if (gpu_active_ && observation_dump_path.empty()) {
+    if (gpu_active_ && !observation_dump) {
         // The smoothed corr is already resident (scaling + smooth-G ran on the device, no round-trip).
         const int nf = gpu_->Combine(rawrun_group.data(), min_partiality, capture_uncertainty_coeff,
                                      min_captured_fraction, max_frame_gap, fulls_dropped_overloaded);

image_analysis/scale_merge/RotationScaleMerge.h

@@ -139,7 +139,7 @@ public:
                        int scaling_iterations,
                        size_t nthreads,
                        Logger &logger,
-                       std::string observation_dump_path = {});
+                       std::function<void(const std::string &)> observation_dump = {});
 
     // Copy the per-frame partials into the flat buffers. Call once before the first Run().
     void Ingest();
@@ -395,7 +395,8 @@ private:
     std::optional<UnitCell> reference_cell;
     size_t nthreads;
     Logger *logger;
-    std::string observation_dump_path;
+    // Handed the text of the diagnostic observation dump each time the combine runs (CPU only).
+    std::function<void(const std::string &)> observation_dump;
     // The group the current Run merges in (see Run); null outside a Run that was given one.
     const gemmi::SpaceGroup *run_group = nullptr;
     const gemmi::SpaceGroup &Group() const { return run_group ? *run_group : x.GetSpaceGroupOrP1(); }

image_analysis/structure_refinement/CMakeLists.txt

@@ -5,6 +5,7 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC
         MolecularReplacement.cpp MolecularReplacement.h
         $<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:MolecularReplacementGPU.cu> MolecularReplacementGPU.h
         ModelValidation.cpp ModelValidation.h
+        RugnuxModel.h
         RigidBodyRefine.cpp RigidBodyRefine.h
         $<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RigidBodyGPU.cpp>
         $<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RigidBodyGPU.cu>

image_analysis/structure_refinement/ModelValidation.cpp

@@ -21,16 +21,12 @@
 #include <vector>
 #include <unordered_map>
 
-#include <gemmi/mmread_gz.hpp> // read_structure_gz
 #include <gemmi/assembly.hpp>   // expand_ncs
-#include <gemmi/gz.hpp>        // MaybeGzipped
 #include <gemmi/it92.hpp>      // IT92 x-ray form factors
 #include <gemmi/dencalc.hpp>   // DensityCalculator
 #include <gemmi/fourier.hpp>   // get_size_for_hkl, get_f_phi_on_grid (the maps)
 #include <gemmi/solmask.hpp>   // SolventMasker
 #include <gemmi/scaling.hpp>   // Scaling (bulk solvent + anisotropic B)
-#include <gemmi/ccp4.hpp>      // Ccp4 map I/O
-#include <gemmi/mtz.hpp>       // Mtz (map-coefficient output)
 #include <gemmi/stats.hpp>     // calculate_data_statistics
 
 #include "../../common/CorrelationCoefficient.h"
@@ -82,22 +78,9 @@ gemmi::Grid<float> map_from_coefficients(gemmi::AsuData<std::complex<float>> &co
     return MapFromFPhi(gemmi::get_f_phi_on_grid<float>(coef, size, true));
 }
 
-// A file this run may already have written - the validation in the model's setting writes over the
-// first one's maps - is removed before it is written again, not truncated: XFS (and ext4) flush a file
-// truncated and rewritten when it is closed, and three 150 MB maps forced out to a disk that way cost
-// seconds where a fresh file costs nothing.
-void remove_before_rewriting(const std::string &path) {
-    std::remove(path.c_str());
-}
-
-// Write a map as CCP4; return its RMS (the sigma the map is read in).
-double write_ccp4(const gemmi::Grid<float> &map, const std::string &path) {
-    gemmi::Ccp4<float> ccp4;
-    ccp4.grid = map;
-    ccp4.update_ccp4_header(2);
-    remove_before_rewriting(path);
-    ccp4.write_ccp4_map(path);
-    return ccp4.hstats.rms;
+// The RMS of a map, the sigma it is read in: the one its CCP4 header states (Ccp4::update_ccp4_header).
+double map_rms(const gemmi::Grid<float> &map) {
+    return gemmi::calculate_data_statistics(map.data).rms;
 }
 
 // Cubic, not the default linear, for reading a map at a point. The maps are sampled every d_min/3,
@@ -538,8 +521,7 @@ namespace {
 // validation either way.
 ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                                const UnitCell &cell,
-                               const std::string &model_path,
-                               const std::string &output_prefix,
+                               const RugnuxModel &model,
                                Logger &logger,
                                const gemmi::SpaceGroup *data_space_group,
                                bool probe_indexing_ambiguity,
@@ -549,32 +531,29 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                                const ModelValidationSchedule &schedule,
                                const gemmi::Op *twin_law,
                                bool rigid_body_gpu,
-                               const gemmi::Structure *model) {
+                               const gemmi::Structure *placed) {
     ModelValidationResult result;
-    result.model_path = model_path;
+    result.model_name = model.name;
 
-    // --- read the atomic model ---
+    // --- the atomic model ---
     ModelState mdl;
     gemmi::Structure &st = mdl.st;   // the real model, the one the maps and the report describe
     try {
-        // Detect, not the default: without it GEMMI picks the format from the extension and only
-        // falls back to the content when it does not recognise one. A model arrives named however
-        // whoever produced it named it, so the file itself is the better authority.
-        st = model != nullptr ? *model : gemmi::read_structure_gz(model_path, gemmi::CoorFormat::Detect);
+        st = placed != nullptr ? *placed : model.structure;
         expand_strict_ncs(st);
     } catch (const std::exception &e) {
-        result.failure_reason = fmt::format("cannot read model {}: {}", model_path, e.what());
+        result.failure_reason = fmt::format("cannot read model {}: {}", model.name, e.what());
         logger.Error("Model validation: {}", result.failure_reason);
         return result;
     }
     if (st.models.empty() || !st.cell.is_crystal()) {
-        result.failure_reason = fmt::format("model {} has no atoms or no unit cell", model_path);
+        result.failure_reason = fmt::format("model {} has no atoms or no unit cell", model.name);
         logger.Error("Model validation: {}", result.failure_reason);
         return result;
     }
     const gemmi::SpaceGroup *sg = st.find_spacegroup();
     if (!sg) {
-        result.failure_reason = fmt::format("model {} has no usable space group", model_path);
+        result.failure_reason = fmt::format("model {} has no usable space group", model.name);
         logger.Error("Model validation: {}", result.failure_reason);
         return result;
     }
@@ -660,7 +639,7 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
     // and their maps and reflection files can be compared directly. Moving the model would leave
     // each dataset in whichever indexing its own run happened to pick.
     // A model handed over already placed in the data's cell (molecular replacement) is on the data's axes.
-    if (data_cell.is_crystal() && model == nullptr) {
+    if (data_cell.is_crystal() && placed == nullptr) {
         const gemmi::GroupOps model_gops = sg->operations();
         const bool same_point_group =
             data_space_group != nullptr && same_rotations(data_space_group->operations(), model_gops);
@@ -1734,23 +1713,27 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
         mapfofc.v.push_back({terms[i].hkl, delfwt[i] * ph});
     }
 
-    // A validation started before the one it follows had decided writes nothing until that one has,
-    // and nothing at all where it decided otherwise.
-    if (schedule.write_gate.valid() && !schedule.write_gate.get()) {
-        result.failure_reason = "superseded before its files were written";
+    // A validation started before the one it follows had decided makes no maps until that one has,
+    // and none at all where it decided otherwise.
+    if (schedule.maps_gate.valid() && !schedule.maps_gate.get()) {
+        result.failure_reason = "superseded before its maps were made";
         return result;
     }
-    // Where the validation in the model's setting, started beside this one, is going to be kept, it
-    // writes the maps and the map coefficients on the axes the files are written on, and this one's would
-    // be overwritten unread: they are not made at all.
+    // Where the validation in the model's setting, started beside this one, is going to be kept, its
+    // maps and map coefficients are on the axes the files are written on, and this one's would be
+    // thrown away unread: they are not made at all.
     const bool maps_superseded = schedule.maps_superseded && schedule.maps_superseded(result);
+    std::shared_ptr<ModelMaps> maps;
 
     if (!maps_superseded) {
-        // --- write the maps and score the 2mFo-DFc map at atom centres (a real map peaks there) ---
+        maps = std::make_shared<ModelMaps>();
+        // --- make the maps and score the 2mFo-DFc map at atom centres (a real map peaks there) ---
         // The three maps - this one, the difference map and the anomalous map - share nothing but what
-        // they read, so the other two are made and written on threads of their own beside this one.
+        // they read, so the other two are made on threads of their own beside this one. Nothing here
+        // reads the difference map, so it is made only where it is kept.
         std::future<void> fofc_map = std::async(std::launch::async, [&] {
-            write_ccp4(map_from_coefficients(mapfofc, sf_gpu), output_prefix + "_fofc.ccp4");
+            if (schedule.keep_map_grids)
+                maps->map_fofc = map_from_coefficients(mapfofc, sf_gpu);
         });
 
         // --- anomalous difference map, where the merge kept the Bijvoet split ---
@@ -1764,13 +1747,15 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                                            result.indexing_op, fmodel);
             result.anomalous_pairs = static_cast<int>(mapanom.v.size());
             if (!mapanom.v.empty()) {
-                const gemmi::Grid<float> grid = map_from_coefficients(mapanom, sf_gpu);
-                read_anomalous_map(grid, write_ccp4(grid, output_prefix + "_anom.ccp4"), st, result);
+                gemmi::Grid<float> grid = map_from_coefficients(mapanom, sf_gpu);
+                read_anomalous_map(grid, map_rms(grid), st, result);
+                if (schedule.keep_map_grids)
+                    maps->map_anom = std::move(grid);
             }
         });
 
-        const gemmi::Grid<float> grid2fofc = map_from_coefficients(map2fofc, sf_gpu);
-        const double rms2 = write_ccp4(grid2fofc, output_prefix + "_2fofc.ccp4");
+        gemmi::Grid<float> grid2fofc = map_from_coefficients(map2fofc, sf_gpu);
+        const double rms2 = map_rms(grid2fofc);
         {
             double s = 0; int n = 0;
             for (gemmi::Model &m : st.models)
@@ -1779,6 +1764,8 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                         for (gemmi::Atom &a : r.atoms) { s += grid2fofc.interpolate_value(a.pos, MAP_INTERPOLATION_ORDER); ++n; }
             result.mean_atom_density_sigma = (n > 0 && rms2 > 0) ? (s / n) / rms2 : 0;
         }
+        if (schedule.keep_map_grids)
+            maps->map_2fofc = std::move(grid2fofc);
         fofc_map.get();
         anomalous_map.get();
     }
@@ -1803,60 +1790,53 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                            : "the model does not fit these data");
 
     if (!maps_superseded) {
-        // --- MTZ of map coefficients so the maps can be re-opened / rebuilt in Coot etc. ---
-        try {
-            gemmi::Mtz mtz(true);
-            // The group the REFLECTIONS end up in, which is the model's only where its hand was adopted -
-            // AdoptModelFrame decides this the same way a few lines below. An enantiomorphic pair indexes
-            // identically, so the coefficients are the same numbers either way and only the label moves;
-            // but the two groups have different screw translations, so a reader that expands symmetry out
-            // of this file works in the wrong one if the label disagrees with the .mtz beside it.
-            mtz.spacegroup = result.adopted_model_enantiomorph
-                                 ? sg
-                                 // No data group means the caller merged in P1, and P1 is what the
-                                 // reflection files beside this one carry - not the model's group.
-                                 : (data_space_group != nullptr ? data_space_group
-                                                                : gemmi::find_spacegroup_by_number(1));
-            mtz.set_cell_for_all(ucell);
-            mtz.add_dataset("model_validation");
-            mtz.datasets.back().wavelength = wavelength_A;
-            mtz.add_column("FP", 'F', -1, -1, false);
-            mtz.add_column("FC", 'F', -1, -1, false);
-            mtz.add_column("PHIC", 'P', -1, -1, false);
-            mtz.add_column("FWT", 'F', -1, -1, false);
-            mtz.add_column("PHWT", 'P', -1, -1, false);
-            mtz.add_column("DELFWT", 'F', -1, -1, false);
-            mtz.add_column("PHDELWT", 'P', -1, -1, false);
-            // The figure of merit the coefficients carry, so the weighting can be read off - and undone -
-            // from the file rather than having to be taken on trust.
-            mtz.add_column("FOM", 'W', -1, -1, false);
-            mtz.add_column("FREE", 'I', -1, -1, false);
-            std::vector<float> data;
-            for (size_t i = 0; i < terms.size(); i++) {
-                // [0, 360), the convention the MTZ 'P' columns carried before sigma_A weighting: std::arg
-                // returns (-180, 180] and a P column is not supposed to.
-                const double phi_wrapped = terms[i].phi * 180.0 / PI;
-                const auto phi_deg = static_cast<float>(phi_wrapped < 0 ? phi_wrapped + 360.0 : phi_wrapped);
-                data.insert(data.end(), {static_cast<float>(terms[i].hkl[0]), static_cast<float>(terms[i].hkl[1]),
-                                         static_cast<float>(terms[i].hkl[2]),
-                                         static_cast<float>(terms[i].fo), static_cast<float>(terms[i].fc),
-                                         phi_deg,
-                                         fwt[i], phi_deg,
-                                         delfwt[i], phi_deg,
-                                         static_cast<float>(sigma_a.weight[i].m),
-                                         terms[i].free ? 0.0f : 1.0f});
-            }
-            mtz.nreflections = static_cast<int>(terms.size());
-            mtz.data = std::move(data);
-            remove_before_rewriting(output_prefix + "_maps.mtz");
-            mtz.write_to_file(output_prefix + "_maps.mtz");
-        } catch (const std::exception &e) {
-            logger.Warning("Model validation: could not write map MTZ: {}", e.what());
+        // --- the map coefficients, so the maps can be re-opened / rebuilt in Coot etc. ---
+        MtzTable &mtz = maps->coefficients;
+        // The group the REFLECTIONS end up in, which is the model's only where its hand was adopted -
+        // AdoptModelFrame decides this the same way a few lines below. An enantiomorphic pair indexes
+        // identically, so the coefficients are the same numbers either way and only the label moves;
+        // but the two groups have different screw translations, so a reader that expands symmetry out
+        // of this file works in the wrong one if the label disagrees with the .mtz beside it.
+        mtz.space_group_name = result.adopted_model_enantiomorph
+                               ? sg->xhm()
+                               // No data group means the caller merged in P1, and P1 is what the
+                               // reflection files beside this one carry - not the model's group.
+                               : (data_space_group != nullptr ? data_space_group->xhm()
+                                                              : gemmi::find_spacegroup_by_number(1)->xhm());
+        mtz.unit_cell = UnitCell{static_cast<float>(ucell.a), static_cast<float>(ucell.b),
+                                 static_cast<float>(ucell.c), static_cast<float>(ucell.alpha),
+                                 static_cast<float>(ucell.beta), static_cast<float>(ucell.gamma)};
+        mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                        {"model_validation", "model_validation", "model_validation",
+                         static_cast<float>(wavelength_A)}};
+        // The figure of merit the coefficients carry, so the weighting can be read off - and undone -
+        // from the file rather than having to be taken on trust.
+        const std::pair<const char *, char> columns[] = {{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"FP", 'F'},
+                                                         {"FC", 'F'}, {"PHIC", 'P'}, {"FWT", 'F'}, {"PHWT", 'P'},
+                                                         {"DELFWT", 'F'}, {"PHDELWT", 'P'}, {"FOM", 'W'},
+                                                         {"FREE", 'I'}};
+        for (const auto &[label, type] : columns)
+            mtz.columns.push_back({label, type, type == 'H' ? 0 : 1, {}});
+        for (size_t i = 0; i < terms.size(); i++) {
+            // [0, 360), the convention the MTZ 'P' columns carried before sigma_A weighting: std::arg
+            // returns (-180, 180] and a P column is not supposed to.
+            const double phi_wrapped = terms[i].phi * 180.0 / PI;
+            const auto phi_deg = static_cast<float>(phi_wrapped < 0 ? phi_wrapped + 360.0 : phi_wrapped);
+            const float row[] = {static_cast<float>(terms[i].hkl[0]), static_cast<float>(terms[i].hkl[1]),
+                                 static_cast<float>(terms[i].hkl[2]),
+                                 static_cast<float>(terms[i].fo), static_cast<float>(terms[i].fc),
+                                 phi_deg,
+                                 fwt[i], phi_deg,
+                                 delfwt[i], phi_deg,
+                                 static_cast<float>(sigma_a.weight[i].m),
+                                 terms[i].free ? 0.0f : 1.0f};
+            for (size_t c = 0; c < mtz.columns.size(); c++)
+                mtz.columns[c].values.push_back(row[c]);
         }
     }
 
     result.ok = true;
-    result.maps_prefix = output_prefix;
+    result.maps = std::move(maps);
     // The placed coordinates, for the caller to write out beside the maps once the frame is settled.
     result.placed_model = std::make_shared<gemmi::Structure>(st);
     logger.Info("Model validation: R-work={:.4f} ({} refl)  R-free={:.4f} ({} refl)  "
@@ -1885,12 +1865,7 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
                        "data were indexed in the wrong hand, and reindexing would hide which",
                        result.anomalous_deepest_site, result.anomalous_deepest_sigma);
     if (maps_superseded)
-        logger.Info("Model validation: no maps made here - the validation in the model's setting writes them");
-    else
-        logger.Info("Model validation: wrote {}_2fofc.ccp4, {}_fofc.ccp4{}, {}_maps.mtz",
-                    output_prefix, output_prefix,
-                    result.anomalous_sites.empty() ? "" : ", " + output_prefix + "_anom.ccp4",
-                    output_prefix);
+        logger.Info("Model validation: no maps made here - the validation in the model's setting makes them");
     if (result.fit_tested && !result.model_fits)
         logger.Info("Model validation: the model was rejected, so nothing downstream moved - the "
                     "reflection files are byte for byte the ones a run with no model would have "
@@ -1975,8 +1950,7 @@ std::vector<size_t> OtherCardsTotalMemory(int device) {
 
 ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &merged,
                                            const UnitCell &cell,
-                                           const std::string &model_path,
-                                           const std::string &output_prefix,
+                                           const RugnuxModel &model,
                                            Logger &logger,
                                            const gemmi::SpaceGroup *data_space_group,
                                            bool probe_indexing_ambiguity,
@@ -1985,18 +1959,18 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
                                            const std::vector<float> &report_shell_d_min,
                                            const ModelValidationSchedule &schedule,
                                            const gemmi::Op *twin_law,
-                                           const gemmi::Structure *model) {
+                                           const gemmi::Structure *placed) {
 #ifdef JFJOCH_USE_CUDA
     // A CUDA failure ends the validation, not the run, and nothing is moved to the CPU part way through:
     // the validation reports why it did not finish, and a validation that did not finish decides nothing,
     // so the reflection files are those of a run without a model.
     try {
-        return Validate(merged, cell, model_path, output_prefix, logger, data_space_group,
+        return Validate(merged, cell, model, logger, data_space_group,
                         probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, schedule, twin_law,
-                        true, model);
+                        true, placed);
     } catch (const RigidBodyGPUFailure &e) {
         ModelValidationResult failed;
-        failed.model_path = model_path;
+        failed.model_name = model.name;
         failed.failure_reason = fmt::format("the GPU failed during the validation ({})", e.what());
         logger.Error("Model validation: {}", failed.failure_reason);
         cuda_throw_if_context_lost();   // a lost device ends the run here, not at its next user
@@ -2004,8 +1978,8 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
         return failed;
     }
 #else
-    return Validate(merged, cell, model_path, output_prefix, logger, data_space_group, probe_indexing_ambiguity,
-                    nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, false, model);
+    return Validate(merged, cell, model, logger, data_space_group, probe_indexing_ambiguity,
+                    nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, false, placed);
 #endif
 }
 
@@ -2161,7 +2135,7 @@ void KeepModelVerdict(ModelValidationResult &remade, const ModelValidationResult
     remade.model_space_group_number = decided.model_space_group_number;
 }
 
-std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model_path,
+std::vector<MergedReflection> ModelReferenceIntensities(const RugnuxModel &model,
                                                         const std::optional<UnitCell> &cell,
                                                         const gemmi::SpaceGroup *space_group,
                                                         double d_min,
@@ -2172,19 +2146,15 @@ std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model
         return out;
     }
 
-    gemmi::Structure st;
+    gemmi::Structure st = model.structure;
     try {
-        // Detect, not the default: without it GEMMI picks the format from the extension and only
-        // falls back to the content when it does not recognise one. A model arrives named however
-        // whoever produced it named it, so the file itself is the better authority.
-        st = gemmi::read_structure_gz(model_path, gemmi::CoorFormat::Detect);
         expand_strict_ncs(st);
     } catch (const std::exception &e) {
-        logger.Error("Model reference: cannot read model {}: {}", model_path, e.what());
+        logger.Error("Model reference: cannot read model {}: {}", model.name, e.what());
         return out;
     }
     if (st.models.empty() || !st.cell.is_crystal()) {
-        logger.Error("Model reference: model {} has no atoms or no unit cell", model_path);
+        logger.Error("Model reference: model {} has no atoms or no unit cell", model.name);
         return out;
     }
 
@@ -2207,7 +2177,7 @@ std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model
         st.spacegroup_hm = space_group->xhm();
     const gemmi::SpaceGroup *sg = st.find_spacegroup();
     if (!sg) {
-        logger.Error("Model reference: model {} has no usable space group", model_path);
+        logger.Error("Model reference: model {} has no usable space group", model.name);
         return out;
     }
     st.setup_cell_images();
@@ -2257,6 +2227,6 @@ std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model
     }
 
     logger.Info("Model reference: {} intensities computed from {} to {:.2f} A, space group {}",
-                out.size(), model_path, d_min, sg->short_name());
+                out.size(), model.name, d_min, sg->short_name());
     return out;
 }

image_analysis/structure_refinement/ModelValidation.h

@@ -11,18 +11,29 @@
 #include <string>
 #include <vector>
 
+#include "gemmi/grid.hpp"
 #include "gemmi/symmetry.hpp"
 
 #include "../../common/Reflection.h"   // MergedReflection
 #include "../../common/UnitCell.h"
+#include "../../common/MtzTable.h"
+#include "RugnuxModel.h"
 
 class Logger;
 namespace gemmi { struct Structure; }
 
+// The maps of a validation: their coefficients, as <prefix>_maps.mtz holds them, and the maps themselves,
+// as the .ccp4 files hold them, where the caller asked for them (ModelValidationSchedule::keep_map_grids).
+// The anomalous map is there only where the merge kept the Bijvoet split.
+struct ModelMaps {
+    MtzTable coefficients;
+    std::optional<gemmi::Grid<float>> map_2fofc, map_fofc, map_anom;
+};
+
 // Result of validating merged intensities against an atomic model.
 struct ModelValidationResult {
     bool ok = false;
-    std::string model_path;        // the file that was asked for, so the report can name it
+    std::string model_name;        // RugnuxModel::name, so the report can name the model that was asked for
     // Why it did not run, when ok is false: the file could not be read, had no atoms, no cell or no
     // usable space group. A caller that asked for a model must be able to say so rather than finish
     // quietly with no R-free and no maps, which reads exactly like a run that was never asked.
@@ -57,7 +68,11 @@ struct ModelValidationResult {
     double radial_misfit = NAN;
     int shell_scale_bins = 0;
     double mean_atom_density_sigma = 0.0;  // mean 2mFo-DFc value at atom centres, in sigma
-    std::string maps_prefix;               // where the .ccp4 / _maps.mtz were written
+    // The maps, where this validation made them; the caller takes them over (null after that).
+    std::shared_ptr<ModelMaps> maps;
+    // Where the caller puts the maps beside the reflection files: "" for the answer's, "_mr" for the
+    // model's choice. Set by the caller; the report names <prefix><maps_suffix>.
+    std::string maps_suffix;
 
     // The rigid-body placement of the model against these data, where it was committed. The step is
     // taken only when it lowers R-free, so an unmoved model means the data said no, not that nothing
@@ -208,11 +223,10 @@ struct ModelValidationResult {
     std::shared_ptr<gemmi::Structure> placed_model;
 };
 
-// Given merged intensities and an atomic model (PDB or mmCIF, gzipped or not - the format is taken
-// from the file's content), scale the model structure factors (with an
+// Given merged intensities and an atomic model, scale the model structure factors (with an
 // optimized flat bulk-solvent contribution and an overall anisotropic B) to the WORKING observed
-// amplitudes, then report R-work / R-free and write 2Fo-Fc and Fo-Fc electron-density maps
-// (CCP4) plus an MTZ of map coefficients next to output_prefix. Nothing the reported R-free is
+// amplitudes, then report R-work / R-free and make 2Fo-Fc and Fo-Fc electron-density maps and
+// their coefficients (ModelValidationResult::maps). Nothing the reported R-free is
 // computed against was fitted on the free reflections: the scale, like the rigid-body placement
 // below, sees the working set only and is then applied to all of them.
 //
@@ -242,8 +256,8 @@ struct ModelValidationResult {
 // twin_law, where given, is a twin law of the data's group (a Miller-index operator in the data's
 // basis): the model is then also scored against the data as twinned by it (twin_target and below).
 //
-// model, where given, is the model itself - one already placed in the data's cell, by molecular
-// replacement - and model_path only names it; no other setting is then probed for it.
+// placed, where given, is the model itself - one already placed in the data's cell, by molecular
+// replacement - and `model` only names it; no other setting is then probed for it.
 //
 // report_shell_d_min is the merge statistics' own shell bounds, coarse to fine, and it is what
 // cc_model_shells is binned on. Sharing the grid is the point: a reader has to be able to put a
@@ -266,10 +280,10 @@ struct ModelFrameForecast {
 struct ModelValidationSchedule {
     // Called once, before the null, with the forecast above.
     std::function<void(const ModelFrameForecast &)> on_forecast;
-    // Where valid, waited on before the first file is written: false returns without writing any.
-    std::shared_future<bool> write_gate;
+    // Where valid, waited on before the maps are made: false returns without making any.
+    std::shared_future<bool> maps_gate;
     // Asked once the verdict, the setting and the indexing are decided, before any map is made: true
-    // where another validation will write this one's maps and map coefficients, which are then not made.
+    // where another validation's maps and map coefficients are the ones kept, so this one's are not made.
     std::function<bool(const ModelValidationResult &)> maps_superseded;
     // The null that says whether the model fits. False where the verdict is another validation's
     // (KeepModelVerdict) or was reached elsewhere (a molecular-replacement placement, scored for its maps).
@@ -280,6 +294,8 @@ struct ModelValidationSchedule {
     // GPU memory another validation running beside this one holds there. It limits how many engines
     // this one takes and never what runs on the GPU, so no number depends on whether the other runs.
     size_t gpu_beside_bytes = 0;
+    // Keep the maps themselves (ModelMaps), not only their coefficients: rugnux writes them as .ccp4.
+    bool keep_map_grids = false;
 };
 
 // What a model validation needs of its card, part by part: the bytes of one engine of each, how many
@@ -344,8 +360,7 @@ std::vector<size_t> OtherCardsTotalMemory(int device);
 
 ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &merged,
                                            const UnitCell &cell,
-                                           const std::string &model_path,
-                                           const std::string &output_prefix,
+                                           const RugnuxModel &model,
                                            Logger &logger,
                                            const gemmi::SpaceGroup *data_space_group = nullptr,
                                            bool probe_indexing_ambiguity = true,
@@ -354,7 +369,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
                                            const std::vector<float> &report_shell_d_min = {},
                                            const ModelValidationSchedule &schedule = {},
                                            const gemmi::Op *twin_law = nullptr,
-                                           const gemmi::Structure *model = nullptr);
+                                           const gemmi::Structure *placed = nullptr);
 
 // Reindex `merged` into the frame ValidateAgainstModel reported, so the reflection files that are
 // written describe the same indexing as the R-factors and the maps. Returns the space group they are
@@ -393,7 +408,7 @@ ModelValidationResult ReadModelAnomalousMap(const std::vector<MergedReflection>
 // serial-stills run: there the ambiguity has to be broken per image, at integration time, long before
 // there is a merge to fit a model to. `cell` and `space_group` override the model's own, where
 // the run already knows them. Empty on any failure (which is logged).
-std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model_path,
+std::vector<MergedReflection> ModelReferenceIntensities(const RugnuxModel &model,
                                                         const std::optional<UnitCell> &cell,
                                                         const gemmi::SpaceGroup *space_group,
                                                         double d_min,

image_analysis/structure_refinement/MolecularReplacement.cpp

@@ -18,15 +18,11 @@
 #include <tuple>
 #include <random>
 #include <unordered_map>
-#include <fstream>
 
 #include <fftw3.h>
-#include <nlohmann/json.hpp>
 
-#include <gemmi/mmread_gz.hpp>   // read_structure_gz
 #include <gemmi/modify.hpp>      // remove_hydrogens, remove_alternative_conformations
 #include <gemmi/polyheur.hpp>    // setup_entities, remove_ligands_and_waters
-#include <gemmi/read_cif.hpp>    // read_cif_gz
 #include <gemmi/dencalc.hpp>     // DensityCalculator
 #include <gemmi/it92.hpp>        // IT92 x-ray form factors
 #include <gemmi/scaling.hpp>     // Scaling (bulk solvent + anisotropic B)
@@ -906,19 +902,10 @@ void finish_search_model(SearchModel &m) {
 
 }  // namespace
 
-SearchModel PrepareSearchModel(const std::string &path) {
+SearchModel PrepareSearchModel(const RugnuxModel &model_in) {
     SearchModel out;
-    bool modelcif = false;
-    try {
-        out.st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect);
-        if (out.st.input_format == gemmi::CoorFormat::Mmcif) {
-            const gemmi::cif::Document doc = gemmi::read_cif_gz(path);
-            modelcif = !doc.blocks.empty() && doc.blocks[0].has_mmcif_category("_ma_qa_metric");
-        }
-    } catch (const std::exception &e) {
-        out.failure_reason = std::string("cannot read the model: ") + e.what();
-        return out;
-    }
+    const bool modelcif = model_in.modelcif;
+    out.st = model_in.structure;
     if (out.st.models.empty()) {
         out.failure_reason = "the model has no atoms";
         return out;
@@ -992,39 +979,15 @@ SearchModel PrepareSearchModel(const std::string &path) {
 
 // --- domains from the predicted aligned error ---
 
-std::string PaePathFor(const std::string &model_path) {
-    const size_t slash = model_path.find_last_of("/\\");
-    const std::string dir = slash == std::string::npos ? std::string() : model_path.substr(0, slash + 1);
-    const std::string name = model_path.substr(dir.size());
-    const size_t at = name.rfind("-model_v");
-    if (at == std::string::npos)
-        return {};
-    const size_t dot = name.find('.', at);
-    const std::string version = name.substr(at + 8, dot == std::string::npos ? std::string::npos : dot - at - 8);
-    return dir + name.substr(0, at) + "-predicted_aligned_error_v" + version + ".json";
-}
-
 // A prediction cut into domains by its predicted aligned error before molecular replacement, as Oeffner et
 // al. (2022) Acta Cryst. D78, 1303-1314 do; here by average linkage.
-std::vector<std::vector<int>> PaeDomains(const std::string &pae_path, const SearchModel &model) {
+std::vector<std::vector<int>> PaeDomains(const std::vector<std::vector<double>> &pae, const SearchModel &model) {
     // Two domains a prediction does not place relative to each other with confidence are this far apart in
     // mean PAE (within one domain it is a few angstroms); a domain is at least this many residues.
     constexpr double PAE_DOMAIN_A = 10.0;
     constexpr int DOMAIN_MIN_RESIDUES = 40;
     // The clustering starts from runs of this many consecutive residues kept.
     constexpr int PIECE = 5;
-    std::vector<std::vector<double>> pae;
-    try {
-        std::ifstream in(pae_path);
-        if (!in)
-            return {};
-        const nlohmann::json j = nlohmann::json::parse(in);
-        const nlohmann::json &m = j.is_array() ? j.at(0).at("predicted_aligned_error") : j.at("predicted_aligned_error");
-        for (const auto &row : m)
-            pae.push_back(row.get<std::vector<double>>());
-    } catch (const std::exception &) {
-        return {};
-    }
     const size_t n = pae.size();
     if (n != static_cast<size_t>(model.residues_read)
         || std::any_of(pae.begin(), pae.end(), [n](const std::vector<double> &r) { return r.size() != n; }))
@@ -3576,7 +3539,7 @@ std::vector<TranslationPeak> SplitTranslationFunction(const std::vector<Molecula
 }
 #endif
 
-MolecularReplacementResult RunMolecularReplacement(const std::string &model_path, const std::vector<MRMerge> &merges,
+MolecularReplacementResult RunMolecularReplacement(const RugnuxModel &input, const std::vector<MRMerge> &merges,
                                                    Logger &logger, size_t nthreads, size_t gpu_resident_bytes) {
     const auto t0 = std::chrono::steady_clock::now();
     // What the search may hold on the card: the budget model validation keeps to. What the stream-ordered pool
@@ -3596,25 +3559,24 @@ MolecularReplacementResult RunMolecularReplacement(const std::string &model_path
 #else
     (void) gpu_resident_bytes;
 #endif
-    SearchModel model = PrepareSearchModel(model_path);
+    SearchModel model = PrepareSearchModel(input);
     if (!model.failure_reason.empty()) {
         MolecularReplacementResult result;
-        result.model_path = model_path;
+        result.model_name = input.name;
         result.residues_read = model.residues_read;
         result.failure_reason = model.failure_reason;
         return result;
     }
     model.description = "the whole model";
     // Only a prediction has a predicted aligned error.
-    const std::string pae_path = model.predicted ? PaePathFor(model_path) : std::string();
-    const std::vector<std::vector<int>> domains = pae_path.empty() ? std::vector<std::vector<int>>()
-                                                                   : PaeDomains(pae_path, model);
+    const bool pae = model.predicted && input.pae.has_value();
+    const std::vector<std::vector<int>> domains = pae ? PaeDomains(*input.pae, model) : std::vector<std::vector<int>>();
     const double t_model = seconds_since(t0);
-    if (!pae_path.empty())
-        logger.Info("Molecular replacement: {} rigid domain(s) in the predicted aligned error {}", domains.size(), pae_path);
+    if (pae)
+        logger.Info("Molecular replacement: {} rigid domain(s) in the predicted aligned error", domains.size());
 
     MolecularReplacementResult result = search_model(model, merges, logger, nthreads, gpu_budget, other_budgets);
-    result.model_path = model_path;
+    result.model_name = input.name;
     result.pae_domains = static_cast<int>(domains.size());
     result.t_model += t_model;
     // Where the whole model is placed nowhere and its prediction falls apart into domains whose relative
@@ -3664,7 +3626,7 @@ MolecularReplacementResult RunMolecularReplacement(const std::string &model_path
                                           found[best].candidates[found[best].chosen].space_group.xhm());
         } else if (best >= 0 && agree) {
             MolecularReplacementResult chosen = std::move(found[best]);
-            chosen.model_path = model_path;
+            chosen.model_name = input.name;
             chosen.pae_domains = result.pae_domains;
             chosen.t_model = result.t_model;
             chosen.t_transform = result.t_transform;

image_analysis/structure_refinement/MolecularReplacement.h

@@ -35,6 +35,7 @@
 
 #include "../../common/Reflection.h"   // MergedReflection
 #include "../../common/UnitCell.h"
+#include "RugnuxModel.h"
 
 class Logger;
 
@@ -55,19 +56,16 @@ struct SearchModel {
 };
 // A prediction where the file says so - ModelCIF's _ma_qa_metric - or where it has no cell and its B column
 // holds one value per residue, from 0 to 100; an experimental structure otherwise.
-SearchModel PrepareSearchModel(const std::string &path);
+SearchModel PrepareSearchModel(const RugnuxModel &model);
 
-// The rigid domains of a prediction from its predicted aligned error (PAE, the AlphaFold DB's JSON: a matrix
+// The rigid domains of a prediction from its predicted aligned error (PAE, the AlphaFold DB's matrix
 // "predicted_aligned_error" over the residues of the whole prediction): the kept residues clustered by average
 // linkage on the symmetrised PAE until every two clusters are a mean PAE_DOMAIN_A apart; clusters of at least
-// DOMAIN_MIN_RESIDUES, largest first, each a list of positions in the prediction. Empty where the file cannot
-// be read or does not match the model.
-std::vector<std::vector<int>> PaeDomains(const std::string &pae_path, const SearchModel &model);
+// DOMAIN_MIN_RESIDUES, largest first, each a list of positions in the prediction. Empty where the matrix does
+// not match the model.
+std::vector<std::vector<int>> PaeDomains(const std::vector<std::vector<double>> &pae, const SearchModel &model);
 // The search model of one domain: the residues of `model` at those positions, recentred.
 SearchModel DomainModel(const SearchModel &model, const std::vector<int> &residues);
-// The PAE file the AlphaFold DB puts beside a model: AF-<id>-F1-model_v<n>.cif ->
-// AF-<id>-F1-predicted_aligned_error_v<n>.json in the same directory; empty where the name does not follow it.
-std::string PaePathFor(const std::string &model_path);
 
 // Rotations covering SO(3) with neighbours about `step_deg` apart: the identity, then shells of
 // rotation angle kappa = step, 2 step, ... 180 deg, each with its axes on a golden-spiral lattice of
@@ -342,7 +340,7 @@ struct MRCandidate {
 struct MolecularReplacementResult {
     bool ok = false;
     std::string failure_reason;
-    std::string model_path;
+    std::string model_name;                          // RugnuxModel::name
     int residues_read = 0, residues_kept = 0;
     double mass_da = 0;
     std::string search_model;                        // the whole model, or the PAE domain the decision was made with
@@ -442,7 +440,7 @@ double MolecularReplacementGPUSeconds(const MolecularReplacementGPUNeed &need, c
 // counts alone, never on free memory.
 bool PackMolecularTransformGPU(const MolecularReplacementGPUNeed &need, size_t packed);
 
-// Search `merges` (the answer's first) for the predicted model at `model_path`, in every space group with
+// Search `merges` (the answer's first) for the predicted `model`, in every space group with
 // the rotations of each merge's group - one copy, then further copies one at a time while each new one
 // stands out, packs and lowers R-free - and decide. A candidate is PLACED where a copy's translation
 // function stands out (TF Z >= 8), the copies pack, one copy fits the cell, and its R-work beats its own
@@ -450,7 +448,7 @@ bool PackMolecularTransformGPU(const MolecularReplacementGPUNeed &need, size_t p
 // every other by an R-free margin; it chooses the twin hypothesis' merge only where none of the
 // answer's groups can hold the model at all (Matthews, packing) and a group of the subgroup can.
 // `gpu_resident_bytes`: what is held on the card meanwhile by what came before (a merge engine still at work).
-MolecularReplacementResult RunMolecularReplacement(const std::string &model_path, const std::vector<MRMerge> &merges,
+MolecularReplacementResult RunMolecularReplacement(const RugnuxModel &model, const std::vector<MRMerge> &merges,
                                                    Logger &logger, size_t nthreads, size_t gpu_resident_bytes = 0);
 
 // The hand check of an ENANTIOMORPH decision, filled into result.hand_check: taken into the answer by

image_analysis/structure_refinement/RugnuxModel.h

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "gemmi/model.hpp"
+
+// An atomic model handed to a run (rugnux --model / --mr, the broker's upload), as its file held it. The
+// caller parses it (ParseRugnuxModel, writer/RugnuxModelFile.h); the analysis reads no file.
+struct RugnuxModel {
+    std::string name;              // what the report calls it: the file it was given as
+    gemmi::Structure structure;    // as read, input_format kept
+    bool modelcif = false;         // a ModelCIF file (_ma_qa_metric): a prediction, whatever its B column holds
+    // A prediction's predicted aligned error over the residues of the whole prediction, where it came with one.
+    std::optional<std::vector<std::vector<double>>> pae;
+};

image_pusher/HDF5FilePusher.cpp

@@ -66,6 +66,7 @@ bool HDF5FilePusher::EndDataCollection(const EndMessage &message) {
         if (!writer)
             throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Image pusher not ready for writing images");
         writer->WriteHDF5(message);
+        writer->WriteResultFiles(message, logger);
         writer->Finalize();
     } catch (...) {
         writer.reset();

reader/CMakeLists.txt

@@ -19,6 +19,8 @@ ADD_LIBRARY(JFJochReader STATIC
         HDF5ImageLocator.cpp
         JFJochStartMessageReader.cpp
         JFJochStartMessageReader.h
+        JFJochImageBufferReader.cpp
+        JFJochImageBufferReader.h
         HDF5ImageLocator.h
         HDF5ImageSource.cpp
         HDF5ImageSource.h

reader/JFJochImageBufferReader.cpp

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "JFJochImageBufferReader.h"
+
+#include <algorithm>
+#include <cstring>
+
+#include "JFJochStartMessageReader.h"
+#include "../common/JFJochException.h"
+#include "../frame_serialize/CBORStream2Deserializer.h"
+
+JFJochImageBufferReader::JFJochImageBufferReader(ImageBuffer &buffer) : buffer_(buffer) {}
+
+void JFJochImageBufferReader::Open(const StartMessage &start) {
+    {
+        std::unique_lock ul(m_);
+        number_of_images_ = start.number_of_images;
+        present_.assign(number_of_images_, false);
+        max_present_ = -1;
+        ended_ = false;
+    }
+    SetStartMessage(DatasetFromStartMessage(start, default_experiment));
+}
+
+void JFJochImageBufferReader::Ended() {
+    {
+        std::unique_lock ul(m_);
+        ended_ = true;
+    }
+    cv_.notify_all();
+}
+
+uint64_t JFJochImageBufferReader::GetNumberOfImages() const {
+    return number_of_images_;
+}
+
+void JFJochImageBufferReader::Close() {
+    SetStartMessage({});
+}
+
+void JFJochImageBufferReader::Poll(std::unique_lock<std::mutex> &ul) {
+    polling_ = true;
+    next_poll_ = std::chrono::steady_clock::now() + POLL_PERIOD;
+    ul.unlock();
+    const std::vector<int64_t> images = buffer_.GetStatus().images_in_the_buffer;
+    ul.lock();
+    polling_ = false;
+    for (const int64_t i: images)
+        if (i >= 0 && static_cast<uint64_t>(i) < number_of_images_) {
+            present_[i] = true;
+            max_present_ = std::max(max_present_, i);
+        }
+    cv_.notify_all();
+}
+
+bool JFJochImageBufferReader::WaitForMessage(int64_t image_number, std::vector<uint8_t> &cbor) {
+    if (image_number < 0 || static_cast<uint64_t>(image_number) >= number_of_images_)
+        throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Image out of bounds");
+    std::unique_lock ul(m_);
+    // Images arrive in order, give or take a few: one still missing when the buffer already holds an
+    // image MAX_IMAGE_LAG past it is lost, not late. After the end, one is asked for once more, as it
+    // may have come in just before.
+    while (!present_[image_number] && !ended_ && max_present_ < image_number + MAX_IMAGE_LAG) {
+        const auto now = std::chrono::steady_clock::now();
+        if (!polling_ && now >= next_poll_)
+            Poll(ul);
+        else
+            cv_.wait_until(ul, polling_ ? now + POLL_PERIOD : next_poll_);
+    }
+    ul.unlock();
+    return buffer_.GetImage(cbor, image_number);
+}
+
+bool JFJochImageBufferReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &ret) {
+    if (!WaitForMessage(image_number, ret.read_buffer))
+        return false;
+    const auto msg = CBORStream2Deserialize(ret.read_buffer);
+    if (msg->msg_type != CBORImageType::IMAGE || !msg->data_message)
+        return false;
+    // Copied out of the message: read_buffer is scratch a caller may hand to the next read.
+    const CompressedImage &image = msg->data_message->image;
+    ret.image_buffer.resize(image.GetCompressedSize());
+    memcpy(ret.image_buffer.data(), image.GetCompressed(), image.GetCompressedSize());
+    ret.image = CompressedImage(ret.image_buffer.data(), ret.image_buffer.size(), image.GetWidth(),
+                                image.GetHeight(), image.GetMode(), image.GetCompressionAlgorithm());
+    return true;
+}
+
+bool JFJochImageBufferReader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset, DataMessage &message,
+                                          std::vector<uint8_t> &buffer, int64_t image_number, bool) {
+    if (!dataset || !WaitForMessage(image_number, buffer))
+        return false;
+    const auto msg = CBORStream2Deserialize(buffer);
+    if (msg->msg_type != CBORImageType::IMAGE || !msg->data_message)
+        return false;
+    message = *msg->data_message;
+    return true;
+}

reader/JFJochImageBufferReader.h

@@ -0,0 +1,59 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "JFJochReader.h"
+#include "../common/ImageBuffer.h"
+
+// Reads a collection inside the broker while it arrives: the dataset comes from its start message,
+// the images from the broker's image buffer, where the receiver puts each one as it forwards it. A
+// read of an image not there yet waits for it; after Ended it returns false for an image that never
+// came, and so does a read of an image still missing when one MAX_IMAGE_LAG past it is there. The
+// buffer must hold the whole collection (slots >= images) - a slot taken again for a later
+// image would hand out that image instead. However many threads wait, one at a time looks at the buffer,
+// every POLL_PERIOD, so the waiting stays off the lock the receiver puts the images in under.
+class JFJochImageBufferReader : public JFJochReader {
+public:
+    constexpr static int64_t MAX_IMAGE_LAG = 100;
+    constexpr static auto POLL_PERIOD = std::chrono::milliseconds(2);
+
+    explicit JFJochImageBufferReader(ImageBuffer &buffer);
+
+    void Open(const StartMessage &start);
+    // No image comes after this: the collection is over, or abandoned.
+    void Ended();
+
+    [[nodiscard]] uint64_t GetNumberOfImages() const override;
+    void Close() override;
+    bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override;
+    std::vector<SpotToSave> ReadSpots(int64_t image) const override { return {}; }
+
+private:
+    bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset, DataMessage &message,
+                     std::vector<uint8_t> &buffer, int64_t image_number, bool update_dataset) override;
+    // The image's CBOR message, as the receiver put it in the buffer; false when it never came.
+    bool WaitForMessage(int64_t image_number, std::vector<uint8_t> &cbor);
+
+    // Looks at the buffer (GetStatus) and notes the images it holds in present_; under m_, released while
+    // the buffer is asked.
+    void Poll(std::unique_lock<std::mutex> &ul);
+
+    ImageBuffer &buffer_;
+    uint64_t number_of_images_ = 0;
+    std::atomic<bool> ended_ = false;
+
+    std::mutex m_;
+    std::condition_variable cv_;
+    bool polling_ = false;
+    std::chrono::steady_clock::time_point next_poll_;
+    std::vector<bool> present_;
+    int64_t max_present_ = -1;
+};

reader/JFJochStartMessageReader.cpp

@@ -15,78 +15,77 @@ namespace {
         const std::string prefix = std::filesystem::path(start.file_prefix).filename().string();
         return fmt::format("{:s}_data_{:06d}.h5", prefix, file_number + 1);
     }
+}
 
-    // The dataset as the start message describes it - the same quantities an HDF5 master states, taken
-    // from the message the writer wrote it from.
-    std::shared_ptr<JFJochReaderDataset> Dataset(const StartMessage &start, const DiffractionExperiment &defaults) {
-        auto dataset = std::make_shared<JFJochReaderDataset>();
-        dataset->arm_date = start.arm_date;
-        dataset->jfjoch_release = start.jfjoch_release;
-        dataset->error_value = start.error_value;
-        dataset->file_detect_ice_rings = start.detect_ice_rings;
+std::shared_ptr<JFJochReaderDataset> DatasetFromStartMessage(const StartMessage &start,
+                                                             const DiffractionExperiment &defaults) {
+    auto dataset = std::make_shared<JFJochReaderDataset>();
+    dataset->arm_date = start.arm_date;
+    dataset->jfjoch_release = start.jfjoch_release;
+    dataset->error_value = start.error_value;
+    dataset->file_detect_ice_rings = start.detect_ice_rings;
 
-        auto &x = dataset->experiment;
-        x = defaults;
-        // The reader hands every image out as signed 32-bit, as JFJochHDF5Reader does.
-        x.BitDepthImage(32);
-        x.PixelSigned(true);
-        x.FilePrefix(std::filesystem::path(start.file_prefix).filename().string());
-        x.BeamX_pxl(start.beam_center_x);
-        x.BeamY_pxl(start.beam_center_y);
-        x.DetectorDistance_mm(start.detector_distance * 1000.0f);
-        x.PoniRot1_rad(start.poni_rot1.value_or(0.0f));
-        x.PoniRot2_rad(start.poni_rot2.value_or(0.0f));
-        x.PoniRot3_rad(start.poni_rot3.value_or(0.0f));
-        x.IncidentEnergy_keV(WVL_1A_IN_KEV / start.incident_wavelength);
-        if (start.incident_wavelength_spread)
-            x.BandwidthFWHM(start.incident_wavelength_spread.value() / start.incident_wavelength);
-        x.DetectIceRings(start.detect_ice_rings.value_or(false));
+    auto &x = dataset->experiment;
+    x = defaults;
+    // The reader hands every image out as signed 32-bit, as JFJochHDF5Reader does.
+    x.BitDepthImage(32);
+    x.PixelSigned(true);
+    x.FilePrefix(std::filesystem::path(start.file_prefix).filename().string());
+    x.BeamX_pxl(start.beam_center_x);
+    x.BeamY_pxl(start.beam_center_y);
+    x.DetectorDistance_mm(start.detector_distance * 1000.0f);
+    x.PoniRot1_rad(start.poni_rot1.value_or(0.0f));
+    x.PoniRot2_rad(start.poni_rot2.value_or(0.0f));
+    x.PoniRot3_rad(start.poni_rot3.value_or(0.0f));
+    x.IncidentEnergy_keV(WVL_1A_IN_KEV / start.incident_wavelength);
+    if (start.incident_wavelength_spread)
+        x.BandwidthFWHM(start.incident_wavelength_spread.value() / start.incident_wavelength);
+    x.DetectIceRings(start.detect_ice_rings.value_or(false));
 
-        DetectorSetup detector = DetDECTRIS(start.image_size_x, start.image_size_y, start.detector_description, {});
-        detector.PixelSize_um(start.pixel_size_x * 1e6f);
-        detector.MirrorY(start.mirror_y);
-        detector.ImageOrientation(DetectorOrientation(start.detector_orientation_mirror_y,
-                                                      start.detector_orientation_quarter_turns));
-        detector.SensorThickness_um(start.sensor_thickness * 1e6f);
-        detector.SensorMaterial(start.sensor_material);
-        detector.SaturationLimit(SaturationLimitFromValue(start.saturation_value));
-        detector.BitDepthImage(32);
-        detector.MinFrameTime(std::chrono::microseconds(0));
-        detector.MinCountTime(std::chrono::microseconds(0));
-        detector.ReadOutTime(std::chrono::nanoseconds(0));
-        x.Detector(detector);
+    DetectorSetup detector = DetDECTRIS(start.image_size_x, start.image_size_y, start.detector_description, {});
+    detector.PixelSize_um(start.pixel_size_x * 1e6f);
+    detector.MirrorY(start.mirror_y);
+    detector.ImageOrientation(DetectorOrientation(start.detector_orientation_mirror_y,
+                                                  start.detector_orientation_quarter_turns));
+    detector.SensorThickness_um(start.sensor_thickness * 1e6f);
+    detector.SensorMaterial(start.sensor_material);
+    detector.SaturationLimit(SaturationLimitFromValue(start.saturation_value));
+    detector.BitDepthImage(32);
+    detector.MinFrameTime(std::chrono::microseconds(0));
+    detector.MinCountTime(std::chrono::microseconds(0));
+    detector.ReadOutTime(std::chrono::nanoseconds(0));
+    x.Detector(detector);
 
-        x.FrameTime(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<float>(start.frame_time)),
-                    std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<float>(start.count_time)));
+    x.FrameTime(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<float>(start.frame_time)),
+                std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<float>(start.count_time)));
 
-        x.Goniometer(start.goniometer);
-        x.GridScan(start.grid_scan);
-        x.Smargon(start.smargon_position);
-        x.NumTriggers(1);
-        x.ImagesPerTrigger(static_cast<int64_t>(start.number_of_images));
+    x.Goniometer(start.goniometer);
+    x.GridScan(start.grid_scan);
+    x.Smargon(start.smargon_position);
+    x.NumTriggers(1);
+    x.ImagesPerTrigger(static_cast<int64_t>(start.number_of_images));
 
-        InstrumentMetadata instrument;
-        instrument.SourceName(start.source_name);
-        instrument.SourceType(start.source_type);
-        instrument.InstrumentName(start.instrument_name);
-        x.ImportInstrumentMetadata(instrument);
-        x.SampleName(start.sample_name);
-        x.SampleTemperature_K(start.sample_temperature_K);
-        x.RingCurrent_mA(start.ring_current_mA);
-        x.TotalFlux(start.total_flux);
-        x.AttenuatorTransmission(start.attenuator_transmission);
-        if (start.beam_size_x && start.beam_size_y) {
-            x.BeamSizeX_um(start.beam_size_x.value() * 1e6f);
-            x.BeamSizeY_um(start.beam_size_y.value() * 1e6f);
-        }
-
-        if (!start.pixel_mask.empty())
-            dataset->pixel_mask = std::make_shared<const PixelMask>(start.pixel_mask.begin()->second);
-        else
-            dataset->pixel_mask = std::make_shared<const PixelMask>(
-                    std::vector<uint32_t>(start.image_size_x * start.image_size_y));
-        return dataset;
+    InstrumentMetadata instrument;
+    instrument.SourceName(start.source_name);
+    instrument.SourceType(start.source_type);
+    instrument.InstrumentName(start.instrument_name);
+    x.ImportInstrumentMetadata(instrument);
+    x.SampleName(start.sample_name);
+    x.SampleTemperature_K(start.sample_temperature_K);
+    x.RingCurrent_mA(start.ring_current_mA);
+    x.TotalFlux(start.total_flux);
+    x.AttenuatorTransmission(start.attenuator_transmission);
+    if (start.beam_size_x && start.beam_size_y) {
+        x.BeamSizeX_um(start.beam_size_x.value() * 1e6f);
+        x.BeamSizeY_um(start.beam_size_y.value() * 1e6f);
     }
+
+    if (!start.pixel_mask.empty())
+        dataset->pixel_mask = std::make_shared<const PixelMask>(start.pixel_mask.begin()->second);
+    else
+        dataset->pixel_mask = std::make_shared<const PixelMask>(
+                std::vector<uint32_t>(start.image_size_x * start.image_size_y));
+    return dataset;
 }
 
 void JFJochStartMessageReader::Open(const StartMessage &start, const std::string &master_path) {
@@ -114,7 +113,7 @@ void JFJochStartMessageReader::Open(const StartMessage &start, const std::string
         ended_ = false;
         lost_.clear();
     }
-    SetStartMessage(Dataset(start, default_experiment));
+    SetStartMessage(DatasetFromStartMessage(start, default_experiment));
 }
 
 void JFJochStartMessageReader::FileClosed(uint64_t file_number, uint64_t image_count) {

reader/JFJochStartMessageReader.h

@@ -19,6 +19,11 @@ public:
     explicit LiveCollectionLost(const std::string &what) : std::runtime_error(what) {}
 };
 
+// The dataset as the start message describes it - the same quantities an HDF5 master states, taken
+// from the message the writer wrote it from.
+std::shared_ptr<JFJochReaderDataset> DatasetFromStartMessage(const StartMessage &start,
+                                                             const DiffractionExperiment &defaults);
+
 // Reads a collection while it is being written: the dataset comes from its start message, as the
 // broker sends it, and the images straight from the data files - /entry/data/data in each - once the
 // broker reports a file closed and renamed into place (FileClosed). A read of an image whose file has

receiver/CMakeLists.txt

@@ -11,7 +11,9 @@ ADD_LIBRARY(JFJochReceiver STATIC
         JFJochReceiverCurrentStatus.h
         JFJochReceiverOutput.h
         JFJochReceiverLite.h JFJochReceiverLite.cpp
+        JFJochReceiverRugnux.h JFJochReceiverRugnux.cpp
+        RugnuxLiveStatus.h RugnuxLiveStatus.cpp
         JFJochReceiver.cpp
         JFJochReceiver.h)
 
-TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImagePuller JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview)
+TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImagePuller JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview JFJochRugnux)

receiver/JFJochReceiver.cpp

@@ -197,6 +197,7 @@ void JFJochReceiver::SendEndMessage() {
         message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram();
 
     scan_result.FillEndMessage(message);
+    AddToEndMessage(message);
 
     if (push_images_to_writer) {
         if (!image_pusher.EndDataCollection(message))
@@ -205,6 +206,11 @@ void JFJochReceiver::SendEndMessage() {
         logger.Info("Disconnected from writers");
     }
 
+    // The result blocks (AddToEndMessage) are megabytes only the writer needs.
+    message.merged.reset();
+    message.map_coefficients.reset();
+    message.placed_model.reset();
+
     if (zmq_metadata_socket != nullptr)
         zmq_metadata_socket->EndDataCollection(message);
 

receiver/JFJochReceiver.h

@@ -96,6 +96,9 @@ protected:
     void SendStartMessage();
     void SendEndMessage();
     void SaveStartMessageToImageBuffer(const StartMessage &msg);
+    // The last word on the end message before it is sent: a receiver whose analysis is not the
+    // per-image one fills in what it found.
+    virtual void AddToEndMessage(EndMessage &message) {}
     virtual void Cancel(const JFJochException &e);
 public:
     JFJochReceiver(const DiffractionExperiment& experiment,

receiver/JFJochReceiverLite.cpp

@@ -7,38 +7,36 @@
 
 using namespace std::chrono_literals;
 
-namespace {
-    // Serialize a processed image into its buffer slot. The slot holds the compressed
-    // image plus the per-image CBOR metadata (spot list, reflection list, ...); if an
-    // unusually rich frame makes that metadata too large to fit alongside the image,
-    // drop just this frame (log + recycle the slot) instead of letting the serializer
-    // throw, which would abort the whole data collection. Returns true when the image
-    // was serialized and the slot is ready for the caller to commit.
-    bool SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg,
-                              ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger) {
-        try {
-            serializer.SerializeImage(msg);
-            return true;
-        } catch (const JFJochException &) {
-            // Confirm this is a slot-overflow (not some other CBOR error) by re-serializing
-            // the metadata alone with an empty image placeholder: that always fits when the
-            // image is what overflowed, and gives the exact metadata size for the log. If it
-            // throws too, it is not a simple overflow and the exception propagates.
-            const auto image = msg.image;
-            msg.image = CompressedImage(nullptr, 0, image.GetWidth(), image.GetHeight(),
-                                        image.GetMode(), image.GetCompressionAlgorithm(),
-                                        image.GetChannel());
-            CBORStream2Serializer meas(static_cast<uint8_t *>(loc.GetImage()), slot_size);
-            meas.SerializeImage(msg);
-            const size_t metadata_size = meas.GetImageAppendOffset();
-            msg.image = image;
+// Serialize a processed image into its buffer slot. The slot holds the compressed
+// image plus the per-image CBOR metadata (spot list, reflection list, ...); if an
+// unusually rich frame makes that metadata too large to fit alongside the image,
+// drop just this frame (log + recycle the slot) instead of letting the serializer
+// throw, which would abort the whole data collection. Returns true when the image
+// was serialized and the slot is ready for the caller to commit.
+bool JFJochReceiverLite::SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg,
+                                              ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger) {
+    try {
+        serializer.SerializeImage(msg);
+        return true;
+    } catch (const JFJochException &) {
+        // Confirm this is a slot-overflow (not some other CBOR error) by re-serializing
+        // the metadata alone with an empty image placeholder: that always fits when the
+        // image is what overflowed, and gives the exact metadata size for the log. If it
+        // throws too, it is not a simple overflow and the exception propagates.
+        const auto image = msg.image;
+        msg.image = CompressedImage(nullptr, 0, image.GetWidth(), image.GetHeight(),
+                                    image.GetMode(), image.GetCompressionAlgorithm(),
+                                    image.GetChannel());
+        CBORStream2Serializer meas(static_cast<uint8_t *>(loc.GetImage()), slot_size);
+        meas.SerializeImage(msg);
+        const size_t metadata_size = meas.GetImageAppendOffset();
+        msg.image = image;
 
-            logger.Error("Dropping image {} - CBOR metadata too large to store image: "
-                         "metadata {} B + image {} B do not fit in buffer slot {} B",
-                         msg.number, metadata_size, image.GetCompressedSize(), slot_size);
-            loc.release();
-            return false;
-        }
+        logger.Error("Dropping image {} - CBOR metadata too large to store image: "
+                     "metadata {} B + image {} B do not fit in buffer slot {} B",
+                     msg.number, metadata_size, image.GetCompressedSize(), slot_size);
+        loc.release();
+        return false;
     }
 }
 
@@ -153,7 +151,7 @@ void JFJochReceiverLite::MeasurementThread() {
             return; // just quit the function - no measurement, no results
         }
 
-        Configure(msg->cbor->start_message.value());
+        ApplyStartMessage(experiment, msg->cbor->start_message.value());
         image_buffer.StartMeasurement(experiment); // Only at this point we know bit-depth of the images
         start_time = std::chrono::system_clock::now();
         // Send new start message out
@@ -204,7 +202,7 @@ void JFJochReceiverLite::MeasurementThread() {
     current_status.SetStatus(GetStatus());
 }
 
-void JFJochReceiverLite::Configure(const StartMessage &msg) {
+void JFJochReceiverLite::ApplyStartMessage(DiffractionExperiment &experiment, const StartMessage &msg) {
     if ((experiment.GetXPixelsNum() != msg.image_size_x)
     || (experiment.GetYPixelsNum() != msg.image_size_y))
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,

receiver/JFJochReceiverLite.h

@@ -56,8 +56,12 @@ class JFJochReceiverLite : public JFJochReceiver {
     void MaskThread(uint32_t id);
     void MeasurementThread();
     void DataAnalysisThread(uint32_t id);
-    void Configure(const StartMessage &message);
 public:
+    // Shared with JFJochReceiverRugnux, which forwards a collection the same way.
+    static bool SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg,
+                                     ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger);
+    // Takes what the detector's start message decides into the experiment.
+    static void ApplyStartMessage(DiffractionExperiment &experiment, const StartMessage &message);
 
     static int64_t NumberOfDataAnalysisThreads(int64_t requested_thread_number, const DiffractionExperiment& experiment);
 

receiver/JFJochReceiverOutput.h

@@ -26,6 +26,9 @@ struct JFJochReceiverOutput {
     std::string writer_err;
     std::optional<int64_t> images_written;
     MeanProcessingTime processing_time;
+    // A problem with the analysis alone (the images are all there), for the broker's status: why a live
+    // rugnux run was skipped. Empty otherwise.
+    std::string analysis_warning;
 };
 
 

receiver/JFJochReceiverRugnux.cpp

@@ -0,0 +1,548 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "JFJochReceiverRugnux.h"
+
+#include <cmath>
+#include <linux/capability.h>
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include "spdlog/fmt/fmt.h"
+
+#include "JFJochReceiverLite.h"
+#include "../common/ParallelFor.h"
+#include "../frame_serialize/CBORStream2Deserializer.h"
+#include "../frame_serialize/CBORStream2Serializer.h"
+#include "../rugnux/RugnuxLive.h"
+#include "../rugnux/ResultReport.h"
+#include "../writer/RugnuxResultBlocks.h"
+
+using namespace std::chrono_literals;
+
+namespace {
+    int64_t SteadyNowNs() {
+        return std::chrono::duration_cast<std::chrono::nanoseconds>(
+                std::chrono::steady_clock::now().time_since_epoch()).count();
+    }
+
+    // Whether this process may set a thread back to nice 0 once it has lowered it: CAP_SYS_NICE, or an
+    // RLIMIT_NICE that reaches 0 (the limit is 20 - nice).
+    bool CanRaiseNiceToZero() {
+        __user_cap_header_struct header{_LINUX_CAPABILITY_VERSION_3, 0};
+        __user_cap_data_struct caps[_LINUX_CAPABILITY_U32S_3]{};
+        if (syscall(SYS_capget, &header, caps) == 0 && (caps[0].effective & (1u << CAP_SYS_NICE)))
+            return true;
+        rlimit limit{};
+        return getrlimit(RLIMIT_NICE, &limit) == 0 && (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur >= 20);
+    }
+
+    double SecondsSince(int64_t ns) {
+        return static_cast<double>(SteadyNowNs() - ns) / 1e9;
+    }
+
+    // A run reachable for Cut while it exists: registered in `slot` (under `m`, cancelled at once where the
+    // cut came first) and taken off again before it is destroyed, an exception included.
+    class Reachable {
+        std::mutex &m;
+        Rugnux *&slot;
+    public:
+        Reachable(std::mutex &m, Rugnux *&slot, Rugnux &run, const bool &cut, const std::atomic<bool> &cancelled)
+            : m(m), slot(slot) {
+            std::unique_lock ul(m);
+            slot = &run;
+            if (cut || cancelled)
+                run.Cancel();
+        }
+        ~Reachable() {
+            std::unique_lock ul(m);
+            slot = nullptr;
+        }
+    };
+
+    // What rugnux reports as it goes: the per-image results of its canonical pass into the receiver's
+    // plots and scan result - the plots accumulate, so an image reported by the pre-pass as well would be
+    // counted twice - and its steps into the log.
+    class LiveObserver : public RugnuxObserver {
+        JFJochReceiverPlots &plots;
+        ScanResultGenerator &scan_result;
+        const AzimuthalIntegrationMapping &mapping;
+        RugnuxLiveStatus &status;
+        Logger &logger;
+        std::atomic<bool> canonical{false};
+    public:
+        LiveObserver(JFJochReceiverPlots &plots, ScanResultGenerator &scan_result,
+                     const AzimuthalIntegrationMapping &mapping, RugnuxLiveStatus &status, Logger &logger)
+            : plots(plots), scan_result(scan_result), mapping(mapping), status(status), logger(logger) {}
+
+        void OnImagePass(bool c) override { canonical = c; }
+        void OnImageProcessed(const DataMessage &msg) override {
+            if (!canonical)
+                return;
+            plots.Add(msg, AzimuthalIntegrationProfile(mapping));
+            scan_result.Add(msg);
+        }
+        void OnProgress(uint64_t done, uint64_t total) override { status.Progress(done, total); }
+        void OnStep(const std::string &title) override {
+            status.Step(title);
+            logger.Info("rugnux: {}", title);
+        }
+        void OnNote(const std::string &text) override {
+            status.Note(text);
+            logger.Info("rugnux:   {}", text);
+        }
+    };
+}
+
+JFJochReceiverRugnux::JFJochReceiverRugnux(const DiffractionExperiment &in_experiment,
+                                           const PixelMask &in_pixel_mask,
+                                           ImagePuller &in_image_puller,
+                                           ImagePusher &in_image_pusher,
+                                           Logger &in_logger,
+                                           int64_t in_forward_nthreads,
+                                           int in_rugnux_nthreads,
+                                           std::shared_ptr<const RugnuxModel> in_model,
+                                           double in_deadline_s,
+                                           RugnuxLiveStatus &in_live_status,
+                                           const SpotFindingSettings &in_spot_finding_settings,
+                                           JFJochReceiverCurrentStatus &in_current_status,
+                                           JFJochReceiverPlots &in_plots,
+                                           ImageBuffer &in_image_buffer,
+                                           ZMQPreviewSocket *in_zmq_preview_socket,
+                                           ZMQMetadataSocket *in_zmq_metadata_socket)
+    : JFJochReceiver(in_experiment, in_image_buffer, in_image_pusher, in_current_status, in_plots,
+                     in_spot_finding_settings, in_logger, in_pixel_mask, in_zmq_preview_socket,
+                     in_zmq_metadata_socket, nullptr),
+      image_puller(in_image_puller),
+      reader(in_image_buffer),
+      forward_nthreads(JFJochReceiverLite::NumberOfDataAnalysisThreads(in_forward_nthreads, in_experiment)),
+      rugnux_nthreads(in_rugnux_nthreads),
+      model(std::move(in_model)),
+      deadline_s(in_deadline_s),
+      live_status(in_live_status),
+      measurement_started(1),
+      forward_started(forward_nthreads) {
+    // The ParallelFor pool is made by the first thread to use it and its workers take that thread's
+    // priority. Made here, at the broker's own priority, before the rugnux thread lowers its own - so
+    // the pool is lowered for the analysis only where it can be raised back (see RugnuxThread).
+    parallel_detail::WorkerPool::Instance();
+    logger.Info("Starting {} forward threads; rugnux runs on {} threads while the images arrive",
+                forward_nthreads, rugnux_nthreads);
+    for (int i = 0; i < forward_nthreads; i++)
+        forward_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverRugnux::ForwardThread, this, i));
+    measurement = std::async(std::launch::async, &JFJochReceiverRugnux::MeasurementThread, this);
+    forward_started.wait();
+}
+
+JFJochReceiverRugnux::~JFJochReceiverRugnux() {
+    cancelled = true;
+    reader.Ended();
+    {
+        std::unique_lock ul(rugnux_m);
+        if (rugnux != nullptr)
+            rugnux->Cancel();
+        if (probe != nullptr)
+            probe->Cancel();
+    }
+    try {
+        if (measurement.valid())
+            measurement.get();
+    } catch (const std::exception &e) {
+        logger.Error("ReceiverRugnux teardown failed: {}", e.what());
+    }
+}
+
+void JFJochReceiverRugnux::MeasurementThread() {
+    std::future<void> rugnux_done, probe_done;
+    try {
+        // Waiting for the start message, as JFJochReceiverLite::MeasurementThread.
+        std::optional<std::chrono::steady_clock::time_point> give_up_at;
+        auto msg = image_puller.PollImage();
+        while (!cancelled && (!msg.has_value() || !msg->cbor || !msg->cbor->start_message)) {
+            if (detector_finished && !give_up_at)
+                give_up_at = std::chrono::steady_clock::now() + StartMessageGrace;
+            if (give_up_at && (std::chrono::steady_clock::now() > *give_up_at)) {
+                logger.Error("Detector finished, but no start message arrived - abandoning measurement");
+                cancelled = true;
+                break;
+            }
+            msg = image_puller.PollImage();
+        }
+        if (cancelled) {
+            current_status.SetProgress({});
+            current_status.SetStatus(GetStatus());
+            measurement_started.count_down();
+            return;
+        }
+
+        JFJochReceiverLite::ApplyStartMessage(experiment, msg->cbor->start_message.value());
+        image_buffer.StartMeasurement(experiment);
+        start_time = std::chrono::system_clock::now();
+        SendStartMessage();
+
+        // Rugnux reads the images back from the buffer, so it needs one slot per image: a slot taken
+        // again would hand it a later image. Without that the collection is only forwarded.
+        const auto slots = image_buffer.GetStatus().total_slots;
+        live_status.Start(experiment.GetImageNum());
+        if (slots < experiment.GetImageNum()) {
+            skipped_warning = fmt::format("no rugnux analysis: the image buffer holds {} images, the collection "
+                                          "has {}", slots, experiment.GetImageNum());
+            logger.Error("{}", skipped_warning);
+            live_status.Finish(RugnuxLiveState::Skipped, "the image buffer cannot hold the whole collection");
+        } else
+            StartRugnux(rugnux_done, probe_done);
+        measurement_started.count_down();
+    } catch (const JFJochException &e) {
+        logger.ErrorException(e);
+        Cancel(e);
+        measurement_started.count_down();
+        throw;
+    } catch (const std::exception &e) {
+        logger.Error("Cancelling data collection due to exception: {}", e.what());
+        cancelled = true;
+        measurement_started.count_down();
+        throw;
+    }
+
+    logger.Info("Receiving started");
+    for (auto &f: forward_futures)
+        f.get();
+    reader.Ended();
+    images_done = true;
+    live_status.LastImage();
+    const int64_t last_image = last_image_ns.load();
+    logger.Info("Receiving finished, {} images", images_collected.load());
+
+    if (rugnux_done.valid()) {
+        // The fixed cutoff: the analysis gets deadline_s after the last image, then gives what it has.
+        if (deadline_s > 0 && rugnux_done.wait_for(std::chrono::duration<double>(deadline_s))
+                              == std::future_status::timeout)
+            Cut("deadline");
+        rugnux_done.wait();
+        logger.Info("rugnux live: done {:.2f} s after the last image", SecondsSince(last_image));
+    }
+    if (probe_done.valid())
+        probe_done.wait();
+
+    current_status.SetProgress(1.0);
+    current_status.SetStatus(GetStatus());
+
+    SendEndMessage();
+    const double end_sent_s = SecondsSince(last_image);
+
+    if (!image_buffer.CheckIfBufferReturned(SendFinalizeTimeout)) {
+        logger.Error("Send commands not finalized in {} seconds", SendFinalizeTimeout.count());
+        throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in "
+                                                               + std::to_string(SendFinalizeTimeout.count()) + " seconds");
+    }
+    if (push_images_to_writer)
+        writer_error = image_pusher.Finalize();
+    if (last_image > 0 && rugnux_done.valid()) {
+        const double written_s = SecondsSince(last_image);
+        live_status.Latency(written_s);
+        logger.Info("rugnux live: last image -> all written {:.2f} s (end message sent at {:.2f} s)",
+                    written_s, end_sent_s);
+    }
+
+    end_time = std::chrono::system_clock::now();
+    current_status.SetProgress({});
+    current_status.SetStatus(GetStatus());
+}
+
+void JFJochReceiverRugnux::StartRugnux(std::future<void> &rugnux_done, std::future<void> &probe_done) {
+    // A failure here costs the analysis only: the images go on to the writer as they would without it.
+    try {
+        // The start message as the writer gets it, so rugnux sees the dataset an HDF5 master states.
+        std::vector<uint8_t> start_cbor;
+        image_buffer.GetStartMessage(start_cbor);
+        reader.Open(CBORStream2Deserialize(start_cbor)->start_message.value());
+        rugnux_done = std::async(std::launch::async, &JFJochReceiverRugnux::RugnuxThread, this);
+        probe_done = std::async(std::launch::async, &JFJochReceiverRugnux::ProbeThread, this);
+    } catch (const std::exception &e) {
+        logger.Error("rugnux live: cannot start - {}", e.what());
+        live_status.Finish(RugnuxLiveState::Failed, e.what());
+    }
+}
+
+void JFJochReceiverRugnux::RugnuxThread() {
+    // Below the forward threads: every thread rugnux starts inherits this, so an image arriving finds a
+    // core however busy rugnux keeps the machine.
+    // The process-wide ParallelFor pool was made before (in the constructor), so it does not inherit
+    // this and is lowered separately.
+    setpriority(PRIO_PROCESS, static_cast<id_t>(syscall(SYS_gettid)), 10);
+    // Only where it can be raised back afterwards: the pool outlives the collection, and left at nice 10
+    // it would slow every analysis the broker runs after it.
+    const bool pool_niced = CanRaiseNiceToZero();
+    if (pool_niced)
+        parallel_detail::WorkerPool::Instance().SetNice(10);
+    else {
+        static std::once_flag logged;
+        std::call_once(logged, [this] {
+            logger.Warning("rugnux live: the ParallelFor pool is left at its priority - lowering it needs "
+                           "the right to raise it back (CAP_SYS_NICE or RLIMIT_NICE)");
+        });
+    }
+    try {
+        const auto dataset = reader.GetDataset();
+        const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, model);
+        LiveObserver observer(plots, scan_result, *az_int_mapping, live_status, logger);
+        ProcessResult result;
+        {
+            Rugnux process(reader, setup.experiment, *dataset->pixel_mask, setup.config);
+            Reachable reachable(rugnux_m, rugnux, process, cut, cancelled);
+            result = process.Run(&observer);
+        }
+        // The verdict and summary of the report a `rugnux` run writes, without writing it.
+        const ReportDocument doc = BuildReportDocument(experiment.GetFilePrefix(), experiment.GetFilePrefix(),
+                                                       setup.experiment, result);
+        std::vector<std::string> warnings;
+        for (const auto &w: doc.warnings)
+            if (!w.developer_only)
+                warnings.push_back(w.text);
+        live_status.Result(doc.verdict, doc.verdict_text, doc.summary_rows, warnings);
+        live_status.Finish(result.cancelled ? RugnuxLiveState::Cancelled : RugnuxLiveState::Done);
+        std::unique_lock ul(rugnux_m);
+        rugnux_result = std::move(result);
+    } catch (const std::exception &e) {
+        logger.Error("rugnux live: {}", e.what());
+        live_status.Finish(RugnuxLiveState::Failed, e.what());
+    }
+    if (pool_niced && !parallel_detail::WorkerPool::Instance().SetNice(0))
+        logger.Warning("rugnux live: the ParallelFor pool could not be raised back to nice 0");
+}
+
+void JFJochReceiverRugnux::ProbeThread() {
+    // Rugnux waits for half the sweep before it decides on a lattice. In the meantime the machine is
+    // idle, so a small run of its own indexes the first 60 degrees and says at once whether there is
+    // a crystal - quick feedback for the person at the beamline, above all where there is none. It
+    // decides nothing for the run, which goes on as it would without it.
+    constexpr double PROBE_DEG = 60.0;
+    setpriority(PRIO_PROCESS, static_cast<id_t>(syscall(SYS_gettid)), 10);
+    try {
+        const auto dataset = reader.GetDataset();
+        const auto gonio = dataset->experiment.GetGoniometer();
+        if (!gonio || gonio->GetIncrement_deg() == 0.0f)
+            return;
+        const int64_t frames = std::lround(PROBE_DEG / std::abs(gonio->GetIncrement_deg()));
+        if (2 * frames > static_cast<int64_t>(reader.GetNumberOfImages()))
+            return;   // a sweep this short is decided on soon enough by the run itself
+        // A quarter of the threads: it runs beside the main run, and each of its workers holds GPU memory too.
+        RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, std::max(2, rugnux_nthreads / 4), nullptr);
+        setup.config.end_image = static_cast<int>(frames);
+        setup.config.rotation_postrefine_geometry = false;
+        setup.config.run_scaling = false;
+        ProcessResult r;
+        {
+            Rugnux probe_run(reader, setup.experiment, *dataset->pixel_mask, setup.config);
+            Reachable reachable(rugnux_m, probe, probe_run, cut, cancelled);
+            r = probe_run.Run(nullptr);
+        }
+        if (r.consensus_cell && r.rotation_lattice_type) {
+            const auto &c = *r.consensus_cell;
+            live_status.Probe(fmt::format("lattice on the first {:.0f} deg: {}-centred {}, {:.1f} {:.1f} {:.1f} "
+                                          "{:.1f} {:.1f} {:.1f}, {:.0f}% of the images indexed", PROBE_DEG,
+                                          r.rotation_lattice_type->centering,
+                                          gemmi::crystal_system_str(r.rotation_lattice_type->crystal_system),
+                                          c.a, c.b, c.c, c.alpha, c.beta, c.gamma,
+                                          100.0 * r.indexing_rate.value_or(0.0f)));
+        } else
+            live_status.Probe(fmt::format("no lattice on the first {:.0f} deg", PROBE_DEG));
+    } catch (const NoLatticeFound &e) {
+        live_status.Probe(fmt::format("no lattice on the first {:.0f} deg: {}", PROBE_DEG, e.reason));
+    } catch (const std::exception &e) {
+        logger.Warning("rugnux live: the lattice probe failed: {}", e.what());
+    }
+    logger.Info("rugnux live: lattice probe - {}", live_status.Get().probe);
+}
+
+void JFJochReceiverRugnux::AddToEndMessage(EndMessage &message) {
+    // A failure here costs the end message the analysis only: it goes out as the receiver made it.
+    try {
+        std::unique_lock ul(rugnux_m);
+        if (!rugnux_result)
+            return;
+        const EndMessage &r = rugnux_result->end_message;
+        message.indexing_rate = r.indexing_rate;
+        message.bkg_estimate = r.bkg_estimate;
+        message.spindle_blind_fraction = r.spindle_blind_fraction;
+        message.ice_ring_score_mean = r.ice_ring_score_mean;
+        message.az_int_result = r.az_int_result;
+        message.rotation_lattice = r.rotation_lattice;
+        message.rotation_lattice_type = r.rotation_lattice_type;
+        message.rotation_extra_lattices = r.rotation_extra_lattices;
+        message.unit_cell = r.unit_cell;
+        message.space_group_name = r.space_group_name;
+        message.space_group_number = r.space_group_number;
+        message.reindex_matrix = r.reindex_matrix;
+        message.refined_beam_center_x = r.refined_beam_center_x;
+        message.refined_beam_center_y = r.refined_beam_center_y;
+        message.refined_poni_rot1 = r.refined_poni_rot1;
+        message.refined_poni_rot2 = r.refined_poni_rot2;
+        message.refined_poni_rot3 = r.refined_poni_rot3;
+        message.refined_rotation_axis = r.refined_rotation_axis;
+        // Merged data, map coefficients and placed model, for the writer to write beside the master. Moved:
+        // the result is not read again.
+        if (!rugnux_result->outputs)
+            return;
+        RugnuxOutputs &o = *rugnux_result->outputs;
+        if (o.merged)
+            message.merged = std::move(o.merged->mtz);
+        if (o.maps)
+            message.map_coefficients = std::move(o.maps->coefficients);
+        if (o.placed_model)
+            message.placed_model = PlacedModelFromStructure(*o.placed_model->model, o.placed_model->cell,
+                                                            o.placed_model->space_group.xhm());
+
+        // Every pusher serialises END into a buffer of MESSAGE_SIZE_FOR_START_END, and an END that does
+        // not fit there fails the collection. So it is serialised here first, into a buffer of the same
+        // size (uninitialised: only the bytes written are touched); one that does not fit goes without
+        // the blocks.
+        const auto scratch = std::make_unique_for_overwrite<uint8_t[]>(MESSAGE_SIZE_FOR_START_END);
+        CBORStream2Serializer serializer(scratch.get(), MESSAGE_SIZE_FOR_START_END);
+        try {
+            serializer.SerializeSequenceEnd(message);
+        } catch (const JFJochException &e) {
+            const std::string text = fmt::format("the merged data, map coefficients and placed model do not "
+                                                 "fit in the end message ({} MiB) and are not written",
+                                                 MESSAGE_SIZE_FOR_START_END / (1024 * 1024));
+            logger.Error("rugnux live: {} - {}", text, e.what());
+            live_status.Warning(text);
+            message.merged.reset();
+            message.map_coefficients.reset();
+            message.placed_model.reset();
+        }
+    } catch (const std::exception &e) {
+        logger.Error("rugnux live: the end message goes without the analysis results - {}", e.what());
+        message.merged.reset();
+        message.map_coefficients.reset();
+        message.placed_model.reset();
+    }
+}
+
+void JFJochReceiverRugnux::ForwardThread(uint32_t id) {
+    forward_started.count_down();
+    measurement_started.wait();
+
+    while (!cancelled && !end_message_received) {
+        try {
+            auto msg = image_puller.PollImage(std::chrono::milliseconds(5));
+            if (!msg.has_value() || !msg->cbor)
+                continue;
+            if (msg->cbor->end_message) {
+                end_message_received = true;
+            } else if (msg->cbor->calibration) {
+                if (push_images_to_writer)
+                    image_pusher.SendCalibration(msg->cbor->calibration.value());
+            } else if (msg->cbor->data_message) {
+                last_image_ns = SteadyNowNs();
+                live_status.ImageReceived();
+                DataMessage data_msg = msg->cbor->data_message.value();
+                ++images_collected;
+
+                const auto compressed_size_img = data_msg.image.GetCompressedSize();
+                const auto uncompressed_size_img = data_msg.image.GetUncompressedSize();
+                compressed_size += compressed_size_img;
+                uncompressed_size += uncompressed_size_img;
+                UpdateMaxImageReceived(data_msg.number);
+
+                data_msg.original_number = data_msg.number;
+                data_msg.user_data = experiment.GetImageAppendix();
+                data_msg.run_number = experiment.GetRunNumber();
+                data_msg.run_name = experiment.GetRunName();
+                data_msg.receiver_buf_available = image_buffer.GetAvailSlots();
+                data_msg.receiver_aq_dev_delay = image_puller.GetCurrentFifoUtilization();
+                if (compressed_size_img > 0)
+                    data_msg.compression_ratio = static_cast<float>(uncompressed_size_img)
+                                                 / static_cast<float>(compressed_size_img);
+
+                auto loc = image_buffer.GetImageSlot();
+                if (loc == nullptr)
+                    writer_queue_full = true;
+                else {
+                    auto writer_buffer = (uint8_t *) loc->GetImage();
+                    CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize());
+                    if (JFJochReceiverLite::SerializeImageOrDrop(serializer, data_msg, *loc,
+                                                                 experiment.GetImageBufferLocationSize(), logger)) {
+                        loc->SetImageNumber(data_msg.number);
+                        loc->SetImageSize(serializer.GetBufferSize());
+                        loc->SetIndexed(false);
+                        loc->ReadyToSend();
+
+                        if (zmq_preview_socket != nullptr)
+                            zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize());
+                        if (zmq_metadata_socket != nullptr)
+                            zmq_metadata_socket->AddDataMessage(data_msg);
+                        if (push_images_to_writer) {
+                            if (image_pusher.SendImage(*loc))
+                                ++images_sent;
+                        } else
+                            loc->release();
+                        UpdateMaxImageSent(data_msg.number);
+                    }
+                }
+            }
+            UpdateMaxDelay(image_puller.GetCurrentFifoUtilization());
+            current_status.SetProgress(GetProgress());
+            current_status.SetStatus(GetStatus());
+        } catch (const JFJochException &e) {
+            logger.ErrorException(e);
+            Cancel(e);
+        } catch (const std::exception &e) {
+            logger.Error("Cancelling data collection due to exception: {}", e.what());
+            cancelled = true;
+        }
+    }
+}
+
+void JFJochReceiverRugnux::Cancel(bool silent) {
+    detector_finished = true;
+    // The silent cancel is the detector reading idle at the end of every collection: rugnux goes on.
+    if (silent)
+        return;
+    // /cancel is the one way in for the beamline, which cannot tell the collection from its analysis: it
+    // ends the collection while the images still arrive, as JFJochReceiverLite does, and in every case
+    // cuts the analysis, so the end message goes out at once with what rugnux has established.
+    if (!images_done)
+        JFJochReceiver::Cancel(silent);
+    Cut("cancel");
+}
+
+void JFJochReceiverRugnux::Cut(const std::string &reason) {
+    std::unique_lock ul(rugnux_m);
+    if (cut || rugnux_result)
+        return;
+    cut = true;
+    reader.Ended();   // no image is waited for any more
+    if (rugnux != nullptr)
+        rugnux->Cancel();
+    if (probe != nullptr)
+        probe->Cancel();
+    live_status.Cut(reason);
+    logger.Info("rugnux live: analysis cut ({}) - the end message carries what is established", reason);
+}
+
+void JFJochReceiverRugnux::StopReceiver() {
+    if (measurement.valid()) {
+        measurement.get();
+        logger.Info("Receiver stopped");
+    }
+}
+
+float JFJochReceiverRugnux::GetEfficiency() const {
+    if (experiment.GetFrameNum() == 0)
+        return 0;
+    return static_cast<float>(images_collected) / static_cast<float>(experiment.GetFrameNum());
+}
+
+JFJochReceiverOutput JFJochReceiverRugnux::GetFinalStatistics() const {
+    JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics();
+    ret.analysis_warning = skipped_warning;
+    return ret;
+}
+
+float JFJochReceiverRugnux::GetProgress() const {
+    if (experiment.GetFrameNum() == 0)
+        return 0.0;
+    return static_cast<float>(max_image_number_received) / static_cast<float>(experiment.GetFrameNum());
+}

receiver/JFJochReceiverRugnux.h

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <future>
+#include <latch>
+
+#include "JFJochReceiver.h"
+#include "../image_puller/ImagePuller.h"
+#include "../reader/JFJochImageBufferReader.h"
+#include "../rugnux/Rugnux.h"
+#include "RugnuxLiveStatus.h"
+
+// A DECTRIS rotation collection forwarded to the writer as JFJochReceiverLite does, but analysed by
+// rugnux while it arrives instead of image by image: the forward threads put each image into the
+// image buffer and send it on, and one rugnux thread reads the collection back from that buffer. The
+// end message waits for rugnux, and carries what it found - the lattice, refined geometry, per-image
+// results and its merged data, map coefficients and placed model (see AddToEndMessage).
+class JFJochReceiverRugnux : public JFJochReceiver {
+    using JFJochReceiver::Cancel;
+
+    ImagePuller &image_puller;
+    JFJochImageBufferReader reader;
+
+    std::atomic<bool> end_message_received{false};
+    std::atomic<bool> detector_finished{false};   // see JFJochReceiverLite
+    constexpr static auto StartMessageGrace = std::chrono::seconds(5);
+
+    const int64_t forward_nthreads;
+    const int rugnux_nthreads;
+    const std::shared_ptr<const RugnuxModel> model;
+    const double deadline_s;   // after the last image, the analysis is cut (0 = never)
+    RugnuxLiveStatus &live_status;
+    std::string skipped_warning;   // why the collection was not analysed, for the broker's status
+
+    std::vector<std::future<void>> forward_futures;
+    std::future<void> measurement;
+    std::latch measurement_started;
+    std::latch forward_started;
+
+    std::mutex rugnux_m;
+    Rugnux *rugnux = nullptr;   // while it runs, for Cut
+    Rugnux *probe = nullptr;    // the same for the lattice probe
+    std::optional<ProcessResult> rugnux_result;
+    bool cut = false;
+    std::atomic<bool> images_done{false};   // the forward threads have ended
+
+    // The latency of the demonstration: from the last image in to every result in place.
+    std::atomic<int64_t> last_image_ns{0};
+
+    void MeasurementThread();
+    // Opens the reader on the collection and starts the rugnux and probe threads; a failure is the
+    // analysis's alone (live status Failed), never the collection's.
+    void StartRugnux(std::future<void> &rugnux_done, std::future<void> &probe_done);
+    void ForwardThread(uint32_t id);
+    void RugnuxThread();
+    void ProbeThread();
+    // Stops the analysis where it is: rugnux returns what it has established (see Rugnux::Cancel), and
+    // the end message goes out with that. Reason: "deadline" or "cancel".
+    void Cut(const std::string &reason);
+    void AddToEndMessage(EndMessage &message) override;
+public:
+    JFJochReceiverRugnux(const DiffractionExperiment &experiment,
+                         const PixelMask &pixel_mask,
+                         ImagePuller &image_puller,
+                         ImagePusher &image_pusher,
+                         Logger &logger,
+                         int64_t forward_nthreads,
+                         int rugnux_nthreads,
+                         std::shared_ptr<const RugnuxModel> model,
+                         double deadline_s,
+                         RugnuxLiveStatus &live_status,
+                         const SpotFindingSettings &spot_finding_settings,
+                         JFJochReceiverCurrentStatus &current_status,
+                         JFJochReceiverPlots &plots,
+                         ImageBuffer &image_buffer,
+                         ZMQPreviewSocket *zmq_preview_socket = nullptr,
+                         ZMQMetadataSocket *zmq_metadata_socket = nullptr);
+    ~JFJochReceiverRugnux() override;
+
+    void StopReceiver() override;
+    void Cancel(bool silent) override;
+
+    float GetEfficiency() const override;
+    float GetProgress() const override;
+    JFJochReceiverOutput GetFinalStatistics() const override;
+
+    void SetSpotFindingSettings(const SpotFindingSettings &spot_finding_settings) override {}
+};

receiver/JFJochReceiverService.cpp

@@ -2,6 +2,9 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 #include "JFJochReceiverService.h"
+#include "JFJochReceiverRugnux.h"
+#include "../common/ThreadAffinity.h"
+#include "../rugnux/RugnuxDefaults.h"
 #include "JFJochReceiverFPGA.h"
 #include "JFJochReceiverLite.h"
 #include "../preview/JFJochJPEG.h"
@@ -17,6 +20,23 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic
           spot_finding_settings(DiffractionExperiment::DefaultDataProcessingSettings()) {
 }
 
+JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
+                                                         int reserved_cpus, double deadline_s) {
+    std::unique_lock ul_state(state_mutex);
+    if (state != ReceiverState::Idle)
+        throw JFJochException(JFJochExceptionCategory::WrongDAQState,
+                              "Cannot change rugnux settings during data collection");
+    rugnux_enable = enable;
+    rugnux_model = std::move(model);
+    rugnux_reserved_cpus = reserved_cpus;
+    rugnux_deadline_s = deadline_s;
+    return *this;
+}
+
+RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const {
+    return rugnux_status.Get();
+}
+
 JFJochReceiverService &JFJochReceiverService::NumThreads(int64_t input) {
     if (input <= 0)
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Thread number must be above zero");
@@ -103,6 +123,14 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
                     image_puller = std::make_shared<ZMQImagePuller>(
                             experiment.GetDetectorSetup().GetDECTRISStream2Addr());
                 }
+                if (rugnux_enable && RugnuxRotationData(experiment)) {
+                    receiver = std::make_unique<JFJochReceiverRugnux>(
+                            experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads_local,
+                            std::max<int>(1, static_cast<int>(AvailableCpus()) - rugnux_reserved_cpus),
+                            rugnux_model, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots, image_buffer,
+                            zmq_preview_socket.get(), zmq_metadata_socket.get());
+                    break;
+                }
                 receiver = std::make_unique<JFJochReceiverLite>(experiment,
                                                                 pixel_mask,
                                                                 *image_puller,

receiver/JFJochReceiverService.h

@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include "RugnuxLiveStatus.h"
+
 #include <mutex>
 #include <memory>
 
@@ -12,6 +14,8 @@
 #include "../preview/ZMQMetadataSocket.h"
 #include "../preview/ZMQPreviewSocket.h"
 
+struct RugnuxModel;
+
 class JFJochReceiverService {
     std::unique_ptr<JFJochReceiver> receiver;
     AcquisitionDeviceGroup &aq_devices;
@@ -21,6 +25,14 @@ class JFJochReceiverService {
     std::shared_ptr<ImagePuller> image_puller;
     int64_t nthreads = 8;
 
+    // Rotation collections from a DECTRIS detector analysed by rugnux while they arrive
+    // (JFJochReceiverRugnux), instead of image by image. Off unless asked for.
+    bool rugnux_enable = false;
+    std::shared_ptr<const RugnuxModel> rugnux_model;
+    int rugnux_reserved_cpus = 4;   // left to the forward threads and the pusher
+    double rugnux_deadline_s = 0;   // the analysis is cut this long after the last image (0 = never)
+    RugnuxLiveStatus rugnux_status;   // of the last collection, kept after it ends
+
     enum class ReceiverState {Idle, Running};
     ReceiverState state = ReceiverState::Idle;
     mutable std::mutex state_mutex;
@@ -43,6 +55,9 @@ public:
                           ImagePusher &pusher,
                           size_t send_buffer_size_MiB = 512);
     JFJochReceiverService& NumThreads(int64_t input);
+    JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model, int reserved_cpus,
+                                      double deadline_s);
+    RugnuxLiveSnapshot GetRugnuxStatus() const;
     JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional<int32_t> &watermark = {});
     JFJochReceiverService& MetadataSocket(const std::string &addr);
 

receiver/RugnuxLiveStatus.cpp

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "RugnuxLiveStatus.h"
+
+double RugnuxLiveStatus::Now() const {
+    return std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
+}
+
+void RugnuxLiveStatus::CloseStep() {
+    if (!s.steps.empty() && !s.steps.back().duration_s)
+        s.steps.back().duration_s = Now() - s.steps.back().started_s;
+}
+
+void RugnuxLiveStatus::Start(int64_t images_expected) {
+    std::unique_lock ul(m);
+    s = RugnuxLiveSnapshot{};
+    s.state = RugnuxLiveState::Running;
+    s.images_expected = images_expected;
+    start = std::chrono::steady_clock::now();
+}
+
+void RugnuxLiveStatus::ImageReceived() {
+    std::unique_lock ul(m);
+    ++s.images_received;
+}
+
+void RugnuxLiveStatus::Step(const std::string &title) {
+    std::unique_lock ul(m);
+    CloseStep();
+    s.steps.push_back(RugnuxLiveStep{.title = title, .started_s = Now()});
+    s.step_progress.reset();
+}
+
+void RugnuxLiveStatus::Note(const std::string &text) {
+    std::unique_lock ul(m);
+    if (s.steps.empty())
+        s.steps.push_back(RugnuxLiveStep{.title = "", .started_s = Now()});
+    s.steps.back().notes.push_back(text);
+}
+
+void RugnuxLiveStatus::Progress(uint64_t done, uint64_t total) {
+    std::unique_lock ul(m);
+    if (total > 0)
+        s.step_progress = static_cast<float>(done) / static_cast<float>(total);
+}
+
+void RugnuxLiveStatus::Finish(RugnuxLiveState state, const std::string &message) {
+    std::unique_lock ul(m);
+    CloseStep();
+    s.elapsed_s = Now();
+    s.state = state;
+    s.message = message;
+    s.step_progress.reset();
+}
+
+void RugnuxLiveStatus::Result(const std::string &verdict, const std::string &verdict_text,
+                              const std::vector<std::pair<std::string, std::string>> &summary,
+                              const std::vector<std::string> &warnings) {
+    std::unique_lock ul(m);
+    s.verdict = verdict;
+    s.verdict_text = verdict_text;
+    s.summary = summary;
+    s.warnings = warnings;
+}
+
+void RugnuxLiveStatus::Latency(double last_image_to_written_s) {
+    std::unique_lock ul(m);
+    s.last_image_to_written_s = last_image_to_written_s;
+}
+
+void RugnuxLiveStatus::LastImage() {
+    std::unique_lock ul(m);
+    s.last_image_s = Now();
+}
+
+void RugnuxLiveStatus::Cut(const std::string &reason) {
+    std::unique_lock ul(m);
+    s.cut = reason;
+    s.cut_s = Now();
+}
+
+void RugnuxLiveStatus::Probe(const std::string &text) {
+    std::unique_lock ul(m);
+    s.probe = text;
+    s.probe_s = Now();
+}
+
+void RugnuxLiveStatus::Warning(const std::string &text) {
+    std::unique_lock ul(m);
+    s.warnings.push_back(text);
+}
+
+RugnuxLiveSnapshot RugnuxLiveStatus::Get() const {
+    std::unique_lock ul(m);
+    RugnuxLiveSnapshot ret = s;
+    if (s.state == RugnuxLiveState::Running)
+        ret.elapsed_s = Now();
+    return ret;
+}

receiver/RugnuxLiveStatus.h

@@ -0,0 +1,72 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <chrono>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+// Where the live rugnux run of the last collection is, for the broker's status route: the steps it has
+// taken with what each found ("lattice found", "space group P 21 21 21"), the step it is on and how far
+// into its images, and - once it is over - its verdict and summary as the report states them, and the
+// latency from the last image to every result written.
+enum class RugnuxLiveState { Idle, Running, Done, Failed, Cancelled, Skipped };
+
+struct RugnuxLiveStep {
+    std::string title;
+    double started_s = 0.0;               // since the collection started
+    std::optional<double> duration_s;     // set once the next step starts or the run ends
+    std::vector<std::string> notes;
+};
+
+struct RugnuxLiveSnapshot {
+    RugnuxLiveState state = RugnuxLiveState::Idle;
+    std::string message;                  // why it failed or was skipped
+    int64_t images_expected = 0;
+    int64_t images_received = 0;
+    double elapsed_s = 0.0;
+    std::optional<float> step_progress;   // fraction of the current pass's images
+    std::vector<RugnuxLiveStep> steps;
+    // The lattice probe on the first 60 degrees (JFJochReceiverRugnux::ProbeThread): an early answer
+    // to "is there a crystal", seconds into the collection; the run's own answer comes later.
+    std::string probe;
+    std::optional<double> probe_s;        // when it answered, since the collection started
+    std::optional<double> last_image_s;   // when the last image arrived, since the collection started
+    // The analysis was cut - "deadline" (the broker's deadline_s after the last image) or "cancel" - at
+    // cut_s since the collection started; what it had established went out, the steps after it were left out.
+    std::string cut;
+    std::optional<double> cut_s;
+    std::optional<double> last_image_to_written_s;
+    std::string verdict;                  // OK | WARNINGS | UNUSABLE | FAILED
+    std::string verdict_text;
+    std::vector<std::pair<std::string, std::string>> summary;
+    std::vector<std::string> warnings;
+};
+
+class RugnuxLiveStatus {
+    mutable std::mutex m;
+    RugnuxLiveSnapshot s;
+    std::chrono::steady_clock::time_point start;
+    double Now() const;
+    void CloseStep();
+public:
+    void Start(int64_t images_expected);
+    void ImageReceived();
+    void Step(const std::string &title);
+    void Note(const std::string &text);
+    void Progress(uint64_t done, uint64_t total);
+    void Finish(RugnuxLiveState state, const std::string &message = "");
+    void Result(const std::string &verdict, const std::string &verdict_text,
+                const std::vector<std::pair<std::string, std::string>> &summary,
+                const std::vector<std::string> &warnings);
+    void Latency(double last_image_to_written_s);
+    void LastImage();
+    void Cut(const std::string &reason);
+    void Probe(const std::string &text);
+    void Warning(const std::string &text);
+    RugnuxLiveSnapshot Get() const;
+};

rugnux/CMakeLists.txt

@@ -17,6 +17,8 @@ ADD_LIBRARY(JFJochRugnux STATIC
         RugnuxScaleMerge.cpp
         RugnuxDefaults.cpp
         RugnuxDefaults.h
+        RugnuxLive.cpp
+        RugnuxLive.h
         Rugnux.h
         RugnuxCommandLine.cpp
         RugnuxCommandLine.h
@@ -29,13 +31,18 @@ ADD_LIBRARY(JFJochRugnux STATIC
         BrokerFeed.h
 )
 
-TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter JFJochModelWriter JFJochPreview gemmi fftw3f
+TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi fftw3f
                       httplib::httplib)
 
+# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux
+# returns: the library itself writes none.
+ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h)
+TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter)
+
 # rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the
 # default), azint, scale (re-scale/merge stored reflections) or calibration (powder-ring geometry).
 ADD_EXECUTABLE(rugnux rugnux_cli.cpp RugnuxConsole.cpp RugnuxConsole.h)
-TARGET_LINK_LIBRARIES(rugnux JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter)
+TARGET_LINK_LIBRARIES(rugnux JFJochRugnuxOutput JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter)
 INSTALL(TARGETS rugnux RUNTIME COMPONENT rugnux)
 # macOS: sign the installed binary with the Developer ID (hardened runtime, secure timestamp) so the
 # .tgz can be notarized - see JFJOCH_MACOS_SIGNING_IDENTITY in the top-level CMakeLists.txt. The

rugnux/DiagnosticOutput.cpp

@@ -5,7 +5,6 @@
 
 #include <algorithm>
 #include <cmath>
-#include <fstream>
 
 #include <spdlog/fmt/fmt.h>
 
@@ -103,17 +102,13 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
     return WriteJPEGToMem(rgb_image, 85);
 }
 
-void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plot,
-                       const std::optional<GoniometerAxis> &goniometer, size_t image_count) {
-    const std::string path = output_prefix + "_plot.txt";
-    std::ofstream file(path, std::ofstream::out | std::ofstream::trunc);
-    if (!file)
-        throw JFJochException(JFJochExceptionCategory::FileWriteError,
-                              "Cannot open " + path + " for writing");
+std::string PerImagePlotText(const PerImagePlot &plot) {
+    const std::optional<GoniometerAxis> &goniometer = plot.goniometer;
+    std::string file;
 
     // One comment line, its names over their own columns, and nothing else a plotting program has
     // to be told to skip. The '#' takes the first character of the image-number field.
-    file << "#" << fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
+    file += "#" + fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
                                " {:>7} {:>11}\n",
                                "image", "angle_deg", "bkg", "resolution_A", "spots", "scale_G",
                                "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition");
@@ -125,7 +120,7 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo
                            || std::any_of(plot.scale_g.begin(), plot.scale_g.end(),
                                           [](float g) { return std::isfinite(g); });
 
-    for (size_t i = 0; i < image_count; i++) {
+    for (size_t i = 0; i < plot.image_count; i++) {
         const float angle = goniometer
                                 ? goniometer->GetAngle_deg(static_cast<float>(i))
                                       + goniometer->GetWedge_deg() / 2.0f
@@ -152,11 +147,12 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo
         } else if (merge_ran && std::isfinite(scale_g))
             disposition = 1.0f;
 
-        file << fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}"
+        file += fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}"
                             " {:>12.4f} {:>8} {:>7} {:>11}\n",
                             i, angle, Element(plot.bkg_estimate, i), Element(plot.resolution_A, i),
                             Element(plot.spot_count, i), scale_g, Element(plot.mosaicity_deg, i),
                             Element(plot.cc_to_merge, i), Element(plot.cc_n, i),
                             merge_ran ? std::to_string(merged) : "nan", disposition);
     }
+    return file;
 }

rugnux/DiagnosticOutput.h

@@ -40,9 +40,10 @@ struct PerImagePlot {
     std::vector<float> cc_to_merge;
     std::vector<int> cc_n;
     std::vector<uint8_t> frame_disposition;
+    std::optional<GoniometerAxis> goniometer;   // the angle column; none: nan
+    size_t image_count = 0;                     // the rows
 };
 
-// Writes <output_prefix>_plot.txt: one '#' legend line naming the columns, then one whitespace-
-// separated row per image, so `plot '<f>' using 1:4` works with nothing in between.
-void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plot,
-                       const std::optional<GoniometerAxis> &goniometer, size_t image_count);
+// The text of <prefix>_plot.txt: one '#' legend line naming the columns, then one whitespace-separated
+// row per image, so `plot '<f>' using 1:4` works with nothing in between.
+std::string PerImagePlotText(const PerImagePlot &plot);

rugnux/ResultReport.cpp

@@ -4,7 +4,6 @@
 #include <cmath>
 #include <algorithm>
 #include <cctype>
-#include <fstream>
 #include <sstream>
 
 #include <spdlog/fmt/fmt.h>
@@ -1459,7 +1458,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                                  domains ? ", or spots of the further lattice domains found land on those positions" : "",
                                  result.doubled_cell
                                  ? fmt::format("Both cells were merged; the other one is in {}.mtz - compare them in refinement",
-                                               result.doubled_cell->other_prefix)
+                                               output_prefix + result.doubled_cell->other_suffix)
                                  : fmt::format("The doubled cell {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} was not merged",
                                                sc.doubled_cell.a, sc.doubled_cell.b, sc.doubled_cell.c,
                                                sc.doubled_cell.alpha, sc.doubled_cell.beta, sc.doubled_cell.gamma)));
@@ -1467,7 +1466,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                 const auto &h = *result.doubled_cell;
                 Add(s, KeyBool("DOUBLED_CELL_ADOPTED", h.adopted));
                 Add(s, KeyText("DOUBLED_CELL_DECISION", h.decision));
-                Add(s, KeyText("DOUBLED_CELL_OTHER_FILE", h.other_prefix + ".mtz"));
+                Add(s, KeyText("DOUBLED_CELL_OTHER_FILE", output_prefix + h.other_suffix + ".mtz"));
                 Add(s, KeyText("DOUBLED_CELL_OTHER_CELL", CellString(h.other_cell)));
                 Add(s, KeyText("DOUBLED_CELL_OTHER_SPACE_GROUP", h.other_space_group));
                 Add(s, KeyReal("DOUBLED_CELL_OTHER_D_MIN", h.other_d_min_A, "{:.2f}"));
@@ -1932,7 +1931,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
             }
             if (result.twin_subgroup_merge) {
                 const auto &t = *result.twin_subgroup_merge;
-                Add(s, KeyText("TWIN_SUBGROUP_FILE", t.file_prefix + ".mtz"));
+                Add(s, KeyText("TWIN_SUBGROUP_FILE", output_prefix + t.file_suffix + ".mtz"));
                 Add(s, KeyText("TWIN_SUBGROUP_SPACE_GROUP", t.space_group));
                 Add(s, KeyText("TWIN_SUBGROUP_LAW", t.twin_law));
                 Add(s, KeyReal("TWIN_SUBGROUP_FRACTION", t.twin_fraction, "{:.3f}"));
@@ -1952,7 +1951,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                         " as _pdbx_reflns_twin). Refine against it with that twin law where the model prefers it.\n"
                         " Nothing in this report or in the answer's files depends on it.{}",
                         t.adopted_space_group, t.reason, t.space_group, t.adopted_space_group, t.space_group,
-                        t.space_group, t.file_prefix, t.twin_law, t.twin_fraction,
+                        t.space_group, output_prefix + t.file_suffix, t.twin_law, t.twin_fraction,
                         t.other_subgroups.empty() ? std::string()
                                                   : "\n Other subgroups the search confirmed, each a twin hypothesis of\n"
                                                     " its own (re-run with -S to merge in one): see\n"
@@ -2329,7 +2328,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
         ModelValidationResult unplaced;
         if (!result.model_validation) {
             const auto &mp = *result.model_prior;
-            unplaced.model_path = mp.model_path;
+            unplaced.model_name = mp.model_name;
             unplaced.failure_reason = mp.ok ? "molecular replacement did not place the model"
                                             : "the model could not be searched for: " + mp.failure_reason;
         }
@@ -2338,7 +2337,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
         const bool mr_hand = by_mr && result.model_prior->hand_check.adopted;
         ReportSection s;
         s.title = "5. MODEL VALIDATION";
-        Add(s, KeyText("MODEL_FILE", mv.model_path));
+        Add(s, KeyText("MODEL_FILE", mv.model_name));
         if (!result.model_placement.empty())
             Add(s, KeyEnum("MODEL_PLACEMENT", result.model_placement, {"RIGID_BODY", "MR", "NOT_PLACED"}));
         if (!mv.ok) {
@@ -2441,12 +2440,12 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                                            mv.indexing_margin_null_sd), true));
                 Add(s, KeyReal("MODEL_INDEXING_MARGIN_SIGMA", mv.indexing_margin_sigma, "{:+.2f}", true));
             }
-            if (!mv.maps_prefix.empty())
-                Add(s, KeyText("MAPS_PREFIX", mv.maps_prefix));
+            if (mv.ok && !output_prefix.empty())
+                Add(s, KeyText("MAPS_PREFIX", output_prefix + mv.maps_suffix));
             // The model's choice of another group or of the twin hypothesis' subgroup, by either route: files of
             // their own, never the answer's.
-            if (!result.model_prior_file_prefix.empty()) {
-                Add(s, KeyText("MODEL_CHOICE_FILE_PREFIX", result.model_prior_file_prefix));
+            if (!result.model_prior_file_suffix.empty()) {
+                Add(s, KeyText("MODEL_CHOICE_FILE_PREFIX", output_prefix + result.model_prior_file_suffix));
                 Add(s, KeyText("MODEL_CHOICE_SPACE_GROUP", result.model_prior_file_space_group));
             }
             // The twin hypothesis (section 4): the model scored against that merge too, and which of
@@ -2474,7 +2473,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                         " fraction a scanned on the working set. The maps and the placed model come from {},\n"
                         " the merge the model fits better on R-free ({:.4f} against {:.4f}). This chooses the\n"
                         " maps only: the answer's reflection files and every number above are the answer's.",
-                        t.space_group, t.file_prefix, t.twin_law,
+                        t.space_group, output_prefix + t.file_suffix, t.twin_law,
                         t.maps_from_this_merge ? "the merge in " + t.space_group : "the answer's merge",
                         tv.twin_target ? std::min(tv.r_free, tv.r_free_twin) : tv.r_free, mv.r_free)));
             }
@@ -2704,7 +2703,8 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                         " atoms that is not inverted (ENANTIOMORPH_VETOED otherwise) - and nothing else; another group it\n"
                         " chooses is written as {}.mtz/.cif with the placed model.",
                         mp.d_max_search, mp.d_min_search,
-                        result.model_prior_file_prefix.empty() ? std::string("<prefix>_mr") : result.model_prior_file_prefix)));
+                        result.model_prior_file_suffix.empty() ? std::string("<prefix>_mr")
+                                                               : output_prefix + result.model_prior_file_suffix)));
                 Add(s, Prose(fmt::format(
                         " Seconds: model {:.2f}, transform {:.2f}, rotation {:.2f}, translation {:.2f}, rigid body and\n"
                         " R-free {:.2f}, null {:.2f}, PAE domains {:.2f}, hand check {:.3f}.", mp.t_model, mp.t_transform,
@@ -3136,26 +3136,3 @@ std::string RenderResultReport(const std::string &output_prefix,
             BuildReportDocument(output_prefix, input_file, experiment, result, provenance),
             provenance.developer);
 }
-
-ReportDocument WriteResultReport(const std::string &output_prefix,
-                                 const std::string &input_file,
-                                 const DiffractionExperiment &experiment,
-                                 const ProcessResult &result,
-                                 Logger &logger,
-                                 const RunProvenance &provenance) {
-    ReportDocument doc = BuildReportDocument(output_prefix, input_file, experiment, result, provenance);
-    if (output_prefix.empty())
-        return doc;   // "compute the statistics, persist nothing"
-
-    const std::string filename = output_prefix + "_report.txt";
-    // The report is unconditional, so it must never be the reason a run fails: a run that produced a
-    // good .mtz must survive an unwritable path or a full disk. Report the failure and carry on.
-    try {
-        std::ofstream file(filename);
-        file.exceptions(std::ios::failbit | std::ios::badbit);
-        file << RenderReportText(doc, provenance.developer);
-    } catch (const std::exception &e) {
-        logger.Warning("Could not write the results report {}: {}", filename, e.what());
-    }
-    return doc;
-}

rugnux/ResultReport.h

@@ -72,14 +72,3 @@ std::string RenderResultReport(const std::string &output_prefix,
                                const DiffractionExperiment &experiment,
                                const ProcessResult &result,
                                const RunProvenance &provenance = {});
-
-// Renders and writes <output_prefix>_report.txt, and returns the document it rendered (the CLI's
-// closing summary reads it). Writes nothing when the prefix is empty (the "compute statistics,
-// persist nothing" mode). Never throws: a run that produced good reflections must not be lost
-// because a side file could not be written, so a failure is logged and swallowed.
-ReportDocument WriteResultReport(const std::string &output_prefix,
-                                 const std::string &input_file,
-                                 const DiffractionExperiment &experiment,
-                                 const ProcessResult &result,
-                                 Logger &logger,
-                                 const RunProvenance &provenance = {});

rugnux/Rugnux.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -41,7 +39,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"
@@ -434,18 +431,26 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) {
         result = RunAllPasses(observer);
     } catch (const NoLatticeFound &) {
         // A run that found no lattice after deciding a near tie early is made again deciding late.
-        if (!near_tie_rechecks_->carried)
+        if (!near_tie_rechecks_->carried || config_.fast_live)
             throw;
         lost = true;
     }
-    if (lost || (!result.cancelled && NearTieOverturned())) {
+    if (lost || (!result.cancelled && !config_.fast_live && NearTieOverturned())) {
         JoinBeamCenterCapture();
         Logger logger("Rugnux");
         logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, "
                     "deciding near ties on all the frames");
+        result.outputs.reset();   // replaced: not held through the run made again
         result = again->RunAllPasses(observer);
         run = again.get();
     }
+    // Cut short in fast_live: a pass cut before its image loop read anything has no lattice and no merge,
+    // so the run's answer is the last pass that ran to its end. (A canonical pass cut inside its image
+    // loop merges the images it read - see ImagePass.)
+    if (config_.fast_live && result.cancelled && established_) {
+        result = *established_;
+        result.cancelled = true;
+    }
     result.total_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - run_start).count();
     // For the report's beam-centre warning: what the background measured, and what this crystal asks
     // of the centre - at the file's geometry, as the pre-scan's check asks it, but on the cell the run
@@ -743,7 +748,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     // skip the search that is the whole point of running again.
     // The run as this pass finds it, for the re-check of a near tie its first pass may decide early.
     std::shared_ptr<Rugnux> recheck;
-    if (!decide_late_ && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing
+    if (!decide_late_ && !config_.fast_live && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing
         && config_.two_pass_rotation && !force_rotation_result_.has_value()
         && !config_.forced_rotation_lattice.has_value() && experiment_.IsRotationIndexing())
         recheck = std::make_shared<Rugnux>(*this);
@@ -782,7 +787,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     const bool calibration = (config_.mode == ProcessMode::Calibration);
     const bool calibration_spots = calibration && config_.calibration_method == CalibrationMethod::Spots;
     const bool per_image_analysis = full || calibration;
-    const bool write_files = write_output && !config_.output_prefix.empty();
+    // The output pass makes what the run returns to its caller (ProcessResult::outputs).
+    if (write_output && !geometry_prepass)
+        result.outputs = std::make_shared<RugnuxOutputs>();
     // Set below the first merge, once the pass can tell that RunAllPasses is going to throw it away and
     // run again - see there for what that turns off and why it is knowable at that point.
     bool superseded = false;
@@ -792,9 +799,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     experiment_.BitDepthImage(32).PixelSigned(true);
     experiment_.Mode(DetectorMode::Standard);
     experiment_.OverwriteExistingFiles(true);
-    // Offline processing: the output prefix is a local path the operator chose (rugnux -o or the
-    // viewer's "next to the input file"), so use the trusted setter that permits an absolute path.
-    experiment_.FilePrefixTrusted(config_.output_prefix.empty() ? "output" : config_.output_prefix);
     experiment_.SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
     experiment_.ImagesPerTrigger(images_to_process);
     experiment_.NumTriggers(1);
@@ -868,7 +872,8 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     JFJochReceiverPlots plots;
     plots.Setup(experiment_, mapping);
 
-    // Output file (NXmxIntegrated master that links back to the original images).
+    // The per-image analysis' start message (an NXmxIntegrated master that links back to the original
+    // images); the image sink names the file.
     StartMessage start_message;
     experiment_.FillMessage(start_message);
     start_message.arm_date = dataset->arm_date;
@@ -919,7 +924,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     }
 
     // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
-    // but the process file reads them. Copy them where a writer is actually built rather than here:
+    // but the process file reads them. Copy them where the start message is handed on rather than here:
     // the default merging path writes no process file at all.
     auto fill_writer_maps = [&] {
         start_message.pixel_mask["default"] = pixel_mask_.GetMask(experiment_);
@@ -927,18 +932,18 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             start_message.roi_map = experiment_.ExportROIMap();
     };
 
-    std::unique_ptr<FileWriter> writer;
+    // The output pass streams its per-image analysis to the caller's image sink, where there is one.
+    RugnuxImageSink *const sink = write_output && !geometry_prepass ? config_.image_sink : nullptr;
     std::unique_ptr<ProcessFileWriter> writer_queue;
     // Whether anything is in place to break the indexing ambiguity, for the warning further down: a
     // reference MTZ, or (serial stills) the reference computed from a model just below.
     bool ambiguity_reference = !config_.reference_data.empty();
-    if (write_files && config_.write_process_h5) {
+    if (sink) {
         fill_writer_maps();
-        writer = std::make_unique<FileWriter>(start_message, /*check_overwrite_at_start=*/true,
-                                              /*trusted_path=*/true);
+        sink->Start(start_message, output_label_);
         // Deep enough that a worker never waits for the writer in the normal case, shallow enough that
         // the backlog is bounded by the worker count rather than by the length of the run.
-        writer_queue = std::make_unique<ProcessFileWriter>(*writer, 4 * std::max(1, config_.nthreads));
+        writer_queue = std::make_unique<ProcessFileWriter>(*sink, 4 * std::max(1, config_.nthreads));
     }
 
     const char *mode_name = full ? "full analysis"
@@ -997,12 +1002,12 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
         std::vector<MergedReflection> model_reference;
         if (!config_.reference_data.empty()) {
             indexer->ReferenceIntensities(config_.reference_data);
-        } else if (!config_.model_path.empty() && !experiment_.IsRotationIndexing()
+        } else if (config_.model && !experiment_.IsRotationIndexing()
                    && experiment_.GetUnitCell().has_value() && experiment_.GetGemmiSpaceGroup().has_value()) {
             const auto d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A()
                 .value_or(experiment_.GetDetectorMaxResolution_A());
             model_reference = ModelReferenceIntensities(
-                config_.model_path, experiment_.GetUnitCell(),
+                *config_.model, experiment_.GetUnitCell(),
                 &experiment_.GetSpaceGroupOrP1(), d_min, logger);
             if (!model_reference.empty()) {
                 indexer->ReferenceIntensities(model_reference);
@@ -1032,26 +1037,23 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image;
 
     // Declared here rather than where they are first filled, so that the phases below can share them
-    // through PipelineLocals: the bytes the image loop read, the end message (also written to the file),
-    // and the unmerged MTZ when it is built beside the reflection files (see there) with the
-    // experiment it is built from - which it points into for its space group, so it lives until the
-    // file is written.
+    // through PipelineLocals: the bytes the image loop read, the end message (also handed to the sink),
+    // and the unmerged MTZ when it is built beside the reflection files (see there).
     std::atomic<uint64_t> total_uncompressed_bytes = 0;
     EndMessage end_msg;
-    std::optional<DiffractionExperiment> unmerged_experiment;
-    std::future<gemmi::Mtz> unmerged_mtz;
+    std::future<MtzTable> unmerged_mtz;
     PipelineLocals p{
             .logger = logger, .result = result, .observer = observer,
             .geometry_prepass = geometry_prepass, .full = full, .calibration = calibration,
             .calibration_spots = calibration_spots, .per_image_analysis = per_image_analysis,
-            .write_files = write_files, .start_image = start_image, .end_image = end_image,
+            .write_output = write_output, .start_image = start_image, .end_image = end_image,
             .images_to_process = images_to_process, .prepass_images = prepass_images, .loop_end = loop_end,
             .dataset = dataset, .mapping = mapping, .plots = plots,
-            .writer = writer, .writer_queue = writer_queue, .indexer_pool = indexer_pool, .indexer = indexer,
+            .sink = sink, .writer_queue = writer_queue, .indexer_pool = indexer_pool, .indexer = indexer,
             .indexing_bound_A = indexing_bound_A, .census_future = census_future, .superseded = superseded,
             .ambiguity_reference = ambiguity_reference, .start_time = start_time,
             .total_uncompressed_bytes = total_uncompressed_bytes, .end_msg = end_msg,
-            .unmerged_experiment = unmerged_experiment, .unmerged_mtz = unmerged_mtz};
+            .unmerged_mtz = unmerged_mtz};
 
     // First pass of two-pass rotation indexing (full analysis only).
     if (full && force_rotation_result_.has_value()) {
@@ -1108,7 +1110,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
     image_pass_mark.Stop();
 
     // Scaling and merging (full analysis only).
-    if (full && !cancelled_ && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found)
+    // As ImagePass's stopped(): in fast_live a pass that read all its images merges whatever comes after.
+    const bool images_cut = config_.fast_live ? result.cancelled : static_cast<bool>(cancelled_);
+    if (full && !images_cut && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found)
         && (config_.run_scaling || !config_.reference_data.empty())) {
         TimingMark mark(logger, "scale, merge and symmetry");
         std::optional<GPUMerge> gpu_merge;   // what the speculative probe waits for where both do not fit
@@ -1127,47 +1131,42 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             return result;
     }
 
-    TimingMark output_mark(logger, "output files");
+    TimingMark output_mark(logger, "outputs");
 
     // Unmerged observations. After the merge, because that is where the space group is settled and
     // the file has to be written in it, but from the integrated observations rather than the merged
     // ones: the partiality and the per-image scale are left for the reading program, which fits a
-    // scale model of its own. write_files, not write_output - without an output prefix there is
-    // nowhere to put it, and the name would come out as a bare "_unmerged.mtz".
-    if (full && !cancelled_ && write_files && !geometry_prepass && !superseded && result.consensus_cell) {
+    // scale model of its own.
+    if (full && !cancelled_ && result.outputs && !superseded && result.consensus_cell) {
         if (config_.export_unmerged) {
-            if (observer) observer->OnPhase("Writing unmerged reflections");
-            const std::string path = config_.output_prefix + "_unmerged.mtz";
-            if (unmerged_mtz.valid()) {
-                gemmi::Mtz mtz = unmerged_mtz.get();
-                SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
-                mtz.write_to_file(path);
-            } else
-                WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell,
-                                            experiment_, true, config_.provenance, path, config_.nthreads);
-            logger.Info("Unmerged observations written to {}", path);
+            if (observer) observer->OnPhase("Unmerged reflections");
+            MtzTable mtz = unmerged_mtz.valid()
+                           ? unmerged_mtz.get()
+                           : UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell,
+                                              experiment_, true, config_.provenance, config_.nthreads);
+            SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
+            result.outputs->unmerged = std::move(mtz);
         }
         if (config_.export_unmerged_partials) {
-            if (observer) observer->OnPhase("Writing unmerged reflections");
-            const std::string path = config_.output_prefix + "_unmerged_partials.mtz";
-            WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell,
-                                        experiment_, false, config_.provenance, path, config_.nthreads);
-            logger.Info("Unmerged partials written to {}", path);
+            if (observer) observer->OnPhase("Unmerged reflections");
+            MtzTable mtz = UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell,
+                                            experiment_, false, config_.provenance, config_.nthreads);
+            SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
+            result.outputs->unmerged_partials = std::move(mtz);
         }
     }
 
     // When merging is on, the _process.h5 is skipped because the merged reflections are the wanted
     // output and the per-image file is large. If nothing indexed there are no merged reflections, so
-    // the run would finish successfully having written no file at all. Write one now: the per-image
-    // messages went past while there was no writer, but the end message still carries the per-image
+    // the run would finish successfully having written no file at all. Hand the caller one: the
+    // per-image messages went past with no sink, but the end message still carries the per-image
     // scalars (spot counts, background, resolution estimate) and the dataset metadata - which is what
     // anyone asking "why did nothing index" needs, and it is small precisely because nothing indexed.
-    if (!writer && full && write_files && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) {
-        logger.Warning("No image indexed, so there are no merged reflections to write - writing "
-                       "{}_process.h5 with the per-image analysis instead", config_.output_prefix);
+    if (!sink && full && result.outputs && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) {
+        logger.Warning("No image indexed, so there are no merged reflections - the per-image analysis is "
+                       "kept for a _process.h5 instead");
         fill_writer_maps();
-        writer = std::make_unique<FileWriter>(start_message, /*check_overwrite_at_start=*/true,
-                                              /*trusted_path=*/true);
+        result.outputs->process_start = start_message;
     }
 
     // The per-frame sigma_M the merge wrote back onto the outcomes (the _plots.txt column), for the report.
@@ -1183,22 +1182,13 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             result.mosaicity_deg_p90 = mos[mos.size() * 9 / 10];
         }
     }
-    // The two files a person checks the run by eye with. Not on the geometry pre-pass or a pass that
-    // is thrown away, which would otherwise leave a stale _01 copy beside the real one. The table is
-    // written on a stills run too - it is the only per-image scaling table there is, and the columns
+    // The two files a person checks the run by eye with. Not on a pass that is thrown away. The table
+    // is made on a stills run too - it is the only per-image scaling table there is, and the columns
     // a stills run cannot fill come out as nan; the picture is rotation-only at its source, since
     // only the rotation pre-scan builds the projection it draws.
-    if (full && write_files && !cancelled_ && !geometry_prepass && !superseded) {
-        const std::string jpeg = detector_diagnostic_jpeg_.valid() ? detector_diagnostic_jpeg_.get() : std::string();
-        if (!jpeg.empty()) {
-            const std::string path = config_.output_prefix + "_detector.jpg";
-            std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
-            if (f) {
-                f.write(jpeg.data(), static_cast<std::streamsize>(jpeg.size()));
-                logger.Info("Detector diagnostic image written to {}", path);
-            } else
-                logger.Warning("Cannot open {} for writing", path);
-        }
+    if (full && result.outputs && !cancelled_ && !superseded) {
+        if (detector_diagnostic_jpeg_.valid())
+            result.outputs->detector_jpeg = detector_diagnostic_jpeg_.get();
 
         PerImagePlot plot;
         plots.GetPlotRaw(plot.bkg_estimate, PlotType::BkgEstimate, "");
@@ -1217,16 +1207,17 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
         }
         if (result.has_merge_statistics)
             plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition;
-        WritePerImagePlot(config_.output_prefix, plot, experiment_.GetGoniometer(),
-                          static_cast<size_t>(images_to_process));
-        logger.Info("Per-image table written to {}_plot.txt", config_.output_prefix);
+        plot.goniometer = experiment_.GetGoniometer();
+        plot.image_count = static_cast<size_t>(images_to_process);
+        result.outputs->plot = std::move(plot);
     }
 
-    if (writer) {
-        writer->WriteHDF5(end_msg);
-        writer->Finalize();
-        result.written_master_path = config_.output_prefix + "_process.h5";
-    }
+    if (sink)
+        sink->End(end_msg);
+    result.end_message = end_msg;
+    // A pass that is to be run again makes nothing.
+    if (superseded)
+        result.outputs.reset();
 
     output_mark.Stop();
 

rugnux/Rugnux.h

@@ -44,6 +44,7 @@
 #include "../image_analysis/WriteReflections.h"   // OutputProvenance
 #include "../image_analysis/bragg_integration/SpotWidth.h"       // FluxCurve
 #include "../image_analysis/bragg_integration/SpotFootprint.h"   // FootprintSpot
+#include "DiagnosticOutput.h"                                     // PerImagePlot
 
 class JFJochReader;
 class JFJochHDF5Reader;
@@ -51,17 +52,31 @@ struct JFJochReaderRawImage;
 struct PipelineLocals;   // RugnuxPipeline.h, private to the implementation
 
 // Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux (all of its --mode
-// values that read images) and (later) the viewer. The full workflow lives here, not in the CLIs:
+// values that read images), the viewer and the broker. The full workflow lives here, not in the CLIs:
 // setup, an optional two-pass rotation-indexing pre-pass, a parallel per-image loop (std::thread),
-// an optional scaling/merging post-pass, and the _process.h5 output. The detector geometry and all
-// algorithm settings are configured on the DiffractionExperiment by the caller; ProcessConfig only
-// carries run control. Cancellable from any thread (e.g. SIGINT or a GUI button) via Cancel().
+// an optional scaling/merging post-pass. It reads and writes no file: what it makes is returned in
+// ProcessResult (RugnuxOutputs) and handed to RugnuxImageSink, and the caller writes it
+// (RugnuxOutputFiles.h). The detector geometry and all algorithm settings are configured on the
+// DiffractionExperiment by the caller; ProcessConfig only carries run control. Cancellable from any
+// thread (e.g. SIGINT or a GUI button) via Cancel().
 enum class ProcessMode {
     AzimuthalIntegration,   // preprocess + azimuthal integration only (rugnux --mode=azint)
     FullAnalysis,           // spot finding + indexing + refinement + integration (rugnux --mode=mx)
     Calibration             // detector geometry from powder rings (rugnux --mode=calibration)
 };
 
+// Where the per-image analysis of an output pass goes as it is made: rugnux and the viewer put it in
+// <prefix>_process.h5 (ProcessFileSink, RugnuxOutputFiles.h). Called from one thread at a time.
+class RugnuxImageSink {
+public:
+    virtual ~RugnuxImageSink() = default;
+    // A pass starts. `label` tells its files apart from the answer's: empty for every pass but the
+    // doubled cell's, which is "_doubled" (ProcessResult::DoubledCell).
+    virtual void Start(const StartMessage &start, const std::string &label) = 0;
+    virtual void Write(const DataMessage &msg) = 0;
+    virtual void End(const EndMessage &end) = 0;
+};
+
 struct ProcessConfig {
     ProcessMode mode = ProcessMode::FullAnalysis;
 
@@ -78,20 +93,23 @@ struct ProcessConfig {
     // setting it (the tests do) changes the run's time and its memory, and nothing it writes.
     size_t host_memory_budget_bytes = 0;
 
-    // Output prefix for the _process.h5 (and scaled reflections). Empty => process without writing.
-    std::string output_prefix;
-    // What the reflection files record about where they came from (WriteReflections).
+    // What the reflection files record about where they came from (the MTZ HISTORY, WriteReflections).
     OutputProvenance provenance;
 
-    // Write the per-image _process.h5. Defaults true; rugnux turns it off when merging
-    // (the .mtz/.cif is the wanted output and the h5 is large) unless --write-process-h5 is given.
-    bool write_process_h5 = true;
+    // Where the per-image analysis of each output pass goes (the _process.h5); none: it is not kept.
+    // rugnux gives one when not merging, or with --write-process-h5.
+    RugnuxImageSink *image_sink = nullptr;
 
-    // Write the merged reflections (.mtz/.cif). Defaults true (the CLI always writes them when
-    // merging); the viewer can turn them off independently of the _process.h5 (e.g. a quick run that
-    // only wants the ISa/R-meas numbers). Only relevant to FullAnalysis with scaling.
+    // Make the merged reflections' outputs (RugnuxOutputs::merged and the twin hypothesis' and finalist
+    // ledger's merges). Defaults true (the CLI always writes them when merging); the viewer can turn them
+    // off (e.g. a quick run that only wants the ISa/R-meas numbers). Only relevant to FullAnalysis with
+    // scaling.
     bool write_merged = true;
 
+    // Keep the model validation's maps themselves (ModelMaps), not only their coefficients: rugnux writes
+    // them as .ccp4. The broker never asks for them.
+    bool keep_map_grids = false;
+
     SpotFindingSettings spot_finding;   // FullAnalysis spot finding (and Calibration by spots)
 
     // Calibration: how the powder rings are measured, and the calibrant's rings as q = 2*pi/d (1/A).
@@ -115,6 +133,13 @@ struct ProcessConfig {
     // are decided (Rugnux::decide_late_).
     bool live = false;
 
+    // Live in the broker, where the run has to end a fixed time after the last frame: every decision
+    // stands where it was first taken, and nothing the run would otherwise do again once the sweep is
+    // in - the near-tie re-checks and the run made again, the rotation-scale walk, the metric-symmetry
+    // arms, the geometry walk, the supercell and quality guards, the doubled cell - is done. A
+    // difficult crystal can come out worse; an easy one comes out the same, sooner.
+    bool fast_live = false;
+
     // Rotation indexing (FullAnalysis)
     bool rotation_indexing = false;
     bool two_pass_rotation = true;
@@ -189,15 +214,16 @@ struct ProcessConfig {
     std::string reference_path;
     std::string reference_column;
 
-    // Diagnostic: if set, the -P rot3d combine writes the unmerged fulls here (for comparison vs XDS).
-    std::string observation_dump_path;
+    // Diagnostic: if set, handed the unmerged fulls of every -P rot3d combine as text, for comparison
+    // with XDS (--dump-observations).
+    std::function<void(const std::string &)> observation_dump;
 
-    // Write the integrated observations as <prefix>_unmerged.mtz, for aimless / pointless / careless
-    // and anything else that scales unmerged data itself - each rocking event summed into one full.
-    // On by default (--no-export-unmerged turns it off): it is how a run's observations reach the
-    // rest of the field, and a user who does not know the flag exists should still get the file.
-    // It is the largest file a run produces, so a batch that only wants the merged numbers turns it
-    // off. Needs an output prefix.
+    // Make the integrated observations as <prefix>_unmerged.mtz (RugnuxOutputs::unmerged), for aimless /
+    // pointless / careless and anything else that scales unmerged data itself - each rocking event summed
+    // into one full. On by default (--no-export-unmerged turns it off): it is how a run's observations
+    // reach the rest of the field, and a user who does not know the flag exists should still get the
+    // file. It is the largest file a run produces, so a batch that only wants the merged numbers turns
+    // it off.
     bool export_unmerged = true;
 
     // The same as <prefix>_unmerged_partials.mtz, with each partial written as its own row for the
@@ -205,10 +231,10 @@ struct ProcessConfig {
     // run can write both.
     bool export_unmerged_partials = false;
 
-    // If set, after merging, validate the merged intensities against this atomic model (PDB or
-    // mmCIF, gzipped or not): compute R-free with an optimized bulk solvent and write 2Fo-Fc /
-    // Fo-Fc maps (--model).
-    std::string model_path;
+    // If set, after merging, validate the merged intensities against this atomic model: compute
+    // R-free with an optimized bulk solvent and write 2Fo-Fc / Fo-Fc maps (--model). The caller parses
+    // it (ParseRugnuxModel, writer/RugnuxModelFile.h).
+    std::shared_ptr<const RugnuxModel> model;
 
     // --mr: the model is searched for by molecular replacement straight away, without the rigid-body
     // placement first. Without it a --model that is a prediction, or an experimental structure the rigid body
@@ -320,6 +346,48 @@ struct SearchMergeSignal {
     int64_t inner_reflections = 0; // ... of this many reflections
 };
 
+// The reflection files of one merge, <prefix><suffix>.mtz and .cif (and the answer's .hkl): the MTZ as its
+// table, and what the mmCIF and the HKLF-4 file are written from (WriteReflections.h).
+struct MergedOutput {
+    std::string suffix;                          // "", "_twin_<group>" or "_mr"
+    MtzTable mtz;
+    std::vector<MergedReflection> reflections;
+    UnitCell cell{};
+    DiffractionExperiment experiment;            // in the group the files are written in
+    MergeStatistics statistics;
+    ErrorModelReport error_model;
+    TwinningAnalysisResult twinning;             // a twin hypothesis' law and fraction, for its mmCIF
+    std::vector<ScaledFull> scaled_fulls;        // the .hkl's records, where the merge made them (rotation)
+};
+
+// An atomic model as placed against the data (<prefix>_model.cif, WritePlacedModel): `cell` and
+// `space_group` are those of the reflection files beside it.
+struct PlacedModelOutput {
+    std::shared_ptr<gemmi::Structure> model;
+    UnitCell cell{};
+    gemmi::SpaceGroup space_group;
+};
+
+// What a run makes for its caller to keep - the files rugnux writes, each named here after the file it
+// goes to at the caller's prefix. Made by the output pass, the one whose result the run returns.
+struct RugnuxOutputs {
+    std::optional<MergedOutput> merged;                       // <prefix>.mtz/.cif/.hkl
+    std::optional<MergedOutput> twin;                         // the twin hypothesis' merge
+    std::optional<MergedOutput> model_prior;                  // the model's choice, <prefix>_mr.mtz/.cif
+    std::shared_ptr<ModelMaps> maps;                          // <prefix>_maps.mtz, _2fofc/_fofc/_anom.ccp4
+    std::shared_ptr<ModelMaps> model_prior_maps;              // the same at <prefix>_mr
+    std::optional<PlacedModelOutput> placed_model;            // <prefix>_model.cif/.pdb
+    std::optional<PlacedModelOutput> model_prior_placed_model;   // <prefix>_mr_model.cif/.pdb
+    std::optional<MtzTable> unmerged;                         // <prefix>_unmerged.mtz
+    std::optional<MtzTable> unmerged_partials;                // <prefix>_unmerged_partials.mtz
+    std::string detector_jpeg;                                // <prefix>_detector.jpg; empty: none drawn
+    std::optional<PerImagePlot> plot;                         // <prefix>_plot.txt
+    // A run that indexed nothing, with no per-image analysis streamed (ProcessConfig::image_sink): the
+    // start message of a <prefix>_process.h5 holding the end message (ProcessResult::end_message), so
+    // there is a file that says why nothing indexed.
+    std::optional<StartMessage> process_start;
+};
+
 struct ProcessResult {
     bool cancelled = false;
     uint64_t images_processed = 0;
@@ -331,6 +399,11 @@ struct ProcessResult {
     // beam-centre pre-scan that runs before the first one. processing_time_s is the last pass alone
     // and so understates a two-pass run by more than a factor of two.
     double total_time_s = 0.0;
+    // The end message of this pass, as the _process.h5 records it: the lattice, refined geometry,
+    // reindexing and the run means - what the broker puts into its own when it runs rugnux live.
+    EndMessage end_message;
+    // What this pass made for the caller to keep - the output pass's only; null on every other pass.
+    std::shared_ptr<RugnuxOutputs> outputs;
     // Workers the per-image loop actually ran, which is not config.nthreads once the loop is capped.
     int image_loop_threads = 0;
     double frame_rate_hz = 0.0;
@@ -387,7 +460,6 @@ struct ProcessResult {
     // every other class, or where the free refinement is not available.
     std::optional<double> lattice_angle_departure_deg;
     MeanProcessingTime mean_processing_time{};
-    std::optional<std::string> written_master_path;
     std::string merge_statistics_text;   // populated when scaling/merging ran
 
     // Structured merge statistics (per-shell + overall), populated when scaling/merging ran. ISa is
@@ -564,7 +636,7 @@ struct ProcessResult {
         std::string reason;                  // why the hypothesis was written
         std::string adopted_space_group;     // the group of the answer, as written
         std::string space_group;             // the subgroup, as written (standard setting)
-        std::string file_prefix;             // <prefix>_twin_<SG>
+        std::string file_suffix;             // _twin_<SG>, or _mr: the files are <prefix><file_suffix>
         std::string twin_law;                // hkl triplet, on the subgroup file's axes
         double twin_fraction = NAN;          // EstimateTwinLawFraction: a lower bound
         int twin_fraction_pairs = 0;
@@ -586,7 +658,8 @@ struct ProcessResult {
     struct DoubledCell {
         bool adopted = false;              // the answer is the doubled cell, the other one the probed cell
         std::string decision;              // why, in the words the run reports it in
-        std::string other_prefix;          // <prefix>_halfcell or <prefix>_doubled
+        std::string other_suffix;          // _halfcell or _doubled: the other cell's files are <prefix><other_suffix>
+        std::shared_ptr<RugnuxOutputs> other_outputs;
         UnitCell other_cell{};
         std::string other_space_group;
         double other_d_min_A = NAN, other_cc_half = NAN, other_r_free = NAN;
@@ -606,9 +679,9 @@ struct ProcessResult {
     std::string model_placement;
     // What molecular replacement found, where it ran; failure_reason says why where it could not.
     std::optional<MolecularReplacementResult> model_prior;
-    // The file prefix the model's choice of another group or the twin hypothesis' subgroup was written under
-    // (<prefix>_mr), and that group; empty where nothing was.
-    std::string model_prior_file_prefix;
+    // The file suffix the model's choice of another group or the twin hypothesis' subgroup is written under
+    // (_mr), and that group; empty where nothing was.
+    std::string model_prior_file_suffix;
     std::string model_prior_file_space_group;
 
     // Space group used for the final re-scale/merge and written to the master file: determined by the
@@ -793,6 +866,10 @@ public:
     virtual void OnPhase(const std::string &phase) {}
     virtual void OnProgress(uint64_t done, uint64_t total) {}
     virtual void OnImageProcessed(const DataMessage &msg) {}
+    // An image pass starts: OnImageProcessed then reports its images. `canonical` on the pass whose
+    // per-image results the run keeps - every image once, at the final geometry; a pre-pass or a probe pass
+    // reports some of the images again, or at another geometry.
+    virtual void OnImagePass(bool canonical) {}
     virtual void OnStep(const std::string &title) {}
     virtual void OnNote(const std::string &text) {}
 };
@@ -820,6 +897,9 @@ class Rugnux {
         CancelFlag Own() const { CancelFlag f; f.run = flag; return f; }
     };
     CancelFlag cancelled_;
+    // fast_live: the last pass that ran to its end. Where a cancel cuts the pass after it short, Run()
+    // returns this one, so the broker sends what is established (lattice, geometry) rather than nothing.
+    std::optional<ProcessResult> established_;
 
     // Where the steps and notes of the pass now running go: the observer on a pass that makes the
     // result, nullptr on a probe pass and on a copy run beside the main one. Set by RunPipeline.
@@ -877,8 +957,11 @@ class Rugnux {
     std::optional<RotationIndexerResult> force_rotation_result_;
     // The indexing result the last full pass integrated on, which the doubled-cell hypothesis doubles.
     std::optional<RotationIndexerResult> last_rotation_result_;
-    // Keep the written merged reflections on the result (ProcessResult::merged_reflections).
+    // Keep the merged reflections on the result (ProcessResult::merged_reflections).
     bool keep_merged_reflections_ = false;
+    // What the image sink is told the output pass is (RugnuxImageSink::Start): "_doubled" for the doubled
+    // cell's pass, empty otherwise.
+    std::string output_label_;
 
     // The space group the USER fixed (-S on the CLI, the settings panel in the viewer), captured before
     // anything can clear it - a pass that determines a group of its own writes it onto the experiment,
@@ -1264,7 +1347,8 @@ class Rugnux {
                               RugnuxObserver *observer);
 
     // One full run of the pipeline (index + integrate + optional scale/merge/space-group + output).
-    // write_output == false suppresses all file output (the geometry pre-pass writes nothing).
+    // write_output == false makes no outputs (ProcessResult::outputs) and streams nothing to the image sink;
+    // nor does the geometry pre-pass.
     // geometry_prepass == true stops after post-refining the detector geometry from the first integration
     // and applies it (+ the refined lattice) to experiment_, so the next pass re-integrates with it.
     ProcessResult RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass);
@@ -1341,7 +1425,6 @@ class Rugnux {
     };
     // What the refined passes start from.
     struct RefinedPassStart {
-        std::string base_prefix;
         std::optional<GoniometerAxis> gonio_snapshot;
         std::array<float, 5> header_geometry{};
         RotationScaleOutcome scale_walk;   // where the walk has ended before they start
@@ -1350,8 +1433,7 @@ class Rugnux {
     // the rotation-scale walk and it overturned them.
     // After the refined passes: where the supercell probe's class stands clear, integrate and merge the
     // doubled cell too, and return whichever of the two is the answer (ProcessResult::doubled_cell).
-    ProcessResult DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half,
-                                        const std::string &base_prefix);
+    ProcessResult DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half);
     std::optional<ProcessResult> RefinedPasses(RugnuxObserver *observer, ProcessResult pass1,
                                                const RefinedPassStart &start);
     // The rotation-scale walk, its probes on copies of the run, while the refined passes run on a copy
@@ -1369,6 +1451,9 @@ public:
     const PowderRings &GetPowderRings() const { return powder_; }
 
     // Request cancellation; safe to call from any thread (the worker loop checks between images).
+    // fast_live makes it a cut: a pass that is past its images still merges and makes its outputs, the
+    // canonical pass cut in its images merges those it read, only the model is left out, and any other
+    // pass cut short gives way to the last complete one.
     void Cancel() { cancelled_ = true; }
     bool IsCancelled() const { return cancelled_; }
 };

rugnux/RugnuxCalibration.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -39,7 +37,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"

rugnux/RugnuxCommandLine.cpp

@@ -53,6 +53,8 @@ namespace {
 std::string RugnuxCommandLine(const ProcessConfig &config,
                                      const DiffractionExperiment &experiment,
                                      const std::string &input_file,
+                                     const std::string &output_prefix,
+                                     bool write_process_h5,
                                      const std::string &calibrant_name) {
     std::vector<std::string> args;
     const bool azint = (config.mode == ProcessMode::AzimuthalIntegration);
@@ -71,8 +73,8 @@ std::string RugnuxCommandLine(const ProcessConfig &config,
         args.push_back(val);
     };
 
-    if (!config.output_prefix.empty())
-        add("-o", config.output_prefix);
+    if (!output_prefix.empty())
+        add("-o", output_prefix);
     // Every CPU the run may use is what rugnux takes without -N, on whatever machine runs the command.
     if (!config.nthreads_auto)
         add("-N", std::to_string(config.nthreads));
@@ -82,8 +84,8 @@ std::string RugnuxCommandLine(const ProcessConfig &config,
             if (!config.reference_column.empty())
                 add("--reference-column", config.reference_column);
         }
-        if (!config.model_path.empty())
-            add(config.force_mr ? "--mr" : "--model", config.model_path);
+        if (config.model)
+            add(config.force_mr ? "--mr" : "--model", config.model->name);
     }
     if (config.start_image != 0)
         add("-s", std::to_string(config.start_image));
@@ -224,7 +226,7 @@ std::string RugnuxCommandLine(const ProcessConfig &config,
             // When merging, the CLI skips the large _process.h5 unless asked; emit the flag when it is
             // wanted so a copied command matches the GUI's "Save _process.h5" choice. (write_merged has
             // no CLI equivalent - the CLI always writes the .mtz/.cif when merging.)
-            if (config.write_process_h5)
+            if (write_process_h5)
                 args.emplace_back("--write-process-h5");
             // The low-resolution limit is on by default at the same value in both, so emit it only when
             // it differs - including 0, which is how the CLI spells "no limit".

rugnux/RugnuxCommandLine.h

@@ -14,7 +14,11 @@ class DiffractionExperiment;
 // matter for the run, not every obscure flag; geometry is taken from the input file, so geometry
 // overrides are not emitted. calibrant_name is only used by ProcessMode::Calibration - the config
 // carries the calibrant's rings, and --calibrant takes the name they were resolved from.
+// output_prefix and write_process_h5 are where the run puts its files and whether a merging run writes
+// its _process.h5 too.
 std::string RugnuxCommandLine(const ProcessConfig &config,
                                      const DiffractionExperiment &experiment,
                                      const std::string &input_file,
+                                     const std::string &output_prefix,
+                                     bool write_process_h5,
                                      const std::string &calibrant_name = "");

rugnux/RugnuxFirstPass.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -39,7 +37,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"

rugnux/RugnuxFirstPassAccept.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -39,7 +37,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"

rugnux/RugnuxFirstPassRescues.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -39,7 +37,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"

rugnux/RugnuxImagePass.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -19,7 +18,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -39,7 +37,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"
@@ -357,8 +354,10 @@ void Rugnux::ImagePass(PipelineLocals &p) {
         bragg_overlapped_major += bragg.overlapped_major;
     };
 
-    if (observer)
+    if (observer) {
         observer->OnPhase("Processing images");
+        observer->OnImagePass(p.write_output && !geometry_prepass);
+    }
     Step(fmt::format("{}{}, {} images",
                      !per_image_analysis ? "Azimuthal integration"
                      : calibration ? "Image pass"
@@ -408,8 +407,19 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     result.image_loop_time_s = std::chrono::duration<double>(
             std::chrono::steady_clock::now() - image_loop_start).count();
 
-    result.cancelled = cancelled_;
     result.images_processed = finished_count.load();
+    // fast_live: a cut inside the canonical pass's image loop ends its sweep at the images read so far, and
+    // the pass merges those, a partial sweep, as a pre-pass merges its share. Any other pass a cut stops.
+    const bool partial_sweep = config_.fast_live && cancelled_ && p.write_output && !geometry_prepass
+                               && result.images_processed > 0;
+    result.cancelled = cancelled_ && !partial_sweep;
+    if (partial_sweep)
+        Note(fmt::format("cut: the sweep ends at image {} of {}", max_written_ordinal.load() + 1,
+                         images_to_process));
+    // Whether the pass stops here. In fast_live a cut that comes after the last image read lets the pass
+    // finish - its lattice, its merge and its files are what the run has established - and only the model
+    // after them is left out (ScaleMergeAndSymmetry).
+    const auto stopped = [&] { return config_.fast_live ? result.cancelled : static_cast<bool>(cancelled_); };
     Note(fmt::format("{} images, {:.0f} per second", result.images_processed,
                      result.images_processed / std::max(1e-6, result.image_loop_time_s)));
 
@@ -420,11 +430,13 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     // cross-validation of its smoothness then took every scale in the sweep to NaN.
     if (geometry_prepass || prepass_frames_only_)
         indexer->GetIntegrationOutcome().resize(prepass_images);
+    else if (partial_sweep)
+        indexer->GetIntegrationOutcome().resize(max_written_ordinal.load() + 1);
 
     // The footprint of the reflection rather than of the spot: the pre-scan's widths plus where the
     // spots sit against their predictions (SpotFootprint.h). Like the widths it acts only where the
     // reflection outgrows the r1 disk, and like them it is held back for the canonical pass.
-    if (measure_offsets && !cancelled_) {
+    if (measure_offsets && !stopped()) {
         const SpotFootprint fp = FootprintWithOffsets(*prescan_footprint_, footprint_offsets);
         const BraggIntegrationSettings canonical = bragg_adaptive_.value_or(experiment_.GetBraggIntegrationSettings());
         bool outgrows = false;
@@ -446,7 +458,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
         }
     }
 
-    if (full && indexer && experiment_.IsRotationIndexing() && !cancelled_) {
+    if (full && indexer && experiment_.IsRotationIndexing() && !stopped()) {
         const SupercellProbe probe = indexer->GetSupercellProbe();
         if (probe[0][0].n > 0 && probe[0][1].n > 0) {
             std::string table;
@@ -594,7 +606,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
             predicted += p_i;
             crowded += c_i;
         }
-        if (predicted > 0 && !cancelled_) {
+        if (predicted > 0 && !stopped()) {
             const double f = static_cast<double>(crowded) / static_cast<double>(predicted);
             const double n = static_cast<double>(crowding_frames.size());
             double ss = 0.0;
@@ -611,7 +623,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
 
     // Every image failing is a total failure, not a run that produced nothing: it used to be reported
     // only as per-image log lines while the process still exited 0 with no output file.
-    if (!cancelled_ && images_to_process > 0 && result.images_processed == 0)
+    if (!stopped() && images_to_process > 0 && result.images_processed == 0)
         throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                               "No image could be analyzed - see the per-image errors above");
 
@@ -621,7 +633,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     // Powder calibration: the whole run has now been accumulated - the summed (q x azimuth) profile in
     // plots, or the pooled spot list - so fit the detector geometry to it. The caller reports it and
     // writes the PONI file; nothing here is stored in the _process.h5.
-    if (calibration && !cancelled_)
+    if (calibration && !stopped())
         FitCalibration(p, calibration_spot_list);
 
     // End message (also written to the file).
@@ -642,7 +654,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
     end_msg.az_int_result["dataset"] = plots.GetAzIntProfile();
     end_msg.indexing_rate = result.indexing_rate;
 
-    if (full && !cancelled_) {
+    if (full && !stopped()) {
         const auto rot = indexer->FinalizeRotationIndexing();
         indexer->RethrowIndexerError();
         if (rot.has_value()) {
@@ -709,7 +721,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
             // it is written in - the pair the volume comparisons in Run() need.
             result.consensus_centering = rot->search_result.centering;
             // Log only: what the rest of the sweep says about the decisions taken on its first part.
-            if (!p.geometry_prepass && p.write_files && experiment_.IsRotationIndexing())
+            if (!p.geometry_prepass && p.write_output && experiment_.IsRotationIndexing())
                 FarEndCheck(p, *rot);
         }
         result.consensus_cell = indexer->GetConsensusUnitCell();

rugnux/RugnuxLive.cpp

@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "RugnuxLive.h"
+
+#include "RugnuxDefaults.h"
+
+RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads,
+                                    std::shared_ptr<const RugnuxModel> model) {
+    RugnuxLiveSetup s;
+
+    // The experiment as rugnux_cli.cpp sets it up for a rotation sweep with no options given.
+    DiffractionExperiment &x = s.experiment;
+    x = dataset.experiment;
+    ApplyRugnuxExperimentDefaults(x);
+    x.NumTriggers(1);
+    ClearStoredCrystal(x);
+    x.MaxSpotCount(RUGNUX_MAX_SPOT_COUNT);
+
+    IndexingSettings indexing;
+    indexing.Algorithm(IndexingAlgorithmEnum::Auto);
+    indexing.RotationIndexing(true);
+    indexing.GeomRefinementAlgorithm(GeomRefinementAlgorithmEnum::BeamCenter);
+    indexing.IndexIceRings(false);
+    x.ImportIndexingSettings(indexing);
+    x.DetectIceRings(RugnuxDetectIceRings(dataset.file_detect_ice_rings, true));
+
+    ScalingSettings scaling = RugnuxDefaultScalingSettings(true);
+    scaling.ScaleFulls(true);
+    scaling.RelativeBDegrees(0.0);
+    scaling.MergeFriedel(true);
+    scaling.MinPartiality(0.02);
+    scaling.MinCCForImage(0.0);
+    x.ImportScalingSettings(scaling);
+
+    ProcessConfig &c = s.config;
+    c.mode = ProcessMode::FullAnalysis;
+    c.nthreads = nthreads;
+    // As a bare `rugnux` with no -N: the per-image loop keeps to what a card can take (ImagePass).
+    c.nthreads_auto = true;
+    c.export_unmerged = false;
+    c.spot_finding = RugnuxDefaultSpotFinding(true);
+    c.rotation_indexing = true;
+    c.two_pass_rotation = true;
+    c.rotation_postrefine_geometry = true;
+    c.rotation_indexing_image_count = RUGNUX_ROTATION_INDEXING_IMAGES;
+    c.detect_beam_stop = RUGNUX_DETECT_BEAM_STOP_FRAMES;
+    c.beam_center_check = true;
+    c.measure_spot_budget = true;
+    c.run_scaling = true;
+    c.model = std::move(model);
+    c.live = true;
+    c.fast_live = true;
+    // Decided late, on the whole sweep, as it re-checks its radius there - which fast_live does not.
+    c.adaptive_integration_radius = false;
+    return s;
+}

rugnux/RugnuxLive.h

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <string>
+
+#include "Rugnux.h"
+#include "../reader/JFJochReaderDataset.h"
+
+// A rotation collection processed inside the broker while its frames arrive: what a bare `rugnux` on
+// the same file would set, with the run made fast_live. What it makes is returned in memory
+// (ProcessResult::outputs), for the broker to build the END message from, and the model, where one is
+// given, is placed and validated as `rugnux --model` does.
+struct RugnuxLiveSetup {
+    DiffractionExperiment experiment;
+    ProcessConfig config;
+};
+
+RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads,
+                                    std::shared_ptr<const RugnuxModel> model);

rugnux/RugnuxOutputFiles.cpp

@@ -0,0 +1,176 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "RugnuxOutputFiles.h"
+
+#include <filesystem>
+#include <fstream>
+
+#include <gemmi/ccp4.hpp>
+
+#include "../writer/FileWriter.h"
+#include "../writer/WriteModel.h"
+#include "../writer/WriteRugnuxFiles.h"
+
+namespace {
+
+void WriteCcp4(const gemmi::Grid<float> &map, const std::string &path) {
+    gemmi::Ccp4<float> ccp4;
+    ccp4.grid = map;
+    ccp4.update_ccp4_header(2);
+    ccp4.write_ccp4_map(path);
+}
+
+// The files of one output pass at `prefix`, all but its _process.h5.
+void WriteOutputs(const RugnuxOutputs &o, const std::string &prefix, const OutputProvenance &provenance,
+                  size_t nthreads, Logger &logger) {
+    if (o.merged)
+        WriteMergedOutput(*o.merged, prefix, provenance, nthreads, /*hkl=*/true);
+    if (o.twin)
+        WriteMergedOutput(*o.twin, prefix, provenance, nthreads, /*hkl=*/false);
+    if (o.model_prior)
+        WriteMergedOutput(*o.model_prior, prefix, provenance, nthreads, /*hkl=*/false);
+    if (o.maps)
+        WriteModelMaps(*o.maps, prefix, logger);
+    if (o.model_prior_maps)
+        WriteModelMaps(*o.model_prior_maps, prefix + "_mr", logger);
+    if (o.placed_model)
+        WritePlacedModel(*o.placed_model->model, o.placed_model->cell, o.placed_model->space_group, prefix, logger);
+    if (o.model_prior_placed_model)
+        WritePlacedModel(*o.model_prior_placed_model->model, o.model_prior_placed_model->cell,
+                         o.model_prior_placed_model->space_group, prefix + "_mr", logger);
+    if (o.unmerged) {
+        WriteMtz(*o.unmerged, prefix + "_unmerged.mtz");
+        logger.Info("Unmerged observations written to {}_unmerged.mtz", prefix);
+    }
+    if (o.unmerged_partials) {
+        WriteMtz(*o.unmerged_partials, prefix + "_unmerged_partials.mtz");
+        logger.Info("Unmerged partials written to {}_unmerged_partials.mtz", prefix);
+    }
+    if (!o.detector_jpeg.empty()) {
+        const std::string path = prefix + "_detector.jpg";
+        std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
+        if (f) {
+            f.write(o.detector_jpeg.data(), static_cast<std::streamsize>(o.detector_jpeg.size()));
+            logger.Info("Detector diagnostic image written to {}", path);
+        } else
+            logger.Warning("Cannot open {} for writing", path);
+    }
+    if (o.plot) {
+        WritePerImagePlot(prefix, *o.plot);
+        logger.Info("Per-image table written to {}_plot.txt", prefix);
+    }
+}
+
+}  // namespace
+
+ProcessFileSink::ProcessFileSink(std::string prefix) : prefix_(std::move(prefix)) {}
+
+ProcessFileSink::~ProcessFileSink() = default;
+
+void ProcessFileSink::Start(const StartMessage &start, const std::string &label) {
+    writer_.reset();
+    StartMessage s = start;
+    s.file_prefix = prefix_ + label;
+    writer_ = std::make_unique<FileWriter>(s, /*check_overwrite_at_start=*/true, /*trusted_path=*/true);
+}
+
+void ProcessFileSink::Write(const DataMessage &msg) {
+    writer_->Write(msg);
+}
+
+void ProcessFileSink::End(const EndMessage &end) {
+    writer_->WriteHDF5(end);
+    writer_->Finalize();
+    writer_.reset();
+}
+
+void WriteRugnuxOutputs(const ProcessResult &result, const std::string &prefix, const OutputProvenance &provenance,
+                        size_t nthreads, Logger &logger) {
+    if (result.outputs) {
+        WriteOutputs(*result.outputs, prefix, provenance, nthreads, logger);
+        if (result.outputs->process_start) {
+            StartMessage start = *result.outputs->process_start;
+            start.file_prefix = prefix;
+            FileWriter writer(start, /*check_overwrite_at_start=*/true, /*trusted_path=*/true);
+            writer.WriteHDF5(result.end_message);
+            writer.Finalize();
+        }
+    }
+    if (result.doubled_cell) {
+        const ProcessResult::DoubledCell &h = *result.doubled_cell;
+        if (h.other_outputs)
+            WriteOutputs(*h.other_outputs, prefix + h.other_suffix, provenance, nthreads, logger);
+        // The doubled cell's pass put its per-image analysis under a name of its own; where it is the answer,
+        // the two _process.h5 files trade names, as the other files of the two passes are named.
+        std::error_code ec;
+        if (h.adopted && std::filesystem::exists(prefix + "_doubled_process.h5", ec)) {
+            if (std::filesystem::exists(prefix + "_process.h5", ec))
+                std::filesystem::rename(prefix + "_process.h5", prefix + h.other_suffix + "_process.h5", ec);
+            std::filesystem::rename(prefix + "_doubled_process.h5", prefix + "_process.h5", ec);
+        }
+    }
+}
+
+void WriteMergedOutput(const MergedOutput &merged, const std::string &prefix, const OutputProvenance &provenance,
+                       size_t nthreads, bool hkl) {
+    // An MTZ, an mmCIF and a SHELX HKLF-4 .hkl - each has its uses downstream (MTZ for the CCP4 / phenix
+    // reflection tools, mmCIF for deposition and as the self-describing native format, HKLF-4 for SHELXL
+    // refinement and SHELXC / ANODE substructure solution). The .hkl holds the unmerged scaled fulls where the
+    // merge provides them (rotation), the merged reflections otherwise (stills).
+    const std::string file = prefix + merged.suffix;
+    WriteMtz(merged.mtz, file + ".mtz");
+    WriteMmcifReflections(merged.reflections, merged.cell, merged.experiment, merged.statistics, merged.error_model,
+                          merged.twinning, provenance, file + ".cif", nthreads);
+    if (!hkl)
+        return;
+    if (!merged.scaled_fulls.empty())
+        WriteShelxHklReflections(merged.scaled_fulls, file + ".hkl", nthreads);
+    else
+        WriteShelxHklReflections(merged.reflections, merged.experiment, file + ".hkl", nthreads);
+}
+
+void WriteModelMaps(const ModelMaps &maps, const std::string &prefix, Logger &logger) {
+    WriteMtz(maps.coefficients, prefix + "_maps.mtz");
+    if (maps.map_2fofc)
+        WriteCcp4(*maps.map_2fofc, prefix + "_2fofc.ccp4");
+    if (maps.map_fofc)
+        WriteCcp4(*maps.map_fofc, prefix + "_fofc.ccp4");
+    if (maps.map_anom)
+        WriteCcp4(*maps.map_anom, prefix + "_anom.ccp4");
+    logger.Info("Model validation: wrote {}_maps.mtz{}", prefix,
+                maps.map_2fofc ? fmt::format(", {0}_2fofc.ccp4, {0}_fofc.ccp4{1}", prefix,
+                                             maps.map_anom ? ", " + prefix + "_anom.ccp4" : std::string())
+                               : std::string());
+}
+
+void WritePerImagePlot(const std::string &prefix, const PerImagePlot &plot) {
+    const std::string path = prefix + "_plot.txt";
+    std::ofstream file(path, std::ofstream::out | std::ofstream::trunc);
+    if (!file)
+        throw JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot open " + path + " for writing");
+    file << PerImagePlotText(plot);
+}
+
+ReportDocument WriteResultReport(const std::string &output_prefix,
+                                 const std::string &input_file,
+                                 const DiffractionExperiment &experiment,
+                                 const ProcessResult &result,
+                                 Logger &logger,
+                                 const RunProvenance &provenance) {
+    ReportDocument doc = BuildReportDocument(output_prefix, input_file, experiment, result, provenance);
+    if (output_prefix.empty())
+        return doc;   // "compute the statistics, persist nothing"
+
+    const std::string filename = output_prefix + "_report.txt";
+    // The report is unconditional, so it must never be the reason a run fails: a run that produced a
+    // good .mtz must survive an unwritable path or a full disk. Report the failure and carry on.
+    try {
+        std::ofstream file(filename);
+        file.exceptions(std::ios::failbit | std::ios::badbit);
+        file << RenderReportText(doc, provenance.developer);
+    } catch (const std::exception &e) {
+        logger.Warning("Could not write the results report {}: {}", filename, e.what());
+    }
+    return doc;
+}

rugnux/RugnuxOutputFiles.h

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "Rugnux.h"
+#include "ResultReport.h"
+
+class FileWriter;
+
+// The files of a run, written by the program that ran it - rugnux and the viewer - from what the analysis
+// returned: the library itself writes none.
+
+// <prefix><label>_process.h5: the per-image analysis of each output pass, as it is made
+// (ProcessConfig::image_sink).
+class ProcessFileSink : public RugnuxImageSink {
+    std::string prefix_;
+    std::unique_ptr<FileWriter> writer_;
+public:
+    explicit ProcessFileSink(std::string prefix);
+    ~ProcessFileSink() override;
+    void Start(const StartMessage &start, const std::string &label) override;
+    void Write(const DataMessage &msg) override;
+    void End(const EndMessage &end) override;
+};
+
+// Every file `result` holds, under `prefix` with the names rugnux gives them: the reflection files (.mtz,
+// .cif, .hkl, the twin hypothesis' and the model's choice's), the unmerged observations, the maps and the
+// placed models, _detector.jpg, _plot.txt, and the _process.h5 of a run that indexed nothing; the other
+// cell's beside them where the doubled cell was merged too. `provenance` and `nthreads` are what the mmCIF
+// is written with.
+void WriteRugnuxOutputs(const ProcessResult &result, const std::string &prefix, const OutputProvenance &provenance,
+                        size_t nthreads, Logger &logger);
+
+// The reflection files of one merge at <prefix><suffix>: .mtz and .cif, and the .hkl where `hkl`.
+void WriteMergedOutput(const MergedOutput &merged, const std::string &prefix, const OutputProvenance &provenance,
+                       size_t nthreads, bool hkl);
+
+// A model validation's maps at `prefix`: _maps.mtz, and _2fofc.ccp4, _fofc.ccp4 and _anom.ccp4 for the maps
+// it kept (ModelValidationSchedule::keep_map_grids).
+void WriteModelMaps(const ModelMaps &maps, const std::string &prefix, Logger &logger);
+
+// <prefix>_plot.txt (PerImagePlotText).
+void WritePerImagePlot(const std::string &prefix, const PerImagePlot &plot);
+
+// Renders and writes <output_prefix>_report.txt, and returns the document it rendered (the CLI's
+// closing summary reads it). Writes nothing when the prefix is empty (the "compute statistics,
+// persist nothing" mode). Never throws: a run that produced good reflections must not be lost
+// because a side file could not be written, so a failure is logged and swallowed.
+ReportDocument WriteResultReport(const std::string &output_prefix,
+                                 const std::string &input_file,
+                                 const DiffractionExperiment &experiment,
+                                 const ProcessResult &result,
+                                 Logger &logger,
+                                 const RunProvenance &provenance = {});

rugnux/RugnuxPasses.cpp

@@ -4,7 +4,6 @@
 #include <cstdlib>
 #include "Rugnux.h"
 #include "../image_analysis/structure_refinement/ModelValidation.h"
-#include "../writer/WriteModel.h"
 #include "../image_analysis/indexing/SpindleCuspLoss.h"
 #include "../image_analysis/bragg_integration/SpotWidth.h"
 #include "../image_analysis/bragg_integration/SpotFootprint.h"
@@ -20,8 +19,6 @@
 #include <cmath>
 #include <condition_variable>
 #include <deque>
-#include <filesystem>
-#include <fstream>
 #include <cstring>
 #include <functional>
 #include <future>
@@ -41,7 +38,6 @@
 #include "../common/CUDAWrapper.h"
 #include "../common/JFJochException.h"
 #include "../common/time_utc.h"
-#include "../writer/FileWriter.h"
 #include "../image_analysis/MXAnalysisWithoutFPGA.h"
 #include "../image_analysis/beam_stop/ShadowFinder.h"
 #include "../image_analysis/IndexAndRefine.h"
@@ -353,9 +349,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
     // Rotation two-pass geometry post-refinement: the first pass integrates at the header geometry and
     // post-refines the detector geometry (distance + beam from the observed spot positions, cell scale + axis
     // from phi_obs); the second pass re-indexes and re-integrates with the refined geometry. Only the second
-    // (refined-geometry) pass writes merged files, under the plain "<prefix>_*" name. The first pass still runs
-    // under a "<prefix>_01" prefix so that the _process.h5 it writes, when one is asked for, does not collide
-    // with the second pass's - the ice-ring flags and geometry stored in each are the ones that pass used.
+    // (refined-geometry) pass makes outputs; the first makes none, nor a _process.h5.
     if (config_.mode == ProcessMode::FullAnalysis && config_.rotation_postrefine_geometry
         && experiment_.IsRotationIndexing()) {
         Logger logger("Rugnux");
@@ -363,7 +357,6 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
         // post none.
         const auto step = [observer](const std::string &title) { if (observer) observer->OnStep(title); };
         const auto note = [observer](const std::string &text) { if (observer) observer->OnNote(text); };
-        const std::string base_prefix = config_.output_prefix;
         const auto gonio_snapshot = experiment_.GetGoniometer();
         // What the first pass starts from, so a second first pass at a different beam centre can start
         // from the same place (the beam-centre arbitration below).
@@ -378,15 +371,13 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
         beam_center_same_lattice_alternative_.reset();
         lattice_arm_check_.reset();
 
-        logger.Info("Rotation two-pass geometry post-refinement: first pass (header geometry) -> {}_01_*",
-                    base_prefix);
-        // Keep an empty prefix empty (the "compute stats, persist nothing" mode): the "_01" suffix would
-        // make it non-empty and RunPipeline would then write stray _01_* files despite no output wanted.
-        config_.output_prefix = base_prefix.empty() ? base_prefix : base_prefix + "_01";
+        logger.Info("Rotation two-pass geometry post-refinement: first pass (header geometry)");
         auto pass1 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/true);
         pass1.pass_number = 1;
         pass1.pass_count = 2;
-        if (cancelled_) { config_.output_prefix = base_prefix; return pass1; }
+        if (cancelled_) return pass1;
+        if (config_.fast_live)
+            established_ = pass1;
 
         // Beam centre as a LATTICE hypothesis, decided on the merge. The first pass indexed both
         // centres, found the same cell with a different metric symmetry read off it, and measured that
@@ -597,7 +588,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
             beam_center_alternative_.reset();
             beam_center_alternative_state_.reset();
             beam_center_same_lattice_alternative_.reset();
-            if (cancelled_) { config_.output_prefix = base_prefix; return pass1; }
+            if (cancelled_) return pass1;
         }
 
         // Undo the pre-pass goniometer shift (a sub-range or strided run moves start/increment inside
@@ -630,7 +621,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
             experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2])
                        .PoniRot1_rad(g[3]).PoniRot2_rad(g[4]);
         }
-        RefinedPassStart start{base_prefix, gonio_snapshot, header_geometry, {}};
+        RefinedPassStart start{gonio_snapshot, header_geometry, {}};
 
         // The goniometer rotation scale: the angles in the file are the commanded ones, so a stage that
         // turned at the wrong rate is a geometry error like any other - but one pass 1's fit reads only
@@ -658,7 +649,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
         // pass off and throws it away, so the walk runs first, on its own, and the refined passes
         // after it. Either way the run ends where the sequential walk leaves it.
         constexpr double ROTATION_SCALE_STEP = 0.01;
-        if (!cancelled_ && gonio_snapshot && pass1.post_refine && !config_.rotation_scale) {
+        if (!cancelled_ && !config_.fast_live && gonio_snapshot && pass1.post_refine && !config_.rotation_scale) {
             const auto check = std::make_shared<RotationScaleCheck>();
             RotationScaleWalk walk;
             if (std::fabs(pass1.post_refine->rotation_scale - 1.0) >= ROTATION_SCALE_STEP) {
@@ -687,7 +678,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
                 if (!check->overturns) {
                     if (output_error)
                         std::rethrow_exception(output_error);
-                    return output->DoubledCellHypothesis(observer, std::move(*result), base_prefix);
+                    return output->DoubledCellHypothesis(observer, std::move(*result));
                 }
             }
 
@@ -706,7 +697,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
                                walk.scale > 1.0f ? "further" : "less far");
             }
         }
-        return DoubledCellHypothesis(observer, *RefinedPasses(observer, std::move(pass1), start), base_prefix);
+        return DoubledCellHypothesis(observer, *RefinedPasses(observer, std::move(pass1), start));
     }
     return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
 }
@@ -880,7 +871,6 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
     Logger logger("Rugnux");
     const auto step = [observer](const std::string &title) { if (observer) observer->OnStep(title); };
     const auto note = [observer](const std::string &text) { if (observer) observer->OnNote(text); };
-    const std::string &base_prefix = start.base_prefix;
     const auto &gonio_snapshot = start.gonio_snapshot;
     const auto &header_geometry = start.header_geometry;
     RotationScaleOutcome scale_walk = start.scale_walk;
@@ -940,8 +930,8 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
 
     // Let every refined pass from here on ask the quality guard's question of itself, so one that
     // is going to be thrown away can skip the report and the files it would have superseded.
-    quality_guard_pass1_ = pass1;
-    config_.output_prefix = base_prefix;   // the refined pass is the canonical result (no _02 suffix)
+    if (!config_.fast_live)
+        quality_guard_pass1_ = pass1;
     // The canonical pass measures the post-refinement again where the pre-pass moved the geometry
     // (it applies nothing), so that the walk below has something to decide on.
     postrefine_probe_ = prepass_detector_geometry_.has_value();
@@ -997,7 +987,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
     double split = 0.0;
     std::optional<ProcessResult> demoted_arm;   // run where the data resolve the equality
     std::shared_ptr<Rugnux> demoted_recheck;    // the demoted arm again on all the frames, for a near tie
-    if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) {
+    if (!cancelled_ && !config_.fast_live && (length_arm || angle_arm) && pass1.rotation_lattice_type) {
         split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0;
         const double dx = std::max(experiment_.GetBeamX_pxl(),
                                    static_cast<float>(experiment_.GetXPixelsNum()) - experiment_.GetBeamX_pxl());
@@ -1099,10 +1089,9 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
         }
     }
 
-    logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical) -> {}_*",
-                base_prefix);
-    speculate_geometry_probe_ = true;   // a walk that follows starts with an indexing probe
-    rerun_if_starved_ = true;
+    logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical)");
+    speculate_geometry_probe_ = !config_.fast_live;   // a walk that follows starts with an indexing probe
+    rerun_if_starved_ = !config_.fast_live;
     const DiffractionExperiment before_pass2 = experiment_;
     auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
     if (radius_recheck_) {
@@ -1156,6 +1145,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
             demoted_settings.LatticeClassDemoted(length_arm);
             experiment_.ImportIndexingSettings(demoted_settings);
             lattice_angles_demoted_ = angle_arm;
+            pass2.outputs.reset();   // replaced: not held through the pass run again
             pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
         } else if (!cancelled_) {
             note(fmt::format("the metric symmetry stands: {}", promoted_class));
@@ -1296,7 +1286,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
     // from here, not from where its own probe left the run.
     DiffractionExperiment last_round_start = experiment_;
     bool walk_shown = false;
-    while (!cancelled_ && prepass_detector_geometry_
+    while (!cancelled_ && !config_.fast_live && prepass_detector_geometry_
            && pass2.post_refine && pass2.post_refine->detector_refined) {
         if (!walk_shown) {
             step("Geometry check");
@@ -1339,6 +1329,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
         // hand before the merge, so the files are written once, below, at the round the walk keeps.
         speculate_geometry_probe_ = !large_walk;
         postrefine_probe_only_ = true;
+        pass2.outputs.reset();   // replaced: not held through the round
         pass2 = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false);
         postrefine_probe_only_ = false;
         speculate_geometry_probe_ = false;
@@ -1437,7 +1428,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
     // RunPipeline) - so each is followed by it. After the first re-run the radius is the fixed one
     // and this never fires again.
     const auto rerun_if_starved = [&](ProcessResult &pass, const std::string decision) {
-        if (cancelled_ || !bragg_adaptive_ || !bkg_starved_fraction_
+        if (cancelled_ || config_.fast_live || !bragg_adaptive_ || !bkg_starved_fraction_
             || *bkg_starved_fraction_ <= spot_width::BKG_STARVED_MAX_FRACTION)
             return;
         logger.Info("Two-pass: at the widened integration radius r1={:.1f} the neighbouring "
@@ -1454,6 +1445,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
         bragg_adaptive_.reset();
         note(fmt::format("{:.1f}% of the background rings starved: the pass runs again at r1 {:.1f} px",
                          100.0 * starved, bragg_before_adaptive_->GetR1()));
+        pass.outputs.reset();   // replaced: not held through the pass run again
         pass = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
         pass.post_refine = pass1.post_refine;
         pass.pass_number = passes;
@@ -1487,7 +1479,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
         };
         const double v1 = primitive_volume(*pass1.consensus_cell, pass1.consensus_centering);
         const double v2 = primitive_volume(*pass2.consensus_cell, pass2.consensus_centering);
-        if (v1 > 0.0 && v2 > 1.5 * v1 && !supercell_settled_) {
+        if (v1 > 0.0 && v2 > 1.5 * v1 && !supercell_settled_ && !config_.fast_live) {
             logger.Info("Two-pass: second-pass primitive cell volume {:.0f} A^3 is {:.2f}x pass-1's "
                         "({:.0f} A^3) - a spurious supercell; re-running the second pass with "
                         "pass-1's result forced",
@@ -1501,6 +1493,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
                 force_rotation_result_->axis =
                         ScaleRotation(*force_rotation_result_->axis, *prepass_rotation_scale_);
             rerun_if_starved_ = true;
+            pass2.outputs.reset();   // replaced: not held through the pass run again
             pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
             rerun_if_starved_ = false;
             pass2.post_refine = pass1.post_refine;
@@ -1519,7 +1512,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
     // but it is not guaranteed to be, and the guard that decides is RefinedPassIsWorse above. It
     // has already been asked once, inside the pass itself, which is how a pass that is about to be
     // thrown away knows not to write a report; this is where the answer is acted on.
-    if (!cancelled_ && pass1.has_merge_statistics && pass2.has_merge_statistics) {
+    if (!cancelled_ && !config_.fast_live && pass1.has_merge_statistics && pass2.has_merge_statistics) {
         logger.Info("Two-pass quality guard: the refined pass holds {} reflections at I/sigma >= 2 and "
                     "{} at <= -2 of {} in a {:.0f} A^3 primitive cell, the header-geometry pass {} and {} "
                     "of {} in {:.0f} A^3; low-order axial {} against {}",
@@ -1550,7 +1543,6 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
                 prepass_rotation_scale_.reset();
                 pass1.post_refine->rotation_scale_suspect = false;
             }
-            config_.output_prefix = base_prefix;
             // This pass is the answer whatever it measures - the guard has had its one chance -
             // so it must not conclude from its own search merge that it is going to be re-run.
             quality_guard_pass1_.reset();
@@ -1563,6 +1555,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
                 force_rotation_result_ = *prepass_result_;
             rerun_if_starved_ = true;
             note("the refined geometry is worse: the pass runs again at the header geometry");
+            pass2.outputs.reset();   // redo is the answer: pass 2's outputs are not held through it
             auto redo = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
             rerun_if_starved_ = false;
             redo.post_refine = pass1.post_refine;
@@ -1798,9 +1791,8 @@ namespace {
     }
 }
 
-ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half,
-                                            const std::string &base_prefix) {
-    if (cancelled_ || user_cell_ || user_fixed_sg_ || !half.supercell || !last_rotation_result_
+ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half) {
+    if (cancelled_ || config_.fast_live || user_cell_ || user_fixed_sg_ || !half.supercell || !last_rotation_result_
         || !half.has_merge_statistics || !half.consensus_cell || !DoubledCellWorthMerging(*half.supercell))
         return half;
     Logger logger("Rugnux");
@@ -1836,9 +1828,7 @@ ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessRes
     if (observer)
         observer->OnStep("Doubled-cell hypothesis");
 
-    const std::string half_prefix = base_prefix + "_halfcell";
-    const std::string doubled_prefix = base_prefix + "_doubled";
-    config_.output_prefix = base_prefix.empty() ? base_prefix : doubled_prefix;
+    output_label_ = "_doubled";
     force_rotation_result_ = doubled;
     keep_merged_reflections_ = true;
 
@@ -1854,7 +1844,7 @@ ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessRes
     }
     force_rotation_result_.reset();
     keep_merged_reflections_ = false;
-    config_.output_prefix = base_prefix;
+    output_label_.clear();
     if (!d.has_merge_statistics || !d.consensus_cell) {
         logger.Warning("Doubled-cell hypothesis: the doubled cell merged nothing - the probed cell stays the answer");
         return half;
@@ -1908,28 +1898,12 @@ ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessRes
                           weakest->d_high_A);
     logger.Info("Doubled-cell hypothesis: {}", h.decision);
 
-    // The answer's files go under the plain prefix and the other cell's beside it: where the doubled cell
-    // is adopted, every file it wrote takes the plain name, and the probed cell's file of the same name
-    // moves to <prefix>_halfcell.
-    if (adopt && !base_prefix.empty()) {
-        const std::filesystem::path dp(doubled_prefix);
-        const std::filesystem::path dir = dp.has_parent_path() ? dp.parent_path() : std::filesystem::path(".");
-        const std::string stem = dp.filename().string();
-        std::error_code ec;
-        std::vector<std::string> suffixes;
-        for (const auto &f : std::filesystem::directory_iterator(dir, ec))
-            if (const std::string name = f.path().filename().string(); name.starts_with(stem))
-                suffixes.push_back(name.substr(stem.size()));
-        for (const auto &x : suffixes) {
-            if (std::filesystem::exists(base_prefix + x, ec))
-                std::filesystem::rename(base_prefix + x, half_prefix + x, ec);
-            std::filesystem::rename(doubled_prefix + x, base_prefix + x, ec);
-        }
-    }
-
+    // The answer's outputs are the ones under the plain prefix and the other cell's beside them, at
+    // <prefix>_halfcell where the doubled cell is adopted and at <prefix>_doubled where it is not.
     ProcessResult &answer = adopt ? d : half;
     const ProcessResult &other = adopt ? half : d;
-    h.other_prefix = adopt ? half_prefix : doubled_prefix;
+    h.other_suffix = adopt ? "_halfcell" : "_doubled";
+    h.other_outputs = other.outputs;
     h.other_cell = *other.consensus_cell;
     h.other_space_group = other.space_group ? other.space_group->xhm() : std::string();
     h.other_d_min_A = other.merge_statistics.overall.d_min;