From 585cc777d522851ca645faf5d4e83d70e97a0d31 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 18:09:39 +0100 Subject: [PATCH 01/44] GPU analysis of converted JUNGFRAU frames with a per-run pre-pass With rugnux enabled, a serial (non-rotation) collection in the FPGA workflow is analysed as the DECTRIS frames are: the frame transformation threads compress each converted frame and hand it to StillsGPUAnalysis, whose threads run MXAnalysisWithoutFPGA (GPU spot finding, indexing, integration) and pass the result back to the receiver to be written. Before any frame is analysed, the first 100 are run through rugnux as `rugnux --no-merge` on them (RunStillsPrePass, in memory): its beam centre (the direct beam, from the scattered background and, where frames index, the stills geometry trial or the bundle refinement with a given cell) and its mask (plus the beam-stop shadow and the defective pixels) are what every frame of the collection is then analysed with. Frames arriving meanwhile are held. The start message waits for the pre-pass, so it states that centre and mask; the END message carries the centre as refined_beam_center_x/y. The start message dataset now carries the detector modules, unit cell and space group, which the pre-pass needs. tools/jfjoch_stills_gpu_replay replays a stored serial collection through the same analysis (pre-pass, indexed fraction, sustained rate). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- reader/JFJochStartMessageReader.cpp | 6 + receiver/CMakeLists.txt | 2 + receiver/JFJochReceiver.cpp | 37 +++--- receiver/JFJochReceiver.h | 4 + receiver/JFJochReceiverFPGA.cpp | 113 ++++++++++++++++++- receiver/JFJochReceiverFPGA.h | 11 +- receiver/JFJochReceiverService.cpp | 5 +- receiver/StillsGPUAnalysis.cpp | 137 +++++++++++++++++++++++ receiver/StillsGPUAnalysis.h | 85 ++++++++++++++ receiver/StillsPrePass.cpp | 64 +++++++++++ receiver/StillsPrePass.h | 32 ++++++ rugnux/Rugnux.h | 3 + rugnux/RugnuxLive.cpp | 39 +++++++ rugnux/RugnuxLive.h | 5 + tests/JFJochReceiverProcessingTest.cpp | 81 ++++++++++++++ tools/CMakeLists.txt | 4 + tools/jfjoch_stills_gpu_replay.cpp | 149 +++++++++++++++++++++++++ 17 files changed, 755 insertions(+), 22 deletions(-) create mode 100644 receiver/StillsGPUAnalysis.cpp create mode 100644 receiver/StillsGPUAnalysis.h create mode 100644 receiver/StillsPrePass.cpp create mode 100644 receiver/StillsPrePass.h create mode 100644 tools/jfjoch_stills_gpu_replay.cpp diff --git a/reader/JFJochStartMessageReader.cpp b/reader/JFJochStartMessageReader.cpp index f0c5fc14a..6ff183b7d 100644 --- a/reader/JFJochStartMessageReader.cpp +++ b/reader/JFJochStartMessageReader.cpp @@ -55,6 +55,12 @@ std::shared_ptr DatasetFromStartMessage(const StartMessage detector.MinCountTime(std::chrono::microseconds(0)); detector.ReadOutTime(std::chrono::nanoseconds(0)); x.Detector(detector); + // The modules one by one, where the start message states them (a JUNGFRAU's, as imaged). + if (!start.detector_modules.empty()) + x.Modules(DetectorModules(start.detector_modules)); + x.SetUnitCell(start.unit_cell); + if (start.space_group_number) + x.SpaceGroupNumber(static_cast(*start.space_group_number)); x.FrameTime(std::chrono::duration_cast(std::chrono::duration(start.frame_time)), std::chrono::duration_cast(std::chrono::duration(start.count_time))); diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index 4eaf0898a..9096e604c 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -12,6 +12,8 @@ ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverOutput.h JFJochReceiverLite.h JFJochReceiverLite.cpp JFJochReceiverRugnux.h JFJochReceiverRugnux.cpp + StillsPrePass.h StillsPrePass.cpp + StillsGPUAnalysis.h StillsGPUAnalysis.cpp RugnuxLiveStatus.h RugnuxLiveStatus.cpp RugnuxStructure.h RugnuxStructure.cpp JFJochReceiver.cpp diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 8ecc77425..9ffa7badc 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -107,28 +107,33 @@ JFJochReceiverStatus JFJochReceiver::GetStatus() const { } -void JFJochReceiver::SendStartMessage() { +StartMessage JFJochReceiver::MakeStartMessage(const DiffractionExperiment &x, const PixelMask &mask, + const AzimuthalIntegrationMapping &mapping) { StartMessage message{}; - experiment.FillMessage(message); + x.FillMessage(message); message.arm_date = time_UTC(std::chrono::system_clock::now()); - message.az_int_q_bin_count = az_int_mapping->GetQBinCount(); - message.az_int_bin_to_q = az_int_mapping->GetBinToQ(); - message.az_int_bin_to_two_theta = az_int_mapping->GetBinToTwoTheta(); - message.az_int_phi_bin_count = az_int_mapping->GetAzimuthalBinCount(); - if (az_int_mapping->GetAzimuthalBinCount() > 1) { - message.az_int_bin_to_phi = az_int_mapping->GetBinToPhi(); - message.az_int_map = az_int_mapping->GetPixelToBin(); + message.az_int_q_bin_count = mapping.GetQBinCount(); + message.az_int_bin_to_q = mapping.GetBinToQ(); + message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta(); + message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount(); + if (mapping.GetAzimuthalBinCount() > 1) { + message.az_int_bin_to_phi = mapping.GetBinToPhi(); + message.az_int_map = mapping.GetPixelToBin(); } message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress(); - message.rois = experiment.ROI().ExportMetadata(); - if (!experiment.ROI().empty()) - message.roi_map = experiment.ExportROIMap(); - message.max_spot_count = experiment.GetMaxSpotCount(); + message.rois = x.ROI().ExportMetadata(); + if (!x.ROI().empty()) + message.roi_map = x.ExportROIMap(); + message.max_spot_count = x.GetMaxSpotCount(); + message.pixel_mask["default"] = mask.GetMask(x); + return message; +} +void JFJochReceiver::SendStartMessage() { + SendStartMessage(MakeStartMessage(experiment, pixel_mask, *az_int_mapping)); +} - std::vector nexus_mask; - message.pixel_mask["default"] = pixel_mask.GetMask(experiment); - +void JFJochReceiver::SendStartMessage(StartMessage message) { SaveStartMessageToImageBuffer(message); if (push_images_to_writer) diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 93fb213ae..f0f76db20 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -93,7 +93,11 @@ protected: void UpdateMaxDelay(uint64_t delay); JFJochReceiverStatus GetStatus() const; + // The start message of this collection, stating that experiment, mask and azimuthal mapping. + StartMessage MakeStartMessage(const DiffractionExperiment &x, const PixelMask &mask, + const AzimuthalIntegrationMapping &mapping); void SendStartMessage(); + void SendStartMessage(StartMessage message); void SendEndMessage(); void SaveStartMessageToImageBuffer(const StartMessage &msg); // The last word on the end message before it is sent: a receiver whose analysis is not the diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index d22f64de4..4f8969587 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -6,6 +6,9 @@ #include #include "ImageMetadata.h" +#include "JFJochReceiverLite.h" +#include "../common/CUDAWrapper.h" +#include "../image_analysis/MXAnalysisWithoutFPGA.h" JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_pixel_mask, @@ -19,7 +22,8 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen ImageBuffer &in_send_buf_ctrl, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, - IndexerThreadPool *indexing_thread_pool) + IndexerThreadPool *indexing_thread_pool, + bool in_gpu_analysis) : JFJochReceiver(in_experiment, in_send_buf_ctrl, in_image_sender, @@ -39,7 +43,9 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4), summation_nthreads(2), frame_transformation_ready((experiment.GetImageNum() > 0) ? frame_transformation_nthreads : 0), - data_acquisition_ready(ndatastreams) { + data_acquisition_ready(ndatastreams), + gpu_analysis(in_gpu_analysis && experiment.GetImageNum() > 0 + && experiment.GetDetectorMode() == DetectorMode::Standard) { // The FPGA integration core has a fixed bin-count limit. Reject configurations it cannot // handle before starting acquisition, unless azimuthal integration is forced onto the CPU. const auto &azint_settings = experiment.GetAzimuthalIntegrationSettings(); @@ -118,8 +124,21 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen } logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads); } else if (experiment.GetImageNum() > 0) { - SendStartMessage(); - SendCalibration(); + if (gpu_analysis) { + // The start message and the calibration go out once the pre-pass is done. + gpu = std::make_unique( + experiment, pixel_mask, MakeStartMessage(experiment, pixel_mask, *az_int_mapping), + indexing_thread_pool, std::min(16, frame_transformation_nthreads), logger, + [this](const StillsGPUAnalysis::Setup &s) { + SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping)); + SendCalibration(); + }, + [this](DataMessage &m, const AzimuthalIntegrationProfile &p) { GPUSink(m, p); }, + [this] { return GetSpotFindingSettings(); }); + } else { + SendStartMessage(); + SendCalibration(); + } for (int i = 0; i < experiment.GetImageNum(); i++) images_to_go.Put(i); @@ -420,6 +439,23 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { experiment.GetImageMode(), CompressionAlgorithm::NO_COMPRESSION); + if (gpu_analysis) { + // Compressed here, as it is written, and analysed by GPUAnalysisThread. + auto held = std::make_shared(); + held->image.resize(experiment.GetImageBufferLocationSize()); + held->image.resize(transformation.CompressImage(held->image.data(), held->image.size())); + held->message = std::move(message); + held->message.image = CompressedImage(held->image.data(), held->image.size(), + experiment.GetXPixelsNum(), experiment.GetYPixelsNum(), + experiment.GetImageMode(), + experiment.GetCompressionAlgorithm()); + compressed_size += held->image.size(); + uncompressed_size += experiment.GetModulesNum() * RAW_MODULE_SIZE * experiment.GetByteDepthImage(); + ++images_collected; + gpu->Hold(held); + continue; + } + // No-op unless the CPU backend is forced; fills az_int_profile_image from the assembled image analyzer->RunAzimuthalIntegration(transformation.GetImage(), az_int_profile_image); @@ -598,6 +634,9 @@ void JFJochReceiverFPGA::FinalizeMeasurement() { logger.Debug("All processing threads done"); } + if (gpu) + gpu->Finish(); + current_status.SetProgress(1.0); current_status.SetStatus(GetStatus()); @@ -703,3 +742,69 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) { // Initialize data processing acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping); } + +void JFJochReceiverFPGA::GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile) { + try { + const auto status = image_buffer.GetStatus(); + message.receiver_buf_available = status.available_slots; + message.receiver_buf_in_preparation = status.preparation_slots; + message.receiver_buf_in_sending = status.sending_slots; + if (message.image.GetCompressedSize() > 0) + message.compression_ratio = static_cast(experiment.GetPixelsNum() * experiment.GetByteDepthImage()) + / static_cast(message.image.GetCompressedSize()); + + if (message.image_collection_efficiency == 1.0f) { + saturated_pixels.Add(message.saturated_pixel_count); + error_pixels.Add(message.error_pixel_count); + if (message.roi.contains("beam")) { + roi_beam_npixel.Add(message.roi["beam"].pixels); + roi_beam_sum.Add(message.roi["beam"].sum); + } + } + + scan_result.Add(message); + + if (!serialmx_filter.ApplyFilter(message)) + ++images_skipped; + else if (auto loc = image_buffer.GetImageSlot(); loc == nullptr) + writer_queue_full = true; + else { + auto writer_buffer = (uint8_t *) loc->GetImage(); + CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize()); + if (JFJochReceiverLite::SerializeImageOrDrop(serializer, message, *loc, + experiment.GetImageBufferLocationSize(), logger)) { + loc->SetImageNumber(message.number); + loc->SetImageSize(serializer.GetBufferSize()); + loc->SetIndexed(message.indexing_result.value_or(false)); + loc->ReadyToSend(); + + if (zmq_preview_socket != nullptr) + zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize()); + if (zmq_metadata_socket != nullptr) + zmq_metadata_socket->AddDataMessage(message); + + if (push_images_to_writer) { + if (image_pusher.SendImage(*loc)) + ++images_sent; + } else + loc->release(); + UpdateMaxImageSent(message.number); + } + } + + plots.Add(message, profile); + current_status.SetProgress(GetProgress()); + current_status.SetStatus(GetStatus()); + } catch (const JFJochException &e) { + logger.ErrorException(e); + Cancel(e); + } +} + +void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) { + if (!gpu) + return; + message.unit_cell = gpu->GetConsensusUnitCell(); + message.refined_beam_center_x = gpu->GetBeamX_pxl(); + message.refined_beam_center_y = gpu->GetBeamY_pxl(); +} diff --git a/receiver/JFJochReceiverFPGA.h b/receiver/JFJochReceiverFPGA.h index d4bcdf3d0..4de57d708 100644 --- a/receiver/JFJochReceiverFPGA.h +++ b/receiver/JFJochReceiverFPGA.h @@ -40,6 +40,7 @@ #include "../image_analysis/MXAnalysisAfterFPGA.h" #include "../common/MovingAverage.h" #include "JFJochReceiverOutput.h" +#include "StillsGPUAnalysis.h" class JFJochReceiverFPGA : public JFJochReceiver { const JFCalibration *calibration; @@ -86,6 +87,13 @@ class JFJochReceiverFPGA : public JFJochReceiver { void Cancel(const JFJochException &e) override; void FinalizeMeasurement(); + // Serial data with rugnux on: the converted frames are analysed on the GPU (StillsGPUAnalysis), and + // the start message waits for its pre-pass. + const bool gpu_analysis; + std::unique_ptr gpu; + void GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile); + void AddToEndMessage(EndMessage &message) override; + void SendCalibration(); void SendPedestal(const std::string &prefix, const std::vector &v, int gain, int sc); void RetrievePedestal(std::vector &output) const; @@ -102,7 +110,8 @@ public: ImageBuffer &image_buffer, ZMQPreviewSocket *zmq_preview_socket, ZMQMetadataSocket *zmq_metadata_socket, - IndexerThreadPool *indexing_thread_pool); + IndexerThreadPool *indexing_thread_pool, + bool gpu_analysis = false); ~JFJochReceiverFPGA(); JFJochReceiverFPGA(const JFJochReceiverFPGA &other) = delete; JFJochReceiverFPGA& operator=(const JFJochReceiverFPGA &other) = delete; diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index fce06c914..95bef030b 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -114,7 +114,10 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(), - indexer_thread_pool.get()); + indexer_thread_pool.get(), + // Serial data with rugnux on: the frames are + // analysed on the GPU, after a pre-pass. + rugnux_enable && !RugnuxRotationData(experiment)); break; case DetectorType::DECTRIS: if (puller) diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp new file mode 100644 index 000000000..fba263555 --- /dev/null +++ b/receiver/StillsGPUAnalysis.cpp @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "StillsGPUAnalysis.h" + +#include + +#include "../common/CUDAWrapper.h" +#include "../image_analysis/MXAnalysisWithoutFPGA.h" + +StillsGPUAnalysis::StillsGPUAnalysis(const DiffractionExperiment &in_experiment, const PixelMask &in_mask, + const StartMessage &in_start, IndexerThreadPool *in_indexing_thread_pool, + int64_t nthreads, Logger &in_logger, Ready in_ready, Sink in_sink, + Settings in_settings) + : experiment(in_experiment), mask(in_mask), start(in_start), indexing_thread_pool(in_indexing_thread_pool), + logger(in_logger), ready(std::move(in_ready)), sink(std::move(in_sink)), settings(std::move(in_settings)) { + prepass_future = std::async(std::launch::async, &StillsGPUAnalysis::PrePassThread, this); + for (int64_t i = 0; i < nthreads; i++) + analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysis::AnalysisThread, this)); + logger.Info("GPU analysis: {} threads, the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES); +} + +StillsGPUAnalysis::~StillsGPUAnalysis() { + if (!finished) + Finish(); +} + +void StillsGPUAnalysis::Hold(const std::shared_ptr &image) { + { + std::unique_lock ul(prepass_mutex); + if (!prepass_go) { + if (image->message.number < PREPASS_IMAGES) + prepass_images[image->message.number] = image; + ++images_held; + // Every one of the first images is there - or, where some never come, twice as many are. + prepass_go = (prepass_images.size() == std::min(PREPASS_IMAGES, experiment.GetImageNum())) + || (images_held >= 2 * PREPASS_IMAGES); + } + } + prepass_cv.notify_all(); + queue.PutBlocking(image); +} + +void StillsGPUAnalysis::StartPrePass() { + { + std::unique_lock ul(prepass_mutex); + prepass_go = true; + } + prepass_cv.notify_all(); +} + +void StillsGPUAnalysis::Finish() { + finished = true; + StartPrePass(); + prepass_future.get(); + for (size_t i = 0; i < analysis_futures.size(); i++) + queue.PutBlocking(nullptr); + for (auto &f: analysis_futures) + f.get(); +} + +void StillsGPUAnalysis::PrePassThread() { + std::map> sample; + { + std::unique_lock ul(prepass_mutex); + prepass_cv.wait(ul, [this] { return prepass_go; }); + sample.swap(prepass_images); + } + + const auto masked = [](const PixelMask &m) { + return std::count_if(m.GetMask().begin(), m.GetMask().end(), [](uint32_t v) { return v != 0; }); + }; + setup.experiment = experiment; + setup.mask = mask; + if (!sample.empty()) { + try { + const auto t0 = std::chrono::steady_clock::now(); + const auto r = RunStillsPrePass(start, sample, static_cast(std::thread::hardware_concurrency())); + setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl); + setup.mask = r.mask; + logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, " + "masked pixels {} -> {}", r.images, + std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), + experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), r.beam_x_pxl, r.beam_y_pxl, + masked(mask), masked(r.mask)); + } catch (const std::exception &e) { + logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask - {}", + e.what()); + } + } + sample.clear(); + setup.mapping = std::make_unique(setup.experiment, setup.mask); + indexer = std::make_unique(setup.experiment, indexing_thread_pool, false, true); + + ready(setup); + prepass_done.count_down(); +} + +void StillsGPUAnalysis::AnalysisThread() { + try { + pin_gpu(); + } catch (const JFJochException &e) { + logger.Warning("Error pinning GPU {}", e.what()); + } + + prepass_done.wait(); + + MXAnalysisWithoutFPGA analysis(setup.experiment, *setup.mapping, setup.mask, *indexer, + /*enable_fused_adaptive_gpu=*/true); + + while (auto held = queue.GetBlocking()) { + DataMessage &message = held->message; + AzimuthalIntegrationProfile profile(*setup.mapping); + const auto start_time = std::chrono::steady_clock::now(); + try { + analysis.Analyze(message, profile, settings()); + } catch (const JFJochException &e) { + // The frame is still written, without its analysis. + logger.ErrorException(e); + } + message.processing_time_s = std::chrono::duration(std::chrono::steady_clock::now() + - start_time).count(); + sink(message, profile); + } +} + +std::optional StillsGPUAnalysis::GetConsensusUnitCell() const { + return indexer->GetConsensusUnitCell(); +} + +float StillsGPUAnalysis::GetBeamX_pxl() const { + return setup.experiment.GetBeamX_pxl(); +} + +float StillsGPUAnalysis::GetBeamY_pxl() const { + return setup.experiment.GetBeamY_pxl(); +} diff --git a/receiver/StillsGPUAnalysis.h b/receiver/StillsGPUAnalysis.h new file mode 100644 index 000000000..1f043219e --- /dev/null +++ b/receiver/StillsGPUAnalysis.h @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "StillsPrePass.h" +#include "../common/DiffractionExperiment.h" +#include "../common/AzimuthalIntegrationMapping.h" +#include "../common/AzimuthalIntegrationProfile.h" +#include "../common/Logger.h" +#include "../common/ThreadSafeFIFO.h" +#include "../image_analysis/IndexAndRefine.h" + +// Serial frames analysed on the GPU as the DECTRIS ones are (MXAnalysisWithoutFPGA), after a pre-pass on +// the first PREPASS_IMAGES (RunStillsPrePass) has decided the beam centre and the mask they are analysed +// with. Every frame given before the pre-pass is done is held; `ready` is called once it is, with the +// geometry and mask the frames are then analysed at, before any of them is analysed; `sink` gets each +// frame once analysed, from one of the analysis threads. +class StillsGPUAnalysis { +public: + constexpr static int64_t PREPASS_IMAGES = 100; + + struct Setup { + DiffractionExperiment experiment; + PixelMask mask; + std::unique_ptr mapping; + }; + using Ready = std::function; + using Sink = std::function; + using Settings = std::function; + + // `start`: the collection's start message, as it would be sent without the pre-pass. + StillsGPUAnalysis(const DiffractionExperiment &experiment, const PixelMask &mask, const StartMessage &start, + IndexerThreadPool *indexing_thread_pool, int64_t nthreads, Logger &logger, + Ready ready, Sink sink, Settings settings); + ~StillsGPUAnalysis(); + + void Hold(const std::shared_ptr &image); + // No frame comes after this: the pre-pass runs on what came, if it has not, and every frame is + // analysed before it returns. + void Finish(); + + // After Finish. + [[nodiscard]] std::optional GetConsensusUnitCell() const; + [[nodiscard]] float GetBeamX_pxl() const; + [[nodiscard]] float GetBeamY_pxl() const; + +private: + const DiffractionExperiment &experiment; + const PixelMask &mask; + const StartMessage start; + IndexerThreadPool *indexing_thread_pool; + Logger &logger; + Ready ready; + Sink sink; + Settings settings; + + Setup setup; + std::unique_ptr indexer; + + std::mutex prepass_mutex; + std::condition_variable prepass_cv; + bool prepass_go = false; + int64_t images_held = 0; + std::map> prepass_images; + std::latch prepass_done{1}; + + ThreadSafeFIFO> queue; + std::future prepass_future; + std::vector> analysis_futures; + bool finished = false; + + void StartPrePass(); + void PrePassThread(); + void AnalysisThread(); +}; diff --git a/receiver/StillsPrePass.cpp b/receiver/StillsPrePass.cpp new file mode 100644 index 000000000..3719ac156 --- /dev/null +++ b/receiver/StillsPrePass.cpp @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "StillsPrePass.h" + +#include "../common/JFJochException.h" +#include "../reader/JFJochReader.h" +#include "../reader/JFJochStartMessageReader.h" +#include "../rugnux/RugnuxLive.h" + +namespace { + // The held images as a collection of their own, numbered from 0 as the collection numbers them. + class HeldImageReader : public JFJochReader { + const std::map> &images; + uint64_t number_of_images; + + bool LoadImage_i(std::shared_ptr &dataset, DataMessage &message, + std::vector &, int64_t image_number, bool) override { + const auto it = images.find(image_number); + if (!dataset || it == images.end()) + return false; + message = it->second->message; + return true; + } + public: + HeldImageReader(const StartMessage &start, const std::map> &in_images) + : images(in_images), number_of_images(in_images.empty() ? 0 : in_images.rbegin()->first + 1) { + SetStartMessage(DatasetFromStartMessage(start, default_experiment)); + } + + [[nodiscard]] uint64_t GetNumberOfImages() const override { return number_of_images; } + void Close() override {} + + bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &ret) override { + const auto it = images.find(image_number); + if (it == images.end()) + return false; + ret.image = it->second->message.image; + return true; + } + + std::vector ReadSpots(int64_t) const override { return {}; } + }; +} + +StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map> &images, + int nthreads) { + HeldImageReader reader(start, images); + const auto dataset = reader.GetDataset(); + const RugnuxLiveSetup setup = MakeRugnuxStillsPrePassSetup(*dataset, nthreads); + + Rugnux rugnux(reader, setup.experiment, *dataset->pixel_mask, setup.config); + const ProcessResult result = rugnux.Run(); + Rugnux::ReleasePinnedPool(); + + return StillsPrePassResult{ + // Where the beam lands: the receiver analyses with the detector square to it, as stated, and a tilt + // the run refined moves its PONI, not this. + .beam_x_pxl = static_cast(result.used_direct_beam_x_pxl), + .beam_y_pxl = static_cast(result.used_direct_beam_y_pxl), + .mask = rugnux.GetPixelMask(), + .images = images.size() + }; +} diff --git a/receiver/StillsPrePass.h b/receiver/StillsPrePass.h new file mode 100644 index 000000000..efcb44fb4 --- /dev/null +++ b/receiver/StillsPrePass.h @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +#include "../common/JFJochMessages.h" +#include "../common/PixelMask.h" + +// An image the receiver holds until the pre-pass is done: its message, with the image pointing into +// `image`, the compressed frame it owns. +struct HeldImage { + DataMessage message; + std::vector image; +}; + +struct StillsPrePassResult { + float beam_x_pxl; + float beam_y_pxl; + PixelMask mask; + size_t images = 0; +}; + +// The first frames of a serial collection through rugnux, as `rugnux --no-merge` on them: the beam centre +// it decides (from the scattered background, and from the frames that index where there are any) and the +// mask it ends with (the collection's, plus the beam-stop shadow and the defective pixels). `images` by +// number; the start message is the collection's. +StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map> &images, + int nthreads); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 4a8bea15f..02ed28edb 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -1889,6 +1889,9 @@ public: static void ReleasePinnedPool(); // The pre-scan's powder measurement, for a report on a run that ended without a result. const PowderRings &GetPowderRings() const { return powder_; } + // The mask the run ended with: the input's, plus the beam-stop shadow and the defective pixels the + // pre-scan found. + const PixelMask &GetPixelMask() const { return pixel_mask_; } // Request cancellation; safe to call from any thread (the worker loop checks between images). // fast_live makes it a cut: a pass that is past its images still merges and makes its outputs, the diff --git a/rugnux/RugnuxLive.cpp b/rugnux/RugnuxLive.cpp index 90cf5f755..bade93af5 100644 --- a/rugnux/RugnuxLive.cpp +++ b/rugnux/RugnuxLive.cpp @@ -65,6 +65,45 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr return s; } +RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads) { + RugnuxLiveSetup s; + + // The experiment as rugnux_cli.cpp sets it up for serial data with no options given. + DiffractionExperiment &x = s.experiment; + x = dataset.experiment; + ApplyRugnuxExperimentDefaults(x); + x.NumTriggers(1); + x.MaxSpotCount(RUGNUX_MAX_SPOT_COUNT); + + IndexingSettings indexing; + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + indexing.RotationIndexing(false); + indexing.GeomRefinementAlgorithm(GeomRefinementAlgorithmEnum::BeamCenter); + indexing.IndexIceRings(false); + x.ImportIndexingSettings(indexing); + x.DetectIceRings(RugnuxDetectIceRings(dataset.file_detect_ice_rings, false)); + ApplyRugnuxIntegrationRadii(x, false); + + ProcessConfig &c = s.config; + c.mode = ProcessMode::FullAnalysis; + c.nthreads = nthreads; + c.nthreads_auto = true; + c.export_unmerged = false; + c.spot_finding = RugnuxDefaultSpotFinding(false); + // The frames come in photons: the receiver converted them. + ApplyRugnuxPixelGain(x, c.spot_finding, 1.0f); + c.rotation_indexing = false; + c.detect_beam_stop = RUGNUX_DETECT_BEAM_STOP_FRAMES; + c.beam_center_check = true; + c.adaptive_integration_radius = true; + c.measure_spot_budget = true; + // As `rugnux -C`: the geometry bundle adjustment needs the cell. + if (x.GetUnitCell().has_value()) + c.refine_geometry = RUGNUX_REFINE_GEOMETRY_FRAMES; + c.run_scaling = false; + return s; +} + void WarmUpRugnuxLiveGPU(const RugnuxLiveSetup &setup) { #ifdef JFJOCH_USE_CUDA // As Rugnux sets them for its first-pass pool (the measured centre's indexing and pass 2 use it as diff --git a/rugnux/RugnuxLive.h b/rugnux/RugnuxLive.h index cda7d1fb7..6372d38a4 100644 --- a/rugnux/RugnuxLive.h +++ b/rugnux/RugnuxLive.h @@ -20,6 +20,11 @@ struct RugnuxLiveSetup { RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, std::shared_ptr model); +// The first frames of a serial collection, processed inside the broker before the rest are analysed: +// what a bare `rugnux --no-merge` (with -C where the collection states a cell) on those frames sets. +// The run's beam centre and mask are what the broker then analyses the whole collection with. +RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads); + // Builds on every card, at the same time, the GPU FFT indexers a live run of this setup is known to // need - its first pass's and its short-axis pass's (the settings Rugnux gives them) - and the cuFFT // plans of its beam-centre capture, and with them the card's CUDA context and cuFFT's one-time diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index ac1e72260..ed36b2fce 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -107,6 +107,87 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_crystal_spot_and_index", "[JFJochReceiver]" REQUIRE_NOTHROW(writer_future.get()); } +TEST_CASE("JFJochIntegrationTest_ZMQ_crystal_spot_and_index_GPUStills", "[JFJochReceiver]") { + Logger logger(Catch::getResultCapture().getCurrentTestName()); + + RegisterHDF5Filter(); + + const uint16_t nthreads = 4; + + DiffractionExperiment experiment(DetJF4M()); + experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2) + .FilePrefix("crystal_test_gpu_stills").JungfrauConvPhotonCnt(false).SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true) + .DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4) + .SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90}); + experiment.SampleTemperature_K(123.0).RingCurrent_mA(115); + + PixelMask pixel_mask(experiment); + + // Load example image + HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5"); + HDF5DataSet dataset(data, "/entry/data/data"); + HDF5DataSpace file_space(dataset); + + REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum()); + REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum()); + std::vector image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); + + std::vector start = {4,0,0}; + std::vector file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; + dataset.ReadVector(image_conv, start, file_size); + + std::vector image_raw_geom(experiment.GetModulesNum() * RAW_MODULE_SIZE); + ConvertedToRawGeometry(experiment, image_raw_geom.data(), image_conv.data()); + logger.Info("Loaded image"); + + // Setup acquisition device + AcquisitionDeviceGroup aq_devices; + std::unique_ptr test = std::make_unique(0, 64); + for (int m = 0; m < experiment.GetModulesNum(); m++) + test->SetInternalGeneratorFrame((uint16_t *) image_raw_geom.data() + m * RAW_MODULE_SIZE, m); + + aq_devices.Add(std::move(test)); + + ZMQStream2Pusher pusher({"ipc://*"}); + pusher.WriterNotificationSocket("ipc://*"); + + ZMQImagePuller puller(pusher.GetAddress()[0]); + StreamWriter writer(logger, puller); + auto writer_future = std::async(std::launch::async, &StreamWriter::Run, &writer); + + JFJochReceiverService service(aq_devices, logger, pusher); + service.NumThreads(nthreads); + service.Indexing(experiment.GetIndexingSettings()); + // Serial data with rugnux on: the converted frames are analysed on the GPU, after the pre-pass. + service.RugnuxLive(true, nullptr, 4, 60); + + // No progress value at the start of measurement + REQUIRE(!service.GetProgress().has_value()); + + SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); + settings.signal_to_noise_threshold = 2.5; + settings.photon_count_threshold = 5; + settings.min_pix_per_spot = 1; + settings.max_pix_per_spot = 200; + settings.high_resolution_limit = 2.0; + settings.low_resolution_limit = 50.0; + service.SetSpotFindingSettings(settings); + + service.Start(experiment, pixel_mask, nullptr); + auto receiver_out = service.Stop(); + + CHECK(receiver_out.efficiency == 1.0); + REQUIRE(receiver_out.status.indexing_rate); + CHECK(receiver_out.status.indexing_rate.value() == 1.0); + CHECK(receiver_out.status.images_sent == experiment.GetImageNum()); + CHECK(!receiver_out.status.cancelled); + + // No progress value at the end of measurement + REQUIRE(!service.GetProgress().has_value()); + + REQUIRE_NOTHROW(writer_future.get()); +} + TEST_CASE("JFJochIntegrationTest_ZMQ_crystal_spot_and_index_refinement", "[JFJochReceiver]") { Logger logger(Catch::getResultCapture().getCurrentTestName()); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 12c0e427f..18abcf779 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -46,6 +46,10 @@ IF(NOT JFJOCH_PORTABLE_ONLY) ADD_EXECUTABLE(jfjoch_stream2_replay jfjoch_stream2_replay.cpp) TARGET_LINK_LIBRARIES(jfjoch_stream2_replay JFJochReader JFJochZMQ CBORStream2FrameSerialize JFJochCommon) + # Not installed: replays a stored serial dataset through the GPU analysis of converted JUNGFRAU frames. + ADD_EXECUTABLE(jfjoch_stills_gpu_replay jfjoch_stills_gpu_replay.cpp) + TARGET_LINK_LIBRARIES(jfjoch_stills_gpu_replay JFJochReceiver JFJochWriter JFJochReader) + ADD_EXECUTABLE(jfjoch_pcie_status jfjoch_pcie_status.cpp) TARGET_LINK_LIBRARIES(jfjoch_pcie_status JFJochDevice JFJochCommon) INSTALL(TARGETS jfjoch_pcie_status RUNTIME COMPONENT jfjoch) diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp new file mode 100644 index 000000000..7744e00f4 --- /dev/null +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Replays the frames of a stored serial collection through the broker's GPU analysis of converted +// JUNGFRAU frames (StillsGPUAnalysis): the pre-pass on the first images, then every frame analysed with the +// beam centre and mask it decided. Reports the pre-pass, the indexed fraction and the sustained rate, and +// writes the mask the frames were analysed with (_mask.u32, uint32, row by row). + +#include +#include +#include +#include +#include +#include +#include + +#include "../common/Logger.h" +#include "../image_analysis/indexing/IndexerThreadPool.h" +#include "../reader/JFJochHDF5Reader.h" +#include "../receiver/StillsGPUAnalysis.h" +#include "../writer/HDF5Objects.h" + +void print_usage() { + std::cout << "Usage: jfjoch_stills_gpu_replay [options] " << std::endl; + std::cout << " -o output prefix (default stills_gpu_replay)" << std::endl; + std::cout << " -s first image (default 0)" << std::endl; + std::cout << " -n images to replay (default all)" << std::endl; + std::cout << " -r images given per second (default 0 = as fast as they are read)" << std::endl; + std::cout << " -N analysis threads (default 16)" << std::endl; + std::cout << " -x , -y beam centre, overriding the file" << std::endl; + std::cout << " -C unit cell of the collection" << std::endl; + std::cout << " -c leave the file's unit cell out of the collection" << std::endl; +} + +int main(int argc, char **argv) { + Logger logger("jfjoch_stills_gpu_replay"); + RegisterHDF5Filter(); + + std::string prefix = "stills_gpu_replay"; + int64_t first = 0; + std::optional images; + double rate = 0.0; + int nthreads = 16; + std::optional beam_x, beam_y; + std::optional cell; + bool no_cell = false; + + int opt; + while ((opt = getopt(argc, argv, "o:s:n:r:N:x:y:C:c")) != -1) { + switch (opt) { + case 'o': prefix = optarg; break; + case 's': first = atoll(optarg); break; + case 'n': images = atoll(optarg); break; + case 'r': rate = atof(optarg); break; + case 'N': nthreads = atoi(optarg); break; + case 'x': beam_x = atof(optarg); break; + case 'y': beam_y = atof(optarg); break; + case 'c': no_cell = true; break; + case 'C': { + UnitCell c{}; + if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) { + print_usage(); + return EXIT_FAILURE; + } + cell = c; + break; + } + default: print_usage(); return EXIT_FAILURE; + } + } + if (optind + 1 != argc) { + print_usage(); + return EXIT_FAILURE; + } + + JFJochHDF5Reader reader; + reader.ReadFile(argv[optind]); + const auto dataset = reader.GetDataset(); + const int64_t n = std::min(images.value_or(reader.GetNumberOfImages() - first), + reader.GetNumberOfImages() - first); + const auto stored = reader.GetStoredPixelFormat(); + + DiffractionExperiment x(dataset->experiment); + x.ImagesPerTrigger(n).NumTriggers(1); + if (beam_x) + x.BeamX_pxl(*beam_x); + if (beam_y) + x.BeamY_pxl(*beam_y); + if (no_cell) + x.SetUnitCell(std::nullopt); + if (cell) + x.SetUnitCell(cell); + const PixelMask &mask = *dataset->pixel_mask; + + StartMessage start; + x.FillMessage(start); + start.number_of_images = n; + start.bit_depth_image = stored.bit_depth; + start.pixel_signed = stored.is_signed; + start.pixel_mask["default"] = mask.GetMask(x); + + IndexerThreadPool pool(x.GetIndexingSettings()); + + std::atomic analysed = 0, indexed = 0; + std::chrono::steady_clock::time_point t_ready; + const auto t0 = std::chrono::steady_clock::now(); + { + StillsGPUAnalysis gpu(x, mask, start, &pool, nthreads, logger, + [&](const StillsGPUAnalysis::Setup &s) { + t_ready = std::chrono::steady_clock::now(); + const auto &m = s.mask.GetMask(); + std::ofstream f(prefix + "_mask.u32", std::ios::binary); + f.write(reinterpret_cast(m.data()), m.size() * sizeof(uint32_t)); + logger.Info("Ready after {:.1f} s: beam centre {:.2f} {:.2f}", + std::chrono::duration(t_ready - t0).count(), + s.experiment.GetBeamX_pxl(), s.experiment.GetBeamY_pxl()); + }, + [&](DataMessage &m, const AzimuthalIntegrationProfile &) { + ++analysed; + if (m.indexing_result.value_or(false)) + ++indexed; + }, + [] { return DiffractionExperiment::DefaultDataProcessingSettings(); }); + + JFJochReaderRawImage raw; + for (int64_t i = 0; i < n; i++) { + if (rate > 0) + std::this_thread::sleep_until(t0 + std::chrono::duration(static_cast(i) / rate)); + reader.ReadRawImage(first + i, raw); + auto held = std::make_shared(); + held->image.assign(raw.image.GetCompressed(), raw.image.GetCompressed() + raw.image.GetCompressedSize()); + held->message.number = i; + held->message.image = CompressedImage(held->image.data(), held->image.size(), raw.image.GetWidth(), + raw.image.GetHeight(), raw.image.GetMode(), + raw.image.GetCompressionAlgorithm()); + held->message.image_collection_efficiency = 1.0f; + gpu.Hold(held); + } + logger.Info("Gave {} images in {:.1f} s", n, + std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); + gpu.Finish(); + } + const auto t1 = std::chrono::steady_clock::now(); + const double after_ready = std::chrono::duration(t1 - t_ready).count(); + logger.Info("Analysed {} images, {} indexed ({:.1f}%); {:.1f} s after the pre-pass = {:.1f} Hz; total {:.1f} s", + analysed.load(), indexed.load(), 100.0 * indexed / std::max(1, analysed), after_ready, + analysed / after_ready, std::chrono::duration(t1 - t0).count()); + return EXIT_SUCCESS; +} -- 2.54.0 From 10c5e0d1e288bd76b804d35d7854327f9fdb9f34 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 18:47:35 +0100 Subject: [PATCH 02/44] jfjoch_stills_gpu_replay: -c indexes de novo (Auto indexer) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- tools/jfjoch_stills_gpu_replay.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp index 7744e00f4..047bd9dfa 100644 --- a/tools/jfjoch_stills_gpu_replay.cpp +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -86,8 +86,13 @@ int main(int argc, char **argv) { x.BeamX_pxl(*beam_x); if (beam_y) x.BeamY_pxl(*beam_y); - if (no_cell) + if (no_cell) { + // Indexed de novo, as the broker does where no cell is stated. x.SetUnitCell(std::nullopt); + IndexingSettings indexing = x.GetIndexingSettings(); + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + x.ImportIndexingSettings(indexing); + } if (cell) x.SetUnitCell(cell); const PixelMask &mask = *dataset->pixel_mask; -- 2.54.0 From 6195e7121075e6c2cf293f511cdb7b74e000d98b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 19:21:21 +0100 Subject: [PATCH 03/44] Stills pre-pass: apply the refined detector distance, report it in END The GPU analysis of converted frames now takes the pre-pass's detector distance (rugnux refines it with the geometry bundle where a cell is given) along with the beam centre. END carries it as refined_detector_distance_mm; that field and refined_beam_center_x/y are now in the CBOR END message, so they reach a writer over the wire, not only in process. The receiver's plots are set up again on the pre-pass geometry: a moved distance changes the azimuthal binning, and combining profiles of the two binnings threw and cancelled the collection. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- common/JFJochMessages.h | 2 ++ docs/CBOR.md | 3 +++ frame_serialize/CBORStream2Deserializer.cpp | 6 ++++++ frame_serialize/CBORStream2Serializer.cpp | 3 +++ receiver/JFJochReceiverFPGA.cpp | 3 +++ receiver/StillsGPUAnalysis.cpp | 9 +++++++-- receiver/StillsGPUAnalysis.h | 1 + receiver/StillsPrePass.cpp | 1 + receiver/StillsPrePass.h | 7 ++++--- tests/CBORTest.cpp | 20 ++++++++++++++++++++ 10 files changed, 50 insertions(+), 5 deletions(-) diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 0aa2ee4fb..7f25b8d7b 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -488,6 +488,8 @@ struct EndMessage { std::optional refined_poni_rot2; std::optional refined_poni_rot3; std::optional refined_rotation_axis; + // The detector distance the frames were analysed at, where it was refined (the broker's stills pre-pass). + std::optional refined_detector_distance_mm; // Vectors with end result: std::vector data_collection_efficiency; diff --git a/docs/CBOR.md b/docs/CBOR.md index ccaaef192..247fa1f2a 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -308,6 +308,9 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | adu_histogram_bin_width | uint64 | Width of bins in the above histogram \[ADU\] | | | max_receiver_delay | uint64 | Internal performance of Jungfraujoch | | | bkg_estimate | float | Mean background estimate for the whole run | | +| refined_beam_center_x | float | Beam centre X the frames were analysed at, where the analysis refined it \[pixel\] | | +| refined_beam_center_y | float | Beam centre Y the frames were analysed at, where the analysis refined it \[pixel\] | | +| refined_detector_distance_mm | float | Detector distance the frames were analysed at, where the analysis refined it \[mm\] | | | spindle_blind_fraction | float | Run mean of the per-image spindle_blind_fraction, over the frames that had one | | | spindle_lost_unique_fraction | float | Fraction (0-1) of unique reflections the mounting made unmeasurable, exact under the measured point group; offline (Rugnux) only | | | indexing_rate | float | Mean indexing rate for the whole run | | diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index f5e56cc52..f540f75a7 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1733,6 +1733,12 @@ namespace { ProcessADUHistogramElement(message, value); else if (key == "adu_histogram_bin_width") message.adu_histogram_bin_width = GetCBORUInt(value); + else if (key == "refined_beam_center_x") + message.refined_beam_center_x = GetCBORFloat(value); + else if (key == "refined_beam_center_y") + message.refined_beam_center_y = GetCBORFloat(value); + else if (key == "refined_detector_distance_mm") + message.refined_detector_distance_mm = GetCBORFloat(value); else if (key == "bkg_estimate") message.bkg_estimate = GetCBORFloat(value); else if (key == "spindle_blind_fraction") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index ec07c26f3..8fe15d49c 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -946,6 +946,9 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) { CBOR_ENC(mapEncoder, "adu_histogram_bin_width", message.adu_histogram_bin_width); CBOR_ENC(mapEncoder, "max_receiver_delay", message.max_receiver_delay); CBOR_ENC(mapEncoder, "indexing_rate", message.indexing_rate); + CBOR_ENC(mapEncoder, "refined_beam_center_x", message.refined_beam_center_x); + CBOR_ENC(mapEncoder, "refined_beam_center_y", message.refined_beam_center_y); + CBOR_ENC(mapEncoder, "refined_detector_distance_mm", message.refined_detector_distance_mm); CBOR_ENC(mapEncoder, "bkg_estimate", message.bkg_estimate); CBOR_ENC(mapEncoder, "spindle_blind_fraction", message.spindle_blind_fraction); CBOR_ENC(mapEncoder, "spindle_lost_unique_fraction", message.spindle_lost_unique_fraction); diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 4f8969587..10a77b872 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -130,6 +130,8 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen experiment, pixel_mask, MakeStartMessage(experiment, pixel_mask, *az_int_mapping), indexing_thread_pool, std::min(16, frame_transformation_nthreads), logger, [this](const StillsGPUAnalysis::Setup &s) { + // The profiles are binned on the pre-pass's geometry. + plots.Setup(s.experiment, *s.mapping); SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping)); SendCalibration(); }, @@ -807,4 +809,5 @@ void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) { message.unit_cell = gpu->GetConsensusUnitCell(); message.refined_beam_center_x = gpu->GetBeamX_pxl(); message.refined_beam_center_y = gpu->GetBeamY_pxl(); + message.refined_detector_distance_mm = gpu->GetDetectorDistance_mm(); } diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index fba263555..7d306c713 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -76,12 +76,13 @@ void StillsGPUAnalysis::PrePassThread() { try { const auto t0 = std::chrono::steady_clock::now(); const auto r = RunStillsPrePass(start, sample, static_cast(std::thread::hardware_concurrency())); - setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl); + setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl).DetectorDistance_mm(r.distance_mm); setup.mask = r.mask; logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, " - "masked pixels {} -> {}", r.images, + "distance {:.2f} -> {:.2f} mm, masked pixels {} -> {}", r.images, std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), r.beam_x_pxl, r.beam_y_pxl, + experiment.GetDetectorDistance_mm(), r.distance_mm, masked(mask), masked(r.mask)); } catch (const std::exception &e) { logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask - {}", @@ -135,3 +136,7 @@ float StillsGPUAnalysis::GetBeamX_pxl() const { float StillsGPUAnalysis::GetBeamY_pxl() const { return setup.experiment.GetBeamY_pxl(); } + +float StillsGPUAnalysis::GetDetectorDistance_mm() const { + return setup.experiment.GetDetectorDistance_mm(); +} diff --git a/receiver/StillsGPUAnalysis.h b/receiver/StillsGPUAnalysis.h index 1f043219e..0f7a4f796 100644 --- a/receiver/StillsGPUAnalysis.h +++ b/receiver/StillsGPUAnalysis.h @@ -53,6 +53,7 @@ public: [[nodiscard]] std::optional GetConsensusUnitCell() const; [[nodiscard]] float GetBeamX_pxl() const; [[nodiscard]] float GetBeamY_pxl() const; + [[nodiscard]] float GetDetectorDistance_mm() const; private: const DiffractionExperiment &experiment; diff --git a/receiver/StillsPrePass.cpp b/receiver/StillsPrePass.cpp index 3719ac156..4bbaa69ea 100644 --- a/receiver/StillsPrePass.cpp +++ b/receiver/StillsPrePass.cpp @@ -58,6 +58,7 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map(result.used_direct_beam_x_pxl), .beam_y_pxl = static_cast(result.used_direct_beam_y_pxl), + .distance_mm = result.used_distance_mm, .mask = rugnux.GetPixelMask(), .images = images.size() }; diff --git a/receiver/StillsPrePass.h b/receiver/StillsPrePass.h index efcb44fb4..ea5c8c021 100644 --- a/receiver/StillsPrePass.h +++ b/receiver/StillsPrePass.h @@ -20,13 +20,14 @@ struct HeldImage { struct StillsPrePassResult { float beam_x_pxl; float beam_y_pxl; + float distance_mm; PixelMask mask; size_t images = 0; }; // The first frames of a serial collection through rugnux, as `rugnux --no-merge` on them: the beam centre -// it decides (from the scattered background, and from the frames that index where there are any) and the -// mask it ends with (the collection's, plus the beam-stop shadow and the defective pixels). `images` by -// number; the start message is the collection's. +// it decides (from the scattered background, and from the frames that index where there are any), the +// distance (refined where a cell is given) and the mask it ends with (the collection's, plus the beam-stop +// shadow and the defective pixels). `images` by number; the start message is the collection's. StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map> &images, int nthreads); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index 76cbe36ce..5ab96b460 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -578,6 +578,26 @@ TEST_CASE("CBORSerialize_End_SpaceGroup", "[CBOR]") { CHECK(*out.space_group_number == static_cast(sg->number)); } +TEST_CASE("CBORSerialize_End_RefinedGeometry", "[CBOR]") { + std::vector buffer(1024 * 1024); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + + EndMessage message{}; + message.max_image_number = 0; + message.refined_beam_center_x = 1615.5f; + message.refined_beam_center_y = 1659.8f; + message.refined_detector_distance_mm = 115.17f; + REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); + + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->end_message); + const EndMessage &out = *deserialized->end_message; + CHECK(out.refined_beam_center_x == message.refined_beam_center_x); + CHECK(out.refined_beam_center_y == message.refined_beam_center_y); + CHECK(out.refined_detector_distance_mm == message.refined_detector_distance_mm); +} + TEST_CASE("CBORSerialize_End_UnitCellEsd", "[CBOR]") { std::vector buffer(1024 * 1024); CBORStream2Serializer serializer(buffer.data(), buffer.size()); -- 2.54.0 From 72eccd6b71e6e9e466af76c9affba0fee3564260 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 19:29:18 +0100 Subject: [PATCH 04/44] GPU analysis of converted frames: rugnux's stills spot finder The live analysis indexed 56.3% of the frames of a serial lysozyme run where offline rugnux indexes 59.3% of the same frames. The whole gap is the spot finder: the broker's default fixed threshold with two-pixel minimum spots against rugnux's adaptive threshold that keeps single-pixel spots. With those two settings (on top of the broker's others) the live analysis indexes 59.3% of all 24960 frames, as offline; the sustained rate is unchanged (~1080 Hz on the workstation). The pre-pass decided the centre and the mask with the same spot finder. jfjoch_stills_gpu_replay analyses as the receiver does; -F keeps the broker's fixed-threshold finder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- receiver/JFJochReceiverFPGA.cpp | 10 +++++++++- tools/jfjoch_stills_gpu_replay.cpp | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 10a77b872..e21c9f930 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -136,7 +136,15 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen SendCalibration(); }, [this](DataMessage &m, const AzimuthalIntegrationProfile &p) { GPUSink(m, p); }, - [this] { return GetSpotFindingSettings(); }); + [this] { + // The broker's settings, with the spot finder rugnux uses on stills (the pre-pass + // decided the centre and mask with it): adaptive threshold, single-pixel spots kept. + // With the broker's default fixed threshold 3 points fewer frames index. + SpotFindingSettings s = GetSpotFindingSettings(); + s.adaptive_threshold = true; + s.min_pix_per_spot = std::nullopt; + return s; + }); } else { SendStartMessage(); SendCalibration(); diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp index 047bd9dfa..d7161dbec 100644 --- a/tools/jfjoch_stills_gpu_replay.cpp +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -29,6 +29,7 @@ void print_usage() { std::cout << " -N analysis threads (default 16)" << std::endl; std::cout << " -x , -y beam centre, overriding the file" << std::endl; std::cout << " -C unit cell of the collection" << std::endl; + std::cout << " -F the broker's default fixed-threshold spot finder (default: adaptive, as the receiver)" << std::endl; std::cout << " -c leave the file's unit cell out of the collection" << std::endl; } @@ -44,9 +45,10 @@ int main(int argc, char **argv) { std::optional beam_x, beam_y; std::optional cell; bool no_cell = false; + bool fixed_threshold = false; int opt; - while ((opt = getopt(argc, argv, "o:s:n:r:N:x:y:C:c")) != -1) { + while ((opt = getopt(argc, argv, "o:s:n:r:N:x:y:C:cF")) != -1) { switch (opt) { case 'o': prefix = optarg; break; case 's': first = atoll(optarg); break; @@ -56,6 +58,7 @@ int main(int argc, char **argv) { case 'x': beam_x = atof(optarg); break; case 'y': beam_y = atof(optarg); break; case 'c': no_cell = true; break; + case 'F': fixed_threshold = true; break; case 'C': { UnitCell c{}; if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) { @@ -104,6 +107,13 @@ int main(int argc, char **argv) { start.pixel_signed = stored.is_signed; start.pixel_mask["default"] = mask.GetMask(x); + SpotFindingSettings spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); + // As JFJochReceiverFPGA analyses converted frames. + if (!fixed_threshold) { + spot_finding.adaptive_threshold = true; + spot_finding.min_pix_per_spot = std::nullopt; + } + IndexerThreadPool pool(x.GetIndexingSettings()); std::atomic analysed = 0, indexed = 0; @@ -125,7 +135,7 @@ int main(int argc, char **argv) { if (m.indexing_result.value_or(false)) ++indexed; }, - [] { return DiffractionExperiment::DefaultDataProcessingSettings(); }); + [&] { return spot_finding; }); JFJochReaderRawImage raw; for (int64_t i = 0; i < n; i++) { -- 2.54.0 From 4c38fe6718dc5a02d2d8e489e5b846d04fc8f023 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 19:36:25 +0100 Subject: [PATCH 05/44] Stills pre-pass: no whole-range distance search The broker's distance is the beamline's, refined by the geometry bundle; the 30-3000 mm search (for files whose distance can be off by a factor) took 3.4 of the 9.8 s pre-pass on 100 JUNGFRAU 8M frames. Without it 6.0 s, the same beam centre (within 0.01 px from the file's centre and from 8-19 px off), the same distance and a bit-identical mask. Offline rugnux keeps the search (ProcessConfig::stills_distance_search, default on). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- rugnux/Rugnux.h | 4 ++++ rugnux/RugnuxLive.cpp | 2 ++ rugnux/RugnuxStills.cpp | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 02ed28edb..8367b68a9 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -192,6 +192,10 @@ struct ProcessConfig { // The detector distance was given on the command line: on stills it is held - not searched // (DecideStillsGeometry), not refined (RefineStillsGeometry). bool detector_distance_given = false; + // Stills with a cell: the distance is searched over the whole range (DecideStillsGeometry) - for files whose + // distance can be off by a factor. Off where it comes from the beamline (the broker's pre-pass); it is still + // refined (RefineStillsGeometry). + bool stills_distance_search = true; // The detector tilt was given on the command line (--rot1/--rot2): on stills it is held, not refined // (DecideStillsTilt). bool detector_tilt_given = false; diff --git a/rugnux/RugnuxLive.cpp b/rugnux/RugnuxLive.cpp index bade93af5..4cd31f0a4 100644 --- a/rugnux/RugnuxLive.cpp +++ b/rugnux/RugnuxLive.cpp @@ -100,6 +100,8 @@ RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, // As `rugnux -C`: the geometry bundle adjustment needs the cell. if (x.GetUnitCell().has_value()) c.refine_geometry = RUGNUX_REFINE_GEOMETRY_FRAMES; + // The broker's distance is the beamline's: refined, not searched for. + c.stills_distance_search = false; c.run_scaling = false; return s; } diff --git a/rugnux/RugnuxStills.cpp b/rugnux/RugnuxStills.cpp index 1911dadc6..9537b80d7 100644 --- a/rugnux/RugnuxStills.cpp +++ b/rugnux/RugnuxStills.cpp @@ -968,7 +968,7 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) { // value. A distance the user gave (--detector-distance) is not searched: it is the user's decision, e.g. a // calibrated distance where the reference cell is the less certain of the two. float distance = file_distance; - if (experiment_.GetUnitCell().has_value() && !config_.detector_distance_given) { + if (experiment_.GetUnitCell().has_value() && !config_.detector_distance_given && config_.stills_distance_search) { std::vector grid; for (float d = MIN_DISTANCE_MM; d <= MAX_DISTANCE_MM; d *= STEP) grid.push_back(d); -- 2.54.0 From 4a54e0c6f58c52507a6f966a4ea1a2427a372464 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 2 Nov 2026 20:22:34 +0100 Subject: [PATCH 06/44] /rugnux/status: the serial pre-pass and GPU analysis A serial collection analysed on the GPU after the pre-pass now reports in the rugnux live status: the run as steps (Pre-pass, GPU analysis) with images received and progress, as rotation live does, and a `stills` block - the pre-pass state (holding N of 100, running, done, failed) and duration, the stated and decided beam centre and distance, the beam-stop shadow and defective pixels it masked, and the images analysed and indexed and the analysis rate since. Schema rugnux_live_stills; C++ model, frontend client and Redoc regenerated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJKPUFLoeAchjzv3x4 --- broker/OpenAPIConvert.cpp | 32 ++ broker/gen/model/Rugnux_live_status.cpp | 30 +- broker/gen/model/Rugnux_live_status.h | 10 + broker/gen/model/Rugnux_live_stills.cpp | 392 ++++++++++++++++++++++++ broker/gen/model/Rugnux_live_stills.h | 189 ++++++++++++ broker/gen/model/Rugnux_settings.h | 6 +- broker/jfjoch_api.yaml | 77 ++++- broker/redoc-static.html | 6 +- frontend/src/client/index.ts | 2 +- frontend/src/client/types.gen.ts | 57 +++- frontend/src/client/zod.gen.ts | 33 +- receiver/JFJochReceiverFPGA.cpp | 5 +- receiver/JFJochReceiverFPGA.h | 3 +- receiver/JFJochReceiverService.cpp | 3 +- receiver/RugnuxLiveStatus.cpp | 5 + receiver/RugnuxLiveStatus.h | 20 ++ receiver/StillsGPUAnalysis.cpp | 82 ++++- receiver/StillsGPUAnalysis.h | 12 +- 18 files changed, 942 insertions(+), 22 deletions(-) create mode 100644 broker/gen/model/Rugnux_live_stills.cpp create mode 100644 broker/gen/model/Rugnux_live_stills.h diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index c67f159a5..63cf22b24 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -1287,6 +1287,38 @@ org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSna ret.setDMin(input.d_min.value()); if (input.isa) ret.setIsa(input.isa.value()); + if (input.stills) { + const RugnuxLiveStills &st = input.stills.value(); + org::openapitools::server::model::Rugnux_live_stills stills; + switch (st.prepass) { + case RugnuxLivePrePass::Holding: stills.setPrepass("Holding"); break; + case RugnuxLivePrePass::Running: stills.setPrepass("Running"); break; + case RugnuxLivePrePass::Done: stills.setPrepass("Done"); break; + case RugnuxLivePrePass::Failed: stills.setPrepass("Failed"); break; + } + stills.setImagesHeld(st.images_held); + stills.setImagesNeeded(st.images_needed); + if (st.prepass_s) + stills.setPrepassS(st.prepass_s.value()); + stills.setFileBeamXPxl(st.file_beam_x_pxl); + stills.setFileBeamYPxl(st.file_beam_y_pxl); + stills.setFileDistanceMm(st.file_distance_mm); + if (st.beam_x_pxl) + stills.setBeamXPxl(st.beam_x_pxl.value()); + if (st.beam_y_pxl) + stills.setBeamYPxl(st.beam_y_pxl.value()); + if (st.distance_mm) + stills.setDistanceMm(st.distance_mm.value()); + if (st.shadow_pixels) + stills.setShadowPixels(st.shadow_pixels.value()); + if (st.bad_pixels) + stills.setBadPixels(st.bad_pixels.value()); + stills.setImagesAnalysed(st.images_analysed); + stills.setImagesIndexed(st.images_indexed); + if (st.analysis_rate_hz) + stills.setAnalysisRateHz(st.analysis_rate_hz.value()); + ret.setStills(stills); + } if (input.model) { const RugnuxLiveModel &m = input.model.value(); org::openapitools::server::model::Rugnux_live_model model; diff --git a/broker/gen/model/Rugnux_live_status.cpp b/broker/gen/model/Rugnux_live_status.cpp index 52e8a4183..a98b7d45c 100644 --- a/broker/gen/model/Rugnux_live_status.cpp +++ b/broker/gen/model/Rugnux_live_status.cpp @@ -27,6 +27,7 @@ Rugnux_live_status::Rugnux_live_status() m_Anomalous_map = false; m_Anomalous_signal = false; m_ModelIsSet = false; + m_StillsIsSet = false; m_Space_group = ""; m_Space_groupIsSet = false; m_D_min = 0.0; @@ -78,7 +79,7 @@ bool Rugnux_live_status::validate(std::stringstream& msg, const std::string& pat bool success = true; const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_status" : pathPrefix; - + if (stepProgressIsSet()) { const float& value = m_Step_progress; @@ -188,6 +189,9 @@ bool Rugnux_live_status::operator==(const Rugnux_live_status& rhs) const ((!modelIsSet() && !rhs.modelIsSet()) || (modelIsSet() && rhs.modelIsSet() && getModel() == rhs.getModel())) && + ((!stillsIsSet() && !rhs.stillsIsSet()) || (stillsIsSet() && rhs.stillsIsSet() && getStills() == rhs.getStills())) && + + ((!spaceGroupIsSet() && !rhs.spaceGroupIsSet()) || (spaceGroupIsSet() && rhs.spaceGroupIsSet() && getSpaceGroup() == rhs.getSpaceGroup())) && @@ -268,6 +272,8 @@ void to_json(nlohmann::json& j, const Rugnux_live_status& o) j["anomalous_signal"] = o.m_Anomalous_signal; if(o.modelIsSet()) j["model"] = o.m_Model; + if(o.stillsIsSet()) + j["stills"] = o.m_Stills; if(o.spaceGroupIsSet()) j["space_group"] = o.m_Space_group; if(o.dMinIsSet()) @@ -315,6 +321,11 @@ void from_json(const nlohmann::json& j, Rugnux_live_status& o) j.at("model").get_to(o.m_Model); o.m_ModelIsSet = true; } + if(j.find("stills") != j.end()) + { + j.at("stills").get_to(o.m_Stills); + o.m_StillsIsSet = true; + } if(j.find("space_group") != j.end()) { j.at("space_group").get_to(o.m_Space_group); @@ -440,6 +451,23 @@ void Rugnux_live_status::unsetModel() { m_ModelIsSet = false; } +org::openapitools::server::model::Rugnux_live_stills Rugnux_live_status::getStills() const +{ + return m_Stills; +} +void Rugnux_live_status::setStills(org::openapitools::server::model::Rugnux_live_stills const& value) +{ + m_Stills = value; + m_StillsIsSet = true; +} +bool Rugnux_live_status::stillsIsSet() const +{ + return m_StillsIsSet; +} +void Rugnux_live_status::unsetStills() +{ + m_StillsIsSet = false; +} std::string Rugnux_live_status::getSpaceGroup() const { return m_Space_group; diff --git a/broker/gen/model/Rugnux_live_status.h b/broker/gen/model/Rugnux_live_status.h index 16b0bd600..8dae39871 100644 --- a/broker/gen/model/Rugnux_live_status.h +++ b/broker/gen/model/Rugnux_live_status.h @@ -24,6 +24,7 @@ #include "Rugnux_live_model.h" #include "Rugnux_live_step.h" #include +#include "Rugnux_live_stills.h" #include namespace org::openapitools::server::model @@ -95,6 +96,13 @@ public: bool modelIsSet() const; void unsetModel(); /// + /// + /// + org::openapitools::server::model::Rugnux_live_stills getStills() const; + void setStills(org::openapitools::server::model::Rugnux_live_stills const& value); + bool stillsIsSet() const; + void unsetStills(); + /// /// Space group of the result (the summary's \"Space group\" row); absent until the run is over /// std::string getSpaceGroup() const; @@ -237,6 +245,8 @@ protected: org::openapitools::server::model::Rugnux_live_model m_Model; bool m_ModelIsSet; + org::openapitools::server::model::Rugnux_live_stills m_Stills; + bool m_StillsIsSet; std::string m_Space_group; bool m_Space_groupIsSet; double m_D_min; diff --git a/broker/gen/model/Rugnux_live_stills.cpp b/broker/gen/model/Rugnux_live_stills.cpp new file mode 100644 index 000000000..9fcb6bd56 --- /dev/null +++ b/broker/gen/model/Rugnux_live_stills.cpp @@ -0,0 +1,392 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.181 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_live_stills.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_live_stills::Rugnux_live_stills() +{ + m_Prepass = ""; + m_Images_held = 0L; + m_Images_needed = 0L; + m_Prepass_s = 0.0; + m_Prepass_sIsSet = false; + m_File_beam_x_pxl = 0.0; + m_File_beam_y_pxl = 0.0; + m_File_distance_mm = 0.0; + m_Beam_x_pxl = 0.0; + m_Beam_x_pxlIsSet = false; + m_Beam_y_pxl = 0.0; + m_Beam_y_pxlIsSet = false; + m_Distance_mm = 0.0; + m_Distance_mmIsSet = false; + m_Shadow_pixels = 0L; + m_Shadow_pixelsIsSet = false; + m_Bad_pixels = 0L; + m_Bad_pixelsIsSet = false; + m_Images_analysed = 0L; + m_Images_indexed = 0L; + m_Analysis_rate_hz = 0.0; + m_Analysis_rate_hzIsSet = false; + +} + +void Rugnux_live_stills::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_live_stills::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_live_stills::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_stills" : pathPrefix; + + + return success; +} + +bool Rugnux_live_stills::operator==(const Rugnux_live_stills& rhs) const +{ + return + + + (getPrepass() == rhs.getPrepass()) + && + + (getImagesHeld() == rhs.getImagesHeld()) + && + + (getImagesNeeded() == rhs.getImagesNeeded()) + && + + + ((!prepassSIsSet() && !rhs.prepassSIsSet()) || (prepassSIsSet() && rhs.prepassSIsSet() && getPrepassS() == rhs.getPrepassS())) && + + (getFileBeamXPxl() == rhs.getFileBeamXPxl()) + && + + (getFileBeamYPxl() == rhs.getFileBeamYPxl()) + && + + (getFileDistanceMm() == rhs.getFileDistanceMm()) + && + + + ((!beamXPxlIsSet() && !rhs.beamXPxlIsSet()) || (beamXPxlIsSet() && rhs.beamXPxlIsSet() && getBeamXPxl() == rhs.getBeamXPxl())) && + + + ((!beamYPxlIsSet() && !rhs.beamYPxlIsSet()) || (beamYPxlIsSet() && rhs.beamYPxlIsSet() && getBeamYPxl() == rhs.getBeamYPxl())) && + + + ((!distanceMmIsSet() && !rhs.distanceMmIsSet()) || (distanceMmIsSet() && rhs.distanceMmIsSet() && getDistanceMm() == rhs.getDistanceMm())) && + + + ((!shadowPixelsIsSet() && !rhs.shadowPixelsIsSet()) || (shadowPixelsIsSet() && rhs.shadowPixelsIsSet() && getShadowPixels() == rhs.getShadowPixels())) && + + + ((!badPixelsIsSet() && !rhs.badPixelsIsSet()) || (badPixelsIsSet() && rhs.badPixelsIsSet() && getBadPixels() == rhs.getBadPixels())) && + + (getImagesAnalysed() == rhs.getImagesAnalysed()) + && + + (getImagesIndexed() == rhs.getImagesIndexed()) + && + + + ((!analysisRateHzIsSet() && !rhs.analysisRateHzIsSet()) || (analysisRateHzIsSet() && rhs.analysisRateHzIsSet() && getAnalysisRateHz() == rhs.getAnalysisRateHz())) + + ; +} + +bool Rugnux_live_stills::operator!=(const Rugnux_live_stills& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_live_stills& o) +{ + j = nlohmann::json::object(); + j["prepass"] = o.m_Prepass; + j["images_held"] = o.m_Images_held; + j["images_needed"] = o.m_Images_needed; + if(o.prepassSIsSet()) + j["prepass_s"] = o.m_Prepass_s; + j["file_beam_x_pxl"] = o.m_File_beam_x_pxl; + j["file_beam_y_pxl"] = o.m_File_beam_y_pxl; + j["file_distance_mm"] = o.m_File_distance_mm; + if(o.beamXPxlIsSet()) + j["beam_x_pxl"] = o.m_Beam_x_pxl; + if(o.beamYPxlIsSet()) + j["beam_y_pxl"] = o.m_Beam_y_pxl; + if(o.distanceMmIsSet()) + j["distance_mm"] = o.m_Distance_mm; + if(o.shadowPixelsIsSet()) + j["shadow_pixels"] = o.m_Shadow_pixels; + if(o.badPixelsIsSet()) + j["bad_pixels"] = o.m_Bad_pixels; + j["images_analysed"] = o.m_Images_analysed; + j["images_indexed"] = o.m_Images_indexed; + if(o.analysisRateHzIsSet()) + j["analysis_rate_hz"] = o.m_Analysis_rate_hz; + +} + +void from_json(const nlohmann::json& j, Rugnux_live_stills& o) +{ + j.at("prepass").get_to(o.m_Prepass); + j.at("images_held").get_to(o.m_Images_held); + j.at("images_needed").get_to(o.m_Images_needed); + if(j.find("prepass_s") != j.end()) + { + j.at("prepass_s").get_to(o.m_Prepass_s); + o.m_Prepass_sIsSet = true; + } + j.at("file_beam_x_pxl").get_to(o.m_File_beam_x_pxl); + j.at("file_beam_y_pxl").get_to(o.m_File_beam_y_pxl); + j.at("file_distance_mm").get_to(o.m_File_distance_mm); + if(j.find("beam_x_pxl") != j.end()) + { + j.at("beam_x_pxl").get_to(o.m_Beam_x_pxl); + o.m_Beam_x_pxlIsSet = true; + } + if(j.find("beam_y_pxl") != j.end()) + { + j.at("beam_y_pxl").get_to(o.m_Beam_y_pxl); + o.m_Beam_y_pxlIsSet = true; + } + if(j.find("distance_mm") != j.end()) + { + j.at("distance_mm").get_to(o.m_Distance_mm); + o.m_Distance_mmIsSet = true; + } + if(j.find("shadow_pixels") != j.end()) + { + j.at("shadow_pixels").get_to(o.m_Shadow_pixels); + o.m_Shadow_pixelsIsSet = true; + } + if(j.find("bad_pixels") != j.end()) + { + j.at("bad_pixels").get_to(o.m_Bad_pixels); + o.m_Bad_pixelsIsSet = true; + } + j.at("images_analysed").get_to(o.m_Images_analysed); + j.at("images_indexed").get_to(o.m_Images_indexed); + if(j.find("analysis_rate_hz") != j.end()) + { + j.at("analysis_rate_hz").get_to(o.m_Analysis_rate_hz); + o.m_Analysis_rate_hzIsSet = true; + } + +} + +std::string Rugnux_live_stills::getPrepass() const +{ + return m_Prepass; +} +void Rugnux_live_stills::setPrepass(std::string const& value) +{ + m_Prepass = value; +} +int64_t Rugnux_live_stills::getImagesHeld() const +{ + return m_Images_held; +} +void Rugnux_live_stills::setImagesHeld(int64_t const value) +{ + m_Images_held = value; +} +int64_t Rugnux_live_stills::getImagesNeeded() const +{ + return m_Images_needed; +} +void Rugnux_live_stills::setImagesNeeded(int64_t const value) +{ + m_Images_needed = value; +} +double Rugnux_live_stills::getPrepassS() const +{ + return m_Prepass_s; +} +void Rugnux_live_stills::setPrepassS(double const value) +{ + m_Prepass_s = value; + m_Prepass_sIsSet = true; +} +bool Rugnux_live_stills::prepassSIsSet() const +{ + return m_Prepass_sIsSet; +} +void Rugnux_live_stills::unsetPrepass_s() +{ + m_Prepass_sIsSet = false; +} +double Rugnux_live_stills::getFileBeamXPxl() const +{ + return m_File_beam_x_pxl; +} +void Rugnux_live_stills::setFileBeamXPxl(double const value) +{ + m_File_beam_x_pxl = value; +} +double Rugnux_live_stills::getFileBeamYPxl() const +{ + return m_File_beam_y_pxl; +} +void Rugnux_live_stills::setFileBeamYPxl(double const value) +{ + m_File_beam_y_pxl = value; +} +double Rugnux_live_stills::getFileDistanceMm() const +{ + return m_File_distance_mm; +} +void Rugnux_live_stills::setFileDistanceMm(double const value) +{ + m_File_distance_mm = value; +} +double Rugnux_live_stills::getBeamXPxl() const +{ + return m_Beam_x_pxl; +} +void Rugnux_live_stills::setBeamXPxl(double const value) +{ + m_Beam_x_pxl = value; + m_Beam_x_pxlIsSet = true; +} +bool Rugnux_live_stills::beamXPxlIsSet() const +{ + return m_Beam_x_pxlIsSet; +} +void Rugnux_live_stills::unsetBeam_x_pxl() +{ + m_Beam_x_pxlIsSet = false; +} +double Rugnux_live_stills::getBeamYPxl() const +{ + return m_Beam_y_pxl; +} +void Rugnux_live_stills::setBeamYPxl(double const value) +{ + m_Beam_y_pxl = value; + m_Beam_y_pxlIsSet = true; +} +bool Rugnux_live_stills::beamYPxlIsSet() const +{ + return m_Beam_y_pxlIsSet; +} +void Rugnux_live_stills::unsetBeam_y_pxl() +{ + m_Beam_y_pxlIsSet = false; +} +double Rugnux_live_stills::getDistanceMm() const +{ + return m_Distance_mm; +} +void Rugnux_live_stills::setDistanceMm(double const value) +{ + m_Distance_mm = value; + m_Distance_mmIsSet = true; +} +bool Rugnux_live_stills::distanceMmIsSet() const +{ + return m_Distance_mmIsSet; +} +void Rugnux_live_stills::unsetDistance_mm() +{ + m_Distance_mmIsSet = false; +} +int64_t Rugnux_live_stills::getShadowPixels() const +{ + return m_Shadow_pixels; +} +void Rugnux_live_stills::setShadowPixels(int64_t const value) +{ + m_Shadow_pixels = value; + m_Shadow_pixelsIsSet = true; +} +bool Rugnux_live_stills::shadowPixelsIsSet() const +{ + return m_Shadow_pixelsIsSet; +} +void Rugnux_live_stills::unsetShadow_pixels() +{ + m_Shadow_pixelsIsSet = false; +} +int64_t Rugnux_live_stills::getBadPixels() const +{ + return m_Bad_pixels; +} +void Rugnux_live_stills::setBadPixels(int64_t const value) +{ + m_Bad_pixels = value; + m_Bad_pixelsIsSet = true; +} +bool Rugnux_live_stills::badPixelsIsSet() const +{ + return m_Bad_pixelsIsSet; +} +void Rugnux_live_stills::unsetBad_pixels() +{ + m_Bad_pixelsIsSet = false; +} +int64_t Rugnux_live_stills::getImagesAnalysed() const +{ + return m_Images_analysed; +} +void Rugnux_live_stills::setImagesAnalysed(int64_t const value) +{ + m_Images_analysed = value; +} +int64_t Rugnux_live_stills::getImagesIndexed() const +{ + return m_Images_indexed; +} +void Rugnux_live_stills::setImagesIndexed(int64_t const value) +{ + m_Images_indexed = value; +} +double Rugnux_live_stills::getAnalysisRateHz() const +{ + return m_Analysis_rate_hz; +} +void Rugnux_live_stills::setAnalysisRateHz(double const value) +{ + m_Analysis_rate_hz = value; + m_Analysis_rate_hzIsSet = true; +} +bool Rugnux_live_stills::analysisRateHzIsSet() const +{ + return m_Analysis_rate_hzIsSet; +} +void Rugnux_live_stills::unsetAnalysis_rate_hz() +{ + m_Analysis_rate_hzIsSet = false; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_live_stills.h b/broker/gen/model/Rugnux_live_stills.h new file mode 100644 index 000000000..79c5bdacc --- /dev/null +++ b/broker/gen/model/Rugnux_live_stills.h @@ -0,0 +1,189 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.181 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_live_stills.h + * + * Serial frames converted by the FPGA and analysed on the GPU: the pre-pass on the first images (the beam centre, distance and mask the whole collection is analysed with), and the analysis since. + */ + +#ifndef Rugnux_live_stills_H_ +#define Rugnux_live_stills_H_ + + +#include +#include + +namespace org::openapitools::server::model +{ + +/// +/// Serial frames converted by the FPGA and analysed on the GPU: the pre-pass on the first images (the beam centre, distance and mask the whole collection is analysed with), and the analysis since. +/// +class Rugnux_live_stills +{ +public: + Rugnux_live_stills(); + virtual ~Rugnux_live_stills() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_live_stills& rhs) const; + bool operator!=(const Rugnux_live_stills& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_live_stills members + + /// + /// Holding the first images, running on them, or done (Failed - the stated geometry and mask are used) + /// + std::string getPrepass() const; + void setPrepass(std::string const& value); + /// + /// Images held for the pre-pass + /// + int64_t getImagesHeld() const; + void setImagesHeld(int64_t const value); + /// + /// Images the pre-pass runs on + /// + int64_t getImagesNeeded() const; + void setImagesNeeded(int64_t const value); + /// + /// How long the pre-pass took + /// + double getPrepassS() const; + void setPrepassS(double const value); + bool prepassSIsSet() const; + void unsetPrepass_s(); + /// + /// + /// + double getFileBeamXPxl() const; + void setFileBeamXPxl(double const value); + /// + /// + /// + double getFileBeamYPxl() const; + void setFileBeamYPxl(double const value); + /// + /// + /// + double getFileDistanceMm() const; + void setFileDistanceMm(double const value); + /// + /// Beam centre the pre-pass decided (where the beam lands) + /// + double getBeamXPxl() const; + void setBeamXPxl(double const value); + bool beamXPxlIsSet() const; + void unsetBeam_x_pxl(); + /// + /// + /// + double getBeamYPxl() const; + void setBeamYPxl(double const value); + bool beamYPxlIsSet() const; + void unsetBeam_y_pxl(); + /// + /// Detector distance the pre-pass decided + /// + double getDistanceMm() const; + void setDistanceMm(double const value); + bool distanceMmIsSet() const; + void unsetDistance_mm(); + /// + /// Pixels masked as the beam-stop shadow + /// + int64_t getShadowPixels() const; + void setShadowPixels(int64_t const value); + bool shadowPixelsIsSet() const; + void unsetShadow_pixels(); + /// + /// Pixels masked as defective (hot, dead, error value) + /// + int64_t getBadPixels() const; + void setBadPixels(int64_t const value); + bool badPixelsIsSet() const; + void unsetBad_pixels(); + /// + /// + /// + int64_t getImagesAnalysed() const; + void setImagesAnalysed(int64_t const value); + /// + /// + /// + int64_t getImagesIndexed() const; + void setImagesIndexed(int64_t const value); + /// + /// Images analysed per second since the pre-pass + /// + double getAnalysisRateHz() const; + void setAnalysisRateHz(double const value); + bool analysisRateHzIsSet() const; + void unsetAnalysis_rate_hz(); + + friend void to_json(nlohmann::json& j, const Rugnux_live_stills& o); + friend void from_json(const nlohmann::json& j, Rugnux_live_stills& o); +protected: + std::string m_Prepass; + + int64_t m_Images_held; + + int64_t m_Images_needed; + + double m_Prepass_s; + bool m_Prepass_sIsSet; + double m_File_beam_x_pxl; + + double m_File_beam_y_pxl; + + double m_File_distance_mm; + + double m_Beam_x_pxl; + bool m_Beam_x_pxlIsSet; + double m_Beam_y_pxl; + bool m_Beam_y_pxlIsSet; + double m_Distance_mm; + bool m_Distance_mmIsSet; + int64_t m_Shadow_pixels; + bool m_Shadow_pixelsIsSet; + int64_t m_Bad_pixels; + bool m_Bad_pixelsIsSet; + int64_t m_Images_analysed; + + int64_t m_Images_indexed; + + double m_Analysis_rate_hz; + bool m_Analysis_rate_hzIsSet; + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_live_stills_H_ */ diff --git a/broker/gen/model/Rugnux_settings.h b/broker/gen/model/Rugnux_settings.h index 875a71579..c5ed7b904 100644 --- a/broker/gen/model/Rugnux_settings.h +++ b/broker/gen/model/Rugnux_settings.h @@ -12,7 +12,7 @@ /* * Rugnux_settings.h * - * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. + * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. Serial collections converted by the FPGA (JUNGFRAU) are analysed on the GPU after a pre-pass on their first images that decides the beam centre, distance and mask (`stills` in `/rugnux/status`). */ #ifndef Rugnux_settings_H_ @@ -25,7 +25,7 @@ namespace org::openapitools::server::model { /// -/// Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. +/// Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. Serial collections converted by the FPGA (JUNGFRAU) are analysed on the GPU after a pre-pass on their first images that decides the beam centre, distance and mask (`stills` in `/rugnux/status`). /// class Rugnux_settings { @@ -58,7 +58,7 @@ public: /// Rugnux_settings members /// - /// Process DECTRIS rotation collections with rugnux live + /// Process DECTRIS rotation collections with rugnux live, and FPGA serial collections on the GPU after a pre-pass /// bool isEnable() const; void setEnable(bool const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 8db6a50d4..6071f98a3 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -2690,12 +2690,14 @@ components: type: object description: | Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux - inside the broker while they arrive, and the merged result is sent to the writer. + inside the broker while they arrive, and the merged result is sent to the writer. Serial + collections converted by the FPGA (JUNGFRAU) are analysed on the GPU after a pre-pass on their + first images that decides the beam centre, distance and mask (`stills` in `/rugnux/status`). properties: enable: type: boolean default: false - description: Process DECTRIS rotation collections with rugnux live + description: Process DECTRIS rotation collections with rugnux live, and FPGA serial collections on the GPU after a pre-pass reserved_cpus: type: integer format: int64 @@ -2953,6 +2955,75 @@ components: type: number format: double description: Absent where no atom is within 12 Angstrom + rugnux_live_stills: + type: object + description: | + Serial frames converted by the FPGA and analysed on the GPU: the pre-pass on the first images (the beam + centre, distance and mask the whole collection is analysed with), and the analysis since. + required: + - prepass + - images_held + - images_needed + - file_beam_x_pxl + - file_beam_y_pxl + - file_distance_mm + - images_analysed + - images_indexed + properties: + prepass: + type: string + enum: [Holding, Running, Done, Failed] + description: Holding the first images, running on them, or done (Failed - the stated geometry and mask are used) + images_held: + type: integer + format: int64 + description: Images held for the pre-pass + images_needed: + type: integer + format: int64 + description: Images the pre-pass runs on + prepass_s: + type: number + format: double + description: How long the pre-pass took + file_beam_x_pxl: + type: number + format: double + file_beam_y_pxl: + type: number + format: double + file_distance_mm: + type: number + format: double + beam_x_pxl: + type: number + format: double + description: Beam centre the pre-pass decided (where the beam lands) + beam_y_pxl: + type: number + format: double + distance_mm: + type: number + format: double + description: Detector distance the pre-pass decided + shadow_pixels: + type: integer + format: int64 + description: Pixels masked as the beam-stop shadow + bad_pixels: + type: integer + format: int64 + description: Pixels masked as defective (hot, dead, error value) + images_analysed: + type: integer + format: int64 + images_indexed: + type: integer + format: int64 + analysis_rate_hz: + type: number + format: double + description: Images analysed per second since the pre-pass rugnux_live_status: type: object description: | @@ -2999,6 +3070,8 @@ components: description: The merged data carry usable anomalous signal (the summary's "Anomalous" row says so) model: $ref: '#/components/schemas/rugnux_live_model' + stills: + $ref: '#/components/schemas/rugnux_live_stills' space_group: type: string description: Space group of the result (the summary's "Space group" row); absent until the run is over diff --git a/broker/redoc-static.html b/broker/redoc-static.html index f70387820..1a3c98def 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -836,7 +836,7 @@ Changing detector will set detector to Inactive state and will requ

Responses

Response samples

Content type
application/json
{
  • "detectors": [
    ],
  • "current_id": 0
}

Change rugnux live mode settings

This can only be done when detector is Idle, Error or Inactive states.

-
Request Body schema: application/json
enable
boolean
Default: false

Process DECTRIS rotation collections with rugnux live

+
Request Body schema: application/json
enable
boolean
Default: false

Process DECTRIS rotation collections with rugnux live, and FPGA serial collections on the GPU after a pre-pass

reserved_cpus
integer <int64> >= 0
Default: 4

CPU threads left to the receiver and the image pusher while rugnux runs

deadline_s
number <double> >= 0
Default: 60

Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as POST /cancel after the last image does). 0 means no cutoff.

@@ -864,7 +864,7 @@ and validation of every following rugnux live collection, until it is removed wi
Authorizations:
bearerAuth

Responses

Response samples

Content type
application/json
{
  • "state": "Idle",
  • "placed_model": true,
  • "refined_model": true,
  • "anomalous_map": true,
  • "anomalous_signal": true,
  • "model": {
    },
  • "space_group": "string",
  • "d_min": 0.1,
  • "isa": 0.1,
  • "probe": "string",
  • "probe_s": 0.1,
  • "start_to_first_image_s": 0.1,
  • "message": "string",
  • "images_expected": 0,
  • "expected_collection_s": 0.1,
  • "images_received": 0,
  • "elapsed_s": 0.1,
  • "step_progress": 1,
  • "steps": [
    ],
  • "last_image_s": 0.1,
  • "cut": "string",
  • "cut_s": 0.1,
  • "last_image_to_written_s": 0.1,
  • "verdict": "string",
  • "verdict_text": "string",
  • "summary": [
    ],
  • "warnings": [
    ]
}

Get the model as placed by the last rugnux live run

The model placed against the data by molecular replacement or validation, as mmCIF - the same text as the +

http://localhost:5232/rugnux/status

Response samples

Content type
application/json
{
  • "state": "Idle",
  • "placed_model": true,
  • "refined_model": true,
  • "anomalous_map": true,
  • "anomalous_signal": true,
  • "model": {
    },
  • "stills": {
    },
  • "space_group": "string",
  • "d_min": 0.1,
  • "isa": 0.1,
  • "probe": "string",
  • "probe_s": 0.1,
  • "start_to_first_image_s": 0.1,
  • "message": "string",
  • "images_expected": 0,
  • "expected_collection_s": 0.1,
  • "images_received": 0,
  • "elapsed_s": 0.1,
  • "step_progress": 1,
  • "steps": [
    ],
  • "last_image_s": 0.1,
  • "cut": "string",
  • "cut_s": 0.1,
  • "last_image_to_written_s": 0.1,
  • "verdict": "string",
  • "verdict_text": "string",
  • "summary": [
    ],
  • "warnings": [
    ]
}

Get the model as placed by the last rugnux live run

The model placed against the data by molecular replacement or validation, as mmCIF - the same text as the _model.cif file written beside the data: the data's cell and space group, with the input model's own metadata (entities, sequences, references, authors, software) kept. Kept as long as /rugnux/status. Where the run refined the model, the refined model unless asked for stage=placed.

@@ -1123,7 +1123,7 @@ then image might be replaced in the buffer between calling /images and /image.cb