Files
Jungfraujoch/receiver/JFJochReceiverService.cpp
leonarski_f 7f2ebd8209
Build Packages / Build documentation (push) Successful in 46s
Build Packages / Create release (push) Successful in 8s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m25s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 9m46s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 13m25s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 16m41s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 23m46s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 25m37s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 24m31s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 28m9s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 20m47s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 26m21s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 27m36s
Build Packages / Generate python client (push) Successful in 31s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 29m27s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 31m46s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 25m22s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 30m36s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 31m49s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 23m57s
Build Packages / Unit tests (push) Successful in 1h47m52s
Build Packages / Validation Battery (push) Successful in 44m55s
v1.0.0-rc.184 (#95)
* Rugnux: Rotation processing recovers most of the speed lost since 1.0.0-rc.180; live processing in jfjoch_broker finishes sooner.
* Rugnux: More reliable rotation processing and indexing, including small molecules and supercells.
* Rugnux: More robust serial data and model refinement; molecular replacement handles multi-chain models.
* jfjoch_broker (experimental): Live serial processing keeps the detector geometry fixed for the whole run.
* jfjoch_broker: Pixel sum without an FPGA, a mask bit for inactive pixels, and several stability fixes.
* jfjoch_broker: Rotation indexing and quick integration moved to rugnux live mode; their API settings are deprecated and ignored.
* jfjoch_broker, Rugnux: Azimuthal integration can use logarithmic (log10) Q bins or a given number of bins, set from the API, the web interface and the Rugnux command line.
* jfjoch_viewer: Build and display fixes; the README is rewritten.

Reviewed-on: #95
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-11-10 22:06:51 +01:00

464 lines
20 KiB
C++

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "JFJochReceiverService.h"
#include "JFJochReceiverRugnuxRotation.h"
#include "JFJochReceiverRugnuxStills.h"
#include "../rugnux/RugnuxDefaults.h"
#include "../common/ThreadAffinity.h"
#include "JFJochReceiverFPGA.h"
#include "JFJochReceiverLite.h"
#include "JFJochReceiverDarkMask.h"
#include "JFJochReceiverPedestal.h"
#include "../preview/JFJochJPEG.h"
#include "../preview/JFJochTIFF.h"
JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devices,
Logger &in_logger, ImagePusher &pusher,
size_t send_buffer_size_MiB)
: aq_devices(in_aq_devices),
logger(in_logger),
image_buffer(send_buffer_size_MiB * 1024 * 1024),
image_pusher(pusher),
spot_finding_settings(DiffractionExperiment::DefaultDataProcessingSettings()) {
}
JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
int reserved_cpus, double deadline_s) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change rugnux settings during data collection");
rugnux_enable = enable;
rugnux_model = std::move(model);
rugnux_reserved_cpus = reserved_cpus;
rugnux_deadline_s = deadline_s;
return *this;
}
JFJochReceiverService &JFJochReceiverService::UseFPGAImagePuller(bool enable, size_t slots) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change the receiver path during data collection");
fpga_image_puller = enable;
fpga_image_puller_slots = slots;
return *this;
}
RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const {
return rugnux_status.Get();
}
JFJochReceiverService &JFJochReceiverService::NumThreads(int64_t input) {
if (input <= 0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Thread number must be above zero");
nthreads = input;
return *this;
}
void JFJochReceiverService::FinalizeMeasurementChangeState() {
std::unique_lock ul(state_mutex);
state = ReceiverState::Idle;
measurement_done.notify_all();
}
void JFJochReceiverService::FinalizeMeasurement() {
try {
receiver->StopReceiver();
if (fpga_puller)
fpga_puller->Stop();
} catch (...) {
FinalizeMeasurementChangeState();
throw;
}
FinalizeMeasurementChangeState();
}
std::optional<JFJochReceiverStatus> JFJochReceiverService::GetStatus() {
return receiver_status.GetStatus();
}
void JFJochReceiverService::Preflight(const DiffractionExperiment &experiment) {
// As JFJochReceiver decides whether it pushes to a writer at all
if ((experiment.GetImageNum() <= 0) || experiment.GetFilePrefix().empty())
return;
// FillMessage alone, so this frame is small: the per-pixel arrays are added by JFJochReceiver::SendStartMessage
StartMessage message{};
experiment.FillMessage(message);
image_pusher.Preflight(message);
}
void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
const PixelMask &pixel_mask,
const JFCalibration *calibration,
std::shared_ptr<ImagePuller> puller) {
std::unique_lock ul_state(state_mutex); // unique lock, as it will destroy and create receiver object
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Receiver not idle, cannot start");
try {
// First clean-up old measurement
receiver.reset();
analysis_warning.clear();
if (fpga_puller) {
image_puller.reset();
fpga_puller.reset();
}
preview_image.Configure(experiment, pixel_mask);
// Pedestal runs go to JFJochReceiverPedestal, host summation stays with JFJochReceiverFPGA
bool stream_input = (experiment.GetDetectorType() == DetectorType::DECTRIS);
if (fpga_image_puller && !stream_input && (experiment.GetImageNum() > 0)) {
if (experiment.IsCPUSummation())
logger.Warning("Summation above what the card can do needs JFJochReceiverFPGA: FPGA image puller not used");
else {
fpga_puller = std::make_shared<FPGAImagePuller>(experiment, pixel_mask, calibration, aq_devices, logger,
nthreads, fpga_image_puller_slots);
puller = fpga_puller;
stream_input = true;
}
}
if (!stream_input && experiment.IsPedestalRun()) {
receiver = std::make_unique<JFJochReceiverPedestal>(experiment, pixel_mask,
aq_devices, image_pusher,
logger,
spot_finding_settings,
receiver_status,
plots,
image_buffer);
} else if (!stream_input) {
// Summation on the host runs threads of its own for each image
auto fpga_nthreads = nthreads;
if (experiment.IsCPUSummation())
fpga_nthreads = 4;
receiver = std::make_unique<JFJochReceiverFPGA>(experiment, pixel_mask,
calibration,
aq_devices, image_pusher,
logger,
fpga_nthreads,
spot_finding_settings,
receiver_status,
plots,
image_buffer,
zmq_preview_socket.get(),
zmq_metadata_socket.get(),
indexer_thread_pool.get());
} else {
if (puller)
image_puller = puller;
else
image_puller = std::make_shared<ZMQImagePuller>(experiment.GetDetectorSetup().GetDECTRISStream2Addr());
bool rugnux_rotation = rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard)
&& RugnuxRotationData(experiment);
// Rugnux reads the images back from the buffer: a collection that does not fit gets the per-image analysis instead
const int64_t slots = image_buffer.GetBufferSize() / experiment.GetImageBufferLocationSize();
if (rugnux_rotation && (slots < experiment.GetImageNum())) {
analysis_warning = fmt::format("no rugnux analysis: the image buffer holds {} images, the collection "
"has {}", slots, experiment.GetImageNum());
logger.Warning("{}", analysis_warning);
rugnux_status.Start(experiment.GetImageNum(), {}, std::chrono::steady_clock::now());
rugnux_status.Finish(RugnuxLiveState::Skipped, "the image buffer cannot hold the whole collection");
rugnux_rotation = false;
}
// The dark frames of the mask sequence have a receiver of their own, with rugnux on as well.
if (experiment.GetDetectorMode() == DetectorMode::DarkMask)
receiver = std::make_unique<JFJochReceiverDarkMask>(experiment,
pixel_mask,
*image_puller,
image_pusher,
logger,
nthreads,
spot_finding_settings,
receiver_status,
plots,
image_buffer);
else if (rugnux_rotation)
receiver = std::make_unique<JFJochReceiverRugnuxRotation>(
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
std::max<int>(1, static_cast<int>(AvailableCpus()) - rugnux_reserved_cpus),
rugnux_model, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots,
image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get());
else if (rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard)
&& !RugnuxRotationData(experiment))
receiver = std::make_unique<JFJochReceiverRugnuxStills>(
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
rugnux_model, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots,
image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(), indexer_thread_pool.get());
else
receiver = std::make_unique<JFJochReceiverLite>(experiment,
pixel_mask,
*image_puller,
image_pusher,
logger,
nthreads,
spot_finding_settings,
receiver_status,
plots,
image_buffer,
zmq_preview_socket.get(),
zmq_metadata_socket.get(),
indexer_thread_pool.get());
}
measurement = std::async(std::launch::async, &JFJochReceiverService::FinalizeMeasurement, this);
state = ReceiverState::Running;
} catch (const std::exception &e) {
receiver.reset(); // its threads may be polling the puller reset below
// The receiver never started: /status must not keep reporting a run that never happened
receiver_status.Clear();
receiver_status.SetProgress({});
if (fpga_puller) {
image_puller.reset();
fpga_puller.reset();
}
logger.ErrorException(e);
throw;
}
}
void JFJochReceiverService::Cancel(bool silent) {
std::unique_lock ul(state_mutex);
if (state == ReceiverState::Running) {
receiver->Cancel(silent);
if (fpga_puller)
fpga_puller->Cancel();
}
}
JFJochReceiverOutput JFJochReceiverService::Stop() {
std::unique_lock ul(state_mutex);
measurement_done.wait(ul, [this] { return (state != ReceiverState::Running); });
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongReceiverState, "Receiver in weird state");
try {
if (measurement.valid())
measurement.get();
} catch (JFJochException &e) {
logger.ErrorException(e);
throw;
}
if (!receiver) {
logger.Warning("Request to stop while receiver not running");
throw JFJochException(JFJochExceptionCategory::WrongReceiverState, "Receiver idle, cannot stop");
}
auto ret = receiver->GetFinalStatistics();
if (!analysis_warning.empty())
ret.analysis_warning = analysis_warning;
return ret;
}
void JFJochReceiverService::SetSpotFindingSettings(const SpotFindingSettings &settings) {
try {
std::unique_lock ul(state_mutex);
DiffractionExperiment::CheckDataProcessingSettings(settings);
spot_finding_settings = settings;
if (state != ReceiverState::Idle)
receiver->SetSpotFindingSettings(settings);
} catch (std::exception &e) {
logger.ErrorException(e);
throw;
}
}
MultiLinePlot JFJochReceiverService::GetDataProcessingPlot(const PlotRequest &request) {
return plots.GetPlots(request);
}
void JFJochReceiverService::GetPlotRaw(std::vector<float> &v, PlotType type, const std::string &roi) {
plots.GetPlotRaw(v, type, roi);
}
std::vector<AcquisitionDeviceNetConfig> JFJochReceiverService::GetNetworkConfig() {
return aq_devices.GetNetworkConfig();
}
void JFJochReceiverService::LoadInternalGeneratorImage(const DiffractionExperiment &experiment,
const std::vector<uint16_t> &image,
uint64_t image_number) {
std::vector<uint16_t> raw_geom, eiger_geom;
const uint16_t *frame;
if (image.size() == RAW_MODULE_SIZE * experiment.GetModulesNum()) {
frame = image.data();
} else if (image.size() == experiment.GetPixelsNum()) {
raw_geom.resize(RAW_MODULE_SIZE * experiment.GetModulesNum());
ConvertedToRawGeometry(experiment, raw_geom.data(), image.data());
frame = raw_geom.data();
} else
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Size of input array with raw expected image is wrong");
for (int i = 0; i < experiment.GetDataStreamsNum(); i++) {
uint32_t module0 = experiment.GetFirstModuleOfDataStream(i);
switch (experiment.GetDetectorSetup().GetDetectorType()) {
case DetectorType::EIGER:
eiger_geom.resize(RAW_MODULE_SIZE);
for (int m = 0; m < experiment.GetModulesNum(i); m++) {
RawToEigerInput(eiger_geom.data(), frame + (module0 + m) * RAW_MODULE_SIZE);
aq_devices[i].SetInternalGeneratorFrame(eiger_geom.data(),
m + experiment.GetModulesNum(i) * image_number);
}
break;
case DetectorType::JUNGFRAU:
for (int m = 0; m < experiment.GetModulesNum(i); m++)
aq_devices[i].SetInternalGeneratorFrame(frame + (module0 + m) * RAW_MODULE_SIZE,
m + experiment.GetModulesNum(i) * image_number);
break;
default:
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Detector not supported");
}
}
}
void JFJochReceiverService::GetXFELEventCode(std::vector<uint64_t> &v) const {
plots.GetXFELEventCode(v);
}
void JFJochReceiverService::GetXFELPulseID(std::vector<uint64_t> &v) const {
plots.GetXFELPulseID(v);
}
std::vector<DeviceStatus> JFJochReceiverService::GetDeviceStatus() const {
return aq_devices.GetDeviceStatus();
}
std::optional<float> JFJochReceiverService::GetProgress() const {
return receiver_status.GetProgress();
}
JFJochReceiverService &JFJochReceiverService::PreviewSocket(const std::string &addr, const std::optional<int32_t> &watermark) {
if (!addr.empty()) {
logger.Info("ZeroMQ preview socket available at {}", addr);
zmq_preview_socket = std::make_unique<ZMQPreviewSocket>(addr, watermark);
}
return *this;
}
JFJochReceiverService &JFJochReceiverService::MetadataSocket(const std::string &addr) {
if (!addr.empty()) {
logger.Info("ZeroMQ metadata socket available at {}", addr);
zmq_metadata_socket = std::make_unique<ZMQMetadataSocket>(addr);
}
return *this;
}
std::string JFJochReceiverService::GetPreviewSocketAddress() const {
if (zmq_preview_socket)
return zmq_preview_socket->GetAddress();
return "";
}
std::string JFJochReceiverService::GetMetadataSocketAddress() const {
if (zmq_metadata_socket)
return zmq_metadata_socket->GetAddress();
return "";
}
JFJochReceiverService &JFJochReceiverService::PreviewSocketSettings(const ZMQPreviewSettings &input) {
if (zmq_preview_socket)
zmq_preview_socket->ImportSettings(input);
return *this;
}
JFJochReceiverService &JFJochReceiverService::MetadataSocketSettings(const ZMQMetadataSettings &input) {
if (zmq_metadata_socket)
zmq_metadata_socket->ImportSettings(input);
return *this;
}
ZMQPreviewSettings JFJochReceiverService::GetPreviewSocketSettings() const {
if (zmq_preview_socket)
return zmq_preview_socket->GetSettings();
return {};
}
ZMQMetadataSettings JFJochReceiverService::GetMetadataSocketSettings() const {
if (zmq_metadata_socket)
return zmq_metadata_socket->GetSettings();
return {};
}
void JFJochReceiverService::GetStartMessageFromBuffer(std::vector<uint8_t> &v) {
image_buffer.GetStartMessage(v);
}
bool JFJochReceiverService::GetImageFromBuffer(std::vector<uint8_t> &v, int64_t image_number) {
return image_buffer.GetImage(v, image_number);
}
std::string JFJochReceiverService::GetJPEGFromBuffer(const PreviewImageSettings &settings, int64_t image_number) {
std::vector<uint8_t> cbor_image;
if (!image_buffer.GetImage(cbor_image, image_number))
return {};
return preview_image.GenerateImage(settings, cbor_image);
}
std::string JFJochReceiverService::GetTIFFFromBuffer(int64_t image_number) {
std::vector<uint8_t> cbor_image;
if (!image_buffer.GetImage(cbor_image, image_number))
return {};
return PreviewImage::GenerateTIFF(cbor_image);
}
ImageBufferStatus JFJochReceiverService::GetImageBufferStatus() const {
return image_buffer.GetStatus();
}
void JFJochReceiverService::ClearResults() {
std::unique_lock ul(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot clear results during data collection");
receiver_status.Clear();
receiver_status.SetProgress({});
plots.Clear();
rugnux_status.Clear();
image_buffer.Finalize(std::chrono::milliseconds(2500));
}
void JFJochReceiverService::ClearImageBuffer() {
std::unique_lock ul(state_mutex);
// Clearing image buffer during data collection could be catastrophic, so better protect here
if (state == ReceiverState::Idle)
image_buffer.Finalize(std::chrono::milliseconds(2500));
else
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot clear image buffer during data collection");
}
JFJochReceiverService &JFJochReceiverService::Indexing(const IndexingSettings &input) {
std::unique_lock ul(state_mutex);
if (state == ReceiverState::Idle) {
// The receiver goes first: it holds a raw pointer to the pool, and a stale one's GPU resources can fail the new indexers
receiver.reset();
logger.Info("Resetting indexing thread pool");
indexer_thread_pool.reset();
ReleaseKeptFFTIndexers();
if (input.GetAlgorithm() != IndexingAlgorithmEnum::None) {
logger.Info("Creating indexing thread pool...");
indexer_thread_pool = std::make_unique<IndexerThreadPool>(input);
logger.Info(" ... done");
}
return *this;
} else
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change indexing settings during data collection");
}
ImagePusherStatus JFJochReceiverService::GetImagePusherStatus() const {
return image_pusher.GetStatus();
}