Files
leonarski_f aecd48b9d3
Build Packages / Build documentation (push) Successful in 48s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 15m47s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 23m19s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 18m47s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m54s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 11m27s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 26m59s
Build Packages / Unit tests (push) Successful in 1h50m25s
Build Packages / Validation Battery (push) Successful in 46m38s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 32m52s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 32m32s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 28m34s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 29m19s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 30m11s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 33m58s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 31m26s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 34m4s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 32m17s
Build Packages / Generate python client (push) Successful in 32s
Build Packages / Create release (push) Successful in 9s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 31m50s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 19m38s
v1.0.0-rc.186 (#97)
* jfjoch_broker (experimental): Live Rugnux covers serial collections from DECTRIS detectors; a rotation collection too large for the image buffer falls back to standard per-image processing with a warning, and a stream that ends without its end message finishes the collection instead of hanging it.
* jfjoch_broker (experimental): Live Rugnux settings to build the solvent during refinement and to process a rotation collection as stills, a Clear action for the last result, a `_report.txt` for live rotation runs, R-free reported for the refined model, and ice-ring detection on by default for rotation as in the command line.
* jfjoch_broker (experimental): More complete and reproducible live serial merges, analysed for twinning, pseudo-translation and anisotropy, with merging statistics by resolution shell in the live status.
* jfjoch_writer: Live results with large models are written reliably; update jfjoch_writer together with jfjoch_broker.
* Rugnux: Several masters can be processed together with `--force-still`, including grid scans; more robust and faster merging of several sweeps; improved twinning, pseudo-translation, anisotropy, anomalous-signal and space-group decisions.
* Rugnux: Molecular replacement places the copies of a multi-copy solution together; the pseudo-translation warning quotes the Patterson peak and is not raised once a model is placed; the report advises on extending the rotation or reorienting the crystal where the coverage would gain from it.
* Web frontend: Light and dark modes, a phone layout, a hideable page list, a redesigned Rugnux live page with step-through of the Fo-Fc peaks in the structure view.
* jfjoch_viewer: A broker URL given on the command line follows the running collection.

Reviewed-on: #97
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-11-13 20:13:43 +01:00

338 lines
14 KiB
C++

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "JFJochReceiver.h"
#include "../common/time_utc.h"
#include "../frame_serialize/CBORStream2Serializer.h"
#include "../image_analysis/IndexAndRefine.h"
JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment,
ImageBuffer &in_image_buffer,
ImagePusher &in_image_pusher,
JFJochReceiverCurrentStatus &in_current_status,
JFJochReceiverPlots &in_plots,
const SpotFindingSettings &spot_finding_settings,
Logger &logger,
const PixelMask &in_pixel_mask,
ZMQPreviewSocket *in_zmq_preview_socket,
ZMQMetadataSocket *in_zmq_metadata_socket)
: logger(logger),
experiment(in_experiment),
spot_finding_settings(spot_finding_settings),
image_buffer(in_image_buffer),
image_pusher(in_image_pusher),
zmq_preview_socket(in_zmq_preview_socket),
zmq_metadata_socket(in_zmq_metadata_socket),
current_status(in_current_status),
plots(in_plots),
scan_result(in_experiment),
serialmx_filter(in_experiment),
pixel_mask(in_pixel_mask) {
logger.Debug("Initializing receiver");
// Ensure there is nothing running for now
if (!image_buffer.Finalize(std::chrono::seconds(1)))
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"There are unfinished preview/sending jobs in the buffer");
logger.Debug("Image buffer from previous run finalized");
current_status.SetProgress(0);
current_status.SetStatus(JFJochReceiverStatus{}); // GetStatus() calls virtual functions and cannot be called yet
auto start_time_point = std::chrono::steady_clock::now();
az_int_mapping = std::make_unique<AzimuthalIntegrationMapping>(experiment, pixel_mask);
auto end_time_point = std::chrono::steady_clock::now();
auto duration = std::chrono::duration<float>(end_time_point - start_time_point);
logger.Debug("Azimuthal integration mapping done in {:.5f} s with {} threads", duration.count(), az_int_mapping->GetNThreads());
plots.Setup(experiment, *az_int_mapping);
push_images_to_writer = (experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty());
}
JFJochReceiver::~JFJochReceiver() = default;
SpotFindingSettings JFJochReceiver::GetSpotFindingSettings() {
std::unique_lock ul(spot_finding_settings_mutex);
return spot_finding_settings;
}
void JFJochReceiver::SetSpotFindingSettings(const SpotFindingSettings &in_spot_finding_settings) {
std::unique_lock ul(spot_finding_settings_mutex);
DiffractionExperiment::CheckDataProcessingSettings(in_spot_finding_settings);
spot_finding_settings = in_spot_finding_settings;
}
void JFJochReceiver::UpdateMaxImageSent(int64_t image_number) {
std::unique_lock ul(max_image_number_sent_mutex);
if (image_number + 1 > max_image_number_sent)
max_image_number_sent = image_number + 1;
}
void JFJochReceiver::UpdateMaxImageReceived(int64_t image_number) {
std::unique_lock ul(max_image_number_received_mutex);
if (image_number + 1 > max_image_number_received)
max_image_number_received = image_number + 1;
}
void JFJochReceiver::UpdateMaxDelay(uint64_t delay) {
std::unique_lock ul(max_delay_mutex);
if (!max_delay || (delay > max_delay))
max_delay = delay;
}
JFJochReceiverStatus JFJochReceiver::GetStatus() const {
JFJochReceiverStatus ret;
ret.indexing_rate = plots.GetIndexingRate();
ret.bkg_estimate = plots.GetBkgEstimate();
if ((experiment.GetImageNum() > 0) && (compressed_size > 0)) {
ret.compressed_ratio = static_cast<double>(uncompressed_size) / static_cast<double>(compressed_size);
}
ret.saturated_pixels = saturated_pixels.Read();
ret.error_pixels = error_pixels.Read();
ret.roi_beam_npixel = roi_beam_npixel.Read();
ret.roi_beam_sum = roi_beam_sum.Read();
ret.compressed_size = compressed_size;
ret.max_receive_delay = max_delay;
ret.max_image_number_sent = max_image_number_sent;
ret.images_collected = images_collected;
ret.images_sent = images_sent;
ret.images_skipped = images_skipped;
ret.images_dropped = GetImagesDropped();
ret.images_written = image_pusher.GetImagesWritten();
ret.cancelled = cancelled;
ret.efficiency = GetEfficiency();
return ret;
}
StartMessage JFJochReceiver::MakeStartMessage(const DiffractionExperiment &in_experiment, const PixelMask &mask,
const AzimuthalIntegrationMapping &mapping) {
StartMessage message{};
in_experiment.FillMessage(message);
message.arm_date = time_UTC(std::chrono::system_clock::now());
message.az_int_q_bin_count = mapping.GetQBinCount();
message.az_int_bin_to_q = mapping.GetBinToQ();
message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta();
message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
if (mapping.GetAzimuthalBinCount() > 1) {
message.az_int_bin_to_phi = mapping.GetBinToPhi();
message.az_int_map = mapping.GetPixelToBin();
}
message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress();
message.rois = in_experiment.ROI().ExportMetadata();
if (!in_experiment.ROI().empty())
message.roi_map = in_experiment.ExportROIMap();
message.max_spot_count = in_experiment.GetMaxSpotCount();
message.pixel_mask["default"] = mask.GetMask(in_experiment);
return message;
}
void JFJochReceiver::SendStartMessage() {
SendStartMessage(MakeStartMessage(experiment, pixel_mask, *az_int_mapping));
}
void JFJochReceiver::SendStartMessage(StartMessage message) {
SaveStartMessageToImageBuffer(message);
if (push_images_to_writer)
image_pusher.StartDataCollection(message);
if (zmq_preview_socket != nullptr)
zmq_preview_socket->StartDataCollection(message);
if (zmq_metadata_socket != nullptr)
zmq_metadata_socket->StartDataCollection(message);
}
void JFJochReceiver::SaveStartMessageToImageBuffer(const StartMessage &msg) {
std::vector<uint8_t> buffer(MESSAGE_SIZE_FOR_START_END);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
serializer.SerializeSequenceStart(msg);
buffer.resize(serializer.GetBufferSize());
image_buffer.SaveStartMessage(buffer);
}
// A frame whose metadata does not fit its slot is dropped, without aborting the collection
bool JFJochReceiver::SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg,
ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger) {
try {
serializer.SerializeImage(msg);
return true;
} catch (const JFJochException &) {
// The metadata alone fits if it is the image that overflowed; if not, the exception propagates
const auto image = msg.image;
msg.image = CompressedImage(nullptr, 0, image.GetWidth(), image.GetHeight(),
image.GetMode(), image.GetCompressionAlgorithm(),
image.GetChannel());
CBORStream2Serializer meas(static_cast<uint8_t *>(loc.GetImage()), slot_size);
meas.SerializeImage(msg);
const size_t metadata_size = meas.GetImageAppendOffset();
msg.image = image;
logger.Error("Dropping image {} - CBOR metadata too large to store image: "
"metadata {} B + image {} B do not fit in buffer slot {} B",
msg.number, metadata_size, image.GetCompressedSize(), slot_size);
loc.release();
return false;
}
}
void JFJochReceiver::CountImage(const DataMessage &message) {
++images_collected;
collection_efficiency_sum += message.image_collection_efficiency.value_or(1.0f);
compressed_size += message.image.GetCompressedSize();
uncompressed_size += message.image.GetUncompressedSize();
UpdateMaxImageReceived(message.number);
}
void JFJochReceiver::StampImage(DataMessage &message, size_t fifo_delay) {
message.original_number = message.number;
message.user_data = experiment.GetImageAppendix();
message.run_number = experiment.GetRunNumber();
message.run_name = experiment.GetRunName();
message.receiver_buf_available = image_buffer.GetAvailSlots();
message.receiver_aq_dev_delay = fifo_delay;
// Uncompressed/compressed (e.g. 7x), as JFJochReceiverFPGA and JFJochReceiver::GetStatus
if (message.image.GetCompressedSize() > 0)
message.compression_ratio = static_cast<float>(message.image.GetUncompressedSize())
/ static_cast<float>(message.image.GetCompressedSize());
}
void JFJochReceiver::AddToAverages(const DataMessage &message) {
// A stream that does not state it (DECTRIS) delivers whole images
if (message.image_collection_efficiency.value_or(1.0f) != 1.0f)
return;
saturated_pixels.Add(message.saturated_pixel_count);
error_pixels.Add(message.error_pixel_count);
if (message.roi.contains("beam")) {
roi_beam_npixel.Add(message.roi.at("beam").pixels);
roi_beam_sum.Add(message.roi.at("beam").sum);
}
}
bool JFJochReceiver::ForwardImage(DataMessage &message, ZeroCopyReturnValue *loc) {
if (loc == nullptr) {
writer_queue_full = true;
return false;
}
auto writer_buffer = (uint8_t *) loc->GetImage();
CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize());
if (!SerializeImageOrDrop(serializer, message, *loc, experiment.GetImageBufferLocationSize(), logger))
return false;
loc->SetImageNumber(message.number);
loc->SetImageSize(serializer.GetBufferSize());
loc->SetIndexed(message.indexing_result.value_or(false));
loc->ReadyToSend();
if (zmq_preview_socket != nullptr)
zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize());
if (zmq_metadata_socket != nullptr)
zmq_metadata_socket->AddDataMessage(message);
if (push_images_to_writer) {
// Only count images the pusher accepted; TCP refuses them on a broken/absent connection
if (image_pusher.SendImage(*loc))
++images_sent;
} else
loc->release();
UpdateMaxImageSent(message.number);
return true;
}
void JFJochReceiver::SendEndMessage() {
EndMessage message{};
message.max_image_number = max_image_number_sent;
message.images_collected_count = images_collected;
message.images_sent_to_write_count = images_sent;
message.max_receiver_delay = max_delay;
message.efficiency = GetEfficiency();
message.end_date = time_UTC(std::chrono::system_clock::now());
message.run_number = experiment.GetRunNumber();
message.run_name = experiment.GetRunName();
message.bkg_estimate = plots.GetBkgEstimate();
message.spindle_blind_fraction = plots.GetSpindleBlindFraction();
message.ice_ring_score_mean = plots.GetIceRingScore();
message.indexing_rate = plots.GetIndexingRate();
message.az_int_result["dataset"] = plots.GetAzIntProfile();
for (int i = 0; i < adu_histogram_module.size(); i++)
message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram();
scan_result.FillEndMessage(message);
AddToEndMessage(message);
if (push_images_to_writer) {
if (!image_pusher.EndDataCollection(message))
logger.Error("End message not sent via ZeroMQ (time-out)");
logger.Debug("Disconnected from writers");
}
// The result blocks and the report (AddToEndMessage) are only for the writer.
message.merged.reset();
message.map_coefficients.reset();
message.placed_model.reset();
message.refined_model.reset();
message.refined_map_coefficients.reset();
message.blobs.reset();
message.report.reset();
if (zmq_metadata_socket != nullptr)
zmq_metadata_socket->EndDataCollection(message);
if (zmq_preview_socket != nullptr)
zmq_preview_socket->EndDataCollection(message);
}
void JFJochReceiver::AddIndexingToEndMessage(EndMessage &message, const IndexAndRefine &indexer) {
// A given cell is what the master file records; the consensus of the refined cells only stands in
// for one that was not given.
if (!experiment.GetUnitCell())
message.unit_cell = indexer.GetConsensusUnitCell();
// The images are written regardless, but the indexing rate is then not a statement about the crystal
if (const int64_t error_count = indexer.GetIndexerErrorCount(); error_count > 0) {
try {
indexer.RethrowIndexerError();
} catch (const std::exception &e) {
logger.Error("Indexing failed {} times in this collection, first with: {}. Those attempts are "
"left out of the indexing rate", error_count, e.what());
}
}
}
JFJochReceiverOutput JFJochReceiver::GetFinalStatistics() const {
JFJochReceiverOutput ret;
ret.efficiency = GetEfficiency();
ret.start_time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(start_time.time_since_epoch()).count();
ret.end_time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_time.time_since_epoch()).count();
ret.writer_queue_full_warning = writer_queue_full;
ret.status = GetStatus();
ret.writer_err = writer_error;
ret.scan_result = scan_result.GetResult();
ret.processing_time = plots.GetMeanProcessingTime();
return ret;
}
void JFJochReceiver::Cancel(bool silent) {
if (!silent) {
// Remote abort: This tells FPGAs to stop but doesn't do anything to CPU code
logger.Warning("Cancelling on request");
cancelled = true;
}
}
void JFJochReceiver::Cancel(const JFJochException &e) {
logger.Error("Cancelling data collection due to exception");
logger.ErrorException(e);
// Error abort: This tells FPGAs to stop and also prevents deadlock in CPU code by setting abort to 1
cancelled = true;
}