Build Packages / Build documentation (push) Successful in 46s
Build Packages / Create release (push) Successful in 8s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m25s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 9m46s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 13m25s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 16m41s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 23m46s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 25m37s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 24m31s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 28m9s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 20m47s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 26m21s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 27m36s
Build Packages / Generate python client (push) Successful in 31s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 29m27s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 31m46s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 25m22s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 30m36s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 31m49s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 23m57s
Build Packages / Unit tests (push) Successful in 1h47m52s
Build Packages / Validation Battery (push) Successful in 44m55s
* Rugnux: Rotation processing recovers most of the speed lost since 1.0.0-rc.180; live processing in jfjoch_broker finishes sooner. * Rugnux: More reliable rotation processing and indexing, including small molecules and supercells. * Rugnux: More robust serial data and model refinement; molecular replacement handles multi-chain models. * jfjoch_broker (experimental): Live serial processing keeps the detector geometry fixed for the whole run. * jfjoch_broker: Pixel sum without an FPGA, a mask bit for inactive pixels, and several stability fixes. * jfjoch_broker: Rotation indexing and quick integration moved to rugnux live mode; their API settings are deprecated and ignored. * jfjoch_broker, Rugnux: Azimuthal integration can use logarithmic (log10) Q bins or a given number of bins, set from the API, the web interface and the Rugnux command line. * jfjoch_viewer: Build and display fixes; the README is rewritten. Reviewed-on: #95 Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
334 lines
14 KiB
C++
334 lines
14 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "JFJochReceiver.h"
|
|
#include "../common/time_utc.h"
|
|
#include "../frame_serialize/CBORStream2Serializer.h"
|
|
#include "../image_analysis/IndexAndRefine.h"
|
|
|
|
JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment,
|
|
ImageBuffer &in_image_buffer,
|
|
ImagePusher &in_image_pusher,
|
|
JFJochReceiverCurrentStatus &in_current_status,
|
|
JFJochReceiverPlots &in_plots,
|
|
const SpotFindingSettings &spot_finding_settings,
|
|
Logger &logger,
|
|
const PixelMask &in_pixel_mask,
|
|
ZMQPreviewSocket *in_zmq_preview_socket,
|
|
ZMQMetadataSocket *in_zmq_metadata_socket)
|
|
: logger(logger),
|
|
experiment(in_experiment),
|
|
spot_finding_settings(spot_finding_settings),
|
|
image_buffer(in_image_buffer),
|
|
image_pusher(in_image_pusher),
|
|
zmq_preview_socket(in_zmq_preview_socket),
|
|
zmq_metadata_socket(in_zmq_metadata_socket),
|
|
current_status(in_current_status),
|
|
plots(in_plots),
|
|
scan_result(in_experiment),
|
|
serialmx_filter(in_experiment),
|
|
pixel_mask(in_pixel_mask) {
|
|
logger.Debug("Initializing receiver");
|
|
// Ensure there is nothing running for now
|
|
if (!image_buffer.Finalize(std::chrono::seconds(1)))
|
|
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
|
|
"There are unfinished preview/sending jobs in the buffer");
|
|
logger.Debug("Image buffer from previous run finalized");
|
|
|
|
current_status.SetProgress(0);
|
|
current_status.SetStatus(JFJochReceiverStatus{}); // GetStatus() calls virtual functions and cannot be called yet
|
|
|
|
auto start_time_point = std::chrono::steady_clock::now();
|
|
az_int_mapping = std::make_unique<AzimuthalIntegrationMapping>(experiment, pixel_mask);
|
|
auto end_time_point = std::chrono::steady_clock::now();
|
|
auto duration = std::chrono::duration<float>(end_time_point - start_time_point);
|
|
logger.Debug("Azimuthal integration mapping done in {:.5f} s with {} threads", duration.count(), az_int_mapping->GetNThreads());
|
|
|
|
plots.Setup(experiment, *az_int_mapping);
|
|
|
|
push_images_to_writer = (experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty());
|
|
}
|
|
|
|
JFJochReceiver::~JFJochReceiver() = default;
|
|
|
|
SpotFindingSettings JFJochReceiver::GetSpotFindingSettings() {
|
|
std::unique_lock ul(spot_finding_settings_mutex);
|
|
return spot_finding_settings;
|
|
}
|
|
|
|
void JFJochReceiver::SetSpotFindingSettings(const SpotFindingSettings &in_spot_finding_settings) {
|
|
std::unique_lock ul(spot_finding_settings_mutex);
|
|
DiffractionExperiment::CheckDataProcessingSettings(in_spot_finding_settings);
|
|
spot_finding_settings = in_spot_finding_settings;
|
|
}
|
|
|
|
void JFJochReceiver::UpdateMaxImageSent(int64_t image_number) {
|
|
std::unique_lock ul(max_image_number_sent_mutex);
|
|
if (image_number + 1 > max_image_number_sent)
|
|
max_image_number_sent = image_number + 1;
|
|
}
|
|
|
|
void JFJochReceiver::UpdateMaxImageReceived(int64_t image_number) {
|
|
std::unique_lock ul(max_image_number_received_mutex);
|
|
if (image_number + 1 > max_image_number_received)
|
|
max_image_number_received = image_number + 1;
|
|
}
|
|
|
|
void JFJochReceiver::UpdateMaxDelay(uint64_t delay) {
|
|
std::unique_lock ul(max_delay_mutex);
|
|
if (!max_delay || (delay > max_delay))
|
|
max_delay = delay;
|
|
}
|
|
|
|
JFJochReceiverStatus JFJochReceiver::GetStatus() const {
|
|
JFJochReceiverStatus ret;
|
|
|
|
ret.indexing_rate = plots.GetIndexingRate();
|
|
ret.bkg_estimate = plots.GetBkgEstimate();
|
|
|
|
if ((experiment.GetImageNum() > 0) && (compressed_size > 0)) {
|
|
ret.compressed_ratio = static_cast<double>(uncompressed_size) / static_cast<double>(compressed_size);
|
|
}
|
|
|
|
ret.saturated_pixels = saturated_pixels.Read();
|
|
ret.error_pixels = error_pixels.Read();
|
|
ret.roi_beam_npixel = roi_beam_npixel.Read();
|
|
ret.roi_beam_sum = roi_beam_sum.Read();
|
|
|
|
ret.compressed_size = compressed_size;
|
|
ret.max_receive_delay = max_delay;
|
|
ret.max_image_number_sent = max_image_number_sent;
|
|
ret.images_collected = images_collected;
|
|
ret.images_sent = images_sent;
|
|
ret.images_skipped = images_skipped;
|
|
ret.images_dropped = GetImagesDropped();
|
|
ret.images_written = image_pusher.GetImagesWritten();
|
|
ret.cancelled = cancelled;
|
|
ret.efficiency = GetEfficiency();
|
|
return ret;
|
|
}
|
|
|
|
|
|
StartMessage JFJochReceiver::MakeStartMessage(const DiffractionExperiment &in_experiment, const PixelMask &mask,
|
|
const AzimuthalIntegrationMapping &mapping) {
|
|
StartMessage message{};
|
|
in_experiment.FillMessage(message);
|
|
message.arm_date = time_UTC(std::chrono::system_clock::now());
|
|
message.az_int_q_bin_count = mapping.GetQBinCount();
|
|
message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta();
|
|
message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
|
|
if (mapping.GetAzimuthalBinCount() > 1) {
|
|
message.az_int_bin_to_phi = mapping.GetBinToPhi();
|
|
message.az_int_map = mapping.GetPixelToBin();
|
|
}
|
|
message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress();
|
|
message.rois = in_experiment.ROI().ExportMetadata();
|
|
if (!in_experiment.ROI().empty())
|
|
message.roi_map = in_experiment.ExportROIMap();
|
|
message.max_spot_count = in_experiment.GetMaxSpotCount();
|
|
message.pixel_mask["default"] = mask.GetMask(in_experiment);
|
|
return message;
|
|
}
|
|
|
|
void JFJochReceiver::SendStartMessage() {
|
|
SendStartMessage(MakeStartMessage(experiment, pixel_mask, *az_int_mapping));
|
|
}
|
|
|
|
void JFJochReceiver::SendStartMessage(StartMessage message) {
|
|
SaveStartMessageToImageBuffer(message);
|
|
|
|
if (push_images_to_writer)
|
|
image_pusher.StartDataCollection(message);
|
|
|
|
if (zmq_preview_socket != nullptr)
|
|
zmq_preview_socket->StartDataCollection(message);
|
|
|
|
if (zmq_metadata_socket != nullptr)
|
|
zmq_metadata_socket->StartDataCollection(message);
|
|
}
|
|
|
|
void JFJochReceiver::SaveStartMessageToImageBuffer(const StartMessage &msg) {
|
|
std::vector<uint8_t> buffer(MESSAGE_SIZE_FOR_START_END);
|
|
CBORStream2Serializer serializer(buffer.data(), buffer.size());
|
|
serializer.SerializeSequenceStart(msg);
|
|
buffer.resize(serializer.GetBufferSize());
|
|
image_buffer.SaveStartMessage(buffer);
|
|
}
|
|
|
|
// A frame whose metadata does not fit its slot is dropped, without aborting the collection
|
|
bool JFJochReceiver::SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg,
|
|
ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger) {
|
|
try {
|
|
serializer.SerializeImage(msg);
|
|
return true;
|
|
} catch (const JFJochException &) {
|
|
// The metadata alone fits if it is the image that overflowed; if not, the exception propagates
|
|
const auto image = msg.image;
|
|
msg.image = CompressedImage(nullptr, 0, image.GetWidth(), image.GetHeight(),
|
|
image.GetMode(), image.GetCompressionAlgorithm(),
|
|
image.GetChannel());
|
|
CBORStream2Serializer meas(static_cast<uint8_t *>(loc.GetImage()), slot_size);
|
|
meas.SerializeImage(msg);
|
|
const size_t metadata_size = meas.GetImageAppendOffset();
|
|
msg.image = image;
|
|
|
|
logger.Error("Dropping image {} - CBOR metadata too large to store image: "
|
|
"metadata {} B + image {} B do not fit in buffer slot {} B",
|
|
msg.number, metadata_size, image.GetCompressedSize(), slot_size);
|
|
loc.release();
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void JFJochReceiver::CountImage(const DataMessage &message) {
|
|
++images_collected;
|
|
collection_efficiency_sum += message.image_collection_efficiency.value_or(1.0f);
|
|
compressed_size += message.image.GetCompressedSize();
|
|
uncompressed_size += message.image.GetUncompressedSize();
|
|
UpdateMaxImageReceived(message.number);
|
|
}
|
|
|
|
void JFJochReceiver::StampImage(DataMessage &message, size_t fifo_delay) {
|
|
message.original_number = message.number;
|
|
message.user_data = experiment.GetImageAppendix();
|
|
message.run_number = experiment.GetRunNumber();
|
|
message.run_name = experiment.GetRunName();
|
|
message.receiver_buf_available = image_buffer.GetAvailSlots();
|
|
message.receiver_aq_dev_delay = fifo_delay;
|
|
// Uncompressed/compressed (e.g. 7x), as JFJochReceiverFPGA and JFJochReceiver::GetStatus
|
|
if (message.image.GetCompressedSize() > 0)
|
|
message.compression_ratio = static_cast<float>(message.image.GetUncompressedSize())
|
|
/ static_cast<float>(message.image.GetCompressedSize());
|
|
}
|
|
|
|
void JFJochReceiver::AddToAverages(const DataMessage &message) {
|
|
// A stream that does not state it (DECTRIS) delivers whole images
|
|
if (message.image_collection_efficiency.value_or(1.0f) != 1.0f)
|
|
return;
|
|
saturated_pixels.Add(message.saturated_pixel_count);
|
|
error_pixels.Add(message.error_pixel_count);
|
|
if (message.roi.contains("beam")) {
|
|
roi_beam_npixel.Add(message.roi.at("beam").pixels);
|
|
roi_beam_sum.Add(message.roi.at("beam").sum);
|
|
}
|
|
}
|
|
|
|
bool JFJochReceiver::ForwardImage(DataMessage &message, ZeroCopyReturnValue *loc) {
|
|
if (loc == nullptr) {
|
|
writer_queue_full = true;
|
|
return false;
|
|
}
|
|
auto writer_buffer = (uint8_t *) loc->GetImage();
|
|
CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize());
|
|
if (!SerializeImageOrDrop(serializer, message, *loc, experiment.GetImageBufferLocationSize(), logger))
|
|
return false;
|
|
loc->SetImageNumber(message.number);
|
|
loc->SetImageSize(serializer.GetBufferSize());
|
|
loc->SetIndexed(message.indexing_result.value_or(false));
|
|
loc->ReadyToSend();
|
|
|
|
if (zmq_preview_socket != nullptr)
|
|
zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize());
|
|
|
|
if (zmq_metadata_socket != nullptr)
|
|
zmq_metadata_socket->AddDataMessage(message);
|
|
|
|
if (push_images_to_writer) {
|
|
// Only count images the pusher accepted; TCP refuses them on a broken/absent connection
|
|
if (image_pusher.SendImage(*loc))
|
|
++images_sent;
|
|
} else
|
|
loc->release();
|
|
UpdateMaxImageSent(message.number);
|
|
return true;
|
|
}
|
|
|
|
void JFJochReceiver::SendEndMessage() {
|
|
EndMessage message{};
|
|
message.max_image_number = max_image_number_sent;
|
|
message.images_collected_count = images_collected;
|
|
message.images_sent_to_write_count = images_sent;
|
|
|
|
message.max_receiver_delay = max_delay;
|
|
message.efficiency = GetEfficiency();
|
|
message.end_date = time_UTC(std::chrono::system_clock::now());
|
|
message.run_number = experiment.GetRunNumber();
|
|
message.run_name = experiment.GetRunName();
|
|
|
|
message.bkg_estimate = plots.GetBkgEstimate();
|
|
message.spindle_blind_fraction = plots.GetSpindleBlindFraction();
|
|
message.ice_ring_score_mean = plots.GetIceRingScore();
|
|
message.indexing_rate = plots.GetIndexingRate();
|
|
|
|
message.az_int_result["dataset"] = plots.GetAzIntProfile();
|
|
|
|
for (int i = 0; i < adu_histogram_module.size(); i++)
|
|
message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram();
|
|
|
|
scan_result.FillEndMessage(message);
|
|
AddToEndMessage(message);
|
|
|
|
if (push_images_to_writer) {
|
|
if (!image_pusher.EndDataCollection(message))
|
|
logger.Error("End message not sent via ZeroMQ (time-out)");
|
|
|
|
logger.Debug("Disconnected from writers");
|
|
}
|
|
|
|
// The result blocks (AddToEndMessage) are megabytes only the writer needs.
|
|
message.merged.reset();
|
|
message.map_coefficients.reset();
|
|
message.placed_model.reset();
|
|
message.refined_model.reset();
|
|
message.refined_map_coefficients.reset();
|
|
message.blobs.reset();
|
|
|
|
if (zmq_metadata_socket != nullptr)
|
|
zmq_metadata_socket->EndDataCollection(message);
|
|
|
|
if (zmq_preview_socket != nullptr)
|
|
zmq_preview_socket->EndDataCollection(message);
|
|
}
|
|
|
|
void JFJochReceiver::AddIndexingToEndMessage(EndMessage &message, const IndexAndRefine &indexer) {
|
|
message.unit_cell = indexer.GetConsensusUnitCell();
|
|
// The images are written regardless, but the indexing rate is then not a statement about the crystal
|
|
if (const int64_t error_count = indexer.GetIndexerErrorCount(); error_count > 0) {
|
|
try {
|
|
indexer.RethrowIndexerError();
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Indexing failed {} times in this collection, first with: {}. Those attempts are "
|
|
"left out of the indexing rate", error_count, e.what());
|
|
}
|
|
}
|
|
}
|
|
|
|
JFJochReceiverOutput JFJochReceiver::GetFinalStatistics() const {
|
|
JFJochReceiverOutput ret;
|
|
ret.efficiency = GetEfficiency();
|
|
ret.start_time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(start_time.time_since_epoch()).count();
|
|
ret.end_time_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_time.time_since_epoch()).count();
|
|
ret.writer_queue_full_warning = writer_queue_full;
|
|
ret.status = GetStatus();
|
|
ret.writer_err = writer_error;
|
|
ret.scan_result = scan_result.GetResult();
|
|
ret.processing_time = plots.GetMeanProcessingTime();
|
|
return ret;
|
|
}
|
|
|
|
void JFJochReceiver::Cancel(bool silent) {
|
|
if (!silent) {
|
|
// Remote abort: This tells FPGAs to stop but doesn't do anything to CPU code
|
|
logger.Warning("Cancelling on request");
|
|
cancelled = true;
|
|
}
|
|
}
|
|
|
|
void JFJochReceiver::Cancel(const JFJochException &e) {
|
|
logger.Error("Cancelling data collection due to exception");
|
|
logger.ErrorException(e);
|
|
// Error abort: This tells FPGAs to stop and also prevents deadlock in CPU code by setting abort to 1
|
|
cancelled = true;
|
|
}
|