// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include "JFJochReceiver.h" #include "../common/CUDAWrapper.h" #include "../common/time_utc.h" #include "JFJochCompressor.h" #include "../frame_serialize/CBORStream2Serializer.h" JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, ImageBuffer &in_image_buffer, ImagePusher &in_image_pusher, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, const SpotFindingSettings &spot_finding_settings, Logger &logger, const PixelMask &in_pixel_mask, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, IndexerThreadPool *indexing_thread_pool) : logger(logger), experiment(in_experiment), spot_finding_settings(spot_finding_settings), image_buffer(in_image_buffer), image_pusher(in_image_pusher), zmq_preview_socket(in_zmq_preview_socket), zmq_metadata_socket(in_zmq_metadata_socket), current_status(in_current_status), plots(in_plots), scan_result(in_experiment), serialmx_filter(in_experiment), pixel_mask(in_pixel_mask), // retain_outcomes=false: no online scaling/merge at the end, so the integration outcomes are not used indexer(experiment, indexing_thread_pool, false, /*real_time=*/true) { logger.Debug("Initializing receiver"); // Ensure there is nothing running for now if (!image_buffer.Finalize(std::chrono::seconds(1))) throw JFJochException(JFJochExceptionCategory::WrongDAQState, "There are unfinished preview/sending jobs in the buffer"); logger.Debug("Image buffer from previous run finalized"); current_status.SetProgress(0); current_status.SetEfficiency({}); current_status.SetStatus(JFJochReceiverStatus{}); // GetStatus() is virtual function and cannot be called yet! auto start_time_point = std::chrono::steady_clock::now(); az_int_mapping = std::make_unique(experiment, pixel_mask); auto end_time_point = std::chrono::steady_clock::now(); auto duration = std::chrono::duration(end_time_point - start_time_point); logger.Debug("Azimuthal integration mapping done in {:.5f} s with {} threads", duration.count(), az_int_mapping->GetNThreads()); plots.Setup(experiment, *az_int_mapping); push_images_to_writer = (experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty()); } JFJochReceiver::~JFJochReceiver() = default; SpotFindingSettings JFJochReceiver::GetSpotFindingSettings() { std::unique_lock ul(spot_finding_settings_mutex); return spot_finding_settings; } void JFJochReceiver::SetSpotFindingSettings(const SpotFindingSettings &in_spot_finding_settings) { std::unique_lock ul(spot_finding_settings_mutex); DiffractionExperiment::CheckDataProcessingSettings(in_spot_finding_settings); spot_finding_settings = in_spot_finding_settings; } void JFJochReceiver::UpdateMaxImageSent(int64_t image_number) { std::unique_lock ul(max_image_number_sent_mutex); if (image_number + 1 > max_image_number_sent) max_image_number_sent = image_number + 1; } void JFJochReceiver::UpdateMaxImageReceived(int64_t image_number) { std::unique_lock ul(max_image_number_received_mutex); if (image_number + 1 > max_image_number_received) max_image_number_received = image_number + 1; } void JFJochReceiver::UpdateMaxDelay(uint64_t delay) { std::unique_lock ul(max_delay_mutex); if (!max_delay || (delay > max_delay)) max_delay = delay; } JFJochReceiverStatus JFJochReceiver::GetStatus() const { JFJochReceiverStatus ret; ret.indexing_rate = plots.GetIndexingRate(); ret.bkg_estimate = plots.GetBkgEstimate(); if ((experiment.GetImageNum() > 0) && (compressed_size > 0)) { ret.compressed_ratio = static_cast(uncompressed_size) / static_cast(compressed_size); } ret.saturated_pixels = saturated_pixels.Read(); ret.error_pixels = error_pixels.Read(); ret.roi_beam_npixel = roi_beam_npixel.Read(); ret.roi_beam_sum = roi_beam_sum.Read(); ret.compressed_size = compressed_size; ret.max_receive_delay = max_delay; ret.max_image_number_sent = max_image_number_sent; ret.images_collected = images_collected; ret.images_sent = images_sent; ret.images_skipped = images_skipped; ret.images_dropped = GetImagesDropped(); ret.images_written = image_pusher.GetImagesWritten(); ret.cancelled = cancelled; ret.efficiency = GetEfficiency(); return ret; } StartMessage JFJochReceiver::MakeStartMessage(const DiffractionExperiment &in_experiment, const PixelMask &mask, const AzimuthalIntegrationMapping &mapping) { StartMessage message{}; in_experiment.FillMessage(message); message.arm_date = time_UTC(std::chrono::system_clock::now()); message.az_int_q_bin_count = mapping.GetQBinCount(); message.az_int_bin_to_q = mapping.GetBinToQ(); message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta(); message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount(); if (mapping.GetAzimuthalBinCount() > 1) { message.az_int_bin_to_phi = mapping.GetBinToPhi(); message.az_int_map = mapping.GetPixelToBin(); } message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress(); message.rois = in_experiment.ROI().ExportMetadata(); if (!in_experiment.ROI().empty()) message.roi_map = in_experiment.ExportROIMap(); message.max_spot_count = in_experiment.GetMaxSpotCount(); message.pixel_mask["default"] = mask.GetMask(in_experiment); return message; } void JFJochReceiver::SendStartMessage() { SendStartMessage(MakeStartMessage(experiment, pixel_mask, *az_int_mapping)); } void JFJochReceiver::SendStartMessage(StartMessage message) { SaveStartMessageToImageBuffer(message); if (push_images_to_writer) image_pusher.StartDataCollection(message); if (zmq_preview_socket != nullptr) zmq_preview_socket->StartDataCollection(message); if (zmq_metadata_socket != nullptr) zmq_metadata_socket->StartDataCollection(message); } void JFJochReceiver::SaveStartMessageToImageBuffer(const StartMessage &msg) { std::vector buffer(MESSAGE_SIZE_FOR_START_END); CBORStream2Serializer serializer(buffer.data(), buffer.size()); serializer.SerializeSequenceStart(msg); buffer.resize(serializer.GetBufferSize()); image_buffer.SaveStartMessage(buffer); } // A frame whose metadata does not fit its slot is dropped, without aborting the collection bool JFJochReceiver::SerializeImageOrDrop(CBORStream2Serializer &serializer, DataMessage &msg, ZeroCopyReturnValue &loc, size_t slot_size, Logger &logger) { try { serializer.SerializeImage(msg); return true; } catch (const JFJochException &) { // The metadata alone fits if it is the image that overflowed; if not, the exception propagates const auto image = msg.image; msg.image = CompressedImage(nullptr, 0, image.GetWidth(), image.GetHeight(), image.GetMode(), image.GetCompressionAlgorithm(), image.GetChannel()); CBORStream2Serializer meas(static_cast(loc.GetImage()), slot_size); meas.SerializeImage(msg); const size_t metadata_size = meas.GetImageAppendOffset(); msg.image = image; logger.Error("Dropping image {} - CBOR metadata too large to store image: " "metadata {} B + image {} B do not fit in buffer slot {} B", msg.number, metadata_size, image.GetCompressedSize(), slot_size); loc.release(); return false; } } void JFJochReceiver::CountImage(const DataMessage &message) { ++images_collected; compressed_size += message.image.GetCompressedSize(); uncompressed_size += message.image.GetUncompressedSize(); UpdateMaxImageReceived(message.number); } void JFJochReceiver::StampImage(DataMessage &message, size_t fifo_delay) { message.original_number = message.number; message.user_data = experiment.GetImageAppendix(); message.run_number = experiment.GetRunNumber(); message.run_name = experiment.GetRunName(); message.receiver_buf_available = image_buffer.GetAvailSlots(); message.receiver_aq_dev_delay = fifo_delay; // Uncompressed/compressed (e.g. 7x), as JFJochReceiverFPGA and JFJochReceiver::GetFinalStatistics if (message.image.GetCompressedSize() > 0) message.compression_ratio = static_cast(message.image.GetUncompressedSize()) / static_cast(message.image.GetCompressedSize()); } void JFJochReceiver::AddToAverages(const DataMessage &message) { // A stream that does not state it (DECTRIS) delivers whole images if (message.image_collection_efficiency.value_or(1.0f) != 1.0f) return; saturated_pixels.Add(message.saturated_pixel_count); error_pixels.Add(message.error_pixel_count); if (message.roi.contains("beam")) { roi_beam_npixel.Add(message.roi.at("beam").pixels); roi_beam_sum.Add(message.roi.at("beam").sum); } } bool JFJochReceiver::ForwardImage(DataMessage &message, ZeroCopyReturnValue *loc) { if (loc == nullptr) { writer_queue_full = true; return false; } auto writer_buffer = (uint8_t *) loc->GetImage(); CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize()); if (!SerializeImageOrDrop(serializer, message, *loc, experiment.GetImageBufferLocationSize(), logger)) return false; loc->SetImageNumber(message.number); loc->SetImageSize(serializer.GetBufferSize()); loc->SetIndexed(message.indexing_result.value_or(false)); loc->ReadyToSend(); if (zmq_preview_socket != nullptr) zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize()); if (zmq_metadata_socket != nullptr) zmq_metadata_socket->AddDataMessage(message); if (push_images_to_writer) { // Only count images the pusher accepted; TCP refuses them on a broken/absent connection if (image_pusher.SendImage(*loc)) ++images_sent; } else loc->release(); UpdateMaxImageSent(message.number); return true; } void JFJochReceiver::SendEndMessage() { EndMessage message{}; message.max_image_number = max_image_number_sent; message.images_collected_count = images_collected; message.images_sent_to_write_count = images_sent; message.max_receiver_delay = max_delay; message.efficiency = GetEfficiency(); message.end_date = time_UTC(std::chrono::system_clock::now()); message.run_number = experiment.GetRunNumber(); message.run_name = experiment.GetRunName(); message.bkg_estimate = plots.GetBkgEstimate(); message.spindle_blind_fraction = plots.GetSpindleBlindFraction(); message.ice_ring_score_mean = plots.GetIceRingScore(); message.indexing_rate = plots.GetIndexingRate(); message.az_int_result["dataset"] = plots.GetAzIntProfile(); const auto rotation_indexer_ret = indexer.FinalizeRotationIndexing(); if (rotation_indexer_ret.has_value()) { message.rotation_lattice = rotation_indexer_ret->lattice; message.rotation_lattice_type = LatticeMessage{ .centering = rotation_indexer_ret->search_result.centering, .niggli_class = rotation_indexer_ret->search_result.niggli_class, .crystal_system = rotation_indexer_ret->search_result.system }; message.rotation_extra_lattices = rotation_indexer_ret->extra_lattices; rotation_indexing_lattice = rotation_indexer_ret->lattice; rotation_indexing_lattice_type = message.rotation_lattice_type; } message.unit_cell = indexer.GetConsensusUnitCell(); // The images are written regardless, but the indexing rate is then not a statement about the crystal if (const int64_t error_count = indexer.GetIndexerErrorCount(); error_count > 0) { try { indexer.RethrowIndexerError(); } catch (const std::exception &e) { logger.Error("Indexing failed {} times in this collection, first with: {}. Those attempts are " "left out of the indexing rate, and a rotation lattice may be missing for the same " "reason", error_count, e.what()); } } for (int i = 0; i < adu_histogram_module.size(); i++) message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram(); scan_result.FillEndMessage(message); AddToEndMessage(message); if (push_images_to_writer) { if (!image_pusher.EndDataCollection(message)) logger.Error("End message not sent via ZeroMQ (time-out)"); logger.Debug("Disconnected from writers"); } // The result blocks (AddToEndMessage) are megabytes only the writer needs. message.merged.reset(); message.map_coefficients.reset(); message.placed_model.reset(); message.refined_model.reset(); message.refined_map_coefficients.reset(); message.blobs.reset(); if (zmq_metadata_socket != nullptr) zmq_metadata_socket->EndDataCollection(message); if (zmq_preview_socket != nullptr) zmq_preview_socket->EndDataCollection(message); } JFJochReceiverOutput JFJochReceiver::GetFinalStatistics() const { JFJochReceiverOutput ret; ret.efficiency = GetEfficiency(); ret.start_time_ms = std::chrono::duration_cast(start_time.time_since_epoch()).count(); ret.end_time_ms = std::chrono::duration_cast(end_time.time_since_epoch()).count(); ret.writer_queue_full_warning = writer_queue_full; ret.status = GetStatus(); ret.writer_err = writer_error; ret.scan_result = scan_result.GetResult(); ret.scan_result.rotation_lattice = rotation_indexing_lattice; if (rotation_indexing_lattice_type) { ret.scan_result.rotation_crystal_system = rotation_indexing_lattice_type->crystal_system; ret.scan_result.rotation_centering = rotation_indexing_lattice_type->centering; } ret.images_written = images_written; ret.processing_time = plots.GetMeanProcessingTime(); return ret; } void JFJochReceiver::Cancel(bool silent) { if (!silent) { // Remote abort: This tells FPGAs to stop but doesn't do anything to CPU code logger.Warning("Cancelling on request"); cancelled = true; } } void JFJochReceiver::Cancel(const JFJochException &e) { logger.Error("Cancelling data collection due to exception"); logger.ErrorException(e); // Error abort: This tells FPGAs to stop and also prevents deadlock in CPU code by setting abort to 1 cancelled = true; }