// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include "JFJochReceiverStream.h" using namespace std::chrono_literals; int64_t JFJochReceiverStream::NumberOfThreads(int64_t requested_thread_number, const DiffractionExperiment &in_experiment) { int64_t number_of_images = in_experiment.GetImageNum(); auto image_time = in_experiment.GetImageTime(); if (requested_thread_number <= 0) return 1; // For very small datasets no need to go multithreaded if (number_of_images < 4) return 1; if (number_of_images < 16) return std::min(2, requested_thread_number); // For 100 Hz, there is no reason to go for a very large number of threads if (number_of_images < 256 || (image_time >= 10ms)) return std::min(16, requested_thread_number); return requested_thread_number; } void JFJochReceiverStream::ApplyStartMessage(DiffractionExperiment &experiment, const StartMessage &msg) { // A PSI detector's images are what the experiment set the card up to produce: nothing to learn here. if (experiment.GetDetectorType() != DetectorType::DECTRIS) return; if ((experiment.GetXPixelsNum() != msg.image_size_x) || (experiment.GetYPixelsNum() != msg.image_size_y)) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in detector size"); if (fabs(experiment.GetPixelSize_mm() - msg.pixel_size_x * 1e3) > 1e-7) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in pixel size"); experiment.Detector().Description(msg.detector_description); experiment.Detector().SerialNumber(msg.detector_serial_number); experiment.Detector().SensorMaterial(msg.sensor_material); experiment.Detector().SensorThickness_um(msg.sensor_thickness * 1e6); experiment.Detector().SaturationLimit(SaturationLimitFromValue(msg.saturation_value)); // The outgoing start message is rebuilt from the experiment, NXmx would record these as not applied experiment.Detector().CountRateCorrectionApplied(msg.countrate_correction_enabled); experiment.Detector().CountRateCorrectionLookupTable(msg.countrate_correction_lookup_table); experiment.Detector().FlatfieldApplied(msg.flatfield_enabled); experiment.Detector().VirtualPixelInterpolationApplied(msg.virtual_pixel_interpolation_enabled); // The mask written with the data is ours, never uploaded to the detector experiment.ApplyPixelMask(false); // Images are forwarded byte-for-byte, so the stream decides both width and sign experiment.Detector().BitDepthImage(msg.bit_depth_image); experiment.PixelSigned(msg.pixel_signed); } JFJochReceiverStream::JFJochReceiverStream(const DiffractionExperiment &in_experiment, const PixelMask &in_pixel_mask, ImagePuller &in_image_puller, ImagePusher &in_image_pusher, Logger &in_logger, int64_t requested_nthreads, const SpotFindingSettings &in_spot_finding_settings, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, ImageBuffer &in_image_buffer, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, IndexerThreadPool *indexing_thread_pool) : JFJochReceiver(in_experiment, in_image_buffer, in_image_pusher, in_current_status, in_plots, in_spot_finding_settings, in_logger, in_pixel_mask, in_zmq_preview_socket, in_zmq_metadata_socket, indexing_thread_pool), image_puller(in_image_puller), nthreads(NumberOfThreads(requested_nthreads, in_experiment)), measurement_started(1), images_started(nthreads) { } void JFJochReceiverStream::StartThreads() { logger.Debug("Starting {} image threads", nthreads); for (int i = 0; i < nthreads; i++) image_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverStream::ImageThread, this, i)); measurement = std::async(std::launch::async, &JFJochReceiverStream::MeasurementThread, this); images_started.wait(); logger.Debug("Image threads ready"); } void JFJochReceiverStream::JoinFuture(std::future &future) { try { if (future.valid()) future.get(); } catch (const std::exception &e) { logger.Error("Receiver teardown failed: {}", e.what()); } catch (...) { logger.Error("Receiver teardown failed with unknown exception"); } } void JFJochReceiverStream::JoinThreads() { JoinFuture(measurement); // The measurement thread joins them, unless it left with an error for (auto &f: image_futures) JoinFuture(f); } void JFJochReceiverStream::MeasurementThread() { try { // Once the detector has reported it is done, a start message in flight may still land, then give up std::optional give_up_at; auto msg = image_puller.PollImage(); while (!cancelled && (!msg.has_value() || !msg->cbor || !msg->cbor->start_message)) { if (detector_finished && !give_up_at) give_up_at = std::chrono::steady_clock::now() + StartMessageGrace; if (give_up_at && (std::chrono::steady_clock::now() > *give_up_at)) { logger.Error("Detector finished, but no start message arrived - abandoning measurement"); cancelled = true; break; } msg = image_puller.PollImage(); } if (cancelled) { current_status.SetProgress({}); current_status.SetStatus(GetStatus()); measurement_started.count_down(); return; // just quit the function - no measurement, no results } ApplyStartMessage(experiment, msg->cbor->start_message.value()); image_buffer.StartMeasurement(experiment); // Only at this point we know bit-depth of the images start_time = std::chrono::system_clock::now(); OnStart(msg->received); measurement_started.count_down(); } catch (const JFJochException &e) { logger.ErrorException(e); Cancel(e); measurement_started.count_down(); throw; } catch (const std::exception &e) { logger.Error("Cancelling data collection due to exception: {}", e.what()); cancelled = true; measurement_started.count_down(); throw; } logger.Info("Receiving started"); for (auto &f: image_futures) f.get(); logger.Debug("Image threads finished"); FinishCollection(); } void JFJochReceiverStream::OnStart(std::chrono::steady_clock::time_point received) { SendStartMessage(); } void JFJochReceiverStream::ImageThread(uint32_t id) { WaitForImages(); PollImages(id); } void JFJochReceiverStream::WaitForImages() { images_started.count_down(); measurement_started.wait(); } void JFJochReceiverStream::PollImages(uint32_t id) { while (!cancelled && !end_message_received) { try { auto msg = image_puller.PollImage(std::chrono::milliseconds(5)); if (!msg.has_value() || !msg->cbor) { // Message not received or not parsed continue; } else if (msg->cbor->end_message) { end_adu_histogram = msg->cbor->end_message->adu_histogram; end_message_received = true; logger.Debug("Thread {} end message received", id); } else if (msg->cbor->data_message) { DataMessage message = msg->cbor->data_message.value(); OnImage(id, message, msg->received); } UpdateMaxDelay(image_puller.GetCurrentFifoUtilization()); current_status.SetProgress(GetProgress()); current_status.SetStatus(GetStatus()); } catch (const JFJochException &e) { logger.ErrorException(e); Cancel(e); } catch (const std::exception &e) { logger.Error("Cancelling data collection due to exception: {}", e.what()); cancelled = true; } } } void JFJochReceiverStream::CheckIfImagesSent() { if (!image_buffer.CheckIfBufferReturned(SendFinalizeTimeout)) { logger.Error("Send commands not finalized in {} seconds", SendFinalizeTimeout.count()); throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in " + std::to_string(SendFinalizeTimeout.count()) + " seconds"); } } void JFJochReceiverStream::WriterFinalize() { if (push_images_to_writer) writer_error = image_pusher.Finalize(); } void JFJochReceiverStream::FinishCollection() { current_status.SetProgress(1.0); current_status.SetStatus(GetStatus()); SendEndMessage(); CheckIfImagesSent(); logger.Debug("All images sent"); WriterFinalize(); logger.Debug("Writing process finalized"); end_time = std::chrono::system_clock::now(); current_status.SetProgress({}); current_status.SetStatus(GetStatus()); } void JFJochReceiverStream::StopReceiver() { if (measurement.valid()) { measurement.get(); logger.Debug("Receiver stopped"); } } void JFJochReceiverStream::Cancel(bool silent) { JFJochReceiver::Cancel(silent); // A silent cancel is not a request to abort, it says the detector has finished. It bounds the wait for the start message. detector_finished = true; } void JFJochReceiverStream::AddToEndMessage(EndMessage &message) { if (!end_adu_histogram.empty()) message.adu_histogram = end_adu_histogram; } // With the card summing, an image is many frames. int64_t JFJochReceiverStream::ExpectedImages() const { return experiment.GetImageNum(); } float JFJochReceiverStream::GetEfficiency() const { if (ExpectedImages() == 0) return 0; return static_cast(images_collected) / static_cast(ExpectedImages()); } float JFJochReceiverStream::GetProgress() const { if (ExpectedImages() == 0) return 0.0; return static_cast(max_image_number_received) / static_cast(ExpectedImages()); }