Receivers: rugnux live only through FPGAImagePuller, pedestal runs on JFJochReceiverPedestal
- JFJochReceiverFPGA no longer runs rugnux: its live stills (StillsGPUAnalysisFPGA, GPUSink, the merged data in END) and its pedestal code go; the receiver is the one of rc186 (rc186-old), which differs from what it replaces only in these two parts. - JFJochReceiverPedestal back from rc186 (242587c58,7080109a4) unchanged: raw frames from the devices into JFPedestalCalc, nothing on any socket. In rc186 it took every PSI pedestal run directly from the devices, not through the puller; so it does here. - Dispatch for a PSI detector: pedestal -> JFJochReceiverPedestal; rugnux live on (Standard mode, no host summation) -> FPGAImagePuller and JFJochReceiverRugnuxStills or ...Rotation; otherwise JFJochReceiverFPGA, unchanged. With host summation rugnux live is not run (a warning in the status). - The FPGAImagePullerUse enum goes: FPGAImagePullerSettings(slots, blocking, rate, always), where always (every collection through the puller, without rugnux to JFJochReceiverLite) is kept for the tests comparing the puller with JFJochReceiverFPGA pixel for pixel. - jfjoch_fpga_stills_test loses -F; StillsGPUAnalysisFPGATest goes, the GPU-stills processing tests run the puller path only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
1 parent
4e84c94e19
commit
f2e4dc63b4
15 files changed
+381
-1112
No files matched your search
+3
-4
@@ -74,7 +74,7 @@ synthesis C model on the CPU, so no FPGA device is required.
|
||||
Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector:
|
||||
|
||||
```
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-F] [-K[<MiB>]] <master.h5>
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-d] [-v] [-K[<MiB>]] <master.h5>
|
||||
```
|
||||
|
||||
- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement
|
||||
@@ -87,9 +87,8 @@ jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C
|
||||
one is saturated or the sum exceeds 16 bits it is saturated.
|
||||
- The frames are loaded into the card's internal frame generator and the run is made with the JUNGFRAU
|
||||
conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon
|
||||
counts and rugnux live stills analyses them through `FPGAImagePuller` (`-F`: through
|
||||
`JFJochReceiverFPGA`'s own live stills, for comparison; that path does not block: frames past what its
|
||||
analysis holds are lost and counted). Beam centre, distance, energy and the cell
|
||||
counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live
|
||||
on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s). Beam centre, distance, energy and the cell
|
||||
come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is
|
||||
written.
|
||||
- Each card's generator holds module frames of its own modules only: as many images as its memory holds
|
||||
|
||||
@@ -7,6 +7,7 @@ ADD_LIBRARY(JFJochReceiver STATIC
|
||||
JFJochReceiverService.cpp JFJochReceiverService.h
|
||||
JFJochReceiverTest.cpp JFJochReceiverTest.h
|
||||
JFJochReceiverFPGA.cpp JFJochReceiverFPGA.h
|
||||
JFJochReceiverPedestal.cpp JFJochReceiverPedestal.h
|
||||
ImageMetadata.cpp
|
||||
ImageMetadata.h
|
||||
LossyFilter.cpp
|
||||
@@ -21,7 +22,6 @@ ADD_LIBRARY(JFJochReceiver STATIC
|
||||
JFJochReceiverRugnuxStills.h JFJochReceiverRugnuxStills.cpp
|
||||
StillsPrePass.h StillsPrePass.cpp
|
||||
StillsGPUAnalysis.h StillsGPUAnalysis.cpp
|
||||
StillsGPUAnalysisFPGA.h StillsGPUAnalysisFPGA.cpp
|
||||
RugnuxLiveStatus.h RugnuxLiveStatus.cpp
|
||||
RugnuxStructure.h RugnuxStructure.cpp
|
||||
JFJochReceiver.cpp
|
||||
|
||||
+34
-328
@@ -3,14 +3,9 @@
|
||||
|
||||
#include "JFJochReceiverFPGA.h"
|
||||
|
||||
#include "../image_analysis/WriteReflections.h"
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "ImageMetadata.h"
|
||||
#include "JFJochReceiverLite.h"
|
||||
#include "../common/CUDAWrapper.h"
|
||||
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
||||
|
||||
JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment,
|
||||
const PixelMask &in_pixel_mask,
|
||||
@@ -24,9 +19,7 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
|
||||
ImageBuffer &in_send_buf_ctrl,
|
||||
ZMQPreviewSocket *in_zmq_preview_socket,
|
||||
ZMQMetadataSocket *in_zmq_metadata_socket,
|
||||
IndexerThreadPool *indexing_thread_pool,
|
||||
bool in_gpu_analysis,
|
||||
RugnuxLiveStatus *in_rugnux_status)
|
||||
IndexerThreadPool *indexing_thread_pool)
|
||||
: JFJochReceiver(in_experiment,
|
||||
in_send_buf_ctrl,
|
||||
in_image_sender,
|
||||
@@ -44,13 +37,9 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
|
||||
ndatastreams(experiment.GetDataStreamsNum()),
|
||||
frame_transformation_nthreads(
|
||||
in_forward_and_sum_nthreads),
|
||||
pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4),
|
||||
summation_nthreads(2),
|
||||
frame_transformation_ready((experiment.GetImageNum() > 0) ? frame_transformation_nthreads : 0),
|
||||
data_acquisition_ready(ndatastreams),
|
||||
gpu_analysis(in_gpu_analysis && experiment.GetImageNum() > 0
|
||||
&& experiment.GetDetectorMode() == DetectorMode::Standard),
|
||||
rugnux_status(in_rugnux_status) {
|
||||
frame_transformation_ready(frame_transformation_nthreads),
|
||||
data_acquisition_ready(ndatastreams) {
|
||||
// The FPGA integration core has a fixed bin-count limit. Reject configurations it cannot
|
||||
// handle before starting acquisition, unless azimuthal integration is forced onto the CPU.
|
||||
const auto &azint_settings = experiment.GetAzimuthalIntegrationSettings();
|
||||
@@ -92,92 +81,23 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
|
||||
|
||||
logger.Debug("Data acquisition devices ready");
|
||||
|
||||
if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0)
|
||||
|| (experiment.GetDetectorMode() == DetectorMode::PedestalG1)
|
||||
|| (experiment.GetDetectorMode() == DetectorMode::PedestalG2)) {
|
||||
if (experiment.GetImageNum() > 0) {
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"Saving and calculating pedestal is not supported for the time being");
|
||||
}
|
||||
SendStartMessage();
|
||||
SendCalibration();
|
||||
|
||||
switch (experiment.GetDetectorMode()) {
|
||||
case DetectorMode::PedestalG1:
|
||||
only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2);
|
||||
break;
|
||||
case DetectorMode::PedestalG2:
|
||||
only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2);
|
||||
break;
|
||||
default:
|
||||
only_2nd_sc_pedestal = false;
|
||||
break;
|
||||
}
|
||||
for (int i = 0; i < experiment.GetImageNum(); i++)
|
||||
images_to_go.Put(i);
|
||||
|
||||
int64_t pedestal_count = (only_2nd_sc_pedestal)
|
||||
? experiment.GetModulesNum()
|
||||
: experiment.GetModulesNum() * experiment.GetStorageCellNumber();
|
||||
for (int i = 0; i < pedestal_count; i++)
|
||||
pedestal.emplace_back(std::make_unique<JFPedestalCalc>(experiment));
|
||||
|
||||
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
|
||||
bool ignore = only_2nd_sc_pedestal && (s == 0);
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
for (int m = 0; m < experiment.GetModulesNum(d); m++)
|
||||
for (int n = 0; n < pedestal_nthreads; n++)
|
||||
frame_transformation_futures.emplace_back(std::async(std::launch::async,
|
||||
&JFJochReceiverFPGA::MeasurePedestalThread,
|
||||
this, d, m, s, n, ignore));
|
||||
}
|
||||
logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads);
|
||||
} else if (experiment.GetImageNum() > 0) {
|
||||
if (gpu_analysis) {
|
||||
// The start message and the calibration go out once the pre-pass is done.
|
||||
gpu = std::make_unique<StillsGPUAnalysisFPGA>(
|
||||
experiment, pixel_mask, MakeStartMessage(experiment, pixel_mask, *az_int_mapping),
|
||||
indexing_thread_pool, std::min<int64_t>(16, frame_transformation_nthreads),
|
||||
// Half the image buffer for the frames waiting, the other half for those sent.
|
||||
image_buffer.GetStatus().total_slots / 2, logger,
|
||||
[this](const StillsGPUAnalysisFPGA::Setup &s) {
|
||||
// The profiles are binned on the pre-pass's geometry.
|
||||
plots.Setup(s.experiment, *s.mapping);
|
||||
SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping));
|
||||
SendCalibration();
|
||||
},
|
||||
[this](DataMessage &m, const AzimuthalIntegrationProfile &p) { GPUSink(m, p); },
|
||||
[this] {
|
||||
// The broker's settings, with the spot finder rugnux uses on stills (the pre-pass
|
||||
// decided the centre and mask with it): adaptive threshold, single-pixel spots kept.
|
||||
// With the broker's default fixed threshold 3 points fewer frames index.
|
||||
// Every frame is integrated: integration is rugnux's, not the broker's to switch off.
|
||||
SpotFindingSettings s = GetSpotFindingSettings();
|
||||
s.adaptive_threshold = true;
|
||||
s.min_pix_per_spot = std::nullopt;
|
||||
s.quick_integration = true;
|
||||
return s;
|
||||
},
|
||||
// The start message did not go out (or the analysis could not be set up): nothing can
|
||||
// be written, so the collection stops, as it does on an error in a frame thread.
|
||||
[this](const JFJochException &e) { Cancel(e); },
|
||||
rugnux_status);
|
||||
} else {
|
||||
SendStartMessage();
|
||||
SendCalibration();
|
||||
}
|
||||
|
||||
for (int i = 0; i < experiment.GetImageNum(); i++)
|
||||
images_to_go.Put(i);
|
||||
|
||||
// Setup frames summation and forwarding
|
||||
for (uint32_t i = 0; i < frame_transformation_nthreads; i++) {
|
||||
auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread,
|
||||
this, i);
|
||||
frame_transformation_futures.emplace_back(std::move(handle));
|
||||
}
|
||||
|
||||
logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads);
|
||||
frame_transformation_ready.wait();
|
||||
logger.Debug("Image compression/forwarding threads ready");
|
||||
// Setup frames summation and forwarding
|
||||
for (uint32_t i = 0; i < frame_transformation_nthreads; i++) {
|
||||
auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread,
|
||||
this, i);
|
||||
frame_transformation_futures.emplace_back(std::move(handle));
|
||||
}
|
||||
|
||||
logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads);
|
||||
frame_transformation_ready.wait();
|
||||
logger.Debug("Image compression/forwarding threads ready");
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverFPGA::AcquireThread,
|
||||
this, d));
|
||||
@@ -251,49 +171,11 @@ void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) {
|
||||
}
|
||||
|
||||
|
||||
void JFJochReceiverFPGA::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell,
|
||||
uint32_t threadid, bool ignore) {
|
||||
JFPedestalCalc pedestal_calc(experiment);
|
||||
|
||||
uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber();
|
||||
uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads;
|
||||
uint32_t storage_cell_header = UINT32_MAX;
|
||||
|
||||
try {
|
||||
for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) {
|
||||
// Frame will be processed only if one already collects frame+2
|
||||
acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number);
|
||||
if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) {
|
||||
auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number);
|
||||
// Partial packets will bring more problems, than benefit
|
||||
pedestal_calc.AnalyzeImage((uint16_t *) output->pixels);
|
||||
storage_cell_header = (output->module_statistics.debug >> 8) & 0xF;
|
||||
}
|
||||
acquisition_device[data_stream].FrameBufferRelease(frame, module_number);
|
||||
|
||||
UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number));
|
||||
current_status.SetProgress(GetProgress());
|
||||
current_status.SetStatus(GetStatus());
|
||||
}
|
||||
|
||||
uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
|
||||
if (!only_2nd_sc_pedestal)
|
||||
offset += experiment.GetModulesNum() * storage_cell;
|
||||
|
||||
if (!ignore)
|
||||
*pedestal[offset] += pedestal_calc;
|
||||
} catch (const JFJochException &e) {
|
||||
Cancel(e);
|
||||
}
|
||||
logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done",
|
||||
data_stream, module_number, storage_cell, storage_cell_header);
|
||||
}
|
||||
|
||||
int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
|
||||
int64_t image_number,
|
||||
uint16_t module_number,
|
||||
uint32_t threadid,
|
||||
ModuleSummation &summation) {
|
||||
void JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
|
||||
int64_t image_number,
|
||||
uint16_t module_number,
|
||||
uint32_t threadid,
|
||||
ModuleSummation &summation) {
|
||||
ModuleSummation local_summation(experiment);
|
||||
int64_t starting_frame = image_number * experiment.GetSummation();
|
||||
|
||||
@@ -317,7 +199,6 @@ int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
|
||||
|
||||
if (!summation.empty())
|
||||
summation.AddFPGAOutput(local_summation.GetOutput(), 4);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
|
||||
@@ -380,7 +261,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
|
||||
for (int i = 0; i < experiment.GetModulesNum(); i++)
|
||||
summation.emplace_back(std::make_unique<ModuleSummation>(experiment));
|
||||
|
||||
std::vector<std::future<int64_t> > futures;
|
||||
std::vector<std::future<void>> futures;
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++) {
|
||||
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
|
||||
@@ -455,41 +336,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
message.image = CompressedImage(transformation.GetImage(),
|
||||
experiment.GetPixelsNum() * experiment.GetByteDepthImage(),
|
||||
experiment.GetXPixelsNum(),
|
||||
experiment.GetYPixelsNum(),
|
||||
experiment.GetImageMode(),
|
||||
CompressionAlgorithm::NO_COMPRESSION);
|
||||
|
||||
if (gpu_analysis) {
|
||||
// Compressed here into a slot of the image buffer, where it waits for the GPU analysis; GPUSink
|
||||
// writes it, with its analysis, to a second slot, which is sent. A frame that finds no slot
|
||||
// free, or as many frames waiting as the analysis is to hold, is dropped, as the receiver drops
|
||||
// any frame that finds the image buffer full.
|
||||
++images_collected;
|
||||
uncompressed_size += experiment.GetModulesNum() * RAW_MODULE_SIZE * experiment.GetByteDepthImage();
|
||||
auto held = std::make_shared<HeldImage>();
|
||||
held->slot = image_buffer.GetImageSlot();
|
||||
if (held->slot == nullptr) {
|
||||
writer_queue_full = true;
|
||||
continue;
|
||||
}
|
||||
const size_t size = transformation.CompressImage(static_cast<uint8_t *>(held->slot->GetImage()),
|
||||
experiment.GetImageBufferLocationSize());
|
||||
held->message = std::move(message);
|
||||
held->message.image = CompressedImage(held->slot->GetImage(), size,
|
||||
experiment.GetXPixelsNum(), experiment.GetYPixelsNum(),
|
||||
experiment.GetImageMode(),
|
||||
experiment.GetCompressionAlgorithm());
|
||||
if (!gpu->Hold(held)) {
|
||||
writer_queue_full = true;
|
||||
continue;
|
||||
}
|
||||
compressed_size += size;
|
||||
continue;
|
||||
}
|
||||
|
||||
// No-op unless the CPU backend is forced; fills az_int_profile_image from the assembled image
|
||||
analyzer->RunAzimuthalIntegration(transformation.GetImage(), az_int_profile_image);
|
||||
|
||||
@@ -616,12 +462,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
|
||||
}
|
||||
|
||||
float JFJochReceiverFPGA::GetEfficiency() const {
|
||||
|
||||
uint64_t expected_packets;
|
||||
if (experiment.GetImageNum() == 0)
|
||||
expected_packets = expected_packets_per_image * experiment.GetFrameNum();
|
||||
else
|
||||
expected_packets = expected_packets_per_image * experiment.GetImageNum();
|
||||
uint64_t expected_packets = expected_packets_per_image * experiment.GetImageNum();
|
||||
uint64_t received_packets = 0;
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++) {
|
||||
@@ -652,43 +493,29 @@ float JFJochReceiverFPGA::GetProgress() const {
|
||||
if (experiment.IsCPUSummation())
|
||||
frames *= experiment.GetSummation();
|
||||
|
||||
if (frames == 0)
|
||||
frames = experiment.GetFrameNum();
|
||||
if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished()))
|
||||
if (acquisition_device[0].Counters().IsAcquisitionFinished())
|
||||
return 1.0;
|
||||
|
||||
return static_cast<float>(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast<float>(frames);
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) {
|
||||
AddIndexingToEndMessage(message, indexer);
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::FinalizeMeasurement() {
|
||||
if (!frame_transformation_futures.empty()) {
|
||||
for (auto &future: frame_transformation_futures)
|
||||
future.get();
|
||||
for (auto &future: frame_transformation_futures)
|
||||
future.get();
|
||||
|
||||
logger.Debug("All processing threads done");
|
||||
}
|
||||
|
||||
// Where the start message never went out (or the analysis could not be set up), there is no end
|
||||
// message either: the devices are stopped all the same, and the error is the collection's result.
|
||||
std::exception_ptr gpu_error;
|
||||
if (gpu) {
|
||||
try {
|
||||
gpu->Finish();
|
||||
} catch (const std::exception &) {
|
||||
gpu_error = std::current_exception();
|
||||
}
|
||||
}
|
||||
logger.Debug("All processing threads done");
|
||||
|
||||
current_status.SetProgress(1.0);
|
||||
current_status.SetStatus(GetStatus());
|
||||
|
||||
if (!gpu_error)
|
||||
SendEndMessage();
|
||||
SendEndMessage();
|
||||
|
||||
if (experiment.GetImageNum() > 0) {
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
acquisition_device[d].Cancel();
|
||||
}
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
acquisition_device[d].Cancel();
|
||||
|
||||
end_time = std::chrono::system_clock::now();
|
||||
|
||||
@@ -702,18 +529,10 @@ void JFJochReceiverFPGA::FinalizeMeasurement() {
|
||||
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in "
|
||||
+ std::to_string(SendFinalizeTimeout.count()) + " seconds");
|
||||
}
|
||||
if (gpu_error) {
|
||||
current_status.SetProgress({});
|
||||
current_status.SetStatus(GetStatus());
|
||||
std::rethrow_exception(gpu_error);
|
||||
}
|
||||
logger.Info("Receiving data done");
|
||||
|
||||
if (push_images_to_writer)
|
||||
writer_error = image_pusher.Finalize();
|
||||
// Last image -> all written, as the rugnux receiver reports it
|
||||
if (gpu)
|
||||
gpu->Written();
|
||||
|
||||
current_status.SetProgress({});
|
||||
current_status.SetStatus(GetStatus());
|
||||
@@ -758,29 +577,10 @@ JFJochReceiverOutput JFJochReceiverFPGA::GetFinalStatistics() const {
|
||||
ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m));
|
||||
}
|
||||
}
|
||||
RetrievePedestal(ret.pedestal_result);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::RetrievePedestal(std::vector<JFModulePedestal> &output) const {
|
||||
time_t curr_time = std::chrono::system_clock::to_time_t(start_time);
|
||||
for (const auto &pc: pedestal) {
|
||||
JFModulePedestal mp;
|
||||
if (experiment.GetDetectorMode() == DetectorMode::PedestalG0)
|
||||
pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT);
|
||||
else
|
||||
pc->Export(mp);
|
||||
mp.SetCollectionTime(curr_time);
|
||||
output.emplace_back(std::move(mp));
|
||||
}
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) {
|
||||
if (experiment.IsPedestalRun()) {
|
||||
acquisition_device[data_stream].InitializeEmptyPixelMask(experiment);
|
||||
return; // No calibration loaded for pedestal
|
||||
}
|
||||
|
||||
if (calibration != nullptr)
|
||||
acquisition_device[data_stream].InitializeCalibration(experiment, *calibration);
|
||||
|
||||
@@ -793,97 +593,3 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) {
|
||||
// Initialize data processing
|
||||
acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping);
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile) {
|
||||
try {
|
||||
const auto status = image_buffer.GetSlotStatus();
|
||||
message.receiver_buf_available = status.available_slots;
|
||||
message.receiver_buf_in_preparation = status.preparation_slots;
|
||||
message.receiver_buf_in_sending = status.sending_slots;
|
||||
if (message.image.GetCompressedSize() > 0)
|
||||
message.compression_ratio = static_cast<float>(experiment.GetPixelsNum() * experiment.GetByteDepthImage())
|
||||
/ static_cast<float>(message.image.GetCompressedSize());
|
||||
|
||||
if (message.image_collection_efficiency == 1.0f) {
|
||||
saturated_pixels.Add(message.saturated_pixel_count);
|
||||
error_pixels.Add(message.error_pixel_count);
|
||||
if (message.roi.contains("beam")) {
|
||||
roi_beam_npixel.Add(message.roi["beam"].pixels);
|
||||
roi_beam_sum.Add(message.roi["beam"].sum);
|
||||
}
|
||||
}
|
||||
|
||||
scan_result.Add(message);
|
||||
|
||||
if (!serialmx_filter.ApplyFilter(message))
|
||||
++images_skipped;
|
||||
else if (auto loc = image_buffer.GetImageSlot(); loc == nullptr)
|
||||
writer_queue_full = true;
|
||||
else {
|
||||
auto writer_buffer = (uint8_t *) loc->GetImage();
|
||||
CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize());
|
||||
if (JFJochReceiverLite::SerializeImageOrDrop(serializer, message, *loc,
|
||||
experiment.GetImageBufferLocationSize(), logger)) {
|
||||
loc->SetImageNumber(message.number);
|
||||
loc->SetImageSize(serializer.GetBufferSize());
|
||||
loc->SetIndexed(message.indexing_result.value_or(false));
|
||||
loc->ReadyToSend();
|
||||
|
||||
if (zmq_preview_socket != nullptr)
|
||||
zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize());
|
||||
if (zmq_metadata_socket != nullptr)
|
||||
zmq_metadata_socket->AddDataMessage(message);
|
||||
|
||||
if (push_images_to_writer) {
|
||||
if (image_pusher.SendImage(*loc))
|
||||
++images_sent;
|
||||
} else
|
||||
loc->release();
|
||||
UpdateMaxImageSent(message.number);
|
||||
}
|
||||
}
|
||||
|
||||
plots.Add(message, profile);
|
||||
current_status.SetProgress(GetProgress());
|
||||
current_status.SetStatus(GetStatus());
|
||||
} catch (const JFJochException &e) {
|
||||
logger.ErrorException(e);
|
||||
Cancel(e);
|
||||
} catch (const std::exception &e) {
|
||||
const JFJochException error(JFJochExceptionCategory::WrongDAQState,
|
||||
fmt::format("Image {} not written - {}", message.number, e.what()));
|
||||
logger.ErrorException(error);
|
||||
Cancel(error);
|
||||
}
|
||||
}
|
||||
|
||||
void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) {
|
||||
AddIndexingToEndMessage(message, indexer);
|
||||
if (!gpu)
|
||||
return;
|
||||
message.unit_cell = gpu->GetConsensusUnitCell();
|
||||
message.refined_beam_center_x = gpu->GetBeamX_pxl();
|
||||
message.refined_beam_center_y = gpu->GetBeamY_pxl();
|
||||
message.refined_detector_distance_mm = gpu->GetDetectorDistance_mm();
|
||||
// The merged data, for the writer to put beside the master file.
|
||||
if (const auto &m = gpu->GetMerged(); m && gpu->GetMergeExperiment().GetUnitCell())
|
||||
message.merged = MergedMtzTable(m->merge.merged, *gpu->GetMergeExperiment().GetUnitCell(),
|
||||
gpu->GetMergeExperiment(), OutputProvenance{});
|
||||
if (!message.merged)
|
||||
return;
|
||||
// Every pusher serialises END into a buffer of MESSAGE_SIZE_FOR_START_END, and an END that does not fit there
|
||||
// fails the collection. So it is serialised here first, as JFJochReceiverRugnuxRotation does; one that does not fit
|
||||
// goes without the merged data.
|
||||
const auto scratch = std::make_unique_for_overwrite<uint8_t[]>(MESSAGE_SIZE_FOR_START_END);
|
||||
CBORStream2Serializer serializer(scratch.get(), MESSAGE_SIZE_FOR_START_END);
|
||||
try {
|
||||
serializer.SerializeSequenceEnd(message);
|
||||
} catch (const JFJochException &e) {
|
||||
const std::string text = fmt::format("the merged data do not fit in the end message ({} MiB) and are not "
|
||||
"written", MESSAGE_SIZE_FOR_START_END / (1024 * 1024));
|
||||
logger.Error("GPU analysis: {} - {}", text, e.what());
|
||||
if (rugnux_status)
|
||||
rugnux_status->Warning(text);
|
||||
message.merged.reset();
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <atomic>
|
||||
#include <future>
|
||||
#include <latch>
|
||||
@@ -21,18 +20,14 @@
|
||||
#include "../common/ImageBuffer.h"
|
||||
#include "../common/PixelMask.h"
|
||||
|
||||
#include "../image_analysis/spot_finding/StrongPixelSet.h"
|
||||
#include "../common/AzimuthalIntegrationMapping.h"
|
||||
#include "../common/AzimuthalIntegrationProfile.h"
|
||||
|
||||
#include "../jungfrau/JFCalibration.h"
|
||||
#include "../common/ADUHistogram.h"
|
||||
#include "../common/Plot.h"
|
||||
#include "../common/JFJochReceiverPlots.h"
|
||||
#include "../preview/PreviewImage.h"
|
||||
#include "LossyFilter.h"
|
||||
#include "JFJochReceiverCurrentStatus.h"
|
||||
#include "../jungfrau/JFPedestalCalc.h"
|
||||
#include "../preview/ZMQPreviewSocket.h"
|
||||
#include "../preview/ZMQMetadataSocket.h"
|
||||
#include "ImageMetadata.h"
|
||||
@@ -40,15 +35,12 @@
|
||||
#include "../image_analysis/MXAnalysisAfterFPGA.h"
|
||||
#include "../common/MovingAverage.h"
|
||||
#include "JFJochReceiverOutput.h"
|
||||
#include "StillsGPUAnalysisFPGA.h"
|
||||
|
||||
class JFJochReceiverFPGA : public JFJochReceiver {
|
||||
const JFCalibration *calibration;
|
||||
|
||||
IndexAndRefine indexer;
|
||||
|
||||
std::vector<std::unique_ptr<JFPedestalCalc>> pedestal;
|
||||
|
||||
std::vector<uint16_t> roi_map;
|
||||
|
||||
std::vector<std::future<void>> frame_transformation_futures;
|
||||
@@ -62,11 +54,9 @@ class JFJochReceiverFPGA : public JFJochReceiver {
|
||||
uint16_t ndatastreams{0};
|
||||
|
||||
const int64_t frame_transformation_nthreads;
|
||||
const int64_t pedestal_nthreads;
|
||||
const int64_t summation_nthreads;
|
||||
|
||||
int64_t expected_packets_per_image;
|
||||
bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells
|
||||
|
||||
std::latch frame_transformation_ready;
|
||||
std::latch data_acquisition_ready;
|
||||
@@ -75,31 +65,18 @@ class JFJochReceiverFPGA : public JFJochReceiver {
|
||||
void LoadCalibrationToFPGA(uint16_t data_stream);
|
||||
|
||||
void FrameTransformationThread(uint32_t threadid);
|
||||
void MeasurePedestalThread(uint16_t data_stream,
|
||||
uint16_t module_number,
|
||||
uint16_t storage_cell,
|
||||
uint32_t threadid,
|
||||
bool ignore);
|
||||
|
||||
int64_t SummationThread(uint16_t data_stream,
|
||||
void SummationThread(uint16_t data_stream,
|
||||
int64_t image_number,
|
||||
uint16_t module_number,
|
||||
uint32_t threadid,
|
||||
ModuleSummation &summation);
|
||||
void Cancel(const JFJochException &e) override;
|
||||
void FinalizeMeasurement();
|
||||
|
||||
// Serial data with rugnux on: the converted frames are analysed on the GPU (StillsGPUAnalysisFPGA), and
|
||||
// the start message waits for its pre-pass.
|
||||
const bool gpu_analysis;
|
||||
RugnuxLiveStatus *const rugnux_status;
|
||||
std::unique_ptr<StillsGPUAnalysisFPGA> gpu;
|
||||
void GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile);
|
||||
void AddToEndMessage(EndMessage &message) override;
|
||||
void FinalizeMeasurement();
|
||||
|
||||
void SendCalibration();
|
||||
void SendPedestal(const std::string &prefix, const std::vector<uint8_t> &v, int gain, int sc);
|
||||
void RetrievePedestal(std::vector<JFModulePedestal> &output) const;
|
||||
public:
|
||||
JFJochReceiverFPGA(const DiffractionExperiment& experiment,
|
||||
const PixelMask &pixel_mask,
|
||||
@@ -113,9 +90,7 @@ public:
|
||||
ImageBuffer &image_buffer,
|
||||
ZMQPreviewSocket *zmq_preview_socket,
|
||||
ZMQMetadataSocket *zmq_metadata_socket,
|
||||
IndexerThreadPool *indexing_thread_pool,
|
||||
bool gpu_analysis = false,
|
||||
RugnuxLiveStatus *rugnux_status = nullptr);
|
||||
IndexerThreadPool *indexing_thread_pool);
|
||||
~JFJochReceiverFPGA();
|
||||
JFJochReceiverFPGA(const JFJochReceiverFPGA &other) = delete;
|
||||
JFJochReceiverFPGA& operator=(const JFJochReceiverFPGA &other) = delete;
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "JFJochReceiverPedestal.h"
|
||||
|
||||
JFJochReceiverPedestal::JFJochReceiverPedestal(const DiffractionExperiment &in_experiment,
|
||||
const PixelMask &in_pixel_mask,
|
||||
AcquisitionDeviceGroup &in_aq_device,
|
||||
ImagePusher &in_image_sender,
|
||||
Logger &in_logger,
|
||||
const SpotFindingSettings &in_spot_finding_settings,
|
||||
JFJochReceiverCurrentStatus &in_current_status,
|
||||
JFJochReceiverPlots &in_plots,
|
||||
ImageBuffer &in_send_buf_ctrl)
|
||||
: JFJochReceiver(in_experiment,
|
||||
in_send_buf_ctrl,
|
||||
in_image_sender,
|
||||
in_current_status,
|
||||
in_plots,
|
||||
in_spot_finding_settings,
|
||||
in_logger,
|
||||
in_pixel_mask),
|
||||
acquisition_device(in_aq_device),
|
||||
ndatastreams(experiment.GetDataStreamsNum()),
|
||||
pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4),
|
||||
data_acquisition_ready(ndatastreams) {
|
||||
image_buffer.StartMeasurement(experiment);
|
||||
|
||||
if (acquisition_device.size() < ndatastreams)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"Number of acquisition devices has to match data streams");
|
||||
|
||||
expected_packets_per_image = 0;
|
||||
for (int d = 0; d < ndatastreams; d++) {
|
||||
acquisition_device[d].PrepareAction(experiment);
|
||||
expected_packets_per_image += acquisition_device[d].Counters().GetExpectedPacketsPerImage();
|
||||
|
||||
logger.Debug("Acquisition device {} prepared", d);
|
||||
}
|
||||
|
||||
logger.Debug("Data acquisition devices ready");
|
||||
|
||||
switch (experiment.GetDetectorMode()) {
|
||||
case DetectorMode::PedestalG1:
|
||||
only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2);
|
||||
break;
|
||||
case DetectorMode::PedestalG2:
|
||||
only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2);
|
||||
break;
|
||||
default:
|
||||
only_2nd_sc_pedestal = false;
|
||||
break;
|
||||
}
|
||||
|
||||
int64_t pedestal_count = (only_2nd_sc_pedestal)
|
||||
? experiment.GetModulesNum()
|
||||
: experiment.GetModulesNum() * experiment.GetStorageCellNumber();
|
||||
for (int i = 0; i < pedestal_count; i++)
|
||||
pedestal.emplace_back(std::make_unique<JFPedestalCalc>(experiment));
|
||||
|
||||
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
|
||||
bool ignore = only_2nd_sc_pedestal && (s == 0);
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
for (int m = 0; m < experiment.GetModulesNum(d); m++)
|
||||
for (int n = 0; n < pedestal_nthreads; n++)
|
||||
pedestal_futures.emplace_back(std::async(std::launch::async,
|
||||
&JFJochReceiverPedestal::MeasurePedestalThread,
|
||||
this, d, m, s, n, ignore));
|
||||
}
|
||||
logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads);
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverPedestal::AcquireThread,
|
||||
this, d));
|
||||
|
||||
data_acquisition_ready.wait();
|
||||
start_time = std::chrono::system_clock::now();
|
||||
measurement = std::async(std::launch::async, &JFJochReceiverPedestal::FinalizeMeasurement, this);
|
||||
logger.Info("Receiving data started");
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::AcquireThread(uint16_t data_stream) {
|
||||
try {
|
||||
acquisition_device[data_stream].InitializeEmptyPixelMask(experiment);
|
||||
logger.Debug("Device thread {} start action", data_stream);
|
||||
acquisition_device[data_stream].StartAction(experiment);
|
||||
} catch (const JFJochException &e) {
|
||||
Cancel(e);
|
||||
data_acquisition_ready.count_down();
|
||||
logger.ErrorException(e);
|
||||
logger.Warning("Device thread {} done due to an error", data_stream);
|
||||
return;
|
||||
}
|
||||
|
||||
data_acquisition_ready.count_down();
|
||||
|
||||
try {
|
||||
logger.Debug("Device thread {} wait for action complete", data_stream);
|
||||
acquisition_device[data_stream].WaitForActionComplete();
|
||||
} catch (const JFJochException &e) {
|
||||
logger.ErrorException(e);
|
||||
Cancel(e);
|
||||
logger.ErrorException(e);
|
||||
logger.Warning("Device thread {} done due to an error", data_stream);
|
||||
return;
|
||||
}
|
||||
logger.Debug("Device thread {} done", data_stream);
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell,
|
||||
uint32_t threadid, bool ignore) {
|
||||
JFPedestalCalc pedestal_calc(experiment);
|
||||
|
||||
uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber();
|
||||
uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads;
|
||||
uint32_t storage_cell_header = UINT32_MAX;
|
||||
|
||||
try {
|
||||
for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) {
|
||||
// Frame will be processed only if one already collects frame+2
|
||||
acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number);
|
||||
if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) {
|
||||
auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number);
|
||||
// Partial packets will bring more problems, than benefit
|
||||
pedestal_calc.AnalyzeImage((uint16_t *) output->pixels);
|
||||
storage_cell_header = (output->module_statistics.debug >> 8) & 0xF;
|
||||
}
|
||||
acquisition_device[data_stream].FrameBufferRelease(frame, module_number);
|
||||
|
||||
UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number));
|
||||
current_status.SetProgress(GetProgress());
|
||||
current_status.SetStatus(GetStatus());
|
||||
}
|
||||
|
||||
uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
|
||||
if (!only_2nd_sc_pedestal)
|
||||
offset += experiment.GetModulesNum() * storage_cell;
|
||||
|
||||
if (!ignore)
|
||||
*pedestal[offset] += pedestal_calc;
|
||||
} catch (const JFJochException &e) {
|
||||
Cancel(e);
|
||||
}
|
||||
logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done",
|
||||
data_stream, module_number, storage_cell, storage_cell_header);
|
||||
}
|
||||
|
||||
float JFJochReceiverPedestal::GetEfficiency() const {
|
||||
uint64_t expected_packets = expected_packets_per_image * experiment.GetFrameNum();
|
||||
uint64_t received_packets = 0;
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++) {
|
||||
received_packets += acquisition_device[d].Counters().GetTotalPackets();
|
||||
}
|
||||
|
||||
if ((expected_packets == received_packets) || (expected_packets == 0))
|
||||
return 1.0;
|
||||
return received_packets / static_cast<double>(expected_packets);
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::Cancel(bool silent) {
|
||||
JFJochReceiver::Cancel(silent);
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
acquisition_device[d].Cancel();
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::Cancel(const JFJochException &e) {
|
||||
JFJochReceiver::Cancel(e);
|
||||
|
||||
for (int d = 0; d < ndatastreams; d++)
|
||||
acquisition_device[d].Cancel();
|
||||
}
|
||||
|
||||
float JFJochReceiverPedestal::GetProgress() const {
|
||||
int64_t frames = experiment.GetFrameNum();
|
||||
if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished()))
|
||||
return 1.0;
|
||||
|
||||
return static_cast<float>(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast<float>(frames);
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::FinalizeMeasurement() {
|
||||
for (auto &future: pedestal_futures)
|
||||
future.get();
|
||||
|
||||
logger.Debug("All processing threads done");
|
||||
|
||||
current_status.SetProgress(1.0);
|
||||
current_status.SetStatus(GetStatus());
|
||||
|
||||
end_time = std::chrono::system_clock::now();
|
||||
|
||||
for (auto &future: data_acquisition_futures)
|
||||
future.get();
|
||||
|
||||
logger.Debug("Devices stopped");
|
||||
logger.Info("Receiving data done");
|
||||
|
||||
current_status.SetProgress({});
|
||||
current_status.SetStatus(GetStatus());
|
||||
}
|
||||
|
||||
void JFJochReceiverPedestal::StopReceiver() {
|
||||
if (measurement.valid()) {
|
||||
measurement.get();
|
||||
logger.Debug("Receiver stopped");
|
||||
}
|
||||
}
|
||||
|
||||
JFJochReceiverPedestal::~JFJochReceiverPedestal() {
|
||||
try {
|
||||
if (measurement.valid())
|
||||
measurement.get();
|
||||
} catch (const std::exception &e) {
|
||||
logger.Error("ReceiverPedestal teardown failed: {}", e.what());
|
||||
} catch (...) {
|
||||
logger.Error("ReceiverPedestal teardown failed with unknown exception");
|
||||
}
|
||||
}
|
||||
|
||||
JFJochReceiverOutput JFJochReceiverPedestal::GetFinalStatistics() const {
|
||||
JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics();
|
||||
for (int d = 0; d < ndatastreams; d++) {
|
||||
for (int m = 0; m < acquisition_device[d].Counters().GetModuleNumber(); m++) {
|
||||
ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule());
|
||||
ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m));
|
||||
}
|
||||
}
|
||||
|
||||
time_t curr_time = std::chrono::system_clock::to_time_t(start_time);
|
||||
for (const auto &pc: pedestal) {
|
||||
JFModulePedestal mp;
|
||||
if (experiment.GetDetectorMode() == DetectorMode::PedestalG0)
|
||||
pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT);
|
||||
else
|
||||
pc->Export(mp);
|
||||
mp.SetCollectionTime(curr_time);
|
||||
ret.pedestal_result.emplace_back(std::move(mp));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <future>
|
||||
#include <latch>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "JFJochReceiver.h"
|
||||
#include "../acquisition_device/AcquisitionDeviceGroup.h"
|
||||
#include "../jungfrau/JFPedestalCalc.h"
|
||||
|
||||
// A pedestal run: raw frames from the devices go into JFPedestalCalc; no image buffer slots, nothing goes out on any socket
|
||||
class JFJochReceiverPedestal : public JFJochReceiver {
|
||||
std::vector<std::unique_ptr<JFPedestalCalc>> pedestal;
|
||||
|
||||
std::vector<std::future<void>> pedestal_futures;
|
||||
std::vector<std::future<void>> data_acquisition_futures;
|
||||
|
||||
std::future<void> measurement;
|
||||
|
||||
AcquisitionDeviceGroup &acquisition_device;
|
||||
uint16_t ndatastreams{0};
|
||||
|
||||
const int64_t pedestal_nthreads;
|
||||
|
||||
int64_t expected_packets_per_image;
|
||||
bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells
|
||||
|
||||
std::latch data_acquisition_ready;
|
||||
|
||||
void AcquireThread(uint16_t data_stream);
|
||||
void MeasurePedestalThread(uint16_t data_stream,
|
||||
uint16_t module_number,
|
||||
uint16_t storage_cell,
|
||||
uint32_t threadid,
|
||||
bool ignore);
|
||||
void Cancel(const JFJochException &e) override;
|
||||
void FinalizeMeasurement();
|
||||
public:
|
||||
JFJochReceiverPedestal(const DiffractionExperiment &experiment,
|
||||
const PixelMask &pixel_mask,
|
||||
AcquisitionDeviceGroup &acquisition_devices,
|
||||
ImagePusher &image_pusher,
|
||||
Logger &logger,
|
||||
const SpotFindingSettings &spot_finding_settings,
|
||||
JFJochReceiverCurrentStatus ¤t_status,
|
||||
JFJochReceiverPlots &plots,
|
||||
ImageBuffer &image_buffer);
|
||||
~JFJochReceiverPedestal() override;
|
||||
JFJochReceiverPedestal(const JFJochReceiverPedestal &other) = delete;
|
||||
JFJochReceiverPedestal &operator=(const JFJochReceiverPedestal &other) = delete;
|
||||
|
||||
void StopReceiver() override;
|
||||
|
||||
float GetEfficiency() const override;
|
||||
void Cancel(bool silent) override;
|
||||
float GetProgress() const override;
|
||||
|
||||
JFJochReceiverOutput GetFinalStatistics() const override;
|
||||
};
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "JFJochReceiverFPGA.h"
|
||||
#include "JFJochReceiverLite.h"
|
||||
#include "JFJochReceiverDarkMask.h"
|
||||
#include "JFJochReceiverPedestal.h"
|
||||
#include "../preview/JFJochJPEG.h"
|
||||
#include "../preview/JFJochTIFF.h"
|
||||
|
||||
@@ -38,16 +39,16 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
|
||||
return *this;
|
||||
}
|
||||
|
||||
JFJochReceiverService &JFJochReceiverService::UseFPGAImagePuller(FPGAImagePullerUse use, size_t slots, bool blocking,
|
||||
double frame_rate_hz) {
|
||||
JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t slots, bool blocking, double frame_rate_hz,
|
||||
bool always) {
|
||||
std::unique_lock ul_state(state_mutex);
|
||||
if (state != ReceiverState::Idle)
|
||||
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
|
||||
"Cannot change the receiver path during data collection");
|
||||
fpga_image_puller = use;
|
||||
fpga_image_puller_slots = slots;
|
||||
fpga_image_puller_blocking = blocking;
|
||||
fpga_image_puller_rate_hz = frame_rate_hz;
|
||||
fpga_image_puller_always = always;
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -56,19 +57,12 @@ double JFJochReceiverService::GetFPGAImagePullerSlotWaitTime_s() const {
|
||||
return fpga_puller ? fpga_puller->GetSlotWaitTime_s() : 0.0;
|
||||
}
|
||||
|
||||
// Pedestal runs and summation on the host need JFJochReceiverFPGA
|
||||
// A PSI collection analysed by rugnux live; summation on the host needs JFJochReceiverFPGA
|
||||
bool JFJochReceiverService::FPGAImagePullerFor(const DiffractionExperiment &experiment) const {
|
||||
if ((experiment.GetDetectorType() == DetectorType::DECTRIS) || experiment.IsPedestalRun()
|
||||
|| experiment.IsCPUSummation() || (experiment.GetImageNum() <= 0))
|
||||
return false;
|
||||
switch (fpga_image_puller) {
|
||||
case FPGAImagePullerUse::Always:
|
||||
return true;
|
||||
case FPGAImagePullerUse::RugnuxLive:
|
||||
return rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
return fpga_image_puller_always || (rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard));
|
||||
}
|
||||
|
||||
RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const {
|
||||
@@ -155,7 +149,15 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
|
||||
puller = fpga_puller;
|
||||
}
|
||||
|
||||
if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) {
|
||||
if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && experiment.IsPedestalRun()) {
|
||||
receiver = std::make_unique<JFJochReceiverPedestal>(experiment, pixel_mask, aq_devices, image_pusher,
|
||||
logger, spot_finding_settings, receiver_status,
|
||||
plots, image_buffer);
|
||||
} else if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) {
|
||||
if (rugnux_enable && experiment.IsCPUSummation()) {
|
||||
analysis_warning = "no rugnux analysis: summation on the host";
|
||||
logger.Warning("{}", analysis_warning);
|
||||
}
|
||||
// Summation on the host runs threads of its own for each image
|
||||
auto fpga_nthreads = nthreads;
|
||||
if (experiment.IsCPUSummation())
|
||||
@@ -171,11 +173,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
|
||||
image_buffer,
|
||||
zmq_preview_socket.get(),
|
||||
zmq_metadata_socket.get(),
|
||||
indexer_thread_pool.get(),
|
||||
// Serial data with rugnux on: the frames are
|
||||
// analysed on the GPU, after a pre-pass.
|
||||
rugnux_enable && !RugnuxRotationData(experiment),
|
||||
&rugnux_status);
|
||||
indexer_thread_pool.get());
|
||||
} else {
|
||||
if (puller)
|
||||
image_puller = puller;
|
||||
|
||||
@@ -20,9 +20,6 @@
|
||||
|
||||
struct RugnuxModel;
|
||||
|
||||
// RugnuxLive: collections analysed by rugnux live; Always: also the others, through JFJochReceiverLite;
|
||||
// Never: all on JFJochReceiverFPGA, its own live stills included
|
||||
enum class FPGAImagePullerUse { RugnuxLive, Always, Never };
|
||||
|
||||
class JFJochReceiverService {
|
||||
std::unique_ptr<JFJochReceiver> receiver;
|
||||
@@ -43,8 +40,8 @@ class JFJochReceiverService {
|
||||
RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends
|
||||
std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status
|
||||
|
||||
// Which PSI collections go through FPGAImagePuller and the stream receivers
|
||||
FPGAImagePullerUse fpga_image_puller = FPGAImagePullerUse::RugnuxLive;
|
||||
// A PSI collection analysed by rugnux live goes through FPGAImagePuller to the stream receivers
|
||||
bool fpga_image_puller_always = false;
|
||||
size_t fpga_image_puller_slots = FPGAImagePuller::DefaultSlots;
|
||||
bool fpga_image_puller_blocking = false;
|
||||
double fpga_image_puller_rate_hz = 0;
|
||||
@@ -77,9 +74,11 @@ public:
|
||||
double deadline_s, bool refine_solvent = false, bool force_still = false);
|
||||
RugnuxLiveSnapshot GetRugnuxStatus() const;
|
||||
void ClearRugnuxStatus(); // back to Idle, as before any collection
|
||||
// blocking, frame_rate_hz: see FPGAImagePuller
|
||||
JFJochReceiverService& UseFPGAImagePuller(FPGAImagePullerUse use, size_t slots = FPGAImagePuller::DefaultSlots,
|
||||
bool blocking = false, double frame_rate_hz = 0);
|
||||
// slots, blocking, frame_rate_hz: see FPGAImagePuller. always: every PSI collection but pedestal and host
|
||||
// summation through the puller, without rugnux to JFJochReceiverLite (for tests comparing it with
|
||||
// JFJochReceiverFPGA)
|
||||
JFJochReceiverService& FPGAImagePullerSettings(size_t slots = FPGAImagePuller::DefaultSlots, bool blocking = false,
|
||||
double frame_rate_hz = 0, bool always = false);
|
||||
// Of the last collection through FPGAImagePuller: FPGAImagePuller::GetSlotWaitTime_s, 0 without one
|
||||
double GetFPGAImagePullerSlotWaitTime_s() const;
|
||||
JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional<int32_t> &watermark = {});
|
||||
|
||||
@@ -1,419 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "StillsGPUAnalysisFPGA.h"
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "../common/CUDAWrapper.h"
|
||||
#include "../common/ThreadAffinity.h"
|
||||
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
||||
#include "../rugnux/RugnuxDefaults.h"
|
||||
|
||||
StillsGPUAnalysisFPGA::StillsGPUAnalysisFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_mask,
|
||||
const StartMessage &in_start, IndexerThreadPool *in_indexing_thread_pool,
|
||||
int64_t nthreads, int64_t in_max_held, Logger &in_logger, Ready in_ready,
|
||||
Sink in_sink, Settings in_settings, Failed in_failed,
|
||||
RugnuxLiveStatus *in_status, bool in_merge)
|
||||
: experiment(in_experiment), mask(in_mask), start(in_start), indexing_thread_pool(in_indexing_thread_pool),
|
||||
logger(in_logger), ready(std::move(in_ready)), sink(std::move(in_sink)), settings(std::move(in_settings)),
|
||||
setup_failed(std::move(in_failed)), status(in_status), merge_frames(in_merge),
|
||||
max_held(std::max<int64_t>(1, in_max_held)), queue(max_held) {
|
||||
live.images_needed = std::min(PREPASS_IMAGES, experiment.GetImageNum());
|
||||
live.file_beam_x_pxl = experiment.GetBeamX_pxl();
|
||||
live.file_beam_y_pxl = experiment.GetBeamY_pxl();
|
||||
live.file_distance_mm = experiment.GetDetectorDistance_mm();
|
||||
if (status) {
|
||||
status->Start(experiment.GetImageNum(),
|
||||
static_cast<double>(experiment.GetImageNum())
|
||||
* std::chrono::duration<double>(experiment.GetImageTime()).count(),
|
||||
std::chrono::steady_clock::now());
|
||||
status->Step("Pre-pass");
|
||||
Publish();
|
||||
}
|
||||
prepass_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::PrePassThread, this);
|
||||
for (int64_t i = 0; i < nthreads; i++)
|
||||
analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysisFPGA::AnalysisThread, this));
|
||||
logger.Info("GPU analysis: {} threads, at least the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES);
|
||||
}
|
||||
|
||||
StillsGPUAnalysisFPGA::~StillsGPUAnalysisFPGA() {
|
||||
try {
|
||||
if (!finished)
|
||||
Finish();
|
||||
} catch (const std::exception &e) {
|
||||
logger.Error("GPU analysis: {}", e.what());
|
||||
}
|
||||
if (status && outcome)
|
||||
status->Finish(outcome->first, outcome->second);
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::AnalysisError(const std::string &text) {
|
||||
logger.Error("GPU analysis: {}", text);
|
||||
std::unique_lock ul(error_mutex);
|
||||
if (analysis_error.empty())
|
||||
analysis_error = text;
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::Publish() {
|
||||
std::unique_lock ul(live_mutex);
|
||||
if (live.prepass == RugnuxLivePrePass::Done || live.prepass == RugnuxLivePrePass::Failed) {
|
||||
const double t = std::chrono::duration<double>(std::chrono::steady_clock::now() - ready_time).count();
|
||||
if (t > 0 && live.images_analysed > 0)
|
||||
live.analysis_rate_hz = static_cast<double>(live.images_analysed) / t;
|
||||
}
|
||||
status->Stills(live);
|
||||
}
|
||||
|
||||
bool StillsGPUAnalysisFPGA::Hold(const std::shared_ptr<HeldImage> &image) {
|
||||
if (status)
|
||||
status->ImageReceived(std::chrono::steady_clock::now());
|
||||
if (!queue.Put(image)) {
|
||||
// As many frames wait as the buffer is to hold: the pre-pass runs on what it has, for the last time.
|
||||
StartPrePass();
|
||||
return false;
|
||||
}
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
if (!prepass_over) {
|
||||
prepass_images[image->message.number] = image;
|
||||
std::unique_lock ul_live(live_mutex);
|
||||
live.images_held = static_cast<int64_t>(prepass_images.size());
|
||||
}
|
||||
}
|
||||
prepass_cv.notify_all();
|
||||
if (status)
|
||||
Publish();
|
||||
return true;
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::StartPrePass() {
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
prepass_last = true;
|
||||
}
|
||||
prepass_cv.notify_all();
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::Finish() {
|
||||
finished = true;
|
||||
StartPrePass();
|
||||
prepass_future.get();
|
||||
for (size_t i = 0; i < analysis_futures.size(); i++)
|
||||
queue.PutBlocking(nullptr);
|
||||
for (auto &f: analysis_futures)
|
||||
f.get();
|
||||
if (status) {
|
||||
last_image = status->LastImage();
|
||||
Publish();
|
||||
}
|
||||
if (merge_future.valid()) {
|
||||
if (status)
|
||||
status->Step("Merge");
|
||||
merge_queue.Put(-1);
|
||||
merge_future.get();
|
||||
if (merged && status) {
|
||||
PublishMerge(*merged, "the collection");
|
||||
status->Headline(merge_experiment.GetSpaceGroupOrP1().xhm(), merged->merge.statistics.overall.d_min,
|
||||
std::nullopt);
|
||||
}
|
||||
}
|
||||
if (status) {
|
||||
if (setup_error)
|
||||
status->Finish(RugnuxLiveState::Failed, "GPU analysis setup failed");
|
||||
else {
|
||||
// Reported by Written, once everything is written - the end message and the writer go on meanwhile.
|
||||
status->Step("Writing");
|
||||
std::unique_lock ul(error_mutex);
|
||||
if (analysis_error.empty())
|
||||
outcome.emplace(RugnuxLiveState::Done, "");
|
||||
else
|
||||
outcome.emplace(RugnuxLiveState::Failed, analysis_error);
|
||||
}
|
||||
}
|
||||
if (setup_error)
|
||||
std::rethrow_exception(setup_error);
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::Written() {
|
||||
if (!status || !outcome)
|
||||
return;
|
||||
if (last_image)
|
||||
status->Latency(std::chrono::duration<double>(std::chrono::steady_clock::now() - last_image.value()).count());
|
||||
status->Finish(outcome->first, outcome->second);
|
||||
outcome.reset();
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::PrePassThread() {
|
||||
const auto masked = [](const PixelMask &m) {
|
||||
return std::count_if(m.GetMask().begin(), m.GetMask().end(), [](uint32_t v) { return v != 0; });
|
||||
};
|
||||
setup.experiment = experiment;
|
||||
setup.mask = mask;
|
||||
const int64_t all_images = experiment.GetImageNum();
|
||||
std::optional<StillsPrePassResult> result;
|
||||
std::chrono::steady_clock::time_point prepass_start = std::chrono::steady_clock::now();
|
||||
// At a low hit rate the first frames may not be enough to fit the geometry on, or may not hold a crystal: with
|
||||
// a cell to fit, the pre-pass runs again on all that is held once that has doubled, until the fit has its
|
||||
// frames, or the collection ended, or the sample is as large as it may grow (the last try): bounded so that
|
||||
// the last run is short and the other half of the queue takes the frames arriving meanwhile.
|
||||
const int64_t max_sample = std::min(PREPASS_MAX_IMAGES, max_held / 2);
|
||||
for (int64_t needed = std::min(PREPASS_IMAGES, all_images); ;) {
|
||||
std::map<int64_t, std::shared_ptr<HeldImage>> sample;
|
||||
bool last;
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
prepass_cv.wait(ul, [&] {
|
||||
return prepass_last || static_cast<int64_t>(prepass_images.size()) >= needed;
|
||||
});
|
||||
// Frames that arrived during the previous run do not make this one larger than the bound.
|
||||
for (const auto &[n, image]: prepass_images) {
|
||||
if (static_cast<int64_t>(sample.size()) >= std::max(needed, max_sample))
|
||||
break;
|
||||
sample.emplace(n, image);
|
||||
}
|
||||
last = prepass_last || static_cast<int64_t>(sample.size()) >= std::min(all_images, max_sample);
|
||||
}
|
||||
if (sample.empty())
|
||||
break;
|
||||
{
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.prepass = RugnuxLivePrePass::Running;
|
||||
}
|
||||
if (status)
|
||||
Publish();
|
||||
try {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
auto r = RunStillsPrePass(start, sample, static_cast<int>(std::thread::hardware_concurrency()));
|
||||
logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, "
|
||||
"distance {:.2f} -> {:.2f} mm on {} frames, masked pixels {} -> {}", r.images,
|
||||
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
|
||||
experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), r.beam_x_pxl, r.beam_y_pxl,
|
||||
experiment.GetDetectorDistance_mm(), r.distance_mm, r.geometry_frames,
|
||||
masked(mask), masked(r.mask));
|
||||
result = std::move(r);
|
||||
} catch (const std::exception &e) {
|
||||
logger.Warning("Pre-pass on {} images failed - {}", sample.size(), e.what());
|
||||
}
|
||||
if (last || !experiment.GetUnitCell()
|
||||
|| (result && result->geometry_frames >= RUGNUX_REFINE_GEOMETRY_FRAMES))
|
||||
break;
|
||||
needed = std::min({2 * static_cast<int64_t>(sample.size()), all_images, max_sample});
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.prepass = RugnuxLivePrePass::Holding;
|
||||
live.images_needed = needed;
|
||||
}
|
||||
{
|
||||
// The frames are analysed from here on: their slots go back to the buffer once they are sent.
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
prepass_over = true;
|
||||
prepass_images.clear();
|
||||
}
|
||||
const bool failed = !result;
|
||||
int geometry_frames = 0;
|
||||
if (result) {
|
||||
const auto &r = *result;
|
||||
setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl).DetectorDistance_mm(r.distance_mm);
|
||||
setup.mask = r.mask;
|
||||
geometry_frames = r.geometry_frames;
|
||||
const auto &m = r.mask.GetMask();
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.beam_x_pxl = r.beam_x_pxl;
|
||||
live.beam_y_pxl = r.beam_y_pxl;
|
||||
live.distance_mm = r.distance_mm;
|
||||
live.shadow_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) {
|
||||
return (v & (1u << PixelMask::BeamStopPixelBit)) != 0;
|
||||
});
|
||||
live.bad_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) {
|
||||
return (v & (1u << PixelMask::HotPixelBit)) != 0;
|
||||
});
|
||||
} else
|
||||
logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask");
|
||||
try {
|
||||
setup.mapping = std::make_unique<AzimuthalIntegrationMapping>(setup.experiment, setup.mask);
|
||||
// Retained for the merge, which takes each image's reflections as soon as it is analysed.
|
||||
indexer = std::make_unique<IndexAndRefine>(setup.experiment, indexing_thread_pool, merge_frames, true);
|
||||
if (geometry_frames == 0) {
|
||||
const std::string text = fmt::format(
|
||||
"The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the "
|
||||
"whole run is analysed at the distance in the header, {:.2f} mm, beam centre ({:.2f}, {:.2f}) px",
|
||||
setup.experiment.GetDetectorDistance_mm(), setup.experiment.GetBeamX_pxl(),
|
||||
setup.experiment.GetBeamY_pxl());
|
||||
logger.Warning("{}", text);
|
||||
if (status)
|
||||
status->Note(text);
|
||||
}
|
||||
if (merge_frames) {
|
||||
merge_experiment = setup.experiment;
|
||||
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
|
||||
merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
|
||||
setup.experiment.GetUnitCell(), std::nullopt,
|
||||
setup.experiment.IsDetectIceRings(),
|
||||
SpotFindingSettings().ice_ring_width_Q_recipA, logger);
|
||||
merge_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::MergeThread, this);
|
||||
}
|
||||
ready(setup);
|
||||
} catch (const std::exception &e) {
|
||||
// Nothing can be written without the start message: the frames are dropped (AnalysisThread), and
|
||||
// the collection is stopped.
|
||||
const auto *j = dynamic_cast<const JFJochException *>(&e);
|
||||
const JFJochException error = j ? *j : JFJochException(JFJochExceptionCategory::WrongDAQState,
|
||||
std::string("GPU analysis setup failed: ") + e.what());
|
||||
logger.ErrorException(error);
|
||||
setup_error = j ? std::current_exception() : std::make_exception_ptr(error);
|
||||
if (setup_failed)
|
||||
setup_failed(error);
|
||||
prepass_done.count_down();
|
||||
return;
|
||||
}
|
||||
{
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.prepass = failed ? RugnuxLivePrePass::Failed : RugnuxLivePrePass::Done;
|
||||
live.prepass_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - prepass_start).count();
|
||||
ready_time = std::chrono::steady_clock::now();
|
||||
}
|
||||
if (status) {
|
||||
status->Step("GPU analysis");
|
||||
Publish();
|
||||
}
|
||||
prepass_done.count_down();
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::AnalysisThread() {
|
||||
try {
|
||||
pin_gpu();
|
||||
} catch (const std::exception &e) {
|
||||
logger.Warning("Error pinning GPU {}", e.what());
|
||||
}
|
||||
|
||||
prepass_done.wait();
|
||||
|
||||
if (setup_error) {
|
||||
while (queue.GetBlocking()) {}
|
||||
return;
|
||||
}
|
||||
|
||||
// A thread whose analysis cannot be built still passes its frames on, unanalysed.
|
||||
std::unique_ptr<MXAnalysisWithoutFPGA> analysis;
|
||||
try {
|
||||
analysis = std::make_unique<MXAnalysisWithoutFPGA>(setup.experiment, *setup.mapping, setup.mask,
|
||||
*indexer, /*enable_fused_adaptive_gpu=*/true);
|
||||
} catch (const std::exception &e) {
|
||||
AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ")
|
||||
+ e.what());
|
||||
}
|
||||
|
||||
while (auto held = queue.GetBlocking()) {
|
||||
DataMessage &message = held->message;
|
||||
std::optional<AzimuthalIntegrationProfile> profile;
|
||||
const auto start_time = std::chrono::steady_clock::now();
|
||||
profile.emplace(*setup.mapping);
|
||||
try {
|
||||
if (analysis)
|
||||
analysis->Analyze(message, *profile, settings());
|
||||
} catch (const JFJochException &e) {
|
||||
// The frame is still written, without its analysis.
|
||||
logger.ErrorException(e);
|
||||
} catch (const std::exception &e) {
|
||||
AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what()));
|
||||
}
|
||||
message.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now()
|
||||
- start_time).count();
|
||||
try {
|
||||
sink(message, *profile);
|
||||
if (merge)
|
||||
merge_queue.Put(message.number);
|
||||
if (status) {
|
||||
int64_t analysed;
|
||||
{
|
||||
std::unique_lock ul(live_mutex);
|
||||
analysed = ++live.images_analysed;
|
||||
if (message.indexing_result.value_or(false))
|
||||
++live.images_indexed;
|
||||
}
|
||||
status->Progress(analysed, experiment.GetImageNum());
|
||||
Publish();
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
// The thread goes on with the next frame, so that the queue is drained.
|
||||
AnalysisError(fmt::format("image {} lost - {}", message.number, e.what()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<UnitCell> StillsGPUAnalysisFPGA::GetConsensusUnitCell() const {
|
||||
return indexer->GetConsensusUnitCell();
|
||||
}
|
||||
|
||||
float StillsGPUAnalysisFPGA::GetBeamX_pxl() const {
|
||||
return setup.experiment.GetBeamX_pxl();
|
||||
}
|
||||
|
||||
float StillsGPUAnalysisFPGA::GetBeamY_pxl() const {
|
||||
return setup.experiment.GetBeamY_pxl();
|
||||
}
|
||||
|
||||
float StillsGPUAnalysisFPGA::GetDetectorDistance_mm() const {
|
||||
return setup.experiment.GetDetectorDistance_mm();
|
||||
}
|
||||
|
||||
const std::optional<StillsMergeOutput> &StillsGPUAnalysisFPGA::GetMerged() const {
|
||||
return merged;
|
||||
}
|
||||
|
||||
const DiffractionExperiment &StillsGPUAnalysisFPGA::GetMergeExperiment() const {
|
||||
return merge_experiment;
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::MergeThread() {
|
||||
constexpr int64_t PREPARE_EVERY = 200; // frames between two Prepare calls after a checkpoint
|
||||
const int64_t images = experiment.GetImageNum();
|
||||
// The cell merged in, as rugnux merges it: the consensus of the frames indexed by the first merge (the
|
||||
// collection's cell only picks the frames that are merged).
|
||||
bool cell_set = false;
|
||||
const auto have_cell = [&] {
|
||||
if (!cell_set)
|
||||
if (const auto c = indexer->GetConsensusUnitCell()) {
|
||||
merge_experiment.SetUnitCell(*c);
|
||||
merge->Cell(*c);
|
||||
cell_set = true;
|
||||
}
|
||||
return cell_set;
|
||||
};
|
||||
size_t next = 0;
|
||||
try {
|
||||
const auto add = [&](int64_t n, IntegrationOutcome &&o) {
|
||||
merge->Add(n, std::move(o));
|
||||
if (next < CHECKPOINTS.size()
|
||||
&& merge->Added() >= std::llround(CHECKPOINTS[next] * static_cast<double>(images))) {
|
||||
if (have_cell())
|
||||
PublishMerge(merge->Checkpoint(MERGE_THREADS),
|
||||
fmt::format("{:.0f}% of the images (provisional)", 100.0 * CHECKPOINTS[next]));
|
||||
++next;
|
||||
} else if (next > 0 && merge->Added() % PREPARE_EVERY == 0)
|
||||
merge->Prepare(MERGE_THREADS);
|
||||
};
|
||||
for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking())
|
||||
add(n, indexer->TakeIntegrationOutcome(n));
|
||||
if (have_cell()) {
|
||||
merged = merge->Final(static_cast<int>(AvailableCpus()));
|
||||
logger.Info("Stills merge after the last image: {:.2f} s", merged->seconds);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
logger.Warning("Stills merge failed: {}", e.what());
|
||||
merged.reset();
|
||||
}
|
||||
}
|
||||
|
||||
void StillsGPUAnalysisFPGA::PublishMerge(const StillsMergeOutput &out, const std::string &title) {
|
||||
const auto &o = out.merge.statistics.overall;
|
||||
const std::string text = fmt::format("Merge of {}: {} crystals, {:.2f} A, CC1/2 {:.3f}, R_split {:.3f}, "
|
||||
"completeness {:.1f}%, <I/sigma> {:.1f} ({:.1f} s)", title, out.images,
|
||||
o.d_min, o.cc_half, o.r_split,
|
||||
o.possible_unique_reflections > 0
|
||||
? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0,
|
||||
o.mean_i_over_sigma, out.seconds);
|
||||
logger.Info("{}", text);
|
||||
if (status)
|
||||
status->Note(text);
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <condition_variable>
|
||||
#include <exception>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <latch>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#include "StillsPrePass.h"
|
||||
#include "RugnuxLiveStatus.h"
|
||||
#include "../common/DiffractionExperiment.h"
|
||||
#include "../common/JFJochException.h"
|
||||
#include "../common/AzimuthalIntegrationMapping.h"
|
||||
#include "../common/AzimuthalIntegrationProfile.h"
|
||||
#include "../common/Logger.h"
|
||||
#include "../common/ThreadSafeFIFO.h"
|
||||
#include "../image_analysis/IndexAndRefine.h"
|
||||
#include "../rugnux/StillsLiveMerge.h"
|
||||
|
||||
// JFJochReceiverFPGA's live stills, kept as they are at rc.185 while the stream receivers take StillsGPUAnalysis.
|
||||
// Serial frames analysed on the GPU as the DECTRIS ones are (MXAnalysisWithoutFPGA), after a pre-pass on
|
||||
// the first PREPASS_IMAGES (RunStillsPrePass) has decided the beam centre and the mask they are analysed
|
||||
// with. Every frame given before the pre-pass is done is held; `ready` is called once it is, with the
|
||||
// geometry and mask the frames are then analysed at, before any of them is analysed; `sink` gets each
|
||||
// frame once analysed, from one of the analysis threads. Where `status` is given, the run is reported there
|
||||
// (pre-pass, its decisions, the analysis).
|
||||
// At most `max_held` frames wait for analysis: Hold refuses one more, so the slots held stay bounded.
|
||||
// Where the setup after the pre-pass fails (`ready` included), `failed` is called at once, the frames are
|
||||
// dropped unanalysed and Finish throws the error. An analysis thread that fails passes its frames on
|
||||
// unanalysed, and the run is reported as failed.
|
||||
// The indexed frames are merged as `rugnux --mode scale` merges them (StillsLiveMerge), beside the analysis:
|
||||
// a provisional merge at each of CHECKPOINTS of the images, on MERGE_THREADS threads, reported as a step
|
||||
// of the status, and the merge of the whole collection once the last frame is analysed, on every thread,
|
||||
// before Finish returns (GetMerged). The cell is the run's consensus at the first merge, as rugnux merges in
|
||||
// it (the collection's cell picks the frames merged); the space group the collection's, else P1.
|
||||
// merge = false leaves it out.
|
||||
// With a cell given, the pre-pass runs again on all the frames held each time they have doubled, until the geometry
|
||||
// fit has its frames, the collection ends or the sample reaches PREPASS_MAX_IMAGES or half of `max_held`; the
|
||||
// analysis starts after that and catches up from the frames held. The bound keeps the last run short and leaves
|
||||
// room in the queue for the frames arriving meanwhile, which are otherwise not written. The geometry is fixed when the pre-pass ends: every frame is analysed and merged at it. Where the pre-pass could
|
||||
// not refine it (too few of the frames it had indexed: a small image buffer, a serial hit rate, a chip whose first
|
||||
// frames are empty; or no cell given), the header's distance stays for the whole run, with a warning in the log
|
||||
// and the status.
|
||||
class StillsGPUAnalysisFPGA {
|
||||
public:
|
||||
// Enough frames for the distance to be refined at a serial hit rate: of 100 frames of a collection indexed
|
||||
// at 18%, 7 indexed at a distance 2.6% off and the geometry refinement did not run; of 500, 34 did and
|
||||
// refined it to 0.4%. They wait in the image buffer, which has to hold them in half its slots.
|
||||
constexpr static int64_t PREPASS_IMAGES = 500;
|
||||
constexpr static int64_t PREPASS_MAX_IMAGES = 4 * PREPASS_IMAGES;
|
||||
constexpr static std::array<double, 2> CHECKPOINTS = {0.5, 0.75};
|
||||
constexpr static int MERGE_THREADS = 8;
|
||||
|
||||
struct Setup {
|
||||
DiffractionExperiment experiment;
|
||||
PixelMask mask;
|
||||
std::unique_ptr<AzimuthalIntegrationMapping> mapping;
|
||||
};
|
||||
using Ready = std::function<void(const Setup &)>;
|
||||
using Sink = std::function<void(DataMessage &, const AzimuthalIntegrationProfile &)>;
|
||||
using Settings = std::function<SpotFindingSettings()>;
|
||||
using Failed = std::function<void(const JFJochException &)>;
|
||||
|
||||
// `start`: the collection's start message, as it would be sent without the pre-pass.
|
||||
StillsGPUAnalysisFPGA(const DiffractionExperiment &experiment, const PixelMask &mask, const StartMessage &start,
|
||||
IndexerThreadPool *indexing_thread_pool, int64_t nthreads, int64_t max_held, Logger &logger,
|
||||
Ready ready, Sink sink, Settings settings, Failed failed = {},
|
||||
RugnuxLiveStatus *status = nullptr, bool merge = true);
|
||||
~StillsGPUAnalysisFPGA();
|
||||
|
||||
// False where `max_held` frames already wait: the frame is not taken.
|
||||
bool Hold(const std::shared_ptr<HeldImage> &image);
|
||||
// No frame comes after this: the pre-pass runs on what came, if it has not, and every frame is
|
||||
// analysed before it returns. Throws where the setup after the pre-pass failed.
|
||||
void Finish();
|
||||
// After Finish, once the end message is out and the writer is done: the status gets the latency from the last
|
||||
// image and then the run's outcome. Without it the outcome is reported when the analysis goes, with no latency.
|
||||
void Written();
|
||||
|
||||
// After Finish.
|
||||
[[nodiscard]] std::optional<UnitCell> GetConsensusUnitCell() const;
|
||||
[[nodiscard]] float GetBeamX_pxl() const;
|
||||
[[nodiscard]] float GetBeamY_pxl() const;
|
||||
[[nodiscard]] float GetDetectorDistance_mm() const;
|
||||
// The merge of the collection, where one was made.
|
||||
[[nodiscard]] const std::optional<StillsMergeOutput> &GetMerged() const;
|
||||
[[nodiscard]] const DiffractionExperiment &GetMergeExperiment() const;
|
||||
|
||||
private:
|
||||
const DiffractionExperiment &experiment;
|
||||
const PixelMask &mask;
|
||||
const StartMessage start;
|
||||
IndexerThreadPool *indexing_thread_pool;
|
||||
Logger &logger;
|
||||
Ready ready;
|
||||
Sink sink;
|
||||
Settings settings;
|
||||
Failed setup_failed;
|
||||
RugnuxLiveStatus *status;
|
||||
const bool merge_frames;
|
||||
|
||||
std::mutex live_mutex;
|
||||
RugnuxLiveStills live;
|
||||
std::chrono::steady_clock::time_point ready_time;
|
||||
void Publish();
|
||||
// Kept by Finish for Written
|
||||
std::optional<std::chrono::steady_clock::time_point> last_image;
|
||||
std::optional<std::pair<RugnuxLiveState, std::string>> outcome;
|
||||
|
||||
Setup setup;
|
||||
std::unique_ptr<IndexAndRefine> indexer;
|
||||
|
||||
std::mutex prepass_mutex;
|
||||
std::condition_variable prepass_cv;
|
||||
bool prepass_last = false; // what is held is all there will be
|
||||
bool prepass_over = false;
|
||||
std::map<int64_t, std::shared_ptr<HeldImage>> prepass_images;
|
||||
std::latch prepass_done{1};
|
||||
std::exception_ptr setup_error; // set before prepass_done
|
||||
|
||||
std::mutex error_mutex;
|
||||
std::string analysis_error; // the first, where an analysis thread failed
|
||||
void AnalysisError(const std::string &text);
|
||||
|
||||
const int64_t max_held;
|
||||
ThreadSafeFIFO<std::shared_ptr<HeldImage>> queue;
|
||||
std::future<void> prepass_future;
|
||||
std::vector<std::future<void>> analysis_futures;
|
||||
bool finished = false;
|
||||
|
||||
// The frames analysed, by number, for the merge (-1 after the last); it takes their outcomes from the indexer.
|
||||
ThreadSafeFIFO<int64_t> merge_queue;
|
||||
std::unique_ptr<StillsLiveMerge> merge; // after the indexer: its reflections are in the indexer's arena
|
||||
DiffractionExperiment merge_experiment;
|
||||
std::optional<StillsMergeOutput> merged;
|
||||
std::future<void> merge_future;
|
||||
|
||||
void StartPrePass();
|
||||
void PrePassThread();
|
||||
void AnalysisThread();
|
||||
void MergeThread();
|
||||
void PublishMerge(const StillsMergeOutput &out, const std::string &title);
|
||||
};
|
||||
@@ -132,7 +132,6 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
RugnuxLiveTriageTest.cpp
|
||||
RugnuxScaleInputsTest.cpp
|
||||
StillsGPUAnalysisTest.cpp
|
||||
StillsGPUAnalysisFPGATest.cpp
|
||||
RugnuxStructureTest.cpp
|
||||
SweepLayoutTest.cpp
|
||||
BrukerReaderTest.cpp
|
||||
|
||||
@@ -165,7 +165,7 @@ namespace {
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(4);
|
||||
service.SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(puller ? FPGAImagePullerUse::Always : FPGAImagePullerUse::Never);
|
||||
service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller);
|
||||
service.RugnuxLive(rugnux, nullptr, 2, 30);
|
||||
|
||||
PixelMask mask(x);
|
||||
@@ -419,7 +419,7 @@ TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") {
|
||||
CollectingPusher pusher;
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(4).SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(puller ? FPGAImagePullerUse::Always : FPGAImagePullerUse::Never);
|
||||
service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller);
|
||||
PixelMask mask(x);
|
||||
service.Start(x, mask, nullptr);
|
||||
const auto out = service.Stop();
|
||||
@@ -541,7 +541,7 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Receiver", "[FPGAImagePuller]") {
|
||||
pusher.delay = 3s;
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(1).SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(FPGAImagePullerUse::Always, 2);
|
||||
service.FPGAImagePullerSettings(2, false, 0, true);
|
||||
|
||||
PixelMask mask(x);
|
||||
const JFCalibration calib = Calibration(x);
|
||||
@@ -654,7 +654,7 @@ TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") {
|
||||
CollectingPusher pusher;
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(FPGAImagePullerUse::Always);
|
||||
service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true);
|
||||
PixelMask mask(x);
|
||||
const JFCalibration calib = Calibration(x);
|
||||
service.Start(x, mask, &calib);
|
||||
@@ -689,7 +689,7 @@ TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]")
|
||||
CollectingPusher pusher;
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(4).SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(FPGAImagePullerUse::Always);
|
||||
service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true);
|
||||
PixelMask mask(x);
|
||||
const JFCalibration calib = Calibration(x);
|
||||
service.Start(x, mask, &calib);
|
||||
@@ -860,7 +860,7 @@ TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") {
|
||||
HDF5FilePusher pusher;
|
||||
JFJochReceiverService service(devices, logger, pusher, 256);
|
||||
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
||||
service.UseFPGAImagePuller(FPGAImagePullerUse::Always);
|
||||
service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true);
|
||||
PixelMask mask(x);
|
||||
const JFCalibration calib = Calibration(x);
|
||||
service.Start(x, mask, &calib);
|
||||
|
||||
@@ -157,8 +157,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_crystal_spot_and_index_GPUStills", "[JFJoch
|
||||
service.Indexing(experiment.GetIndexingSettings());
|
||||
// Serial data with rugnux on: the converted frames are analysed on the GPU, after the pre-pass.
|
||||
service.RugnuxLive(true, nullptr, 4, 60);
|
||||
// The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills
|
||||
service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never));
|
||||
|
||||
// No progress value at the start of measurement
|
||||
REQUIRE(!service.GetProgress().has_value());
|
||||
@@ -236,8 +234,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_GPUStills_writer_down", "[JFJochReceiver]")
|
||||
service.NumThreads(4);
|
||||
service.Indexing(experiment.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60);
|
||||
// The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills
|
||||
service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never));
|
||||
|
||||
service.Start(experiment, pixel_mask, nullptr);
|
||||
REQUIRE_THROWS_AS(service.Stop(), JFJochException);
|
||||
@@ -279,8 +275,6 @@ TEST_CASE("JFJochIntegrationTest_GPUStills_cancel_during_prepass", "[JFJochRecei
|
||||
service.NumThreads(4);
|
||||
service.Indexing(experiment.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60);
|
||||
// The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills
|
||||
service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never));
|
||||
|
||||
service.Start(experiment, pixel_mask, nullptr);
|
||||
std::this_thread::sleep_for(std::chrono::seconds(3));
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "../common/ImageBuffer.h"
|
||||
#include "../receiver/StillsGPUAnalysisFPGA.h"
|
||||
|
||||
namespace {
|
||||
DiffractionExperiment StillsExperiment() {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
x.ImagesPerTrigger(StillsGPUAnalysisFPGA::PREPASS_IMAGES + 200).NumTriggers(1).DetectorDistance_mm(100).BeamX_pxl(500).BeamY_pxl(250)
|
||||
.IncidentEnergy_keV(12.4);
|
||||
return x;
|
||||
}
|
||||
|
||||
// A frame after the pre-pass's sample (numbers from PREPASS_IMAGES on are never in it), in a slot of `buffer`.
|
||||
std::shared_ptr<HeldImage> Frame(ImageBuffer &buffer, int64_t number) {
|
||||
auto held = std::make_shared<HeldImage>();
|
||||
held->slot = buffer.GetImageSlot();
|
||||
REQUIRE(held->slot != nullptr);
|
||||
held->message.number = StillsGPUAnalysisFPGA::PREPASS_IMAGES + number;
|
||||
held->message.image_collection_efficiency = 1.0f;
|
||||
return held;
|
||||
}
|
||||
|
||||
StartMessage Start(const DiffractionExperiment &x) {
|
||||
StartMessage start;
|
||||
x.FillMessage(start);
|
||||
return start;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("StillsGPUAnalysisFPGA_StartFailureReleasesThreads", "[StillsGPUAnalysisFPGA]") {
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
const DiffractionExperiment x = StillsExperiment();
|
||||
const PixelMask mask(x);
|
||||
ImageBuffer buffer(64 * 1024 * 1024);
|
||||
buffer.StartMeasurement(x);
|
||||
RugnuxLiveStatus status;
|
||||
|
||||
std::atomic<int> failed = 0, sunk = 0;
|
||||
{
|
||||
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 4, 100, logger,
|
||||
[](const StillsGPUAnalysisFPGA::Setup &) {
|
||||
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message");
|
||||
},
|
||||
[&](DataMessage &, const AzimuthalIntegrationProfile &) { ++sunk; },
|
||||
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); },
|
||||
[&](const JFJochException &) { ++failed; }, &status);
|
||||
for (int i = 0; i < 3; i++)
|
||||
REQUIRE(gpu.Hold(Frame(buffer, i)));
|
||||
REQUIRE_THROWS_AS(gpu.Finish(), JFJochException);
|
||||
}
|
||||
CHECK(failed == 1);
|
||||
CHECK(sunk == 0);
|
||||
CHECK(status.Get().state == RugnuxLiveState::Failed);
|
||||
// Every slot the frames were held in is back.
|
||||
CHECK(buffer.GetAvailSlots() == buffer.GetStatus().total_slots);
|
||||
}
|
||||
|
||||
TEST_CASE("StillsGPUAnalysisFPGA_HoldIsBounded", "[StillsGPUAnalysisFPGA]") {
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
const DiffractionExperiment x = StillsExperiment();
|
||||
const PixelMask mask(x);
|
||||
ImageBuffer buffer(64 * 1024 * 1024);
|
||||
buffer.StartMeasurement(x);
|
||||
REQUIRE(buffer.GetStatus().total_slots >= 4);
|
||||
|
||||
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 2, 3, logger,
|
||||
[](const StillsGPUAnalysisFPGA::Setup &) {
|
||||
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "no writer");
|
||||
},
|
||||
[](DataMessage &, const AzimuthalIntegrationProfile &) {},
|
||||
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); });
|
||||
// The analysis waits for the pre-pass, which waits for its sample: nothing is taken off the queue.
|
||||
for (int i = 0; i < 3; i++)
|
||||
REQUIRE(gpu.Hold(Frame(buffer, i)));
|
||||
{
|
||||
const auto refused = Frame(buffer, 3);
|
||||
CHECK(!gpu.Hold(refused));
|
||||
}
|
||||
CHECK(buffer.GetStatus().preparation_slots == 3);
|
||||
REQUIRE_THROWS(gpu.Finish());
|
||||
CHECK(buffer.GetStatus().preparation_slots == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("StillsGPUAnalysisFPGA_FailingFrameDoesNotStopAnalysis", "[StillsGPUAnalysisFPGA]") {
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
const DiffractionExperiment x = StillsExperiment();
|
||||
const PixelMask mask(x);
|
||||
ImageBuffer buffer(64 * 1024 * 1024);
|
||||
buffer.StartMeasurement(x);
|
||||
RugnuxLiveStatus status;
|
||||
|
||||
std::atomic<int> sunk = 0;
|
||||
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 1, 100, logger,
|
||||
[](const StillsGPUAnalysisFPGA::Setup &) {},
|
||||
[&](DataMessage &m, const AzimuthalIntegrationProfile &) {
|
||||
++sunk;
|
||||
if (m.number == StillsGPUAnalysisFPGA::PREPASS_IMAGES)
|
||||
throw std::runtime_error("not a JFJochException");
|
||||
},
|
||||
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); },
|
||||
{}, &status);
|
||||
// The frames carry no image, so their analysis fails too - they are passed on all the same.
|
||||
for (int i = 0; i < 3; i++)
|
||||
REQUIRE(gpu.Hold(Frame(buffer, i)));
|
||||
REQUIRE_NOTHROW(gpu.Finish());
|
||||
CHECK(sunk == 3);
|
||||
// The outcome waits for everything to be written, and comes after the latency.
|
||||
CHECK(status.Get().state == RugnuxLiveState::Running);
|
||||
CHECK(!status.Get().last_image_to_written_s);
|
||||
gpu.Written();
|
||||
const auto s = status.Get();
|
||||
CHECK(s.state == RugnuxLiveState::Failed);
|
||||
CHECK(!s.message.empty());
|
||||
REQUIRE(s.last_image_to_written_s);
|
||||
REQUIRE(s.last_image_s);
|
||||
CHECK(s.elapsed_s >= s.last_image_s.value() + s.last_image_to_written_s.value() - 1e-6);
|
||||
REQUIRE(!s.steps.empty());
|
||||
CHECK(s.steps.back().title == "Writing");
|
||||
CHECK(s.steps.back().duration_s);
|
||||
CHECK(buffer.GetStatus().preparation_slots == 0);
|
||||
}
|
||||
@@ -41,8 +41,6 @@ void print_usage(Logger &logger) {
|
||||
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes)");
|
||||
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
|
||||
logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped");
|
||||
logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller (not blocking: frames");
|
||||
logger.Info(" past what its analysis holds are lost)");
|
||||
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
|
||||
logger.Info(" 128 module frames)");
|
||||
}
|
||||
@@ -88,7 +86,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
|
||||
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
|
||||
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
|
||||
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, size_t puller_slots,
|
||||
bool fpga_receiver, const std::vector<double> &rates, bool dropping) {
|
||||
const std::vector<double> &rates, bool dropping) {
|
||||
RegisterHDF5Filter();
|
||||
JFJochHDF5Reader reader;
|
||||
reader.ReadFile(file);
|
||||
@@ -166,13 +164,10 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
|
||||
for (const double rate: rates) {
|
||||
// Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer,
|
||||
// the cards hold their frames and their generators stall. JFJochReceiverFPGA's live stills cannot block.
|
||||
service.UseFPGAImagePuller(fpga_receiver ? FPGAImagePullerUse::Never : FPGAImagePullerUse::RugnuxLive,
|
||||
puller_slots, !dropping, rate);
|
||||
// the cards hold their frames and their generators stall
|
||||
service.FPGAImagePullerSettings(puller_slots, !dropping, rate);
|
||||
logger.Info("");
|
||||
logger.Info("=== {}, {} ===", fpga_receiver ? "JFJochReceiverFPGA (not blocking)"
|
||||
: fmt::format("FPGAImagePuller ({} slots, {})", puller_slots,
|
||||
dropping ? "dropping" : "blocking"),
|
||||
logger.Info("=== FPGAImagePuller ({} slots, {}), {} ===", puller_slots, dropping ? "dropping" : "blocking",
|
||||
(rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes"));
|
||||
|
||||
rusage usage_start{};
|
||||
@@ -289,14 +284,13 @@ int main(int argc, char **argv) {
|
||||
IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto;
|
||||
uint16_t indexing_threads = 4;
|
||||
size_t puller_slots = FPGAImagePuller::DefaultSlots;
|
||||
bool fpga_receiver = false;
|
||||
std::optional<uint32_t> mock_card_MiB;
|
||||
bool verbose = false;
|
||||
std::vector<double> rates = {0};
|
||||
bool dropping = false;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dFK::")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dK::")) != -1) {
|
||||
switch (opt) {
|
||||
case 'v':
|
||||
verbose = true;
|
||||
@@ -346,9 +340,6 @@ int main(int argc, char **argv) {
|
||||
case 'p':
|
||||
puller_slots = atol(optarg);
|
||||
break;
|
||||
case 'F':
|
||||
fpga_receiver = true;
|
||||
break;
|
||||
case 'd':
|
||||
dropping = true;
|
||||
break;
|
||||
@@ -387,7 +378,7 @@ int main(int argc, char **argv) {
|
||||
bool ret = false;
|
||||
try {
|
||||
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
|
||||
i_settings, cell, puller_slots, fpga_receiver, rates, dropping);
|
||||
i_settings, cell, puller_slots, rates, dropping);
|
||||
} catch (std::exception &e) {
|
||||
logger.Error(e.what());
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user