Merge branch 'rc188-i' into rc188-int

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-18 22:23:42 +01:00
commit fdeb3cfddb
90 files changed
+3344 -1253

No files matched your search

+1 -1
View File
@@ -31,7 +31,7 @@ void AcquisitionCounters::Reset(const DiffractionExperiment &experiment, uint16_
curr_frame_number[i] = 0;
handle_for_frame = std::vector<uint64_t>(expected_frames * nmodules, HandleNotFound);
packets_collected = std::vector<uint16_t>(expected_frames * nmodules);
packets_collected = std::vector<uint32_t>(expected_frames * nmodules);
packets_per_module = std::vector<uint64_t>(nmodules);
total_packets = 0;
expected_packets_per_module = 512 * experiment.GetFPGASummation();
+1 -1
View File
@@ -21,7 +21,7 @@ class AcquisitionCounters {
mutable std::condition_variable_any data_updated;
std::vector<uint64_t> handle_for_frame;
std::vector<uint16_t> packets_collected;
std::vector<uint32_t> packets_collected;
uint64_t total_packets;
std::vector<uint64_t> packets_per_module;
+10
View File
@@ -1,6 +1,7 @@
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <bit>
#include <thread>
#include <fstream>
#include <cmath>
@@ -54,6 +55,7 @@ void AcquisitionDevice::StartAction(const DiffractionExperiment &experiment, uin
counters.Reset(experiment, data_stream);
expected_frames = experiment.GetFrameNum() / experiment.GetFPGASummation();
fpga_summation = experiment.GetFPGASummation();
// Ensure internal WR queue is empty
work_request_queue.Clear();
@@ -92,6 +94,14 @@ void AcquisitionDevice::WaitForActionComplete() {
}
if (!skip_analysis) {
// The card counts the packets of a summed module in 16 bits, which wraps from 128 frames on (512 a frame):
// counted here from those present in all the frames
if (512 * fpga_summation > UINT16_MAX) {
uint32_t packets = 0;
for (const auto &mask: output->module_statistics.packet_mask)
packets += std::popcount(mask);
output->module_statistics.packet_count = packets * fpga_summation;
}
c.module_number = output->module_statistics.module_number;
c.packet_count = output->module_statistics.packet_count;
c.frame_number = output->module_statistics.frame_number;
+2
View File
@@ -38,6 +38,7 @@ class AcquisitionDevice {
std::chrono::time_point<std::chrono::system_clock> end_time;
int64_t expected_frames;
int64_t fpga_summation = 1;
virtual void Start(const DiffractionExperiment& experiment, uint32_t flags) = 0;
virtual void Finalize() {}; // do clean-up after action is done
@@ -123,5 +124,6 @@ public:
void RunInternalGenerator(const DiffractionExperiment& experiment);
// The module frame of the internal frame generator's images, throws if the card memory cannot hold it
void CheckInternalGeneratorFrame(size_t module_frame) const;
uint64_t GetInternalGeneratorModuleFrames() const { return internal_generator_module_frames; }
};
+1 -1
View File
@@ -14,7 +14,7 @@ struct Completion {
uint64_t frame_number;
uint32_t handle;
uint16_t packet_count;
uint32_t packet_count;
uint16_t data_collection_id;
uint16_t module_number;
};
+1 -8
View File
@@ -40,15 +40,8 @@ void FPGAAcquisitionDevice::ReadWorkCompletionThread() {
}
if (read) {
// Frame and module of an image completion are read from its buffer later (WaitForActionComplete)
c = parse_hw_completion(values);
if (logger) {
logger->Debug("Stream {} Completion data collection ID {} type {} frame {} module {}",
data_stream,
c.data_collection_id,
(int)c.type,
c.frame_number,
c.module_number);
}
if (c.data_collection_id == data_collection_id) {
work_completion_queue.PutBlocking(c);
if (c.type == Completion::Type::End)
+10 -6
View File
@@ -6,9 +6,6 @@
#include "MockAcquisitionDevice.h"
namespace {
// Memory of one HBM interface, as the HLS simulation has it: 64 module frames for the internal generator
constexpr uint32_t CardMemoryBytes = 32 * 1024 * 1024;
bool MaskEmpty(const uint64_t packet_mask[8]) {
for (int i = 0; i < 8; i++) {
if (packet_mask[i] != 0)
@@ -18,8 +15,10 @@ namespace {
}
}
MockAcquisitionDevice::MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules)
MockAcquisitionDevice::MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules,
uint32_t in_card_memory_bytes)
: FPGAAcquisitionDevice(data_stream),
card_memory_bytes(in_card_memory_bytes),
collected_modules(2 * MAX_MODULES_FPGA),
summation_completions(MAX_MODULES_FPGA * MAX_FPGA_SUMMATION),
summation_mask(MAX_MODULES_FPGA, std::vector<bool>(MAX_FPGA_SUMMATION)),
@@ -29,7 +28,7 @@ MockAcquisitionDevice::MockAcquisitionDevice(uint16_t data_stream, size_t in_fra
ipv4_addr = 0x0132010A;
max_modules = MAX_MODULES_FPGA;
internal_generator_module_frames = GeneratorModuleFrames(CardMemoryBytes);
internal_generator_module_frames = GeneratorModuleFrames(card_memory_bytes);
generator_frames.resize(internal_generator_module_frames);
buffers.reserve(in_frame_buffer_size_modules);
@@ -268,6 +267,7 @@ int MockAcquisitionDevice::FrameGenerator(FrameGeneratorConfig config) {
void MockAcquisitionDevice::DataCollectionThread() {
cancel = false;
stop_frame_generator = false;
stalls_host = 0;
for (auto &c: collected_modules)
c = ModuleCompletion{.frame_number = UINT64_MAX};
@@ -456,11 +456,14 @@ void MockAcquisitionDevice::WriteToHost(const ModuleCompletion &c) {
// host_writer.cpp: waits for a host buffer; only when the data collection is cancelled the module
// is dropped instead
uint32_t handle;
const auto t0 = std::chrono::steady_clock::now();
while (!work_requests.Get(handle)) {
if (cancel)
return;
std::this_thread::sleep_for(std::chrono::microseconds(10));
}
// timer_host: the wait in cycles of the card's 200 MHz clock
stalls_host += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count() / 5;
if (handle >= buffer_device.size())
return;
DeviceOutput *output = buffer_device[handle];
@@ -534,11 +537,12 @@ DataCollectionStatus MockAcquisitionDevice::GetDataCollectionStatus() const {
status.ctrl_reg = idle ? (1 << 4) : 0;
status.max_modules = MAX_MODULES_FPGA;
status.hbm_size_bytes = CardMemoryBytes;
status.hbm_size_bytes = card_memory_bytes;
status.run_counter = run_counter;
status.current_pulseid = pulse_id;
status.packets_udp = udp_packets;
status.packets_sls = sls_packets;
status.pipeline_stalls_host = stalls_host;
return status;
}
+7 -1
View File
@@ -62,6 +62,7 @@ class MockAcquisitionDevice : public FPGAAcquisitionDevice {
uint32_t generator_slot;
};
const uint32_t card_memory_bytes;
std::vector<std::vector<std::unique_ptr<DeviceOutput>>> outputs;
// The frames of the internal generator, by slot (module frame of the card memory)
std::vector<std::vector<uint16_t>> generator_frames;
@@ -99,6 +100,7 @@ class MockAcquisitionDevice : public FPGAAcquisitionDevice {
std::atomic<uint64_t> current_pulse_id = UINT64_MAX;
std::atomic<uint64_t> udp_packets = 0;
std::atomic<uint64_t> sls_packets = 0;
std::atomic<uint64_t> stalls_host = 0;
void HW_ReadActionRegister(DataCollectionConfig *job) override;
void HW_WriteActionRegister(const DataCollectionConfig *job) override;
@@ -123,7 +125,11 @@ class MockAcquisitionDevice : public FPGAAcquisitionDevice {
void WriteToHost(const ModuleCompletion &c);
void EndDataCollection();
public:
MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules);
// Memory of one HBM interface, as the HLS simulation had it: 64 module frames for the internal generator
static constexpr uint32_t DefaultCardMemoryBytes = 32 * 1024 * 1024;
MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules,
uint32_t card_memory_bytes = DefaultCardMemoryBytes);
~MockAcquisitionDevice() override;
void SetOutput(uint64_t image, uint32_t module_number, const DeviceOutput &output);
+2 -1
View File
@@ -1018,6 +1018,7 @@ void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) {
ret.setDeadlineS(s.deadline_s);
ret.setRefineSolvent(s.refine_solvent);
ret.setForceStill(s.force_still);
ret.setStillsDownsample(s.stills_downsample);
ret.setModelLoaded(s.model != nullptr);
if (s.model)
ret.setModelSizeBytes(s.model_size_bytes);
@@ -1028,7 +1029,7 @@ void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings,
httplib::Response &response) {
state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(),
rugnuxSettings.getDeadlineS(), rugnuxSettings.isRefineSolvent(),
rugnuxSettings.isForceStill());
rugnuxSettings.isForceStill(), rugnuxSettings.getStillsDownsample());
response.status = 200;
}
+2 -1
View File
@@ -283,7 +283,8 @@ void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings
.reserved_cpus = input.getRugnux().getReservedCpus(),
.deadline_s = input.getRugnux().getDeadlineS(),
.refine_solvent = input.getRugnux().isRefineSolvent(),
.force_still = input.getRugnux().isForceStill()
.force_still = input.getRugnux().isForceStill(),
.stills_downsample = input.getRugnux().getStillsDownsample()
});
}
+1 -1
View File
@@ -385,7 +385,7 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) {
void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) {
if (receiver)
receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s, input.refine_solvent,
input.force_still);
input.force_still, input.stills_downsample);
rugnux = input;
}
+1
View File
@@ -20,6 +20,7 @@ struct RugnuxLiveSettings {
double deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
bool refine_solvent = false; // the refinement of a placed model builds the solvent (rugnux --refine-solvent)
bool force_still = false; // rotation collections are processed as stills (rugnux --force-still)
int64_t stills_downsample = 1; // serial collections: 1 of every this many images analysed on average
std::shared_ptr<const RugnuxModel> model; // null = no model
int64_t model_size_bytes = 0; // as uploaded
};
+3 -1
View File
@@ -343,6 +343,7 @@ void JFJochStateMachine::Initialize() {
s.enable = false;
s.refine_solvent = false;
s.force_still = false;
s.stills_downsample = 1;
s.model = nullptr;
s.model_size_bytes = 0;
services.SetRugnuxLive(s);
@@ -1226,7 +1227,7 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const {
}
void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent,
bool force_still) {
bool force_still, int64_t stills_downsample) {
std::unique_lock ul(m);
if (IsRunning())
throw WrongDAQStateException("Cannot change rugnux settings during data collection");
@@ -1237,6 +1238,7 @@ void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, doubl
s.deadline_s = deadline_s;
s.refine_solvent = refine_solvent;
s.force_still = force_still;
s.stills_downsample = stills_downsample;
services.SetRugnuxLive(s);
}
+2 -1
View File
@@ -248,7 +248,8 @@ public:
void SetIndexingSettings(const IndexingSettings &input);
IndexingSettings GetIndexingSettings() const;
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still);
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still,
int64_t stills_downsample = 1);
void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes); // null = remove the model
RugnuxLiveSettings GetRugnuxLive() const;
RugnuxLiveSnapshot GetRugnuxStatus() const;
+2 -1
View File
@@ -728,7 +728,8 @@ DatasetSettings Convert(const org::openapitools::server::model::Dataset_settings
ret.SampleTemperature_K(input.getSampleTemperatureK());
ret.SpotFindingEnable(input.isSpotFinding());
ret.MaxSpotCount(input.getMaxSpotCount());
if (input.maxSpotCountIsSet())
ret.MaxSpotCount(input.getMaxSpotCount());
if (input.detectIceRingsIsSet())
ret.DetectIceRings(input.isDetectIceRings());
+1 -1
View File
@@ -85,7 +85,7 @@ Dataset_settings::Dataset_settings()
m_Spot_finding = true;
m_Spot_findingIsSet = false;
m_SmargonIsSet = false;
m_Max_spot_count = 250;
m_Max_spot_count = 0;
m_Max_spot_countIsSet = false;
m_Detect_ice_rings = false;
m_Detect_ice_ringsIsSet = false;
+2 -2
View File
@@ -81,7 +81,7 @@ public:
bool ntriggerIsSet() const;
void unsetNtrigger();
/// <summary>
/// Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time. In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved.
/// Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of frames is summed on the host entirely). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved.
/// </summary>
int64_t getImageTimeUs() const;
void setImageTimeUs(int64_t const value);
@@ -318,7 +318,7 @@ public:
bool smargonIsSet() const;
void unsetSmargon();
/// <summary>
/// Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected
/// Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected. Left out, the broker&#39;s per-image indexing keeps 250 and rugnux live stills 1000.
/// </summary>
int32_t getMaxSpotCount() const;
void setMaxSpotCount(int32_t const value);
+45 -2
View File
@@ -31,6 +31,8 @@ Rugnux_settings::Rugnux_settings()
m_Refine_solventIsSet = false;
m_Force_still = false;
m_Force_stillIsSet = false;
m_Stills_downsample = 1L;
m_Stills_downsampleIsSet = false;
}
@@ -81,7 +83,21 @@ bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPr
}
}
if (stillsDownsampleIsSet())
{
const int64_t& value = m_Stills_downsample;
const std::string currentValuePath = _pathPrefix + ".stillsDownsample";
if (value < 1ll)
{
success = false;
msg << currentValuePath << ": must be greater than or equal to 1;";
}
}
return success;
}
@@ -103,7 +119,10 @@ bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const
((!refineSolventIsSet() && !rhs.refineSolventIsSet()) || (refineSolventIsSet() && rhs.refineSolventIsSet() && isRefineSolvent() == rhs.isRefineSolvent())) &&
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill()))
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill())) &&
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample()))
;
}
@@ -126,6 +145,8 @@ void to_json(nlohmann::json& j, const Rugnux_settings& o)
j["refine_solvent"] = o.m_Refine_solvent;
if(o.forceStillIsSet())
j["force_still"] = o.m_Force_still;
if(o.stillsDownsampleIsSet())
j["stills_downsample"] = o.m_Stills_downsample;
}
@@ -156,6 +177,11 @@ void from_json(const nlohmann::json& j, Rugnux_settings& o)
j.at("force_still").get_to(o.m_Force_still);
o.m_Force_stillIsSet = true;
}
if(j.find("stills_downsample") != j.end())
{
j.at("stills_downsample").get_to(o.m_Stills_downsample);
o.m_Stills_downsampleIsSet = true;
}
}
@@ -244,6 +270,23 @@ void Rugnux_settings::unsetForce_still()
{
m_Force_stillIsSet = false;
}
int64_t Rugnux_settings::getStillsDownsample() const
{
return m_Stills_downsample;
}
void Rugnux_settings::setStillsDownsample(int64_t const value)
{
m_Stills_downsample = value;
m_Stills_downsampleIsSet = true;
}
bool Rugnux_settings::stillsDownsampleIsSet() const
{
return m_Stills_downsampleIsSet;
}
void Rugnux_settings::unsetStills_downsample()
{
m_Stills_downsampleIsSet = false;
}
} // namespace org::openapitools::server::model
+9
View File
@@ -92,6 +92,13 @@ public:
void setForceStill(bool const value);
bool forceStillIsSet() const;
void unsetForce_still();
/// <summary>
/// Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by &#x60;POST /initialize&#x60;.
/// </summary>
int64_t getStillsDownsample() const;
void setStillsDownsample(int64_t const value);
bool stillsDownsampleIsSet() const;
void unsetStills_downsample();
friend void to_json(nlohmann::json& j, const Rugnux_settings& o);
friend void from_json(const nlohmann::json& j, Rugnux_settings& o);
@@ -106,6 +113,8 @@ protected:
bool m_Refine_solventIsSet;
bool m_Force_still;
bool m_Force_stillIsSet;
int64_t m_Stills_downsample;
bool m_Stills_downsampleIsSet;
};
+30 -1
View File
@@ -27,6 +27,8 @@ Rugnux_status::Rugnux_status()
m_Deadline_sIsSet = false;
m_Refine_solvent = false;
m_Force_still = false;
m_Stills_downsample = 0L;
m_Stills_downsampleIsSet = false;
m_Model_loaded = false;
m_Model_size_bytes = 0L;
m_Model_size_bytesIsSet = false;
@@ -52,7 +54,7 @@ bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPref
bool success = true;
const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix;
return success;
}
@@ -76,6 +78,9 @@ bool Rugnux_status::operator==(const Rugnux_status& rhs) const
(isForceStill() == rhs.isForceStill())
&&
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample())) &&
(isModelLoaded() == rhs.isModelLoaded())
&&
@@ -99,6 +104,8 @@ void to_json(nlohmann::json& j, const Rugnux_status& o)
j["deadline_s"] = o.m_Deadline_s;
j["refine_solvent"] = o.m_Refine_solvent;
j["force_still"] = o.m_Force_still;
if(o.stillsDownsampleIsSet())
j["stills_downsample"] = o.m_Stills_downsample;
j["model_loaded"] = o.m_Model_loaded;
if(o.modelSizeBytesIsSet())
j["model_size_bytes"] = o.m_Model_size_bytes;
@@ -116,6 +123,11 @@ void from_json(const nlohmann::json& j, Rugnux_status& o)
}
j.at("refine_solvent").get_to(o.m_Refine_solvent);
j.at("force_still").get_to(o.m_Force_still);
if(j.find("stills_downsample") != j.end())
{
j.at("stills_downsample").get_to(o.m_Stills_downsample);
o.m_Stills_downsampleIsSet = true;
}
j.at("model_loaded").get_to(o.m_Model_loaded);
if(j.find("model_size_bytes") != j.end())
{
@@ -174,6 +186,23 @@ void Rugnux_status::setForceStill(bool const value)
{
m_Force_still = value;
}
int64_t Rugnux_status::getStillsDownsample() const
{
return m_Stills_downsample;
}
void Rugnux_status::setStillsDownsample(int64_t const value)
{
m_Stills_downsample = value;
m_Stills_downsampleIsSet = true;
}
bool Rugnux_status::stillsDownsampleIsSet() const
{
return m_Stills_downsampleIsSet;
}
void Rugnux_status::unsetStills_downsample()
{
m_Stills_downsampleIsSet = false;
}
bool Rugnux_status::isModelLoaded() const
{
return m_Model_loaded;
+9
View File
@@ -85,6 +85,13 @@ public:
bool isForceStill() const;
void setForceStill(bool const value);
/// <summary>
/// Serial collections, 1 of every this many images analysed on average
/// </summary>
int64_t getStillsDownsample() const;
void setStillsDownsample(int64_t const value);
bool stillsDownsampleIsSet() const;
void unsetStills_downsample();
/// <summary>
/// A model was uploaded with &#x60;PUT /config/rugnux/model&#x60;
/// </summary>
bool isModelLoaded() const;
@@ -110,6 +117,8 @@ protected:
bool m_Force_still;
int64_t m_Stills_downsample;
bool m_Stills_downsampleIsSet;
bool m_Model_loaded;
int64_t m_Model_size_bytes;
+22 -3
View File
@@ -409,7 +409,10 @@ components:
description: |
Image time.
If not provided (or zero value) the frame time is assumed as default.
For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time.
For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA.
Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image
(250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of
frames is summed on the host entirely).
In XFEL mode: summation happens for frames collected with multiple triggers.
Ignored for storage cells and if raw data are saved.
beam_x_pxl:
@@ -667,8 +670,9 @@ components:
type: integer
minimum: 10
maximum: 2000
default: 250
description: Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected
description: |
Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected.
Left out, the broker's per-image indexing keeps 250 and rugnux live stills 1000.
detect_ice_rings:
type: boolean
description: |
@@ -2841,6 +2845,17 @@ components:
description: |
Process DECTRIS rotation collections as stills, on the serial path (pre-pass, GPU analysis, stills merge),
as `rugnux --force-still` does. The written file keeps its goniometer. Reset to false by `POST /initialize`.
stills_downsample:
type: integer
format: int64
minimum: 1
default: 1
description: |
Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came,
without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
rugnux_status:
type: object
required:
@@ -2867,6 +2882,10 @@ components:
force_still:
type: boolean
description: DECTRIS rotation collections are processed as stills
stills_downsample:
type: integer
format: int64
description: Serial collections, 1 of every this many images analysed on average
model_loaded:
type: boolean
description: A model was uploaded with `PUT /config/rugnux/model`
+4 -1
View File
@@ -29,7 +29,6 @@ DatasetSettings::DatasetSettings() {
poni_rot_2_rad = 0.0f;
poni_rot_3_rad = 0.0f;
max_spot_count = std::min(MAX_SPOT_COUNT, 250);
}
DatasetSettings &DatasetSettings::ImagesPerTrigger(int64_t input) {
@@ -557,6 +556,10 @@ const XrayFluorescenceSpectrum & DatasetSettings::GetFluorescenceSpectrum() cons
}
int64_t DatasetSettings::GetMaxSpotCount() const {
return max_spot_count.value_or(DEFAULT_MAX_SPOT_COUNT);
}
std::optional<int64_t> DatasetSettings::GetMaxSpotCountGiven() const {
return max_spot_count;
}
+3 -2
View File
@@ -73,7 +73,7 @@ class DatasetSettings {
float poni_rot_3_rad;
bool spot_finding_enable;
int64_t max_spot_count;
std::optional<int64_t> max_spot_count; // unset: each analysis takes its own default
// Special handling of ice rings; unset where the user did not ask either way, so that the analysis takes its own
// default (rugnux: on for a rotation sweep, off for stills) and the master file carries no detect_ice_rings
@@ -186,7 +186,8 @@ public:
std::optional<float> GetSampleTemperature_K() const;
bool IsSpotFindingEnabled() const;
int64_t GetMaxSpotCount() const;
int64_t GetMaxSpotCount() const; // DEFAULT_MAX_SPOT_COUNT where none is given
std::optional<int64_t> GetMaxSpotCountGiven() const;
bool IsDetectIceRings() const; // false where unset
std::optional<bool> GetDetectIceRings() const;
+2
View File
@@ -47,6 +47,8 @@ constexpr float DEFAULT_G2_FACTOR = -0.1145f;
constexpr float DEFAULT_HG0_FACTOR = 100.0f;
constexpr int MAX_SPOT_COUNT = 64 * 1024;
// Spots kept per image where none is given: the broker's per-image indexing (rugnux live stills takes its own default)
constexpr int DEFAULT_MAX_SPOT_COUNT = 250;
// Three spots fit any lattice at all; one short of ViableCellMinSpots (default 9) is where a fitted
// lattice stops being distinguishable from a random one, so a spot list shorter than this is never
+2 -2
View File
@@ -13,9 +13,9 @@ DetectorSettings &DetectorSettings::InternalGeneratorEnable(bool input) {
return *this;
}
DetectorSettings &DetectorSettings::InternalGeneratorImages(int64_t input) {
DetectorSettings &DetectorSettings::InternalGeneratorImages(int64_t input, int64_t max_images) {
check_min("Internal packet generator images", input, 1);
check_max("Internal packet generator images", input, 64);
check_max("Internal packet generator images", input, max_images);
internal_fpga_packet_generator_images = input;
return *this;
}
+3 -1
View File
@@ -37,7 +37,9 @@ class DetectorSettings {
DetectorTiming timing = DetectorTiming::Trigger;
public:
DetectorSettings& InternalGeneratorEnable(bool input);
DetectorSettings& InternalGeneratorImages(int64_t input);
// The broker's limit; the firmware's own is FPGA_INTERNAL_GENERATOR_MAX_IMAGES
static constexpr int64_t DefaultMaxInternalGeneratorImages = 64;
DetectorSettings& InternalGeneratorImages(int64_t input, int64_t max_images = DefaultMaxInternalGeneratorImages);
DetectorSettings& StorageCells(int64_t input);
DetectorSettings& StorageCellStart(int64_t input);
DetectorSettings& StorageCellDelay(const std::chrono::nanoseconds &input);
+25 -16
View File
@@ -599,6 +599,10 @@ DiffractionExperiment &DiffractionExperiment::MaxSpotCount(int64_t input) {
return *this;
}
std::optional<int64_t> DiffractionExperiment::GetMaxSpotCountGiven() const {
return dataset.GetMaxSpotCountGiven();
}
int64_t DiffractionExperiment::GetMaxSpotCount() const {
if (!IsSpotFindingEnabled())
return 0;
@@ -795,12 +799,12 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const {
if (GetSummation() == 1)
message.summation_mode = "none";
else {
if (IsCPUSummation())
message.summation_mode = "cpu";
else
message.summation_mode = "fpga";
}
else if (!IsCPUSummation())
message.summation_mode = "fpga";
else if (GetFPGASummation() > 1)
message.summation_mode = "hybrid";
else
message.summation_mode = "cpu";
}
float DiffractionExperiment::GetPixelSize_mm() const {
@@ -877,14 +881,23 @@ int64_t DiffractionExperiment::GetSummation() const {
}
int64_t DiffractionExperiment::GetFPGASummation() const {
if (IsCPUSummation())
const int64_t n = GetSummation();
if (cpu_summation)
return 1;
return GetSummation();
for (int64_t d = std::min<int64_t>(n, MAX_FPGA_SUMMATION); d > 1; d--)
if (n % d == 0)
return d;
return 1;
}
int64_t DiffractionExperiment::GetHostSummation() const {
return GetSummation() / GetFPGASummation();
}
int64_t DiffractionExperiment::GetByteDepthFPGA() const {
// Summed on the card as well: 32-bit, the host adds to it
if (IsCPUSummation())
return 2;
return (GetFPGASummation() > 1) ? 4 : 2;
return GetByteDepthImage();
}
@@ -1094,8 +1107,8 @@ float DiffractionExperiment::GetPhotonEnergyForConversion_keV() const {
return GetIncidentEnergy_keV();
}
DiffractionExperiment &DiffractionExperiment::InternalPacketGeneratorImages(int64_t input) {
detector_settings.InternalGeneratorImages(input);
DiffractionExperiment &DiffractionExperiment::InternalPacketGeneratorImages(int64_t input, int64_t max_images) {
detector_settings.InternalGeneratorImages(input, max_images);
return *this;
}
@@ -1634,11 +1647,7 @@ DiffractionExperiment &DiffractionExperiment::ApplyPixelMask(bool input) {
}
bool DiffractionExperiment::IsCPUSummation() const {
if (summation == 1)
return false;
if (summation >= MAX_FPGA_SUMMATION)
return true;
return cpu_summation;
return GetHostSummation() > 1;
}
DiffractionExperiment & DiffractionExperiment::ElectronSource(bool input) {
+8 -1
View File
@@ -119,7 +119,8 @@ public:
DiffractionExperiment& ZMQPreviewPeriod(const std::optional<std::chrono::microseconds> &input);
DiffractionExperiment& UseInternalPacketGenerator(bool input);
DiffractionExperiment& InternalPacketGeneratorImages(int64_t input);
DiffractionExperiment& InternalPacketGeneratorImages(int64_t input,
int64_t max_images = DetectorSettings::DefaultMaxInternalGeneratorImages);
DiffractionExperiment& MaskModuleEdges(bool input);
DiffractionExperiment& MaskChipEdges(bool input);
@@ -304,6 +305,7 @@ public:
int64_t GetStorageCellStart() const;
int64_t GetMaxSpotCount() const;
std::optional<int64_t> GetMaxSpotCountGiven() const; // as the user gave it, unset where not
float GetPixelSize_mm() const;
std::string GetSourceName() const;
@@ -355,7 +357,10 @@ public:
std::optional<int64_t> GetSpaceGroupNumber() const;
bool GetSaveCalibration() const;
int64_t GetSummation() const;
// The card sums the largest divisor of the summation up to MAX_FPGA_SUMMATION, the host the card's images of
// one image: GetSummation() = GetFPGASummation() * GetHostSummation()
int64_t GetFPGASummation() const;
int64_t GetHostSummation() const;
std::string GetSampleName() const;
float GetIncidentEnergy_keV() const;
float GetWavelength_A() const;
@@ -422,7 +427,9 @@ public:
[[nodiscard]] bool IsMaskPixelsWithoutG0() const;
[[nodiscard]] bool IsApplyPixelMask() const;
// All of the summation on the host (tests)
DiffractionExperiment& CPUSummation(bool input);
// The host sums part of the image (GetHostSummation() > 1)
[[nodiscard]] bool IsCPUSummation() const;
[[nodiscard]] FileWriterFormat GetFileWriterFormat() const;
+8
View File
@@ -17,6 +17,13 @@ ModuleSummation::ModuleSummation(bool in_pixel_signed, uint32_t in_fpga_depth_by
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "FPGA byte depth can be only 1, 2 or 4");
output = std::make_unique<DeviceOutput>();
Clear();
}
void ModuleSummation::Clear() {
std::unique_lock ul(module_summation_mutex);
first = true;
is_empty = false;
memset(output.get(), 0, sizeof(DeviceOutput));
}
@@ -69,6 +76,7 @@ void ModuleSummation::AddFPGAOutput(const DeviceOutput &input, const std::option
output->module_statistics.err_pixels += input.module_statistics.err_pixels;
output->module_statistics.saturated_pixels += input.module_statistics.saturated_pixels;
output->module_statistics.packet_count += input.module_statistics.packet_count;
output->module_statistics.pixel_sum += input.module_statistics.pixel_sum;
}
// The only function not handled is spot finding
+2
View File
@@ -23,6 +23,8 @@ public:
void AddFPGAOutput(const DeviceOutput& input, const std::optional<uint32_t> &in_fpga_depth_bytes = {});
void AddEmptyOutput();
// Empty, for the next image
void Clear();
[[nodiscard]] const DeviceOutput& GetOutput() const;
[[nodiscard]] DeviceOutput& GetOutput();
+12
View File
@@ -8,6 +8,18 @@
#include "Coord.h"
#include "DiffractionExperiment.h"
// An assembled image back to raw module frames, the multipixels summed (SqueezeMultipixelsToModule)
template<class Td, class Ts>
void SqueezeToRawGeometry(const DiffractionExperiment &experiment, Td *destination, const Ts *source,
Ts min, Ts max, Td dest_min, Td dest_max) {
for (size_t module_number = 0; module_number < experiment.GetModulesNum(); module_number++)
SqueezeMultipixelsToModule<Td, Ts>(destination + module_number * RAW_MODULE_SIZE, source,
experiment.GetModuleSlowDirectionStep(module_number), min, max,
dest_min, dest_max,
experiment.GetModuleFastDirectionStep(module_number),
experiment.GetPixel0OfModuleConv(module_number));
}
// Input coord is column + 1024 * (line + 512 * module)
// Copies result over multipixel - can be used for example for mask calculation
template<class Td, class Ts>
+50
View File
@@ -139,6 +139,56 @@ void TransferModuleAdjustMultipixels(Td *destination, const Ts *source, int64_t
}
}
// Inverse of TransferModuleAdjustMultipixels: each raw pixel of the module is the sum of the pixels its value was
// split over (2 for a multipixel, 4 at a chip corner), as the split conserves the count. A raw pixel is an error
// pixel where any of them is at or below min, saturated where any is at or above max or the sum reaches dest_max.
template<class Td, class Ts>
void SqueezeMultipixelsToModule(Td *destination, const Ts *source, int64_t slow_dir_step, Ts min, Ts max,
Td dest_min, Td dest_max, int64_t fast_dir_step = 1, int64_t offset_0 = 0) {
for (int64_t line = 0; line < RAW_MODULE_LINES; line++) {
int64_t y[2] = {line + ((line > 255) ? 2 : 0), -1};
if (line == 255)
y[1] = 256;
else if (line == 256)
y[1] = 257;
for (int64_t col = 0; col < RAW_MODULE_COLS; col++) {
const int64_t chip = col / 256, i = col % 256;
int64_t x[2] = {i + chip * 258, -1};
if ((i == 255) && (chip < 3))
x[1] = x[0] + 1;
else if ((i == 0) && (chip > 0))
x[1] = x[0] - 1;
int64_t sum = 0;
bool error = false, saturated = false;
for (int64_t yy: y) {
for (int64_t xx: x) {
if ((yy < 0) || (xx < 0))
continue;
const Ts v = source[offset_0 + yy * slow_dir_step + xx * fast_dir_step];
if (v <= min)
error = true;
else if (v >= max)
saturated = true;
else
sum += v;
}
}
Td &out = destination[line * RAW_MODULE_COLS + col];
if (error)
out = dest_min;
else if (saturated || (sum >= dest_max))
out = dest_max;
else if (sum <= dest_min)
out = dest_min + 1;
else
out = static_cast<Td>(sum);
}
}
}
template<class T>
void LineConvtToRaw(T *destination, const T *source, size_t fast_direction_step) {
for (int chip = 0; chip < 4; chip++) {
+2 -1
View File
@@ -857,7 +857,8 @@ jackknife" (1979), Ann. Statist. 7, 1-26
[doi:10.1214/aos/1176344552](https://doi.org/10.1214/aos/1176344552). Its draws are Poisson weights
hashed with the SplitMix64 finaliser of G. L. Steele Jr., D. Lea and C. H. Flood, "Fast splittable
pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). The same finaliser chooses the images rugnux live
stills analyses with `stills_downsample`.
**Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an
Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU
+1 -1
View File
@@ -100,7 +100,7 @@ There are minor differences at the moment:
| - attenuator_transmission | float | Attenuator transmission \[\] | |
| - total_flux | float | Total flux \[ph/s\] | |
| - space_group_number | uint64 | Space group number | |
| - summation_mode | string | Summation mode (internal\|fpga\|cpu) | |
| - summation_mode | string | Summation mode (none\|fpga\|cpu\|hybrid: the card and the host both sum) | |
| - overwrite | bool | Overwrite existing HDF5 files | |
| - file_format | int | File writer format: 0 = only data files, 1 = NXmx legacy external links, 2 = NXmx VDS, 3 = NXmx integrated, 4 = CBF (retired; rejected), 5 = TIFF (retired; rejected), 6 = no file written | |
| - roi | Array(object) | ROI configurations; each has a `name` (the key of its `roi_integrals` entry and of `/entry/roi/<name>` in HDF5) and is one of: | |
+1 -1
View File
@@ -595,7 +595,7 @@ one keep them only where every input has them.
| Path | Class | Content |
|------|-------|---------|
| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode`, `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/` (`bin_width` and one vector per storage cell), `data_collection_efficiency_image`, `direct_beam_x`, `direct_beam_y`, `pixel_mask` (hard-linked as `/entry/instrument/detector/pixel_mask`), `storage_cell_image` (VDS, with storage cells) |
| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode` (`none`, `fpga`, `cpu` or `hybrid`: card and host both sum), `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/` (`bin_width` and one vector per storage cell), `data_collection_efficiency_image`, `direct_beam_x`, `direct_beam_y`, `pixel_mask` (hard-linked as `/entry/instrument/detector/pixel_mask`), `storage_cell_image` (VDS, with storage cells) |
| `/entry/instrument/detector/calibration` | NXcollection | per-channel pedestal / calibration images (bitshuffle-compressed) |
| `/entry/instrument/fluorescence` | NXcollection | XRF spectrum: `energy` [eV], `data` |
| `/entry/user` | NXcollection | scalar values supplied under `header_appendix.hdf5` |
+36 -16
View File
@@ -204,8 +204,7 @@ back to the broker, so with it the route answers 409.
## Online analysis
Without rugnux live mode, and in the FPGA workflow also for a rotation collection with it, the broker analyses each
image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
Without rugnux live mode the broker analyses each image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
of the image and refinement of its geometry (`geom_refinement_algorithm`). A rotation collection is indexed image by image, as a serial
one. The broker neither integrates Bragg reflections nor determines a lattice for the whole rotation sweep, so the
reflections, the per-image B-factor and integrated-reflection count, and the rotation lattice are filled only where
@@ -221,7 +220,9 @@ goniometer and the rugnux live setting:
| Input | Run | Receiver | What it does |
|---|---|---|---|
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | every run | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer; pedestal runs into the pedestal and its RMS; with rugnux live on, a serial run analysed on the GPU after a rugnux pre-pass (see [Serial collections](#serial-collections-experimental)) |
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | pedestal run | `JFJochReceiverPedestal` | the raw frames into the pedestal and its RMS; nothing on any stream |
| FPGA cards | rugnux live off | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer |
| FPGA cards | rugnux live on | `FPGAImagePuller`, then `JFJochReceiverRugnuxRotation` or `JFJochReceiverRugnuxStills` (`JFJochReceiverLite` for a rotation run the image buffer cannot hold) | the cards' frames assembled (and summed above what the cards sum) and compressed on the host into a stream as a DECTRIS detector gives, then taken as the DECTRIS rows below; the pedestal and gain calibration is not written to the file on this path |
| DECTRIS stream | dark-mask sequence | `JFJochReceiverDarkMask` | the dark frames into the mask; nothing on any stream |
| DECTRIS stream | rugnux live off, or a rotation run the image buffer cannot hold | `JFJochReceiverLite` | each image on the GPU: masking, azimuthal and ROI integration, spot finding, indexing |
| DECTRIS stream | rugnux live on, rotation that fits the image buffer | `JFJochReceiverRugnuxRotation` | images forwarded as they come; rugnux reads the collection back from the image buffer and runs the whole pipeline |
@@ -235,13 +236,19 @@ built for earlier indexing settings are released when the settings change. `azim
moves the azimuthal integration of the FPGA workflow from the card to the host, which a profile with
more bins than the card holds needs.
A card sums up to 256 frames into an image. For more, it sums the largest number of frames up to 256 that divides
the frames of an image (250 of 1000) and the host adds up the card's images (4 of 250 each), in 32 bits, saturating;
a pixel in error or saturated in any of them is so in the sum, and a module missing from any of them is missing
from the image. A prime number of frames above 256 is summed on the host entirely, at the cost of reading every
frame. The start message and the master file's `summation_mode` say which: `fpga`, `hybrid` or `cpu`.
## Rugnux live mode
With rugnux live mode on, collections from a DECTRIS detector are processed inside the broker while they
arrive: rotation collections by rugnux, and serial (stills) collections on the GPU with the geometry and mask a
rugnux pre-pass decided (see [Serial collections](#serial-collections-experimental) below). In the FPGA workflow
(PSI JUNGFRAU and EIGER) serial collections are analysed on the GPU after a pre-pass as well, with the differences
given there; rotation collections run image by image, as with the mode off. It is off by default; the
(PSI JUNGFRAU and EIGER) the frames reach the same receivers through `FPGAImagePuller`, rotation and serial alike.
It is off by default; the
configuration file switches it on with
`"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads
left to the receiver and the image pusher.
@@ -278,7 +285,14 @@ a placed model build the solvent as well: waters added at difference-map peaks a
hold them, waters and ions relabelled; `model.refinement` in `/rugnux/status` then carries `waters_added` and
`waters_removed`. `force_still` (`rugnux --force-still`) sends a rotation collection from a DECTRIS detector the
serial way: pre-pass, GPU analysis and stills merge (see [Serial collections](#serial-collections-experimental)).
The analysis takes the frames without the goniometer; the master file keeps it. It does not change the FPGA workflow.
The analysis takes the frames without the goniometer; the master file keeps it.
`stills_downsample` (default 1) lets a serial collection analyse on average 1 of every this many images, to keep up
with a stream faster than the analysis. Each image is chosen on its own, with probability 1 / `stills_downsample`
from a hash of its number: not every n-th image, which would be in step with a periodic pattern of the stream
(pump-probe on/off, storage cells) and could take, for example, only the dark frames. The choice is reproducible.
The first 2000 images, the largest sample the pre-pass may use, are all analysed. An image not chosen is written as it
came, without its analysis. Reset to 1 by `POST /initialize`, as `force_still` and `refine_solvent` are.
The end message, and with it the end of the collection, waits for the analysis. `deadline_s` (default
60 s; 0 for no cutoff) bounds that wait: this long after the last image the analysis is cut. `POST /cancel` is
@@ -365,17 +379,25 @@ of its frames indexed, as with a small image buffer, a low hit rate or a chip wh
cell given), the distance in the header is used for the whole run: this is a warning in the log and a note of the
live status, and a wrong header distance then costs resolution and indexed frames.
Frames arriving during the pre-pass, or faster than the analysis, wait in the image buffer, in all its slots but
one per analysis thread, kept for sending; a frame that finds it full is dropped and counted as for a full image
buffer, so analysis slower than the frame rate costs frames, not memory. Where the writer lags behind, the analysis
waits for slots to come back. A pre-pass that fails leaves the stated geometry and
Frames arriving during the pre-pass, or faster than the analysis, wait in at most half of the image buffer, as their
compressed bytes, several to a slot; the image routes serve them from there, by number, while they wait. The other
half is for the analysed frames, which go to the writer from a slot each, with one slot kept per analysis thread.
The pre-pass takes at most 2000 frames, fewer where the waiting half fills first: then it starts on what it has. The
receiver's image memory is the image buffer (`image_buffer_MiB`, `-B` of `jfjoch_fpga_stills_test`) and nothing
beside it. No stage on the host discards a frame: a frame that finds the waiting half full waits until the analysis
has made room, and the wait goes back to the source - from FPGA cards, `FPGAImagePuller` holds the cards' buffers and
the cards wait for the host, so analysis slower than the frame rate costs frames at the card, not memory. An analysed
frame that finds no slot to be written from within 2 s is not written and is counted as for a full image buffer. A pre-pass that fails leaves the stated geometry and
mask in use. Where the start message cannot be sent after the pre-pass (no writer listening) or the analysis
cannot be set up, the collection ends with that error, reported by `/status` and `/wait_till_done`, and the
next `/start` runs as usual.
The frames are indexed on at most `max_spot_count` spots of the dataset settings where it is given; left out, on as
many as the pre-pass measured (at most 1000, as `rugnux`), while the per-image analysis without rugnux live keeps 250.
The indexed frames are also merged, as `rugnux --mode scale` merges stills, in memory - in the consensus cell
of the frames indexed by the first merge (the cell of the dataset settings only picks the frames that are
merged) and the space group of the dataset settings (else P1).
merged) and the space group of the dataset settings (else P1): a space group given for the collection is binding.
At half and at three quarters of the images the frames so far are merged beside the analysis, on 8 threads, and
the result is reported as a provisional step note: crystals, resolution, CC1/2, R_split, completeness and
⟨I/σ⟩. The first of these merges scales and post-refines every crystal so far and decides the indexing
@@ -418,11 +440,9 @@ pre-pass state and duration, the stated and decided beam centre and distance, th
images analysed and indexed with the analysis rate. The "Rugnux live" page of the web frontend shows the
same. `tools/jfjoch_stills_gpu_replay` replays a stored serial collection through the same analysis.
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is analysed by the receiver of the
card (`JFJochReceiverFPGA`) with the pre-pass, the GPU analysis and the live merge described above, with three
differences: the converted frames wait for the analysis in at most half of the image buffer's slots, a frame that
finds them taken is dropped, and there is no model step - an uploaded model is not placed or refined on these
merges. The end message carries the merged data, as for a DECTRIS stream.
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is assembled on the host by
`FPGAImagePuller` and analysed by `JFJochReceiverRugnuxStills` exactly as a DECTRIS stream.
`tools/jfjoch_fpga_stills_test` measures this path on the cards with a stored dataset (see [Tools](TOOLS.md)).
## Setting up a local test for Jungfraujoch
For development, it is possible to set up a local installation of Jungfraujoch.
+1 -1
View File
@@ -286,7 +286,7 @@ For very long axes the direction grid's angular resolution binds well below 500
| `-C, --unit-cell <cell>` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with this cell and `-S P1`. The lattice is indexed in that system, never above or below it, on the axes of the cell as typed (an F- or R-centred one in its conventional setting), and only the space group within it is searched |
| `--fft-min-unit-cell <num>` | Shortest cell axis the FFT search accepts, Å (default: 10; accepted 5-40). A candidate with a shorter axis is discarded — but on rotation data a row shorter than the floor, down to 3 Å, is recognised under the multiple of it the search reports past the floor, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given. On stills a crystal below the floor cannot be indexed unless this is lowered |
| `--min-indexed-spots <num>` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
| `-S, --space-group <num\|symbol>` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). Without `-C` the group fixes only the symmetry, not the cell: the lattice is found and reduced as in a run without `-S` (a supercell the indexer returns is still stepped down to the crystal's cell, a long axis is still looked for), indexing puts the crystal in the group's own Bravais class, nothing promotes or demotes it past that class, and the group is applied to the cell found, in its conventional setting. With `-C` as well, the lattice and its setting are the ones given and nothing reduces or doubles the cell. A primitive group on a lattice with no primitive cell of its system (P2 on a C-centred monoclinic crystal, P3 on a rhombohedral one) is merged on the centred cell, and the positions the centring extinguishes are kept (they read about zero). With `-C`, the cell is written on the given cell's axes |
| `-S, --space-group <num\|symbol>` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). Without `-C` the group fixes only the symmetry, not the cell: the lattice is found and reduced as in a run without `-S` (a supercell the indexer returns is still stepped down to the crystal's cell, a long axis is still looked for), indexing puts the crystal in the group's own Bravais class, nothing promotes or demotes it past that class, and the group is applied to the cell found, in its conventional setting. With `-C` as well, the lattice and its setting are the ones given and nothing reduces or doubles the cell. A primitive group on a lattice with no primitive cell of its system (P2 on a C-centred monoclinic crystal, P3 on a rhombohedral one) is merged on the centred cell, and the positions the centring extinguishes are kept (they read about zero). With `-C`, the cell is written on the given cell's axes. A space group (and a cell) stored in the input file is ignored, with a line in the log saying so: only `-S` (and `-C`) fix them (`--mode scale` takes the group a `_process.h5` was written in) |
| `-r, --geom-refine <txt>` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default; on rotation data restrained toward the sweep's geometry, [§7.4](CPU_DATA_ANALYSIS_INDEXING.md#74-multi-stage-tightening-of-inlier-tolerance)) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) |
| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) |
| `--force-rotation-lattice <vec>` | Force rotation lattice (9 floats, Å), skipping the first pass |
+57
View File
@@ -70,6 +70,63 @@ UDP packet simulator used to test the Jungfraujoch FPGA receiver.
Exercises and benchmarks the FPGA data path and receiver. With `-H` it runs the high-level
synthesis C model on the CPU, so no FPGA device is required.
### jfjoch_fpga_stills_test
Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector:
```
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-S <group>] [-X <indexer>] [-t <threads>] [-k <F>] [-m <frames>] [-v] [-K[<MiB>]] <master.h5>
```
- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement
and orientation): the layout is the file's - its NXmx module groups, else the modules the gaps of its pixel
mask mark (where the file does not give a module's readout direction, the module is read out from the
top-left corner of its box; the multipixels are symmetric, so the frames are the same). Each image is folded back into module
frames, the inverse of the host's geometry transformation: the inter-chip pixels the assembled image
spreads one raw pixel over (2, or 4 at a chip corner) are summed back into it (the split conserves
the count); where one of them is an error, gap or masked pixel the raw pixel is an error pixel, where
one is saturated or the sum exceeds 16 bits it is saturated.
- The frames are loaded into the card's internal frame generator and the run is made with the JUNGFRAU
conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon
counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live
on (`-v`: each card's progress every 0.5 s; `-k <F>`: analyse 1 of every
F images on average, the rugnux live setting `stills_downsample`; `-m <N>`: N frames summed into each image, by
the card up to 256 and above that the largest divisor of N up to 256 on the card and the rest on the host, 32-bit
images). Beam centre, distance, energy and the cell
come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is
written.
- Each card's generator holds module frames of its own modules only: as many images as its memory holds
module frames (less one, which the host keeps free) divided by the modules of that card - e.g. 128 module
frames: a 9M (18 modules) on one card 7 images, on two cards of 9 modules 14. The run takes the smallest
over the cards (and at most 128, the firmware's image counter; the broker's limit of 64 images does not
apply here) and prints the mapping. These are the first ones of the file, cycled for the whole run of `-i` images
(default: as many as the file has). **Images repeat**, so throughput, latency, spots per image and
indexing rate are meaningful, the merge and its statistics are not.
- One collection per invocation, as fast as it goes. Nothing on the host discards an image on its way to the
analysis: `FPGAImagePuller` waits for room in its queue to the receiver and the stills receiver for room in the
image buffer, so the card
keeps its frames, its host writer waits for host buffers and the stall reaches the internal generator through
the card's AXI-stream back-pressure (the generator sits before the network stack; a real detector's UDP
packets would be lost at that point instead). The one exception is the writer: an analysed image that finds
no image-buffer slot for the writer within the wait is not written and is counted (`writer_queue_full`). The
card's own count of the cycles it waited for host buffers is printed per card: how long the chain behind it
held the source.
- The cards' generators run free, each as fast as its modules' bandwidth allows, so cards with different
numbers of modules (a 9M on 4 cards: 5, 5, 4, 4) run at different speeds - unlike the modules of a detector,
which a common trigger keeps together; the run then goes at the pace of the slowest card.
- It reports the rate images were delivered (first to last image), the sustained rate (images / wall time,
start to the end of the analysis) and the analysis rate after the pre-pass.
- At the end it prints images collected/sent/dropped, receiving time and frame rate, CPU use, indexing
rate and spots per image, per-image times of the analysis stages, the pre-pass (time, beam centre and
distance it decided), the latency from the last image to the end of the analysis and rugnux's steps.
- The image buffer (`-B`, MiB, default 16384) has to hold the pre-pass's frames
(`StillsGPUAnalysis::PREPASS_IMAGES`) in half its slots.
- `-K` runs on the mock of the card (`MockAcquisitionDevice`, which returns the generator's frames as
loaded) instead of `/dev/jfjoch*`, so the program runs without a card; the optional value is the mock's
generator memory in MiB (default 64 MiB = 128 module frames).
- To compare with the offline analysis of the same images, run it with `-i` equal to the number of
distinct images it reports (no repeats) and `rugnux --force-still -e <that number>` or
`jfjoch_stills_gpu_replay -n <that number>` on the same file.
### jfjoch_lite_perf_test
Performance test of the lite (CPU/GPU) analysis path — spot finding, indexing and optional file
writing.
+2 -2
View File
@@ -7,7 +7,7 @@ Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**images_per_trigger** | **int** | For standard synchrotron data collection - this is number of images collected per one TTL trigger For XFEL (pulsed source) - this number is ignored and set to 1 For storage cell mode - this number is ignored and set to number of storage cells | [optional] [default to 1]
**ntrigger** | **int** | Number of TTL trigger that the detector is expected to receive during data collection | [optional] [default to 1]
**image_time_us** | **int** | Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time. In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. | [optional]
**image_time_us** | **int** | Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of frames is summed on the host entirely). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. | [optional]
**beam_x_pxl** | **float** | /entry/detector/beam_center_x in NXmx Beam center in X direction [pixels] |
**beam_y_pxl** | **float** | /entry/detector/beam_center_y in NXmx Beam center in X direction [pixels] |
**detector_distance_mm** | **float** | /entry/detector/distance in NXmx Detector distance [mm] |
@@ -42,7 +42,7 @@ Name | Type | Description | Notes
**unit_cell** | [**UnitCell**](UnitCell.md) | | [optional]
**spot_finding** | **bool** | Enable spot finding and save spots | [optional] [default to True]
**smargon** | [**DatasetSettingsSmargon**](DatasetSettingsSmargon.md) | | [optional]
**max_spot_count** | **int** | Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected | [optional] [default to 250]
**max_spot_count** | **int** | Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected. Left out, the broker&#39;s per-image indexing keeps 250 and rugnux live stills 1000. | [optional]
**detect_ice_rings** | **bool** | Flag spots as ice rings and reduce their effect on indexing. Left out, the broker&#39;s own spot finding treats no spot as an ice ring, rugnux live decides as a bare &#x60;rugnux&#x60; would (on for a rotation sweep, off for stills), and the master file carries no &#x60;detect_ice_rings&#x60;. | [optional]
**async_start** | **bool** | When set to true, &#x60;/start&#x60; will not wait for detector and Jungfraujoch to be ready for the measurement. | [optional] [default to False]
**module_placement** | [**List[DetectorModulePlacement]**](DetectorModulePlacement.md) | Where the detector&#39;s modules lie for this dataset, as rugnux measures them: rugnux writes &#x60;&lt;prefix&gt;_module_placement.json&#x60;, which holds exactly this field, to be merged into the dataset settings. Each entry names its module by its readout pixel 0 and readout axes in the stored image (&#x60;x0&#x60;, &#x60;y0&#x60;, &#x60;fast_axis&#x60;, &#x60;slow_axis&#x60;, after &#x60;mirror_y&#x60;; see &#x60;detector_module_placement&#x60;), in any order; every module must be named once, and only modules of the detector, else &#x60;/start&#x60; fails with 400. The image is assembled as configured (&#x60;custom_geometry&#x60;); the placement goes into the start message and the NXmx &#x60;module_NN&#x60; groups, and the analysis takes it. Omitted or empty: modules as assembled. | [optional]
@@ -11,6 +11,7 @@ Name | Type | Description | Notes
**deadline_s** | **float** | Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as &#x60;POST /cancel&#x60; after the last image does). 0 means no cutoff. | [optional] [default to 60.0]
**refine_solvent** | **bool** | With a model: the refinement also builds the solvent - waters added at difference-map peaks and removed where the maps no longer hold them, waters and ions relabelled (as &#x60;rugnux --refine-solvent&#x60;). Reset to false by &#x60;POST /initialize&#x60;. | [optional] [default to False]
**force_still** | **bool** | Process DECTRIS rotation collections as stills, on the serial path (pre-pass, GPU analysis, stills merge), as &#x60;rugnux --force-still&#x60; does. The written file keeps its goniometer. Reset to false by &#x60;POST /initialize&#x60;. | [optional] [default to False]
**stills_downsample** | **int** | Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by &#x60;POST /initialize&#x60;. | [optional] [default to 1]
## Example
+1
View File
@@ -10,6 +10,7 @@ Name | Type | Description | Notes
**deadline_s** | **float** | Seconds after the last image at which the analysis is cut; 0 means no cutoff | [optional]
**refine_solvent** | **bool** | The refinement of a placed model also builds the solvent |
**force_still** | **bool** | DECTRIS rotation collections are processed as stills |
**stills_downsample** | **int** | Serial collections, 1 of every this many images analysed on average | [optional]
**model_loaded** | **bool** | A model was uploaded with &#x60;PUT /config/rugnux/model&#x60; |
**model_size_bytes** | **int** | Size of the uploaded model file | [optional]
+3
View File
@@ -58,6 +58,9 @@ typedef __u64 uint64_t;
// MAX_MODULES_FPGA is read by FPGA setup scripts and must be decimal, no expression!
#define MAX_MODULES_FPGA 32
#define MAX_FPGA_SUMMATION 256
// frame_generator.cpp: images in its memory (frame_cell, 7 bits) and module frames it addresses (mem_cell, 10 bits)
#define FPGA_INTERNAL_GENERATOR_MAX_IMAGES 128
#define FPGA_INTERNAL_GENERATOR_MAX_MODULE_FRAMES 1024
#define ADU_HISTO_BIN_WIDTH 32
#define ADU_HISTO_BIN_COUNT (65536/ ADU_HISTO_BIN_WIDTH)
+27 -4
View File
@@ -166,7 +166,10 @@ export type dataset_settings = {
/**
* Image time.
* If not provided (or zero value) the frame time is assumed as default.
* For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time.
* For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA.
* Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image
* (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of
* frames is summed on the host entirely).
* In XFEL mode: summation happens for frames collected with multiple triggers.
* Ignored for storage cells and if raw data are saved.
*
@@ -382,7 +385,9 @@ export type dataset_settings = {
];
};
/**
* Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected
* Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected.
* Left out, the broker's per-image indexing keeps 250 and rugnux live stills 1000.
*
*/
max_spot_count?: number;
/**
@@ -2043,6 +2048,15 @@ export type rugnux_settings = {
*
*/
force_still?: boolean;
/**
* Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
* image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
* which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
* may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came,
* without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
*
*/
stills_downsample?: number;
};
export type rugnux_status = {
@@ -2066,6 +2080,10 @@ export type rugnux_status = {
* DECTRIS rotation collections are processed as stills
*/
force_still: boolean;
/**
* Serial collections, 1 of every this many images analysed on average
*/
stills_downsample?: number;
/**
* A model was uploaded with `PUT /config/rugnux/model`
*/
@@ -2626,7 +2644,10 @@ export type dataset_settingsWritable = {
/**
* Image time.
* If not provided (or zero value) the frame time is assumed as default.
* For JUNGFRAU image time must be multiple of frame time and max value is 256 * frame_time.
* For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA.
* Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image
* (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of
* frames is summed on the host entirely).
* In XFEL mode: summation happens for frames collected with multiple triggers.
* Ignored for storage cells and if raw data are saved.
*
@@ -2842,7 +2863,9 @@ export type dataset_settingsWritable = {
];
};
/**
* Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected
* Maximum number of spots that are saved/used for indexing; spots with highest intensity are selected.
* Left out, the broker's per-image indexing keeps 250 and rugnux live stills 1000.
*
*/
max_spot_count?: number;
/**
+5 -3
View File
@@ -752,7 +752,7 @@ export const zDatasetSettings = z.object({
1
])
}).optional(),
max_spot_count: z.int().gte(10).lte(2000).optional().default(250),
max_spot_count: z.int().gte(10).lte(2000).optional(),
detect_ice_rings: z.boolean().optional(),
async_start: z.boolean().optional().default(false),
module_placement: z.array(zDetectorModulePlacement).optional(),
@@ -856,7 +856,8 @@ export const zRugnuxSettings = z.object({
reserved_cpus: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(4)),
deadline_s: z.number().gte(0).optional().default(60),
refine_solvent: z.boolean().optional().default(false),
force_still: z.boolean().optional().default(false)
force_still: z.boolean().optional().default(false),
stills_downsample: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1))
});
export const zRugnuxStatus = z.object({
@@ -865,6 +866,7 @@ export const zRugnuxStatus = z.object({
deadline_s: z.number().optional(),
refine_solvent: z.boolean(),
force_still: z.boolean(),
stills_downsample: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
model_loaded: z.boolean(),
model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional()
});
@@ -1226,7 +1228,7 @@ export const zDatasetSettingsWritable = z.object({
1
])
}).optional(),
max_spot_count: z.int().gte(10).lte(2000).optional().default(250),
max_spot_count: z.int().gte(10).lte(2000).optional(),
detect_ice_rings: z.boolean().optional(),
async_start: z.boolean().optional().default(false),
module_placement: z.array(zDetectorModulePlacement).optional(),
+11 -4
View File
@@ -50,6 +50,8 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
const [cpusError, setCpusError] = useState(false);
const [deadline, setDeadline] = useState(0);
const [deadlineError, setDeadlineError] = useState(false);
const [downsample, setDownsample] = useState(1);
const [downsampleError, setDownsampleError] = useState(false);
const [counter, setCounter] = useState(0);
const [file, setFile] = useState<File>();
@@ -60,19 +62,21 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
setForceStill(s.force_still);
setCpus(s.reserved_cpus);
setDeadline(s.deadline_s ?? 0);
setDownsample(s.stills_downsample ?? 1);
setCounter(c => c + 1);
}
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s]);
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s, s?.stills_downsample]);
const dirty = s !== undefined && (enable !== s.enable || refineSolvent !== s.refine_solvent
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0));
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0)
|| downsample !== (s.stills_downsample ?? 1));
return (
<Stack direction={{xs: "column", md: "row"}} spacing={2} alignItems="stretch">
<SettingsPanel title="Live processing" dirty={dirty} spacing={2}
onUpload={() => settings.submit({enable, refine_solvent: refineSolvent, force_still: forceStill,
reserved_cpus: cpus, deadline_s: deadline})}
uploadDisabled={measuring || cpusError || deadlineError || settings.pending}
reserved_cpus: cpus, deadline_s: deadline, stills_downsample: downsample})}
uploadDisabled={measuring || cpusError || deadlineError || downsampleError || settings.pending}
snackbar={settings.snackbar}>
<Stack spacing={1.5} alignItems="flex-start" sx={{textAlign: "left"}}>
<FormControlLabel label="Process collections with rugnux live"
@@ -90,6 +94,9 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
<NumberTextField label="Cut the analysis this long after the last image (0 = never)" units="s"
start_val={deadline} min={0} float decimals={1} counter={counter} disabled={measuring}
sx={{width: "100%"}} callback={(v, err) => { setDeadline(v); setDeadlineError(err); }}/>
<NumberTextField label="Serial: analyse 1 of this many images (1 = all)" start_val={downsample} min={1}
max={1000000} counter={counter} disabled={measuring} sx={{width: "100%"}}
callback={(v, err) => { setDownsample(v); setDownsampleError(err); }}/>
</SettingsPanel>
<SettingsPanel title="Model for molecular replacement" spacing={2}>
@@ -49,6 +49,9 @@ IF (JFJOCH_CUDA_AVAILABLE)
# note in image_analysis/indexing/CMakeLists.txt.
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft_static)
ELSEIF (TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochGeomRefinement
$<IF:$<BOOL:$<TARGET_PROPERTY:JFJOCH_STATIC_CUFFT>>,CUDA::cufft_static,CUDA::cufft>)
ELSE()
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft)
ENDIF()
+5
View File
@@ -54,8 +54,13 @@ IF (JFJOCH_CUDA_AVAILABLE)
# Both halves matter: the server build keeps the shared library (its .deb/.rpm take CUDA from the
# distro, and every executable there -- broker, tests, tools -- would otherwise need the device
# link below), and Windows has no static cuFFT to link even for the portable products.
# One exception in the server build: an executable with the JFJOCH_STATIC_CUFFT property (set on
# jfjoch_fpga_stills_test, which is copied to the FPGA servers on its own) gets the static library.
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft_static)
ELSEIF (TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static
$<IF:$<BOOL:$<TARGET_PROPERTY:JFJOCH_STATIC_CUFFT>>,CUDA::cufft_static,CUDA::cufft>)
ELSE()
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft)
ENDIF()
-3
View File
@@ -75,9 +75,6 @@ public:
virtual bool SendAck(const PullerAckMessage &) { return false; }
virtual void Disconnect() = 0;
// Images the puller had no room for, which the receiver therefore never gets.
virtual size_t GetDroppedImages() const { return 0; }
};
+2 -1
View File
@@ -3,9 +3,11 @@
ADD_LIBRARY(JFJochReceiver STATIC
FrameTransformation.cpp FrameTransformation.h
FPGAImagePuller.cpp FPGAImagePuller.h
JFJochReceiverService.cpp JFJochReceiverService.h
JFJochReceiverTest.cpp JFJochReceiverTest.h
JFJochReceiverFPGA.cpp JFJochReceiverFPGA.h
JFJochReceiverPedestal.cpp JFJochReceiverPedestal.h
ImageMetadata.cpp
ImageMetadata.h
LossyFilter.cpp
@@ -20,7 +22,6 @@ ADD_LIBRARY(JFJochReceiver STATIC
JFJochReceiverRugnuxStills.h JFJochReceiverRugnuxStills.cpp
StillsPrePass.h StillsPrePass.cpp
StillsGPUAnalysis.h StillsGPUAnalysis.cpp
StillsGPUAnalysisFPGA.h StillsGPUAnalysisFPGA.cpp
RugnuxLiveStatus.h RugnuxLiveStatus.cpp
RugnuxStructure.h RugnuxStructure.cpp
JFJochReceiver.cpp
+268
View File
@@ -0,0 +1,268 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "FPGAImagePuller.h"
#include <thread>
#include <utility>
FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask,
const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices,
Logger &in_logger, int64_t image_threads)
: ImagePuller(Queue + 2), // the images waiting, the start and the end message
experiment(in_experiment),
logger(in_logger),
devices(in_devices),
host_summation(in_experiment.GetHostSummation()) {
const int ndatastreams = experiment.GetDataStreamsNum();
if (devices.size() < ndatastreams)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Number of acquisition devices has to match data streams");
for (int m = 0; m < experiment.GetModulesNum(); m++)
adu_histogram.emplace_back(std::make_unique<ADUHistogram>());
for (int d = 0; d < ndatastreams; d++) {
devices[d].PrepareAction(experiment);
expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() * host_summation;
}
if (experiment.GetDetectorType() != DetectorType::JUNGFRAU)
calibration = nullptr;
// The receiver's own start message is built from its experiment; this one only says the run has begun.
StartMessage start{};
experiment.FillMessage(start);
outside_fifo.Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(start),
.received = std::chrono::steady_clock::now()});
for (int i = 0; i < image_threads; i++)
image_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::ImageThread, this));
std::latch ready(ndatastreams);
for (int d = 0; d < ndatastreams; d++)
acquire_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::AcquireThread, this,
static_cast<uint16_t>(d), std::cref(mask), calibration,
std::ref(ready)));
ready.wait();
finish_future = std::async(std::launch::async, &FPGAImagePuller::FinalizeMeasurement, this);
if (failed) {
try {
Stop();
} catch (...) {
outside_fifo.Clear(); // the destructor does not run
throw;
}
}
}
FPGAImagePuller::~FPGAImagePuller() {
try {
Stop();
} catch (const std::exception &e) {
logger.Error("FPGA image puller teardown failed: {}", e.what());
}
}
void FPGAImagePuller::SetError(std::exception_ptr e) {
std::unique_lock ul(error_mutex);
if (!error)
error = e;
failed = true;
}
void FPGAImagePuller::StopCards() {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
devices[d].Cancel();
}
void FPGAImagePuller::Cancel() {
cancelled = true;
StopCards();
}
// Cancels first: a receiver that stopped on an error takes no more images, which would hold the threads otherwise
void FPGAImagePuller::Stop() {
Cancel();
std::call_once(finish_once, [this] { finish_future.get(); });
std::unique_lock ul(error_mutex);
if (error)
std::rethrow_exception(std::exchange(error, nullptr));
}
void FPGAImagePuller::AcquireThread(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration,
std::latch &ready) {
auto &device = devices[data_stream];
try {
// Only what the card needs for the pixels: its analysis results are not read.
if (calibration != nullptr)
device.InitializeCalibration(experiment, *calibration);
device.InitializePixelMask(experiment, mask);
device.StartAction(experiment);
} catch (...) {
SetError(std::current_exception());
StopCards(); // the image threads waiting on the other cards have to come out as well
ready.count_down();
return;
}
ready.count_down();
try {
device.WaitForActionComplete();
} catch (...) {
SetError(std::current_exception());
StopCards();
}
}
// Polled: a card that fails or finishes ends the wait. A frame given up on that comes later is below the device's
// slowest frame then (MissingAfterFrames), and the device returns its buffer itself.
bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) {
const auto &counters = devices[data_stream].Counters();
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
while ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) < frame + 2)
&& !counters.IsAcquisitionFinished() && !failed
&& (static_cast<int64_t>(counters.GetFastestFrameNumber()) < frame + 2 + MissingAfterFrames))
std::this_thread::sleep_for(std::chrono::milliseconds(1));
return counters.IsAnyPacketCollected(frame, module_number);
}
bool FPGAImagePuller::CardsFinishedBefore(int64_t frame) const {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++) {
const auto &counters = devices[d].Counters();
if (!counters.IsAcquisitionFinished() || (static_cast<int64_t>(counters.GetFastestFrameNumber()) >= frame))
return false;
}
return true;
}
void FPGAImagePuller::ImageThread() {
// One image of its own, assembled and compressed here
FrameTransformation transformation(experiment);
// Summation on the host: the cards' images of one image added up, one module each, as in JFJochReceiverFPGA
std::vector<std::unique_ptr<ModuleSummation>> summation;
if (host_summation > 1)
for (int m = 0; m < experiment.GetModulesNum(); m++)
summation.emplace_back(std::make_unique<ModuleSummation>(experiment));
try {
for (int64_t image = next_to_take++; image < experiment.GetImageNum() && !failed; image = next_to_take++) {
// The cards are through and none got as far as this image (a cancel, or fewer frames than asked for):
// it will not come, and is not assembled from missing modules either - as JFJochReceiverFPGA
if (CardsFinishedBefore(image * host_summation)) {
Emit(image, std::nullopt);
continue;
}
ImageMetadata metadata(experiment);
auto take_module = [&](const DeviceOutput *output, int d, int m) {
metadata.Process(output);
adu_histogram[experiment.GetFirstModuleOfDataStream(d) + m]->Add(*output);
transformation.ProcessModule(output, d);
};
for (auto &s: summation)
s->Clear();
// The card's images of this image in their order, all modules of each before the next, as they come
for (int64_t frame = image * host_summation; frame < (image + 1) * host_summation; frame++) {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++) {
auto &device = devices[d];
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const bool collected = WaitForFrame(d, m, frame);
if (!summation.empty()) {
auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (collected)
s.AddFPGAOutput(*device.GetDeviceOutput(frame, m));
else
s.AddEmptyOutput();
} else if (collected)
take_module(device.GetDeviceOutput(frame, m), d, m);
else
transformation.FillNotCollectedModule(m, d);
if (collected)
device.FrameBufferRelease(frame, m);
}
}
}
// A module missing from any of the card's images is missing from the image
if (!summation.empty())
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (s.empty())
transformation.FillNotCollectedModule(m, d);
else
take_module(&s.GetOutput(), d, m);
}
DataMessage message{};
message.number = image;
message.original_number = image;
metadata.Export(message, expected_packets_per_image);
// No module of it came
if (message.image_collection_efficiency == 0.0f) {
Emit(image, std::nullopt);
continue;
}
// The card's spot finder and ROI map were not set up for this run; the receiver reports its own.
message.strong_pixel_count.reset();
message.roi.clear();
// The compressed image, copied once into a buffer of the output's own
const CompressedImage compressed = transformation.GetCompressedImage();
auto bytes = std::make_shared<std::vector<uint8_t>>(compressed.GetCompressed(),
compressed.GetCompressed() + compressed.GetCompressedSize());
message.image = CompressedImage(bytes->data(), bytes->size(), compressed.GetWidth(), compressed.GetHeight(),
compressed.GetMode(), compressed.GetCompressionAlgorithm());
ImagePullerOutput output;
output.cbor = std::make_shared<CBORStream2DeserializerOutput>(message);
output.raw = std::shared_ptr<ReceiveBufferRegion>(new ReceiveBufferRegion{bytes->data(), bytes->size()},
[bytes](ReceiveBufferRegion *region) { delete region; });
Emit(image, std::move(output));
}
} catch (...) {
SetError(std::current_exception());
StopCards();
}
}
// Whichever thread finishes the image next in line lets it, and those waiting behind it, out.
void FPGAImagePuller::Emit(int64_t image_number, std::optional<ImagePullerOutput> output) {
std::unique_lock ul(order_mutex);
images_done.emplace(image_number, std::move(output));
for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image);
it = images_done.erase(it), next_image++) {
if (it->second) {
it->second->received = std::chrono::steady_clock::now();
Put(*it->second);
}
}
}
void FPGAImagePuller::FinalizeMeasurement() {
for (auto &f: image_futures)
f.get();
StopCards();
for (auto &f: acquire_futures)
f.get();
// Only an error leaves images unfinished, holding back those behind them
{
std::unique_lock ul(order_mutex);
for (auto &[n, output]: images_done)
if (output)
Put(*output);
images_done.clear();
}
EndMessage end{};
for (size_t i = 0; i < adu_histogram.size(); i++)
end.adu_histogram["module" + std::to_string(i)] = adu_histogram[i]->GetHistogram();
Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(end), .received = std::chrono::steady_clock::now()});
}
void FPGAImagePuller::Put(const ImagePullerOutput &output) {
while (!outside_fifo.PutTimeout(output, std::chrono::milliseconds(100)) && !cancelled);
}
+88
View File
@@ -0,0 +1,88 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <atomic>
#include <future>
#include <latch>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <vector>
#include "FrameTransformation.h"
#include "ImageMetadata.h"
#include "../acquisition_device/AcquisitionDeviceGroup.h"
#include "../common/ADUHistogram.h"
#include "../common/ModuleSummation.h"
#include "../image_puller/ImagePuller.h"
#include "../jungfrau/JFCalibration.h"
// The images of a collection from the FPGA cards, as a detector stream, in image order. As JFJochReceiverFPGA: each
// thread takes the next image, waits for its modules in the cards' buffers, assembles them into an image of its own
// and compresses it; the image leaves as its compressed bytes, and the cards' buffers go back. The cards' buffers are
// the only window over the images in flight. No image is discarded here: an image waits for room in the queue to the
// receiver, holding the cards' buffers meanwhile, so the cards wait for the host. Images are lost only at the card.
class FPGAImagePuller : public ImagePuller {
public:
// Images compressed and waiting for the receiver
static constexpr size_t Queue = 128;
// A module this far behind the fastest one of its card is taken as missing from that frame. It must stay beyond
// AcquisitionCounters::ThresholdFramesLost: the device then returns that frame's buffer itself if it comes later.
static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2;
private:
const DiffractionExperiment experiment;
Logger &logger;
AcquisitionDeviceGroup &devices;
// The cards' images making one image, summed here (DiffractionExperiment::GetHostSummation)
const int64_t host_summation;
uint64_t expected_packets_per_image = 0;
std::atomic<int64_t> next_to_take = 0;
// What is done, waiting for the images before it. No value: that image will not come.
std::mutex order_mutex;
std::map<int64_t, std::optional<ImagePullerOutput>> images_done;
int64_t next_image = 0;
std::vector<std::unique_ptr<ADUHistogram>> adu_histogram;
std::mutex error_mutex;
std::exception_ptr error;
std::atomic<bool> failed = false; // a failed card may never report finished: nothing waits for it then
std::atomic<bool> cancelled = false; // Cancel(): the receiver takes no more images, nothing waits for it
void SetError(std::exception_ptr e);
std::vector<std::future<void>> acquire_futures;
std::vector<std::future<void>> image_futures;
std::future<void> finish_future;
std::once_flag finish_once;
void AcquireThread(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration, std::latch &ready);
void ImageThread();
void FinalizeMeasurement();
// False where the module is missing from the frame
bool WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame);
// Every card has finished, all of them before this frame
bool CardsFinishedBefore(int64_t frame) const;
void Emit(int64_t image_number, std::optional<ImagePullerOutput> output);
// Waits for room in the queue, as long as the receiver takes from it: after a cancel it no longer does
void Put(const ImagePullerOutput &output);
public:
// Arms the cards; when it returns the detector can be started.
FPGAImagePuller(const DiffractionExperiment &experiment, const PixelMask &mask, const JFCalibration *calibration,
AcquisitionDeviceGroup &devices, Logger &logger, int64_t image_threads);
~FPGAImagePuller() override;
void Disconnect() override {}
// The cards finish: what they have got still comes, then the end message.
void StopCards();
// As StopCards(), and the receiver takes no more images: nothing waits for it any more.
void Cancel();
// After the receiver is done: cancels, waits for the threads, throws what went wrong with the cards.
void Stop();
};
+8 -3
View File
@@ -59,6 +59,13 @@ size_t FrameTransformation::CompressImage(void *output, size_t output_size) {
experiment.GetByteDepthImage());
}
void FrameTransformation::ReserveCompressedBuffer() {
if (experiment.GetCompressionAlgorithm() != CompressionAlgorithm::NO_COMPRESSION)
compressed_buffer.resize(MaxCompressedSize(experiment.GetCompressionAlgorithm(),
experiment.GetPixelsNum(),
experiment.GetByteDepthImage()));
}
CompressedImage FrameTransformation::GetCompressedImage() {
const uint8_t *data;
size_t size;
@@ -66,9 +73,7 @@ CompressedImage FrameTransformation::GetCompressedImage() {
data = reinterpret_cast<uint8_t *>(precompression_buffer.data());
size = experiment.GetPixelsNum() * experiment.GetByteDepthImage();
} else {
compressed_buffer.resize(MaxCompressedSize(experiment.GetCompressionAlgorithm(),
experiment.GetPixelsNum(),
experiment.GetByteDepthImage()));
ReserveCompressedBuffer();
data = reinterpret_cast<uint8_t *>(compressed_buffer.data());
size = CompressImage(compressed_buffer.data(), compressed_buffer.size());
}
+2
View File
@@ -23,6 +23,8 @@ public:
void ProcessModule(const DeviceOutput *output, int data_stream);
void ProcessModule(const void *input, uint16_t module_number, int data_stream);
void FillNotCollectedModule(uint16_t module_number, int data_stream);
// Allocates and writes the buffer GetCompressedImage() compresses into, so its pages are not first touched there
void ReserveCompressedBuffer();
CompressedImage GetCompressedImage();
// Throws CompressionBufferTooSmallException if the image does not fit output_size.
size_t CompressImage(void *output, size_t output_size);
+101
View File
@@ -0,0 +1,101 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <cstdint>
#include <functional>
#include <mutex>
#include <vector>
#include "../common/ImageBuffer.h"
#include "../common/ZeroCopyReturnValue.h"
// The frames waiting for the analysis, as their compressed bytes packed into slots of the image buffer: inside its
// memory, at most `max_slots` of its slots (half of them in the stills receiver), and the slots kept free for sending
// left alone. A slot is reused once the frames in it are all released; with every slot taken and full, Allocate fails.
class HoldArena {
struct Chunk {
ZeroCopyReturnValue *slot;
size_t used = 0;
int64_t frames = 0;
};
ImageBuffer &image_buffer;
const int64_t max_slots;
const int64_t keep_free;
const size_t slot_size;
std::mutex m;
std::vector<Chunk> chunks;
size_t current = SIZE_MAX;
public:
struct Region {
uint8_t *data = nullptr;
std::function<void()> release; // once the frame is gone
};
HoldArena(ImageBuffer &in_image_buffer, int64_t in_max_slots, int64_t in_keep_free, size_t in_slot_size)
: image_buffer(in_image_buffer), max_slots(in_max_slots), keep_free(in_keep_free), slot_size(in_slot_size) {}
~HoldArena() { ReleaseSlots(); }
// The slots go back to the image buffer: once no frame is held any more
void ReleaseSlots() {
std::unique_lock ul(m);
for (auto &c: chunks)
if (c.slot) {
c.slot->discard();
c.slot = nullptr;
}
current = SIZE_MAX;
}
// Empty where there is no room
Region Allocate(size_t size) {
std::unique_lock ul(m);
if (size > slot_size)
return {};
if (current == SIZE_MAX || chunks[current].used + size > slot_size) {
current = SIZE_MAX;
for (size_t i = 0; i < chunks.size(); i++)
if (chunks[i].slot && chunks[i].frames == 0) {
chunks[i].used = 0;
current = i;
break;
}
if (current == SIZE_MAX && static_cast<int64_t>(chunks.size()) < max_slots) {
if (auto slot = image_buffer.GetImageSlot(keep_free)) {
chunks.push_back(Chunk{.slot = slot});
current = chunks.size() - 1;
}
}
if (current == SIZE_MAX)
return {};
}
Chunk &c = chunks[current];
Region ret{.data = static_cast<uint8_t *>(c.slot->GetImage()) + c.used};
c.used += size;
c.frames++;
const size_t index = current;
ret.release = [this, index] {
std::unique_lock ul2(m);
chunks[index].frames--;
};
return ret;
}
// No room for a frame of this size without a frame going first
bool Full(size_t size) {
std::unique_lock ul(m);
if (current != SIZE_MAX && chunks[current].used + size <= slot_size)
return false;
for (const auto &c: chunks)
if (c.slot && c.frames == 0)
return false;
return static_cast<int64_t>(chunks.size()) >= max_slots;
}
int64_t GetSlotsUsed() {
std::unique_lock ul(m);
return static_cast<int64_t>(chunks.size());
}
};
+6
View File
@@ -131,6 +131,12 @@ public:
virtual void SetSpotFindingSettings(const SpotFindingSettings &spot_finding_settings);
virtual JFJochReceiverOutput GetFinalStatistics() const;
// A frame the receiver holds and has not sent to the image buffer yet, serialised as the buffer serves an image;
// false where there is none of that number
virtual bool GetWaitingImage(std::vector<uint8_t> &v, int64_t image_number) { return false; }
// The latest of them
virtual std::optional<int64_t> GetLastWaitingImage() { return {}; }
};
+41 -337
View File
@@ -3,14 +3,9 @@
#include "JFJochReceiverFPGA.h"
#include "../image_analysis/WriteReflections.h"
#include <thread>
#include "ImageMetadata.h"
#include "JFJochReceiverLite.h"
#include "../common/CUDAWrapper.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment,
const PixelMask &in_pixel_mask,
@@ -24,9 +19,7 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
ImageBuffer &in_send_buf_ctrl,
ZMQPreviewSocket *in_zmq_preview_socket,
ZMQMetadataSocket *in_zmq_metadata_socket,
IndexerThreadPool *indexing_thread_pool,
bool in_gpu_analysis,
RugnuxLiveStatus *in_rugnux_status)
IndexerThreadPool *indexing_thread_pool)
: JFJochReceiver(in_experiment,
in_send_buf_ctrl,
in_image_sender,
@@ -44,13 +37,9 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
ndatastreams(experiment.GetDataStreamsNum()),
frame_transformation_nthreads(
in_forward_and_sum_nthreads),
pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4),
summation_nthreads(2),
frame_transformation_ready((experiment.GetImageNum() > 0) ? frame_transformation_nthreads : 0),
data_acquisition_ready(ndatastreams),
gpu_analysis(in_gpu_analysis && experiment.GetImageNum() > 0
&& experiment.GetDetectorMode() == DetectorMode::Standard),
rugnux_status(in_rugnux_status) {
frame_transformation_ready(frame_transformation_nthreads),
data_acquisition_ready(ndatastreams) {
// The FPGA integration core has a fixed bin-count limit. Reject configurations it cannot
// handle before starting acquisition, unless azimuthal integration is forced onto the CPU.
const auto &azint_settings = experiment.GetAzimuthalIntegrationSettings();
@@ -88,96 +77,27 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen
}
if (experiment.IsCPUSummation())
expected_packets_per_image *= experiment.GetSummation();
expected_packets_per_image *= experiment.GetHostSummation();
logger.Debug("Data acquisition devices ready");
if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0)
|| (experiment.GetDetectorMode() == DetectorMode::PedestalG1)
|| (experiment.GetDetectorMode() == DetectorMode::PedestalG2)) {
if (experiment.GetImageNum() > 0) {
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Saving and calculating pedestal is not supported for the time being");
}
SendStartMessage();
SendCalibration();
switch (experiment.GetDetectorMode()) {
case DetectorMode::PedestalG1:
only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2);
break;
case DetectorMode::PedestalG2:
only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2);
break;
default:
only_2nd_sc_pedestal = false;
break;
}
for (int i = 0; i < experiment.GetImageNum(); i++)
images_to_go.Put(i);
int64_t pedestal_count = (only_2nd_sc_pedestal)
? experiment.GetModulesNum()
: experiment.GetModulesNum() * experiment.GetStorageCellNumber();
for (int i = 0; i < pedestal_count; i++)
pedestal.emplace_back(std::make_unique<JFPedestalCalc>(experiment));
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
bool ignore = only_2nd_sc_pedestal && (s == 0);
for (int d = 0; d < ndatastreams; d++)
for (int m = 0; m < experiment.GetModulesNum(d); m++)
for (int n = 0; n < pedestal_nthreads; n++)
frame_transformation_futures.emplace_back(std::async(std::launch::async,
&JFJochReceiverFPGA::MeasurePedestalThread,
this, d, m, s, n, ignore));
}
logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads);
} else if (experiment.GetImageNum() > 0) {
if (gpu_analysis) {
// The start message and the calibration go out once the pre-pass is done.
gpu = std::make_unique<StillsGPUAnalysisFPGA>(
experiment, pixel_mask, MakeStartMessage(experiment, pixel_mask, *az_int_mapping),
indexing_thread_pool, std::min<int64_t>(16, frame_transformation_nthreads),
// Half the image buffer for the frames waiting, the other half for those sent.
image_buffer.GetStatus().total_slots / 2, logger,
[this](const StillsGPUAnalysisFPGA::Setup &s) {
// The profiles are binned on the pre-pass's geometry.
plots.Setup(s.experiment, *s.mapping);
SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping));
SendCalibration();
},
[this](DataMessage &m, const AzimuthalIntegrationProfile &p) { GPUSink(m, p); },
[this] {
// The broker's settings, with the spot finder rugnux uses on stills (the pre-pass
// decided the centre and mask with it): adaptive threshold, single-pixel spots kept.
// With the broker's default fixed threshold 3 points fewer frames index.
// Every frame is integrated: integration is rugnux's, not the broker's to switch off.
SpotFindingSettings s = GetSpotFindingSettings();
s.adaptive_threshold = true;
s.min_pix_per_spot = std::nullopt;
s.quick_integration = true;
return s;
},
// The start message did not go out (or the analysis could not be set up): nothing can
// be written, so the collection stops, as it does on an error in a frame thread.
[this](const JFJochException &e) { Cancel(e); },
rugnux_status);
} else {
SendStartMessage();
SendCalibration();
}
for (int i = 0; i < experiment.GetImageNum(); i++)
images_to_go.Put(i);
// Setup frames summation and forwarding
for (uint32_t i = 0; i < frame_transformation_nthreads; i++) {
auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread,
this, i);
frame_transformation_futures.emplace_back(std::move(handle));
}
logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads);
frame_transformation_ready.wait();
logger.Debug("Image compression/forwarding threads ready");
// Setup frames summation and forwarding
for (uint32_t i = 0; i < frame_transformation_nthreads; i++) {
auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread,
this, i);
frame_transformation_futures.emplace_back(std::move(handle));
}
logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads);
frame_transformation_ready.wait();
logger.Debug("Image compression/forwarding threads ready");
for (int d = 0; d < ndatastreams; d++)
data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverFPGA::AcquireThread,
this, d));
@@ -251,53 +171,15 @@ void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) {
}
void JFJochReceiverFPGA::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell,
uint32_t threadid, bool ignore) {
JFPedestalCalc pedestal_calc(experiment);
uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber();
uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads;
uint32_t storage_cell_header = UINT32_MAX;
try {
for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) {
// Frame will be processed only if one already collects frame+2
acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number);
if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) {
auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number);
// Partial packets will bring more problems, than benefit
pedestal_calc.AnalyzeImage((uint16_t *) output->pixels);
storage_cell_header = (output->module_statistics.debug >> 8) & 0xF;
}
acquisition_device[data_stream].FrameBufferRelease(frame, module_number);
UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number));
current_status.SetProgress(GetProgress());
current_status.SetStatus(GetStatus());
}
uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
if (!only_2nd_sc_pedestal)
offset += experiment.GetModulesNum() * storage_cell;
if (!ignore)
*pedestal[offset] += pedestal_calc;
} catch (const JFJochException &e) {
Cancel(e);
}
logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done",
data_stream, module_number, storage_cell, storage_cell_header);
}
int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
int64_t image_number,
uint16_t module_number,
uint32_t threadid,
ModuleSummation &summation) {
void JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
int64_t image_number,
uint16_t module_number,
uint32_t threadid,
ModuleSummation &summation) {
ModuleSummation local_summation(experiment);
int64_t starting_frame = image_number * experiment.GetSummation();
int64_t starting_frame = image_number * experiment.GetHostSummation();
for (int64_t i = threadid; i < experiment.GetSummation(); i += summation_nthreads) {
for (int64_t i = threadid; i < experiment.GetHostSummation(); i += summation_nthreads) {
const int64_t frame = starting_frame + i;
// Frame will be processed only if one already collects frame+2
@@ -317,7 +199,6 @@ int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream,
if (!summation.empty())
summation.AddFPGAOutput(local_summation.GetOutput(), 4);
return 0;
}
void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
@@ -348,7 +229,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
try {
int64_t expected_frame = image_number;
if (experiment.IsCPUSummation())
expected_frame *= experiment.GetSummation();
expected_frame *= experiment.GetHostSummation();
logger.Debug("Frame transformation thread - trying to get image {}", expected_frame);
// If data acquisition is finished and fastest frame for the first device is behind
@@ -380,7 +261,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
for (int i = 0; i < experiment.GetModulesNum(); i++)
summation.emplace_back(std::make_unique<ModuleSummation>(experiment));
std::vector<std::future<int64_t> > futures;
std::vector<std::future<void>> futures;
for (int d = 0; d < ndatastreams; d++) {
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
@@ -455,41 +336,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
continue;
}
message.image = CompressedImage(transformation.GetImage(),
experiment.GetPixelsNum() * experiment.GetByteDepthImage(),
experiment.GetXPixelsNum(),
experiment.GetYPixelsNum(),
experiment.GetImageMode(),
CompressionAlgorithm::NO_COMPRESSION);
if (gpu_analysis) {
// Compressed here into a slot of the image buffer, where it waits for the GPU analysis; GPUSink
// writes it, with its analysis, to a second slot, which is sent. A frame that finds no slot
// free, or as many frames waiting as the analysis is to hold, is dropped, as the receiver drops
// any frame that finds the image buffer full.
++images_collected;
uncompressed_size += experiment.GetModulesNum() * RAW_MODULE_SIZE * experiment.GetByteDepthImage();
auto held = std::make_shared<HeldImage>();
held->slot = image_buffer.GetImageSlot();
if (held->slot == nullptr) {
writer_queue_full = true;
continue;
}
const size_t size = transformation.CompressImage(static_cast<uint8_t *>(held->slot->GetImage()),
experiment.GetImageBufferLocationSize());
held->message = std::move(message);
held->message.image = CompressedImage(held->slot->GetImage(), size,
experiment.GetXPixelsNum(), experiment.GetYPixelsNum(),
experiment.GetImageMode(),
experiment.GetCompressionAlgorithm());
if (!gpu->Hold(held)) {
writer_queue_full = true;
continue;
}
compressed_size += size;
continue;
}
// No-op unless the CPU backend is forced; fills az_int_profile_image from the assembled image
analyzer->RunAzimuthalIntegration(transformation.GetImage(), az_int_profile_image);
@@ -616,12 +462,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) {
}
float JFJochReceiverFPGA::GetEfficiency() const {
uint64_t expected_packets;
if (experiment.GetImageNum() == 0)
expected_packets = expected_packets_per_image * experiment.GetFrameNum();
else
expected_packets = expected_packets_per_image * experiment.GetImageNum();
uint64_t expected_packets = expected_packets_per_image * experiment.GetImageNum();
uint64_t received_packets = 0;
for (int d = 0; d < ndatastreams; d++) {
@@ -650,45 +491,31 @@ void JFJochReceiverFPGA::Cancel(const JFJochException &e) {
float JFJochReceiverFPGA::GetProgress() const {
int64_t frames = experiment.GetImageNum();
if (experiment.IsCPUSummation())
frames *= experiment.GetSummation();
frames *= experiment.GetHostSummation();
if (frames == 0)
frames = experiment.GetFrameNum();
if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished()))
if (acquisition_device[0].Counters().IsAcquisitionFinished())
return 1.0;
return static_cast<float>(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast<float>(frames);
}
void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) {
AddIndexingToEndMessage(message, indexer);
}
void JFJochReceiverFPGA::FinalizeMeasurement() {
if (!frame_transformation_futures.empty()) {
for (auto &future: frame_transformation_futures)
future.get();
for (auto &future: frame_transformation_futures)
future.get();
logger.Debug("All processing threads done");
}
// Where the start message never went out (or the analysis could not be set up), there is no end
// message either: the devices are stopped all the same, and the error is the collection's result.
std::exception_ptr gpu_error;
if (gpu) {
try {
gpu->Finish();
} catch (const std::exception &) {
gpu_error = std::current_exception();
}
}
logger.Debug("All processing threads done");
current_status.SetProgress(1.0);
current_status.SetStatus(GetStatus());
if (!gpu_error)
SendEndMessage();
SendEndMessage();
if (experiment.GetImageNum() > 0) {
for (int d = 0; d < ndatastreams; d++)
acquisition_device[d].Cancel();
}
for (int d = 0; d < ndatastreams; d++)
acquisition_device[d].Cancel();
end_time = std::chrono::system_clock::now();
@@ -702,18 +529,10 @@ void JFJochReceiverFPGA::FinalizeMeasurement() {
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in "
+ std::to_string(SendFinalizeTimeout.count()) + " seconds");
}
if (gpu_error) {
current_status.SetProgress({});
current_status.SetStatus(GetStatus());
std::rethrow_exception(gpu_error);
}
logger.Info("Receiving data done");
if (push_images_to_writer)
writer_error = image_pusher.Finalize();
// Last image -> all written, as the rugnux receiver reports it
if (gpu)
gpu->Written();
current_status.SetProgress({});
current_status.SetStatus(GetStatus());
@@ -751,36 +570,15 @@ JFJochReceiverOutput JFJochReceiverFPGA::GetFinalStatistics() const {
JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics();
for (int d = 0; d < ndatastreams; d++) {
for (int m = 0; m < acquisition_device[d].Counters().GetModuleNumber(); m++) {
if (experiment.IsCPUSummation())
ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule() * experiment.GetSummation());
else
ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule());
// All the frames of the collection, whoever sums them
ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule());
ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m));
}
}
RetrievePedestal(ret.pedestal_result);
return ret;
}
void JFJochReceiverFPGA::RetrievePedestal(std::vector<JFModulePedestal> &output) const {
time_t curr_time = std::chrono::system_clock::to_time_t(start_time);
for (const auto &pc: pedestal) {
JFModulePedestal mp;
if (experiment.GetDetectorMode() == DetectorMode::PedestalG0)
pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT);
else
pc->Export(mp);
mp.SetCollectionTime(curr_time);
output.emplace_back(std::move(mp));
}
}
void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) {
if (experiment.IsPedestalRun()) {
acquisition_device[data_stream].InitializeEmptyPixelMask(experiment);
return; // No calibration loaded for pedestal
}
if (calibration != nullptr)
acquisition_device[data_stream].InitializeCalibration(experiment, *calibration);
@@ -793,97 +591,3 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) {
// Initialize data processing
acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping);
}
void JFJochReceiverFPGA::GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile) {
try {
const auto status = image_buffer.GetSlotStatus();
message.receiver_buf_available = status.available_slots;
message.receiver_buf_in_preparation = status.preparation_slots;
message.receiver_buf_in_sending = status.sending_slots;
if (message.image.GetCompressedSize() > 0)
message.compression_ratio = static_cast<float>(experiment.GetPixelsNum() * experiment.GetByteDepthImage())
/ static_cast<float>(message.image.GetCompressedSize());
if (message.image_collection_efficiency == 1.0f) {
saturated_pixels.Add(message.saturated_pixel_count);
error_pixels.Add(message.error_pixel_count);
if (message.roi.contains("beam")) {
roi_beam_npixel.Add(message.roi["beam"].pixels);
roi_beam_sum.Add(message.roi["beam"].sum);
}
}
scan_result.Add(message);
if (!serialmx_filter.ApplyFilter(message))
++images_skipped;
else if (auto loc = image_buffer.GetImageSlot(); loc == nullptr)
writer_queue_full = true;
else {
auto writer_buffer = (uint8_t *) loc->GetImage();
CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize());
if (JFJochReceiverLite::SerializeImageOrDrop(serializer, message, *loc,
experiment.GetImageBufferLocationSize(), logger)) {
loc->SetImageNumber(message.number);
loc->SetImageSize(serializer.GetBufferSize());
loc->SetIndexed(message.indexing_result.value_or(false));
loc->ReadyToSend();
if (zmq_preview_socket != nullptr)
zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize());
if (zmq_metadata_socket != nullptr)
zmq_metadata_socket->AddDataMessage(message);
if (push_images_to_writer) {
if (image_pusher.SendImage(*loc))
++images_sent;
} else
loc->release();
UpdateMaxImageSent(message.number);
}
}
plots.Add(message, profile);
current_status.SetProgress(GetProgress());
current_status.SetStatus(GetStatus());
} catch (const JFJochException &e) {
logger.ErrorException(e);
Cancel(e);
} catch (const std::exception &e) {
const JFJochException error(JFJochExceptionCategory::WrongDAQState,
fmt::format("Image {} not written - {}", message.number, e.what()));
logger.ErrorException(error);
Cancel(error);
}
}
void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) {
AddIndexingToEndMessage(message, indexer);
if (!gpu)
return;
message.unit_cell = gpu->GetConsensusUnitCell();
message.refined_beam_center_x = gpu->GetBeamX_pxl();
message.refined_beam_center_y = gpu->GetBeamY_pxl();
message.refined_detector_distance_mm = gpu->GetDetectorDistance_mm();
// The merged data, for the writer to put beside the master file.
if (const auto &m = gpu->GetMerged(); m && gpu->GetMergeExperiment().GetUnitCell())
message.merged = MergedMtzTable(m->merge.merged, *gpu->GetMergeExperiment().GetUnitCell(),
gpu->GetMergeExperiment(), OutputProvenance{});
if (!message.merged)
return;
// Every pusher serialises END into a buffer of MESSAGE_SIZE_FOR_START_END, and an END that does not fit there
// fails the collection. So it is serialised here first, as JFJochReceiverRugnuxRotation does; one that does not fit
// goes without the merged data.
const auto scratch = std::make_unique_for_overwrite<uint8_t[]>(MESSAGE_SIZE_FOR_START_END);
CBORStream2Serializer serializer(scratch.get(), MESSAGE_SIZE_FOR_START_END);
try {
serializer.SerializeSequenceEnd(message);
} catch (const JFJochException &e) {
const std::string text = fmt::format("the merged data do not fit in the end message ({} MiB) and are not "
"written", MESSAGE_SIZE_FOR_START_END / (1024 * 1024));
logger.Error("GPU analysis: {} - {}", text, e.what());
if (rugnux_status)
rugnux_status->Warning(text);
message.merged.reset();
}
}
+3 -28
View File
@@ -3,7 +3,6 @@
#pragma once
#include <condition_variable>
#include <atomic>
#include <future>
#include <latch>
@@ -21,18 +20,14 @@
#include "../common/ImageBuffer.h"
#include "../common/PixelMask.h"
#include "../image_analysis/spot_finding/StrongPixelSet.h"
#include "../common/AzimuthalIntegrationMapping.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include "../jungfrau/JFCalibration.h"
#include "../common/ADUHistogram.h"
#include "../common/Plot.h"
#include "../common/JFJochReceiverPlots.h"
#include "../preview/PreviewImage.h"
#include "LossyFilter.h"
#include "JFJochReceiverCurrentStatus.h"
#include "../jungfrau/JFPedestalCalc.h"
#include "../preview/ZMQPreviewSocket.h"
#include "../preview/ZMQMetadataSocket.h"
#include "ImageMetadata.h"
@@ -40,15 +35,12 @@
#include "../image_analysis/MXAnalysisAfterFPGA.h"
#include "../common/MovingAverage.h"
#include "JFJochReceiverOutput.h"
#include "StillsGPUAnalysisFPGA.h"
class JFJochReceiverFPGA : public JFJochReceiver {
const JFCalibration *calibration;
IndexAndRefine indexer;
std::vector<std::unique_ptr<JFPedestalCalc>> pedestal;
std::vector<uint16_t> roi_map;
std::vector<std::future<void>> frame_transformation_futures;
@@ -62,11 +54,9 @@ class JFJochReceiverFPGA : public JFJochReceiver {
uint16_t ndatastreams{0};
const int64_t frame_transformation_nthreads;
const int64_t pedestal_nthreads;
const int64_t summation_nthreads;
int64_t expected_packets_per_image;
bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells
std::latch frame_transformation_ready;
std::latch data_acquisition_ready;
@@ -75,31 +65,18 @@ class JFJochReceiverFPGA : public JFJochReceiver {
void LoadCalibrationToFPGA(uint16_t data_stream);
void FrameTransformationThread(uint32_t threadid);
void MeasurePedestalThread(uint16_t data_stream,
uint16_t module_number,
uint16_t storage_cell,
uint32_t threadid,
bool ignore);
int64_t SummationThread(uint16_t data_stream,
void SummationThread(uint16_t data_stream,
int64_t image_number,
uint16_t module_number,
uint32_t threadid,
ModuleSummation &summation);
void Cancel(const JFJochException &e) override;
void FinalizeMeasurement();
// Serial data with rugnux on: the converted frames are analysed on the GPU (StillsGPUAnalysisFPGA), and
// the start message waits for its pre-pass.
const bool gpu_analysis;
RugnuxLiveStatus *const rugnux_status;
std::unique_ptr<StillsGPUAnalysisFPGA> gpu;
void GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile);
void AddToEndMessage(EndMessage &message) override;
void FinalizeMeasurement();
void SendCalibration();
void SendPedestal(const std::string &prefix, const std::vector<uint8_t> &v, int gain, int sc);
void RetrievePedestal(std::vector<JFModulePedestal> &output) const;
public:
JFJochReceiverFPGA(const DiffractionExperiment& experiment,
const PixelMask &pixel_mask,
@@ -113,9 +90,7 @@ public:
ImageBuffer &image_buffer,
ZMQPreviewSocket *zmq_preview_socket,
ZMQMetadataSocket *zmq_metadata_socket,
IndexerThreadPool *indexing_thread_pool,
bool gpu_analysis = false,
RugnuxLiveStatus *rugnux_status = nullptr);
IndexerThreadPool *indexing_thread_pool);
~JFJochReceiverFPGA();
JFJochReceiverFPGA(const JFJochReceiverFPGA &other) = delete;
JFJochReceiverFPGA& operator=(const JFJochReceiverFPGA &other) = delete;
+242
View File
@@ -0,0 +1,242 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "JFJochReceiverPedestal.h"
JFJochReceiverPedestal::JFJochReceiverPedestal(const DiffractionExperiment &in_experiment,
const PixelMask &in_pixel_mask,
AcquisitionDeviceGroup &in_aq_device,
ImagePusher &in_image_sender,
Logger &in_logger,
const SpotFindingSettings &in_spot_finding_settings,
JFJochReceiverCurrentStatus &in_current_status,
JFJochReceiverPlots &in_plots,
ImageBuffer &in_send_buf_ctrl)
: JFJochReceiver(in_experiment,
in_send_buf_ctrl,
in_image_sender,
in_current_status,
in_plots,
in_spot_finding_settings,
in_logger,
in_pixel_mask),
acquisition_device(in_aq_device),
ndatastreams(experiment.GetDataStreamsNum()),
pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4),
data_acquisition_ready(ndatastreams) {
image_buffer.StartMeasurement(experiment);
if (acquisition_device.size() < ndatastreams)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Number of acquisition devices has to match data streams");
expected_packets_per_image = 0;
for (int d = 0; d < ndatastreams; d++) {
acquisition_device[d].PrepareAction(experiment);
expected_packets_per_image += acquisition_device[d].Counters().GetExpectedPacketsPerImage();
logger.Debug("Acquisition device {} prepared", d);
}
logger.Debug("Data acquisition devices ready");
switch (experiment.GetDetectorMode()) {
case DetectorMode::PedestalG1:
only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2);
break;
case DetectorMode::PedestalG2:
only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2);
break;
default:
only_2nd_sc_pedestal = false;
break;
}
int64_t pedestal_count = (only_2nd_sc_pedestal)
? experiment.GetModulesNum()
: experiment.GetModulesNum() * experiment.GetStorageCellNumber();
for (int i = 0; i < pedestal_count; i++)
pedestal.emplace_back(std::make_unique<JFPedestalCalc>(experiment));
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
bool ignore = only_2nd_sc_pedestal && (s == 0);
for (int d = 0; d < ndatastreams; d++)
for (int m = 0; m < experiment.GetModulesNum(d); m++)
for (int n = 0; n < pedestal_nthreads; n++)
pedestal_futures.emplace_back(std::async(std::launch::async,
&JFJochReceiverPedestal::MeasurePedestalThread,
this, d, m, s, n, ignore));
}
logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads);
for (int d = 0; d < ndatastreams; d++)
data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverPedestal::AcquireThread,
this, d));
data_acquisition_ready.wait();
start_time = std::chrono::system_clock::now();
measurement = std::async(std::launch::async, &JFJochReceiverPedestal::FinalizeMeasurement, this);
logger.Info("Receiving data started");
}
void JFJochReceiverPedestal::AcquireThread(uint16_t data_stream) {
try {
acquisition_device[data_stream].InitializeEmptyPixelMask(experiment);
logger.Debug("Device thread {} start action", data_stream);
acquisition_device[data_stream].StartAction(experiment);
} catch (const JFJochException &e) {
Cancel(e);
data_acquisition_ready.count_down();
logger.ErrorException(e);
logger.Warning("Device thread {} done due to an error", data_stream);
return;
}
data_acquisition_ready.count_down();
try {
logger.Debug("Device thread {} wait for action complete", data_stream);
acquisition_device[data_stream].WaitForActionComplete();
} catch (const JFJochException &e) {
logger.ErrorException(e);
Cancel(e);
logger.ErrorException(e);
logger.Warning("Device thread {} done due to an error", data_stream);
return;
}
logger.Debug("Device thread {} done", data_stream);
}
void JFJochReceiverPedestal::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell,
uint32_t threadid, bool ignore) {
JFPedestalCalc pedestal_calc(experiment);
uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber();
uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads;
uint32_t storage_cell_header = UINT32_MAX;
try {
for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) {
// Frame will be processed only if one already collects frame+2
acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number);
if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) {
auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number);
// Partial packets will bring more problems, than benefit
pedestal_calc.AnalyzeImage((uint16_t *) output->pixels);
storage_cell_header = (output->module_statistics.debug >> 8) & 0xF;
}
acquisition_device[data_stream].FrameBufferRelease(frame, module_number);
UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number));
current_status.SetProgress(GetProgress());
current_status.SetStatus(GetStatus());
}
uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number;
if (!only_2nd_sc_pedestal)
offset += experiment.GetModulesNum() * storage_cell;
if (!ignore)
*pedestal[offset] += pedestal_calc;
} catch (const JFJochException &e) {
Cancel(e);
}
logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done",
data_stream, module_number, storage_cell, storage_cell_header);
}
float JFJochReceiverPedestal::GetEfficiency() const {
uint64_t expected_packets = expected_packets_per_image * experiment.GetFrameNum();
uint64_t received_packets = 0;
for (int d = 0; d < ndatastreams; d++) {
received_packets += acquisition_device[d].Counters().GetTotalPackets();
}
if ((expected_packets == received_packets) || (expected_packets == 0))
return 1.0;
return received_packets / static_cast<double>(expected_packets);
}
void JFJochReceiverPedestal::Cancel(bool silent) {
JFJochReceiver::Cancel(silent);
for (int d = 0; d < ndatastreams; d++)
acquisition_device[d].Cancel();
}
void JFJochReceiverPedestal::Cancel(const JFJochException &e) {
JFJochReceiver::Cancel(e);
for (int d = 0; d < ndatastreams; d++)
acquisition_device[d].Cancel();
}
float JFJochReceiverPedestal::GetProgress() const {
int64_t frames = experiment.GetFrameNum();
if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished()))
return 1.0;
return static_cast<float>(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast<float>(frames);
}
void JFJochReceiverPedestal::FinalizeMeasurement() {
for (auto &future: pedestal_futures)
future.get();
logger.Debug("All processing threads done");
current_status.SetProgress(1.0);
current_status.SetStatus(GetStatus());
end_time = std::chrono::system_clock::now();
for (auto &future: data_acquisition_futures)
future.get();
logger.Debug("Devices stopped");
logger.Info("Receiving data done");
current_status.SetProgress({});
current_status.SetStatus(GetStatus());
}
void JFJochReceiverPedestal::StopReceiver() {
if (measurement.valid()) {
measurement.get();
logger.Debug("Receiver stopped");
}
}
JFJochReceiverPedestal::~JFJochReceiverPedestal() {
try {
if (measurement.valid())
measurement.get();
} catch (const std::exception &e) {
logger.Error("ReceiverPedestal teardown failed: {}", e.what());
} catch (...) {
logger.Error("ReceiverPedestal teardown failed with unknown exception");
}
}
JFJochReceiverOutput JFJochReceiverPedestal::GetFinalStatistics() const {
JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics();
for (int d = 0; d < ndatastreams; d++) {
for (int m = 0; m < acquisition_device[d].Counters().GetModuleNumber(); m++) {
ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule());
ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m));
}
}
time_t curr_time = std::chrono::system_clock::to_time_t(start_time);
for (const auto &pc: pedestal) {
JFModulePedestal mp;
if (experiment.GetDetectorMode() == DetectorMode::PedestalG0)
pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT);
else
pc->Export(mp);
mp.SetCollectionTime(curr_time);
ret.pedestal_result.emplace_back(std::move(mp));
}
return ret;
}
+63
View File
@@ -0,0 +1,63 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <future>
#include <latch>
#include <memory>
#include <vector>
#include "JFJochReceiver.h"
#include "../acquisition_device/AcquisitionDeviceGroup.h"
#include "../jungfrau/JFPedestalCalc.h"
// A pedestal run: raw frames from the devices go into JFPedestalCalc; no image buffer slots, nothing goes out on any socket
class JFJochReceiverPedestal : public JFJochReceiver {
std::vector<std::unique_ptr<JFPedestalCalc>> pedestal;
std::vector<std::future<void>> pedestal_futures;
std::vector<std::future<void>> data_acquisition_futures;
std::future<void> measurement;
AcquisitionDeviceGroup &acquisition_device;
uint16_t ndatastreams{0};
const int64_t pedestal_nthreads;
int64_t expected_packets_per_image;
bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells
std::latch data_acquisition_ready;
void AcquireThread(uint16_t data_stream);
void MeasurePedestalThread(uint16_t data_stream,
uint16_t module_number,
uint16_t storage_cell,
uint32_t threadid,
bool ignore);
void Cancel(const JFJochException &e) override;
void FinalizeMeasurement();
public:
JFJochReceiverPedestal(const DiffractionExperiment &experiment,
const PixelMask &pixel_mask,
AcquisitionDeviceGroup &acquisition_devices,
ImagePusher &image_pusher,
Logger &logger,
const SpotFindingSettings &spot_finding_settings,
JFJochReceiverCurrentStatus &current_status,
JFJochReceiverPlots &plots,
ImageBuffer &image_buffer);
~JFJochReceiverPedestal() override;
JFJochReceiverPedestal(const JFJochReceiverPedestal &other) = delete;
JFJochReceiverPedestal &operator=(const JFJochReceiverPedestal &other) = delete;
void StopReceiver() override;
float GetEfficiency() const override;
void Cancel(bool silent) override;
float GetProgress() const override;
JFJochReceiverOutput GetFinalStatistics() const override;
};
+67 -31
View File
@@ -8,8 +8,10 @@
#include "../common/time_utc.h"
#include "../frame_serialize/CBORStream2Serializer.h"
#include "../rugnux/Rugnux.h"
#include "../rugnux/RugnuxDefaults.h"
#include "RugnuxStructure.h"
#include "StillsSelection.h"
JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperiment &in_experiment,
const PixelMask &in_pixel_mask,
@@ -27,16 +29,19 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
ImageBuffer &in_image_buffer,
ZMQPreviewSocket *in_zmq_preview_socket,
ZMQMetadataSocket *in_zmq_metadata_socket,
IndexerThreadPool *in_indexing_thread_pool)
IndexerThreadPool *in_indexing_thread_pool,
int64_t in_downsample)
: JFJochReceiverStream(in_experiment, in_pixel_mask, in_image_puller, in_image_pusher, in_logger,
in_forward_nthreads, in_spot_finding_settings, in_current_status, in_plots,
in_image_buffer, in_zmq_preview_socket, in_zmq_metadata_socket),
model(std::move(in_model)),
deadline_s(in_deadline_s),
refine_solvent(in_refine_solvent),
downsample(in_downsample),
live_status(in_live_status),
indexing_thread_pool(in_indexing_thread_pool),
stills_nthreads(std::min<int64_t>(16, in_forward_nthreads)) {
// As offline rugnux's per-image loop: 16 a card, at most the threads given
stills_nthreads(ImageWorkers(in_experiment.GetPixelsNum(), static_cast<int>(in_forward_nthreads))) {
StartThreads();
}
@@ -47,7 +52,10 @@ JFJochReceiverRugnuxStills::~JFJochReceiverRugnuxStills() {
void JFJochReceiverRugnuxStills::OnStart(std::chrono::steady_clock::time_point received) {
const int64_t slots = image_buffer.GetStatus().total_slots;
stills_keep_free = std::min(stills_nthreads, slots / 2);
// The frames that wait take at most half of the image buffer; the other half is for those analysed, sent from
// a slot each, and a slot is kept free for each analysis thread to send with
hold_arena = std::make_unique<HoldArena>(image_buffer, slots / 2, std::min(stills_nthreads, slots / 2),
experiment.GetImageBufferLocationSize());
const StartMessage start = MakeStartMessage(experiment, pixel_mask, *az_int_mapping);
// The frames are shown from their arrival, on the stated geometry until the pre-pass has decided its own.
// To the image buffer only: the writer and the ZeroMQ streams get the one start message, after the pre-pass.
@@ -63,8 +71,8 @@ void JFJochReceiverRugnuxStills::OnStart(std::chrono::steady_clock::time_point r
auto analysis = std::make_unique<StillsGPUAnalysis>(
analysis_experiment, pixel_mask, analysis_start,
indexing_thread_pool, stills_nthreads,
// What the pre-pass holds and fits on; the frames beyond it wait in the image buffer as well.
slots / 2, logger,
// What the pre-pass holds and fits on, at most: the hold full starts it earlier (OnImage)
StillsGPUAnalysis::PREPASS_MAX_IMAGES, logger,
*this, live_status, true, model, deadline_s, refine_solvent);
std::unique_lock ul(stills_mutex);
stills = std::move(analysis);
@@ -82,39 +90,33 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
CountImage(message);
const size_t size = message.image.GetCompressedSize();
// Waits for the analysis in a slot of the image buffer; a slot is kept free for each analysis thread to send with
// Waits for the analysis as its compressed bytes, packed into at most half of the image buffer (HoldArena)
auto held = std::make_shared<HeldImage>();
held->slot = image_buffer.GetImageSlot(stills_keep_free);
if (held->slot == nullptr) {
writer_queue_full = true;
return;
// With the hold full, the frame waits: the pre-pass starts on what is held and the analysis drains it. No frame is
// lost here; a source faster than the analysis loses frames at the card.
HoldArena::Region region = hold_arena->Allocate(size);
if (!region.data || hold_arena->Full(size))
stills->StartPrePass();
while (!region.data && !cancelled) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
region = hold_arena->Allocate(size);
}
if (!region.data)
return;
memcpy(region.data, message.image.GetCompressed(), size);
held->release = std::move(region.release);
// Not chosen (stills_downsample): written as it came, without its analysis
held->analyse = StillsImageSelected(message.number, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES);
held->message = message;
StampImage(held->message, image_puller.GetCurrentFifoUtilization());
// In the slot as an image without analysis, so that it can be seen while it waits; it is not sent. The
// analysis reads the image where it is in the slot.
auto slot_buffer = static_cast<uint8_t *>(held->slot->GetImage());
CBORStream2Serializer serializer(slot_buffer, experiment.GetImageBufferLocationSize());
held->message.image = CompressedImage(nullptr, 0, message.image.GetWidth(), message.image.GetHeight(),
held->message.image = CompressedImage(region.data, size, message.image.GetWidth(), message.image.GetHeight(),
message.image.GetMode(), message.image.GetCompressionAlgorithm(),
message.image.GetChannel());
serializer.SerializeImage(held->message);
const size_t image_offset = serializer.GetImageAppendOffset();
if (image_offset + size + 32 >= static_cast<size_t>(experiment.GetImageBufferLocationSize())) {
logger.Error("Dropping image {} - metadata {} B + image {} B do not fit in buffer slot {} B", message.number,
image_offset, size, experiment.GetImageBufferLocationSize());
return;
{
// Seen while it waits (GetWaitingImage): the message as it came, as the analysis changes its own
std::unique_lock ul(waiting_mutex);
waiting[message.number] = Waiting{.held = held, .message = held->message};
}
memcpy(slot_buffer + image_offset, message.image.GetCompressed(), size);
serializer.AppendImage(size);
held->message.image = CompressedImage(slot_buffer + image_offset, size, message.image.GetWidth(),
message.image.GetHeight(), message.image.GetMode(),
message.image.GetCompressionAlgorithm(), message.image.GetChannel());
held->slot->SetImageNumber(message.number);
held->slot->SetImageSize(serializer.GetBufferSize());
held->slot->SetIndexed(false);
held->slot->ReadyToSend();
stills->Hold(held);
}
@@ -128,6 +130,8 @@ void JFJochReceiverRugnuxStills::FinishCollection() {
} catch (const std::exception &) {
error = std::current_exception();
}
// Every frame is analysed: the slots the frames waited in go back, before the image buffer is checked
hold_arena->ReleaseSlots();
current_status.SetProgress(1.0);
current_status.SetStatus(GetStatus());
@@ -157,6 +161,8 @@ void JFJochReceiverRugnuxStills::StillsReady(const StillsSetup &setup) {
plots.Setup(setup.experiment, *setup.mapping);
DiffractionExperiment written(setup.experiment);
written.Goniometer(experiment.GetGoniometer());
// The cell as stated: the one the frames are indexed on is the analysis's, and the merge reports its own
written.SetUnitCell(experiment.GetUnitCell());
SendStartMessage(MakeStartMessage(written, setup.mask, *setup.mapping));
}
@@ -188,6 +194,10 @@ ZeroCopyReturnValue *JFJochReceiverRugnuxStills::AnalysedImageSlot() {
}
void JFJochReceiverRugnuxStills::SendAnalysedImage(DataMessage &message, const AzimuthalIntegrationProfile &profile) {
{
std::unique_lock ul(waiting_mutex);
waiting.erase(message.number);
}
try {
const auto status = image_buffer.GetSlotStatus();
message.receiver_buf_available = status.available_slots;
@@ -215,6 +225,32 @@ void JFJochReceiverRugnuxStills::SendAnalysedImage(DataMessage &message, const A
}
}
std::optional<int64_t> JFJochReceiverRugnuxStills::GetLastWaitingImage() {
std::unique_lock ul(waiting_mutex);
// A frame the analysis let go of without sending it (a failed setup) is no longer there
while (!waiting.empty() && waiting.rbegin()->second.held.expired())
waiting.erase(std::prev(waiting.end()));
if (waiting.empty())
return {};
return waiting.rbegin()->first;
}
bool JFJochReceiverRugnuxStills::GetWaitingImage(std::vector<uint8_t> &v, int64_t image_number) {
std::unique_lock ul(waiting_mutex);
const auto it = waiting.find(image_number);
if (it == waiting.end())
return false;
// The frame's bytes in the hold stay while this copy is made
const auto held = it->second.held.lock();
if (!held)
return false;
v.resize(experiment.GetImageBufferLocationSize());
CBORStream2Serializer serializer(v.data(), v.size());
serializer.SerializeImage(it->second.message);
v.resize(serializer.GetBufferSize());
return true;
}
void JFJochReceiverRugnuxStills::AddToEndMessage(EndMessage &message) {
JFJochReceiverStream::AddToEndMessage(message);
// A failure here costs the end message the analysis only: it goes out as the receiver made it.
+19 -2
View File
@@ -3,12 +3,14 @@
#pragma once
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include "JFJochReceiverStream.h"
#include "RugnuxLiveStatus.h"
#include "HoldArena.h"
#include "StillsGPUAnalysis.h"
// A serial collection analysed frame by frame at the geometry of a pre-pass, which the start message waits for
@@ -16,11 +18,22 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin
const std::shared_ptr<const RugnuxModel> model;
const double deadline_s; // after the last image, the model step is cut (0 = never)
const bool refine_solvent; // the refinement of the placed model builds the solvent
const int64_t downsample; // 1 of every this many images analysed on average (StillsImageSelected)
RugnuxLiveStatus &live_status;
IndexerThreadPool *const indexing_thread_pool;
const int64_t stills_nthreads;
int64_t stills_keep_free = 0; // slots frames do not take, for the analysis threads to send with
// The frames waiting for the analysis, packed as their compressed bytes into at most half of the image buffer's
// slots (a frame is a fraction of a slot: 1.5 MB of a 24 MB slot on a 9M detector, and the pre-pass holds every
// frame that arrives before it is over); a slot is kept free for each analysis thread to send with
std::unique_ptr<HoldArena> hold_arena;
// The frames waiting, by number, as they came, to be seen while they wait
struct Waiting {
std::weak_ptr<HeldImage> held;
DataMessage message;
};
std::mutex waiting_mutex;
std::map<int64_t, Waiting> waiting;
std::mutex stills_mutex;
bool cut = false;
@@ -53,6 +66,10 @@ public:
ImageBuffer &image_buffer,
ZMQPreviewSocket *zmq_preview_socket = nullptr,
ZMQMetadataSocket *zmq_metadata_socket = nullptr,
IndexerThreadPool *indexing_thread_pool = nullptr);
IndexerThreadPool *indexing_thread_pool = nullptr,
int64_t downsample = 1);
~JFJochReceiverRugnuxStills() override;
bool GetWaitingImage(std::vector<uint8_t> &v, int64_t image_number) override;
std::optional<int64_t> GetLastWaitingImage() override;
};
+69 -14
View File
@@ -9,6 +9,7 @@
#include "JFJochReceiverFPGA.h"
#include "JFJochReceiverLite.h"
#include "JFJochReceiverDarkMask.h"
#include "JFJochReceiverPedestal.h"
#include "../preview/JFJochJPEG.h"
#include "../preview/JFJochTIFF.h"
@@ -24,7 +25,7 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic
JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
int reserved_cpus, double deadline_s, bool refine_solvent,
bool force_still) {
bool force_still, int64_t stills_downsample) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
@@ -35,9 +36,27 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
rugnux_deadline_s = deadline_s;
rugnux_refine_solvent = refine_solvent;
rugnux_force_still = force_still;
rugnux_stills_downsample = stills_downsample;
return *this;
}
JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(bool always) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change the receiver path during data collection");
fpga_image_puller_always = always;
return *this;
}
// A PSI collection analysed by rugnux live
bool JFJochReceiverService::FPGAImagePullerFor(const DiffractionExperiment &experiment) const {
if ((experiment.GetDetectorType() == DetectorType::DECTRIS) || experiment.IsPedestalRun()
|| (experiment.GetImageNum() <= 0))
return false;
return fpga_image_puller_always || (rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard));
}
RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const {
return rugnux_status.Get();
}
@@ -64,13 +83,23 @@ void JFJochReceiverService::FinalizeMeasurementChangeState() {
}
void JFJochReceiverService::FinalizeMeasurement() {
// The puller is stopped however the receiver ended: its threads hold the cards' buffers otherwise
std::exception_ptr error;
try {
receiver->StopReceiver();
} catch (...) {
FinalizeMeasurementChangeState();
throw;
error = std::current_exception();
}
try {
if (fpga_puller)
fpga_puller->Stop();
} catch (...) {
if (!error)
error = std::current_exception();
}
FinalizeMeasurementChangeState();
if (error)
std::rethrow_exception(error);
}
std::optional<JFJochReceiverStatus> JFJochReceiverService::GetStatus() {
@@ -106,11 +135,24 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
try {
// First clean-up old measurement
receiver.reset();
image_puller.reset();
fpga_puller.reset();
analysis_warning.clear();
preview_image.Configure(experiment, pixel_mask);
if (experiment.GetDetectorType() != DetectorType::DECTRIS) {
// A PSI detector's frames through FPGAImagePuller go to the stream receivers, as a DECTRIS stream does
if (FPGAImagePullerFor(experiment)) {
fpga_puller = std::make_shared<FPGAImagePuller>(experiment, pixel_mask, calibration, aq_devices, logger,
nthreads);
puller = fpga_puller;
}
if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && experiment.IsPedestalRun()) {
receiver = std::make_unique<JFJochReceiverPedestal>(experiment, pixel_mask, aq_devices, image_pusher,
logger, spot_finding_settings, receiver_status,
plots, image_buffer);
} else if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) {
// Summation on the host runs threads of its own for each image
auto fpga_nthreads = nthreads;
if (experiment.IsCPUSummation())
@@ -126,11 +168,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
image_buffer,
zmq_preview_socket.get(),
zmq_metadata_socket.get(),
indexer_thread_pool.get(),
// Serial data with rugnux on: the frames are
// analysed on the GPU, after a pre-pass.
rugnux_enable && !RugnuxRotationData(experiment),
&rugnux_status);
indexer_thread_pool.get());
} else {
if (puller)
image_puller = puller;
@@ -175,7 +213,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
rugnux_model, rugnux_deadline_s, rugnux_refine_solvent, rugnux_status, spot_finding_settings,
receiver_status, plots, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(),
indexer_thread_pool.get());
indexer_thread_pool.get(), rugnux_stills_downsample);
else
receiver = std::make_unique<JFJochReceiverLite>(experiment,
pixel_mask,
@@ -195,6 +233,8 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
state = ReceiverState::Running;
} catch (const std::exception &e) {
receiver.reset(); // its threads may be polling the puller
image_puller.reset();
fpga_puller.reset();
// The receiver never started, so drop the status its base constructor had already reset -
// otherwise /status and /statistics keep reporting a zero-progress run that never happened,
// until the next start overwrites it.
@@ -207,8 +247,14 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
void JFJochReceiverService::Cancel(bool silent) {
std::unique_lock ul(state_mutex);
if (state == ReceiverState::Running)
if (state == ReceiverState::Running) {
receiver->Cancel(silent);
// Silent: the detector has finished, and what the cards have got still goes to the receiver
if (fpga_puller && silent)
fpga_puller->StopCards();
else if (fpga_puller)
fpga_puller->Cancel();
}
}
JFJochReceiverOutput JFJochReceiverService::Stop() {
@@ -373,20 +419,29 @@ void JFJochReceiverService::GetStartMessageFromBuffer(std::vector<uint8_t> &v) {
image_buffer.GetStartMessage(v);
}
// The image buffer, then the frames the receiver holds and has not sent yet (live stills, waiting for the analysis)
bool JFJochReceiverService::GetImageFromBuffer(std::vector<uint8_t> &v, int64_t image_number) {
return image_buffer.GetImage(v, image_number);
std::unique_lock ul(state_mutex); // the receiver is replaced at a start
if (receiver && image_number == ImageBuffer::MaxImage) {
const auto waiting = receiver->GetLastWaitingImage();
if (waiting && *waiting > image_buffer.GetStatus().max_image_number)
image_number = *waiting;
}
if (image_buffer.GetImage(v, image_number))
return true;
return receiver && (image_number != ImageBuffer::MaxImage) && receiver->GetWaitingImage(v, image_number);
}
std::string JFJochReceiverService::GetJPEGFromBuffer(const PreviewImageSettings &settings, int64_t image_number) {
std::vector<uint8_t> cbor_image;
if (!image_buffer.GetImage(cbor_image, image_number))
if (!GetImageFromBuffer(cbor_image, image_number))
return {};
return preview_image.GenerateImage(settings, cbor_image);
}
std::string JFJochReceiverService::GetTIFFFromBuffer(int64_t image_number) {
std::vector<uint8_t> cbor_image;
if (!image_buffer.GetImage(cbor_image, image_number))
if (!GetImageFromBuffer(cbor_image, image_number))
return {};
return PreviewImage::GenerateTIFF(cbor_image);
}
+13 -1
View File
@@ -9,6 +9,7 @@
#include <memory>
#include "JFJochReceiver.h"
#include "FPGAImagePuller.h"
#include "../preview/PreviewImage.h"
#include "../image_puller/ZMQImagePuller.h"
#include "../image_analysis/indexing/IndexerThreadPool.h"
@@ -19,6 +20,7 @@
struct RugnuxModel;
class JFJochReceiverService {
std::unique_ptr<JFJochReceiver> receiver;
AcquisitionDeviceGroup &aq_devices;
@@ -35,9 +37,15 @@ class JFJochReceiverService {
double rugnux_deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
bool rugnux_refine_solvent = false; // the refinement of a placed model builds the solvent
bool rugnux_force_still = false; // rotation collections from a DECTRIS detector go the stills way
int64_t rugnux_stills_downsample = 1; // serial collections: 1 of every this many images analysed
RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends
std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status
// A PSI collection analysed by rugnux live goes through FPGAImagePuller to the stream receivers
bool fpga_image_puller_always = false;
std::shared_ptr<FPGAImagePuller> fpga_puller;
bool FPGAImagePullerFor(const DiffractionExperiment &experiment) const;
enum class ReceiverState {Idle, Running};
ReceiverState state = ReceiverState::Idle;
mutable std::mutex state_mutex;
@@ -61,9 +69,13 @@ public:
size_t send_buffer_size_MiB = 512);
JFJochReceiverService& NumThreads(int64_t input);
JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model, int reserved_cpus,
double deadline_s, bool refine_solvent = false, bool force_still = false);
double deadline_s, bool refine_solvent = false, bool force_still = false,
int64_t stills_downsample = 1);
RugnuxLiveSnapshot GetRugnuxStatus() const;
void ClearRugnuxStatus(); // back to Idle, as before any collection
// Every PSI collection but a pedestal run through FPGAImagePuller, without rugnux to JFJochReceiverLite (for tests
// comparing it with JFJochReceiverFPGA)
JFJochReceiverService& FPGAImagePullerSettings(bool always);
JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional<int32_t> &watermark = {});
JFJochReceiverService& MetadataSocket(const std::string &addr);
-1
View File
@@ -57,7 +57,6 @@ protected:
virtual void Cut(const std::string &reason) {} // the analysis that goes on after the images, if there is one
void AddToEndMessage(EndMessage &message) override;
uint64_t GetImagesDropped() const override { return image_puller.GetDroppedImages(); }
public:
// Takes what the detector's start message decides into the experiment.
static void ApplyStartMessage(DiffractionExperiment &experiment, const StartMessage &message);
+18 -9
View File
@@ -191,7 +191,8 @@ void StillsGPUAnalysis::PrePassThread() {
Publish();
try {
const auto t0 = std::chrono::steady_clock::now();
auto ret = RunStillsPrePass(start, sample, static_cast<int>(std::thread::hardware_concurrency()));
auto ret = RunStillsPrePass(start, sample, static_cast<int>(std::thread::hardware_concurrency()),
experiment.GetMaxSpotCountGiven());
logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, "
"distance {:.2f} -> {:.2f} mm on {} frames, masked pixels {} -> {}", ret.images,
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
@@ -218,8 +219,17 @@ void StillsGPUAnalysis::PrePassThread() {
}
const bool failed = !result;
int geometry_frames = 0;
// Spots kept per frame: as given for the collection, else rugnux's default (below: the budget the pre-pass kept)
setup.experiment.MaxSpotCount(experiment.GetMaxSpotCountGiven().value_or(RUGNUX_MAX_SPOT_COUNT));
if (result) {
setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm);
// The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the refined
// distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off)
if (result->cell)
setup.experiment.SetUnitCell(*result->cell);
// and on as many spots as the pre-pass kept, as offline: the count given, or the budget it measured (the
// broker's per-image default of 250 left a 9M frame of ~380 spots a third short of what its lattice is refined on)
setup.experiment.MaxSpotCount(result->max_spot_count);
// The frames are integrated as the pre-pass integrated its own: its polarization and background ring
setup.experiment.PolarizationFactor(result->polarization);
setup.experiment.ImportBraggIntegrationSettings(result->integration);
@@ -260,7 +270,7 @@ void StillsGPUAnalysis::PrePassThread() {
merge_experiment = setup.experiment;
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
setup.experiment.GetUnitCell(),
experiment.GetUnitCell(),
result ? result->run_width : std::nullopt,
setup.experiment.IsDetectIceRings(),
SpotFindingSettings().ice_ring_width_Q_recipA, logger);
@@ -322,7 +332,7 @@ void StillsGPUAnalysis::AnalysisThread() {
const auto start_time = std::chrono::steady_clock::now();
profile.emplace(*setup.mapping);
try {
if (analysis)
if (analysis && held->analyse)
analysis->AnalyzeStreamImage(message, *profile, sink.AnalysisSettings());
} catch (const JFJochException &e) {
// The frame is still written, without its analysis.
@@ -334,16 +344,15 @@ void StillsGPUAnalysis::AnalysisThread() {
- start_time).count();
try {
sink.SendAnalysedImage(message, *profile);
if (merge)
merge_queue.Put(message.number);
int64_t analysed;
{
if (held->analyse) {
if (merge)
merge_queue.Put(message.number);
std::unique_lock ul(live_mutex);
analysed = ++live.images_analysed;
++live.images_analysed;
if (message.indexing_result.value_or(false))
++live.images_indexed;
}
status.Progress(analysed, experiment.GetImageNum());
status.Progress(++images_done, experiment.GetImageNum());
Publish();
} catch (const std::exception &e) {
// The thread goes on with the next frame, so that the queue is drained.
+3 -1
View File
@@ -67,6 +67,8 @@ public:
~StillsGPUAnalysis();
void Hold(const std::shared_ptr<HeldImage> &image);
// The frames held are all the pre-pass gets (the end of the collection, or no room for more): it starts on them
void StartPrePass();
// No frame comes after this: every frame is analysed before it returns. Throws if the setup after the pre-pass failed.
void Finish();
// After Finish, once everything is written; without it the outcome is reported when the analysis goes
@@ -146,6 +148,7 @@ private:
std::condition_variable model_cv;
std::optional<ModelJob> model_next;
std::atomic<bool> step_cut{false}; // the step running is cut
std::atomic<int64_t> images_done{0}; // sent, analysed or not
bool model_stop = false;
bool model_cut = false; // the steps from here on are cut too
bool model_over = false; // the step on the merge of the collection has ended: nothing is left to cut
@@ -158,7 +161,6 @@ private:
void ModelThread();
void PublishModel(const ModelStepResult &result, const std::string &title);
void StartPrePass();
void PrePassThread();
void AnalysisThread();
void MergeThread();
-419
View File
@@ -1,419 +0,0 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "StillsGPUAnalysisFPGA.h"
#include <thread>
#include "../common/CUDAWrapper.h"
#include "../common/ThreadAffinity.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
#include "../rugnux/RugnuxDefaults.h"
StillsGPUAnalysisFPGA::StillsGPUAnalysisFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_mask,
const StartMessage &in_start, IndexerThreadPool *in_indexing_thread_pool,
int64_t nthreads, int64_t in_max_held, Logger &in_logger, Ready in_ready,
Sink in_sink, Settings in_settings, Failed in_failed,
RugnuxLiveStatus *in_status, bool in_merge)
: experiment(in_experiment), mask(in_mask), start(in_start), indexing_thread_pool(in_indexing_thread_pool),
logger(in_logger), ready(std::move(in_ready)), sink(std::move(in_sink)), settings(std::move(in_settings)),
setup_failed(std::move(in_failed)), status(in_status), merge_frames(in_merge),
max_held(std::max<int64_t>(1, in_max_held)), queue(max_held) {
live.images_needed = std::min(PREPASS_IMAGES, experiment.GetImageNum());
live.file_beam_x_pxl = experiment.GetBeamX_pxl();
live.file_beam_y_pxl = experiment.GetBeamY_pxl();
live.file_distance_mm = experiment.GetDetectorDistance_mm();
if (status) {
status->Start(experiment.GetImageNum(),
static_cast<double>(experiment.GetImageNum())
* std::chrono::duration<double>(experiment.GetImageTime()).count(),
std::chrono::steady_clock::now());
status->Step("Pre-pass");
Publish();
}
prepass_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::PrePassThread, this);
for (int64_t i = 0; i < nthreads; i++)
analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysisFPGA::AnalysisThread, this));
logger.Info("GPU analysis: {} threads, at least the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES);
}
StillsGPUAnalysisFPGA::~StillsGPUAnalysisFPGA() {
try {
if (!finished)
Finish();
} catch (const std::exception &e) {
logger.Error("GPU analysis: {}", e.what());
}
if (status && outcome)
status->Finish(outcome->first, outcome->second);
}
void StillsGPUAnalysisFPGA::AnalysisError(const std::string &text) {
logger.Error("GPU analysis: {}", text);
std::unique_lock ul(error_mutex);
if (analysis_error.empty())
analysis_error = text;
}
void StillsGPUAnalysisFPGA::Publish() {
std::unique_lock ul(live_mutex);
if (live.prepass == RugnuxLivePrePass::Done || live.prepass == RugnuxLivePrePass::Failed) {
const double t = std::chrono::duration<double>(std::chrono::steady_clock::now() - ready_time).count();
if (t > 0 && live.images_analysed > 0)
live.analysis_rate_hz = static_cast<double>(live.images_analysed) / t;
}
status->Stills(live);
}
bool StillsGPUAnalysisFPGA::Hold(const std::shared_ptr<HeldImage> &image) {
if (status)
status->ImageReceived(std::chrono::steady_clock::now());
if (!queue.Put(image)) {
// As many frames wait as the buffer is to hold: the pre-pass runs on what it has, for the last time.
StartPrePass();
return false;
}
{
std::unique_lock ul(prepass_mutex);
if (!prepass_over) {
prepass_images[image->message.number] = image;
std::unique_lock ul_live(live_mutex);
live.images_held = static_cast<int64_t>(prepass_images.size());
}
}
prepass_cv.notify_all();
if (status)
Publish();
return true;
}
void StillsGPUAnalysisFPGA::StartPrePass() {
{
std::unique_lock ul(prepass_mutex);
prepass_last = true;
}
prepass_cv.notify_all();
}
void StillsGPUAnalysisFPGA::Finish() {
finished = true;
StartPrePass();
prepass_future.get();
for (size_t i = 0; i < analysis_futures.size(); i++)
queue.PutBlocking(nullptr);
for (auto &f: analysis_futures)
f.get();
if (status) {
last_image = status->LastImage();
Publish();
}
if (merge_future.valid()) {
if (status)
status->Step("Merge");
merge_queue.Put(-1);
merge_future.get();
if (merged && status) {
PublishMerge(*merged, "the collection");
status->Headline(merge_experiment.GetSpaceGroupOrP1().xhm(), merged->merge.statistics.overall.d_min,
std::nullopt);
}
}
if (status) {
if (setup_error)
status->Finish(RugnuxLiveState::Failed, "GPU analysis setup failed");
else {
// Reported by Written, once everything is written - the end message and the writer go on meanwhile.
status->Step("Writing");
std::unique_lock ul(error_mutex);
if (analysis_error.empty())
outcome.emplace(RugnuxLiveState::Done, "");
else
outcome.emplace(RugnuxLiveState::Failed, analysis_error);
}
}
if (setup_error)
std::rethrow_exception(setup_error);
}
void StillsGPUAnalysisFPGA::Written() {
if (!status || !outcome)
return;
if (last_image)
status->Latency(std::chrono::duration<double>(std::chrono::steady_clock::now() - last_image.value()).count());
status->Finish(outcome->first, outcome->second);
outcome.reset();
}
void StillsGPUAnalysisFPGA::PrePassThread() {
const auto masked = [](const PixelMask &m) {
return std::count_if(m.GetMask().begin(), m.GetMask().end(), [](uint32_t v) { return v != 0; });
};
setup.experiment = experiment;
setup.mask = mask;
const int64_t all_images = experiment.GetImageNum();
std::optional<StillsPrePassResult> result;
std::chrono::steady_clock::time_point prepass_start = std::chrono::steady_clock::now();
// At a low hit rate the first frames may not be enough to fit the geometry on, or may not hold a crystal: with
// a cell to fit, the pre-pass runs again on all that is held once that has doubled, until the fit has its
// frames, or the collection ended, or the sample is as large as it may grow (the last try): bounded so that
// the last run is short and the other half of the queue takes the frames arriving meanwhile.
const int64_t max_sample = std::min(PREPASS_MAX_IMAGES, max_held / 2);
for (int64_t needed = std::min(PREPASS_IMAGES, all_images); ;) {
std::map<int64_t, std::shared_ptr<HeldImage>> sample;
bool last;
{
std::unique_lock ul(prepass_mutex);
prepass_cv.wait(ul, [&] {
return prepass_last || static_cast<int64_t>(prepass_images.size()) >= needed;
});
// Frames that arrived during the previous run do not make this one larger than the bound.
for (const auto &[n, image]: prepass_images) {
if (static_cast<int64_t>(sample.size()) >= std::max(needed, max_sample))
break;
sample.emplace(n, image);
}
last = prepass_last || static_cast<int64_t>(sample.size()) >= std::min(all_images, max_sample);
}
if (sample.empty())
break;
{
std::unique_lock ul(live_mutex);
live.prepass = RugnuxLivePrePass::Running;
}
if (status)
Publish();
try {
const auto t0 = std::chrono::steady_clock::now();
auto r = RunStillsPrePass(start, sample, static_cast<int>(std::thread::hardware_concurrency()));
logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, "
"distance {:.2f} -> {:.2f} mm on {} frames, masked pixels {} -> {}", r.images,
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), r.beam_x_pxl, r.beam_y_pxl,
experiment.GetDetectorDistance_mm(), r.distance_mm, r.geometry_frames,
masked(mask), masked(r.mask));
result = std::move(r);
} catch (const std::exception &e) {
logger.Warning("Pre-pass on {} images failed - {}", sample.size(), e.what());
}
if (last || !experiment.GetUnitCell()
|| (result && result->geometry_frames >= RUGNUX_REFINE_GEOMETRY_FRAMES))
break;
needed = std::min({2 * static_cast<int64_t>(sample.size()), all_images, max_sample});
std::unique_lock ul(live_mutex);
live.prepass = RugnuxLivePrePass::Holding;
live.images_needed = needed;
}
{
// The frames are analysed from here on: their slots go back to the buffer once they are sent.
std::unique_lock ul(prepass_mutex);
prepass_over = true;
prepass_images.clear();
}
const bool failed = !result;
int geometry_frames = 0;
if (result) {
const auto &r = *result;
setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl).DetectorDistance_mm(r.distance_mm);
setup.mask = r.mask;
geometry_frames = r.geometry_frames;
const auto &m = r.mask.GetMask();
std::unique_lock ul(live_mutex);
live.beam_x_pxl = r.beam_x_pxl;
live.beam_y_pxl = r.beam_y_pxl;
live.distance_mm = r.distance_mm;
live.shadow_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) {
return (v & (1u << PixelMask::BeamStopPixelBit)) != 0;
});
live.bad_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) {
return (v & (1u << PixelMask::HotPixelBit)) != 0;
});
} else
logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask");
try {
setup.mapping = std::make_unique<AzimuthalIntegrationMapping>(setup.experiment, setup.mask);
// Retained for the merge, which takes each image's reflections as soon as it is analysed.
indexer = std::make_unique<IndexAndRefine>(setup.experiment, indexing_thread_pool, merge_frames, true);
if (geometry_frames == 0) {
const std::string text = fmt::format(
"The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the "
"whole run is analysed at the distance in the header, {:.2f} mm, beam centre ({:.2f}, {:.2f}) px",
setup.experiment.GetDetectorDistance_mm(), setup.experiment.GetBeamX_pxl(),
setup.experiment.GetBeamY_pxl());
logger.Warning("{}", text);
if (status)
status->Note(text);
}
if (merge_frames) {
merge_experiment = setup.experiment;
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
setup.experiment.GetUnitCell(), std::nullopt,
setup.experiment.IsDetectIceRings(),
SpotFindingSettings().ice_ring_width_Q_recipA, logger);
merge_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::MergeThread, this);
}
ready(setup);
} catch (const std::exception &e) {
// Nothing can be written without the start message: the frames are dropped (AnalysisThread), and
// the collection is stopped.
const auto *j = dynamic_cast<const JFJochException *>(&e);
const JFJochException error = j ? *j : JFJochException(JFJochExceptionCategory::WrongDAQState,
std::string("GPU analysis setup failed: ") + e.what());
logger.ErrorException(error);
setup_error = j ? std::current_exception() : std::make_exception_ptr(error);
if (setup_failed)
setup_failed(error);
prepass_done.count_down();
return;
}
{
std::unique_lock ul(live_mutex);
live.prepass = failed ? RugnuxLivePrePass::Failed : RugnuxLivePrePass::Done;
live.prepass_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - prepass_start).count();
ready_time = std::chrono::steady_clock::now();
}
if (status) {
status->Step("GPU analysis");
Publish();
}
prepass_done.count_down();
}
void StillsGPUAnalysisFPGA::AnalysisThread() {
try {
pin_gpu();
} catch (const std::exception &e) {
logger.Warning("Error pinning GPU {}", e.what());
}
prepass_done.wait();
if (setup_error) {
while (queue.GetBlocking()) {}
return;
}
// A thread whose analysis cannot be built still passes its frames on, unanalysed.
std::unique_ptr<MXAnalysisWithoutFPGA> analysis;
try {
analysis = std::make_unique<MXAnalysisWithoutFPGA>(setup.experiment, *setup.mapping, setup.mask,
*indexer, /*enable_fused_adaptive_gpu=*/true);
} catch (const std::exception &e) {
AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ")
+ e.what());
}
while (auto held = queue.GetBlocking()) {
DataMessage &message = held->message;
std::optional<AzimuthalIntegrationProfile> profile;
const auto start_time = std::chrono::steady_clock::now();
profile.emplace(*setup.mapping);
try {
if (analysis)
analysis->Analyze(message, *profile, settings());
} catch (const JFJochException &e) {
// The frame is still written, without its analysis.
logger.ErrorException(e);
} catch (const std::exception &e) {
AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what()));
}
message.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now()
- start_time).count();
try {
sink(message, *profile);
if (merge)
merge_queue.Put(message.number);
if (status) {
int64_t analysed;
{
std::unique_lock ul(live_mutex);
analysed = ++live.images_analysed;
if (message.indexing_result.value_or(false))
++live.images_indexed;
}
status->Progress(analysed, experiment.GetImageNum());
Publish();
}
} catch (const std::exception &e) {
// The thread goes on with the next frame, so that the queue is drained.
AnalysisError(fmt::format("image {} lost - {}", message.number, e.what()));
}
}
}
std::optional<UnitCell> StillsGPUAnalysisFPGA::GetConsensusUnitCell() const {
return indexer->GetConsensusUnitCell();
}
float StillsGPUAnalysisFPGA::GetBeamX_pxl() const {
return setup.experiment.GetBeamX_pxl();
}
float StillsGPUAnalysisFPGA::GetBeamY_pxl() const {
return setup.experiment.GetBeamY_pxl();
}
float StillsGPUAnalysisFPGA::GetDetectorDistance_mm() const {
return setup.experiment.GetDetectorDistance_mm();
}
const std::optional<StillsMergeOutput> &StillsGPUAnalysisFPGA::GetMerged() const {
return merged;
}
const DiffractionExperiment &StillsGPUAnalysisFPGA::GetMergeExperiment() const {
return merge_experiment;
}
void StillsGPUAnalysisFPGA::MergeThread() {
constexpr int64_t PREPARE_EVERY = 200; // frames between two Prepare calls after a checkpoint
const int64_t images = experiment.GetImageNum();
// The cell merged in, as rugnux merges it: the consensus of the frames indexed by the first merge (the
// collection's cell only picks the frames that are merged).
bool cell_set = false;
const auto have_cell = [&] {
if (!cell_set)
if (const auto c = indexer->GetConsensusUnitCell()) {
merge_experiment.SetUnitCell(*c);
merge->Cell(*c);
cell_set = true;
}
return cell_set;
};
size_t next = 0;
try {
const auto add = [&](int64_t n, IntegrationOutcome &&o) {
merge->Add(n, std::move(o));
if (next < CHECKPOINTS.size()
&& merge->Added() >= std::llround(CHECKPOINTS[next] * static_cast<double>(images))) {
if (have_cell())
PublishMerge(merge->Checkpoint(MERGE_THREADS),
fmt::format("{:.0f}% of the images (provisional)", 100.0 * CHECKPOINTS[next]));
++next;
} else if (next > 0 && merge->Added() % PREPARE_EVERY == 0)
merge->Prepare(MERGE_THREADS);
};
for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking())
add(n, indexer->TakeIntegrationOutcome(n));
if (have_cell()) {
merged = merge->Final(static_cast<int>(AvailableCpus()));
logger.Info("Stills merge after the last image: {:.2f} s", merged->seconds);
}
} catch (const std::exception &e) {
logger.Warning("Stills merge failed: {}", e.what());
merged.reset();
}
}
void StillsGPUAnalysisFPGA::PublishMerge(const StillsMergeOutput &out, const std::string &title) {
const auto &o = out.merge.statistics.overall;
const std::string text = fmt::format("Merge of {}: {} crystals, {:.2f} A, CC1/2 {:.3f}, R_split {:.3f}, "
"completeness {:.1f}%, <I/sigma> {:.1f} ({:.1f} s)", title, out.images,
o.d_min, o.cc_half, o.r_split,
o.possible_unique_reflections > 0
? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0,
o.mean_i_over_sigma, out.seconds);
logger.Info("{}", text);
if (status)
status->Note(text);
}
-151
View File
@@ -1,151 +0,0 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <array>
#include <condition_variable>
#include <exception>
#include <functional>
#include <future>
#include <latch>
#include <map>
#include <memory>
#include <mutex>
#include <vector>
#include "StillsPrePass.h"
#include "RugnuxLiveStatus.h"
#include "../common/DiffractionExperiment.h"
#include "../common/JFJochException.h"
#include "../common/AzimuthalIntegrationMapping.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include "../common/Logger.h"
#include "../common/ThreadSafeFIFO.h"
#include "../image_analysis/IndexAndRefine.h"
#include "../rugnux/StillsLiveMerge.h"
// JFJochReceiverFPGA's live stills, kept as they are at rc.185 while the stream receivers take StillsGPUAnalysis.
// Serial frames analysed on the GPU as the DECTRIS ones are (MXAnalysisWithoutFPGA), after a pre-pass on
// the first PREPASS_IMAGES (RunStillsPrePass) has decided the beam centre and the mask they are analysed
// with. Every frame given before the pre-pass is done is held; `ready` is called once it is, with the
// geometry and mask the frames are then analysed at, before any of them is analysed; `sink` gets each
// frame once analysed, from one of the analysis threads. Where `status` is given, the run is reported there
// (pre-pass, its decisions, the analysis).
// At most `max_held` frames wait for analysis: Hold refuses one more, so the slots held stay bounded.
// Where the setup after the pre-pass fails (`ready` included), `failed` is called at once, the frames are
// dropped unanalysed and Finish throws the error. An analysis thread that fails passes its frames on
// unanalysed, and the run is reported as failed.
// The indexed frames are merged as `rugnux --mode scale` merges them (StillsLiveMerge), beside the analysis:
// a provisional merge at each of CHECKPOINTS of the images, on MERGE_THREADS threads, reported as a step
// of the status, and the merge of the whole collection once the last frame is analysed, on every thread,
// before Finish returns (GetMerged). The cell is the run's consensus at the first merge, as rugnux merges in
// it (the collection's cell picks the frames merged); the space group the collection's, else P1.
// merge = false leaves it out.
// With a cell given, the pre-pass runs again on all the frames held each time they have doubled, until the geometry
// fit has its frames, the collection ends or the sample reaches PREPASS_MAX_IMAGES or half of `max_held`; the
// analysis starts after that and catches up from the frames held. The bound keeps the last run short and leaves
// room in the queue for the frames arriving meanwhile, which are otherwise not written. The geometry is fixed when the pre-pass ends: every frame is analysed and merged at it. Where the pre-pass could
// not refine it (too few of the frames it had indexed: a small image buffer, a serial hit rate, a chip whose first
// frames are empty; or no cell given), the header's distance stays for the whole run, with a warning in the log
// and the status.
class StillsGPUAnalysisFPGA {
public:
// Enough frames for the distance to be refined at a serial hit rate: of 100 frames of a collection indexed
// at 18%, 7 indexed at a distance 2.6% off and the geometry refinement did not run; of 500, 34 did and
// refined it to 0.4%. They wait in the image buffer, which has to hold them in half its slots.
constexpr static int64_t PREPASS_IMAGES = 500;
constexpr static int64_t PREPASS_MAX_IMAGES = 4 * PREPASS_IMAGES;
constexpr static std::array<double, 2> CHECKPOINTS = {0.5, 0.75};
constexpr static int MERGE_THREADS = 8;
struct Setup {
DiffractionExperiment experiment;
PixelMask mask;
std::unique_ptr<AzimuthalIntegrationMapping> mapping;
};
using Ready = std::function<void(const Setup &)>;
using Sink = std::function<void(DataMessage &, const AzimuthalIntegrationProfile &)>;
using Settings = std::function<SpotFindingSettings()>;
using Failed = std::function<void(const JFJochException &)>;
// `start`: the collection's start message, as it would be sent without the pre-pass.
StillsGPUAnalysisFPGA(const DiffractionExperiment &experiment, const PixelMask &mask, const StartMessage &start,
IndexerThreadPool *indexing_thread_pool, int64_t nthreads, int64_t max_held, Logger &logger,
Ready ready, Sink sink, Settings settings, Failed failed = {},
RugnuxLiveStatus *status = nullptr, bool merge = true);
~StillsGPUAnalysisFPGA();
// False where `max_held` frames already wait: the frame is not taken.
bool Hold(const std::shared_ptr<HeldImage> &image);
// No frame comes after this: the pre-pass runs on what came, if it has not, and every frame is
// analysed before it returns. Throws where the setup after the pre-pass failed.
void Finish();
// After Finish, once the end message is out and the writer is done: the status gets the latency from the last
// image and then the run's outcome. Without it the outcome is reported when the analysis goes, with no latency.
void Written();
// After Finish.
[[nodiscard]] std::optional<UnitCell> GetConsensusUnitCell() const;
[[nodiscard]] float GetBeamX_pxl() const;
[[nodiscard]] float GetBeamY_pxl() const;
[[nodiscard]] float GetDetectorDistance_mm() const;
// The merge of the collection, where one was made.
[[nodiscard]] const std::optional<StillsMergeOutput> &GetMerged() const;
[[nodiscard]] const DiffractionExperiment &GetMergeExperiment() const;
private:
const DiffractionExperiment &experiment;
const PixelMask &mask;
const StartMessage start;
IndexerThreadPool *indexing_thread_pool;
Logger &logger;
Ready ready;
Sink sink;
Settings settings;
Failed setup_failed;
RugnuxLiveStatus *status;
const bool merge_frames;
std::mutex live_mutex;
RugnuxLiveStills live;
std::chrono::steady_clock::time_point ready_time;
void Publish();
// Kept by Finish for Written
std::optional<std::chrono::steady_clock::time_point> last_image;
std::optional<std::pair<RugnuxLiveState, std::string>> outcome;
Setup setup;
std::unique_ptr<IndexAndRefine> indexer;
std::mutex prepass_mutex;
std::condition_variable prepass_cv;
bool prepass_last = false; // what is held is all there will be
bool prepass_over = false;
std::map<int64_t, std::shared_ptr<HeldImage>> prepass_images;
std::latch prepass_done{1};
std::exception_ptr setup_error; // set before prepass_done
std::mutex error_mutex;
std::string analysis_error; // the first, where an analysis thread failed
void AnalysisError(const std::string &text);
const int64_t max_held;
ThreadSafeFIFO<std::shared_ptr<HeldImage>> queue;
std::future<void> prepass_future;
std::vector<std::future<void>> analysis_futures;
bool finished = false;
// The frames analysed, by number, for the merge (-1 after the last); it takes their outcomes from the indexer.
ThreadSafeFIFO<int64_t> merge_queue;
std::unique_ptr<StillsLiveMerge> merge; // after the indexer: its reflections are in the indexer's arena
DiffractionExperiment merge_experiment;
std::optional<StillsMergeOutput> merged;
std::future<void> merge_future;
void StartPrePass();
void PrePassThread();
void AnalysisThread();
void MergeThread();
void PublishMerge(const StillsMergeOutput &out, const std::string &title);
};
+4 -2
View File
@@ -44,10 +44,10 @@ namespace {
}
StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<int64_t, std::shared_ptr<HeldImage>> &images,
int nthreads) {
int nthreads, std::optional<int64_t> max_spot_count) {
HeldImageReader reader(start, images);
const auto dataset = reader.GetDataset();
const RugnuxLiveSetup setup = MakeRugnuxStillsPrePassSetup(*dataset, nthreads);
const RugnuxLiveSetup setup = MakeRugnuxStillsPrePassSetup(*dataset, nthreads, max_spot_count);
Rugnux rugnux(reader, setup.experiment, *dataset->pixel_mask, setup.config);
const ProcessResult result = rugnux.Run();
@@ -61,6 +61,8 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<i
.mask = rugnux.GetPixelMask(),
.images = images.size(),
.geometry_frames = result.stills_geometry_frames,
.cell = rugnux.GetExperiment().GetUnitCell(),
.max_spot_count = rugnux.GetExperiment().GetMaxSpotCount(),
.polarization = rugnux.GetExperiment().GetPolarizationFactor(),
.integration = rugnux.GetExperiment().GetBraggIntegrationSettings(),
.run_width = rugnux.GetStillsRunWidth(),
+15 -3
View File
@@ -3,6 +3,8 @@
#pragma once
#include <atomic>
#include <functional>
#include <map>
#include <memory>
#include <vector>
@@ -14,11 +16,14 @@
#include "../common/ZeroCopyReturnValue.h"
#include "../image_analysis/IndexAndRefine.h" // StillsWindow
// An image held until analysed, pointing into `slot`, which goes back unsent with the last copy. The slot is
// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds.
// An image held until analysed, pointing into `slot`, which goes back unsent with the last copy (the slot is
// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds), or into bytes
// `release` gives back with the last copy (HoldArena).
struct HeldImage {
DataMessage message;
ZeroCopyReturnValue *slot = nullptr;
std::function<void()> release;
bool analyse = true; // false: written without its analysis (stills_downsample)
HeldImage() = default;
HeldImage(const HeldImage &) = delete;
@@ -26,6 +31,8 @@ struct HeldImage {
~HeldImage() {
if (slot)
slot->discard();
if (release)
release();
}
};
@@ -36,6 +43,10 @@ struct StillsPrePassResult {
PixelMask mask;
size_t images = 0;
int geometry_frames = 0; // the indexed frames the distance and centre were refined on; 0: kept as stated
// The cell the pre-pass indexed its frames on: the one stated, or found de novo, as refined beside the distance
// (its scale trades with the distance, so the two go together)
std::optional<UnitCell> cell;
int64_t max_spot_count = 0; // the spots kept per frame, the strongest
// What the pre-pass integrated its frames with, for the frames after it: the polarization, the integration
// radii and background ring as the pre-scan measured them, and the run's rocking width with the prediction
// window made from it (empty where the pre-pass could not measure it).
@@ -47,5 +58,6 @@ struct StillsPrePassResult {
// The first frames of a serial collection through rugnux: beam centre, distance (with a cell), mask and
// integration settings it decides
// max_spot_count: as given for the collection (DiffractionExperiment::GetMaxSpotCountGiven)
StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<int64_t, std::shared_ptr<HeldImage>> &images,
int nthreads);
int nthreads, std::optional<int64_t> max_spot_count = {});
+22
View File
@@ -0,0 +1,22 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <cstdint>
// Rugnux live stills with a downsizing factor F analyse on average 1 of every F images, each image chosen on its
// own with probability 1/F from a hash of its number. A stride would be in step with any period of the stream
// (pump-probe on/off, storage cells, module or card interleave) and could take e.g. only the dark frames; the hash
// is not, and the choice is reproducible and the same on every thread and card.
// The first `all_before` images are all taken: the pre-pass may use any of them.
inline bool StillsImageSelected(int64_t image_number, int64_t factor, int64_t all_before) {
if ((factor <= 1) || (image_number < all_before))
return true;
// The SplitMix64 finaliser: Steele, Lea & Flood (2014) Proc. OOPSLA 2014, 453-472; salted
uint64_t z = static_cast<uint64_t>(image_number) + 0x5d0f3c1a7b2e9d46ULL + 0x9e3779b97f4a7c15ULL;
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL;
z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL;
z = z ^ (z >> 31);
return z % static_cast<uint64_t>(factor) == 0;
}
+6
View File
@@ -83,6 +83,12 @@ enum class ProcessMode {
// Where the per-image analysis of an output pass goes as it is made: rugnux and the viewer put it in
// <prefix>_process.h5 (ProcessFileSink, RugnuxOutputFiles.h). Called from one thread at a time.
// The image workers a card takes: as many as keep half of its memory (about 24 bytes a detector pixel each), at most
// `most` (Rugnux::GPUWorkersPerCard); and those of a per-image loop: 16 a card, at most nthreads, nthreads without a
// GPU (Rugnux::ImageWorkers). Also the live stills analysis's.
int GPUWorkersPerCard(int64_t pixels, int most);
int ImageWorkers(int64_t pixels, int nthreads);
class RugnuxImageSink {
public:
virtual ~RugnuxImageSink() = default;
+15 -6
View File
@@ -88,13 +88,16 @@ using namespace rugnux_internal;
// detector, measured at 12-22 bytes a pixel on four data sets - and the workers keep to half of the card,
// so a small card gets fewer of them. How many there are changes only how fast a loop runs, never what it
// computes.
int Rugnux::GPUWorkersPerCard(int most) const {
int GPUWorkersPerCard(int64_t pixels, int most) {
constexpr int64_t WORKER_BYTES_PER_PIXEL = 24;
const auto fit = static_cast<int64_t>(get_gpu_total_memory(0) / 2)
/ (WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum());
const auto fit = static_cast<int64_t>(get_gpu_total_memory(0) / 2) / (WORKER_BYTES_PER_PIXEL * pixels);
return static_cast<int>(std::clamp<int64_t>(fit, 1, most));
}
int Rugnux::GPUWorkersPerCard(int most) const {
return ::GPUWorkersPerCard(experiment_.GetPixelsNum(), most);
}
// Every worker submits its own kernels to a card, and a card runs out of room to accept them long before it
// runs out of work: measured on two GPUs, the image loop's own time falls to sixteen workers and then rises
// again, so forty-eight is slower than sixteen on a 16 Mpx set. Sixteen is where that measurement put the
@@ -103,11 +106,17 @@ int Rugnux::GPUWorkersPerCard(int most) const {
// was left at its default - an explicit -N is a deliberate instruction and is obeyed, which is what a
// previous attempt at this got wrong. A worker also holds GPU memory, so a small card gets fewer of them
// (GPUWorkersPerCard).
int Rugnux::ImageWorkers() const {
int ImageWorkers(int64_t pixels, int nthreads) {
const int gpus = get_gpu_count();
if (!config_.nthreads_auto || gpus == 0)
if (gpus == 0)
return nthreads;
return std::min(nthreads, GPUWorkersPerCard(pixels, 16) * gpus);
}
int Rugnux::ImageWorkers() const {
if (!config_.nthreads_auto)
return config_.nthreads;
return std::min(config_.nthreads, GPUWorkersPerCard(16) * gpus);
return ::ImageWorkers(experiment_.GetPixelsNum(), config_.nthreads);
}
void Rugnux::ImagePass(PipelineLocals &p) {
+5 -3
View File
@@ -66,7 +66,8 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr
return s;
}
RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads) {
RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads,
std::optional<int64_t> max_spot_count) {
RugnuxLiveSetup s;
// The experiment as rugnux_cli.cpp sets it up for serial data with no options given.
@@ -74,7 +75,7 @@ RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset,
x = dataset.experiment;
ApplyRugnuxExperimentDefaults(x);
x.NumTriggers(1);
x.MaxSpotCount(RUGNUX_MAX_SPOT_COUNT);
x.MaxSpotCount(max_spot_count.value_or(RUGNUX_MAX_SPOT_COUNT));
IndexingSettings indexing;
indexing.Algorithm(IndexingAlgorithmEnum::Auto);
@@ -97,7 +98,8 @@ RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset,
c.detect_beam_stop = RUGNUX_DETECT_BEAM_STOP_FRAMES;
c.beam_center_check = true;
c.adaptive_integration_radius = true;
c.measure_spot_budget = true;
// A count given is kept, as `rugnux --max-spots` keeps it
c.measure_spot_budget = !max_spot_count.has_value();
// As `rugnux -C`: the geometry bundle adjustment needs the cell.
if (x.GetUnitCell().has_value())
c.refine_geometry = RUGNUX_REFINE_GEOMETRY_FRAMES;
+3 -1
View File
@@ -23,7 +23,9 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr
// The first frames of a serial collection, processed inside the broker before the rest are analysed:
// what a bare `rugnux --no-merge` (with -C where the collection states a cell) on those frames sets.
// The run's beam centre and mask are what the broker then analyses the whole collection with.
RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads);
// max_spot_count: as the user gave it for the collection; unset, rugnux's default, the budget measured as rugnux does
RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset, int nthreads,
std::optional<int64_t> max_spot_count = {});
// Builds on every card, at the same time, the GPU FFT indexers a live run of this setup is known to
// need - its first pass's (the settings Rugnux gives it) - and the cuFFT
+4
View File
@@ -3485,6 +3485,10 @@ int RugnuxInputRun::PrepareRugnuxInput() {
// configure_offline_output applies the rugnux analysis defaults, one of which is the polarization
// factor, so an explicit --polarization has to land after it or it is silently overwritten.
if (polarization_factor) experiment.PolarizationFactor(polarization_factor.value());
// A space group stored in the data file is not the user's: only -S fixes one
if (space_group == nullptr && experiment.GetGemmiSpaceGroup())
logger.Info("Space group {} stored in the input file is ignored; -S fixes the space group",
experiment.GetGemmiSpaceGroup()->hm);
ClearStoredCrystal(experiment); // shared with the viewer; -S / -C below override it
if (space_group != nullptr)
experiment.SetSpaceGroup(*space_group);
+15
View File
@@ -283,6 +283,21 @@ TEST_CASE("CBORSerialize_Start_ThresholdEnergy", "[CBOR]") {
REQUIRE(output_message.threshold_energy == message.threshold_energy);
}
// The values the start message gives, and those of older files, pass as they are
TEST_CASE("CBORSerialize_Start_SummationMode", "[CBOR]") {
for (const std::string mode: {"none", "fpga", "cpu", "hybrid"}) {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
StartMessage message{};
message.summation_mode = mode;
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize());
REQUIRE(deserialized);
REQUIRE(deserialized->start_message);
CHECK(deserialized->start_message->summation_mode == mode);
}
}
TEST_CASE("CBORSerialize_Start_Fluorescence", "[CBOR]") {
std::vector<uint8_t> buffer(8 * 1024 * 1024);
CBORStream2Serializer serializer(buffer.data(), buffer.size());
+4 -1
View File
@@ -132,7 +132,6 @@ ADD_EXECUTABLE(jfjoch_test
RugnuxLiveTriageTest.cpp
RugnuxScaleInputsTest.cpp
StillsGPUAnalysisTest.cpp
StillsGPUAnalysisFPGATest.cpp
RugnuxStructureTest.cpp
SweepLayoutTest.cpp
BrukerReaderTest.cpp
@@ -155,6 +154,10 @@ ADD_EXECUTABLE(jfjoch_test
ImageMetadataTest.cpp
JFJochReceiverLiteTest.cpp
JFJochReceiverRugnuxTest.cpp
FPGAImagePullerTest.cpp
FPGAImagePullerSummationTest.cpp
StillsSelectionTest.cpp
HoldArenaTest.cpp
GridScanSettingsTest.cpp
JFJochReceiverPlotsTest.cpp
GoniometerAxisTest.cpp
+61 -1
View File
@@ -783,10 +783,60 @@ TEST_CASE("DiffractionExperiment_Summation","[DiffractionExperiment]") {
REQUIRE_NOTHROW(x.Summation(500));
CHECK(x.GetByteDepthImage() == 4);
CHECK(x.GetSummation() == 500);
CHECK(x.GetFPGASummation() == 1);
CHECK(x.GetFPGASummation() == 250);
CHECK(x.GetHostSummation() == 2);
CHECK(x.IsCPUSummation());
}
// The card sums the largest divisor up to 256, the host the rest
TEST_CASE("DiffractionExperiment_Summation_CardAndHost","[DiffractionExperiment]") {
DiffractionExperiment x;
for (const auto &[n, card]: std::vector<std::pair<int64_t, int64_t>>{
{1, 1}, {7, 7}, {255, 255}, {256, 256}, {257, 1}, {512, 256}, {1000, 250}, {1024, 256}, {300, 150},
{514, 2}, {1009, 1}}) {
INFO("summation " << n);
x.Summation(n);
CHECK(x.GetSummation() == n);
CHECK(x.GetFPGASummation() == card);
CHECK(x.GetHostSummation() == n / card);
CHECK(x.IsCPUSummation() == (card != n));
}
// 32-bit from the card where it sums as well; single frames at 16 bits
x.Summation(256);
CHECK(x.GetByteDepthFPGA() == 4);
x.Summation(1000);
CHECK(x.GetByteDepthFPGA() == 4);
CHECK(x.GetByteDepthImage() == 4);
x.Summation(257);
CHECK(x.GetByteDepthFPGA() == 2);
CHECK(x.GetByteDepthImage() == 4);
// All of it on the host
x.Summation(7).CPUSummation(true);
CHECK(x.GetFPGASummation() == 1);
CHECK(x.GetHostSummation() == 7);
CHECK(x.IsCPUSummation());
x.Summation(1);
CHECK(!x.IsCPUSummation());
}
// Who sums: the start message's summation_mode
TEST_CASE("DiffractionExperiment_SummationMode","[DiffractionExperiment]") {
DiffractionExperiment x;
StartMessage message;
for (const auto &[n, mode]: std::vector<std::pair<int64_t, std::string>>{
{1, "none"}, {7, "fpga"}, {256, "fpga"}, {1000, "hybrid"}, {514, "hybrid"}, {257, "cpu"}}) {
INFO("summation " << n);
x.Summation(n);
x.FillMessage(message);
CHECK(message.summation_mode == mode);
}
x.Summation(1000).CPUSummation(true);
x.FillMessage(message);
CHECK(message.summation_mode == "cpu");
}
TEST_CASE("DiffractionExperiment_DetectorModuleHostname","[DiffractionExperiment]") {
std::vector<std::string> h = {"mx1", "mx2", "mx3"};
DiffractionExperiment x(DetectorSetup(DetectorGeometryModular(3),
@@ -1334,3 +1384,13 @@ TEST_CASE("DiffractionExperiment_AzIntegration_QBinning_OpenAPI", "[DiffractionE
Catch::Matchers::ContainsSubstring("only to linear"));
}
}
TEST_CASE("DiffractionExperiment_MaxSpotCountGiven","[DiffractionExperiment]") {
DiffractionExperiment x;
// Not given: each analysis takes its own default, the per-image indexing DEFAULT_MAX_SPOT_COUNT
CHECK(!x.GetMaxSpotCountGiven().has_value());
CHECK(x.GetMaxSpotCount() == DEFAULT_MAX_SPOT_COUNT);
x.MaxSpotCount(700);
CHECK(x.GetMaxSpotCountGiven() == 700);
CHECK(x.GetMaxSpotCount() == 700);
}
+181
View File
@@ -0,0 +1,181 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// Summation split between the card and the host, on the mock of the card: the card sums the largest divisor of the
// summation up to 256, the host the card's images of one image. The pixels of both receivers are checked against
// a sum done here.
#include <catch2/catch_all.hpp>
#include <map>
#include <mutex>
#include "../frame_serialize/CBORStream2Deserializer.h"
#include "../image_pusher/ImagePusher.h"
#include "../receiver/FPGAImagePuller.h"
#include "../receiver/FrameTransformation.h"
#include "../receiver/JFJochReceiverService.h"
#include "MockCardOutput.h"
namespace {
struct SumReceived {
DataMessage message;
std::vector<uint8_t> pixels;
};
class SumPusher : public ImagePusher {
public:
std::mutex m;
std::map<int64_t, SumReceived> images;
void StartDataCollection(StartMessage &) override {}
bool EndDataCollection(const EndMessage &) override { return true; }
bool SendCalibration(const CompressedImage &) override { return true; }
bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override {
auto d = CBORStream2Deserialize(image_data, image_size);
SumReceived r;
r.message = d->data_message.value();
r.message.image.GetUncompressed(r.pixels);
r.message.image = CompressedImage();
std::unique_lock ul(m);
images.emplace(image_number, std::move(r));
return true;
}
std::string PrintSetup() const override { return "summation"; }
ImagePusherType GetType() const override { return ImagePusherType::Test; }
};
DiffractionExperiment SumExperiment(int64_t summation, int64_t images) {
DiffractionExperiment x(DetJF(2));
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).InternalPacketGeneratorImages(1)
.ImagesPerTrigger(images).ImagesPerFile(10).IncidentEnergy_keV(12.4)
.DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600);
x.Conversion();
x.Summation(summation).Compression(CompressionAlgorithm::BSHUF_LZ4);
return x;
}
constexpr size_t ErrPixel = 100, SatPixel = 200, LargePixel = 300;
constexpr int64_t CardImageCount = 5;
// What the card writes for its images, in its pixel format: a pattern, one pixel in error in the second image, one
// saturated in the third, and one large enough that three of them saturate in 32 bits
std::vector<CardImage> CardSums(const DiffractionExperiment &x) {
const bool wide = x.GetByteDepthFPGA() == 4;
std::vector<CardImage> ret;
std::vector<int64_t> values(RAW_MODULE_SIZE);
for (int64_t f = 0; f < CardImageCount; f++) {
CardImage image;
for (int m = 0; m < x.GetModulesNum(); m++) {
for (size_t i = 0; i < RAW_MODULE_SIZE; i++)
values[i] = (f * 7 + m * 3 + static_cast<int64_t>(i)) % 1000;
values[ErrPixel] = (f == 1) ? (wide ? INT32_MIN : INT16_MIN) : 5;
values[SatPixel] = (f == 2) ? (wide ? INT32_MAX : INT16_MAX) : 5;
values[LargePixel] = wide ? INT32_MAX / 3 + 1 : 30000;
auto output = std::make_unique<DeviceOutput>();
StorePixels(*output, x, values);
output->module_statistics.pixel_sum = 1000 * f + m;
image.push_back(std::move(output));
}
ret.push_back(std::move(image));
}
return ret;
}
template<class Tsrc>
int32_t Pixel(const DeviceOutput &output, size_t i) {
return reinterpret_cast<const Tsrc *>(output.pixels)[i];
}
// The image summed here from the card's images, by ModuleSummation's rule: an error in any is an error, then a
// saturated one or a sum reaching INT32_MAX is saturated. Assembled by FrameTransformation, as the receivers do.
std::vector<uint8_t> Reference(const DiffractionExperiment &x, const std::vector<CardImage> &cards, int64_t image) {
const bool wide = x.GetByteDepthFPGA() == 4;
const int64_t err_src = wide ? INT32_MIN : INT16_MIN;
const int64_t sat_src = wide ? INT32_MAX : INT16_MAX;
const int64_t h = x.GetHostSummation();
FrameTransformation transformation(x);
for (int m = 0; m < x.GetModulesNum(); m++) {
auto output = std::make_unique<DeviceOutput>();
memset(output.get(), 0, sizeof(DeviceOutput));
output->module_statistics.module_number = m;
auto sum = reinterpret_cast<int32_t *>(output->pixels);
for (size_t i = 0; i < RAW_MODULE_SIZE; i++) {
int64_t acc = 0;
for (int64_t k = 0; k < h; k++) {
const auto &card = *cards[(image * h + k) % CardImageCount][m];
const int64_t v = wide ? Pixel<int32_t>(card, i) : Pixel<int16_t>(card, i);
if (v == err_src || acc == INT32_MIN)
acc = INT32_MIN;
else if (v == sat_src || acc == INT32_MAX || acc + v >= INT32_MAX)
acc = INT32_MAX;
else
acc += v;
}
sum[i] = static_cast<int32_t>(acc);
}
transformation.ProcessModule(output.get(), 0);
}
auto p = static_cast<const uint8_t *>(transformation.GetImage());
return {p, p + x.GetPixelsNum() * x.GetByteDepthImage()};
}
std::map<int64_t, SumReceived> Run(const DiffractionExperiment &x, bool puller) {
auto devices = MockDevices(x, CardSums(x));
Logger logger("FPGAImagePullerSummation");
SumPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
SpotFindingSettings spots = DiffractionExperiment::DefaultDataProcessingSettings();
spots.indexing = false;
service.NumThreads(4).SetSpotFindingSettings(spots);
service.FPGAImagePullerSettings(puller);
PixelMask mask(x);
const JFCalibration calib = GeneratePedestalCalibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
CHECK(out.efficiency == 1.0f);
CHECK(out.received_packets.empty() == puller); // only the FPGA receiver fills these
return std::move(pusher.images);
}
void CheckSummation(int64_t summation, int64_t card, int64_t host) {
const auto x = SumExperiment(summation, 2);
REQUIRE(x.GetFPGASummation() == card);
REQUIRE(x.GetHostSummation() == host);
REQUIRE(x.GetByteDepthImage() == 4);
const auto cards = CardSums(x);
for (const bool puller: {false, true}) {
INFO("puller " << puller);
const auto images = Run(x, puller);
REQUIRE(images.size() == 2);
for (const auto &[n, r]: images) {
INFO("image " << n);
CHECK(r.message.packets_received == r.message.packets_expected);
CHECK(r.message.packets_expected == 512 * summation * x.GetModulesNum());
CHECK(r.pixels == Reference(x, cards, n));
// From the card's module statistics: only the FPGA receiver keeps them
if (!puller) {
int64_t pixel_sum = 0;
for (int64_t k = 0; k < host; k++)
for (int m = 0; m < x.GetModulesNum(); m++)
pixel_sum += 1000 * ((n * host + k) % CardImageCount) + m;
CHECK(r.message.pixel_sum == pixel_sum);
}
}
}
}
}
TEST_CASE("FPGAImagePuller_Summation256_CardOnly", "[FPGAImagePuller]") {
CheckSummation(256, 256, 1);
}
TEST_CASE("FPGAImagePuller_Summation1000_Hybrid", "[FPGAImagePuller]") {
CheckSummation(1000, 250, 4);
}
TEST_CASE("FPGAImagePuller_Summation257_HostOnly", "[FPGAImagePuller]") {
CheckSummation(257, 1, 257);
}
+972
View File
@@ -0,0 +1,972 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The same frames through JFJochReceiverFPGA and through FPGAImagePuller + JFJochReceiverLite, on the mock of the
// card. The pixels must be the same; the analysis differs by design, and its numbers are printed.
#include <catch2/catch_all.hpp>
#include <cmath>
#include <future>
#include <iostream>
#include <map>
#include <mutex>
#include <random>
#include <thread>
#include "../compression/JFJochDecompress.h"
#include "../frame_serialize/CBORStream2Deserializer.h"
#include "../image_pusher/HDF5FilePusher.h"
#include "../image_pusher/ImagePusher.h"
#include "../writer/HDF5Objects.h"
#include "../writer/StreamWriter.h"
#include "../receiver/FPGAImagePuller.h"
#include "../receiver/JFJochReceiverRugnuxRotation.h"
#include "../receiver/JFJochReceiverService.h"
#include "../rugnux/RugnuxDefaults.h"
#include "MockCardOutput.h"
using namespace std::chrono_literals;
namespace {
struct Received {
DataMessage message;
std::vector<uint8_t> compressed;
};
class CollectingPusher : public ImagePusher {
public:
std::mutex m;
std::map<int64_t, Received> images;
std::vector<int64_t> arrival;
std::vector<std::string> sequence;
EndMessage end;
std::chrono::milliseconds delay{0};
bool fail_start = false;
void StartDataCollection(StartMessage &) override {
if (fail_start)
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "writer refused the start message");
std::unique_lock ul(m);
sequence.emplace_back("start");
}
bool EndDataCollection(const EndMessage &message) override {
std::unique_lock ul(m);
sequence.emplace_back("end");
end = message;
return true;
}
bool SendCalibration(const CompressedImage &c) override {
std::unique_lock ul(m);
sequence.emplace_back("calibration " + c.GetChannel());
return true;
}
bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override {
std::this_thread::sleep_for(delay);
auto d = CBORStream2Deserialize(image_data, image_size);
Received r;
r.message = d->data_message.value();
const uint8_t *p = r.message.image.GetCompressed();
r.compressed.assign(p, p + r.message.image.GetCompressedSize());
r.message.image = CompressedImage(r.compressed.data(), r.compressed.size(), r.message.image.GetWidth(),
r.message.image.GetHeight(), r.message.image.GetMode(),
r.message.image.GetCompressionAlgorithm());
std::unique_lock ul(m);
sequence.emplace_back("image");
arrival.push_back(image_number);
images.emplace(image_number, std::move(r));
return true;
}
std::string PrintSetup() const override { return "collecting"; }
ImagePusherType GetType() const override { return ImagePusherType::Test; }
};
float Pedestal(int module, int pixel) {
return 3000 + 10 * module + pixel % 50;
}
JFCalibration Calibration(const DiffractionExperiment &x) {
JFCalibration ret(x);
std::vector<float> g0(RAW_MODULE_SIZE), g1(RAW_MODULE_SIZE), g2(RAW_MODULE_SIZE);
for (int s = 0; s < x.GetStorageCellNumber(); s++)
for (int m = 0; m < x.GetModulesNum(); m++) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
g0[i] = Pedestal(m, i);
g1[i] = 15000 + 10 * m + i % 50;
g2[i] = 14000 + 10 * m + i % 50;
}
ret.Pedestal(m, 0, s).LoadPedestal(g0);
ret.Pedestal(m, 1, s).LoadPedestal(g1);
ret.Pedestal(m, 2, s).LoadPedestal(g2);
}
return ret;
}
// Raw detector frame of a module: background of a few photons, some spots, one saturated and one bad
// pixel, different for each frame.
std::vector<uint16_t> Frame(int64_t frame, int module, bool jungfrau) {
std::vector<uint16_t> ret(RAW_MODULE_SIZE);
std::mt19937 g(static_cast<uint32_t>(frame * 100 + module));
std::uniform_int_distribution<int> background(0, jungfrau ? 300 : 3);
for (int i = 0; i < RAW_MODULE_SIZE; i++)
ret[i] = (jungfrau ? static_cast<uint16_t>(Pedestal(module, i)) : 0) + background(g);
std::uniform_int_distribution<int> where_x(20, RAW_MODULE_COLS - 20), where_y(20, RAW_MODULE_LINES - 20);
for (int s = 0; s < 40; s++) {
const int x0 = where_x(g), y0 = where_y(g);
for (int y = y0; y < y0 + 3; y++)
for (int x = x0; x < x0 + 3; x++)
ret[y * RAW_MODULE_COLS + x] = jungfrau ? 12000 : 2000;
}
ret[50 * RAW_MODULE_COLS + 500] = jungfrau ? 0xC000 : 0xFFFE;
ret[60 * RAW_MODULE_COLS + 600] = 0xFFFF;
return ret;
}
SpotFindingSettings Spots() {
SpotFindingSettings s = DiffractionExperiment::DefaultDataProcessingSettings();
s.signal_to_noise_threshold = 3;
s.photon_count_threshold = 2;
s.min_pix_per_spot = 1;
s.indexing = false;
return s;
}
// What the card writes for the first frames: their conversion
std::vector<CardImage> CardImages(const DiffractionExperiment &x, int64_t distinct_frames) {
const bool jungfrau = x.GetDetectorType() == DetectorType::JUNGFRAU;
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
std::vector<CardImage> ret;
for (int64_t f = 0; f < distinct_frames; f++) {
std::vector<uint16_t> raw;
for (int m = 0; m < x.GetModulesNum(); m++) {
const auto frame = Frame(f, m, jungfrau);
raw.insert(raw.end(), frame.begin(), frame.end());
}
ret.push_back(ConvertedImage(x, calib, mask, raw));
}
return ret;
}
// The card writing zeros
AcquisitionDeviceGroup Devices(const DiffractionExperiment &x) {
return MockDevices(x, {});
}
struct PathResult {
std::map<int64_t, Received> images;
std::vector<int64_t> arrival;
std::vector<std::string> sequence;
EndMessage end;
JFJochReceiverOutput output;
};
PathResult Run(const DiffractionExperiment &x, bool puller, int64_t distinct_frames, bool rugnux = false) {
auto devices = MockDevices(x, CardImages(x, distinct_frames));
Logger logger("FPGAImagePullerTest");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(4);
service.SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(puller);
service.RugnuxLive(rugnux, nullptr, 2, 30);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
PathResult ret;
ret.output = service.Stop();
ret.images = std::move(pusher.images);
ret.arrival = std::move(pusher.arrival);
ret.sequence = std::move(pusher.sequence);
ret.end = pusher.end;
return ret;
}
std::vector<uint8_t> Pixels(const CompressedImage &image) {
std::vector<uint8_t> ret;
image.GetUncompressed(ret);
return ret;
}
template<class T>
void Row(const std::string &name, const std::optional<T> &a, const std::optional<T> &b) {
if (a == b)
return;
std::cout << " differs " << name << ": FPGA ";
if (a) std::cout << *a; else std::cout << "-";
std::cout << " puller ";
if (b) std::cout << *b; else std::cout << "-";
std::cout << std::endl;
}
// Must be the same: asserted. Analysis: printed for the first image and summed over all.
void Compare(const std::string &name, const PathResult &a, const PathResult &b) {
std::cout << "=== " << name << std::endl;
REQUIRE(a.images.size() == b.images.size());
REQUIRE(!a.images.empty());
int64_t spots_a = 0, spots_b = 0, sat_a = 0, sat_b = 0, err_a = 0, err_b = 0, strong_a = 0, strong_b = 0;
double dot = 0, norm_a = 0, norm_b = 0, ratio_sum = 0;
size_t ratio_n = 0, bins_a = 0, bins_b = 0;
for (const auto &[n, ia]: a.images) {
INFO("image " << n);
REQUIRE(b.images.contains(n));
const auto &ib = b.images.at(n);
const auto &ma = ia.message;
const auto &mb = ib.message;
CHECK(ma.image.GetMode() == mb.image.GetMode());
CHECK(ma.image.GetCompressionAlgorithm() == mb.image.GetCompressionAlgorithm());
CHECK(ma.image.GetWidth() == mb.image.GetWidth());
CHECK(ma.image.GetHeight() == mb.image.GetHeight());
CHECK(Pixels(ma.image) == Pixels(mb.image));
CHECK(ia.compressed.size() == ib.compressed.size());
// From the source
CHECK(ma.number == mb.number);
CHECK(ma.original_number == mb.original_number);
CHECK(ma.packets_expected == mb.packets_expected);
CHECK(ma.packets_received == mb.packets_received);
CHECK(ma.image_collection_efficiency == mb.image_collection_efficiency);
CHECK(ma.timestamp == mb.timestamp);
CHECK(ma.timestamp_base == mb.timestamp_base);
CHECK(ma.exptime == mb.exptime);
CHECK(ma.exptime_base == mb.exptime_base);
CHECK(ma.jf_info == mb.jf_info);
CHECK(ma.storage_cell == mb.storage_cell);
CHECK(ma.xfel_pulse_id == mb.xfel_pulse_id);
CHECK(ma.xfel_event_code == mb.xfel_event_code);
CHECK(ma.pixel_sum == mb.pixel_sum);
CHECK(ma.user_data == mb.user_data);
CHECK(ma.run_number == mb.run_number);
CHECK(ma.run_name == mb.run_name);
if (n == a.images.begin()->first) {
Row("saturated_pixel_count", ma.saturated_pixel_count, mb.saturated_pixel_count);
Row("error_pixel_count", ma.error_pixel_count, mb.error_pixel_count);
Row("min_viable_pixel_value", ma.min_viable_pixel_value, mb.min_viable_pixel_value);
Row("max_viable_pixel_value", ma.max_viable_pixel_value, mb.max_viable_pixel_value);
Row("strong_pixel_count", ma.strong_pixel_count, mb.strong_pixel_count);
Row("spot_count", ma.spot_count, mb.spot_count);
Row("bkg_estimate", ma.bkg_estimate, mb.bkg_estimate);
Row("ice_ring_score", ma.ice_ring_score, mb.ice_ring_score);
Row("receiver_aq_dev_delay", ma.receiver_aq_dev_delay, mb.receiver_aq_dev_delay);
Row("receiver_buf_in_preparation", ma.receiver_buf_in_preparation, mb.receiver_buf_in_preparation);
Row("receiver_buf_in_sending", ma.receiver_buf_in_sending, mb.receiver_buf_in_sending);
std::cout << " compression_ratio FPGA " << ma.compression_ratio.value_or(0)
<< " puller " << mb.compression_ratio.value_or(0) << std::endl;
}
spots_a += ma.spots.size();
spots_b += mb.spots.size();
sat_a += ma.saturated_pixel_count.value_or(0);
sat_b += mb.saturated_pixel_count.value_or(0);
err_a += ma.error_pixel_count.value_or(0);
err_b += mb.error_pixel_count.value_or(0);
strong_a += ma.strong_pixel_count.value_or(0);
strong_b += mb.strong_pixel_count.value_or(0);
bins_a = ma.az_int_profile.size();
bins_b = mb.az_int_profile.size();
for (size_t i = 0; i < std::min(bins_a, bins_b); i++) {
if (std::isfinite(ma.az_int_profile[i]) && std::isfinite(mb.az_int_profile[i])) {
dot += ma.az_int_profile[i] * mb.az_int_profile[i];
norm_a += ma.az_int_profile[i] * ma.az_int_profile[i];
norm_b += mb.az_int_profile[i] * mb.az_int_profile[i];
if (ma.az_int_profile[i] != 0) {
ratio_sum += mb.az_int_profile[i] / ma.az_int_profile[i];
ratio_n++;
}
}
}
}
std::cout << " over " << a.images.size() << " images:"
<< " spots FPGA " << spots_a << " puller " << spots_b
<< " | strong " << strong_a << " / " << strong_b
<< " | saturated " << sat_a << " / " << sat_b
<< " | error " << err_a << " / " << err_b
<< " | az bins " << bins_a << " / " << bins_b
<< " cosine " << (norm_a * norm_b > 0 ? dot / std::sqrt(norm_a * norm_b) : 0)
<< " mean ratio puller/FPGA " << (ratio_n > 0 ? ratio_sum / ratio_n : 0) << std::endl;
const auto &sa = a.output.status;
const auto &sb = b.output.status;
std::cout << " status: efficiency " << a.output.efficiency << " / " << b.output.efficiency
<< " | collected " << sa.images_collected << " / " << sb.images_collected
<< " | sent " << sa.images_sent << " / " << sb.images_sent
<< " | dropped " << sa.images_dropped << " / " << sb.images_dropped
<< " | max_image_sent " << sa.max_image_number_sent << " / " << sb.max_image_number_sent
<< " | compressed_size " << sa.compressed_size << " / " << sb.compressed_size
<< " | ratio " << sa.compressed_ratio.value_or(0) << " / " << sb.compressed_ratio.value_or(0)
<< " | max_delay " << sa.max_receive_delay.value_or(0) << " / " << sb.max_receive_delay.value_or(0)
<< std::endl;
CHECK(a.output.efficiency == b.output.efficiency);
CHECK(sa.images_collected == sb.images_collected);
CHECK(sa.images_sent == sb.images_sent);
CHECK(sb.images_dropped == 0);
CHECK(sa.max_image_number_sent == sb.max_image_number_sent);
CHECK(sa.compressed_size == sb.compressed_size);
CHECK(a.end.images_collected_count == b.end.images_collected_count);
CHECK(a.end.images_sent_to_write_count == b.end.images_sent_to_write_count);
CHECK(a.end.max_image_number == b.end.max_image_number);
CHECK(a.end.adu_histogram == b.end.adu_histogram);
// Calibration goes to the writer only from the FPGA receiver
CHECK(a.sequence.front() == "start");
CHECK(b.sequence.front() == "start");
CHECK(a.sequence.back() == "end");
CHECK(b.sequence.back() == "end");
auto sequence_a = a.sequence;
std::erase_if(sequence_a, [](const std::string &s) { return s.starts_with("calibration"); });
CHECK(sequence_a == b.sequence);
int64_t displacement = 0;
for (size_t i = 0; i < b.arrival.size(); i++)
displacement = std::max(displacement, std::abs(b.arrival[i] - static_cast<int64_t>(i)));
std::cout << " max displacement at the pusher (Lite's threads): " << displacement << " images" << std::endl;
}
void CompareBoth(const std::string &name, const DiffractionExperiment &x, int64_t distinct_frames) {
Compare(name, Run(x, false, distinct_frames), Run(x, true, distinct_frames));
}
DiffractionExperiment Base(const DetectorSetup &detector, int64_t images, int64_t distinct_frames) {
DiffractionExperiment x(detector);
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).InternalPacketGeneratorImages(distinct_frames)
.ImagesPerTrigger(images).ImagesPerFile(10).IncidentEnergy_keV(12.4)
.DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600);
return x;
}
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 6, 3);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
CompareBoth("JF conversion, int16, BSHUF_LZ4", x, 3);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_ApplyMask_ROI", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 4, 2);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_ZSTD).ApplyPixelMask(true);
x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}});
const auto a = Run(x, false, 2);
const auto b = Run(x, true, 2);
Compare("JF conversion, applied mask, ROI, BSHUF_ZSTD", a, b);
const auto &ra = a.images.begin()->second.message.roi.at("box");
const auto &rb = b.images.begin()->second.message.roi.at("box");
std::cout << " ROI sum " << ra.sum << " / " << rb.sum << " pixels " << ra.pixels << " / " << rb.pixels
<< " max " << ra.max_count << " / " << rb.max_count << std::endl;
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Unsigned", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 4, 2);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false);
CompareBoth("JF conversion, uint16, BSHUF_LZ4", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Summation", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 3, 2);
x.Conversion();
x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4);
CompareBoth("JF conversion, card sums 2 frames, int32", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_NoGeometry", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 4, 2);
x.Conversion();
x.GeometryTransformation(false).Compression(CompressionAlgorithm::BSHUF_LZ4);
CompareBoth("JF conversion, no geometry transform", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 4, 2);
x.Raw();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
CompareBoth("JF raw, BSHUF_LZ4", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw_NoCompression", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 3, 1);
x.Raw();
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
CompareBoth("JF raw, no compression", x, 1);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Conversion", "[FPGAImagePuller]") {
auto x = Base(DetEIGER(2), 4, 2);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
CompareBoth("EIGER assembled, 16 bit, BSHUF_LZ4", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Raw", "[FPGAImagePuller]") {
auto x = Base(DetEIGER(2), 4, 2);
x.Raw();
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
CompareBoth("EIGER raw layout, 16 bit, BSHUF_ZSTD", x, 2);
}
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") {
auto x = Base(DetEIGER(2), 3, 1);
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
x.EigerBitDepth(32);
// The 16-bit raw frames do not map onto a 32-bit stream: the card writes zeros
auto devices = Devices(x);
Logger logger("FPGAImagePuller_EIGER32");
for (const bool puller: {false, true}) {
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(4).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(puller);
PixelMask mask(x);
service.Start(x, mask, nullptr);
const auto out = service.Stop();
CHECK(out.efficiency == 1.0f);
CHECK(pusher.images.size() == 3);
CHECK(pusher.images.at(0).message.image.GetMode() == x.GetImageMode());
}
}
// A slow consumer: nothing is lost on the host, the images come in order and the card waits for its buffers
TEST_CASE("FPGAImagePuller_SlowConsumer", "[FPGAImagePuller]") {
// More images than the queue and the card's 64 buffers hold together
constexpr int64_t images = 300;
auto x = Base(DetJF(1), images, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_SlowConsumer");
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
std::this_thread::sleep_for(2s);
std::vector<int64_t> numbers;
bool end = false;
const auto t0 = std::chrono::steady_clock::now();
while (!end && std::chrono::steady_clock::now() - t0 < 120s) {
const auto m = puller.PollImage(10ms);
if (!m)
continue;
if (m->cbor->data_message)
numbers.push_back(m->cbor->data_message->number);
end = m->cbor->end_message.has_value();
}
CHECK(end);
puller.Stop();
CHECK(devices[0].GetDataCollectionStatus().pipeline_stalls_host > 0);
REQUIRE(numbers.size() == images);
for (size_t i = 0; i < numbers.size(); i++)
CHECK(numbers[i] == static_cast<int64_t>(i));
}
// Nobody takes the images: a cancel must still end the puller. Run on a thread of its own, so that a hang fails the
// test instead of stopping the run.
TEST_CASE("FPGAImagePuller_CancelWithoutConsumer", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 300, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
auto logger = std::make_shared<Logger>("FPGAImagePuller_CancelWithoutConsumer");
auto mask = std::make_shared<PixelMask>(x);
auto puller = std::make_shared<FPGAImagePuller>(x, *mask, nullptr, *devices, *logger, 2);
std::this_thread::sleep_for(1s);
auto stopped = std::make_shared<std::promise<void>>();
auto done = stopped->get_future();
std::thread([puller, devices, logger, mask, stopped] {
puller->Cancel();
try {
puller->Stop();
} catch (...) {}
stopped->set_value();
}).detach();
REQUIRE(done.wait_for(30s) == std::future_status::ready);
}
// The same behind Lite: a writer that takes its time holds Lite's only thread, and nothing is lost on the way.
TEST_CASE("FPGAImagePuller_SlowConsumer_Receiver", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 24, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_SlowReceiver");
CollectingPusher pusher;
pusher.delay = 200ms;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(1).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
std::cout << "slow consumer, Lite: collected " << out.status.images_collected << " dropped "
<< out.status.images_dropped << " efficiency " << out.efficiency << std::endl;
CHECK(out.status.images_dropped == 0);
CHECK(out.status.images_collected == 24);
CHECK(out.efficiency == 1.0f);
CHECK(pusher.images.size() == 24);
CHECK(pusher.sequence.back() == "end");
}
// The broker ends every run with a silent cancel once the detector is done: every image still reaches the writer,
// however slow it is.
TEST_CASE("FPGAImagePuller_SilentEndDeliversAll", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 24, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_SilentEnd");
CollectingPusher pusher;
pusher.delay = 200ms;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(1).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
// The card is through, the writer far behind
while (!devices[0].Counters().IsAcquisitionFinished())
std::this_thread::sleep_for(10ms);
service.Cancel(true);
const auto out = service.Stop();
CHECK(!out.status.cancelled);
CHECK(out.status.images_collected == 24);
CHECK(pusher.images.size() == 24);
CHECK(pusher.sequence.back() == "end");
}
// An abort early in a long run ends in bounded time: the images the cards never got to are not assembled
// one by one. Run on a thread of its own, so that a hang fails the test instead of stopping the run.
TEST_CASE("FPGAImagePuller_AbortLargeRun", "[FPGAImagePuller]") {
auto x = Base(DetJF(8), 1000000, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
auto logger = std::make_shared<Logger>("FPGAImagePuller_AbortLargeRun");
auto mask = std::make_shared<PixelMask>(x);
auto puller = std::make_shared<FPGAImagePuller>(x, *mask, nullptr, *devices, *logger, 4);
std::this_thread::sleep_for(1s);
auto stopped = std::make_shared<std::promise<void>>();
auto done = stopped->get_future();
std::thread([puller, devices, logger, mask, stopped] {
puller->Cancel();
try {
puller->Stop();
} catch (...) {}
stopped->set_value();
}).detach();
REQUIRE(done.wait_for(30s) == std::future_status::ready);
}
// The cards are stopped while they wait for the host (nobody reads, the queue is full): every image they hold
// still comes, in order, then the end message.
TEST_CASE("FPGAImagePuller_StopCardsWhileStalled", "[FPGAImagePuller]") {
constexpr int64_t images = 1000;
auto x = Base(DetJF(1), images, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_StopCardsWhileStalled");
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
// The queue full, then the card waiting for host buffers
const auto t0 = std::chrono::steady_clock::now();
while (puller.GetCurrentFifoUtilization() < FPGAImagePuller::Queue && std::chrono::steady_clock::now() - t0 < 60s)
std::this_thread::sleep_for(10ms);
REQUIRE(puller.GetCurrentFifoUtilization() >= FPGAImagePuller::Queue);
std::this_thread::sleep_for(1s);
const auto fastest = devices[0].Counters().GetFastestFrameNumber();
std::this_thread::sleep_for(500ms);
REQUIRE(devices[0].Counters().GetFastestFrameNumber() == fastest);
REQUIRE(!devices[0].Counters().IsAcquisitionFinished());
puller.StopCards();
std::vector<int64_t> numbers;
bool end = false;
while (!end && std::chrono::steady_clock::now() - t0 < 120s) {
const auto m = puller.PollImage(10ms);
if (!m)
continue;
if (m->cbor->data_message)
numbers.push_back(m->cbor->data_message->number);
end = m->cbor->end_message.has_value();
}
CHECK(end);
REQUIRE_NOTHROW(puller.Stop());
int64_t on_card = 0;
for (int64_t f = 0; f < images; f++)
if (devices[0].Counters().IsAnyPacketCollected(f, 0))
on_card++;
std::cout << "cards stopped while stalled: " << numbers.size() << " images delivered, " << on_card
<< " on the card" << std::endl;
CHECK(on_card >= static_cast<int64_t>(FPGAImagePuller::Queue));
CHECK(on_card < images);
REQUIRE(static_cast<int64_t>(numbers.size()) == on_card);
for (size_t i = 0; i < numbers.size(); i++)
CHECK(numbers[i] == static_cast<int64_t>(i));
}
// The receiver ends on an error (the writer refuses the start message) while the puller still has images: the
// puller is stopped with it and the card released, not left armed until the next run.
TEST_CASE("FPGAImagePuller_ReceiverErrorStopsPuller", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 1000, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_ReceiverError");
CollectingPusher pusher;
pusher.fail_start = true;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(1).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
CHECK_THROWS(service.Stop());
CHECK(devices[0].Counters().IsAcquisitionFinished());
CHECK(devices[0].Counters().GetFastestFrameNumber() < 999);
}
// A module frame missing in the middle, and a module that dies: the images keep coming with no help
// from the end of the run, and the missing module is filled, as in JFJochReceiverFPGA.
TEST_CASE("FPGAImagePuller_MissingModule", "[FPGAImagePuller]") {
constexpr int64_t images = 62;
constexpr int64_t dies_at = 3;
auto x = Base(DetJF(2), images, 1);
x.UseInternalPacketGenerator(false).Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto device = std::make_unique<MockAcquisitionDevice>(0, 64);
SetOutputs(*device, x, 0, CardImages(x, 1));
for (int64_t f = 0; f < images; f++)
for (int m = 0; m < 2; m++) {
if ((m == 1) && (f == 1 || f >= dies_at))
continue;
device->CreateJFPackets(x, f + 1, 1, m, nullptr);
}
AcquisitionDeviceGroup devices;
devices.Add(std::move(device));
Logger logger("FPGAImagePuller_MissingModule");
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
// No final packet and no cancel yet: images must come through anyway, up to the lag behind the card
std::vector<int64_t> numbers;
std::vector<float> efficiency;
bool stop_requested = false, end = false;
size_t images_before_cancel = 0;
const auto t0 = std::chrono::steady_clock::now();
auto last = t0;
while (!end && std::chrono::steady_clock::now() - t0 < 900s) {
const auto m = puller.PollImage(10ms);
if (!m) {
if (!stop_requested && std::chrono::steady_clock::now() - last > 5s) {
stop_requested = true;
std::cout << "dead module, before cancel: fastest " << devices[0].Counters().GetFastestFrameNumber()
<< " module 0 at " << devices[0].Counters().GetCurrFrameNumber(0)
<< " module 1 at " << devices[0].Counters().GetCurrFrameNumber(1)
<< " images so far " << numbers.size() << std::endl;
puller.StopCards();
}
continue;
}
last = std::chrono::steady_clock::now();
if (m->cbor->data_message) {
const auto &d = *m->cbor->data_message;
numbers.push_back(d.number);
efficiency.push_back(d.image_collection_efficiency.value_or(-1));
if (!stop_requested)
images_before_cancel = numbers.size();
if (d.number == 1) {
// Module 1 absent from this frame: its pixels are the fill value
std::vector<uint8_t> tmp;
const auto *p = reinterpret_cast<const int16_t *>(d.image.GetUncompressedPtr(tmp));
CHECK(p[x.GetPixel0OfModuleConv(1)] == x.GetImageFillValue());
CHECK(p[x.GetPixel0OfModuleConv(0)] != x.GetImageFillValue());
}
}
end = m->cbor->end_message.has_value();
}
CHECK(end);
REQUIRE_NOTHROW(puller.Stop());
std::cout << "dead module: " << numbers.size() << " images, before any cancel: "
<< images_before_cancel << std::endl;
// Image i is let go once the card reports frame i + 2 + MissingAfterFrames of the other module
CHECK(images_before_cancel >= 4);
REQUIRE(numbers.size() >= 4);
for (size_t i = 0; i < numbers.size(); i++)
CHECK(numbers[i] == static_cast<int64_t>(i));
CHECK(efficiency[0] == 1.0f);
CHECK(efficiency[1] == 0.5f);
// Frame 2 of module 1 is sent, but the card holds a module's last frames until newer ones push them
// out: given up on, it comes with the cancel. Its buffer went back.
CHECK(efficiency[3] == 0.5f);
CHECK(efficiency.back() == 0.5f);
}
TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") {
for (const bool silent: {false, true}) {
auto x = Base(DetJF(2), 6, 1);
x.UseInternalPacketGenerator(false).Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto device = std::make_unique<MockAcquisitionDevice>(0, 64);
SetOutputs(*device, x, 0, CardImages(x, 3));
for (int64_t f = 0; f < 3; f++)
for (int m = 0; m < 2; m++)
device->CreateJFPackets(x, f + 1, 1, m, nullptr);
AcquisitionDeviceGroup devices;
devices.Add(std::move(device));
Logger logger("FPGAImagePuller_Cancel");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(2).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
std::this_thread::sleep_for(2s);
service.Cancel(silent);
const auto out = service.Stop();
std::cout << "cancel silent=" << silent << ": cancelled " << out.status.cancelled << " collected "
<< out.status.images_collected << " sent " << out.status.images_sent << " efficiency "
<< out.efficiency << std::endl;
CHECK(out.status.cancelled == !silent);
CHECK(pusher.sequence.front() == "start");
CHECK(pusher.sequence.back() == "end");
CHECK(out.status.images_collected <= 3);
if (silent) {
CHECK(out.status.images_collected == 3);
CHECK(out.efficiency == Catch::Approx(0.5f));
}
}
}
// Summation on the host takes the puller as well
TEST_CASE("FPGAImagePuller_HostSummation", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 2, 1);
x.Conversion();
x.Summation(2).CPUSummation(true).Compression(CompressionAlgorithm::BSHUF_LZ4);
REQUIRE(x.IsCPUSummation());
auto devices = Devices(x);
Logger logger("FPGAImagePuller_HostSummation");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(4).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
CHECK(out.efficiency == 1.0f);
CHECK(out.received_packets.empty()); // only the FPGA receiver fills these
CHECK(out.status.images_collected == 2);
}
// By default only a collection analysed by rugnux live goes through the puller: the others stay on the FPGA receiver
TEST_CASE("FPGAImagePuller_DefaultOnlyWithRugnuxLive", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 3, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4).FilePrefix("fpga_puller_default");
for (const bool rugnux: {false, true}) {
auto devices = Devices(x);
Logger logger("FPGAImagePuller_Default");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(2).SetSpotFindingSettings(Spots());
service.RugnuxLive(rugnux, nullptr, 2, 30);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
CHECK(out.status.images_sent == 3);
CHECK(out.received_packets.empty() == rugnux); // only the FPGA receiver fills these
}
}
// JFJochReceiverRugnuxRotation takes the same stream, with its own forwarding thread.
TEST_CASE("FPGAImagePuller_RugnuxReceiver", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 6, 3);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
x.FilePrefix("fpga_puller_rugnux");
REQUIRE(RugnuxRotationData(x));
auto devices = MockDevices(x, CardImages(x, 3));
Logger logger("FPGAImagePuller_Rugnux");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(2).SetSpotFindingSettings(Spots());
// Rugnux live takes the puller without being asked
service.RugnuxLive(true, nullptr, 2, 30);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
std::cout << "rugnux receiver: efficiency " << out.efficiency << " sent " << out.status.images_sent
<< " warning '" << out.analysis_warning << "'" << std::endl;
CHECK(out.efficiency == 1.0f);
CHECK(out.status.images_sent == 6);
CHECK(pusher.images.size() == 6);
CHECK(pusher.sequence.front() == "start");
CHECK(pusher.sequence.back() == "end");
CHECK(pusher.end.adu_histogram.contains("module0"));
CHECK_FALSE(pusher.sequence[1].starts_with("calibration")); // the writer gets calibration only from the FPGA receiver
const auto a = Pixels(pusher.images.at(0).message.image);
const auto b = Pixels(pusher.images.at(3).message.image);
CHECK(a == b); // the card output repeats after the third frame
}
// In an unsigned image the card gives saturated and error pixels one value: the counts are the card's, on every receiver.
TEST_CASE("FPGAImagePuller_SaturatedAndErrorCountsFromTheCard", "[FPGAImagePuller]") {
auto x = Base(DetJF(2), 4, 2);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false).FilePrefix("fpga_puller_counts");
const auto card = Run(x, false, 2);
PathResult puller;
SECTION("JFJochReceiverLite") {
puller = Run(x, true, 2);
}
SECTION("JFJochReceiverRugnuxStills") {
puller = Run(x, true, 2, true);
}
SECTION("JFJochReceiverRugnuxRotation") {
auto rotation = x;
rotation.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
puller = Run(rotation, true, 2, true);
}
REQUIRE(card.images.size() == 4);
REQUIRE(puller.images.size() == 4);
int64_t saturated = 0, error = 0;
for (const auto &[n, image]: puller.images) {
INFO("image " << n);
CHECK(image.message.saturated_pixel_count == card.images.at(n).message.saturated_pixel_count);
CHECK(image.message.error_pixel_count == card.images.at(n).message.error_pixel_count);
saturated += image.message.saturated_pixel_count.value_or(0);
error += image.message.error_pixel_count.value_or(0);
}
CHECK(saturated == 8);
CHECK(error == 8);
}
// Rotation forwards the images unanalysed, so nothing would replace what the card's ROI map, not loaded for this run, gave.
TEST_CASE("FPGAImagePuller_NoStaleROI", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 4, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
x.FilePrefix("fpga_puller_roi");
x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}});
auto devices = Devices(x);
Logger logger("FPGAImagePuller_NoStaleROI");
CollectingPusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(2).SetSpotFindingSettings(Spots());
// Rugnux live takes the puller without being asked
service.RugnuxLive(true, nullptr, 2, 30);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
service.Stop();
REQUIRE(pusher.images.size() == 4);
for (const auto &[n, image]: pusher.images)
CHECK(image.message.roi.empty());
}
// An image is many frames: efficiency and progress count images.
TEST_CASE("FPGAImagePuller_RugnuxReceiver_Summation", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 3, 2);
x.Conversion();
x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4);
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
x.FilePrefix("fpga_puller_rugnux_sum");
REQUIRE(x.GetImageNum() == 3);
REQUIRE(x.GetFrameNum() == 6);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_RugnuxSum");
CollectingPusher pusher;
ImageBuffer buffer(256 * 1024 * 1024);
JFJochReceiverCurrentStatus status;
JFJochReceiverPlots plots;
RugnuxLiveStatus live;
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
JFJochReceiverRugnuxRotation receiver(x, mask, puller, pusher, logger, 2, 2, nullptr, 30, false, live, Spots(), status,
plots, buffer);
receiver.StopReceiver();
puller.Stop();
CHECK(receiver.GetEfficiency() == 1.0f);
CHECK(receiver.GetProgress() == 1.0f);
CHECK(pusher.images.size() == 3);
}
// The real writer: the master file states what the card did, not what a DECTRIS stream implies.
TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") {
RegisterHDF5Filter();
auto x = Base(DetJF(2), 5, 2);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4).ApplyPixelMask(true).ImagesPerFile(2)
.FilePrefix("fpga_image_puller_writer").SetFileWriterFormat(FileWriterFormat::NXmxVDS)
.OverwriteExistingFiles(true).SaveCalibration(true);
auto devices = Devices(x);
Logger logger("FPGAImagePuller_Writer");
HDF5FilePusher pusher;
JFJochReceiverService service(devices, logger, pusher, 256);
service.NumThreads(2).SetSpotFindingSettings(Spots());
service.FPGAImagePullerSettings(true);
PixelMask mask(x);
const JFCalibration calib = Calibration(x);
service.Start(x, mask, &calib);
const auto out = service.Stop();
CHECK(out.efficiency == 1.0f);
CHECK(out.writer_err.empty());
CHECK(out.status.images_sent == 5);
HDF5ReadOnlyFile master("fpga_image_puller_writer_master.h5");
CHECK(master.GetBool("/entry/instrument/detector/pixel_mask_applied"));
HDF5DataSet dataset(master, "/entry/data/data");
HDF5DataSpace space(dataset);
CHECK(space.GetDimensions()[0] == 5);
CHECK(space.GetDimensions()[1] == x.GetYPixelsNum());
}
+40
View File
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../receiver/HoldArena.h"
TEST_CASE("HoldArena_WithinHalfOfTheImageBuffer", "[HoldArena]") {
constexpr size_t slot_size = 1024 * 1024;
ImageBuffer buffer(64 * slot_size);
buffer.StartMeasurement(slot_size);
const int64_t total = buffer.GetStatus().total_slots;
REQUIRE(total >= 32);
std::vector<HoldArena::Region> held;
{
HoldArena arena(buffer, total / 2, 4, slot_size);
// Frames of a third of a slot: three to a slot, half of the slots at most
while (true) {
auto r = arena.Allocate(slot_size / 3);
if (!r.data)
break;
held.push_back(std::move(r));
}
CHECK(arena.GetSlotsUsed() == total / 2);
CHECK(held.size() == static_cast<size_t>(3 * (total / 2)));
CHECK(buffer.GetSlotStatus().available_slots == total - total / 2);
// Larger than a slot: never held
CHECK(!arena.Allocate(slot_size + 1).data);
// A slot whose frames are all gone is reused, nothing more is taken from the buffer
for (int i = 0; i < 3; i++)
held[i].release();
CHECK(arena.Allocate(slot_size / 3).data != nullptr);
CHECK(arena.GetSlotsUsed() == total / 2);
arena.ReleaseSlots();
CHECK(buffer.GetSlotStatus().available_slots == total);
}
}
+25 -14
View File
@@ -220,7 +220,7 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
}
TEST_CASE("JFJochReceiverRugnux_Stills_ImagesVisibleDuringPrePass", "[JFJochReceiverRugnux]") {
// A frame held for the pre-pass can be seen in the image buffer as it came, with a start message on the stated
// A frame held for the pre-pass can be seen through the image routes as it came, with a start message on the stated
// geometry; once analysed, the copy with the analysis is the only one there. The writer gets each image once.
Logger logger(Catch::getResultCapture().getCurrentTestName());
RegisterHDF5Filter();
@@ -263,7 +263,8 @@ TEST_CASE("JFJochReceiverRugnux_Stills_ImagesVisibleDuringPrePass", "[JFJochRece
}
REQUIRE(service.GetImageFromBuffer(cbor, ImageBuffer::MaxImage));
CHECK(CBORStream2Deserialize(cbor)->data_message->number == held - 1);
CHECK(service.GetImageBufferStatus().sending_slots == held);
// They wait in the receiver's hold, not as images of the buffer: none is in a slot to be sent
CHECK(service.GetImageBufferStatus().sending_slots == 0);
// Nothing goes to the writer before the pre-pass
CHECK(pusher.Events().empty());
@@ -348,10 +349,9 @@ TEST_CASE("JFJochReceiverRugnux_Stills_CancelDuringRun", "[JFJochReceiverRugnux]
PixelMask mask(x);
Stream stream(x, images);
// Twelve slots: the pre-pass ends on six frames, and the analysis starts with most of the run to come.
const size_t buffer_MiB = 12 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1;
const int64_t given = 8;
stream.Frames(0, given);
// Four slots: the pre-pass starts once the two the frames wait in are full, and the analysis starts with most of
// the run to come. The frames are given one by one until it does.
const size_t buffer_MiB = 4 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1;
RecordingPusher pusher;
AcquisitionDeviceGroup group;
@@ -362,6 +362,16 @@ TEST_CASE("JFJochReceiverRugnux_Stills_CancelDuringRun", "[JFJochReceiverRugnux]
service.RugnuxLive(true, std::make_shared<RugnuxModel>(), 4, 60);
service.Start(x, mask, nullptr, stream.puller);
int64_t given = 0;
const auto holding = [&] {
const auto s = service.GetRugnuxStatus();
return !s.stills || s.stills->prepass == RugnuxLivePrePass::Holding;
};
while (holding() && given < images - 1) {
stream.Frames(given++, 1);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
REQUIRE(given < images - 1);
WaitForStatus(service, [](const RugnuxLiveSnapshot &s) {
return s.stills && s.stills->prepass == RugnuxLivePrePass::Done && s.stills->images_analysed > 0;
});
@@ -369,6 +379,7 @@ TEST_CASE("JFJochReceiverRugnux_Stills_CancelDuringRun", "[JFJochReceiverRugnux]
const auto out = service.Stop();
CHECK(out.status.cancelled);
CHECK(out.status.images_collected == given);
CHECK(out.status.images_sent == given);
CHECK(pusher.Events() == std::vector<std::string>{"start", "image", "end"});
const auto live = service.GetRugnuxStatus();
@@ -636,16 +647,16 @@ TEST_CASE("JFJochReceiverRugnux_Stills_ZMQWriterDown", "[JFJochReceiverRugnux]")
}
TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceiverRugnux]") {
// Forty frames, all there before the pre-pass ends, and twelve slots in the image buffer.
// More frames than half of a twelve-slot image buffer holds, all there before the pre-pass ends. The frames that
// wait take at most half of the buffer: past it, a frame waits, the pre-pass starts on what is held and the
// analysis drains it. Every frame is written.
Logger logger(Catch::getResultCapture().getCurrentTestName());
RegisterHDF5Filter();
const int64_t images = 40;
const int64_t images = 200;
DiffractionExperiment x = StillsExperiment(images);
PixelMask mask(x);
Stream stream(x, images);
stream.Frames(0, images);
stream.End();
const size_t buffer_MiB = 12 * x.GetImageBufferLocationSize() / (1024 * 1024) + 1;
@@ -656,17 +667,17 @@ TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceive
service.Indexing(x.GetIndexingSettings());
service.RugnuxLive(true, nullptr, 4, 60);
stream.Frames(0, images);
stream.End();
service.Start(x, mask, nullptr, stream.puller);
const auto out = service.Stop();
// No deadlock, and the end message goes out.
CHECK(!out.status.cancelled);
CHECK(pusher.Events() == std::vector<std::string>{"start", "image", "end"});
// A frame that finds no free slot is not written, and the collection says so.
CHECK(out.status.images_collected == images);
CHECK(out.efficiency == 1.0f);
CHECK(out.writer_queue_full_warning == (out.status.images_sent < images));
CHECK(out.status.images_sent == images);
CHECK(!out.writer_queue_full_warning);
CHECK(out.scan_result.images.size() == out.status.images_sent);
REQUIRE(pusher.End());
CHECK(pusher.End()->images_collected_count == images);
+63
View File
@@ -517,3 +517,66 @@ TEST_CASE("RawToConvertedCoordinate_Coord","[RawToConvertedGeometry]") {
REQUIRE(Catch::Approx(c.x) == CONVERTED_MODULE_COLS + 8 + 800.5 + 6); // col 800 becomes 806 due to 3 chip boundaries
REQUIRE(Catch::Approx(c.y) == CONVERTED_MODULE_LINES * 2 + 36 - 1 - 300.25 - 2); // line 300 becomes 302 due to 1 chip boundary
}
namespace {
// Assembled with the multipixels split, then squeezed back: the raw frames, where the split divides evenly
void CheckSqueezeRoundTrip(const DetectorSetup &detector) {
DiffractionExperiment x(detector);
x.PixelSigned(true).BitDepthImage(16);
std::vector<int16_t> raw(x.GetModulesNum() * RAW_MODULE_SIZE);
for (size_t i = 0; i < raw.size(); i++)
raw[i] = static_cast<int16_t>(4 * (i % 5000) - 400);
raw[255 * RAW_MODULE_COLS + 255] = INT16_MIN; // error, chip corner
raw[RAW_MODULE_SIZE + 10 * RAW_MODULE_COLS + 256] = INT16_MAX; // saturated, multipixel
std::vector<int32_t> image(x.GetPixelsNum(), INT32_MIN);
for (int m = 0; m < x.GetModulesNum(); m++)
TransferModuleAdjustMultipixels<int32_t, int16_t>(image.data(), raw.data() + m * RAW_MODULE_SIZE,
x.GetModuleSlowDirectionStep(m), INT16_MIN, INT16_MAX,
x.GetModuleFastDirectionStep(m),
x.GetPixel0OfModuleConv(m));
for (auto &v: image) {
if (v == INT16_MIN)
v = INT32_MIN;
else if (v == INT16_MAX)
v = INT32_MAX;
}
std::vector<int16_t> squeezed(raw.size());
SqueezeToRawGeometry<int16_t, int32_t>(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX,
INT16_MIN, INT16_MAX);
CHECK(squeezed == raw);
// A sum past the 16-bit range is saturated
image[x.GetPixel0OfModuleConv(0)] = 40000;
SqueezeToRawGeometry<int16_t, int32_t>(x, squeezed.data(), image.data(), INT32_MIN, INT32_MAX,
INT16_MIN, INT16_MAX);
CHECK(squeezed[0] == INT16_MAX);
}
}
TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_4M","[RawToConvertedGeometry]") {
CheckSqueezeRoundTrip(DetJF(8, 2, 8, 36, true));
}
TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_9M","[RawToConvertedGeometry]") {
CheckSqueezeRoundTrip(DetJF(18, 3, 8, 36, true));
}
TEST_CASE("RawToConvertedGeometry_SqueezeToRawGeometry_16_modules_turned","[RawToConvertedGeometry]") {
// 12 modules in rows of two, every other row read out the other way, and 4 standing modules beside them
using Direction = DetectorModuleGeometry::Direction;
std::vector<DetectorModuleGeometry> modules;
for (int row = 0; row < 6; row++)
for (int col = 0; col < 2; col++) {
if (row % 2 == 0)
modules.emplace_back(col * 1038, row * 550, Direction::Xpos, Direction::Ypos);
else
modules.emplace_back(col * 1038 + 1029, row * 550 + 513, Direction::Xneg, Direction::Yneg);
}
for (int row = 0; row < 2; row++)
for (int col = 0; col < 2; col++)
modules.emplace_back(2076 + col * 522 + 513, row * 1038, Direction::Ypos, Direction::Xneg);
CheckSqueezeRoundTrip(DetJF(DetectorGeometryModular(modules)));
}
-127
View File
@@ -1,127 +0,0 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <atomic>
#include "../common/ImageBuffer.h"
#include "../receiver/StillsGPUAnalysisFPGA.h"
namespace {
DiffractionExperiment StillsExperiment() {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(StillsGPUAnalysisFPGA::PREPASS_IMAGES + 200).NumTriggers(1).DetectorDistance_mm(100).BeamX_pxl(500).BeamY_pxl(250)
.IncidentEnergy_keV(12.4);
return x;
}
// A frame after the pre-pass's sample (numbers from PREPASS_IMAGES on are never in it), in a slot of `buffer`.
std::shared_ptr<HeldImage> Frame(ImageBuffer &buffer, int64_t number) {
auto held = std::make_shared<HeldImage>();
held->slot = buffer.GetImageSlot();
REQUIRE(held->slot != nullptr);
held->message.number = StillsGPUAnalysisFPGA::PREPASS_IMAGES + number;
held->message.image_collection_efficiency = 1.0f;
return held;
}
StartMessage Start(const DiffractionExperiment &x) {
StartMessage start;
x.FillMessage(start);
return start;
}
}
TEST_CASE("StillsGPUAnalysisFPGA_StartFailureReleasesThreads", "[StillsGPUAnalysisFPGA]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
const DiffractionExperiment x = StillsExperiment();
const PixelMask mask(x);
ImageBuffer buffer(64 * 1024 * 1024);
buffer.StartMeasurement(x);
RugnuxLiveStatus status;
std::atomic<int> failed = 0, sunk = 0;
{
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 4, 100, logger,
[](const StillsGPUAnalysisFPGA::Setup &) {
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message");
},
[&](DataMessage &, const AzimuthalIntegrationProfile &) { ++sunk; },
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); },
[&](const JFJochException &) { ++failed; }, &status);
for (int i = 0; i < 3; i++)
REQUIRE(gpu.Hold(Frame(buffer, i)));
REQUIRE_THROWS_AS(gpu.Finish(), JFJochException);
}
CHECK(failed == 1);
CHECK(sunk == 0);
CHECK(status.Get().state == RugnuxLiveState::Failed);
// Every slot the frames were held in is back.
CHECK(buffer.GetAvailSlots() == buffer.GetStatus().total_slots);
}
TEST_CASE("StillsGPUAnalysisFPGA_HoldIsBounded", "[StillsGPUAnalysisFPGA]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
const DiffractionExperiment x = StillsExperiment();
const PixelMask mask(x);
ImageBuffer buffer(64 * 1024 * 1024);
buffer.StartMeasurement(x);
REQUIRE(buffer.GetStatus().total_slots >= 4);
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 2, 3, logger,
[](const StillsGPUAnalysisFPGA::Setup &) {
throw JFJochException(JFJochExceptionCategory::ZeroMQ, "no writer");
},
[](DataMessage &, const AzimuthalIntegrationProfile &) {},
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); });
// The analysis waits for the pre-pass, which waits for its sample: nothing is taken off the queue.
for (int i = 0; i < 3; i++)
REQUIRE(gpu.Hold(Frame(buffer, i)));
{
const auto refused = Frame(buffer, 3);
CHECK(!gpu.Hold(refused));
}
CHECK(buffer.GetStatus().preparation_slots == 3);
REQUIRE_THROWS(gpu.Finish());
CHECK(buffer.GetStatus().preparation_slots == 0);
}
TEST_CASE("StillsGPUAnalysisFPGA_FailingFrameDoesNotStopAnalysis", "[StillsGPUAnalysisFPGA]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
const DiffractionExperiment x = StillsExperiment();
const PixelMask mask(x);
ImageBuffer buffer(64 * 1024 * 1024);
buffer.StartMeasurement(x);
RugnuxLiveStatus status;
std::atomic<int> sunk = 0;
StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 1, 100, logger,
[](const StillsGPUAnalysisFPGA::Setup &) {},
[&](DataMessage &m, const AzimuthalIntegrationProfile &) {
++sunk;
if (m.number == StillsGPUAnalysisFPGA::PREPASS_IMAGES)
throw std::runtime_error("not a JFJochException");
},
[] { return DiffractionExperiment::DefaultDataProcessingSettings(); },
{}, &status);
// The frames carry no image, so their analysis fails too - they are passed on all the same.
for (int i = 0; i < 3; i++)
REQUIRE(gpu.Hold(Frame(buffer, i)));
REQUIRE_NOTHROW(gpu.Finish());
CHECK(sunk == 3);
// The outcome waits for everything to be written, and comes after the latency.
CHECK(status.Get().state == RugnuxLiveState::Running);
CHECK(!status.Get().last_image_to_written_s);
gpu.Written();
const auto s = status.Get();
CHECK(s.state == RugnuxLiveState::Failed);
CHECK(!s.message.empty());
REQUIRE(s.last_image_to_written_s);
REQUIRE(s.last_image_s);
CHECK(s.elapsed_s >= s.last_image_s.value() + s.last_image_to_written_s.value() - 1e-6);
REQUIRE(!s.steps.empty());
CHECK(s.steps.back().title == "Writing");
CHECK(s.steps.back().duration_s);
CHECK(buffer.GetStatus().preparation_slots == 0);
}
+42
View File
@@ -0,0 +1,42 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <cmath>
#include "../receiver/StillsSelection.h"
TEST_CASE("StillsImageSelected_FractionAndPeriods", "[StillsSelection]") {
constexpr int64_t first = 2000;
constexpr int64_t images = 400000;
for (const int64_t factor: {2, 3, 4, 10}) {
INFO("factor " << factor);
const double p = 1.0 / static_cast<double>(factor);
int64_t selected = 0;
for (int64_t i = first; i < first + images; i++)
selected += StillsImageSelected(i, factor, first) ? 1 : 0;
// Within 5 standard deviations of the binomial
CHECK(std::abs(static_cast<double>(selected) / images - p) < 5 * std::sqrt(p * (1 - p) / images));
// Every phase of a period-2 and a period-F pattern (pump-probe on/off, a stride) gets its share
for (const int64_t period: {int64_t(2), factor}) {
for (int64_t phase = 0; phase < period; phase++) {
INFO("period " << period << " phase " << phase);
int64_t n = 0, s = 0;
for (int64_t i = first + phase; i < first + images; i += period) {
n++;
s += StillsImageSelected(i, factor, first) ? 1 : 0;
}
CHECK(std::abs(static_cast<double>(s) / n - p) < 5 * std::sqrt(p * (1 - p) / n));
}
}
}
}
TEST_CASE("StillsImageSelected_DeterministicAndPrePass", "[StillsSelection]") {
for (int64_t i = 0; i < 10000; i++) {
CHECK(StillsImageSelected(i, 1, 0)); // factor 1: every image
CHECK(StillsImageSelected(i, 7, 10000)); // before all_before: every image
CHECK(StillsImageSelected(i, 7, 0) == StillsImageSelected(i, 7, 0));
}
}
+9
View File
@@ -68,6 +68,15 @@ IF(NOT JFJOCH_PORTABLE_ONLY)
TARGET_LINK_LIBRARIES(jfjoch_fpga_test JFJochReceiver)
INSTALL(TARGETS jfjoch_fpga_test RUNTIME COMPONENT jfjoch)
ADD_EXECUTABLE(jfjoch_fpga_stills_test jfjoch_fpga_stills_test.cpp)
TARGET_LINK_LIBRARIES(jfjoch_fpga_stills_test JFJochReceiver JFJochReader)
# Copied to the FPGA servers on its own: cuFFT linked statically (see image_analysis/indexing/CMakeLists.txt),
# which needs the device-link step (see rugnux/CMakeLists.txt)
IF (JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
SET_TARGET_PROPERTIES(jfjoch_fpga_stills_test PROPERTIES JFJOCH_STATIC_CUFFT ON CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
INSTALL(TARGETS jfjoch_fpga_stills_test RUNTIME COMPONENT jfjoch)
ADD_EXECUTABLE(jfjoch_simplon_test jfjoch_simplon_test.cpp)
TARGET_LINK_LIBRARIES(jfjoch_simplon_test JFJochDetector JFJochPreview)
INSTALL(TARGETS jfjoch_simplon_test RUNTIME COMPONENT jfjoch)
+374
View File
@@ -0,0 +1,374 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// Rugnux live stills on the FPGA card(s), with a stored dataset as the detector: the first images of a JUNGFRAU
// dataset folded back into module frames, loaded into the card's internal frame generator and returned verbatim
// (no conversion), then assembled on the host and analysed through FPGAImagePuller and JFJochReceiverRugnuxStills.
// The generator cycles through the images its memory holds, so images repeat: throughput, latency and indexing rate
// are meaningful, the merge and its statistics are not.
#include <algorithm>
#include <atomic>
#include <getopt.h>
#include <iostream>
#include <thread>
#include <sys/resource.h>
#include "../acquisition_device/PCIExpressDevice.h"
#include "../acquisition_device/MockAcquisitionDevice.h"
#include "../common/RawToConvertedGeometry.h"
#include "../common/print_license.h"
#include "../image_pusher/NonePusher.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../receiver/JFJochReceiverService.h"
#include "../writer/HDF5Objects.h"
void print_usage(Logger &logger) {
logger.Info("Usage ./jfjoch_fpga_stills_test {<options>} <master.h5>");
logger.Info("The dataset: JUNGFRAU modules, as the file states them (module groups, else the gaps of its mask).");
logger.Info("Its first images - per card, as many as its generator memory holds of that card's modules - are");
logger.Info("loaded into the cards' internal generators, which cycle through them: images repeat, so throughput, latency");
logger.Info("and indexing rate are meaningful, the merge and its statistics are not. See docs/TOOLS.md.");
logger.Info("Options:");
logger.Info(" -v Each card's progress, the receiver's and the analysis' counts every 0.5 s");
logger.Info(" -i<num> Images in the run (default: as many as the file has)");
logger.Info(" -s<num> Number of data streams (cards)");
logger.Info(" -N<num> Number of image processing threads (default 64)");
logger.Info(" -B<num> Size of the image buffer in MiB (default 16384)");
logger.Info(" -C<a,b,c,alpha,beta,gamma> Unit cell (default: the file's)");
logger.Info(" -S<group> Space group of the collection (a space group stored in the file is ignored)");
logger.Info(" -X<txt> Indexing (auto|fft|fftw|ffbidx), auto is default");
logger.Info(" -t<num> Indexing thread pool size (default: 8, as offline rugnux on stills; the broker's default is 4)");
logger.Info(" -k<num> Analyse 1 of every <num> images on average (rugnux live stills_downsample, default 1)");
logger.Info(" -m<num> Frames summed into an image (default 1): the card sums the largest divisor up to 256, the");
logger.Info(" host the rest (1000 = 250 on the card x 4 on the host)");
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
logger.Info(" 128 module frames)");
}
// A module the file gives the readout of keeps it; one it does not (taken from the gaps of its mask, +x +y) is read
// out from the top-left corner of its box, along its 1030-pixel side. As the multipixels are symmetric in a module,
// which corner it is does not change the frames the card returns.
std::optional<DetectorGeometryModular> GeometryFromModules(const DetectorModules &modules) {
using Direction = DetectorModuleGeometry::Direction;
auto along_x = [](Direction d) { return (d == Direction::Xpos) || (d == Direction::Xneg); };
std::vector<DetectorModuleGeometry> ret;
for (const auto &m: modules.Get()) {
if ((m.width == CONVERTED_MODULE_COLS) && (m.height == CONVERTED_MODULE_LINES) && along_x(m.fast))
ret.emplace_back(m.ReadoutX0(), m.ReadoutY0(), m.fast, m.slow);
else if ((m.width == CONVERTED_MODULE_LINES) && (m.height == CONVERTED_MODULE_COLS) && !along_x(m.fast))
ret.emplace_back(m.ReadoutX0(), m.ReadoutY0(), m.fast, m.slow);
else if ((m.width == CONVERTED_MODULE_LINES) && (m.height == CONVERTED_MODULE_COLS))
ret.emplace_back(m.x0, m.y0, Direction::Ypos, Direction::Xpos);
else
return {};
}
if (ret.empty())
return {};
return DetectorGeometryModular(ret);
}
// The images the internal generator holds for this run: what the card memory and the firmware hold (not the broker's
// limit), at most what the run needs
int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExperiment &x, int64_t file_images) {
int64_t ret = std::min<int64_t>(file_images, x.GetImageNum());
ret = std::min<int64_t>(ret, FPGA_INTERNAL_GENERATOR_MAX_IMAGES);
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
// CheckInternalGeneratorFrame leaves the last module frame of the card memory free
const int64_t module_frames = std::min<int64_t>(aq_devices[i].GetInternalGeneratorModuleFrames() - 1,
FPGA_INTERNAL_GENERATOR_MAX_MODULE_FRAMES);
ret = std::min<int64_t>(ret, module_frames / x.GetModulesNum(i));
}
return ret;
}
// A stored dataset through the card and rugnux live stills; returns false where the run failed
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, bool verbose,
int64_t downsample, const std::optional<gemmi::SpaceGroup> &space_group, int64_t summation) {
RegisterHDF5Filter();
JFJochHDF5Reader reader;
reader.ReadFile(file);
const auto dataset = reader.GetDataset();
const auto &file_x = dataset->experiment;
// The modules as the file states them (module groups, else the gaps of its mask)
const auto geometry = GeometryFromModules(file_x.GetModules());
if (!geometry || (geometry->GetWidth(true) != file_x.GetXPixelsNum())
|| (geometry->GetHeight(true) != file_x.GetYPixelsNum())) {
logger.Error("The file does not state JUNGFRAU modules (1030 x 514) that make up its {} x {} image",
file_x.GetXPixelsNum(), file_x.GetYPixelsNum());
return false;
}
DiffractionExperiment x(DetJF(geometry.value()));
x.Modules(file_x.GetModules());
x.ImagesPerTrigger(nimages.value_or(reader.GetNumberOfImages())).NumTriggers(1).PedestalG0Frames(0)
.UseInternalPacketGenerator(true).DataStreams(nstreams);
// The card returns the frames as loaded; the host assembles them as signed 16-bit photon counts
x.JungfrauConvPhotonCnt(false).GeometryTransformation(true).PixelSigned(true).BitDepthImage(16);
// Summed: 32-bit images
if (summation > 1)
x.Summation(summation).BitDepthImage(32);
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
x.IncidentEnergy_keV(file_x.GetIncidentEnergy_keV()).DetectorDistance_mm(file_x.GetDetectorDistance_mm())
.BeamX_pxl(file_x.GetBeamX_pxl()).BeamY_pxl(file_x.GetBeamY_pxl());
x.SetUnitCell(cell ? cell : file_x.GetUnitCell());
// A space group stored in the file is ignored: only -S gives one, as the user gives it for a collection
if (space_group)
x.SetSpaceGroup(*space_group);
else if (file_x.GetGemmiSpaceGroup())
logger.Info("Space group {} stored in the file is ignored; -S gives the collection one",
file_x.GetGemmiSpaceGroup()->hm);
x.ImportIndexingSettings(indexing);
const int64_t generator_images = GeneratorImages(aq_devices, x, reader.GetNumberOfImages());
if (generator_images < 1) {
logger.Error("The card memory cannot hold one image of {} modules", x.GetModulesNum(0));
return false;
}
x.InternalPacketGeneratorImages(generator_images, FPGA_INTERNAL_GENERATOR_MAX_IMAGES);
PixelMask mask(x);
if (dataset->pixel_mask->GetMask().size() == x.GetPixelsNum())
mask.LoadUserMask(x, dataset->pixel_mask->GetMask());
logger.Info("Dataset {}: {} images of {} modules, {} distinct ones in the generator, {} in the run",
file, reader.GetNumberOfImages(), x.GetModulesNum(), generator_images, x.GetImageNum());
for (int i = 0; i < x.GetDataStreamsNum(); i++)
logger.Info(" card {}: modules {}-{}, generator memory {} module frames, {} used", i,
x.GetFirstModuleOfDataStream(i), x.GetFirstModuleOfDataStream(i) + x.GetModulesNum(i) - 1,
aq_devices[i].GetInternalGeneratorModuleFrames(), x.GetModulesNum(i) * generator_images);
for (int m = 0; m < x.GetModulesNum(); m++)
logger.Debug(" module {}: pixel 0 at ({}, {}), fast step {}, slow step {}", m,
x.GetPixel0OfModuleConv(m) % x.GetXPixelsNum(), x.GetPixel0OfModuleConv(m) / x.GetXPixelsNum(),
x.GetModuleFastDirectionStep(m), x.GetModuleSlowDirectionStep(m));
NonePusher pusher;
JFJochReceiverService service(aq_devices, logger, pusher, send_buffer_size_MiB);
service.NumThreads(nthreads);
service.Indexing(x.GetIndexingSettings());
service.RugnuxLive(true, nullptr, 4, 60, false, false, downsample);
// Error, gap and beam-stop pixels of the reader become error pixels of the module frame
std::vector<int16_t> frame(x.GetModulesNum() * RAW_MODULE_SIZE);
std::vector<uint16_t> raw_frame(x.GetModulesNum() * RAW_MODULE_SIZE);
for (int64_t i = 0; i < generator_images; i++) {
const auto image = reader.LoadImage(i);
SqueezeToRawGeometry<int16_t, int32_t>(x, frame.data(), image->Image().data(), BEAM_STOP_PXL_VALUE,
SATURATED_PXL_VALUE, INT16_MIN, INT16_MAX);
memcpy(raw_frame.data(), frame.data(), frame.size() * sizeof(int16_t));
service.LoadInternalGeneratorImage(x, raw_frame, i);
}
logger.Info("");
logger.Info("=== FPGAImagePuller, {} images ===", x.GetImageNum());
rusage usage_start{};
getrusage(RUSAGE_SELF, &usage_start);
const auto wall_start = std::chrono::steady_clock::now();
service.Start(x, mask, nullptr);
// -v: each card's progress, the frames of its fastest and slowest module written to the host
std::atomic<bool> running = true;
std::thread progress([&] {
while (running) {
std::this_thread::sleep_for(std::chrono::milliseconds(500));
std::string line;
for (int i = 0; i < x.GetDataStreamsNum(); i++)
line += fmt::format(" card {}: {} .. {}{}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
aq_devices[i].Counters().GetFastestFrameNumber(),
aq_devices[i].Counters().IsAcquisitionFinished() ? " done" : "");
// Where a run stands, so that one that stops shows which part waits
if (const auto st = service.GetStatus())
line += fmt::format(" receiver: collected {} sent {}", st->images_collected, st->images_sent);
const auto live = service.GetRugnuxStatus();
if (live.stills)
line += fmt::format(" stills: held {} analysed {} pre-pass {}", live.stills->images_held,
live.stills->images_analysed, static_cast<int>(live.stills->prepass));
if (verbose)
logger.Info("{:.1f} s{}", std::chrono::duration<double>(std::chrono::steady_clock::now()
- wall_start).count(), line);
}
});
const auto output = service.Stop();
running = false;
progress.join();
const auto live = service.GetRugnuxStatus();
const double wall_time = std::chrono::duration<double>(std::chrono::steady_clock::now() - wall_start).count();
rusage usage_end{};
getrusage(RUSAGE_SELF, &usage_end);
auto seconds = [](const timeval &t) { return static_cast<double>(t.tv_sec) + static_cast<double>(t.tv_usec) / 1e6; };
const double cpu_time = seconds(usage_end.ru_utime) - seconds(usage_start.ru_utime)
+ seconds(usage_end.ru_stime) - seconds(usage_start.ru_stime);
int64_t spots = 0, spot_images = 0;
for (const auto &e: output.scan_result.images)
if (e.spot_count) {
spots += e.spot_count.value();
spot_images++;
}
// From the first image to the last one into the receiver
const double delivery_s = live.last_image_s.value_or(0);
const double delivered_hz = (delivery_s > 0) ? static_cast<double>(output.status.images_collected - 1) / delivery_s : 0;
logger.Info("Images expected {} collected {} sent {} efficiency {:.4f}",
x.GetImageNum(), output.status.images_collected, output.status.images_sent, output.efficiency);
logger.Info("Delivered {:.1f} Hz (first to last image {:.2f} s)", delivered_hz, delivery_s);
logger.Info("Sustained {:.1f} Hz (images / wall time {:.2f} s, start to analysis done) CPU {:.1f} s = {:.1f} cores"
" max resident memory {:.1f} GiB",
static_cast<double>(output.status.images_collected) / wall_time, wall_time, cpu_time,
cpu_time / wall_time, static_cast<double>(usage_end.ru_maxrss) / (1024.0 * 1024.0));
// The card's own count of the cycles (200 MHz) it waited for a host buffer, i.e. was held by the host: the
// source blocked
for (int i = 0; i < x.GetDataStreamsNum(); i++) {
const auto status = aq_devices[i].GetDataCollectionStatus();
logger.Info("Card {}: waited for host buffers {:.2f} s ({:.1f}% of the delivery), stalls before HBM {} cycles",
i, status.pipeline_stalls_host / 200e6,
(delivery_s > 0) ? 100.0 * status.pipeline_stalls_host / 200e6 / delivery_s : 0.0,
status.pipeline_stalls_hbm);
}
logger.Info("Indexing rate {:.3f} spots per image {:.1f}", output.status.indexing_rate.value_or(0),
static_cast<double>(spots) / std::max<int64_t>(1, spot_images));
logger.Info("Per-image time (mean; microseconds): preprocess {:.0f} spot finding {:.0f} indexing {:.0f} "
"refinement {:.0f} integration {:.0f} total {:.0f}",
output.processing_time.preprocessing * 1e6, output.processing_time.spot_finding * 1e6,
output.processing_time.indexing * 1e6, output.processing_time.refinement * 1e6,
output.processing_time.integration * 1e6, output.processing_time.processing * 1e6);
if (live.stills) {
const auto &st = *live.stills;
logger.Info("Pre-pass {:.2f} s: beam {:.2f} {:.2f} px (file {:.2f} {:.2f}), distance {:.2f} mm (file {:.2f})",
st.prepass_s.value_or(0), st.beam_x_pxl.value_or(NAN), st.beam_y_pxl.value_or(NAN),
st.file_beam_x_pxl, st.file_beam_y_pxl, st.distance_mm.value_or(NAN), st.file_distance_mm);
logger.Info("Analysed {} ({:.3f} of the images) indexed {} analysis rate after the pre-pass {:.1f} Hz",
st.images_analysed, static_cast<double>(st.images_analysed) / x.GetImageNum(),
st.images_indexed, st.analysis_rate_hz.value_or(0));
}
logger.Info("Rugnux: {} {} last image at {:.2f} s, done {:.2f} s after it", live.verdict, live.message,
live.last_image_s.value_or(NAN), live.last_image_to_written_s.value_or(NAN));
for (const auto &step: live.steps)
logger.Info(" step {:.2f} s + {:.2f} s {}", step.started_s, step.duration_s.value_or(NAN), step.title);
return (output.efficiency > 0) && (live.state == RugnuxLiveState::Done);
}
int main(int argc, char **argv) {
print_license("jfjoch_fpga_stills_test");
Logger logger("jfjoch_fpga_stills_test");
logger.Verbose(true);
uint16_t nstreams = 1;
uint16_t nthreads = 64;
size_t send_buffer_size_MiB = 16384;
std::optional<size_t> nimages;
std::optional<UnitCell> cell;
IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto;
uint16_t indexing_threads = 8; // as offline rugnux's stills (Rugnux::StillsIndexerThreads)
std::optional<uint32_t> mock_card_MiB;
bool verbose = false;
int64_t downsample = 1;
int64_t summation = 1;
std::optional<gemmi::SpaceGroup> space_group;
int opt;
while ((opt = getopt(argc, argv, "vi:s:N:B:C:S:X:t:k:m:K::")) != -1) {
switch (opt) {
case 'v':
verbose = true;
break;
case 'i':
nimages = atol(optarg);
break;
case 's':
nstreams = atol(optarg);
break;
case 'N':
nthreads = atol(optarg);
break;
case 'B':
send_buffer_size_MiB = atol(optarg);
break;
case 'C': {
UnitCell c{};
if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) {
print_usage(logger);
exit(EXIT_FAILURE);
}
cell = c;
break;
}
case 'X':
if (std::string(optarg) == "fft" || std::string(optarg) == "FFT")
indexing = IndexingAlgorithmEnum::FFT;
else if (std::string(optarg) == "ffbidx" || std::string(optarg) == "FFBIDX")
indexing = IndexingAlgorithmEnum::FFBIDX;
else if (std::string(optarg) == "fftw" || std::string(optarg) == "FFTW")
indexing = IndexingAlgorithmEnum::FFTW;
else
indexing = IndexingAlgorithmEnum::Auto;
break;
case 't':
indexing_threads = atol(optarg);
break;
case 'k':
downsample = atol(optarg);
break;
case 'm':
summation = atol(optarg);
break;
case 'S': {
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name(optarg);
if (sg == nullptr) {
print_usage(logger);
exit(EXIT_FAILURE);
}
space_group = *sg;
break;
}
case 'K':
mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64;
break;
default:
print_usage(logger);
exit(EXIT_FAILURE);
}
}
if (optind != argc - 1) {
print_usage(logger);
exit(EXIT_FAILURE);
}
// -v gives the progress lines only: the debug log has a line for every module frame
logger.Verbose(false);
AcquisitionDeviceGroup aq_devices;
for (int i = 0; i < nstreams; i++) {
std::unique_ptr<AcquisitionDevice> device;
if (mock_card_MiB)
device = std::make_unique<MockAcquisitionDevice>(i, 128, mock_card_MiB.value() * 1024 * 1024);
else
device = std::make_unique<PCIExpressDevice>(i, "/dev/jfjoch" + std::to_string(i));
device->EnableLogging(&logger);
device->SetIPv4Address((i << 24) + 0x010a0a0a);
aq_devices.Add(std::move(device));
}
IndexingSettings i_settings;
i_settings.Algorithm(indexing);
i_settings.IndexingThreads(indexing_threads);
bool ret = false;
try {
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
i_settings, cell, verbose, downsample, space_group,
summation);
} catch (std::exception &e) {
logger.Error(e.what());
}
exit(ret ? EXIT_SUCCESS : EXIT_FAILURE);
}
+32 -5
View File
@@ -4,7 +4,8 @@
// Replays the frames of a stored serial collection through the broker's live analysis of serial
// frames (StillsGPUAnalysis): the pre-pass on the first images, then every frame analysed with the
// beam centre and mask it decided. Reports the pre-pass, the indexed fraction and the sustained rate, and
// writes the mask the frames were analysed with (<prefix>_mask.u32, uint32, row by row). With a model (-m), the
// writes the mask the frames were analysed with (<prefix>_mask.u32, uint32, row by row), and per frame its spot
// count, indexed spot count and outcome (<prefix>_images.tsv) and its spots (<prefix>_spots.tsv). With a model (-m), the
// model step of the merges as the broker makes it, and the files the end message would carry: <prefix>.mtz (the
// merge, in the model's frame), <prefix>_maps.mtz, <prefix>_model.cif, <prefix>_refined_model.cif,
// <prefix>_refined_maps.mtz.
@@ -26,6 +27,7 @@
#include "../image_analysis/indexing/IndexerThreadPool.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../receiver/StillsGPUAnalysis.h"
#include "../receiver/StillsSelection.h"
#include "../writer/HDF5Objects.h"
#include "../writer/RugnuxModelFile.h"
#include "../writer/WriteModel.h"
@@ -55,10 +57,12 @@ void print_usage() {
std::cout << " -C <a,b,c,alpha,beta,gamma> unit cell of the collection" << std::endl;
std::cout << " -F the broker's default fixed-threshold spot finder (default: adaptive, as the receiver)" << std::endl;
std::cout << " -c leave the file's unit cell out of the collection" << std::endl;
std::cout << " -S <group> space group of the collection (the merge's)" << std::endl;
std::cout << " -S <group> space group of the collection, the merge's (a space group stored in the file is ignored)" << std::endl;
std::cout << " -M no merge (StillsGPUAnalysis merge = false)" << std::endl;
std::cout << " -e <images> the first images given as empty frames (no diffraction)" << std::endl;
std::cout << " -m <file> model (PDB/mmCIF) placed and refined against the merges" << std::endl;
std::cout << " -p <num> spots kept per image (default: as rugnux live stills where none is given, 1000)" << std::endl;
std::cout << " -k <num> analyse 1 of every <num> images on average, as rugnux live stills_downsample" << std::endl;
std::cout << " -D <s> deadline of the model step after the last image (default 60, 0 = none)" << std::endl;
}
@@ -73,6 +77,7 @@ int main(int argc, char **argv) {
int nthreads = 16;
size_t buffer_MiB = 8192;
std::optional<float> beam_x, beam_y;
std::optional<int64_t> max_spots;
std::optional<UnitCell> cell;
bool no_cell = false;
bool fixed_threshold = false;
@@ -81,9 +86,10 @@ int main(int argc, char **argv) {
int64_t empty_frames = 0;
std::shared_ptr<const RugnuxModel> model;
double deadline_s = 60;
int64_t downsample = 1;
int opt;
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:")) != -1) {
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:k:")) != -1) {
switch (opt) {
case 'o': prefix = optarg; break;
case 's': first = atoll(optarg); break;
@@ -97,9 +103,11 @@ int main(int argc, char **argv) {
case 'F': fixed_threshold = true; break;
case 'S': space_group = optarg; break;
case 'M': merge = false; break;
case 'p': max_spots = atoll(optarg); break;
case 'e': empty_frames = atoll(optarg); break;
case 'm': model = std::make_shared<const RugnuxModel>(ReadRugnuxModelFile(optarg)); break;
case 'D': deadline_s = atof(optarg); break;
case 'k': downsample = atoll(optarg); break;
case 'C': {
UnitCell c{};
if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) {
@@ -139,8 +147,12 @@ int main(int argc, char **argv) {
}
if (cell)
x.SetUnitCell(cell);
if (space_group)
x.SetSpaceGroup(*gemmi::find_spacegroup_by_name(*space_group));
if (max_spots)
x.MaxSpotCount(*max_spots);
// A space group stored in the file is ignored: only -S gives one, as the user gives it for a collection
if (!space_group && x.GetGemmiSpaceGroup())
logger.Info("Space group {} stored in the file is ignored; -S gives the collection one", x.GetGemmiSpaceGroup()->hm);
x.SetSpaceGroup(space_group ? std::optional(*gemmi::find_spacegroup_by_name(*space_group)) : std::nullopt);
const PixelMask &mask = *dataset->pixel_mask;
StartMessage start;
@@ -175,6 +187,10 @@ int main(int argc, char **argv) {
std::unique_ptr<AzimuthalIntegrationProfile> run_profile;
RugnuxLiveStatus status; // nobody reads it here
ReplaySink sink;
std::mutex image_log_mutex;
std::ofstream image_log(prefix + "_images.tsv"), spot_log(prefix + "_spots.tsv");
image_log << "image\tspots\tindexed_spots\tindexed\n";
spot_log << "image\tx\ty\tintensity\tindexed\n";
sink.ready = [&](const StillsSetup &setup) {
t_ready = std::chrono::steady_clock::now();
run_profile = std::make_unique<AzimuthalIntegrationProfile>(*setup.mapping);
@@ -186,6 +202,15 @@ int main(int argc, char **argv) {
setup.experiment.GetBeamX_pxl(), setup.experiment.GetBeamY_pxl());
};
sink.analysed = [&](DataMessage &m, const AzimuthalIntegrationProfile &profile) {
{
// One line per frame and per spot, to compare frame by frame with an offline run's _process.h5
std::unique_lock ul(image_log_mutex);
image_log << m.number << '\t' << m.spots.size() << '\t' << m.spot_count_indexed.value_or(0) << '\t'
<< m.indexing_result.value_or(false) << '\n';
for (const auto &s: m.spots)
spot_log << m.number << '\t' << s.x << '\t' << s.y << '\t' << s.intensity << '\t' << s.indexed
<< '\n';
}
{
// As the receiver's plots do: the frames' profiles add up over the run.
std::unique_lock ul(run_profile_mutex);
@@ -208,6 +233,8 @@ int main(int argc, char **argv) {
JFJochReaderRawImage raw;
std::vector<uint8_t> empty;
for (int64_t i = 0; i < n; i++) {
if (!StillsImageSelected(i, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES))
continue;
if (rate > 0)
std::this_thread::sleep_until(t0 + std::chrono::duration<double>(static_cast<double>(i) / rate));
reader.ReadRawImage(first + i, raw);