Merge branch 'rc188-l' into rc188-i

This commit is contained in:
leonarski_f committed 2026-11-18 12:09:04 +01:00
commit 4e84c94e19
7 files changed
+97 -34

No files matched your search

+47 -28
View File
@@ -48,6 +48,7 @@ JFJochReceiverRugnuxStills::~JFJochReceiverRugnuxStills() {
void JFJochReceiverRugnuxStills::OnStart(std::chrono::steady_clock::time_point received) {
const int64_t slots = image_buffer.GetStatus().total_slots;
stills_keep_free = std::min(stills_nthreads, slots / 2);
copy_budget = slots * experiment.GetImageBufferLocationSize();
const StartMessage start = MakeStartMessage(experiment, pixel_mask, *az_int_mapping);
// The frames are shown from their arrival, on the stated geometry until the pre-pass has decided its own.
// To the image buffer only: the writer and the ZeroMQ streams get the one start message, after the pre-pass.
@@ -82,43 +83,59 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
CountImage(message);
const size_t size = message.image.GetCompressedSize();
// Waits for the analysis in a slot of the image buffer; a slot is kept free for each analysis thread to send with
// Waits for the analysis in a slot of the image buffer, where it can be seen while it waits (it is not sent from
// there); a slot is kept free for each analysis thread to send with. Without a slot, in a copy of its own.
auto held = std::make_shared<HeldImage>();
held->slot = image_buffer.GetImageSlot(stills_keep_free);
while ((held->slot == nullptr) && image_puller.IsBlocking() && !cancelled) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
const auto hold = [&] {
held->slot = image_buffer.GetImageSlot(stills_keep_free);
if (held->slot == nullptr) {
if (copy_bytes.fetch_add(static_cast<int64_t>(size)) + static_cast<int64_t>(size) <= copy_budget) {
held->copy.assign(message.image.GetCompressed(), message.image.GetCompressed() + size);
held->copy_bytes = &copy_bytes;
} else
copy_bytes -= static_cast<int64_t>(size);
}
return held->slot != nullptr || held->copy_bytes != nullptr;
};
bool held_ok = hold();
while (!held_ok && image_puller.IsBlocking() && !cancelled) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
held_ok = hold();
}
if (held->slot == nullptr) {
if (!held_ok) {
writer_queue_full = true;
return;
}
held->message = message;
StampImage(held->message, image_puller.GetCurrentFifoUtilization());
// In the slot as an image without analysis, so that it can be seen while it waits; it is not sent. The
// analysis reads the image where it is in the slot.
auto slot_buffer = static_cast<uint8_t *>(held->slot->GetImage());
CBORStream2Serializer serializer(slot_buffer, experiment.GetImageBufferLocationSize());
held->message.image = CompressedImage(nullptr, 0, message.image.GetWidth(), message.image.GetHeight(),
message.image.GetMode(), message.image.GetCompressionAlgorithm(),
message.image.GetChannel());
serializer.SerializeImage(held->message);
const size_t image_offset = serializer.GetImageAppendOffset();
if (image_offset + size + 32 >= static_cast<size_t>(experiment.GetImageBufferLocationSize())) {
logger.Error("Dropping image {} - metadata {} B + image {} B do not fit in buffer slot {} B", message.number,
image_offset, size, experiment.GetImageBufferLocationSize());
return;
}
memcpy(slot_buffer + image_offset, message.image.GetCompressed(), size);
serializer.AppendImage(size);
held->message.image = CompressedImage(slot_buffer + image_offset, size, message.image.GetWidth(),
message.image.GetHeight(), message.image.GetMode(),
message.image.GetCompressionAlgorithm(), message.image.GetChannel());
held->slot->SetImageNumber(message.number);
held->slot->SetImageSize(serializer.GetBufferSize());
held->slot->SetIndexed(false);
held->slot->ReadyToSend();
if (held->slot) {
// In the slot as an image without analysis; the analysis reads the image where it is in the slot.
auto slot_buffer = static_cast<uint8_t *>(held->slot->GetImage());
CBORStream2Serializer serializer(slot_buffer, experiment.GetImageBufferLocationSize());
held->message.image = CompressedImage(nullptr, 0, message.image.GetWidth(), message.image.GetHeight(),
message.image.GetMode(), message.image.GetCompressionAlgorithm(),
message.image.GetChannel());
serializer.SerializeImage(held->message);
const size_t image_offset = serializer.GetImageAppendOffset();
if (image_offset + size + 32 >= static_cast<size_t>(experiment.GetImageBufferLocationSize())) {
logger.Error("Dropping image {} - metadata {} B + image {} B do not fit in buffer slot {} B", message.number,
image_offset, size, experiment.GetImageBufferLocationSize());
return;
}
memcpy(slot_buffer + image_offset, message.image.GetCompressed(), size);
serializer.AppendImage(size);
held->message.image = CompressedImage(slot_buffer + image_offset, size, message.image.GetWidth(),
message.image.GetHeight(), message.image.GetMode(),
message.image.GetCompressionAlgorithm(), message.image.GetChannel());
held->slot->SetImageNumber(message.number);
held->slot->SetImageSize(serializer.GetBufferSize());
held->slot->SetIndexed(false);
held->slot->ReadyToSend();
} else
held->message.image = CompressedImage(held->copy.data(), size, message.image.GetWidth(),
message.image.GetHeight(), message.image.GetMode(),
message.image.GetCompressionAlgorithm(), message.image.GetChannel());
stills->Hold(held);
}
@@ -161,6 +178,8 @@ void JFJochReceiverRugnuxStills::StillsReady(const StillsSetup &setup) {
plots.Setup(setup.experiment, *setup.mapping);
DiffractionExperiment written(setup.experiment);
written.Goniometer(experiment.GetGoniometer());
// The cell as stated: the one the frames are indexed on is the analysis's, and the merge reports its own
written.SetUnitCell(experiment.GetUnitCell());
SendStartMessage(MakeStartMessage(written, setup.mask, *setup.mapping));
}
+5
View File
@@ -21,6 +21,11 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin
IndexerThreadPool *const indexing_thread_pool;
const int64_t stills_nthreads;
int64_t stills_keep_free = 0; // slots frames do not take, for the analysis threads to send with
// Frames that find no free slot while they wait for the analysis are held as their compressed bytes, up to the
// image buffer's size again: the pre-pass holds every frame that arrives before it is over, and a frame is a
// fraction of a slot (1.5 MB of a 24 MB slot on a 9M detector)
int64_t copy_budget = 0;
std::atomic<int64_t> copy_bytes = 0;
std::mutex stills_mutex;
bool cut = false;
+8 -1
View File
@@ -220,6 +220,13 @@ void StillsGPUAnalysis::PrePassThread() {
int geometry_frames = 0;
if (result) {
setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm);
// The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the refined
// distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off)
if (result->cell)
setup.experiment.SetUnitCell(*result->cell);
// and on as many spots as the pre-pass kept, as offline (the broker's default of 250 left a 9M frame of
// ~380 spots a third short of what its lattice is refined on)
setup.experiment.MaxSpotCount(result->max_spot_count);
// The frames are integrated as the pre-pass integrated its own: its polarization and background ring
setup.experiment.PolarizationFactor(result->polarization);
setup.experiment.ImportBraggIntegrationSettings(result->integration);
@@ -260,7 +267,7 @@ void StillsGPUAnalysis::PrePassThread() {
merge_experiment = setup.experiment;
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
setup.experiment.GetUnitCell(),
experiment.GetUnitCell(),
result ? result->run_width : std::nullopt,
setup.experiment.IsDetectIceRings(),
SpotFindingSettings().ice_ring_width_Q_recipA, logger);
+2
View File
@@ -61,6 +61,8 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<i
.mask = rugnux.GetPixelMask(),
.images = images.size(),
.geometry_frames = result.stills_geometry_frames,
.cell = rugnux.GetExperiment().GetUnitCell(),
.max_spot_count = rugnux.GetExperiment().GetMaxSpotCount(),
.polarization = rugnux.GetExperiment().GetPolarizationFactor(),
.integration = rugnux.GetExperiment().GetBraggIntegrationSettings(),
.run_width = rugnux.GetStillsRunWidth(),
+11 -1
View File
@@ -3,6 +3,7 @@
#pragma once
#include <atomic>
#include <map>
#include <memory>
#include <vector>
@@ -15,10 +16,13 @@
#include "../image_analysis/IndexAndRefine.h" // StillsWindow
// An image held until analysed, pointing into `slot`, which goes back unsent with the last copy. The slot is
// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds.
// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds. Where no slot
// was free, the image points into `copy` instead, its compressed bytes, counted in `copy_bytes` while held.
struct HeldImage {
DataMessage message;
ZeroCopyReturnValue *slot = nullptr;
std::vector<uint8_t> copy;
std::atomic<int64_t> *copy_bytes = nullptr;
HeldImage() = default;
HeldImage(const HeldImage &) = delete;
@@ -26,6 +30,8 @@ struct HeldImage {
~HeldImage() {
if (slot)
slot->discard();
if (copy_bytes)
*copy_bytes -= static_cast<int64_t>(copy.size());
}
};
@@ -36,6 +42,10 @@ struct StillsPrePassResult {
PixelMask mask;
size_t images = 0;
int geometry_frames = 0; // the indexed frames the distance and centre were refined on; 0: kept as stated
// The cell the pre-pass indexed its frames on: the one stated, or found de novo, as refined beside the distance
// (its scale trades with the distance, so the two go together)
std::optional<UnitCell> cell;
int64_t max_spot_count = 0; // the spots kept per frame, the strongest
// What the pre-pass integrated its frames with, for the frames after it: the polarization, the integration
// radii and background ring as the pre-scan measured them, and the run's rocking width with the prediction
// window made from it (empty where the pre-pass could not measure it).
+3 -2
View File
@@ -663,10 +663,11 @@ TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceive
CHECK(!out.status.cancelled);
CHECK(pusher.Events() == std::vector<std::string>{"start", "image", "end"});
// A frame that finds no free slot is not written, and the collection says so.
// A frame that finds no free slot waits in a copy of its own, and is written.
CHECK(out.status.images_collected == images);
CHECK(out.efficiency == 1.0f);
CHECK(out.writer_queue_full_warning == (out.status.images_sent < images));
CHECK(out.status.images_sent == images);
CHECK(!out.writer_queue_full_warning);
CHECK(out.scan_result.images.size() == out.status.images_sent);
REQUIRE(pusher.End());
CHECK(pusher.End()->images_collected_count == images);
+21 -2
View File
@@ -4,7 +4,8 @@
// Replays the frames of a stored serial collection through the broker's live analysis of serial
// frames (StillsGPUAnalysis): the pre-pass on the first images, then every frame analysed with the
// beam centre and mask it decided. Reports the pre-pass, the indexed fraction and the sustained rate, and
// writes the mask the frames were analysed with (<prefix>_mask.u32, uint32, row by row). With a model (-m), the
// writes the mask the frames were analysed with (<prefix>_mask.u32, uint32, row by row), and per frame its spot
// count, indexed spot count and outcome (<prefix>_images.tsv) and its spots (<prefix>_spots.tsv). With a model (-m), the
// model step of the merges as the broker makes it, and the files the end message would carry: <prefix>.mtz (the
// merge, in the model's frame), <prefix>_maps.mtz, <prefix>_model.cif, <prefix>_refined_model.cif,
// <prefix>_refined_maps.mtz.
@@ -59,6 +60,7 @@ void print_usage() {
std::cout << " -M no merge (StillsGPUAnalysis merge = false)" << std::endl;
std::cout << " -e <images> the first images given as empty frames (no diffraction)" << std::endl;
std::cout << " -m <file> model (PDB/mmCIF) placed and refined against the merges" << std::endl;
std::cout << " -p <num> spots kept per image (default: the file's, as the broker's)" << std::endl;
std::cout << " -D <s> deadline of the model step after the last image (default 60, 0 = none)" << std::endl;
}
@@ -73,6 +75,7 @@ int main(int argc, char **argv) {
int nthreads = 16;
size_t buffer_MiB = 8192;
std::optional<float> beam_x, beam_y;
std::optional<int64_t> max_spots;
std::optional<UnitCell> cell;
bool no_cell = false;
bool fixed_threshold = false;
@@ -83,7 +86,7 @@ int main(int argc, char **argv) {
double deadline_s = 60;
int opt;
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:")) != -1) {
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:")) != -1) {
switch (opt) {
case 'o': prefix = optarg; break;
case 's': first = atoll(optarg); break;
@@ -97,6 +100,7 @@ int main(int argc, char **argv) {
case 'F': fixed_threshold = true; break;
case 'S': space_group = optarg; break;
case 'M': merge = false; break;
case 'p': max_spots = atoll(optarg); break;
case 'e': empty_frames = atoll(optarg); break;
case 'm': model = std::make_shared<const RugnuxModel>(ReadRugnuxModelFile(optarg)); break;
case 'D': deadline_s = atof(optarg); break;
@@ -139,6 +143,8 @@ int main(int argc, char **argv) {
}
if (cell)
x.SetUnitCell(cell);
if (max_spots)
x.MaxSpotCount(*max_spots);
if (space_group)
x.SetSpaceGroup(*gemmi::find_spacegroup_by_name(*space_group));
const PixelMask &mask = *dataset->pixel_mask;
@@ -175,6 +181,10 @@ int main(int argc, char **argv) {
std::unique_ptr<AzimuthalIntegrationProfile> run_profile;
RugnuxLiveStatus status; // nobody reads it here
ReplaySink sink;
std::mutex image_log_mutex;
std::ofstream image_log(prefix + "_images.tsv"), spot_log(prefix + "_spots.tsv");
image_log << "image\tspots\tindexed_spots\tindexed\n";
spot_log << "image\tx\ty\tintensity\tindexed\n";
sink.ready = [&](const StillsSetup &setup) {
t_ready = std::chrono::steady_clock::now();
run_profile = std::make_unique<AzimuthalIntegrationProfile>(*setup.mapping);
@@ -186,6 +196,15 @@ int main(int argc, char **argv) {
setup.experiment.GetBeamX_pxl(), setup.experiment.GetBeamY_pxl());
};
sink.analysed = [&](DataMessage &m, const AzimuthalIntegrationProfile &profile) {
{
// One line per frame and per spot, to compare frame by frame with an offline run's _process.h5
std::unique_lock ul(image_log_mutex);
image_log << m.number << '\t' << m.spots.size() << '\t' << m.spot_count_indexed.value_or(0) << '\t'
<< m.indexing_result.value_or(false) << '\n';
for (const auto &s: m.spots)
spot_log << m.number << '\t' << s.x << '\t' << s.y << '\t' << s.intensity << '\t' << s.indexed
<< '\n';
}
{
// As the receiver's plots do: the frames' profiles add up over the run.
std::unique_lock ul(run_profile_mutex);