From 17bfca6a22a45abcf7e4bb7eca1ce109e06f201d Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:57:22 +0100 Subject: [PATCH 1/2] Live stills: frames indexed on the pre-pass's refined cell and spot budget, as offline The pre-pass refines the cell beside the distance (their scales trade), and offline indexes every frame on that refined cell with up to 1000 spots. Live kept the stated cell at the refined distance (off in scale by as much as the distance moved, 1.7% on one 9M set) and the broker's 250-spot cap (a third short of a 9M frame's ~380 spots). Both now come from the pre-pass result; the start message keeps the stated cell, and the merge keeps it as the given cell. Replay on the first N frames (live before -> after, offline rugnux on the same frames): 9M 100 frames 0.99 -> 1.00 (offline 0.99), merge CC1/2 0.758 -> 0.798; 9M 30 frames 0.967 -> 1.00; 4M 100 frames 1.00 -> 1.00 (offline 1.00); 8M 200 frames 0.320 -> 0.325 (offline with the file's space group 0.320). Per-image analysis +10% on 9M (16.0 -> 17.6 ms, more spots). The 8M gap reported (live 0.35 vs offline 0.56) was the comparison: the file carries a space group, which live takes from it and offline only with -S. Offline with -S gives 0.349 = live; without it, offline indexes 13-14 more frames of 6-10 spots out of ~40, and merges worse (CC1/2 0.10 vs 0.20). jfjoch_stills_gpu_replay: -p spots per image; per-frame _images.tsv and _spots.tsv to compare frame by frame with an offline _process.h5. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/JFJochReceiverRugnuxStills.cpp | 2 ++ receiver/StillsGPUAnalysis.cpp | 9 ++++++++- receiver/StillsPrePass.cpp | 2 ++ receiver/StillsPrePass.h | 4 ++++ tools/jfjoch_stills_gpu_replay.cpp | 23 +++++++++++++++++++++-- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/receiver/JFJochReceiverRugnuxStills.cpp b/receiver/JFJochReceiverRugnuxStills.cpp index f14ac33ab..3f237f97c 100644 --- a/receiver/JFJochReceiverRugnuxStills.cpp +++ b/receiver/JFJochReceiverRugnuxStills.cpp @@ -161,6 +161,8 @@ void JFJochReceiverRugnuxStills::StillsReady(const StillsSetup &setup) { plots.Setup(setup.experiment, *setup.mapping); DiffractionExperiment written(setup.experiment); written.Goniometer(experiment.GetGoniometer()); + // The cell as stated: the one the frames are indexed on is the analysis's, and the merge reports its own + written.SetUnitCell(experiment.GetUnitCell()); SendStartMessage(MakeStartMessage(written, setup.mask, *setup.mapping)); } diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index 7fdc92baa..57f400846 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -220,6 +220,13 @@ void StillsGPUAnalysis::PrePassThread() { int geometry_frames = 0; if (result) { setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm); + // The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the refined + // distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off) + if (result->cell) + setup.experiment.SetUnitCell(*result->cell); + // and on as many spots as the pre-pass kept, as offline (the broker's default of 250 left a 9M frame of + // ~380 spots a third short of what its lattice is refined on) + setup.experiment.MaxSpotCount(result->max_spot_count); // The frames are integrated as the pre-pass integrated its own: its polarization and background ring setup.experiment.PolarizationFactor(result->polarization); setup.experiment.ImportBraggIntegrationSettings(result->integration); @@ -260,7 +267,7 @@ void StillsGPUAnalysis::PrePassThread() { merge_experiment = setup.experiment; merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false)); merge = std::make_unique(merge_experiment, experiment.GetImageNum(), - setup.experiment.GetUnitCell(), + experiment.GetUnitCell(), result ? result->run_width : std::nullopt, setup.experiment.IsDetectIceRings(), SpotFindingSettings().ice_ring_width_Q_recipA, logger); diff --git a/receiver/StillsPrePass.cpp b/receiver/StillsPrePass.cpp index eafea8ba1..5f5eb7bf2 100644 --- a/receiver/StillsPrePass.cpp +++ b/receiver/StillsPrePass.cpp @@ -61,6 +61,8 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map cell; + int64_t max_spot_count = 0; // the spots kept per frame, the strongest // What the pre-pass integrated its frames with, for the frames after it: the polarization, the integration // radii and background ring as the pre-scan measured them, and the run's rocking width with the prediction // window made from it (empty where the pre-pass could not measure it). diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp index ed9747660..c75647cd9 100644 --- a/tools/jfjoch_stills_gpu_replay.cpp +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -4,7 +4,8 @@ // Replays the frames of a stored serial collection through the broker's live analysis of serial // frames (StillsGPUAnalysis): the pre-pass on the first images, then every frame analysed with the // beam centre and mask it decided. Reports the pre-pass, the indexed fraction and the sustained rate, and -// writes the mask the frames were analysed with (_mask.u32, uint32, row by row). With a model (-m), the +// writes the mask the frames were analysed with (_mask.u32, uint32, row by row), and per frame its spot +// count, indexed spot count and outcome (_images.tsv) and its spots (_spots.tsv). With a model (-m), the // model step of the merges as the broker makes it, and the files the end message would carry: .mtz (the // merge, in the model's frame), _maps.mtz, _model.cif, _refined_model.cif, // _refined_maps.mtz. @@ -59,6 +60,7 @@ void print_usage() { std::cout << " -M no merge (StillsGPUAnalysis merge = false)" << std::endl; std::cout << " -e the first images given as empty frames (no diffraction)" << std::endl; std::cout << " -m model (PDB/mmCIF) placed and refined against the merges" << std::endl; + std::cout << " -p spots kept per image (default: the file's, as the broker's)" << std::endl; std::cout << " -D deadline of the model step after the last image (default 60, 0 = none)" << std::endl; } @@ -73,6 +75,7 @@ int main(int argc, char **argv) { int nthreads = 16; size_t buffer_MiB = 8192; std::optional beam_x, beam_y; + std::optional max_spots; std::optional cell; bool no_cell = false; bool fixed_threshold = false; @@ -83,7 +86,7 @@ int main(int argc, char **argv) { double deadline_s = 60; int opt; - while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:")) != -1) { + while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:")) != -1) { switch (opt) { case 'o': prefix = optarg; break; case 's': first = atoll(optarg); break; @@ -97,6 +100,7 @@ int main(int argc, char **argv) { case 'F': fixed_threshold = true; break; case 'S': space_group = optarg; break; case 'M': merge = false; break; + case 'p': max_spots = atoll(optarg); break; case 'e': empty_frames = atoll(optarg); break; case 'm': model = std::make_shared(ReadRugnuxModelFile(optarg)); break; case 'D': deadline_s = atof(optarg); break; @@ -139,6 +143,8 @@ int main(int argc, char **argv) { } if (cell) x.SetUnitCell(cell); + if (max_spots) + x.MaxSpotCount(*max_spots); if (space_group) x.SetSpaceGroup(*gemmi::find_spacegroup_by_name(*space_group)); const PixelMask &mask = *dataset->pixel_mask; @@ -175,6 +181,10 @@ int main(int argc, char **argv) { std::unique_ptr run_profile; RugnuxLiveStatus status; // nobody reads it here ReplaySink sink; + std::mutex image_log_mutex; + std::ofstream image_log(prefix + "_images.tsv"), spot_log(prefix + "_spots.tsv"); + image_log << "image\tspots\tindexed_spots\tindexed\n"; + spot_log << "image\tx\ty\tintensity\tindexed\n"; sink.ready = [&](const StillsSetup &setup) { t_ready = std::chrono::steady_clock::now(); run_profile = std::make_unique(*setup.mapping); @@ -186,6 +196,15 @@ int main(int argc, char **argv) { setup.experiment.GetBeamX_pxl(), setup.experiment.GetBeamY_pxl()); }; sink.analysed = [&](DataMessage &m, const AzimuthalIntegrationProfile &profile) { + { + // One line per frame and per spot, to compare frame by frame with an offline run's _process.h5 + std::unique_lock ul(image_log_mutex); + image_log << m.number << '\t' << m.spots.size() << '\t' << m.spot_count_indexed.value_or(0) << '\t' + << m.indexing_result.value_or(false) << '\n'; + for (const auto &s: m.spots) + spot_log << m.number << '\t' << s.x << '\t' << s.y << '\t' << s.intensity << '\t' << s.indexed + << '\n'; + } { // As the receiver's plots do: the frames' profiles add up over the run. std::unique_lock ul(run_profile_mutex); From 702b8d4a68c23f7acadf519b94ba27e9308d3193 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 11:49:09 +0100 Subject: [PATCH 2/2] Live stills: a frame with no free slot during the pre-pass waits as its compressed bytes Every frame that arrives before the pre-pass is over waits for the analysis, and each took a slot of the image buffer, sized for an uncompressed frame plus metadata (24 MB on a 9M detector for a frame of ~1.5 MB compressed). Not blocking, the frames past the buffer were dropped: on the mock, 9M, 2 cards, 1500 frames at ~140 Hz, a 13-17 s pre-pass, 683 of 1186-1400 collected were sent. A frame that finds no slot is now held as a copy of its compressed bytes, up to the image buffer's size again; a slot still comes first (the frame can be seen there while it waits), and the frame is sent from a fresh slot after its analysis as before. Same mock run: 1462-1489 of 1462-1489 collected sent (the rest are the mock puller's own module drops), peak RSS 27.1 -> 27.7 GB. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/JFJochReceiverRugnuxStills.cpp | 73 +++++++++++++++---------- receiver/JFJochReceiverRugnuxStills.h | 5 ++ receiver/StillsPrePass.h | 8 ++- tests/JFJochReceiverRugnuxTest.cpp | 5 +- 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/receiver/JFJochReceiverRugnuxStills.cpp b/receiver/JFJochReceiverRugnuxStills.cpp index 3f237f97c..b80066f07 100644 --- a/receiver/JFJochReceiverRugnuxStills.cpp +++ b/receiver/JFJochReceiverRugnuxStills.cpp @@ -48,6 +48,7 @@ JFJochReceiverRugnuxStills::~JFJochReceiverRugnuxStills() { void JFJochReceiverRugnuxStills::OnStart(std::chrono::steady_clock::time_point received) { const int64_t slots = image_buffer.GetStatus().total_slots; stills_keep_free = std::min(stills_nthreads, slots / 2); + copy_budget = slots * experiment.GetImageBufferLocationSize(); const StartMessage start = MakeStartMessage(experiment, pixel_mask, *az_int_mapping); // The frames are shown from their arrival, on the stated geometry until the pre-pass has decided its own. // To the image buffer only: the writer and the ZeroMQ streams get the one start message, after the pre-pass. @@ -82,43 +83,59 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std: CountImage(message); const size_t size = message.image.GetCompressedSize(); - // Waits for the analysis in a slot of the image buffer; a slot is kept free for each analysis thread to send with + // Waits for the analysis in a slot of the image buffer, where it can be seen while it waits (it is not sent from + // there); a slot is kept free for each analysis thread to send with. Without a slot, in a copy of its own. auto held = std::make_shared(); - held->slot = image_buffer.GetImageSlot(stills_keep_free); - while ((held->slot == nullptr) && image_puller.IsBlocking() && !cancelled) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + const auto hold = [&] { held->slot = image_buffer.GetImageSlot(stills_keep_free); + if (held->slot == nullptr) { + if (copy_bytes.fetch_add(static_cast(size)) + static_cast(size) <= copy_budget) { + held->copy.assign(message.image.GetCompressed(), message.image.GetCompressed() + size); + held->copy_bytes = ©_bytes; + } else + copy_bytes -= static_cast(size); + } + return held->slot != nullptr || held->copy_bytes != nullptr; + }; + bool held_ok = hold(); + while (!held_ok && image_puller.IsBlocking() && !cancelled) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + held_ok = hold(); } - if (held->slot == nullptr) { + if (!held_ok) { writer_queue_full = true; return; } held->message = message; StampImage(held->message, image_puller.GetCurrentFifoUtilization()); - // In the slot as an image without analysis, so that it can be seen while it waits; it is not sent. The - // analysis reads the image where it is in the slot. - auto slot_buffer = static_cast(held->slot->GetImage()); - CBORStream2Serializer serializer(slot_buffer, experiment.GetImageBufferLocationSize()); - held->message.image = CompressedImage(nullptr, 0, message.image.GetWidth(), message.image.GetHeight(), - message.image.GetMode(), message.image.GetCompressionAlgorithm(), - message.image.GetChannel()); - serializer.SerializeImage(held->message); - const size_t image_offset = serializer.GetImageAppendOffset(); - if (image_offset + size + 32 >= static_cast(experiment.GetImageBufferLocationSize())) { - logger.Error("Dropping image {} - metadata {} B + image {} B do not fit in buffer slot {} B", message.number, - image_offset, size, experiment.GetImageBufferLocationSize()); - return; - } - memcpy(slot_buffer + image_offset, message.image.GetCompressed(), size); - serializer.AppendImage(size); - held->message.image = CompressedImage(slot_buffer + image_offset, size, message.image.GetWidth(), - message.image.GetHeight(), message.image.GetMode(), - message.image.GetCompressionAlgorithm(), message.image.GetChannel()); - held->slot->SetImageNumber(message.number); - held->slot->SetImageSize(serializer.GetBufferSize()); - held->slot->SetIndexed(false); - held->slot->ReadyToSend(); + if (held->slot) { + // In the slot as an image without analysis; the analysis reads the image where it is in the slot. + auto slot_buffer = static_cast(held->slot->GetImage()); + CBORStream2Serializer serializer(slot_buffer, experiment.GetImageBufferLocationSize()); + held->message.image = CompressedImage(nullptr, 0, message.image.GetWidth(), message.image.GetHeight(), + message.image.GetMode(), message.image.GetCompressionAlgorithm(), + message.image.GetChannel()); + serializer.SerializeImage(held->message); + const size_t image_offset = serializer.GetImageAppendOffset(); + if (image_offset + size + 32 >= static_cast(experiment.GetImageBufferLocationSize())) { + logger.Error("Dropping image {} - metadata {} B + image {} B do not fit in buffer slot {} B", message.number, + image_offset, size, experiment.GetImageBufferLocationSize()); + return; + } + memcpy(slot_buffer + image_offset, message.image.GetCompressed(), size); + serializer.AppendImage(size); + held->message.image = CompressedImage(slot_buffer + image_offset, size, message.image.GetWidth(), + message.image.GetHeight(), message.image.GetMode(), + message.image.GetCompressionAlgorithm(), message.image.GetChannel()); + held->slot->SetImageNumber(message.number); + held->slot->SetImageSize(serializer.GetBufferSize()); + held->slot->SetIndexed(false); + held->slot->ReadyToSend(); + } else + held->message.image = CompressedImage(held->copy.data(), size, message.image.GetWidth(), + message.image.GetHeight(), message.image.GetMode(), + message.image.GetCompressionAlgorithm(), message.image.GetChannel()); stills->Hold(held); } diff --git a/receiver/JFJochReceiverRugnuxStills.h b/receiver/JFJochReceiverRugnuxStills.h index 1a4ab1b94..3f4d8ba4e 100644 --- a/receiver/JFJochReceiverRugnuxStills.h +++ b/receiver/JFJochReceiverRugnuxStills.h @@ -21,6 +21,11 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin IndexerThreadPool *const indexing_thread_pool; const int64_t stills_nthreads; int64_t stills_keep_free = 0; // slots frames do not take, for the analysis threads to send with + // Frames that find no free slot while they wait for the analysis are held as their compressed bytes, up to the + // image buffer's size again: the pre-pass holds every frame that arrives before it is over, and a frame is a + // fraction of a slot (1.5 MB of a 24 MB slot on a 9M detector) + int64_t copy_budget = 0; + std::atomic copy_bytes = 0; std::mutex stills_mutex; bool cut = false; diff --git a/receiver/StillsPrePass.h b/receiver/StillsPrePass.h index 33b564eb7..2a3765664 100644 --- a/receiver/StillsPrePass.h +++ b/receiver/StillsPrePass.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -15,10 +16,13 @@ #include "../image_analysis/IndexAndRefine.h" // StillsWindow // An image held until analysed, pointing into `slot`, which goes back unsent with the last copy. The slot is -// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds. +// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds. Where no slot +// was free, the image points into `copy` instead, its compressed bytes, counted in `copy_bytes` while held. struct HeldImage { DataMessage message; ZeroCopyReturnValue *slot = nullptr; + std::vector copy; + std::atomic *copy_bytes = nullptr; HeldImage() = default; HeldImage(const HeldImage &) = delete; @@ -26,6 +30,8 @@ struct HeldImage { ~HeldImage() { if (slot) slot->discard(); + if (copy_bytes) + *copy_bytes -= static_cast(copy.size()); } }; diff --git a/tests/JFJochReceiverRugnuxTest.cpp b/tests/JFJochReceiverRugnuxTest.cpp index 18789f6c5..167167439 100644 --- a/tests/JFJochReceiverRugnuxTest.cpp +++ b/tests/JFJochReceiverRugnuxTest.cpp @@ -663,10 +663,11 @@ TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceive CHECK(!out.status.cancelled); CHECK(pusher.Events() == std::vector{"start", "image", "end"}); - // A frame that finds no free slot is not written, and the collection says so. + // A frame that finds no free slot waits in a copy of its own, and is written. CHECK(out.status.images_collected == images); CHECK(out.efficiency == 1.0f); - CHECK(out.writer_queue_full_warning == (out.status.images_sent < images)); + CHECK(out.status.images_sent == images); + CHECK(!out.writer_queue_full_warning); CHECK(out.scan_result.images.size() == out.status.images_sent); REQUIRE(pusher.End()); CHECK(pusher.End()->images_collected_count == images);