From 17bfca6a22a45abcf7e4bb7eca1ce109e06f201d Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:57:22 +0100 Subject: [PATCH] Live stills: frames indexed on the pre-pass's refined cell and spot budget, as offline The pre-pass refines the cell beside the distance (their scales trade), and offline indexes every frame on that refined cell with up to 1000 spots. Live kept the stated cell at the refined distance (off in scale by as much as the distance moved, 1.7% on one 9M set) and the broker's 250-spot cap (a third short of a 9M frame's ~380 spots). Both now come from the pre-pass result; the start message keeps the stated cell, and the merge keeps it as the given cell. Replay on the first N frames (live before -> after, offline rugnux on the same frames): 9M 100 frames 0.99 -> 1.00 (offline 0.99), merge CC1/2 0.758 -> 0.798; 9M 30 frames 0.967 -> 1.00; 4M 100 frames 1.00 -> 1.00 (offline 1.00); 8M 200 frames 0.320 -> 0.325 (offline with the file's space group 0.320). Per-image analysis +10% on 9M (16.0 -> 17.6 ms, more spots). The 8M gap reported (live 0.35 vs offline 0.56) was the comparison: the file carries a space group, which live takes from it and offline only with -S. Offline with -S gives 0.349 = live; without it, offline indexes 13-14 more frames of 6-10 spots out of ~40, and merges worse (CC1/2 0.10 vs 0.20). jfjoch_stills_gpu_replay: -p spots per image; per-frame _images.tsv and _spots.tsv to compare frame by frame with an offline _process.h5. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/JFJochReceiverRugnuxStills.cpp | 2 ++ receiver/StillsGPUAnalysis.cpp | 9 ++++++++- receiver/StillsPrePass.cpp | 2 ++ receiver/StillsPrePass.h | 4 ++++ tools/jfjoch_stills_gpu_replay.cpp | 23 +++++++++++++++++++++-- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/receiver/JFJochReceiverRugnuxStills.cpp b/receiver/JFJochReceiverRugnuxStills.cpp index f14ac33ab..3f237f97c 100644 --- a/receiver/JFJochReceiverRugnuxStills.cpp +++ b/receiver/JFJochReceiverRugnuxStills.cpp @@ -161,6 +161,8 @@ void JFJochReceiverRugnuxStills::StillsReady(const StillsSetup &setup) { plots.Setup(setup.experiment, *setup.mapping); DiffractionExperiment written(setup.experiment); written.Goniometer(experiment.GetGoniometer()); + // The cell as stated: the one the frames are indexed on is the analysis's, and the merge reports its own + written.SetUnitCell(experiment.GetUnitCell()); SendStartMessage(MakeStartMessage(written, setup.mask, *setup.mapping)); } diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index 7fdc92baa..57f400846 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -220,6 +220,13 @@ void StillsGPUAnalysis::PrePassThread() { int geometry_frames = 0; if (result) { setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm); + // The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the refined + // distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off) + if (result->cell) + setup.experiment.SetUnitCell(*result->cell); + // and on as many spots as the pre-pass kept, as offline (the broker's default of 250 left a 9M frame of + // ~380 spots a third short of what its lattice is refined on) + setup.experiment.MaxSpotCount(result->max_spot_count); // The frames are integrated as the pre-pass integrated its own: its polarization and background ring setup.experiment.PolarizationFactor(result->polarization); setup.experiment.ImportBraggIntegrationSettings(result->integration); @@ -260,7 +267,7 @@ void StillsGPUAnalysis::PrePassThread() { merge_experiment = setup.experiment; merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false)); merge = std::make_unique(merge_experiment, experiment.GetImageNum(), - setup.experiment.GetUnitCell(), + experiment.GetUnitCell(), result ? result->run_width : std::nullopt, setup.experiment.IsDetectIceRings(), SpotFindingSettings().ice_ring_width_Q_recipA, logger); diff --git a/receiver/StillsPrePass.cpp b/receiver/StillsPrePass.cpp index eafea8ba1..5f5eb7bf2 100644 --- a/receiver/StillsPrePass.cpp +++ b/receiver/StillsPrePass.cpp @@ -61,6 +61,8 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map cell; + int64_t max_spot_count = 0; // the spots kept per frame, the strongest // What the pre-pass integrated its frames with, for the frames after it: the polarization, the integration // radii and background ring as the pre-scan measured them, and the run's rocking width with the prediction // window made from it (empty where the pre-pass could not measure it). diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp index ed9747660..c75647cd9 100644 --- a/tools/jfjoch_stills_gpu_replay.cpp +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -4,7 +4,8 @@ // Replays the frames of a stored serial collection through the broker's live analysis of serial // frames (StillsGPUAnalysis): the pre-pass on the first images, then every frame analysed with the // beam centre and mask it decided. Reports the pre-pass, the indexed fraction and the sustained rate, and -// writes the mask the frames were analysed with (_mask.u32, uint32, row by row). With a model (-m), the +// writes the mask the frames were analysed with (_mask.u32, uint32, row by row), and per frame its spot +// count, indexed spot count and outcome (_images.tsv) and its spots (_spots.tsv). With a model (-m), the // model step of the merges as the broker makes it, and the files the end message would carry: .mtz (the // merge, in the model's frame), _maps.mtz, _model.cif, _refined_model.cif, // _refined_maps.mtz. @@ -59,6 +60,7 @@ void print_usage() { std::cout << " -M no merge (StillsGPUAnalysis merge = false)" << std::endl; std::cout << " -e the first images given as empty frames (no diffraction)" << std::endl; std::cout << " -m model (PDB/mmCIF) placed and refined against the merges" << std::endl; + std::cout << " -p spots kept per image (default: the file's, as the broker's)" << std::endl; std::cout << " -D deadline of the model step after the last image (default 60, 0 = none)" << std::endl; } @@ -73,6 +75,7 @@ int main(int argc, char **argv) { int nthreads = 16; size_t buffer_MiB = 8192; std::optional beam_x, beam_y; + std::optional max_spots; std::optional cell; bool no_cell = false; bool fixed_threshold = false; @@ -83,7 +86,7 @@ int main(int argc, char **argv) { double deadline_s = 60; int opt; - while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:")) != -1) { + while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:")) != -1) { switch (opt) { case 'o': prefix = optarg; break; case 's': first = atoll(optarg); break; @@ -97,6 +100,7 @@ int main(int argc, char **argv) { case 'F': fixed_threshold = true; break; case 'S': space_group = optarg; break; case 'M': merge = false; break; + case 'p': max_spots = atoll(optarg); break; case 'e': empty_frames = atoll(optarg); break; case 'm': model = std::make_shared(ReadRugnuxModelFile(optarg)); break; case 'D': deadline_s = atof(optarg); break; @@ -139,6 +143,8 @@ int main(int argc, char **argv) { } if (cell) x.SetUnitCell(cell); + if (max_spots) + x.MaxSpotCount(*max_spots); if (space_group) x.SetSpaceGroup(*gemmi::find_spacegroup_by_name(*space_group)); const PixelMask &mask = *dataset->pixel_mask; @@ -175,6 +181,10 @@ int main(int argc, char **argv) { std::unique_ptr run_profile; RugnuxLiveStatus status; // nobody reads it here ReplaySink sink; + std::mutex image_log_mutex; + std::ofstream image_log(prefix + "_images.tsv"), spot_log(prefix + "_spots.tsv"); + image_log << "image\tspots\tindexed_spots\tindexed\n"; + spot_log << "image\tx\ty\tintensity\tindexed\n"; sink.ready = [&](const StillsSetup &setup) { t_ready = std::chrono::steady_clock::now(); run_profile = std::make_unique(*setup.mapping); @@ -186,6 +196,15 @@ int main(int argc, char **argv) { setup.experiment.GetBeamX_pxl(), setup.experiment.GetBeamY_pxl()); }; sink.analysed = [&](DataMessage &m, const AzimuthalIntegrationProfile &profile) { + { + // One line per frame and per spot, to compare frame by frame with an offline run's _process.h5 + std::unique_lock ul(image_log_mutex); + image_log << m.number << '\t' << m.spots.size() << '\t' << m.spot_count_indexed.value_or(0) << '\t' + << m.indexing_result.value_or(false) << '\n'; + for (const auto &s: m.spots) + spot_log << m.number << '\t' << s.x << '\t' << s.y << '\t' << s.intensity << '\t' << s.indexed + << '\n'; + } { // As the receiver's plots do: the frames' profiles add up over the run. std::unique_lock ul(run_profile_mutex);