diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index cb5e75c79..a7a9c143c 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -266,6 +266,16 @@ all: on fewer, too few index for the geometry refinement to run, and the stated buffer has to hold them, and the frames that arrive while the pre-pass runs, in half its slots (for a JUNGFRAU 8M, a slot is the size of an uncompressed frame); a smaller buffer starts the pre-pass on as many as it holds. +Even 500 frames give the pre-pass only a few dozen indexed ones at a serial hit rate, and none where the +first frames are empty (a fixed-target chip whose crystals sit further in). So, where a cell is given and the +pre-pass refined the geometry on fewer than 200 frames, the centre and distance are refined again on the frames +as they are analysed, from their spots and lattices, without holding their images: once 200 strong frames (20 +indexed spots or more) are in, or 5000 indexed, the geometry is fitted, as rugnux fits it on its sample, and the +analysis switches to it; a second round does the same on frames analysed at that geometry. Two fits in all, a +fixed cost. The frames analysed before the last switch are written as analysed but left out of the merge (their +reflections were predicted at the earlier geometry, and the images are no longer there to integrate again); +the switches are notes of the live status, and the end message carries the geometry the run ended at. + Frames arriving during the pre-pass, or faster than the analysis, wait in the image buffer, in at most half of its slots; a frame that finds no free slot is dropped and counted as for a full image buffer, so analysis slower than the frame rate costs frames, not memory. A pre-pass that fails leaves the stated geometry and diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 7dc673a16..2bbfcb9ad 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -1128,6 +1128,12 @@ IntegrationOutcome IndexAndRefine::TakeIntegrationOutcome(int64_t image) { return std::move(integration_outcome.at(image)); } +void IndexAndRefine::DropIntegrationOutcome(int64_t image) { + const std::unique_lock ul(reflections_mutex); + integration_outcome.at(image) = {}; + unit_cells.at(image).reset(); +} + std::vector &IndexAndRefine::GetIntegrationOutcome() { return integration_outcome; } diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index f90c3267b..0467e31ad 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -309,5 +309,7 @@ public: // The retained outcome of one image, moved out: what is left in its place is empty. Safe beside the // analysis of other images. IntegrationOutcome TakeIntegrationOutcome(int64_t image); + // The retained outcome of one image dropped, and its cell with it: it is not in the consensus. + void DropIntegrationOutcome(int64_t image); const std::vector &GetIntegrationOutcome() const; }; diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index 6eb10b893..c45d603dc 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -157,6 +157,7 @@ void StillsGPUAnalysis::PrePassThread() { if (status) Publish(); bool failed = sample.empty(); + int geometry_frames = 0; const auto prepass_start = std::chrono::steady_clock::now(); if (!sample.empty()) { try { @@ -164,6 +165,7 @@ void StillsGPUAnalysis::PrePassThread() { const auto r = RunStillsPrePass(start, sample, static_cast(std::thread::hardware_concurrency())); setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl).DetectorDistance_mm(r.distance_mm); setup.mask = r.mask; + geometry_frames = r.geometry_frames; const auto &m = r.mask.GetMask(); std::unique_lock ul(live_mutex); live.beam_x_pxl = r.beam_x_pxl; @@ -192,6 +194,16 @@ void StillsGPUAnalysis::PrePassThread() { setup.mapping = std::make_unique(setup.experiment, setup.mask); // Retained for the merge, which takes each image's reflections as soon as it is analysed. indexer = std::make_unique(setup.experiment, indexing_thread_pool, merge_frames, true); + // The pre-pass's fit rests on the frames of its sample that indexed; where those are fewer than the + // running refinement bundles, the geometry is refined again, from where the pre-pass left it, on the + // frames as they are analysed. + if (geometry_frames < StillsRunningGeometry::STRONG_FRAMES && setup.experiment.GetUnitCell()) { + running_geometry = std::make_unique(setup.experiment, MERGE_THREADS); + frame_epoch.assign(experiment.GetImageNum(), 0); + geometry_state = GEOMETRY_PENDING; + logger.Info("The pre-pass refined the geometry on {} frames: it is refined again on {} strong frames as " + "they are analysed", geometry_frames, StillsRunningGeometry::STRONG_FRAMES); + } if (merge_frames) { merge_experiment = setup.experiment; merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false)); @@ -242,33 +254,56 @@ void StillsGPUAnalysis::AnalysisThread() { return; } - // A thread whose analysis cannot be built still passes its frames on, unanalysed. + // A thread whose analysis cannot be built still passes its frames on, unanalysed. It is built again at + // the geometry the running refinement switches to. std::unique_ptr analysis; - try { - analysis = std::make_unique(setup.experiment, *setup.mapping, setup.mask, *indexer, - /*enable_fused_adaptive_gpu=*/true); - } catch (const std::exception &e) { - AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ") - + e.what()); - } + int epoch = -1; + const auto build = [&] { + epoch = geometry_epoch; + analysis.reset(); + try { + analysis = std::make_unique(setup.experiment, *setup.mapping, setup.mask, + *indexer, /*enable_fused_adaptive_gpu=*/true); + } catch (const std::exception &e) { + AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ") + + e.what()); + } + }; while (auto held = queue.GetBlocking()) { DataMessage &message = held->message; - AzimuthalIntegrationProfile profile(*setup.mapping); + std::optional profile; + bool switch_geometry = false; const auto start_time = std::chrono::steady_clock::now(); - try { - if (analysis) - analysis->Analyze(message, profile, settings()); - } catch (const JFJochException &e) { - // The frame is still written, without its analysis. - logger.ErrorException(e); - } catch (const std::exception &e) { - AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what())); + { + std::unique_lock gate(gate_mutex); + gate_cv.wait(gate, [this] { return !switch_waiting; }); + } + { + std::shared_lock sl(geometry_mutex); + if (epoch != geometry_epoch) + build(); + profile.emplace(*setup.mapping); + try { + if (analysis) + analysis->Analyze(message, *profile, settings()); + } catch (const JFJochException &e) { + // The frame is still written, without its analysis. + logger.ErrorException(e); + } catch (const std::exception &e) { + AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what())); + } + if (geometry_state == GEOMETRY_PENDING) { + frame_epoch.at(message.number) = static_cast(epoch + 1); + switch_geometry = running_geometry->Add(message); + } } message.processing_time_s = std::chrono::duration(std::chrono::steady_clock::now() - start_time).count(); + if (switch_geometry) + SwitchGeometry(); try { - sink(message, profile); + sink(message, *profile); if (merge) merge_queue.Put(message.number); if (status) { @@ -289,6 +324,56 @@ void StillsGPUAnalysis::AnalysisThread() { } } +void StillsGPUAnalysis::SwitchGeometry() { + // One fit, in the analysis thread that offered the last frame it needed; the others go on meanwhile. + const auto t0 = std::chrono::steady_clock::now(); + const auto r = running_geometry->Refine(); + const bool last_round = ++geometry_round == GEOMETRY_ROUNDS; + if (!r) { + logger.Warning("Running geometry refinement, round {}: no fit on {} frames ({} strong); the frames are " + "analysed at the geometry they were", geometry_round, running_geometry->Offered(), + running_geometry->Strong()); + geometry_state = geometry_epoch > 0 ? GEOMETRY_SWITCHED : GEOMETRY_KEPT; + return; + } + const std::string text = fmt::format( + "Geometry refined on {} analysed frames ({} strong), round {}: beam centre ({:.2f}, {:.2f}) -> " + "({:.2f}, {:.2f}) px, distance {:.2f} -> {:.2f} mm ({:.1f} s)", running_geometry->Offered(), + running_geometry->Strong(), geometry_round, setup.experiment.GetBeamX_pxl(), + setup.experiment.GetBeamY_pxl(), r->beam_x_px, r->beam_y_px, setup.experiment.GetDetectorDistance_mm(), + r->distance_mm, std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); + { + std::unique_lock gate(gate_mutex); + switch_waiting = true; + } + { + std::unique_lock ul(geometry_mutex); + setup.experiment.BeamX_pxl(static_cast(r->beam_x_px)).BeamY_pxl(static_cast(r->beam_y_px)) + .DetectorDistance_mm(static_cast(r->distance_mm)); + retired_mappings.push_back(std::move(setup.mapping)); + setup.mapping = std::make_unique(setup.experiment, setup.mask); + ++geometry_epoch; + if (last_round) + geometry_state = GEOMETRY_SWITCHED; + else + running_geometry = std::make_unique(setup.experiment, MERGE_THREADS); + } + { + std::unique_lock gate(gate_mutex); + switch_waiting = false; + } + gate_cv.notify_all(); + { + std::unique_lock ul(live_mutex); + live.beam_x_pxl = r->beam_x_px; + live.beam_y_pxl = r->beam_y_px; + live.distance_mm = r->distance_mm; + } + logger.Info("{}", text); + if (status) + status->Note(text); +} + std::optional StillsGPUAnalysis::GetConsensusUnitCell() const { return indexer->GetConsensusUnitCell(); } @@ -330,8 +415,8 @@ void StillsGPUAnalysis::MergeThread() { }; size_t next = 0; try { - for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking()) { - merge->Add(n, indexer->TakeIntegrationOutcome(n)); + const auto add = [&](int64_t n, IntegrationOutcome &&o) { + merge->Add(n, std::move(o)); if (next < CHECKPOINTS.size() && merge->Added() >= std::llround(CHECKPOINTS[next] * static_cast(images))) { if (have_cell()) @@ -340,7 +425,28 @@ void StillsGPUAnalysis::MergeThread() { ++next; } else if (next > 0 && merge->Added() % PREPARE_EVERY == 0) merge->Prepare(MERGE_THREADS); + }; + // The frames analysed at a geometry the running refinement may still replace wait until it is decided: + // left out where analysed before its last switch, merged as analysed otherwise. + std::vector held_back; + const auto release = [&] { + for (const int64_t h: held_back) + if (frame_epoch[h] - 1 < geometry_epoch) { + indexer->DropIntegrationOutcome(h); + add(h, IntegrationOutcome{}); + } else + add(h, indexer->TakeIntegrationOutcome(h)); + held_back.clear(); + }; + for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking()) { + if (!frame_epoch.empty() && frame_epoch[n] > 0) + held_back.push_back(n); + else + add(n, indexer->TakeIntegrationOutcome(n)); + if (geometry_state != GEOMETRY_PENDING && !held_back.empty()) + release(); } + release(); if (have_cell()) { merged = merge->Final(static_cast(AvailableCpus())); logger.Info("Stills merge after the last image: {:.2f} s", merged->seconds); diff --git a/receiver/StillsGPUAnalysis.h b/receiver/StillsGPUAnalysis.h index f20409479..e87160b82 100644 --- a/receiver/StillsGPUAnalysis.h +++ b/receiver/StillsGPUAnalysis.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include "StillsPrePass.h" @@ -24,6 +26,7 @@ #include "../common/ThreadSafeFIFO.h" #include "../image_analysis/IndexAndRefine.h" #include "../rugnux/StillsLiveMerge.h" +#include "../rugnux/StillsRunningGeometry.h" // Serial frames analysed on the GPU as the DECTRIS ones are (MXAnalysisWithoutFPGA), after a pre-pass on // the first PREPASS_IMAGES (RunStillsPrePass) has decided the beam centre and the mask they are analysed @@ -41,6 +44,13 @@ // before Finish returns (GetMerged). The cell is the run's consensus at the first merge, as rugnux merges in // it (the collection's cell picks the frames merged); the space group the collection's, else P1. // merge = false leaves it out. +// Where the pre-pass refined the centre and distance on fewer frames than StillsRunningGeometry bundles, or +// could not refine them (too few of the frames it had indexed: a small image buffer, a serial hit rate, a chip +// whose first frames are empty), and a cell is given, they are refined again while the frames are analysed +// (StillsRunningGeometry, from the spots and lattices of the frames analysed, no image held), and the analysis +// switches to that geometry as each of GEOMETRY_ROUNDS fits is made. The frames analysed before the last +// switch are not merged (their reflections were predicted at an earlier geometry; the images are gone); where +// no fit can be made they are merged as analysed. class StillsGPUAnalysis { public: // Enough frames for the distance to be refined at a serial hit rate: of 100 frames of a collection indexed @@ -127,6 +137,29 @@ private: std::optional merged; std::future merge_future; + // The geometry refined while the frames are analysed (see above), in GEOMETRY_ROUNDS rounds, each on the + // frames analysed at the last one's geometry (the spots of a frame are indexed at the geometry it was + // analysed at, and a start 2.6% off left the first fit 0.6% off). The analysis takes geometry_mutex shared + // for each frame; a switch takes it exclusive and raises geometry_epoch, which the analysis threads see to + // rebuild their analysis. geometry_state is PENDING until the last round, then SWITCHED or KEPT. + // frame_epoch: per frame number, 1 + the epoch it was analysed at while PENDING (0: after); the merge + // leaves out those analysed before the last switch. + constexpr static int GEOMETRY_ROUNDS = 2; + enum GeometryState : int { GEOMETRY_PENDING = 0, GEOMETRY_SWITCHED = 1, GEOMETRY_KEPT = 2 }; + std::unique_ptr running_geometry; + int geometry_round = 0; + std::shared_mutex geometry_mutex; + // A switch waiting for geometry_mutex closes this gate to new frames, or the frames that keep taking the + // mutex shared would hold it off to the end of the collection. + std::mutex gate_mutex; + std::condition_variable gate_cv; + bool switch_waiting = false; + std::atomic geometry_epoch{0}; + std::atomic geometry_state{GEOMETRY_KEPT}; + std::vector frame_epoch; + std::vector> retired_mappings; // the analyses before the switch read them + void SwitchGeometry(); + void StartPrePass(); void PrePassThread(); void AnalysisThread(); diff --git a/receiver/StillsPrePass.cpp b/receiver/StillsPrePass.cpp index 4bbaa69ea..c3989b6ce 100644 --- a/receiver/StillsPrePass.cpp +++ b/receiver/StillsPrePass.cpp @@ -60,6 +60,7 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map(result.used_direct_beam_y_pxl), .distance_mm = result.used_distance_mm, .mask = rugnux.GetPixelMask(), - .images = images.size() + .images = images.size(), + .geometry_frames = result.stills_geometry_frames }; } diff --git a/receiver/StillsPrePass.h b/receiver/StillsPrePass.h index b43197f2d..634171360 100644 --- a/receiver/StillsPrePass.h +++ b/receiver/StillsPrePass.h @@ -37,6 +37,7 @@ struct StillsPrePassResult { float distance_mm; PixelMask mask; size_t images = 0; + int geometry_frames = 0; // the indexed frames the distance and centre were refined on; 0: kept as stated }; // The first frames of a serial collection through rugnux, as `rugnux --no-merge` on them: the beam centre diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index bd1462509..72ab81c3d 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -25,6 +25,8 @@ ADD_LIBRARY(JFJochRugnux STATIC RugnuxLive.h StillsLiveMerge.cpp StillsLiveMerge.h + StillsRunningGeometry.cpp + StillsRunningGeometry.h RugnuxRefine.cpp RugnuxRefine.h Rugnux.h diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index f3be5f659..c74c1fe59 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1455,6 +1455,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b result.used_beam_x_pxl = experiment_.GetBeamX_pxl(); result.used_beam_y_pxl = experiment_.GetBeamY_pxl(); result.used_distance_mm = experiment_.GetDetectorDistance_mm(); + result.stills_geometry_frames = stills_geometry_frames_; { const auto &g = experiment_.GetDiffractionGeometry(); constexpr double DEG = 180.0 / PI; diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index faeefc850..b3bd6beb0 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -636,6 +636,9 @@ struct ProcessResult { // rotations, which is a geometry no pass ever ran at. std::array used_detector_tilt_deg{}; double used_direct_beam_x_pxl = 0.0, used_direct_beam_y_pxl = 0.0; + // Stills: the frames the geometry refinement on the sample fitted the centre and distance on + // (RefineStillsGeometry); 0 where it kept the input geometry (too few strong frames). + int stills_geometry_frames = 0; // The rotation axis the same pass integrated at - the sign rescue can adopt the opposite of the // file's. The report's XDS keyword block reads it. Empty on stills. std::optional used_rotation_axis; @@ -1237,6 +1240,7 @@ class Rugnux { // The steps RefineStillsGeometry moved the modules by (empty: not moved). std::vector module_shifts_; + int stills_geometry_frames_ = 0; // ProcessResult::stills_geometry_frames // Placed once, by the first RefineStillsGeometry. bool modules_placed_ = false; diff --git a/rugnux/RugnuxStills.cpp b/rugnux/RugnuxStills.cpp index 8e1a124cd..5e457fc9e 100644 --- a/rugnux/RugnuxStills.cpp +++ b/rugnux/RugnuxStills.cpp @@ -666,6 +666,7 @@ void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_ experiment_.BeamX_pxl(r.beam_x_px).BeamY_pxl(r.beam_y_px).DetectorDistance_mm(r.distance_mm); experiment_.SetUnitCell(r.cell); + stills_geometry_frames_ = r.frames_used; } // The detector tilt of a stills run, as a hypothesis against the tilt the run has (the file's, normally none): diff --git a/rugnux/StillsRunningGeometry.cpp b/rugnux/StillsRunningGeometry.cpp new file mode 100644 index 000000000..e5be725dc --- /dev/null +++ b/rugnux/StillsRunningGeometry.cpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "StillsRunningGeometry.h" + +#include + +#include "../image_analysis/scale_merge/CellPopulation.h" + +namespace { + // A frame the geometry refinement can use, as rugnux takes it from its sample: the spots on lattice 0 with + // their index, the others without. + GeomRefineFrame RefineFrame(const DataMessage &msg) { + GeomRefineFrame f; + f.lattice = *msg.indexing_lattice; + f.ordinal = static_cast(msg.number); + for (const auto &s: msg.spots) { + if (s.indexed && s.lattice == 0) + f.spots.push_back(GeomRefineSpot{s.x, s.y, static_cast(s.h), static_cast(s.k), + static_cast(s.l), s.panel}); + else + f.unindexed.push_back(GeomRefineSpot{s.x, s.y, 0, 0, 0, s.panel}); + } + return f; + } +} + +StillsRunningGeometry::StillsRunningGeometry(const DiffractionExperiment &in_experiment, int in_nthreads) + : experiment(in_experiment), nthreads(in_nthreads) {} + +bool StillsRunningGeometry::Add(const DataMessage &msg) { + std::optional f; + if (msg.indexing_result.value_or(false) && msg.indexing_lattice.has_value()) + f = RefineFrame(msg); + std::unique_lock ul(m); + if (done) + return false; + ++offered; + if (f && f->spots.size() >= 6) { + ++indexed; + strong += static_cast(f->spots.size()) >= MIN_STRONG_SPOTS; + frames.push_back(std::move(*f)); + } + done = static_cast(strong) >= STRONG_FRAMES || indexed >= INDEXED_BUDGET; + return done; +} + +int64_t StillsRunningGeometry::Offered() const { + std::unique_lock ul(m); + return offered; +} + +size_t StillsRunningGeometry::Strong() const { + std::unique_lock ul(m); + return strong; +} + +std::optional StillsRunningGeometry::Refine() { + std::unique_lock ul(m); + const auto cell = experiment.GetUnitCell(); + if (!cell) + return {}; + // As RugnuxStills.cpp's sample: only the frames of the given cell's population are fitted, the population + // told from the strong frames. + const double given = CrystalLattice(*cell).CalcVolume(); + std::vector strong_cells; + for (const auto &f: frames) + if (static_cast(f.spots.size()) >= MIN_STRONG_SPOTS) + strong_cells.push_back(f.lattice.GetUnitCell()); + if (strong_cells.empty()) + return {}; + const auto [low, high] = GivenCellPopulation(strong_cells, *cell); + std::vector bundle; + for (auto &f: frames) { + const double r = f.lattice.CalcVolume() / given - 1.0; + if (static_cast(f.spots.size()) >= MIN_STRONG_SPOTS && r >= low && r <= high) + bundle.push_back(std::move(f)); + } + frames.clear(); + std::sort(bundle.begin(), bundle.end(), [](const GeomRefineFrame &a, const GeomRefineFrame &b) { + if (a.spots.size() != b.spots.size()) return a.spots.size() > b.spots.size(); + return a.ordinal < b.ordinal; + }); + if (static_cast(bundle.size()) > STRONG_FRAMES) + bundle.resize(STRONG_FRAMES); + + GeometryRefinerSettings settings; + settings.crystal_system = experiment.GetCrystalSystem(); + settings.num_threads = nthreads; + settings.cell_reg_coeff = 0.0; // cell and distance both free, as rugnux fits them + const GeometryRefinerResult r = RefineGlobalGeometry(experiment.GetDiffractionGeometry(), *cell, bundle, settings); + if (!r.ok) + return {}; + return r; +} diff --git a/rugnux/StillsRunningGeometry.h b/rugnux/StillsRunningGeometry.h new file mode 100644 index 000000000..ad3260673 --- /dev/null +++ b/rugnux/StillsRunningGeometry.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +#include "../common/DiffractionExperiment.h" +#include "../common/JFJochMessages.h" +#include "../image_analysis/geom_refinement/GeometryRefiner.h" + +// The detector centre and distance of a serial collection refined from frames as they are analysed, holding +// no image: the live path's geometry where its pre-pass could not refine it (too few of the frames the image +// buffer holds indexed). The frames' indexed spots and lattices are kept until STRONG_FRAMES strong frames +// (at least MIN_STRONG_SPOTS indexed spots) are in, or INDEXED_BUDGET frames indexed (empty frames - a chip's +// wells without a crystal - do not count); then the strongest +// are bundled and the geometry refined once, as rugnux refines it on its sample (cell and distance free, the +// detector untilted), on the frames of the given cell's population. A fixed cost: one bundle, one fit. +// Thread-safe. +class StillsRunningGeometry { +public: + constexpr static int STRONG_FRAMES = 200; + constexpr static int MIN_STRONG_SPOTS = 20; + constexpr static int64_t INDEXED_BUDGET = 5000; + + // `experiment`: the geometry the frames are analysed at, the cell given. + explicit StillsRunningGeometry(const DiffractionExperiment &experiment, int nthreads); + + // A frame as analysed. True once: for the frame after which Refine is to be called. + bool Add(const DataMessage &msg); + // The refined geometry, where the fit ran. + std::optional Refine(); + // Frames offered and frames kept, for the log + [[nodiscard]] int64_t Offered() const; + [[nodiscard]] size_t Strong() const; + +private: + const DiffractionExperiment experiment; + const int nthreads; + mutable std::mutex m; + std::vector frames; + int64_t offered = 0, indexed = 0; + size_t strong = 0; + bool done = false; +}; diff --git a/tools/jfjoch_stills_gpu_replay.cpp b/tools/jfjoch_stills_gpu_replay.cpp index 5524a37e7..722da95d9 100644 --- a/tools/jfjoch_stills_gpu_replay.cpp +++ b/tools/jfjoch_stills_gpu_replay.cpp @@ -36,6 +36,7 @@ void print_usage() { std::cout << " -c leave the file's unit cell out of the collection" << std::endl; std::cout << " -S space group of the collection (the merge's)" << std::endl; std::cout << " -M no merge (StillsGPUAnalysis merge = false)" << std::endl; + std::cout << " -e the first images given as empty frames (no diffraction)" << std::endl; } int main(int argc, char **argv) { @@ -54,9 +55,10 @@ int main(int argc, char **argv) { bool fixed_threshold = false; std::optional space_group; bool merge = true; + int64_t empty_frames = 0; int opt; - while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:M")) != -1) { + while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:")) != -1) { switch (opt) { case 'o': prefix = optarg; break; case 's': first = atoll(optarg); break; @@ -70,6 +72,7 @@ int main(int argc, char **argv) { case 'F': fixed_threshold = true; break; case 'S': space_group = optarg; break; case 'M': merge = false; break; + case 'e': empty_frames = atoll(optarg); break; case 'C': { UnitCell c{}; if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) { @@ -138,7 +141,8 @@ int main(int argc, char **argv) { std::chrono::steady_clock::time_point t_ready; const auto t0 = std::chrono::steady_clock::now(); { - StillsGPUAnalysis gpu(x, mask, start, &pool, nthreads, buffer.GetStatus().total_slots, logger, + // As the receiver: half the buffer for the frames waiting, the other half for those sent. + StillsGPUAnalysis gpu(x, mask, start, &pool, nthreads, buffer.GetStatus().total_slots / 2, logger, [&](const StillsGPUAnalysis::Setup &s) { t_ready = std::chrono::steady_clock::now(); const auto &m = s.mask.GetMask(); @@ -158,10 +162,17 @@ int main(int argc, char **argv) { [&] { return spot_finding; }, {}, nullptr, merge); JFJochReaderRawImage raw; + std::vector empty; for (int64_t i = 0; i < n; i++) { if (rate > 0) std::this_thread::sleep_until(t0 + std::chrono::duration(static_cast(i) / rate)); reader.ReadRawImage(first + i, raw); + if (i < empty_frames) { + // An empty frame of the same size and pixel type: no compression, every pixel zero. + empty.assign(raw.image.GetUncompressedSize(), 0); + raw.image = CompressedImage(empty.data(), empty.size(), raw.image.GetWidth(), raw.image.GetHeight(), + raw.image.GetMode()); + } auto held = std::make_shared(); // Unlike the receiver, the replay waits for a slot rather than lose the frame. while ((held->slot = buffer.GetImageSlot()) == nullptr)