Rugnux live: stills_downsample, 1 of every F serial images analysed, chosen by a hash

A serial collection can be analysed on a fraction of its images, to keep up
with a stream faster than the analysis. Not a stride, which is in step with
periodic patterns of the stream (pump-probe on/off, storage cells, module or
card interleave) and could take only the dark frames: each image is chosen on
its own with probability 1/F from the SplitMix64 finaliser of its number,
salted (StillsImageSelected), reproducible and the same on every thread and
card. The first StillsGPUAnalysis::PREPASS_MAX_IMAGES (2000) images are all
taken, as the pre-pass may use any of them. An image not chosen is counted as
collected and skipped before the analysis, neither analysed nor written - as
one the serial filter leaves out.

- OpenAPI rugnux_settings / rugnux_status: stills_downsample (int64, >= 1,
  default 1); C++ model and frontend client regenerated; the Rugnux settings
  page gets the field. Kept by POST /initialize, as reserved_cpus.
- JFJochReceiverService::RugnuxLive, JFJochStateMachine::SetRugnuxLive,
  JFJochReceiverRugnuxStills take it.
- jfjoch_fpga_stills_test -k <F>, jfjoch_stills_gpu_replay -k <F>.
- StillsSelectionTest: fraction within 5 sigma for F = 2, 3, 4, 10 over 400k
  images, every phase of a period-2 and a period-F pattern gets its share,
  deterministic, pre-pass images and F = 1 all taken.

Measured (mock, 9M, 6000 images, -k 4): 2984 analysed = 0.497 (expected 0.5
with the first 2000 all in; 0.246 of the rest). Host-only replay of 10000
serial 8M images: 17.0 s -> 10.0 s in all, after the pre-pass 10.6 s -> 3.4 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-18 12:31:39 +01:00
1 parent f2e4dc63b4
commit c52125bf43
26 files changed
+268 -38

No files matched your search

+2 -1
View File
@@ -1018,6 +1018,7 @@ void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) {
ret.setDeadlineS(s.deadline_s);
ret.setRefineSolvent(s.refine_solvent);
ret.setForceStill(s.force_still);
ret.setStillsDownsample(s.stills_downsample);
ret.setModelLoaded(s.model != nullptr);
if (s.model)
ret.setModelSizeBytes(s.model_size_bytes);
@@ -1028,7 +1029,7 @@ void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings,
httplib::Response &response) {
state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(),
rugnuxSettings.getDeadlineS(), rugnuxSettings.isRefineSolvent(),
rugnuxSettings.isForceStill());
rugnuxSettings.isForceStill(), rugnuxSettings.getStillsDownsample());
response.status = 200;
}
+2 -1
View File
@@ -283,7 +283,8 @@ void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings
.reserved_cpus = input.getRugnux().getReservedCpus(),
.deadline_s = input.getRugnux().getDeadlineS(),
.refine_solvent = input.getRugnux().isRefineSolvent(),
.force_still = input.getRugnux().isForceStill()
.force_still = input.getRugnux().isForceStill(),
.stills_downsample = input.getRugnux().getStillsDownsample()
});
}
+1 -1
View File
@@ -385,7 +385,7 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) {
void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) {
if (receiver)
receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s, input.refine_solvent,
input.force_still);
input.force_still, input.stills_downsample);
rugnux = input;
}
+1
View File
@@ -20,6 +20,7 @@ struct RugnuxLiveSettings {
double deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
bool refine_solvent = false; // the refinement of a placed model builds the solvent (rugnux --refine-solvent)
bool force_still = false; // rotation collections are processed as stills (rugnux --force-still)
int64_t stills_downsample = 1; // serial collections: 1 of every this many images analysed on average
std::shared_ptr<const RugnuxModel> model; // null = no model
int64_t model_size_bytes = 0; // as uploaded
};
+2 -1
View File
@@ -1226,7 +1226,7 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const {
}
void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent,
bool force_still) {
bool force_still, int64_t stills_downsample) {
std::unique_lock ul(m);
if (IsRunning())
throw WrongDAQStateException("Cannot change rugnux settings during data collection");
@@ -1237,6 +1237,7 @@ void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, doubl
s.deadline_s = deadline_s;
s.refine_solvent = refine_solvent;
s.force_still = force_still;
s.stills_downsample = stills_downsample;
services.SetRugnuxLive(s);
}
+2 -1
View File
@@ -248,7 +248,8 @@ public:
void SetIndexingSettings(const IndexingSettings &input);
IndexingSettings GetIndexingSettings() const;
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still);
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still,
int64_t stills_downsample = 1);
void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes); // null = remove the model
RugnuxLiveSettings GetRugnuxLive() const;
RugnuxLiveSnapshot GetRugnuxStatus() const;
+45 -2
View File
@@ -31,6 +31,8 @@ Rugnux_settings::Rugnux_settings()
m_Refine_solventIsSet = false;
m_Force_still = false;
m_Force_stillIsSet = false;
m_Stills_downsample = 1L;
m_Stills_downsampleIsSet = false;
}
@@ -81,7 +83,21 @@ bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPr
}
}
if (stillsDownsampleIsSet())
{
const int64_t& value = m_Stills_downsample;
const std::string currentValuePath = _pathPrefix + ".stillsDownsample";
if (value < 1ll)
{
success = false;
msg << currentValuePath << ": must be greater than or equal to 1;";
}
}
return success;
}
@@ -103,7 +119,10 @@ bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const
((!refineSolventIsSet() && !rhs.refineSolventIsSet()) || (refineSolventIsSet() && rhs.refineSolventIsSet() && isRefineSolvent() == rhs.isRefineSolvent())) &&
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill()))
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill())) &&
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample()))
;
}
@@ -126,6 +145,8 @@ void to_json(nlohmann::json& j, const Rugnux_settings& o)
j["refine_solvent"] = o.m_Refine_solvent;
if(o.forceStillIsSet())
j["force_still"] = o.m_Force_still;
if(o.stillsDownsampleIsSet())
j["stills_downsample"] = o.m_Stills_downsample;
}
@@ -156,6 +177,11 @@ void from_json(const nlohmann::json& j, Rugnux_settings& o)
j.at("force_still").get_to(o.m_Force_still);
o.m_Force_stillIsSet = true;
}
if(j.find("stills_downsample") != j.end())
{
j.at("stills_downsample").get_to(o.m_Stills_downsample);
o.m_Stills_downsampleIsSet = true;
}
}
@@ -244,6 +270,23 @@ void Rugnux_settings::unsetForce_still()
{
m_Force_stillIsSet = false;
}
int64_t Rugnux_settings::getStillsDownsample() const
{
return m_Stills_downsample;
}
void Rugnux_settings::setStillsDownsample(int64_t const value)
{
m_Stills_downsample = value;
m_Stills_downsampleIsSet = true;
}
bool Rugnux_settings::stillsDownsampleIsSet() const
{
return m_Stills_downsampleIsSet;
}
void Rugnux_settings::unsetStills_downsample()
{
m_Stills_downsampleIsSet = false;
}
} // namespace org::openapitools::server::model
+9
View File
@@ -92,6 +92,13 @@ public:
void setForceStill(bool const value);
bool forceStillIsSet() const;
void unsetForce_still();
/// <summary>
/// Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written. 1: every image.
/// </summary>
int64_t getStillsDownsample() const;
void setStillsDownsample(int64_t const value);
bool stillsDownsampleIsSet() const;
void unsetStills_downsample();
friend void to_json(nlohmann::json& j, const Rugnux_settings& o);
friend void from_json(const nlohmann::json& j, Rugnux_settings& o);
@@ -106,6 +113,8 @@ protected:
bool m_Refine_solventIsSet;
bool m_Force_still;
bool m_Force_stillIsSet;
int64_t m_Stills_downsample;
bool m_Stills_downsampleIsSet;
};
+30 -1
View File
@@ -27,6 +27,8 @@ Rugnux_status::Rugnux_status()
m_Deadline_sIsSet = false;
m_Refine_solvent = false;
m_Force_still = false;
m_Stills_downsample = 0L;
m_Stills_downsampleIsSet = false;
m_Model_loaded = false;
m_Model_size_bytes = 0L;
m_Model_size_bytesIsSet = false;
@@ -52,7 +54,7 @@ bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPref
bool success = true;
const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix;
return success;
}
@@ -76,6 +78,9 @@ bool Rugnux_status::operator==(const Rugnux_status& rhs) const
(isForceStill() == rhs.isForceStill())
&&
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample())) &&
(isModelLoaded() == rhs.isModelLoaded())
&&
@@ -99,6 +104,8 @@ void to_json(nlohmann::json& j, const Rugnux_status& o)
j["deadline_s"] = o.m_Deadline_s;
j["refine_solvent"] = o.m_Refine_solvent;
j["force_still"] = o.m_Force_still;
if(o.stillsDownsampleIsSet())
j["stills_downsample"] = o.m_Stills_downsample;
j["model_loaded"] = o.m_Model_loaded;
if(o.modelSizeBytesIsSet())
j["model_size_bytes"] = o.m_Model_size_bytes;
@@ -116,6 +123,11 @@ void from_json(const nlohmann::json& j, Rugnux_status& o)
}
j.at("refine_solvent").get_to(o.m_Refine_solvent);
j.at("force_still").get_to(o.m_Force_still);
if(j.find("stills_downsample") != j.end())
{
j.at("stills_downsample").get_to(o.m_Stills_downsample);
o.m_Stills_downsampleIsSet = true;
}
j.at("model_loaded").get_to(o.m_Model_loaded);
if(j.find("model_size_bytes") != j.end())
{
@@ -174,6 +186,23 @@ void Rugnux_status::setForceStill(bool const value)
{
m_Force_still = value;
}
int64_t Rugnux_status::getStillsDownsample() const
{
return m_Stills_downsample;
}
void Rugnux_status::setStillsDownsample(int64_t const value)
{
m_Stills_downsample = value;
m_Stills_downsampleIsSet = true;
}
bool Rugnux_status::stillsDownsampleIsSet() const
{
return m_Stills_downsampleIsSet;
}
void Rugnux_status::unsetStills_downsample()
{
m_Stills_downsampleIsSet = false;
}
bool Rugnux_status::isModelLoaded() const
{
return m_Model_loaded;
+9
View File
@@ -85,6 +85,13 @@ public:
bool isForceStill() const;
void setForceStill(bool const value);
/// <summary>
/// Serial collections, 1 of every this many images analysed on average
/// </summary>
int64_t getStillsDownsample() const;
void setStillsDownsample(int64_t const value);
bool stillsDownsampleIsSet() const;
void unsetStills_downsample();
/// <summary>
/// A model was uploaded with &#x60;PUT /config/rugnux/model&#x60;
/// </summary>
bool isModelLoaded() const;
@@ -110,6 +117,8 @@ protected:
bool m_Force_still;
int64_t m_Stills_downsample;
bool m_Stills_downsampleIsSet;
bool m_Model_loaded;
int64_t m_Model_size_bytes;
+15
View File
@@ -2841,6 +2841,17 @@ components:
description: |
Process DECTRIS rotation collections as stills, on the serial path (pre-pass, GPU analysis, stills merge),
as `rugnux --force-still` does. The written file keeps its goniometer. Reset to false by `POST /initialize`.
stills_downsample:
type: integer
format: int64
minimum: 1
default: 1
description: |
Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written.
1: every image.
rugnux_status:
type: object
required:
@@ -2867,6 +2878,10 @@ components:
force_still:
type: boolean
description: DECTRIS rotation collections are processed as stills
stills_downsample:
type: integer
format: int64
description: Serial collections, 1 of every this many images analysed on average
model_loaded:
type: boolean
description: A model was uploaded with `PUT /config/rugnux/model`
+2 -1
View File
@@ -841,7 +841,8 @@ jackknife" (1979), Ann. Statist. 7, 1-26
[doi:10.1214/aos/1176344552](https://doi.org/10.1214/aos/1176344552). Its draws are Poisson weights
hashed with the SplitMix64 finaliser of G. L. Steele Jr., D. Lea and C. H. Flood, "Fast splittable
pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). The same finaliser chooses the images rugnux live
stills analyses with `stills_downsample`.
**Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an
Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU
+17 -11
View File
@@ -204,8 +204,7 @@ back to the broker, so with it the route answers 409.
## Online analysis
Without rugnux live mode, and in the FPGA workflow also for a rotation collection with it, the broker analyses each
image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
Without rugnux live mode the broker analyses each image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
of the image and refinement of its geometry (`geom_refinement_algorithm`). A rotation collection is indexed image by image, as a serial
one. The broker neither integrates Bragg reflections nor determines a lattice for the whole rotation sweep, so the
reflections, the per-image B-factor and integrated-reflection count, and the rotation lattice are filled only where
@@ -221,7 +220,9 @@ goniometer and the rugnux live setting:
| Input | Run | Receiver | What it does |
|---|---|---|---|
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | every run | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer; pedestal runs into the pedestal and its RMS; with rugnux live on, a serial run analysed on the GPU after a rugnux pre-pass (see [Serial collections](#serial-collections-experimental)) |
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | pedestal run | `JFJochReceiverPedestal` | the raw frames into the pedestal and its RMS; nothing on any stream |
| FPGA cards | rugnux live off, or summation on the host | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer |
| FPGA cards | rugnux live on | `FPGAImagePuller`, then `JFJochReceiverRugnuxRotation` or `JFJochReceiverRugnuxStills` | the cards' frames assembled and compressed on the host into a stream as a DECTRIS detector gives, then taken as the DECTRIS rows below |
| DECTRIS stream | dark-mask sequence | `JFJochReceiverDarkMask` | the dark frames into the mask; nothing on any stream |
| DECTRIS stream | rugnux live off, or a rotation run the image buffer cannot hold | `JFJochReceiverLite` | each image on the GPU: masking, azimuthal and ROI integration, spot finding, indexing |
| DECTRIS stream | rugnux live on, rotation that fits the image buffer | `JFJochReceiverRugnuxRotation` | images forwarded as they come; rugnux reads the collection back from the image buffer and runs the whole pipeline |
@@ -240,8 +241,8 @@ more bins than the card holds needs.
With rugnux live mode on, collections from a DECTRIS detector are processed inside the broker while they
arrive: rotation collections by rugnux, and serial (stills) collections on the GPU with the geometry and mask a
rugnux pre-pass decided (see [Serial collections](#serial-collections-experimental) below). In the FPGA workflow
(PSI JUNGFRAU and EIGER) serial collections are analysed on the GPU after a pre-pass as well, with the differences
given there; rotation collections run image by image, as with the mode off. It is off by default; the
(PSI JUNGFRAU and EIGER) the frames reach the same receivers through `FPGAImagePuller`, rotation and serial alike;
with summation on the host the collection runs image by image, as with the mode off. It is off by default; the
configuration file switches it on with
`"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads
left to the receiver and the image pusher.
@@ -278,7 +279,14 @@ a placed model build the solvent as well: waters added at difference-map peaks a
hold them, waters and ions relabelled; `model.refinement` in `/rugnux/status` then carries `waters_added` and
`waters_removed`. `force_still` (`rugnux --force-still`) sends a rotation collection from a DECTRIS detector the
serial way: pre-pass, GPU analysis and stills merge (see [Serial collections](#serial-collections-experimental)).
The analysis takes the frames without the goniometer; the master file keeps it. It does not change the FPGA workflow.
The analysis takes the frames without the goniometer; the master file keeps it.
`stills_downsample` (default 1) lets a serial collection analyse on average 1 of every this many images, to keep up
with a stream faster than the analysis. Each image is chosen on its own, with probability 1 / `stills_downsample`
from a hash of its number: not every n-th image, which would be in step with a periodic pattern of the stream
(pump-probe on/off, storage cells) and could take, for example, only the dark frames. The choice is reproducible.
The first 2000 images, which the pre-pass may use, are all analysed. An image not chosen is counted as collected,
neither analysed nor written - as an image the serial filter leaves out.
The end message, and with it the end of the collection, waits for the analysis. `deadline_s` (default
60 s; 0 for no cutoff) bounds that wait: this long after the last image the analysis is cut. `POST /cancel` is
@@ -418,11 +426,9 @@ pre-pass state and duration, the stated and decided beam centre and distance, th
images analysed and indexed with the analysis rate. The "Rugnux live" page of the web frontend shows the
same. `tools/jfjoch_stills_gpu_replay` replays a stored serial collection through the same analysis.
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is analysed by the receiver of the
card (`JFJochReceiverFPGA`) with the pre-pass, the GPU analysis and the live merge described above, with three
differences: the converted frames wait for the analysis in at most half of the image buffer's slots, a frame that
finds them taken is dropped, and there is no model step - an uploaded model is not placed or refined on these
merges. The end message carries the merged data, as for a DECTRIS stream.
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is assembled on the host by
`FPGAImagePuller` and analysed by `JFJochReceiverRugnuxStills` exactly as a DECTRIS stream.
`tools/jfjoch_fpga_stills_test` measures this path on the cards with a stored dataset (see [Tools](TOOLS.md)).
## Setting up a local test for Jungfraujoch
For development, it is possible to set up a local installation of Jungfraujoch.
+3 -2
View File
@@ -74,7 +74,7 @@ synthesis C model on the CPU, so no FPGA device is required.
Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector:
```
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-d] [-v] [-K[<MiB>]] <master.h5>
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-k <F>] [-d] [-v] [-K[<MiB>]] <master.h5>
```
- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement
@@ -88,7 +88,8 @@ jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C
- The frames are loaded into the card's internal frame generator and the run is made with the JUNGFRAU
conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon
counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live
on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s). Beam centre, distance, energy and the cell
on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s; `-k <F>`: analyse 1 of every
F images on average, the rugnux live setting `stills_downsample`). Beam centre, distance, energy and the cell
come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is
written.
- Each card's generator holds module frames of its own modules only: as many images as its memory holds
+13
View File
@@ -2043,6 +2043,15 @@ export type rugnux_settings = {
*
*/
force_still?: boolean;
/**
* Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
* image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
* which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
* may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written.
* 1: every image.
*
*/
stills_downsample?: number;
};
export type rugnux_status = {
@@ -2066,6 +2075,10 @@ export type rugnux_status = {
* DECTRIS rotation collections are processed as stills
*/
force_still: boolean;
/**
* Serial collections, 1 of every this many images analysed on average
*/
stills_downsample?: number;
/**
* A model was uploaded with `PUT /config/rugnux/model`
*/
+3 -1
View File
@@ -856,7 +856,8 @@ export const zRugnuxSettings = z.object({
reserved_cpus: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(4)),
deadline_s: z.number().gte(0).optional().default(60),
refine_solvent: z.boolean().optional().default(false),
force_still: z.boolean().optional().default(false)
force_still: z.boolean().optional().default(false),
stills_downsample: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1))
});
export const zRugnuxStatus = z.object({
@@ -865,6 +866,7 @@ export const zRugnuxStatus = z.object({
deadline_s: z.number().optional(),
refine_solvent: z.boolean(),
force_still: z.boolean(),
stills_downsample: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
model_loaded: z.boolean(),
model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional()
});
+11 -4
View File
@@ -50,6 +50,8 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
const [cpusError, setCpusError] = useState(false);
const [deadline, setDeadline] = useState(0);
const [deadlineError, setDeadlineError] = useState(false);
const [downsample, setDownsample] = useState(1);
const [downsampleError, setDownsampleError] = useState(false);
const [counter, setCounter] = useState(0);
const [file, setFile] = useState<File>();
@@ -60,19 +62,21 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
setForceStill(s.force_still);
setCpus(s.reserved_cpus);
setDeadline(s.deadline_s ?? 0);
setDownsample(s.stills_downsample ?? 1);
setCounter(c => c + 1);
}
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s]);
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s, s?.stills_downsample]);
const dirty = s !== undefined && (enable !== s.enable || refineSolvent !== s.refine_solvent
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0));
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0)
|| downsample !== (s.stills_downsample ?? 1));
return (
<Stack direction={{xs: "column", md: "row"}} spacing={2} alignItems="stretch">
<SettingsPanel title="Live processing" dirty={dirty} spacing={2}
onUpload={() => settings.submit({enable, refine_solvent: refineSolvent, force_still: forceStill,
reserved_cpus: cpus, deadline_s: deadline})}
uploadDisabled={measuring || cpusError || deadlineError || settings.pending}
reserved_cpus: cpus, deadline_s: deadline, stills_downsample: downsample})}
uploadDisabled={measuring || cpusError || deadlineError || downsampleError || settings.pending}
snackbar={settings.snackbar}>
<Stack spacing={1.5} alignItems="flex-start" sx={{textAlign: "left"}}>
<FormControlLabel label="Process collections with rugnux live"
@@ -90,6 +94,9 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
<NumberTextField label="Cut the analysis this long after the last image (0 = never)" units="s"
start_val={deadline} min={0} float decimals={1} counter={counter} disabled={measuring}
sx={{width: "100%"}} callback={(v, err) => { setDeadline(v); setDeadlineError(err); }}/>
<NumberTextField label="Serial: analyse 1 of this many images (1 = all)" start_val={downsample} min={1}
max={1000000} counter={counter} disabled={measuring} sx={{width: "100%"}}
callback={(v, err) => { setDownsample(v); setDownsampleError(err); }}/>
</SettingsPanel>
<SettingsPanel title="Model for molecular replacement" spacing={2}>
+9 -1
View File
@@ -10,6 +10,7 @@
#include "../frame_serialize/CBORStream2Serializer.h"
#include "../rugnux/RugnuxDefaults.h"
#include "RugnuxStructure.h"
#include "StillsSelection.h"
JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperiment &in_experiment,
const PixelMask &in_pixel_mask,
@@ -27,13 +28,15 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
ImageBuffer &in_image_buffer,
ZMQPreviewSocket *in_zmq_preview_socket,
ZMQMetadataSocket *in_zmq_metadata_socket,
IndexerThreadPool *in_indexing_thread_pool)
IndexerThreadPool *in_indexing_thread_pool,
int64_t in_downsample)
: JFJochReceiverStream(in_experiment, in_pixel_mask, in_image_puller, in_image_pusher, in_logger,
in_forward_nthreads, in_spot_finding_settings, in_current_status, in_plots,
in_image_buffer, in_zmq_preview_socket, in_zmq_metadata_socket),
model(std::move(in_model)),
deadline_s(in_deadline_s),
refine_solvent(in_refine_solvent),
downsample(in_downsample),
live_status(in_live_status),
indexing_thread_pool(in_indexing_thread_pool),
stills_nthreads(std::min<int64_t>(16, in_forward_nthreads)) {
@@ -81,6 +84,11 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
}
live_status.ImageReceived(received);
CountImage(message);
// Not analysed and not written, as an image the serial filter leaves out
if (!StillsImageSelected(message.number, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES)) {
++images_skipped;
return;
}
const size_t size = message.image.GetCompressedSize();
// Waits for the analysis in a slot of the image buffer, where it can be seen while it waits (it is not sent from
+3 -1
View File
@@ -16,6 +16,7 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin
const std::shared_ptr<const RugnuxModel> model;
const double deadline_s; // after the last image, the model step is cut (0 = never)
const bool refine_solvent; // the refinement of the placed model builds the solvent
const int64_t downsample; // 1 of every this many images analysed on average (StillsImageSelected)
RugnuxLiveStatus &live_status;
IndexerThreadPool *const indexing_thread_pool;
@@ -58,6 +59,7 @@ public:
ImageBuffer &image_buffer,
ZMQPreviewSocket *zmq_preview_socket = nullptr,
ZMQMetadataSocket *zmq_metadata_socket = nullptr,
IndexerThreadPool *indexing_thread_pool = nullptr);
IndexerThreadPool *indexing_thread_pool = nullptr,
int64_t downsample = 1);
~JFJochReceiverRugnuxStills() override;
};
+3 -2
View File
@@ -25,7 +25,7 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic
JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
int reserved_cpus, double deadline_s, bool refine_solvent,
bool force_still) {
bool force_still, int64_t stills_downsample) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
@@ -36,6 +36,7 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
rugnux_deadline_s = deadline_s;
rugnux_refine_solvent = refine_solvent;
rugnux_force_still = force_still;
rugnux_stills_downsample = stills_downsample;
return *this;
}
@@ -218,7 +219,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
rugnux_model, rugnux_deadline_s, rugnux_refine_solvent, rugnux_status, spot_finding_settings,
receiver_status, plots, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(),
indexer_thread_pool.get());
indexer_thread_pool.get(), rugnux_stills_downsample);
else
receiver = std::make_unique<JFJochReceiverLite>(experiment,
pixel_mask,
+3 -1
View File
@@ -37,6 +37,7 @@ class JFJochReceiverService {
double rugnux_deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
bool rugnux_refine_solvent = false; // the refinement of a placed model builds the solvent
bool rugnux_force_still = false; // rotation collections from a DECTRIS detector go the stills way
int64_t rugnux_stills_downsample = 1; // serial collections: 1 of every this many images analysed
RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends
std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status
@@ -71,7 +72,8 @@ public:
size_t send_buffer_size_MiB = 512);
JFJochReceiverService& NumThreads(int64_t input);
JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model, int reserved_cpus,
double deadline_s, bool refine_solvent = false, bool force_still = false);
double deadline_s, bool refine_solvent = false, bool force_still = false,
int64_t stills_downsample = 1);
RugnuxLiveSnapshot GetRugnuxStatus() const;
void ClearRugnuxStatus(); // back to Idle, as before any collection
// slots, blocking, frame_rate_hz: see FPGAImagePuller. always: every PSI collection but pedestal and host
+22
View File
@@ -0,0 +1,22 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <cstdint>
// Rugnux live stills with a downsizing factor F analyse on average 1 of every F images, each image chosen on its
// own with probability 1/F from a hash of its number. A stride would be in step with any period of the stream
// (pump-probe on/off, storage cells, module or card interleave) and could take e.g. only the dark frames; the hash
// is not, and the choice is reproducible and the same on every thread and card.
// The first `all_before` images are all taken: the pre-pass may use any of them.
inline bool StillsImageSelected(int64_t image_number, int64_t factor, int64_t all_before) {
if ((factor <= 1) || (image_number < all_before))
return true;
// The SplitMix64 finaliser: Steele, Lea & Flood (2014) Proc. OOPSLA 2014, 453-472; salted
uint64_t z = static_cast<uint64_t>(image_number) + 0x5d0f3c1a7b2e9d46ULL + 0x9e3779b97f4a7c15ULL;
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL;
z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL;
z = z ^ (z >> 31);
return z % static_cast<uint64_t>(factor) == 0;
}
+1
View File
@@ -155,6 +155,7 @@ ADD_EXECUTABLE(jfjoch_test
JFJochReceiverLiteTest.cpp
JFJochReceiverRugnuxTest.cpp
FPGAImagePullerTest.cpp
StillsSelectionTest.cpp
GridScanSettingsTest.cpp
JFJochReceiverPlotsTest.cpp
GoniometerAxisTest.cpp
+42
View File
@@ -0,0 +1,42 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <cmath>
#include "../receiver/StillsSelection.h"
TEST_CASE("StillsImageSelected_FractionAndPeriods", "[StillsSelection]") {
constexpr int64_t first = 2000;
constexpr int64_t images = 400000;
for (const int64_t factor: {2, 3, 4, 10}) {
INFO("factor " << factor);
const double p = 1.0 / static_cast<double>(factor);
int64_t selected = 0;
for (int64_t i = first; i < first + images; i++)
selected += StillsImageSelected(i, factor, first) ? 1 : 0;
// Within 5 standard deviations of the binomial
CHECK(std::abs(static_cast<double>(selected) / images - p) < 5 * std::sqrt(p * (1 - p) / images));
// Every phase of a period-2 and a period-F pattern (pump-probe on/off, a stride) gets its share
for (const int64_t period: {int64_t(2), factor}) {
for (int64_t phase = 0; phase < period; phase++) {
INFO("period " << period << " phase " << phase);
int64_t n = 0, s = 0;
for (int64_t i = first + phase; i < first + images; i += period) {
n++;
s += StillsImageSelected(i, factor, first) ? 1 : 0;
}
CHECK(std::abs(static_cast<double>(s) / n - p) < 5 * std::sqrt(p * (1 - p) / n));
}
}
}
}
TEST_CASE("StillsImageSelected_DeterministicAndPrePass", "[StillsSelection]") {
for (int64_t i = 0; i < 10000; i++) {
CHECK(StillsImageSelected(i, 1, 0)); // factor 1: every image
CHECK(StillsImageSelected(i, 7, 10000)); // before all_before: every image
CHECK(StillsImageSelected(i, 7, 0) == StillsImageSelected(i, 7, 0));
}
}
+11 -5
View File
@@ -40,6 +40,7 @@ void print_usage(Logger &logger) {
logger.Info(" -t<num> Indexing thread pool size (default: 4)");
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes)");
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
logger.Info(" -k<num> Analyse 1 of every <num> images on average (rugnux live stills_downsample, default 1)");
logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped");
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
logger.Info(" 128 module frames)");
@@ -86,7 +87,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, size_t puller_slots,
const std::vector<double> &rates, bool dropping) {
const std::vector<double> &rates, bool dropping, int64_t downsample) {
RegisterHDF5Filter();
JFJochHDF5Reader reader;
reader.ReadFile(file);
@@ -142,7 +143,7 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
JFJochReceiverService service(aq_devices, logger, pusher, send_buffer_size_MiB);
service.NumThreads(nthreads);
service.Indexing(x.GetIndexingSettings());
service.RugnuxLive(true, nullptr, 4, 60);
service.RugnuxLive(true, nullptr, 4, 60, false, false, downsample);
// Error, gap and beam-stop pixels of the reader become error pixels of the module frame
std::vector<int16_t> frame(x.GetModulesNum() * RAW_MODULE_SIZE);
@@ -243,7 +244,8 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
logger.Info("Pre-pass {:.2f} s: beam {:.2f} {:.2f} px (file {:.2f} {:.2f}), distance {:.2f} mm (file {:.2f})",
st.prepass_s.value_or(0), st.beam_x_pxl.value_or(NAN), st.beam_y_pxl.value_or(NAN),
st.file_beam_x_pxl, st.file_beam_y_pxl, st.distance_mm.value_or(NAN), st.file_distance_mm);
logger.Info("Analysed {} indexed {} analysis rate after the pre-pass {:.1f} Hz", st.images_analysed,
logger.Info("Analysed {} ({:.3f} of the images) indexed {} analysis rate after the pre-pass {:.1f} Hz",
st.images_analysed, static_cast<double>(st.images_analysed) / x.GetImageNum(),
st.images_indexed, after_prepass_hz);
}
logger.Info("Rugnux: {} {} last image at {:.2f} s, done {:.2f} s after it", live.verdict, live.message,
@@ -288,9 +290,10 @@ int main(int argc, char **argv) {
bool verbose = false;
std::vector<double> rates = {0};
bool dropping = false;
int64_t downsample = 1;
int opt;
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dK::")) != -1) {
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dk:K::")) != -1) {
switch (opt) {
case 'v':
verbose = true;
@@ -343,6 +346,9 @@ int main(int argc, char **argv) {
case 'd':
dropping = true;
break;
case 'k':
downsample = atol(optarg);
break;
case 'K':
mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64;
break;
@@ -378,7 +384,7 @@ int main(int argc, char **argv) {
bool ret = false;
try {
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
i_settings, cell, puller_slots, rates, dropping);
i_settings, cell, puller_slots, rates, dropping, downsample);
} catch (std::exception &e) {
logger.Error(e.what());
}
+7 -1
View File
@@ -27,6 +27,7 @@
#include "../image_analysis/indexing/IndexerThreadPool.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../receiver/StillsGPUAnalysis.h"
#include "../receiver/StillsSelection.h"
#include "../writer/HDF5Objects.h"
#include "../writer/RugnuxModelFile.h"
#include "../writer/WriteModel.h"
@@ -61,6 +62,7 @@ void print_usage() {
std::cout << " -e <images> the first images given as empty frames (no diffraction)" << std::endl;
std::cout << " -m <file> model (PDB/mmCIF) placed and refined against the merges" << std::endl;
std::cout << " -p <num> spots kept per image (default: the file's, as the broker's)" << std::endl;
std::cout << " -k <num> analyse 1 of every <num> images on average, as rugnux live stills_downsample" << std::endl;
std::cout << " -D <s> deadline of the model step after the last image (default 60, 0 = none)" << std::endl;
}
@@ -84,9 +86,10 @@ int main(int argc, char **argv) {
int64_t empty_frames = 0;
std::shared_ptr<const RugnuxModel> model;
double deadline_s = 60;
int64_t downsample = 1;
int opt;
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:")) != -1) {
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:k:")) != -1) {
switch (opt) {
case 'o': prefix = optarg; break;
case 's': first = atoll(optarg); break;
@@ -104,6 +107,7 @@ int main(int argc, char **argv) {
case 'e': empty_frames = atoll(optarg); break;
case 'm': model = std::make_shared<const RugnuxModel>(ReadRugnuxModelFile(optarg)); break;
case 'D': deadline_s = atof(optarg); break;
case 'k': downsample = atoll(optarg); break;
case 'C': {
UnitCell c{};
if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) {
@@ -227,6 +231,8 @@ int main(int argc, char **argv) {
JFJochReaderRawImage raw;
std::vector<uint8_t> empty;
for (int64_t i = 0; i < n; i++) {
if (!StillsImageSelected(i, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES))
continue;
if (rate > 0)
std::this_thread::sleep_until(t0 + std::chrono::duration<double>(static_cast<double>(i) / rate));
reader.ReadRawImage(first + i, raw);