Rugnux live: stills_downsample, 1 of every F serial images analysed, chosen by a hash
A serial collection can be analysed on a fraction of its images, to keep up with a stream faster than the analysis. Not a stride, which is in step with periodic patterns of the stream (pump-probe on/off, storage cells, module or card interleave) and could take only the dark frames: each image is chosen on its own with probability 1/F from the SplitMix64 finaliser of its number, salted (StillsImageSelected), reproducible and the same on every thread and card. The first StillsGPUAnalysis::PREPASS_MAX_IMAGES (2000) images are all taken, as the pre-pass may use any of them. An image not chosen is counted as collected and skipped before the analysis, neither analysed nor written - as one the serial filter leaves out. - OpenAPI rugnux_settings / rugnux_status: stills_downsample (int64, >= 1, default 1); C++ model and frontend client regenerated; the Rugnux settings page gets the field. Kept by POST /initialize, as reserved_cpus. - JFJochReceiverService::RugnuxLive, JFJochStateMachine::SetRugnuxLive, JFJochReceiverRugnuxStills take it. - jfjoch_fpga_stills_test -k <F>, jfjoch_stills_gpu_replay -k <F>. - StillsSelectionTest: fraction within 5 sigma for F = 2, 3, 4, 10 over 400k images, every phase of a period-2 and a period-F pattern gets its share, deterministic, pre-pass images and F = 1 all taken. Measured (mock, 9M, 6000 images, -k 4): 2984 analysed = 0.497 (expected 0.5 with the first 2000 all in; 0.246 of the rest). Host-only replay of 10000 serial 8M images: 17.0 s -> 10.0 s in all, after the pre-pass 10.6 s -> 3.4 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
1 parent
f2e4dc63b4
commit
c52125bf43
26 files changed
+268
-38
No files matched your search
@@ -1018,6 +1018,7 @@ void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) {
|
||||
ret.setDeadlineS(s.deadline_s);
|
||||
ret.setRefineSolvent(s.refine_solvent);
|
||||
ret.setForceStill(s.force_still);
|
||||
ret.setStillsDownsample(s.stills_downsample);
|
||||
ret.setModelLoaded(s.model != nullptr);
|
||||
if (s.model)
|
||||
ret.setModelSizeBytes(s.model_size_bytes);
|
||||
@@ -1028,7 +1029,7 @@ void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings,
|
||||
httplib::Response &response) {
|
||||
state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(),
|
||||
rugnuxSettings.getDeadlineS(), rugnuxSettings.isRefineSolvent(),
|
||||
rugnuxSettings.isForceStill());
|
||||
rugnuxSettings.isForceStill(), rugnuxSettings.getStillsDownsample());
|
||||
response.status = 200;
|
||||
}
|
||||
|
||||
|
||||
@@ -283,7 +283,8 @@ void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings
|
||||
.reserved_cpus = input.getRugnux().getReservedCpus(),
|
||||
.deadline_s = input.getRugnux().getDeadlineS(),
|
||||
.refine_solvent = input.getRugnux().isRefineSolvent(),
|
||||
.force_still = input.getRugnux().isForceStill()
|
||||
.force_still = input.getRugnux().isForceStill(),
|
||||
.stills_downsample = input.getRugnux().getStillsDownsample()
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -385,7 +385,7 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) {
|
||||
void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) {
|
||||
if (receiver)
|
||||
receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s, input.refine_solvent,
|
||||
input.force_still);
|
||||
input.force_still, input.stills_downsample);
|
||||
rugnux = input;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ struct RugnuxLiveSettings {
|
||||
double deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
|
||||
bool refine_solvent = false; // the refinement of a placed model builds the solvent (rugnux --refine-solvent)
|
||||
bool force_still = false; // rotation collections are processed as stills (rugnux --force-still)
|
||||
int64_t stills_downsample = 1; // serial collections: 1 of every this many images analysed on average
|
||||
std::shared_ptr<const RugnuxModel> model; // null = no model
|
||||
int64_t model_size_bytes = 0; // as uploaded
|
||||
};
|
||||
|
||||
@@ -1226,7 +1226,7 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const {
|
||||
}
|
||||
|
||||
void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent,
|
||||
bool force_still) {
|
||||
bool force_still, int64_t stills_downsample) {
|
||||
std::unique_lock ul(m);
|
||||
if (IsRunning())
|
||||
throw WrongDAQStateException("Cannot change rugnux settings during data collection");
|
||||
@@ -1237,6 +1237,7 @@ void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, doubl
|
||||
s.deadline_s = deadline_s;
|
||||
s.refine_solvent = refine_solvent;
|
||||
s.force_still = force_still;
|
||||
s.stills_downsample = stills_downsample;
|
||||
services.SetRugnuxLive(s);
|
||||
}
|
||||
|
||||
|
||||
@@ -248,7 +248,8 @@ public:
|
||||
void SetIndexingSettings(const IndexingSettings &input);
|
||||
IndexingSettings GetIndexingSettings() const;
|
||||
|
||||
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still);
|
||||
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still,
|
||||
int64_t stills_downsample = 1);
|
||||
void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes); // null = remove the model
|
||||
RugnuxLiveSettings GetRugnuxLive() const;
|
||||
RugnuxLiveSnapshot GetRugnuxStatus() const;
|
||||
|
||||
@@ -31,6 +31,8 @@ Rugnux_settings::Rugnux_settings()
|
||||
m_Refine_solventIsSet = false;
|
||||
m_Force_still = false;
|
||||
m_Force_stillIsSet = false;
|
||||
m_Stills_downsample = 1L;
|
||||
m_Stills_downsampleIsSet = false;
|
||||
|
||||
}
|
||||
|
||||
@@ -81,7 +83,21 @@ bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPr
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (stillsDownsampleIsSet())
|
||||
{
|
||||
const int64_t& value = m_Stills_downsample;
|
||||
const std::string currentValuePath = _pathPrefix + ".stillsDownsample";
|
||||
|
||||
|
||||
if (value < 1ll)
|
||||
{
|
||||
success = false;
|
||||
msg << currentValuePath << ": must be greater than or equal to 1;";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
@@ -103,7 +119,10 @@ bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const
|
||||
((!refineSolventIsSet() && !rhs.refineSolventIsSet()) || (refineSolventIsSet() && rhs.refineSolventIsSet() && isRefineSolvent() == rhs.isRefineSolvent())) &&
|
||||
|
||||
|
||||
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill()))
|
||||
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill())) &&
|
||||
|
||||
|
||||
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample()))
|
||||
|
||||
;
|
||||
}
|
||||
@@ -126,6 +145,8 @@ void to_json(nlohmann::json& j, const Rugnux_settings& o)
|
||||
j["refine_solvent"] = o.m_Refine_solvent;
|
||||
if(o.forceStillIsSet())
|
||||
j["force_still"] = o.m_Force_still;
|
||||
if(o.stillsDownsampleIsSet())
|
||||
j["stills_downsample"] = o.m_Stills_downsample;
|
||||
|
||||
}
|
||||
|
||||
@@ -156,6 +177,11 @@ void from_json(const nlohmann::json& j, Rugnux_settings& o)
|
||||
j.at("force_still").get_to(o.m_Force_still);
|
||||
o.m_Force_stillIsSet = true;
|
||||
}
|
||||
if(j.find("stills_downsample") != j.end())
|
||||
{
|
||||
j.at("stills_downsample").get_to(o.m_Stills_downsample);
|
||||
o.m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -244,6 +270,23 @@ void Rugnux_settings::unsetForce_still()
|
||||
{
|
||||
m_Force_stillIsSet = false;
|
||||
}
|
||||
int64_t Rugnux_settings::getStillsDownsample() const
|
||||
{
|
||||
return m_Stills_downsample;
|
||||
}
|
||||
void Rugnux_settings::setStillsDownsample(int64_t const value)
|
||||
{
|
||||
m_Stills_downsample = value;
|
||||
m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
bool Rugnux_settings::stillsDownsampleIsSet() const
|
||||
{
|
||||
return m_Stills_downsampleIsSet;
|
||||
}
|
||||
void Rugnux_settings::unsetStills_downsample()
|
||||
{
|
||||
m_Stills_downsampleIsSet = false;
|
||||
}
|
||||
|
||||
|
||||
} // namespace org::openapitools::server::model
|
||||
|
||||
@@ -92,6 +92,13 @@ public:
|
||||
void setForceStill(bool const value);
|
||||
bool forceStillIsSet() const;
|
||||
void unsetForce_still();
|
||||
/// <summary>
|
||||
/// Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written. 1: every image.
|
||||
/// </summary>
|
||||
int64_t getStillsDownsample() const;
|
||||
void setStillsDownsample(int64_t const value);
|
||||
bool stillsDownsampleIsSet() const;
|
||||
void unsetStills_downsample();
|
||||
|
||||
friend void to_json(nlohmann::json& j, const Rugnux_settings& o);
|
||||
friend void from_json(const nlohmann::json& j, Rugnux_settings& o);
|
||||
@@ -106,6 +113,8 @@ protected:
|
||||
bool m_Refine_solventIsSet;
|
||||
bool m_Force_still;
|
||||
bool m_Force_stillIsSet;
|
||||
int64_t m_Stills_downsample;
|
||||
bool m_Stills_downsampleIsSet;
|
||||
|
||||
};
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ Rugnux_status::Rugnux_status()
|
||||
m_Deadline_sIsSet = false;
|
||||
m_Refine_solvent = false;
|
||||
m_Force_still = false;
|
||||
m_Stills_downsample = 0L;
|
||||
m_Stills_downsampleIsSet = false;
|
||||
m_Model_loaded = false;
|
||||
m_Model_size_bytes = 0L;
|
||||
m_Model_size_bytesIsSet = false;
|
||||
@@ -52,7 +54,7 @@ bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPref
|
||||
bool success = true;
|
||||
const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix;
|
||||
|
||||
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
@@ -76,6 +78,9 @@ bool Rugnux_status::operator==(const Rugnux_status& rhs) const
|
||||
(isForceStill() == rhs.isForceStill())
|
||||
&&
|
||||
|
||||
|
||||
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample())) &&
|
||||
|
||||
(isModelLoaded() == rhs.isModelLoaded())
|
||||
&&
|
||||
|
||||
@@ -99,6 +104,8 @@ void to_json(nlohmann::json& j, const Rugnux_status& o)
|
||||
j["deadline_s"] = o.m_Deadline_s;
|
||||
j["refine_solvent"] = o.m_Refine_solvent;
|
||||
j["force_still"] = o.m_Force_still;
|
||||
if(o.stillsDownsampleIsSet())
|
||||
j["stills_downsample"] = o.m_Stills_downsample;
|
||||
j["model_loaded"] = o.m_Model_loaded;
|
||||
if(o.modelSizeBytesIsSet())
|
||||
j["model_size_bytes"] = o.m_Model_size_bytes;
|
||||
@@ -116,6 +123,11 @@ void from_json(const nlohmann::json& j, Rugnux_status& o)
|
||||
}
|
||||
j.at("refine_solvent").get_to(o.m_Refine_solvent);
|
||||
j.at("force_still").get_to(o.m_Force_still);
|
||||
if(j.find("stills_downsample") != j.end())
|
||||
{
|
||||
j.at("stills_downsample").get_to(o.m_Stills_downsample);
|
||||
o.m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
j.at("model_loaded").get_to(o.m_Model_loaded);
|
||||
if(j.find("model_size_bytes") != j.end())
|
||||
{
|
||||
@@ -174,6 +186,23 @@ void Rugnux_status::setForceStill(bool const value)
|
||||
{
|
||||
m_Force_still = value;
|
||||
}
|
||||
int64_t Rugnux_status::getStillsDownsample() const
|
||||
{
|
||||
return m_Stills_downsample;
|
||||
}
|
||||
void Rugnux_status::setStillsDownsample(int64_t const value)
|
||||
{
|
||||
m_Stills_downsample = value;
|
||||
m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
bool Rugnux_status::stillsDownsampleIsSet() const
|
||||
{
|
||||
return m_Stills_downsampleIsSet;
|
||||
}
|
||||
void Rugnux_status::unsetStills_downsample()
|
||||
{
|
||||
m_Stills_downsampleIsSet = false;
|
||||
}
|
||||
bool Rugnux_status::isModelLoaded() const
|
||||
{
|
||||
return m_Model_loaded;
|
||||
|
||||
@@ -85,6 +85,13 @@ public:
|
||||
bool isForceStill() const;
|
||||
void setForceStill(bool const value);
|
||||
/// <summary>
|
||||
/// Serial collections, 1 of every this many images analysed on average
|
||||
/// </summary>
|
||||
int64_t getStillsDownsample() const;
|
||||
void setStillsDownsample(int64_t const value);
|
||||
bool stillsDownsampleIsSet() const;
|
||||
void unsetStills_downsample();
|
||||
/// <summary>
|
||||
/// A model was uploaded with `PUT /config/rugnux/model`
|
||||
/// </summary>
|
||||
bool isModelLoaded() const;
|
||||
@@ -110,6 +117,8 @@ protected:
|
||||
|
||||
bool m_Force_still;
|
||||
|
||||
int64_t m_Stills_downsample;
|
||||
bool m_Stills_downsampleIsSet;
|
||||
bool m_Model_loaded;
|
||||
|
||||
int64_t m_Model_size_bytes;
|
||||
|
||||
@@ -2841,6 +2841,17 @@ components:
|
||||
description: |
|
||||
Process DECTRIS rotation collections as stills, on the serial path (pre-pass, GPU analysis, stills merge),
|
||||
as `rugnux --force-still` does. The written file keeps its goniometer. Reset to false by `POST /initialize`.
|
||||
stills_downsample:
|
||||
type: integer
|
||||
format: int64
|
||||
minimum: 1
|
||||
default: 1
|
||||
description: |
|
||||
Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
|
||||
image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
|
||||
which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
|
||||
may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written.
|
||||
1: every image.
|
||||
rugnux_status:
|
||||
type: object
|
||||
required:
|
||||
@@ -2867,6 +2878,10 @@ components:
|
||||
force_still:
|
||||
type: boolean
|
||||
description: DECTRIS rotation collections are processed as stills
|
||||
stills_downsample:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Serial collections, 1 of every this many images analysed on average
|
||||
model_loaded:
|
||||
type: boolean
|
||||
description: A model was uploaded with `PUT /config/rugnux/model`
|
||||
|
||||
@@ -841,7 +841,8 @@ jackknife" (1979), Ann. Statist. 7, 1-26
|
||||
[doi:10.1214/aos/1176344552](https://doi.org/10.1214/aos/1176344552). Its draws are Poisson weights
|
||||
hashed with the SplitMix64 finaliser of G. L. Steele Jr., D. Lea and C. H. Flood, "Fast splittable
|
||||
pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472
|
||||
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
|
||||
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). The same finaliser chooses the images rugnux live
|
||||
stills analyses with `stills_downsample`.
|
||||
|
||||
**Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an
|
||||
Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU
|
||||
|
||||
+17
-11
@@ -204,8 +204,7 @@ back to the broker, so with it the route answers 409.
|
||||
|
||||
## Online analysis
|
||||
|
||||
Without rugnux live mode, and in the FPGA workflow also for a rotation collection with it, the broker analyses each
|
||||
image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
|
||||
Without rugnux live mode the broker analyses each image on its own: masking, summation, azimuthal and ROI integration, spot finding, and - with indexing on - indexing
|
||||
of the image and refinement of its geometry (`geom_refinement_algorithm`). A rotation collection is indexed image by image, as a serial
|
||||
one. The broker neither integrates Bragg reflections nor determines a lattice for the whole rotation sweep, so the
|
||||
reflections, the per-image B-factor and integrated-reflection count, and the rotation lattice are filled only where
|
||||
@@ -221,7 +220,9 @@ goniometer and the rugnux live setting:
|
||||
|
||||
| Input | Run | Receiver | What it does |
|
||||
|---|---|---|---|
|
||||
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | every run | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer; pedestal runs into the pedestal and its RMS; with rugnux live on, a serial run analysed on the GPU after a rugnux pre-pass (see [Serial collections](#serial-collections-experimental)) |
|
||||
| FPGA cards (PSI JUNGFRAU, PSI EIGER) | pedestal run | `JFJochReceiverPedestal` | the raw frames into the pedestal and its RMS; nothing on any stream |
|
||||
| FPGA cards | rugnux live off, or summation on the host | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer |
|
||||
| FPGA cards | rugnux live on | `FPGAImagePuller`, then `JFJochReceiverRugnuxRotation` or `JFJochReceiverRugnuxStills` | the cards' frames assembled and compressed on the host into a stream as a DECTRIS detector gives, then taken as the DECTRIS rows below |
|
||||
| DECTRIS stream | dark-mask sequence | `JFJochReceiverDarkMask` | the dark frames into the mask; nothing on any stream |
|
||||
| DECTRIS stream | rugnux live off, or a rotation run the image buffer cannot hold | `JFJochReceiverLite` | each image on the GPU: masking, azimuthal and ROI integration, spot finding, indexing |
|
||||
| DECTRIS stream | rugnux live on, rotation that fits the image buffer | `JFJochReceiverRugnuxRotation` | images forwarded as they come; rugnux reads the collection back from the image buffer and runs the whole pipeline |
|
||||
@@ -240,8 +241,8 @@ more bins than the card holds needs.
|
||||
With rugnux live mode on, collections from a DECTRIS detector are processed inside the broker while they
|
||||
arrive: rotation collections by rugnux, and serial (stills) collections on the GPU with the geometry and mask a
|
||||
rugnux pre-pass decided (see [Serial collections](#serial-collections-experimental) below). In the FPGA workflow
|
||||
(PSI JUNGFRAU and EIGER) serial collections are analysed on the GPU after a pre-pass as well, with the differences
|
||||
given there; rotation collections run image by image, as with the mode off. It is off by default; the
|
||||
(PSI JUNGFRAU and EIGER) the frames reach the same receivers through `FPGAImagePuller`, rotation and serial alike;
|
||||
with summation on the host the collection runs image by image, as with the mode off. It is off by default; the
|
||||
configuration file switches it on with
|
||||
`"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads
|
||||
left to the receiver and the image pusher.
|
||||
@@ -278,7 +279,14 @@ a placed model build the solvent as well: waters added at difference-map peaks a
|
||||
hold them, waters and ions relabelled; `model.refinement` in `/rugnux/status` then carries `waters_added` and
|
||||
`waters_removed`. `force_still` (`rugnux --force-still`) sends a rotation collection from a DECTRIS detector the
|
||||
serial way: pre-pass, GPU analysis and stills merge (see [Serial collections](#serial-collections-experimental)).
|
||||
The analysis takes the frames without the goniometer; the master file keeps it. It does not change the FPGA workflow.
|
||||
The analysis takes the frames without the goniometer; the master file keeps it.
|
||||
|
||||
`stills_downsample` (default 1) lets a serial collection analyse on average 1 of every this many images, to keep up
|
||||
with a stream faster than the analysis. Each image is chosen on its own, with probability 1 / `stills_downsample`
|
||||
from a hash of its number: not every n-th image, which would be in step with a periodic pattern of the stream
|
||||
(pump-probe on/off, storage cells) and could take, for example, only the dark frames. The choice is reproducible.
|
||||
The first 2000 images, which the pre-pass may use, are all analysed. An image not chosen is counted as collected,
|
||||
neither analysed nor written - as an image the serial filter leaves out.
|
||||
|
||||
The end message, and with it the end of the collection, waits for the analysis. `deadline_s` (default
|
||||
60 s; 0 for no cutoff) bounds that wait: this long after the last image the analysis is cut. `POST /cancel` is
|
||||
@@ -418,11 +426,9 @@ pre-pass state and duration, the stated and decided beam centre and distance, th
|
||||
images analysed and indexed with the analysis rate. The "Rugnux live" page of the web frontend shows the
|
||||
same. `tools/jfjoch_stills_gpu_replay` replays a stored serial collection through the same analysis.
|
||||
|
||||
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is analysed by the receiver of the
|
||||
card (`JFJochReceiverFPGA`) with the pre-pass, the GPU analysis and the live merge described above, with three
|
||||
differences: the converted frames wait for the analysis in at most half of the image buffer's slots, a frame that
|
||||
finds them taken is dropped, and there is no model step - an uploaded model is not placed or refined on these
|
||||
merges. The end message carries the merged data, as for a DECTRIS stream.
|
||||
In the FPGA workflow (JUNGFRAU), a serial collection with rugnux live mode on is assembled on the host by
|
||||
`FPGAImagePuller` and analysed by `JFJochReceiverRugnuxStills` exactly as a DECTRIS stream.
|
||||
`tools/jfjoch_fpga_stills_test` measures this path on the cards with a stored dataset (see [Tools](TOOLS.md)).
|
||||
|
||||
## Setting up a local test for Jungfraujoch
|
||||
For development, it is possible to set up a local installation of Jungfraujoch.
|
||||
|
||||
+3
-2
@@ -74,7 +74,7 @@ synthesis C model on the CPU, so no FPGA device is required.
|
||||
Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector:
|
||||
|
||||
```
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-d] [-v] [-K[<MiB>]] <master.h5>
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-X <indexer>] [-r <Hz>,...] [-p <slots>] [-k <F>] [-d] [-v] [-K[<MiB>]] <master.h5>
|
||||
```
|
||||
|
||||
- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement
|
||||
@@ -88,7 +88,8 @@ jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C
|
||||
- The frames are loaded into the card's internal frame generator and the run is made with the JUNGFRAU
|
||||
conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon
|
||||
counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live
|
||||
on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s). Beam centre, distance, energy and the cell
|
||||
on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s; `-k <F>`: analyse 1 of every
|
||||
F images on average, the rugnux live setting `stills_downsample`). Beam centre, distance, energy and the cell
|
||||
come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is
|
||||
written.
|
||||
- Each card's generator holds module frames of its own modules only: as many images as its memory holds
|
||||
|
||||
@@ -2043,6 +2043,15 @@ export type rugnux_settings = {
|
||||
*
|
||||
*/
|
||||
force_still?: boolean;
|
||||
/**
|
||||
* Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
|
||||
* image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
|
||||
* which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
|
||||
* may use (the first 2000) are all analysed. Images not chosen are counted, not analysed and not written.
|
||||
* 1: every image.
|
||||
*
|
||||
*/
|
||||
stills_downsample?: number;
|
||||
};
|
||||
|
||||
export type rugnux_status = {
|
||||
@@ -2066,6 +2075,10 @@ export type rugnux_status = {
|
||||
* DECTRIS rotation collections are processed as stills
|
||||
*/
|
||||
force_still: boolean;
|
||||
/**
|
||||
* Serial collections, 1 of every this many images analysed on average
|
||||
*/
|
||||
stills_downsample?: number;
|
||||
/**
|
||||
* A model was uploaded with `PUT /config/rugnux/model`
|
||||
*/
|
||||
|
||||
@@ -856,7 +856,8 @@ export const zRugnuxSettings = z.object({
|
||||
reserved_cpus: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(4)),
|
||||
deadline_s: z.number().gte(0).optional().default(60),
|
||||
refine_solvent: z.boolean().optional().default(false),
|
||||
force_still: z.boolean().optional().default(false)
|
||||
force_still: z.boolean().optional().default(false),
|
||||
stills_downsample: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1))
|
||||
});
|
||||
|
||||
export const zRugnuxStatus = z.object({
|
||||
@@ -865,6 +866,7 @@ export const zRugnuxStatus = z.object({
|
||||
deadline_s: z.number().optional(),
|
||||
refine_solvent: z.boolean(),
|
||||
force_still: z.boolean(),
|
||||
stills_downsample: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
|
||||
model_loaded: z.boolean(),
|
||||
model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional()
|
||||
});
|
||||
|
||||
@@ -50,6 +50,8 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
const [cpusError, setCpusError] = useState(false);
|
||||
const [deadline, setDeadline] = useState(0);
|
||||
const [deadlineError, setDeadlineError] = useState(false);
|
||||
const [downsample, setDownsample] = useState(1);
|
||||
const [downsampleError, setDownsampleError] = useState(false);
|
||||
const [counter, setCounter] = useState(0);
|
||||
const [file, setFile] = useState<File>();
|
||||
|
||||
@@ -60,19 +62,21 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
setForceStill(s.force_still);
|
||||
setCpus(s.reserved_cpus);
|
||||
setDeadline(s.deadline_s ?? 0);
|
||||
setDownsample(s.stills_downsample ?? 1);
|
||||
setCounter(c => c + 1);
|
||||
}
|
||||
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s]);
|
||||
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s, s?.stills_downsample]);
|
||||
|
||||
const dirty = s !== undefined && (enable !== s.enable || refineSolvent !== s.refine_solvent
|
||||
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0));
|
||||
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0)
|
||||
|| downsample !== (s.stills_downsample ?? 1));
|
||||
|
||||
return (
|
||||
<Stack direction={{xs: "column", md: "row"}} spacing={2} alignItems="stretch">
|
||||
<SettingsPanel title="Live processing" dirty={dirty} spacing={2}
|
||||
onUpload={() => settings.submit({enable, refine_solvent: refineSolvent, force_still: forceStill,
|
||||
reserved_cpus: cpus, deadline_s: deadline})}
|
||||
uploadDisabled={measuring || cpusError || deadlineError || settings.pending}
|
||||
reserved_cpus: cpus, deadline_s: deadline, stills_downsample: downsample})}
|
||||
uploadDisabled={measuring || cpusError || deadlineError || downsampleError || settings.pending}
|
||||
snackbar={settings.snackbar}>
|
||||
<Stack spacing={1.5} alignItems="flex-start" sx={{textAlign: "left"}}>
|
||||
<FormControlLabel label="Process collections with rugnux live"
|
||||
@@ -90,6 +94,9 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
<NumberTextField label="Cut the analysis this long after the last image (0 = never)" units="s"
|
||||
start_val={deadline} min={0} float decimals={1} counter={counter} disabled={measuring}
|
||||
sx={{width: "100%"}} callback={(v, err) => { setDeadline(v); setDeadlineError(err); }}/>
|
||||
<NumberTextField label="Serial: analyse 1 of this many images (1 = all)" start_val={downsample} min={1}
|
||||
max={1000000} counter={counter} disabled={measuring} sx={{width: "100%"}}
|
||||
callback={(v, err) => { setDownsample(v); setDownsampleError(err); }}/>
|
||||
</SettingsPanel>
|
||||
|
||||
<SettingsPanel title="Model for molecular replacement" spacing={2}>
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "../frame_serialize/CBORStream2Serializer.h"
|
||||
#include "../rugnux/RugnuxDefaults.h"
|
||||
#include "RugnuxStructure.h"
|
||||
#include "StillsSelection.h"
|
||||
|
||||
JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperiment &in_experiment,
|
||||
const PixelMask &in_pixel_mask,
|
||||
@@ -27,13 +28,15 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
|
||||
ImageBuffer &in_image_buffer,
|
||||
ZMQPreviewSocket *in_zmq_preview_socket,
|
||||
ZMQMetadataSocket *in_zmq_metadata_socket,
|
||||
IndexerThreadPool *in_indexing_thread_pool)
|
||||
IndexerThreadPool *in_indexing_thread_pool,
|
||||
int64_t in_downsample)
|
||||
: JFJochReceiverStream(in_experiment, in_pixel_mask, in_image_puller, in_image_pusher, in_logger,
|
||||
in_forward_nthreads, in_spot_finding_settings, in_current_status, in_plots,
|
||||
in_image_buffer, in_zmq_preview_socket, in_zmq_metadata_socket),
|
||||
model(std::move(in_model)),
|
||||
deadline_s(in_deadline_s),
|
||||
refine_solvent(in_refine_solvent),
|
||||
downsample(in_downsample),
|
||||
live_status(in_live_status),
|
||||
indexing_thread_pool(in_indexing_thread_pool),
|
||||
stills_nthreads(std::min<int64_t>(16, in_forward_nthreads)) {
|
||||
@@ -81,6 +84,11 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
|
||||
}
|
||||
live_status.ImageReceived(received);
|
||||
CountImage(message);
|
||||
// Not analysed and not written, as an image the serial filter leaves out
|
||||
if (!StillsImageSelected(message.number, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES)) {
|
||||
++images_skipped;
|
||||
return;
|
||||
}
|
||||
const size_t size = message.image.GetCompressedSize();
|
||||
|
||||
// Waits for the analysis in a slot of the image buffer, where it can be seen while it waits (it is not sent from
|
||||
|
||||
@@ -16,6 +16,7 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin
|
||||
const std::shared_ptr<const RugnuxModel> model;
|
||||
const double deadline_s; // after the last image, the model step is cut (0 = never)
|
||||
const bool refine_solvent; // the refinement of the placed model builds the solvent
|
||||
const int64_t downsample; // 1 of every this many images analysed on average (StillsImageSelected)
|
||||
RugnuxLiveStatus &live_status;
|
||||
|
||||
IndexerThreadPool *const indexing_thread_pool;
|
||||
@@ -58,6 +59,7 @@ public:
|
||||
ImageBuffer &image_buffer,
|
||||
ZMQPreviewSocket *zmq_preview_socket = nullptr,
|
||||
ZMQMetadataSocket *zmq_metadata_socket = nullptr,
|
||||
IndexerThreadPool *indexing_thread_pool = nullptr);
|
||||
IndexerThreadPool *indexing_thread_pool = nullptr,
|
||||
int64_t downsample = 1);
|
||||
~JFJochReceiverRugnuxStills() override;
|
||||
};
|
||||
@@ -25,7 +25,7 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic
|
||||
|
||||
JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
|
||||
int reserved_cpus, double deadline_s, bool refine_solvent,
|
||||
bool force_still) {
|
||||
bool force_still, int64_t stills_downsample) {
|
||||
std::unique_lock ul_state(state_mutex);
|
||||
if (state != ReceiverState::Idle)
|
||||
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
|
||||
@@ -36,6 +36,7 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
|
||||
rugnux_deadline_s = deadline_s;
|
||||
rugnux_refine_solvent = refine_solvent;
|
||||
rugnux_force_still = force_still;
|
||||
rugnux_stills_downsample = stills_downsample;
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -218,7 +219,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
|
||||
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
|
||||
rugnux_model, rugnux_deadline_s, rugnux_refine_solvent, rugnux_status, spot_finding_settings,
|
||||
receiver_status, plots, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(),
|
||||
indexer_thread_pool.get());
|
||||
indexer_thread_pool.get(), rugnux_stills_downsample);
|
||||
else
|
||||
receiver = std::make_unique<JFJochReceiverLite>(experiment,
|
||||
pixel_mask,
|
||||
|
||||
@@ -37,6 +37,7 @@ class JFJochReceiverService {
|
||||
double rugnux_deadline_s = 60; // the analysis is cut this long after the last image (0 = never)
|
||||
bool rugnux_refine_solvent = false; // the refinement of a placed model builds the solvent
|
||||
bool rugnux_force_still = false; // rotation collections from a DECTRIS detector go the stills way
|
||||
int64_t rugnux_stills_downsample = 1; // serial collections: 1 of every this many images analysed
|
||||
RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends
|
||||
std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status
|
||||
|
||||
@@ -71,7 +72,8 @@ public:
|
||||
size_t send_buffer_size_MiB = 512);
|
||||
JFJochReceiverService& NumThreads(int64_t input);
|
||||
JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model, int reserved_cpus,
|
||||
double deadline_s, bool refine_solvent = false, bool force_still = false);
|
||||
double deadline_s, bool refine_solvent = false, bool force_still = false,
|
||||
int64_t stills_downsample = 1);
|
||||
RugnuxLiveSnapshot GetRugnuxStatus() const;
|
||||
void ClearRugnuxStatus(); // back to Idle, as before any collection
|
||||
// slots, blocking, frame_rate_hz: see FPGAImagePuller. always: every PSI collection but pedestal and host
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
// Rugnux live stills with a downsizing factor F analyse on average 1 of every F images, each image chosen on its
|
||||
// own with probability 1/F from a hash of its number. A stride would be in step with any period of the stream
|
||||
// (pump-probe on/off, storage cells, module or card interleave) and could take e.g. only the dark frames; the hash
|
||||
// is not, and the choice is reproducible and the same on every thread and card.
|
||||
// The first `all_before` images are all taken: the pre-pass may use any of them.
|
||||
inline bool StillsImageSelected(int64_t image_number, int64_t factor, int64_t all_before) {
|
||||
if ((factor <= 1) || (image_number < all_before))
|
||||
return true;
|
||||
// The SplitMix64 finaliser: Steele, Lea & Flood (2014) Proc. OOPSLA 2014, 453-472; salted
|
||||
uint64_t z = static_cast<uint64_t>(image_number) + 0x5d0f3c1a7b2e9d46ULL + 0x9e3779b97f4a7c15ULL;
|
||||
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL;
|
||||
z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL;
|
||||
z = z ^ (z >> 31);
|
||||
return z % static_cast<uint64_t>(factor) == 0;
|
||||
}
|
||||
@@ -155,6 +155,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
JFJochReceiverLiteTest.cpp
|
||||
JFJochReceiverRugnuxTest.cpp
|
||||
FPGAImagePullerTest.cpp
|
||||
StillsSelectionTest.cpp
|
||||
GridScanSettingsTest.cpp
|
||||
JFJochReceiverPlotsTest.cpp
|
||||
GoniometerAxisTest.cpp
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
#include <cmath>
|
||||
|
||||
#include "../receiver/StillsSelection.h"
|
||||
|
||||
TEST_CASE("StillsImageSelected_FractionAndPeriods", "[StillsSelection]") {
|
||||
constexpr int64_t first = 2000;
|
||||
constexpr int64_t images = 400000;
|
||||
for (const int64_t factor: {2, 3, 4, 10}) {
|
||||
INFO("factor " << factor);
|
||||
const double p = 1.0 / static_cast<double>(factor);
|
||||
int64_t selected = 0;
|
||||
for (int64_t i = first; i < first + images; i++)
|
||||
selected += StillsImageSelected(i, factor, first) ? 1 : 0;
|
||||
// Within 5 standard deviations of the binomial
|
||||
CHECK(std::abs(static_cast<double>(selected) / images - p) < 5 * std::sqrt(p * (1 - p) / images));
|
||||
|
||||
// Every phase of a period-2 and a period-F pattern (pump-probe on/off, a stride) gets its share
|
||||
for (const int64_t period: {int64_t(2), factor}) {
|
||||
for (int64_t phase = 0; phase < period; phase++) {
|
||||
INFO("period " << period << " phase " << phase);
|
||||
int64_t n = 0, s = 0;
|
||||
for (int64_t i = first + phase; i < first + images; i += period) {
|
||||
n++;
|
||||
s += StillsImageSelected(i, factor, first) ? 1 : 0;
|
||||
}
|
||||
CHECK(std::abs(static_cast<double>(s) / n - p) < 5 * std::sqrt(p * (1 - p) / n));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("StillsImageSelected_DeterministicAndPrePass", "[StillsSelection]") {
|
||||
for (int64_t i = 0; i < 10000; i++) {
|
||||
CHECK(StillsImageSelected(i, 1, 0)); // factor 1: every image
|
||||
CHECK(StillsImageSelected(i, 7, 10000)); // before all_before: every image
|
||||
CHECK(StillsImageSelected(i, 7, 0) == StillsImageSelected(i, 7, 0));
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ void print_usage(Logger &logger) {
|
||||
logger.Info(" -t<num> Indexing thread pool size (default: 4)");
|
||||
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes)");
|
||||
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
|
||||
logger.Info(" -k<num> Analyse 1 of every <num> images on average (rugnux live stills_downsample, default 1)");
|
||||
logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped");
|
||||
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
|
||||
logger.Info(" 128 module frames)");
|
||||
@@ -86,7 +87,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
|
||||
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
|
||||
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
|
||||
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, size_t puller_slots,
|
||||
const std::vector<double> &rates, bool dropping) {
|
||||
const std::vector<double> &rates, bool dropping, int64_t downsample) {
|
||||
RegisterHDF5Filter();
|
||||
JFJochHDF5Reader reader;
|
||||
reader.ReadFile(file);
|
||||
@@ -142,7 +143,7 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
JFJochReceiverService service(aq_devices, logger, pusher, send_buffer_size_MiB);
|
||||
service.NumThreads(nthreads);
|
||||
service.Indexing(x.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60);
|
||||
service.RugnuxLive(true, nullptr, 4, 60, false, false, downsample);
|
||||
|
||||
// Error, gap and beam-stop pixels of the reader become error pixels of the module frame
|
||||
std::vector<int16_t> frame(x.GetModulesNum() * RAW_MODULE_SIZE);
|
||||
@@ -243,7 +244,8 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
logger.Info("Pre-pass {:.2f} s: beam {:.2f} {:.2f} px (file {:.2f} {:.2f}), distance {:.2f} mm (file {:.2f})",
|
||||
st.prepass_s.value_or(0), st.beam_x_pxl.value_or(NAN), st.beam_y_pxl.value_or(NAN),
|
||||
st.file_beam_x_pxl, st.file_beam_y_pxl, st.distance_mm.value_or(NAN), st.file_distance_mm);
|
||||
logger.Info("Analysed {} indexed {} analysis rate after the pre-pass {:.1f} Hz", st.images_analysed,
|
||||
logger.Info("Analysed {} ({:.3f} of the images) indexed {} analysis rate after the pre-pass {:.1f} Hz",
|
||||
st.images_analysed, static_cast<double>(st.images_analysed) / x.GetImageNum(),
|
||||
st.images_indexed, after_prepass_hz);
|
||||
}
|
||||
logger.Info("Rugnux: {} {} last image at {:.2f} s, done {:.2f} s after it", live.verdict, live.message,
|
||||
@@ -288,9 +290,10 @@ int main(int argc, char **argv) {
|
||||
bool verbose = false;
|
||||
std::vector<double> rates = {0};
|
||||
bool dropping = false;
|
||||
int64_t downsample = 1;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dK::")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dk:K::")) != -1) {
|
||||
switch (opt) {
|
||||
case 'v':
|
||||
verbose = true;
|
||||
@@ -343,6 +346,9 @@ int main(int argc, char **argv) {
|
||||
case 'd':
|
||||
dropping = true;
|
||||
break;
|
||||
case 'k':
|
||||
downsample = atol(optarg);
|
||||
break;
|
||||
case 'K':
|
||||
mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64;
|
||||
break;
|
||||
@@ -378,7 +384,7 @@ int main(int argc, char **argv) {
|
||||
bool ret = false;
|
||||
try {
|
||||
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
|
||||
i_settings, cell, puller_slots, rates, dropping);
|
||||
i_settings, cell, puller_slots, rates, dropping, downsample);
|
||||
} catch (std::exception &e) {
|
||||
logger.Error(e.what());
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
#include "../image_analysis/indexing/IndexerThreadPool.h"
|
||||
#include "../reader/JFJochHDF5Reader.h"
|
||||
#include "../receiver/StillsGPUAnalysis.h"
|
||||
#include "../receiver/StillsSelection.h"
|
||||
#include "../writer/HDF5Objects.h"
|
||||
#include "../writer/RugnuxModelFile.h"
|
||||
#include "../writer/WriteModel.h"
|
||||
@@ -61,6 +62,7 @@ void print_usage() {
|
||||
std::cout << " -e <images> the first images given as empty frames (no diffraction)" << std::endl;
|
||||
std::cout << " -m <file> model (PDB/mmCIF) placed and refined against the merges" << std::endl;
|
||||
std::cout << " -p <num> spots kept per image (default: the file's, as the broker's)" << std::endl;
|
||||
std::cout << " -k <num> analyse 1 of every <num> images on average, as rugnux live stills_downsample" << std::endl;
|
||||
std::cout << " -D <s> deadline of the model step after the last image (default 60, 0 = none)" << std::endl;
|
||||
}
|
||||
|
||||
@@ -84,9 +86,10 @@ int main(int argc, char **argv) {
|
||||
int64_t empty_frames = 0;
|
||||
std::shared_ptr<const RugnuxModel> model;
|
||||
double deadline_s = 60;
|
||||
int64_t downsample = 1;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:k:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'o': prefix = optarg; break;
|
||||
case 's': first = atoll(optarg); break;
|
||||
@@ -104,6 +107,7 @@ int main(int argc, char **argv) {
|
||||
case 'e': empty_frames = atoll(optarg); break;
|
||||
case 'm': model = std::make_shared<const RugnuxModel>(ReadRugnuxModelFile(optarg)); break;
|
||||
case 'D': deadline_s = atof(optarg); break;
|
||||
case 'k': downsample = atoll(optarg); break;
|
||||
case 'C': {
|
||||
UnitCell c{};
|
||||
if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) {
|
||||
@@ -227,6 +231,8 @@ int main(int argc, char **argv) {
|
||||
JFJochReaderRawImage raw;
|
||||
std::vector<uint8_t> empty;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
if (!StillsImageSelected(i, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES))
|
||||
continue;
|
||||
if (rate > 0)
|
||||
std::this_thread::sleep_until(t0 + std::chrono::duration<double>(static_cast<double>(i) / rate));
|
||||
reader.ReadRawImage(first + i, raw);
|
||||
|
||||
Reference in new issue
Block a user