Rugnux live stills: one pre-pass on the first N images, one geometry decision
The serial pre-pass runs once on the first stills_prepass_images frames (new rugnux live setting, default 1000, 100-20000, reset to 1000 by POST /initialize) instead of the doubling stages (500 -> ... -> 2000 frames, stop at 200 geometry frames). One decision: with a cell and at least RUGNUX_LIVE_GEOMETRY_MIN_FRAMES (50) strong frames among them, the beam centre, distance and cell are refined on them; otherwise the header geometry is used exactly as given - the rugnux pre-pass then neither fits nor tries a centre (ProcessConfig::stills_geometry_min_frames), and StillsGPUAnalysis applies beam/distance/cell only where the fit ran. The X-ray-only steps (saturation, shadow, defective pixels, polarization, background ring, powder rings) run as before on the start of the window; the background beam-centre estimate is reported, not applied, when the geometry is kept. Log and live-status notes say "Geometry refined from K of N frames" or "Fewer than 50 strong frames in the first N: header geometry used". Where the hold (at most half of the image buffer) fills before N frames, the pre-pass runs on those held and says so. The first N images are always analysed under stills_downsample. Rotation is unaffected. Two fixes in the stills geometry: - The refit (bundle indexed again at the fitted geometry) was skipped on every modular detector whenever the modules were not being placed, i.e. always live: placing_modules now also needs refine_modules. Offline, where the first fit places the modules and so skips its refit, the fit is run again from the placed geometry, as after a centre the trial moved. - The local beam-centre grid no longer adopts a chance maximum: it moves the centre only from one that puts spots on the lattice. OpenAPI: rugnux_settings/rugnux_status.stills_prepass_images; C++ model, frontend client and python client docs regenerated; frontend settings field. jfjoch_stills_gpu_replay and jfjoch_fpga_stills_test take -P <images>. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3
This commit is contained in:
1 parent
dd0b5687ff
commit
f251178531
38 files changed
+457
-130
No files matched your search
@@ -1019,6 +1019,7 @@ void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) {
|
||||
ret.setRefineSolvent(s.refine_solvent);
|
||||
ret.setForceStill(s.force_still);
|
||||
ret.setStillsDownsample(s.stills_downsample);
|
||||
ret.setStillsPrepassImages(s.stills_prepass_images);
|
||||
ret.setModelLoaded(s.model != nullptr);
|
||||
if (s.model)
|
||||
ret.setModelSizeBytes(s.model_size_bytes);
|
||||
@@ -1029,7 +1030,8 @@ void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings,
|
||||
httplib::Response &response) {
|
||||
state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(),
|
||||
rugnuxSettings.getDeadlineS(), rugnuxSettings.isRefineSolvent(),
|
||||
rugnuxSettings.isForceStill(), rugnuxSettings.getStillsDownsample());
|
||||
rugnuxSettings.isForceStill(), rugnuxSettings.getStillsDownsample(),
|
||||
rugnuxSettings.getStillsPrepassImages());
|
||||
response.status = 200;
|
||||
}
|
||||
|
||||
|
||||
@@ -284,7 +284,8 @@ void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings
|
||||
.deadline_s = input.getRugnux().getDeadlineS(),
|
||||
.refine_solvent = input.getRugnux().isRefineSolvent(),
|
||||
.force_still = input.getRugnux().isForceStill(),
|
||||
.stills_downsample = input.getRugnux().getStillsDownsample()
|
||||
.stills_downsample = input.getRugnux().getStillsDownsample(),
|
||||
.stills_prepass_images = input.getRugnux().getStillsPrepassImages()
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -385,7 +385,7 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) {
|
||||
void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) {
|
||||
if (receiver)
|
||||
receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s, input.refine_solvent,
|
||||
input.force_still, input.stills_downsample);
|
||||
input.force_still, input.stills_downsample, input.stills_prepass_images);
|
||||
rugnux = input;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ struct RugnuxLiveSettings {
|
||||
bool refine_solvent = false; // the refinement of a placed model builds the solvent (rugnux --refine-solvent)
|
||||
bool force_still = false; // rotation collections are processed as stills (rugnux --force-still)
|
||||
int64_t stills_downsample = 1; // serial collections: 1 of every this many images analysed on average
|
||||
int64_t stills_prepass_images = STILLS_PREPASS_IMAGES; // serial collections: the first images the pre-pass looks at
|
||||
std::shared_ptr<const RugnuxModel> model; // null = no model
|
||||
int64_t model_size_bytes = 0; // as uploaded
|
||||
};
|
||||
|
||||
@@ -344,6 +344,7 @@ void JFJochStateMachine::Initialize() {
|
||||
s.refine_solvent = false;
|
||||
s.force_still = false;
|
||||
s.stills_downsample = 1;
|
||||
s.stills_prepass_images = STILLS_PREPASS_IMAGES;
|
||||
s.model = nullptr;
|
||||
s.model_size_bytes = 0;
|
||||
services.SetRugnuxLive(s);
|
||||
@@ -1227,7 +1228,8 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const {
|
||||
}
|
||||
|
||||
void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent,
|
||||
bool force_still, int64_t stills_downsample) {
|
||||
bool force_still, int64_t stills_downsample,
|
||||
int64_t stills_prepass_images) {
|
||||
std::unique_lock ul(m);
|
||||
if (IsRunning())
|
||||
throw WrongDAQStateException("Cannot change rugnux settings during data collection");
|
||||
@@ -1239,6 +1241,7 @@ void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, doubl
|
||||
s.refine_solvent = refine_solvent;
|
||||
s.force_still = force_still;
|
||||
s.stills_downsample = stills_downsample;
|
||||
s.stills_prepass_images = stills_prepass_images;
|
||||
services.SetRugnuxLive(s);
|
||||
}
|
||||
|
||||
|
||||
@@ -249,7 +249,7 @@ public:
|
||||
IndexingSettings GetIndexingSettings() const;
|
||||
|
||||
void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s, bool refine_solvent, bool force_still,
|
||||
int64_t stills_downsample = 1);
|
||||
int64_t stills_downsample = 1, int64_t stills_prepass_images = STILLS_PREPASS_IMAGES);
|
||||
void SetRugnuxModel(std::shared_ptr<const RugnuxModel> model, int64_t size_bytes); // null = remove the model
|
||||
RugnuxLiveSettings GetRugnuxLive() const;
|
||||
RugnuxLiveSnapshot GetRugnuxStatus() const;
|
||||
|
||||
@@ -33,6 +33,8 @@ Rugnux_settings::Rugnux_settings()
|
||||
m_Force_stillIsSet = false;
|
||||
m_Stills_downsample = 1L;
|
||||
m_Stills_downsampleIsSet = false;
|
||||
m_Stills_prepass_images = 1000L;
|
||||
m_Stills_prepass_imagesIsSet = false;
|
||||
|
||||
}
|
||||
|
||||
@@ -97,6 +99,25 @@ bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPr
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (stillsPrepassImagesIsSet())
|
||||
{
|
||||
const int64_t& value = m_Stills_prepass_images;
|
||||
const std::string currentValuePath = _pathPrefix + ".stillsPrepassImages";
|
||||
|
||||
|
||||
if (value < 100ll)
|
||||
{
|
||||
success = false;
|
||||
msg << currentValuePath << ": must be greater than or equal to 100;";
|
||||
}
|
||||
if (value > 20000ll)
|
||||
{
|
||||
success = false;
|
||||
msg << currentValuePath << ": must be less than or equal to 20000;";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return success;
|
||||
}
|
||||
@@ -122,7 +143,10 @@ bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const
|
||||
((!forceStillIsSet() && !rhs.forceStillIsSet()) || (forceStillIsSet() && rhs.forceStillIsSet() && isForceStill() == rhs.isForceStill())) &&
|
||||
|
||||
|
||||
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample()))
|
||||
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample())) &&
|
||||
|
||||
|
||||
((!stillsPrepassImagesIsSet() && !rhs.stillsPrepassImagesIsSet()) || (stillsPrepassImagesIsSet() && rhs.stillsPrepassImagesIsSet() && getStillsPrepassImages() == rhs.getStillsPrepassImages()))
|
||||
|
||||
;
|
||||
}
|
||||
@@ -147,6 +171,8 @@ void to_json(nlohmann::json& j, const Rugnux_settings& o)
|
||||
j["force_still"] = o.m_Force_still;
|
||||
if(o.stillsDownsampleIsSet())
|
||||
j["stills_downsample"] = o.m_Stills_downsample;
|
||||
if(o.stillsPrepassImagesIsSet())
|
||||
j["stills_prepass_images"] = o.m_Stills_prepass_images;
|
||||
|
||||
}
|
||||
|
||||
@@ -182,6 +208,11 @@ void from_json(const nlohmann::json& j, Rugnux_settings& o)
|
||||
j.at("stills_downsample").get_to(o.m_Stills_downsample);
|
||||
o.m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
if(j.find("stills_prepass_images") != j.end())
|
||||
{
|
||||
j.at("stills_prepass_images").get_to(o.m_Stills_prepass_images);
|
||||
o.m_Stills_prepass_imagesIsSet = true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -287,6 +318,23 @@ void Rugnux_settings::unsetStills_downsample()
|
||||
{
|
||||
m_Stills_downsampleIsSet = false;
|
||||
}
|
||||
int64_t Rugnux_settings::getStillsPrepassImages() const
|
||||
{
|
||||
return m_Stills_prepass_images;
|
||||
}
|
||||
void Rugnux_settings::setStillsPrepassImages(int64_t const value)
|
||||
{
|
||||
m_Stills_prepass_images = value;
|
||||
m_Stills_prepass_imagesIsSet = true;
|
||||
}
|
||||
bool Rugnux_settings::stillsPrepassImagesIsSet() const
|
||||
{
|
||||
return m_Stills_prepass_imagesIsSet;
|
||||
}
|
||||
void Rugnux_settings::unsetStills_prepass_images()
|
||||
{
|
||||
m_Stills_prepass_imagesIsSet = false;
|
||||
}
|
||||
|
||||
|
||||
} // namespace org::openapitools::server::model
|
||||
|
||||
@@ -93,12 +93,19 @@ public:
|
||||
bool forceStillIsSet() const;
|
||||
void unsetForce_still();
|
||||
/// <summary>
|
||||
/// Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
|
||||
/// Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass looks at (the first `stills_prepass_images`) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
|
||||
/// </summary>
|
||||
int64_t getStillsDownsample() const;
|
||||
void setStillsDownsample(int64_t const value);
|
||||
bool stillsDownsampleIsSet() const;
|
||||
void unsetStills_downsample();
|
||||
/// <summary>
|
||||
/// Serial collections: the first images the pre-pass looks at. They are held in the image buffer, and the stream waits, while the pre-pass measures the mask, polarization and integration settings on them and, where at least 50 of them are strong indexed frames (a unit cell given), refines the beam centre, distance and cell; with fewer, the geometry in the dataset settings is used as given. The hold takes at most half of the image buffer: where these images do not fit, the pre-pass runs on those held (reported in `/rugnux/status`). Rotation collections are not affected. Reset to 1000 by `POST /initialize`.
|
||||
/// </summary>
|
||||
int64_t getStillsPrepassImages() const;
|
||||
void setStillsPrepassImages(int64_t const value);
|
||||
bool stillsPrepassImagesIsSet() const;
|
||||
void unsetStills_prepass_images();
|
||||
|
||||
friend void to_json(nlohmann::json& j, const Rugnux_settings& o);
|
||||
friend void from_json(const nlohmann::json& j, Rugnux_settings& o);
|
||||
@@ -115,6 +122,8 @@ protected:
|
||||
bool m_Force_stillIsSet;
|
||||
int64_t m_Stills_downsample;
|
||||
bool m_Stills_downsampleIsSet;
|
||||
int64_t m_Stills_prepass_images;
|
||||
bool m_Stills_prepass_imagesIsSet;
|
||||
|
||||
};
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ Rugnux_status::Rugnux_status()
|
||||
m_Force_still = false;
|
||||
m_Stills_downsample = 0L;
|
||||
m_Stills_downsampleIsSet = false;
|
||||
m_Stills_prepass_images = 0L;
|
||||
m_Stills_prepass_imagesIsSet = false;
|
||||
m_Model_loaded = false;
|
||||
m_Model_size_bytes = 0L;
|
||||
m_Model_size_bytesIsSet = false;
|
||||
@@ -54,7 +56,7 @@ bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPref
|
||||
bool success = true;
|
||||
const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix;
|
||||
|
||||
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
@@ -81,6 +83,9 @@ bool Rugnux_status::operator==(const Rugnux_status& rhs) const
|
||||
|
||||
((!stillsDownsampleIsSet() && !rhs.stillsDownsampleIsSet()) || (stillsDownsampleIsSet() && rhs.stillsDownsampleIsSet() && getStillsDownsample() == rhs.getStillsDownsample())) &&
|
||||
|
||||
|
||||
((!stillsPrepassImagesIsSet() && !rhs.stillsPrepassImagesIsSet()) || (stillsPrepassImagesIsSet() && rhs.stillsPrepassImagesIsSet() && getStillsPrepassImages() == rhs.getStillsPrepassImages())) &&
|
||||
|
||||
(isModelLoaded() == rhs.isModelLoaded())
|
||||
&&
|
||||
|
||||
@@ -106,6 +111,8 @@ void to_json(nlohmann::json& j, const Rugnux_status& o)
|
||||
j["force_still"] = o.m_Force_still;
|
||||
if(o.stillsDownsampleIsSet())
|
||||
j["stills_downsample"] = o.m_Stills_downsample;
|
||||
if(o.stillsPrepassImagesIsSet())
|
||||
j["stills_prepass_images"] = o.m_Stills_prepass_images;
|
||||
j["model_loaded"] = o.m_Model_loaded;
|
||||
if(o.modelSizeBytesIsSet())
|
||||
j["model_size_bytes"] = o.m_Model_size_bytes;
|
||||
@@ -128,6 +135,11 @@ void from_json(const nlohmann::json& j, Rugnux_status& o)
|
||||
j.at("stills_downsample").get_to(o.m_Stills_downsample);
|
||||
o.m_Stills_downsampleIsSet = true;
|
||||
}
|
||||
if(j.find("stills_prepass_images") != j.end())
|
||||
{
|
||||
j.at("stills_prepass_images").get_to(o.m_Stills_prepass_images);
|
||||
o.m_Stills_prepass_imagesIsSet = true;
|
||||
}
|
||||
j.at("model_loaded").get_to(o.m_Model_loaded);
|
||||
if(j.find("model_size_bytes") != j.end())
|
||||
{
|
||||
@@ -203,6 +215,23 @@ void Rugnux_status::unsetStills_downsample()
|
||||
{
|
||||
m_Stills_downsampleIsSet = false;
|
||||
}
|
||||
int64_t Rugnux_status::getStillsPrepassImages() const
|
||||
{
|
||||
return m_Stills_prepass_images;
|
||||
}
|
||||
void Rugnux_status::setStillsPrepassImages(int64_t const value)
|
||||
{
|
||||
m_Stills_prepass_images = value;
|
||||
m_Stills_prepass_imagesIsSet = true;
|
||||
}
|
||||
bool Rugnux_status::stillsPrepassImagesIsSet() const
|
||||
{
|
||||
return m_Stills_prepass_imagesIsSet;
|
||||
}
|
||||
void Rugnux_status::unsetStills_prepass_images()
|
||||
{
|
||||
m_Stills_prepass_imagesIsSet = false;
|
||||
}
|
||||
bool Rugnux_status::isModelLoaded() const
|
||||
{
|
||||
return m_Model_loaded;
|
||||
|
||||
@@ -92,6 +92,13 @@ public:
|
||||
bool stillsDownsampleIsSet() const;
|
||||
void unsetStills_downsample();
|
||||
/// <summary>
|
||||
/// Serial collections, the first images the pre-pass looks at
|
||||
/// </summary>
|
||||
int64_t getStillsPrepassImages() const;
|
||||
void setStillsPrepassImages(int64_t const value);
|
||||
bool stillsPrepassImagesIsSet() const;
|
||||
void unsetStills_prepass_images();
|
||||
/// <summary>
|
||||
/// A model was uploaded with `PUT /config/rugnux/model`
|
||||
/// </summary>
|
||||
bool isModelLoaded() const;
|
||||
@@ -119,6 +126,8 @@ protected:
|
||||
|
||||
int64_t m_Stills_downsample;
|
||||
bool m_Stills_downsampleIsSet;
|
||||
int64_t m_Stills_prepass_images;
|
||||
bool m_Stills_prepass_imagesIsSet;
|
||||
bool m_Model_loaded;
|
||||
|
||||
int64_t m_Model_size_bytes;
|
||||
|
||||
+18
-1
@@ -2854,8 +2854,21 @@ components:
|
||||
Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
|
||||
image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
|
||||
which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
|
||||
may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came,
|
||||
looks at (the first `stills_prepass_images`) are all analysed. Images not chosen are written as they came,
|
||||
without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
|
||||
stills_prepass_images:
|
||||
type: integer
|
||||
format: int64
|
||||
minimum: 100
|
||||
maximum: 20000
|
||||
default: 1000
|
||||
description: |
|
||||
Serial collections: the first images the pre-pass looks at. They are held in the image buffer, and the
|
||||
stream waits, while the pre-pass measures the mask, polarization and integration settings on them and,
|
||||
where at least 50 of them are strong indexed frames (a unit cell given), refines the beam centre,
|
||||
distance and cell; with fewer, the geometry in the dataset settings is used as given. The hold takes at
|
||||
most half of the image buffer: where these images do not fit, the pre-pass runs on those held (reported
|
||||
in `/rugnux/status`). Rotation collections are not affected. Reset to 1000 by `POST /initialize`.
|
||||
rugnux_status:
|
||||
type: object
|
||||
required:
|
||||
@@ -2886,6 +2899,10 @@ components:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Serial collections, 1 of every this many images analysed on average
|
||||
stills_prepass_images:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Serial collections, the first images the pre-pass looks at
|
||||
model_loaded:
|
||||
type: boolean
|
||||
description: A model was uploaded with `PUT /config/rugnux/model`
|
||||
|
||||
+28
-25
@@ -291,8 +291,11 @@ The analysis takes the frames without the goniometer; the master file keeps it.
|
||||
with a stream faster than the analysis. Each image is chosen on its own, with probability 1 / `stills_downsample`
|
||||
from a hash of its number: not every n-th image, which would be in step with a periodic pattern of the stream
|
||||
(pump-probe on/off, storage cells) and could take, for example, only the dark frames. The choice is reproducible.
|
||||
The first 2000 images, the largest sample the pre-pass may use, are all analysed. An image not chosen is written as it
|
||||
came, without its analysis. Reset to 1 by `POST /initialize`, as `force_still` and `refine_solvent` are.
|
||||
The images the pre-pass looks at (`stills_prepass_images`, below) are all analysed. An image not chosen is written as
|
||||
it came, without its analysis. Reset to 1 by `POST /initialize`, as `force_still` and `refine_solvent` are.
|
||||
|
||||
`stills_prepass_images` (default 1000, 100 to 20000) is how many of the first images of a serial collection the
|
||||
pre-pass looks at (see [Serial collections](#serial-collections-experimental)). Reset to 1000 by `POST /initialize`.
|
||||
|
||||
The end message, and with it the end of the collection, waits for the analysis. `deadline_s` (default
|
||||
60 s; 0 for no cutoff) bounds that wait: this long after the last image the analysis is cut. `POST /cancel` is
|
||||
@@ -355,35 +358,35 @@ Live processing of serial collections is **experimental**. With rugnux live mode
|
||||
(non-rotation) collection from a DECTRIS detector, in the standard detector mode, is analysed frame by frame: the
|
||||
frames as the stream delivers them go to GPU spot finding, indexing and integration, with rugnux's stills spot
|
||||
finder (adaptive threshold, single-pixel spots kept). Before any frame is analysed, the first
|
||||
500 frames are run through rugnux, in memory, as `rugnux --no-merge` would run on them. That pre-pass
|
||||
decides the beam centre, refines the detector distance (where the dataset settings give a unit cell) and
|
||||
masks the beam-stop shadow and defective pixels; every frame of the collection is then analysed with that
|
||||
geometry and mask. The start message waits for the pre-pass, so it states them, and the end message carries
|
||||
the centre and distance used (`refined_beam_center_x/y`, `refined_detector_distance_mm`,
|
||||
[CBOR](CBOR.md)). The broker's own distance is taken as the beamline's: unlike offline rugnux, the pre-pass
|
||||
does not search the whole distance range. It refines the centre and distance for a detector square to the
|
||||
beam, as the frames are then analysed. At a serial hit rate the distance needs the 500 frames to be refined at
|
||||
all: on fewer, too few index for the geometry refinement to run, and the stated distance is kept. The pre-pass
|
||||
holds and fits on at most half the slots of the image buffer (a slot is the size of an uncompressed
|
||||
frame), so a smaller buffer starts it on as many as that.
|
||||
|
||||
Where a unit cell is given and those frames are too few for the geometry fit (200 indexed frames), frames go on
|
||||
being held and the pre-pass runs again on all of them each time they have doubled, until the fit has its frames, half
|
||||
the image buffer or 2000 frames are held, or the collection ends; frames arriving during a run do not make the next
|
||||
one larger than that, so the last run stays short. The analysis starts when the pre-pass ends and catches up from the
|
||||
frames waiting, so it starts later on a low hit rate.
|
||||
`stills_prepass_images` frames (default 1000) are run through rugnux once, in memory, as `rugnux --no-merge` would run
|
||||
on them. That pre-pass makes one decision about the geometry: where the dataset settings give a unit cell and at least
|
||||
50 of those frames are strong indexed frames, the beam centre, detector distance and cell are refined on them (the
|
||||
bundle indexed again at the fitted geometry until it stays put); otherwise the beam centre, distance and cell of the
|
||||
dataset settings are used exactly as given - neither part of a fit on too few frames nor a centre tried on them. The
|
||||
log and the live status say which: "Geometry refined from K of N frames", or "Fewer than 50 strong frames in the first
|
||||
N: header geometry used" (with, where it was measured, the beam centre the scattered background points to, reported
|
||||
and not applied). Independently of the hits, the same pre-pass measures on the start of those frames (about 60 of them,
|
||||
empty ones as good as any) what needs only the X-ray beam: the beam-stop shadow, defective pixels, saturation, the
|
||||
polarization plane, the background ring and signal radius of the integration, and powder rings. Every frame of the
|
||||
collection is then analysed with that geometry and mask. The start message waits for the pre-pass, so it states them,
|
||||
and the end message carries the centre and distance used (`refined_beam_center_x/y`, `refined_detector_distance_mm`,
|
||||
[CBOR](CBOR.md)). The broker's own distance is taken as the beamline's: unlike offline rugnux, the pre-pass does not
|
||||
search the whole distance range. It refines the centre and distance for a detector square to the beam, as the frames
|
||||
are then analysed. The pre-pass holds its frames in at most half of the image buffer (below): where the
|
||||
`stills_prepass_images` frames do not fit there, it runs on those held, and says so in the log and the live status.
|
||||
The analysis starts when the pre-pass ends and catches up from the frames waiting.
|
||||
|
||||
The geometry is fixed when the pre-pass ends and does not change during the collection, so every frame is analysed
|
||||
and merged at it and the radial profile has one binning throughout. Where the pre-pass could not refine it (too few
|
||||
of its frames indexed, as with a small image buffer, a low hit rate or a chip whose first frames are empty, or no unit
|
||||
cell given), the distance in the header is used for the whole run: this is a warning in the log and a note of the
|
||||
live status, and a wrong header distance then costs resolution and indexed frames.
|
||||
and merged at it and the radial profile has one binning throughout. Where the pre-pass did not refine it (fewer than 50
|
||||
strong frames among its images, as with a low hit rate or a chip whose first frames are empty, or no unit cell
|
||||
given), the header geometry is used for the whole run: a wrong header distance or beam centre then costs resolution
|
||||
and indexed frames, and a larger `stills_prepass_images` gives the fit more frames to find.
|
||||
|
||||
Frames arriving during the pre-pass, or faster than the analysis, wait in at most half of the image buffer, as their
|
||||
compressed bytes, several to a slot; the image routes serve them from there, by number, while they wait. The other
|
||||
half is for the analysed frames, which go to the writer from a slot each, with one slot kept per analysis thread.
|
||||
The pre-pass takes at most 2000 frames, fewer where the waiting half fills first: then it starts on what it has. The
|
||||
receiver's image memory is the image buffer (`image_buffer_MiB`, `-B` of `jfjoch_fpga_stills_test`) and nothing
|
||||
The pre-pass takes `stills_prepass_images` frames, fewer where the waiting half fills first: then it starts on what
|
||||
it has. The receiver's image memory is the image buffer (`image_buffer_MiB`, `-B` of `jfjoch_fpga_stills_test`) and nothing
|
||||
beside it. No stage on the host discards a frame: a frame that finds the waiting half full waits until the analysis
|
||||
has made room, and the wait goes back to the source - from FPGA cards, `FPGAImagePuller` holds the cards' buffers and
|
||||
the cards wait for the host, so analysis slower than the frame rate costs frames at the card, not memory. An analysed
|
||||
|
||||
+3
-2
@@ -73,7 +73,7 @@ Exercises and benchmarks the FPGA data path and receiver on the FPGA card(s).
|
||||
Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector:
|
||||
|
||||
```
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-S <group>] [-X <indexer>] [-t <threads>] [-k <F>] [-m <frames>] [-v] [-K[<MiB>]] <master.h5>
|
||||
jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C a,b,c,al,be,ga] [-S <group>] [-X <indexer>] [-t <threads>] [-k <F>] [-P <images>] [-m <frames>] [-v] [-K[<MiB>]] <master.h5>
|
||||
```
|
||||
|
||||
- The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement
|
||||
@@ -88,7 +88,8 @@ jfjoch_fpga_stills_test [-i <images>] [-s <cards>] [-N <threads>] [-B <MiB>] [-C
|
||||
conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon
|
||||
counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live
|
||||
on (`-v`: each card's progress every 0.5 s; `-k <F>`: analyse 1 of every
|
||||
F images on average, the rugnux live setting `stills_downsample`; `-m <N>`: N frames summed into each image, by
|
||||
F images on average, the rugnux live setting `stills_downsample`; `-P <N>`: the pre-pass on the first N images,
|
||||
the rugnux live setting `stills_prepass_images`; `-m <N>`: N frames summed into each image, by
|
||||
the card up to 256 and above that the largest divisor of N up to 256 on the card and the rest on the host, 32-bit
|
||||
images). Beam centre, distance, energy and the cell
|
||||
come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is
|
||||
|
||||
@@ -11,7 +11,8 @@ Name | Type | Description | Notes
|
||||
**deadline_s** | **float** | Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff. | [optional] [default to 60.0]
|
||||
**refine_solvent** | **bool** | With a model: the refinement also builds the solvent - waters added at difference-map peaks and removed where the maps no longer hold them, waters and ions relabelled (as `rugnux --refine-solvent`). Reset to false by `POST /initialize`. | [optional] [default to False]
|
||||
**force_still** | **bool** | Process DECTRIS rotation collections as stills, on the serial path (pre-pass, GPU analysis, stills merge), as `rugnux --force-still` does. The written file keeps its goniometer. Reset to false by `POST /initialize`. | [optional] [default to False]
|
||||
**stills_downsample** | **int** | Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by `POST /initialize`. | [optional] [default to 1]
|
||||
**stills_downsample** | **int** | Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride, which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass looks at (the first `stills_prepass_images`) are all analysed. Images not chosen are written as they came, without their analysis. 1: every image. Reset to 1 by `POST /initialize`. | [optional] [default to 1]
|
||||
**stills_prepass_images** | **int** | Serial collections: the first images the pre-pass looks at. They are held in the image buffer, and the stream waits, while the pre-pass measures the mask, polarization and integration settings on them and, where at least 50 of them are strong indexed frames (a unit cell given), refines the beam centre, distance and cell; with fewer, the geometry in the dataset settings is used as given. The hold takes at most half of the image buffer: where these images do not fit, the pre-pass runs on those held (reported in `/rugnux/status`). Rotation collections are not affected. Reset to 1000 by `POST /initialize`. | [optional] [default to 1000]
|
||||
|
||||
## Example
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ Name | Type | Description | Notes
|
||||
**refine_solvent** | **bool** | The refinement of a placed model also builds the solvent |
|
||||
**force_still** | **bool** | DECTRIS rotation collections are processed as stills |
|
||||
**stills_downsample** | **int** | Serial collections, 1 of every this many images analysed on average | [optional]
|
||||
**stills_prepass_images** | **int** | Serial collections, the first images the pre-pass looks at | [optional]
|
||||
**model_loaded** | **bool** | A model was uploaded with `PUT /config/rugnux/model` |
|
||||
**model_size_bytes** | **int** | Size of the uploaded model file | [optional]
|
||||
|
||||
|
||||
@@ -2052,11 +2052,21 @@ export type rugnux_settings = {
|
||||
* Serial collections: analyse on average 1 of every this many images, to keep up with a fast stream. Each
|
||||
* image is chosen on its own with probability 1 / stills_downsample from a hash of its number - not a stride,
|
||||
* which would be in step with periodic patterns (pump-probe on/off, storage cells). The images the pre-pass
|
||||
* may use (the first 2000, its largest sample) are all analysed. Images not chosen are written as they came,
|
||||
* looks at (the first `stills_prepass_images`) are all analysed. Images not chosen are written as they came,
|
||||
* without their analysis. 1: every image. Reset to 1 by `POST /initialize`.
|
||||
*
|
||||
*/
|
||||
stills_downsample?: number;
|
||||
/**
|
||||
* Serial collections: the first images the pre-pass looks at. They are held in the image buffer, and the
|
||||
* stream waits, while the pre-pass measures the mask, polarization and integration settings on them and,
|
||||
* where at least 50 of them are strong indexed frames (a unit cell given), refines the beam centre,
|
||||
* distance and cell; with fewer, the geometry in the dataset settings is used as given. The hold takes at
|
||||
* most half of the image buffer: where these images do not fit, the pre-pass runs on those held (reported
|
||||
* in `/rugnux/status`). Rotation collections are not affected. Reset to 1000 by `POST /initialize`.
|
||||
*
|
||||
*/
|
||||
stills_prepass_images?: number;
|
||||
};
|
||||
|
||||
export type rugnux_status = {
|
||||
@@ -2084,6 +2094,10 @@ export type rugnux_status = {
|
||||
* Serial collections, 1 of every this many images analysed on average
|
||||
*/
|
||||
stills_downsample?: number;
|
||||
/**
|
||||
* Serial collections, the first images the pre-pass looks at
|
||||
*/
|
||||
stills_prepass_images?: number;
|
||||
/**
|
||||
* A model was uploaded with `PUT /config/rugnux/model`
|
||||
*/
|
||||
|
||||
@@ -857,7 +857,8 @@ export const zRugnuxSettings = z.object({
|
||||
deadline_s: z.number().gte(0).optional().default(60),
|
||||
refine_solvent: z.boolean().optional().default(false),
|
||||
force_still: z.boolean().optional().default(false),
|
||||
stills_downsample: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1))
|
||||
stills_downsample: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(1)),
|
||||
stills_prepass_images: z.coerce.bigint().gte(BigInt(100)).lte(BigInt(20000)).optional().default(BigInt(1000))
|
||||
});
|
||||
|
||||
export const zRugnuxStatus = z.object({
|
||||
@@ -867,6 +868,7 @@ export const zRugnuxStatus = z.object({
|
||||
refine_solvent: z.boolean(),
|
||||
force_still: z.boolean(),
|
||||
stills_downsample: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
|
||||
stills_prepass_images: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional(),
|
||||
model_loaded: z.boolean(),
|
||||
model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional()
|
||||
});
|
||||
|
||||
@@ -52,6 +52,8 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
const [deadlineError, setDeadlineError] = useState(false);
|
||||
const [downsample, setDownsample] = useState(1);
|
||||
const [downsampleError, setDownsampleError] = useState(false);
|
||||
const [prepassImages, setPrepassImages] = useState(1000);
|
||||
const [prepassImagesError, setPrepassImagesError] = useState(false);
|
||||
const [counter, setCounter] = useState(0);
|
||||
const [file, setFile] = useState<File>();
|
||||
|
||||
@@ -63,20 +65,23 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
setCpus(s.reserved_cpus);
|
||||
setDeadline(s.deadline_s ?? 0);
|
||||
setDownsample(s.stills_downsample ?? 1);
|
||||
setPrepassImages(s.stills_prepass_images ?? 1000);
|
||||
setCounter(c => c + 1);
|
||||
}
|
||||
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s, s?.stills_downsample]);
|
||||
}, [s?.enable, s?.refine_solvent, s?.force_still, s?.reserved_cpus, s?.deadline_s, s?.stills_downsample, s?.stills_prepass_images]);
|
||||
|
||||
const dirty = s !== undefined && (enable !== s.enable || refineSolvent !== s.refine_solvent
|
||||
|| forceStill !== s.force_still || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0)
|
||||
|| downsample !== (s.stills_downsample ?? 1));
|
||||
|| downsample !== (s.stills_downsample ?? 1) || prepassImages !== (s.stills_prepass_images ?? 1000));
|
||||
|
||||
return (
|
||||
<Stack direction={{xs: "column", md: "row"}} spacing={2} alignItems="stretch">
|
||||
<SettingsPanel title="Live processing" dirty={dirty} spacing={2}
|
||||
onUpload={() => settings.submit({enable, refine_solvent: refineSolvent, force_still: forceStill,
|
||||
reserved_cpus: cpus, deadline_s: deadline, stills_downsample: downsample})}
|
||||
uploadDisabled={measuring || cpusError || deadlineError || downsampleError || settings.pending}
|
||||
reserved_cpus: cpus, deadline_s: deadline, stills_downsample: downsample,
|
||||
stills_prepass_images: prepassImages})}
|
||||
uploadDisabled={measuring || cpusError || deadlineError || downsampleError || prepassImagesError
|
||||
|| settings.pending}
|
||||
snackbar={settings.snackbar}>
|
||||
<Stack spacing={1.5} alignItems="flex-start" sx={{textAlign: "left"}}>
|
||||
<FormControlLabel label="Process collections with rugnux live"
|
||||
@@ -97,6 +102,9 @@ function RugnuxSettings({measuring}: { measuring: boolean }) {
|
||||
<NumberTextField label="Serial: analyse 1 of this many images (1 = all)" start_val={downsample} min={1}
|
||||
max={1000000} counter={counter} disabled={measuring} sx={{width: "100%"}}
|
||||
callback={(v, err) => { setDownsample(v); setDownsampleError(err); }}/>
|
||||
<NumberTextField label="Serial: pre-pass on the first this many images" start_val={prepassImages}
|
||||
min={100} max={20000} counter={counter} disabled={measuring} sx={{width: "100%"}}
|
||||
callback={(v, err) => { setPrepassImages(v); setPrepassImagesError(err); }}/>
|
||||
</SettingsPanel>
|
||||
|
||||
<SettingsPanel title="Model for molecular replacement" spacing={2}>
|
||||
|
||||
@@ -30,7 +30,8 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
|
||||
ZMQPreviewSocket *in_zmq_preview_socket,
|
||||
ZMQMetadataSocket *in_zmq_metadata_socket,
|
||||
IndexerThreadPool *in_indexing_thread_pool,
|
||||
int64_t in_downsample)
|
||||
int64_t in_downsample,
|
||||
int64_t in_prepass_images)
|
||||
: JFJochReceiverStream(in_experiment, in_pixel_mask, in_image_puller, in_image_pusher, in_logger,
|
||||
in_forward_nthreads, in_spot_finding_settings, in_current_status, in_plots,
|
||||
in_image_buffer, in_zmq_preview_socket, in_zmq_metadata_socket),
|
||||
@@ -38,6 +39,7 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
|
||||
deadline_s(in_deadline_s),
|
||||
refine_solvent(in_refine_solvent),
|
||||
downsample(in_downsample),
|
||||
prepass_images(in_prepass_images),
|
||||
live_status(in_live_status),
|
||||
indexing_thread_pool(in_indexing_thread_pool),
|
||||
// As offline rugnux's per-image loop: 16 a card, at most the threads given
|
||||
@@ -71,8 +73,8 @@ void JFJochReceiverRugnuxStills::OnStart(std::chrono::steady_clock::time_point r
|
||||
auto analysis = std::make_unique<StillsGPUAnalysis>(
|
||||
analysis_experiment, pixel_mask, analysis_start,
|
||||
indexing_thread_pool, stills_nthreads,
|
||||
// What the pre-pass holds and fits on, at most: the hold full starts it earlier (OnImage)
|
||||
StillsGPUAnalysis::PREPASS_MAX_IMAGES, logger,
|
||||
// What the pre-pass holds and runs on, at most: the hold full starts it earlier (OnImage)
|
||||
prepass_images, logger,
|
||||
*this, live_status, true, model, deadline_s, refine_solvent);
|
||||
std::unique_lock ul(stills_mutex);
|
||||
stills = std::move(analysis);
|
||||
@@ -96,7 +98,7 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
|
||||
// lost here; a source faster than the analysis loses frames at the card.
|
||||
HoldArena::Region region = hold_arena->Allocate(size);
|
||||
if (!region.data || hold_arena->Full(size))
|
||||
stills->StartPrePass();
|
||||
stills->StartPrePass(true);
|
||||
while (!region.data && !cancelled) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
region = hold_arena->Allocate(size);
|
||||
@@ -106,7 +108,7 @@ void JFJochReceiverRugnuxStills::OnImage(uint32_t id, DataMessage &message, std:
|
||||
memcpy(region.data, message.image.GetCompressed(), size);
|
||||
held->release = std::move(region.release);
|
||||
// Not chosen (stills_downsample): written as it came, without its analysis
|
||||
held->analyse = StillsImageSelected(message.number, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES);
|
||||
held->analyse = StillsImageSelected(message.number, downsample, prepass_images);
|
||||
held->message = message;
|
||||
StampImage(held->message, image_puller.GetCurrentFifoUtilization());
|
||||
held->message.image = CompressedImage(region.data, size, message.image.GetWidth(), message.image.GetHeight(),
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "RugnuxLiveStatus.h"
|
||||
#include "HoldArena.h"
|
||||
#include "StillsGPUAnalysis.h"
|
||||
#include "StillsSelection.h"
|
||||
|
||||
// A serial collection analysed frame by frame at the geometry of a pre-pass, which the start message waits for
|
||||
class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSink {
|
||||
@@ -19,6 +20,7 @@ class JFJochReceiverRugnuxStills : public JFJochReceiverStream, public StillsSin
|
||||
const double deadline_s; // after the last image, the model step is cut (0 = never)
|
||||
const bool refine_solvent; // the refinement of the placed model builds the solvent
|
||||
const int64_t downsample; // 1 of every this many images analysed on average (StillsImageSelected)
|
||||
const int64_t prepass_images; // the first images held for the pre-pass
|
||||
RugnuxLiveStatus &live_status;
|
||||
|
||||
IndexerThreadPool *const indexing_thread_pool;
|
||||
@@ -67,7 +69,8 @@ public:
|
||||
ZMQPreviewSocket *zmq_preview_socket = nullptr,
|
||||
ZMQMetadataSocket *zmq_metadata_socket = nullptr,
|
||||
IndexerThreadPool *indexing_thread_pool = nullptr,
|
||||
int64_t downsample = 1);
|
||||
int64_t downsample = 1,
|
||||
int64_t prepass_images = STILLS_PREPASS_IMAGES);
|
||||
~JFJochReceiverRugnuxStills() override;
|
||||
|
||||
bool GetWaitingImage(std::vector<uint8_t> &v, int64_t image_number) override;
|
||||
|
||||
@@ -25,7 +25,8 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic
|
||||
|
||||
JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model,
|
||||
int reserved_cpus, double deadline_s, bool refine_solvent,
|
||||
bool force_still, int64_t stills_downsample) {
|
||||
bool force_still, int64_t stills_downsample,
|
||||
int64_t stills_prepass_images) {
|
||||
std::unique_lock ul_state(state_mutex);
|
||||
if (state != ReceiverState::Idle)
|
||||
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
|
||||
@@ -37,6 +38,7 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
|
||||
rugnux_refine_solvent = refine_solvent;
|
||||
rugnux_force_still = force_still;
|
||||
rugnux_stills_downsample = stills_downsample;
|
||||
rugnux_stills_prepass_images = stills_prepass_images;
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -213,7 +215,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
|
||||
experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads,
|
||||
rugnux_model, rugnux_deadline_s, rugnux_refine_solvent, rugnux_status, spot_finding_settings,
|
||||
receiver_status, plots, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(),
|
||||
indexer_thread_pool.get(), rugnux_stills_downsample);
|
||||
indexer_thread_pool.get(), rugnux_stills_downsample, rugnux_stills_prepass_images);
|
||||
else
|
||||
receiver = std::make_unique<JFJochReceiverLite>(experiment,
|
||||
pixel_mask,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "RugnuxLiveStatus.h"
|
||||
#include "StillsSelection.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
@@ -38,6 +39,7 @@ class JFJochReceiverService {
|
||||
bool rugnux_refine_solvent = false; // the refinement of a placed model builds the solvent
|
||||
bool rugnux_force_still = false; // rotation collections from a DECTRIS detector go the stills way
|
||||
int64_t rugnux_stills_downsample = 1; // serial collections: 1 of every this many images analysed
|
||||
int64_t rugnux_stills_prepass_images = STILLS_PREPASS_IMAGES; // serial collections: the pre-pass's images
|
||||
RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends
|
||||
std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status
|
||||
|
||||
@@ -70,7 +72,8 @@ public:
|
||||
JFJochReceiverService& NumThreads(int64_t input);
|
||||
JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr<const RugnuxModel> model, int reserved_cpus,
|
||||
double deadline_s, bool refine_solvent = false, bool force_still = false,
|
||||
int64_t stills_downsample = 1);
|
||||
int64_t stills_downsample = 1,
|
||||
int64_t stills_prepass_images = STILLS_PREPASS_IMAGES);
|
||||
RugnuxLiveSnapshot GetRugnuxStatus() const;
|
||||
void ClearRugnuxStatus(); // back to Idle, as before any collection
|
||||
// Every PSI collection but a pedestal run through FPGAImagePuller, without rugnux to JFJochReceiverLite (for tests
|
||||
|
||||
@@ -24,7 +24,7 @@ StillsGPUAnalysis::StillsGPUAnalysis(const DiffractionExperiment &in_experiment,
|
||||
logger(in_logger), sink(in_sink), status(in_status), merge_frames(in_merge),
|
||||
hold_limit(std::max<int64_t>(1, max_held)), model(std::move(in_model)), deadline_s(in_deadline_s),
|
||||
refine_solvent(in_refine_solvent) {
|
||||
live.images_needed = std::min({PREPASS_IMAGES, experiment.GetImageNum(), hold_limit});
|
||||
live.images_needed = std::min(experiment.GetImageNum(), hold_limit);
|
||||
live.file_beam_x_pxl = experiment.GetBeamX_pxl();
|
||||
live.file_beam_y_pxl = experiment.GetBeamY_pxl();
|
||||
live.file_distance_mm = experiment.GetDetectorDistance_mm();
|
||||
@@ -37,7 +37,7 @@ StillsGPUAnalysis::StillsGPUAnalysis(const DiffractionExperiment &in_experiment,
|
||||
prepass_future = std::async(std::launch::async, &StillsGPUAnalysis::PrePassThread, this);
|
||||
for (int64_t i = 0; i < nthreads; i++)
|
||||
analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysis::AnalysisThread, this));
|
||||
logger.Info("GPU analysis: {} threads, at least the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES);
|
||||
logger.Info("GPU analysis: {} threads, the first {} images held for the pre-pass", nthreads, live.images_needed);
|
||||
}
|
||||
|
||||
StillsGPUAnalysis::~StillsGPUAnalysis() {
|
||||
@@ -82,11 +82,13 @@ void StillsGPUAnalysis::Hold(const std::shared_ptr<HeldImage> &image) {
|
||||
Publish();
|
||||
}
|
||||
|
||||
void StillsGPUAnalysis::StartPrePass() {
|
||||
void StillsGPUAnalysis::StartPrePass(bool full) {
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
if (prepass_state == PrePassState::Holding)
|
||||
if (prepass_state == PrePassState::Holding) {
|
||||
prepass_state = PrePassState::Closing;
|
||||
hold_full = full;
|
||||
}
|
||||
}
|
||||
prepass_cv.notify_all();
|
||||
}
|
||||
@@ -160,30 +162,26 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
};
|
||||
setup.experiment = experiment;
|
||||
setup.mask = mask;
|
||||
const int64_t all_images = experiment.GetImageNum();
|
||||
const int64_t needed = std::min(experiment.GetImageNum(), hold_limit);
|
||||
std::optional<StillsPrePassResult> result;
|
||||
std::chrono::steady_clock::time_point prepass_start = std::chrono::steady_clock::now();
|
||||
// With a cell, the pre-pass runs again on all that is held once it has doubled, until the fit has its frames, the
|
||||
// end, or PREPASS_MAX_IMAGES
|
||||
const int64_t most_held = std::min({all_images, hold_limit, PREPASS_MAX_IMAGES});
|
||||
for (int64_t needed = std::min(PREPASS_IMAGES, most_held); ;) {
|
||||
std::map<int64_t, std::shared_ptr<HeldImage>> sample;
|
||||
bool last;
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
prepass_cv.wait(ul, [&] {
|
||||
return prepass_state != PrePassState::Holding || static_cast<int64_t>(prepass_images.size()) >= needed;
|
||||
});
|
||||
// Frames that arrived during the previous run do not make this one larger than the bound
|
||||
for (const auto &[n, image]: prepass_images) {
|
||||
if (static_cast<int64_t>(sample.size()) >= most_held)
|
||||
break;
|
||||
sample.emplace(n, image);
|
||||
}
|
||||
last = prepass_state != PrePassState::Holding || static_cast<int64_t>(sample.size()) >= most_held;
|
||||
}
|
||||
if (sample.empty())
|
||||
break;
|
||||
const auto prepass_start = std::chrono::steady_clock::now();
|
||||
std::map<int64_t, std::shared_ptr<HeldImage>> sample;
|
||||
bool full;
|
||||
{
|
||||
std::unique_lock ul(prepass_mutex);
|
||||
prepass_cv.wait(ul, [&] {
|
||||
return prepass_state != PrePassState::Holding || static_cast<int64_t>(prepass_images.size()) >= needed;
|
||||
});
|
||||
sample = prepass_images;
|
||||
full = hold_full && static_cast<int64_t>(sample.size()) < needed;
|
||||
}
|
||||
if (full) {
|
||||
const std::string text = fmt::format("The image buffer held only {} of the {} images asked for the pre-pass: "
|
||||
"it runs on those", sample.size(), needed);
|
||||
logger.Warning("{}", text);
|
||||
status.Note(text);
|
||||
}
|
||||
if (!sample.empty()) {
|
||||
{
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.prepass = RugnuxLivePrePass::Running;
|
||||
@@ -203,13 +201,6 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
} catch (const std::exception &e) {
|
||||
logger.Warning("Pre-pass on {} images failed - {}", sample.size(), e.what());
|
||||
}
|
||||
if (last || !experiment.GetUnitCell()
|
||||
|| (result && result->geometry_frames >= RUGNUX_REFINE_GEOMETRY_FRAMES))
|
||||
break;
|
||||
needed = std::min(2 * static_cast<int64_t>(sample.size()), most_held);
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.prepass = RugnuxLivePrePass::Holding;
|
||||
live.images_needed = needed;
|
||||
}
|
||||
{
|
||||
// The frames are analysed from here on: their slots go back to the buffer once they are sent.
|
||||
@@ -218,15 +209,19 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
prepass_images.clear();
|
||||
}
|
||||
const bool failed = !result;
|
||||
int geometry_frames = 0;
|
||||
// One decision: the geometry the pre-pass refined on enough strong frames, or the one given - nothing between
|
||||
const bool refined = result && result->geometry_frames > 0;
|
||||
// Spots kept per frame: as given for the collection, else rugnux's default (below: the budget the pre-pass kept)
|
||||
setup.experiment.MaxSpotCount(experiment.GetMaxSpotCountGiven().value_or(RUGNUX_MAX_SPOT_COUNT));
|
||||
if (result) {
|
||||
setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm);
|
||||
// The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the refined
|
||||
// distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off)
|
||||
if (result->cell)
|
||||
setup.experiment.SetUnitCell(*result->cell);
|
||||
if (refined) {
|
||||
setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl)
|
||||
.DetectorDistance_mm(result->distance_mm);
|
||||
// The frames are indexed on the cell refined beside that distance, as offline: the stated cell at the
|
||||
// refined distance is off in scale by as much as the distance moved (1.7% on a set whose header was that far off)
|
||||
if (result->cell)
|
||||
setup.experiment.SetUnitCell(*result->cell);
|
||||
}
|
||||
// and on as many spots as the pre-pass kept, as offline: the count given, or the budget it measured (the
|
||||
// broker's per-image default of 250 left a 9M frame of ~380 spots a third short of what its lattice is refined on)
|
||||
setup.experiment.MaxSpotCount(result->max_spot_count);
|
||||
@@ -234,12 +229,11 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
setup.experiment.PolarizationFactor(result->polarization);
|
||||
setup.experiment.ImportBraggIntegrationSettings(result->integration);
|
||||
setup.mask = result->mask;
|
||||
geometry_frames = result->geometry_frames;
|
||||
const auto &result_mask = result->mask.GetMask();
|
||||
std::unique_lock ul(live_mutex);
|
||||
live.beam_x_pxl = result->beam_x_pxl;
|
||||
live.beam_y_pxl = result->beam_y_pxl;
|
||||
live.distance_mm = result->distance_mm;
|
||||
live.beam_x_pxl = setup.experiment.GetBeamX_pxl();
|
||||
live.beam_y_pxl = setup.experiment.GetBeamY_pxl();
|
||||
live.distance_mm = setup.experiment.GetDetectorDistance_mm();
|
||||
live.shadow_pixels = std::count_if(result_mask.begin(), result_mask.end(), [](uint32_t v) {
|
||||
return (v & (1u << PixelMask::BeamStopPixelBit)) != 0;
|
||||
});
|
||||
@@ -248,6 +242,28 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
});
|
||||
} else
|
||||
logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask");
|
||||
if (refined) {
|
||||
const std::string text = fmt::format(
|
||||
"Geometry refined from {} of {} frames: distance {:.2f} -> {:.2f} mm, beam centre ({:.2f}, {:.2f}) -> "
|
||||
"({:.2f}, {:.2f}) px", result->geometry_frames, result->images, experiment.GetDetectorDistance_mm(),
|
||||
setup.experiment.GetDetectorDistance_mm(), experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(),
|
||||
setup.experiment.GetBeamX_pxl(), setup.experiment.GetBeamY_pxl());
|
||||
logger.Info("{}", text);
|
||||
status.Note(text);
|
||||
} else if (result) {
|
||||
std::string text = fmt::format("{}: header geometry used - distance {:.2f} mm, beam centre ({:.2f}, {:.2f}) px",
|
||||
experiment.GetUnitCell()
|
||||
? fmt::format("Fewer than {} strong frames in the first {}",
|
||||
RUGNUX_LIVE_GEOMETRY_MIN_FRAMES, result->images)
|
||||
: std::string("No unit cell given, the geometry is not refined"),
|
||||
experiment.GetDetectorDistance_mm(), experiment.GetBeamX_pxl(),
|
||||
experiment.GetBeamY_pxl());
|
||||
if (result->background_centre)
|
||||
text += fmt::format("; the scattered background puts the beam at ({:.2f}, {:.2f}) px, not applied",
|
||||
result->background_centre->beam_x_pxl, result->background_centre->beam_y_pxl);
|
||||
logger.Warning("{}", text);
|
||||
status.Note(text);
|
||||
}
|
||||
try {
|
||||
setup.mapping = std::make_unique<AzimuthalIntegrationMapping>(setup.experiment, setup.mask);
|
||||
// Retained for the merge, which takes each image's reflections as soon as it is analysed. The frames
|
||||
@@ -257,15 +273,6 @@ void StillsGPUAnalysis::PrePassThread() {
|
||||
// Every frame predicts through the window of the rocking width the pre-pass measured, as offline
|
||||
if (result)
|
||||
indexer->SetStillsWindow(result->window);
|
||||
if (geometry_frames == 0) {
|
||||
const std::string text = fmt::format(
|
||||
"The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the "
|
||||
"whole run is analysed at the distance in the header, {:.2f} mm, beam centre ({:.2f}, {:.2f}) px",
|
||||
setup.experiment.GetDetectorDistance_mm(), setup.experiment.GetBeamX_pxl(),
|
||||
setup.experiment.GetBeamY_pxl());
|
||||
logger.Warning("{}", text);
|
||||
status.Note(text);
|
||||
}
|
||||
if (merge_frames) {
|
||||
merge_experiment = setup.experiment;
|
||||
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
|
||||
|
||||
@@ -47,10 +47,6 @@ public:
|
||||
// Serial frames analysed on the GPU at the geometry of a pre-pass (fixed from there on), and merged beside the analysis
|
||||
class StillsGPUAnalysis {
|
||||
public:
|
||||
// Enough frames for the distance to be refined at a serial hit rate; they wait in the image buffer
|
||||
constexpr static int64_t PREPASS_IMAGES = 500;
|
||||
// The most a pre-pass run on held frames takes: the last run stays short, so the frames arriving meanwhile flow
|
||||
constexpr static int64_t PREPASS_MAX_IMAGES = 4 * PREPASS_IMAGES;
|
||||
constexpr static std::array<double, 2> CHECKPOINTS = {0.5, 0.75};
|
||||
constexpr static int MERGE_THREADS = 8;
|
||||
// The merge after the last frame scales and post-refines every crystal over the whole run (StillsLiveMerge::Final):
|
||||
@@ -59,6 +55,7 @@ public:
|
||||
constexpr static bool FINAL_FULL_SCALING = true;
|
||||
|
||||
// `start`: the collection's start message, as it would be sent without the pre-pass.
|
||||
// `max_held`: the first images held for the pre-pass, at most (fewer where the hold fills: StartPrePass).
|
||||
StillsGPUAnalysis(const DiffractionExperiment &experiment, const PixelMask &mask, const StartMessage &start,
|
||||
IndexerThreadPool *indexing_thread_pool, int64_t nthreads, int64_t max_held, Logger &logger,
|
||||
StillsSink &sink, RugnuxLiveStatus &status, bool merge = true,
|
||||
@@ -67,8 +64,8 @@ public:
|
||||
~StillsGPUAnalysis();
|
||||
|
||||
void Hold(const std::shared_ptr<HeldImage> &image);
|
||||
// The frames held are all the pre-pass gets (the end of the collection, or no room for more): it starts on them
|
||||
void StartPrePass();
|
||||
// The frames held are all the pre-pass gets (the end of the collection, or `full`: no room for more): it starts on them
|
||||
void StartPrePass(bool full = false);
|
||||
// No frame comes after this: every frame is analysed before it returns. Throws if the setup after the pre-pass failed.
|
||||
void Finish();
|
||||
// After Finish, once everything is written; without it the outcome is reported when the analysis goes
|
||||
@@ -112,6 +109,7 @@ private:
|
||||
std::condition_variable prepass_cv;
|
||||
enum class PrePassState {Holding, Closing, Over}; // Closing: what is held is all there will be
|
||||
PrePassState prepass_state = PrePassState::Holding;
|
||||
bool hold_full = false; // Closing as the hold had no room for the images asked
|
||||
std::map<int64_t, std::shared_ptr<HeldImage>> prepass_images;
|
||||
std::latch prepass_done{1};
|
||||
std::exception_ptr setup_error; // set before prepass_done
|
||||
|
||||
@@ -61,6 +61,7 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<i
|
||||
.mask = rugnux.GetPixelMask(),
|
||||
.images = images.size(),
|
||||
.geometry_frames = result.stills_geometry_frames,
|
||||
.background_centre = result.background_beam_centre,
|
||||
.cell = rugnux.GetExperiment().GetUnitCell(),
|
||||
.max_spot_count = rugnux.GetExperiment().GetMaxSpotCount(),
|
||||
.polarization = rugnux.GetExperiment().GetPolarizationFactor(),
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "../common/StillsRunWidth.h"
|
||||
#include "../common/ZeroCopyReturnValue.h"
|
||||
#include "../image_analysis/IndexAndRefine.h" // StillsWindow
|
||||
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
|
||||
|
||||
// An image held until analysed, pointing into `slot`, which goes back unsent with the last copy (the slot is
|
||||
// forgotten then, so that the copy sent with the analysis is the only one the image buffer finds), or into bytes
|
||||
@@ -42,7 +43,11 @@ struct StillsPrePassResult {
|
||||
float distance_mm;
|
||||
PixelMask mask;
|
||||
size_t images = 0;
|
||||
int geometry_frames = 0; // the indexed frames the distance and centre were refined on; 0: kept as stated
|
||||
// The strong frames the distance, centre and cell were refined on; 0: fewer than RUGNUX_LIVE_GEOMETRY_MIN_FRAMES
|
||||
// (or no cell), the geometry kept as stated
|
||||
int geometry_frames = 0;
|
||||
// Where the scattered background puts the beam (the pre-scan's walk), reported only
|
||||
std::optional<BeamCenterEstimate> background_centre;
|
||||
// The cell the pre-pass indexed its frames on: the one stated, or found de novo, as refined beside the distance
|
||||
// (its scale trades with the distance, so the two go together)
|
||||
std::optional<UnitCell> cell;
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
// The first images of a serial collection the live pre-pass looks at, unless set (rugnux_settings.stills_prepass_images);
|
||||
// they wait in the image buffer. The geometry is refined where RUGNUX_LIVE_GEOMETRY_MIN_FRAMES of them are strong.
|
||||
constexpr int64_t STILLS_PREPASS_IMAGES = 1000;
|
||||
|
||||
// Rugnux live stills with a downsizing factor F analyse on average 1 of every F images, each image chosen on its
|
||||
// own with probability 1/F from a hash of its number. A stride would be in step with any period of the stream
|
||||
// (pump-probe on/off, storage cells, module or card interleave) and could take e.g. only the dark frames; the hash
|
||||
|
||||
+9
-3
@@ -1055,10 +1055,16 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
HotPixelFinder::MAX_SAMPLED_FRAMES), observer);
|
||||
|
||||
// The stills geometry refinement above ran from the file's centre and distance, within a few pixels and
|
||||
// millimetres of them; where the frames have just moved either, it is run again from where they are.
|
||||
// millimetres of them; where the frames have just moved either, it is run again from where they are, and so it is
|
||||
// where it placed the modules: that fit rests on spots indexed at the geometry it started from.
|
||||
// Where too few strong frames refined it (stills_geometry_min_frames), the geometry stays as given, untried.
|
||||
const bool geometry_kept = config_.stills_geometry_min_frames > 0 && stills_geometry_frames_ == 0;
|
||||
const bool moved = full && !experiment_.IsRotationIndexing() && config_.beam_center_check && !config_.live
|
||||
&& !geometry_kept && DecideStillsGeometry(start_image, images_to_process);
|
||||
// The fit that placed (or tried to place) the modules was not indexed again at its geometry (RefineStillsGeometry)
|
||||
const bool placed = config_.refine_modules && experiment_.GetModules().size() >= 2 && stills_geometry_frames_ > 0;
|
||||
bool refitted = false;
|
||||
if (full && !experiment_.IsRotationIndexing() && config_.beam_center_check && !config_.live
|
||||
&& DecideStillsGeometry(start_image, images_to_process) && config_.refine_geometry.has_value()) {
|
||||
if (full && (moved || placed) && config_.refine_geometry.has_value()) {
|
||||
RefineStillsGeometry(start_image, end_image, images_to_process, observer);
|
||||
refitted = true;
|
||||
}
|
||||
|
||||
@@ -175,6 +175,9 @@ struct ProcessConfig {
|
||||
// determines the shared beam/distance/cell that per-image refinement cannot reach, and the main pass
|
||||
// re-indexes with it. The value is the number of strong frames fed to the bundle (--refine-geometry).
|
||||
std::optional<int> refine_geometry;
|
||||
// Fewer strong frames than this in the sample and the geometry is kept as given: neither refined nor tried by
|
||||
// the centre and distance trial (DecideStillsGeometry). 0: the fit takes the strong frames there are (offline).
|
||||
int stills_geometry_min_frames = 0;
|
||||
// Within that bundle adjustment, a detector of several modules (the experiment's DetectorModules: the
|
||||
// file's, the reader's panels, or --module-geometry) has each module placed as a rigid block, where
|
||||
// moving them predicts the spots better on frames the move was not fitted to; the placement found is
|
||||
|
||||
@@ -38,6 +38,10 @@ constexpr int RUGNUX_ROTATION_INDEXING_IMAGES = 100;
|
||||
// stills with a known cell.
|
||||
constexpr int RUGNUX_REFINE_GEOMETRY_FRAMES = 200;
|
||||
|
||||
// Strong frames the live stills pre-pass needs among its images to refine the geometry; with fewer, the
|
||||
// collection is analysed at the geometry as given (ProcessConfig::stills_geometry_min_frames).
|
||||
constexpr int RUGNUX_LIVE_GEOMETRY_MIN_FRAMES = 50;
|
||||
|
||||
// Analysis policy. Output policy (file names, formats, compression) belongs to the caller - the CLI
|
||||
// and the viewer write different things in different places. A front end that exposes one of these as
|
||||
// a user option (the CLI's --polarization) applies it AFTER this call, or it is overwritten here.
|
||||
|
||||
@@ -103,6 +103,8 @@ RugnuxLiveSetup MakeRugnuxStillsPrePassSetup(const JFJochReaderDataset &dataset,
|
||||
// As `rugnux -C`: the geometry bundle adjustment needs the cell.
|
||||
if (x.GetUnitCell().has_value())
|
||||
c.refine_geometry = RUGNUX_REFINE_GEOMETRY_FRAMES;
|
||||
// One decision on the images held: refined from enough strong frames, or kept as given - no centre tried on less
|
||||
c.stills_geometry_min_frames = RUGNUX_LIVE_GEOMETRY_MIN_FRAMES;
|
||||
// The broker's distance is the beamline's: refined, not searched for.
|
||||
c.stills_distance_search = false;
|
||||
// The receiver analyses the frames with the detector square to the beam, so the centre and distance are
|
||||
|
||||
+10
-3
@@ -259,8 +259,9 @@ void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_
|
||||
// high-resolution frames reach, and the placement takes every strong frame sampled (see `placing`):
|
||||
// there the sample goes five times further - with or without --panel-refine, so that the geometry of a
|
||||
// run that does not place them is the one it had when they were always measured.
|
||||
const bool placing_modules = !modules_placed_ && experiment_.GetModules().size() >= 2;
|
||||
const int strong_target = refine_frames * (placing_modules ? 20 : 4);
|
||||
const bool modular = !modules_placed_ && experiment_.GetModules().size() >= 2;
|
||||
const bool placing_modules = modular && config_.refine_modules;
|
||||
const int strong_target = refine_frames * (modular ? 20 : 4);
|
||||
|
||||
// The sample is cut into a FIXED number of interleaved stripes - stripe s is sample positions
|
||||
// s, s+STRIPES, s+2*STRIPES, ... - and each stripe stops once it has contributed its share of the
|
||||
@@ -422,6 +423,11 @@ void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_
|
||||
"frames (>= {} spots)",
|
||||
frames.size(), std::min<int>(examined.load(), static_cast<int>(sample.size())),
|
||||
sample.size(), strong.size(), MIN_STRONG_SPOTS);
|
||||
if (static_cast<int>(strong.size()) < config_.stills_geometry_min_frames) {
|
||||
logger.Warning("Geometry refinement: {} strong frames, fewer than the {} it needs; keeping the input geometry",
|
||||
strong.size(), config_.stills_geometry_min_frames);
|
||||
return;
|
||||
}
|
||||
if (ex_n > 2) {
|
||||
const double sxx = ex_xx - ex_x * ex_x / ex_n, sxy = ex_xy - ex_x * ex_y / ex_n, syy = ex_yy - ex_y * ex_y / ex_n;
|
||||
const double slope = sxy / sxx;
|
||||
@@ -1318,7 +1324,8 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
const int64_t here = trial(x, y, distance, frames), there = trial(gx, gy, distance, frames);
|
||||
logger.Info("Stills beam centre grid at {:.2f} mm: ({:.2f},{:.2f}) puts {} sampled spots on the lattice, "
|
||||
"({:.2f},{:.2f}) {}", distance, gx, gy, there, x, y, here);
|
||||
if (here < FILE_KEPT_ABOVE * there) {
|
||||
// Only from a centre that indexes: where nothing does, the grid's best is a chance lattice on a frame or two
|
||||
if (here > 0 && here < FILE_KEPT_ABOVE * there) {
|
||||
x = gx;
|
||||
y = gy;
|
||||
}
|
||||
|
||||
@@ -250,6 +250,66 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_GPUStills_writer_down", "[JFJochReceiver]")
|
||||
REQUIRE_NOTHROW(writer_future.get());
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochIntegrationTest_GPUStills_GeometryRefinedOnModules", "[JFJochReceiver]") {
|
||||
// 60 strong frames of an 8-module detector: the pre-pass refines the geometry on them and indexes them again at
|
||||
// the fit (live, the modules stay as given), so headers 2% apart in distance end at the same distance.
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
RegisterHDF5Filter();
|
||||
|
||||
const auto run = [&](float distance_mm) {
|
||||
DiffractionExperiment experiment(DetJF4M());
|
||||
experiment.ImagesPerTrigger(60).NumTriggers(1).UseInternalPacketGenerator(true)
|
||||
.FilePrefix("gpu_stills_modules").JungfrauConvPhotonCnt(false)
|
||||
.DetectorDistance_mm(distance_mm).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
|
||||
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha = 90, .beta = 90, .gamma = 90});
|
||||
REQUIRE(experiment.GetModulesNum() == 8);
|
||||
PixelMask pixel_mask(experiment);
|
||||
|
||||
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
|
||||
HDF5DataSet dataset(data, "/entry/data/data");
|
||||
HDF5DataSpace file_space(dataset);
|
||||
std::vector<int16_t> image_conv(file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
|
||||
const std::vector<hsize_t> start = {4, 0, 0};
|
||||
const std::vector<hsize_t> size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
|
||||
dataset.ReadVector(image_conv, start, size);
|
||||
std::vector<int16_t> image_raw_geom(experiment.GetModulesNum() * RAW_MODULE_SIZE);
|
||||
ConvertedToRawGeometry(experiment, image_raw_geom.data(), image_conv.data());
|
||||
|
||||
AcquisitionDeviceGroup aq_devices;
|
||||
auto test = std::make_unique<MockAcquisitionDevice>(0, 64);
|
||||
for (int m = 0; m < experiment.GetModulesNum(); m++)
|
||||
test->SetInternalGeneratorFrame((uint16_t *) image_raw_geom.data() + m * RAW_MODULE_SIZE, m);
|
||||
aq_devices.Add(std::move(test));
|
||||
|
||||
NonePusher pusher;
|
||||
JFJochReceiverService service(aq_devices, logger, pusher);
|
||||
service.NumThreads(4);
|
||||
service.Indexing(experiment.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60);
|
||||
service.Start(experiment, pixel_mask, nullptr);
|
||||
service.Stop();
|
||||
return service.GetRugnuxStatus();
|
||||
};
|
||||
const auto refined = [](const RugnuxLiveSnapshot &s) {
|
||||
for (const auto &step: s.steps)
|
||||
for (const auto ¬e: step.notes)
|
||||
if (note.starts_with("Geometry refined from"))
|
||||
return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
const auto a = run(75.0f), b = run(76.5f);
|
||||
REQUIRE(a.stills);
|
||||
REQUIRE(b.stills);
|
||||
CHECK(refined(a));
|
||||
CHECK(refined(b));
|
||||
REQUIRE(a.stills->distance_mm);
|
||||
REQUIRE(b.stills->distance_mm);
|
||||
logger.Info("Refined distance {:.3f} mm from 75 mm, {:.3f} mm from 76.5 mm", a.stills->distance_mm.value(),
|
||||
b.stills->distance_mm.value());
|
||||
CHECK(b.stills->distance_mm.value() == Catch::Approx(a.stills->distance_mm.value()).epsilon(0.001));
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochIntegrationTest_GPUStills_cancel_during_prepass", "[JFJochReceiver]") {
|
||||
// A cancel while the frames are held for the pre-pass ends the collection.
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
|
||||
@@ -186,10 +186,15 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
|
||||
CHECK(end->refined_detector_distance_mm.has_value());
|
||||
REQUIRE(pusher.Start());
|
||||
CHECK(pusher.Start()->beam_center_x == Catch::Approx(end->refined_beam_center_x.value()));
|
||||
// Twelve frames, fewer than the 50 strong ones the geometry fit needs: the header geometry, exactly as given
|
||||
CHECK(end->refined_beam_center_x.value() == x.GetBeamX_pxl());
|
||||
CHECK(end->refined_beam_center_y.value() == x.GetBeamY_pxl());
|
||||
CHECK(end->refined_detector_distance_mm.value() == x.GetDetectorDistance_mm());
|
||||
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
CHECK(live.state == RugnuxLiveState::Done);
|
||||
CHECK(live.cut.empty());
|
||||
CHECK(HasNote(live, "Fewer than 50 strong frames in the first 12: header geometry used"));
|
||||
CHECK(HasNote(live, "Merge of 6 of 12 images (provisional)"));
|
||||
CHECK(HasNote(live, "Merge of 9 of 12 images (provisional)"));
|
||||
CHECK(HasNote(live, "Merge of the collection"));
|
||||
@@ -497,7 +502,7 @@ TEST_CASE("JFJochReceiverRugnux_Stills_NoIndexableFrames", "[JFJochReceiverRugnu
|
||||
CHECK(pusher.End()->refined_beam_center_x == Catch::Approx(x.GetBeamX_pxl()));
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
CHECK(live.state == RugnuxLiveState::Done);
|
||||
CHECK(HasNote(live, "could not refine the geometry"));
|
||||
CHECK(HasNote(live, "header geometry used"));
|
||||
REQUIRE(live.stills);
|
||||
CHECK(live.stills->images_analysed == images);
|
||||
CHECK(live.stills->images_indexed == 0);
|
||||
@@ -686,10 +691,43 @@ TEST_CASE("JFJochReceiverRugnux_Stills_BufferFullDuringPrePass", "[JFJochReceive
|
||||
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
CHECK(live.state == RugnuxLiveState::Done);
|
||||
CHECK(HasNote(live, "The image buffer held only"));
|
||||
REQUIRE(live.stills);
|
||||
CHECK(live.stills->images_analysed == out.status.images_sent);
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochReceiverRugnux_Stills_PrePassImages", "[JFJochReceiverRugnux]") {
|
||||
// The pre-pass looks at the first stills_prepass_images frames only; every frame is analysed
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
RegisterHDF5Filter();
|
||||
|
||||
const int64_t images = 12;
|
||||
DiffractionExperiment x = StillsExperiment(images);
|
||||
PixelMask mask(x);
|
||||
Stream stream(x, images);
|
||||
stream.Frames(0, images);
|
||||
stream.End();
|
||||
|
||||
RecordingPusher pusher;
|
||||
AcquisitionDeviceGroup group;
|
||||
JFJochReceiverService service(group, logger, pusher);
|
||||
service.NumThreads(4);
|
||||
service.Indexing(x.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60, false, false, 1, 5);
|
||||
|
||||
service.Start(x, mask, nullptr, stream.puller);
|
||||
const auto out = service.Stop();
|
||||
|
||||
CHECK(out.status.images_sent == images);
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
CHECK(live.state == RugnuxLiveState::Done);
|
||||
CHECK(HasNote(live, "in the first 5: header geometry used"));
|
||||
CHECK(!HasNote(live, "The image buffer held only"));
|
||||
REQUIRE(live.stills);
|
||||
CHECK(live.stills->images_needed == 5);
|
||||
CHECK(live.stills->images_analysed == images);
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochReceiverRugnux_Rotation", "[JFJochReceiverRugnux]") {
|
||||
// A scan stays with rugnux's own run: the start message goes out at once, and there are no stills results.
|
||||
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "../broker/OpenAPIConvert.h"
|
||||
#include "../broker/ModulePlacementConvert.h"
|
||||
#include "Dataset_settings.h"
|
||||
#include "Rugnux_settings.h"
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using namespace std::literals::chrono_literals;
|
||||
@@ -41,8 +42,9 @@ TEST_CASE("JFJochStateMachine_Initialize_DisablesRugnux") {
|
||||
JFJochStateMachine state_machine(experiment, services, logger);
|
||||
state_machine.AddDetectorSetup(DetJF4M());
|
||||
|
||||
state_machine.SetRugnuxLive(true, 4, 30.0, true, true);
|
||||
state_machine.SetRugnuxLive(true, 4, 30.0, true, true, 2, 5000);
|
||||
REQUIRE(state_machine.GetRugnuxLive().enable);
|
||||
REQUIRE(state_machine.GetRugnuxLive().stills_prepass_images == 5000);
|
||||
REQUIRE(state_machine.GetRugnuxLive().refine_solvent);
|
||||
REQUIRE(state_machine.GetRugnuxLive().force_still);
|
||||
|
||||
@@ -53,11 +55,28 @@ TEST_CASE("JFJochStateMachine_Initialize_DisablesRugnux") {
|
||||
CHECK(!s.enable);
|
||||
CHECK(!s.refine_solvent);
|
||||
CHECK(!s.force_still);
|
||||
CHECK(s.stills_downsample == 1);
|
||||
CHECK(s.stills_prepass_images == 1000);
|
||||
CHECK(s.deadline_s == 30.0);
|
||||
CHECK(s.model == nullptr);
|
||||
CHECK(s.model_size_bytes == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochStateMachine_RugnuxStillsPrepassImages_OpenAPI") {
|
||||
// Left out of a PUT /config/rugnux, the default; outside 100-20000, refused
|
||||
org::openapitools::server::model::Rugnux_settings s;
|
||||
nlohmann::json j = {{"enable", true}, {"reserved_cpus", 4}};
|
||||
j.get_to(s);
|
||||
CHECK(s.getStillsPrepassImages() == STILLS_PREPASS_IMAGES);
|
||||
CHECK_NOTHROW(s.validate());
|
||||
s.setStillsPrepassImages(50);
|
||||
CHECK_THROWS(s.validate());
|
||||
s.setStillsPrepassImages(30000);
|
||||
CHECK_THROWS(s.validate());
|
||||
s.setStillsPrepassImages(5000);
|
||||
CHECK_NOTHROW(s.validate());
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochStateMachine_AdaptiveThreshold_AcceptedOnFPGADetector") {
|
||||
Logger logger("JFJochStateMachine_AdaptiveThreshold_AcceptedOnFPGADetector");
|
||||
JFJochServices services(logger);
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "../common/time_utc.h"
|
||||
#include "../image_analysis/structure_refinement/RugnuxModel.h"
|
||||
#include "../receiver/StillsGPUAnalysis.h"
|
||||
#include "../receiver/StillsSelection.h"
|
||||
|
||||
namespace {
|
||||
struct TestSink : public StillsSink {
|
||||
@@ -26,7 +27,7 @@ namespace {
|
||||
|
||||
DiffractionExperiment StillsExperiment() {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
x.ImagesPerTrigger(StillsGPUAnalysis::PREPASS_IMAGES + 200).NumTriggers(1).DetectorDistance_mm(100).BeamX_pxl(500).BeamY_pxl(250)
|
||||
x.ImagesPerTrigger(STILLS_PREPASS_IMAGES + 200).NumTriggers(1).DetectorDistance_mm(100).BeamX_pxl(500).BeamY_pxl(250)
|
||||
.IncidentEnergy_keV(12.4);
|
||||
return x;
|
||||
}
|
||||
@@ -36,7 +37,7 @@ namespace {
|
||||
auto held = std::make_shared<HeldImage>();
|
||||
held->slot = buffer.GetImageSlot();
|
||||
REQUIRE(held->slot != nullptr);
|
||||
held->message.number = StillsGPUAnalysis::PREPASS_IMAGES + number;
|
||||
held->message.number = STILLS_PREPASS_IMAGES + number;
|
||||
held->message.image_collection_efficiency = 1.0f;
|
||||
return held;
|
||||
}
|
||||
@@ -109,7 +110,7 @@ TEST_CASE("StillsGPUAnalysis_FailingFrameDoesNotStopAnalysis", "[StillsGPUAnalys
|
||||
TestSink sink;
|
||||
sink.analysed = [&](DataMessage &m) {
|
||||
++sunk;
|
||||
if (m.number == StillsGPUAnalysis::PREPASS_IMAGES)
|
||||
if (m.number == STILLS_PREPASS_IMAGES)
|
||||
throw std::runtime_error("not a JFJochException");
|
||||
};
|
||||
StillsGPUAnalysis gpu(x, mask, Start(x), nullptr, 1, 100, logger, sink, status);
|
||||
|
||||
@@ -40,6 +40,8 @@ void print_usage(Logger &logger) {
|
||||
logger.Info(" -X<txt> Indexing (auto|fft|fftw|ffbidx), auto is default");
|
||||
logger.Info(" -t<num> Indexing thread pool size (default: 8, as offline rugnux on stills; the broker's default is 4)");
|
||||
logger.Info(" -k<num> Analyse 1 of every <num> images on average (rugnux live stills_downsample, default 1)");
|
||||
logger.Info(" -P<num> Pre-pass on the first <num> images (rugnux live stills_prepass_images, default {})",
|
||||
STILLS_PREPASS_IMAGES);
|
||||
logger.Info(" -m<num> Frames summed into an image (default 1): the card sums the largest divisor up to 256, the");
|
||||
logger.Info(" host the rest (1000 = 250 on the card x 4 on the host)");
|
||||
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
|
||||
@@ -87,7 +89,8 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
|
||||
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
|
||||
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
|
||||
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, bool verbose,
|
||||
int64_t downsample, const std::optional<gemmi::SpaceGroup> &space_group, int64_t summation) {
|
||||
int64_t downsample, int64_t prepass_images, const std::optional<gemmi::SpaceGroup> &space_group,
|
||||
int64_t summation) {
|
||||
RegisterHDF5Filter();
|
||||
JFJochHDF5Reader reader;
|
||||
reader.ReadFile(file);
|
||||
@@ -150,7 +153,7 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
JFJochReceiverService service(aq_devices, logger, pusher, send_buffer_size_MiB);
|
||||
service.NumThreads(nthreads);
|
||||
service.Indexing(x.GetIndexingSettings());
|
||||
service.RugnuxLive(true, nullptr, 4, 60, false, false, downsample);
|
||||
service.RugnuxLive(true, nullptr, 4, 60, false, false, downsample, prepass_images);
|
||||
|
||||
// Error, gap and beam-stop pixels of the reader become error pixels of the module frame
|
||||
std::vector<int16_t> frame(x.GetModulesNum() * RAW_MODULE_SIZE);
|
||||
@@ -272,11 +275,12 @@ int main(int argc, char **argv) {
|
||||
std::optional<uint32_t> mock_card_MiB;
|
||||
bool verbose = false;
|
||||
int64_t downsample = 1;
|
||||
int64_t prepass_images = STILLS_PREPASS_IMAGES;
|
||||
int64_t summation = 1;
|
||||
std::optional<gemmi::SpaceGroup> space_group;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:S:X:t:k:m:K::")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:S:X:t:k:P:m:K::")) != -1) {
|
||||
switch (opt) {
|
||||
case 'v':
|
||||
verbose = true;
|
||||
@@ -318,6 +322,9 @@ int main(int argc, char **argv) {
|
||||
case 'k':
|
||||
downsample = atol(optarg);
|
||||
break;
|
||||
case 'P':
|
||||
prepass_images = atol(optarg);
|
||||
break;
|
||||
case 'm':
|
||||
summation = atol(optarg);
|
||||
break;
|
||||
@@ -365,7 +372,7 @@ int main(int argc, char **argv) {
|
||||
bool ret = false;
|
||||
try {
|
||||
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
|
||||
i_settings, cell, verbose, downsample, space_group,
|
||||
i_settings, cell, verbose, downsample, prepass_images, space_group,
|
||||
summation);
|
||||
} catch (std::exception &e) {
|
||||
logger.Error(e.what());
|
||||
|
||||
@@ -63,6 +63,8 @@ void print_usage() {
|
||||
std::cout << " -m <file> model (PDB/mmCIF) placed and refined against the merges" << std::endl;
|
||||
std::cout << " -p <num> spots kept per image (default: as rugnux live stills where none is given, 1000)" << std::endl;
|
||||
std::cout << " -k <num> analyse 1 of every <num> images on average, as rugnux live stills_downsample" << std::endl;
|
||||
std::cout << " -P <num> pre-pass on the first <num> images, as rugnux live stills_prepass_images (default "
|
||||
<< STILLS_PREPASS_IMAGES << ")" << std::endl;
|
||||
std::cout << " -D <s> deadline of the model step after the last image (default 60, 0 = none)" << std::endl;
|
||||
}
|
||||
|
||||
@@ -87,9 +89,10 @@ int main(int argc, char **argv) {
|
||||
std::shared_ptr<const RugnuxModel> model;
|
||||
double deadline_s = 60;
|
||||
int64_t downsample = 1;
|
||||
int64_t prepass_images = STILLS_PREPASS_IMAGES;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:k:")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "o:s:n:r:N:b:x:y:C:cFS:Me:m:D:p:k:P:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'o': prefix = optarg; break;
|
||||
case 's': first = atoll(optarg); break;
|
||||
@@ -108,6 +111,7 @@ int main(int argc, char **argv) {
|
||||
case 'm': model = std::make_shared<const RugnuxModel>(ReadRugnuxModelFile(optarg)); break;
|
||||
case 'D': deadline_s = atof(optarg); break;
|
||||
case 'k': downsample = atoll(optarg); break;
|
||||
case 'P': prepass_images = atoll(optarg); break;
|
||||
case 'C': {
|
||||
UnitCell c{};
|
||||
if (sscanf(optarg, "%f,%f,%f,%f,%f,%f", &c.a, &c.b, &c.c, &c.alpha, &c.beta, &c.gamma) != 6) {
|
||||
@@ -227,13 +231,15 @@ int main(int argc, char **argv) {
|
||||
++indexed;
|
||||
};
|
||||
sink.settings = spot_finding;
|
||||
StillsGPUAnalysis gpu(x, mask, start, &pool, nthreads, buffer.GetStatus().total_slots / 2, logger, sink, status,
|
||||
// As the receiver's hold, at most half the buffer: here a frame to a slot
|
||||
StillsGPUAnalysis gpu(x, mask, start, &pool, nthreads,
|
||||
std::min(prepass_images, buffer.GetStatus().total_slots / 2), logger, sink, status,
|
||||
merge, model, deadline_s);
|
||||
|
||||
JFJochReaderRawImage raw;
|
||||
std::vector<uint8_t> empty;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
if (!StillsImageSelected(i, downsample, StillsGPUAnalysis::PREPASS_MAX_IMAGES))
|
||||
if (!StillsImageSelected(i, downsample, prepass_images))
|
||||
continue;
|
||||
if (rate > 0)
|
||||
std::this_thread::sleep_until(t0 + std::chrono::duration<double>(static_cast<double>(i) / rate));
|
||||
|
||||
Reference in new issue
Block a user