stills geometry trial: the hypotheses a search asks are indexed side by side
Each hypothesis used to be indexed on its own: 8 frames over a four-thread indexer pool, one after another, about 700 of them on a run that searches the distance. The trial ran at 1.5 cores with the card two thirds busy. Now the hypotheses a search compares (the distance scan of every centre, the grid about the scan's peaks, the fine step, the centre grid's coarse and fine rings, the centre candidates) are indexed together, 64 at a time, every frame of every hypothesis one work item, on a pool of as many indexer threads as the image pass has (8). The counts are then read back one by one in the order the search compares them, so every decision is taken exactly as before; a hypothesis' count does not depend on what is indexed beside it. Every trial count, the centre and distance chosen are identical on the five stills sets checked (the 511-703 per-hypothesis debug lines match line for line). Trial alone on the 1 Mpx AGIPD set: 8.3-8.5 s -> 6.9 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
1 parent
077fad9379
commit
19a38e538f
2 files changed
+114
-45
No files matched your search
+7
-2
@@ -1418,8 +1418,13 @@ class Rugnux {
|
||||
int64_t on_lattice = 0; // ... of which on the lattice
|
||||
};
|
||||
std::vector<DataMessage> FindStillsSampleSpots(const std::vector<int> &ordinals, int start_image);
|
||||
StillsTrial IndexStillsSample(const DiffractionExperiment &x, IndexerThreadPool &pool,
|
||||
const std::vector<DataMessage> &frames);
|
||||
// A geometry hypothesis and the frames it is indexed on.
|
||||
struct StillsHypothesis {
|
||||
DiffractionExperiment x;
|
||||
const std::vector<DataMessage> *frames;
|
||||
};
|
||||
// Every hypothesis indexed on its frames, all frames of all hypotheses side by side.
|
||||
std::vector<StillsTrial> IndexStillsSample(const std::vector<StillsHypothesis> &hypotheses, IndexerThreadPool &pool);
|
||||
// Returns whether the run's geometry was moved off the file's.
|
||||
bool DecideStillsGeometry(int start_image, int images_to_process);
|
||||
// Stills: the detector tilt as a hypothesis, refined or held, kept where the frames it was not fitted to
|
||||
|
||||
+107
-43
@@ -670,46 +670,57 @@ std::vector<DataMessage> Rugnux::FindStillsSampleSpots(const std::vector<int> &o
|
||||
return frames;
|
||||
}
|
||||
|
||||
Rugnux::StillsTrial Rugnux::IndexStillsSample(const DiffractionExperiment &x, IndexerThreadPool &pool,
|
||||
const std::vector<DataMessage> &frames) {
|
||||
IndexAndRefine indexer(x, &pool, /*retain_outcomes=*/false);
|
||||
const DiffractionGeometry geom = x.GetDiffractionGeometry();
|
||||
std::vector<Rugnux::StillsTrial> Rugnux::IndexStillsSample(const std::vector<StillsHypothesis> &hypotheses,
|
||||
IndexerThreadPool &pool) {
|
||||
auto settings = config_.spot_finding;
|
||||
settings.enable = true;
|
||||
settings.indexing = true;
|
||||
|
||||
std::vector<std::unique_ptr<IndexAndRefine>> indexers;
|
||||
std::vector<std::pair<size_t, size_t>> items; // (hypothesis, frame)
|
||||
for (size_t h = 0; h < hypotheses.size(); h++) {
|
||||
indexers.push_back(std::make_unique<IndexAndRefine>(hypotheses[h].x, &pool, /*retain_outcomes=*/false));
|
||||
for (size_t i = 0; i < hypotheses[h].frames->size(); i++)
|
||||
items.emplace_back(h, i);
|
||||
}
|
||||
|
||||
std::mutex m;
|
||||
StillsTrial total;
|
||||
std::vector<StillsTrial> total(hypotheses.size());
|
||||
std::atomic<size_t> next = 0;
|
||||
auto worker = [&]() {
|
||||
StillsTrial t;
|
||||
for (size_t i = next.fetch_add(1); i < frames.size() && !cancelled_; i = next.fetch_add(1)) {
|
||||
std::vector<StillsTrial> t(hypotheses.size());
|
||||
for (size_t n = next.fetch_add(1); n < items.size() && !cancelled_; n = next.fetch_add(1)) {
|
||||
const auto [h, i] = items[n];
|
||||
const DiffractionGeometry &geom = hypotheses[h].x.GetDiffractionGeometry();
|
||||
// The spots where the frame put them, at this trial's geometry.
|
||||
DataMessage msg = frames[i];
|
||||
DataMessage msg = (*hypotheses[h].frames)[i];
|
||||
for (auto &s : msg.spots) {
|
||||
s.d_A = geom.PxlToRes(s.x, s.y);
|
||||
s.indexed = false;
|
||||
s.lattice = -1;
|
||||
}
|
||||
t.frames++;
|
||||
if (!indexer.IndexFrameOnly(msg, settings))
|
||||
t[h].frames++;
|
||||
if (!indexers[h]->IndexFrameOnly(msg, settings))
|
||||
continue;
|
||||
t.indexed++;
|
||||
t.spots += static_cast<int64_t>(msg.spots.size());
|
||||
t.on_lattice += msg.spot_count_indexed.value_or(0);
|
||||
t[h].indexed++;
|
||||
t[h].spots += static_cast<int64_t>(msg.spots.size());
|
||||
t[h].on_lattice += msg.spot_count_indexed.value_or(0);
|
||||
}
|
||||
indexer.RethrowIndexerError();
|
||||
std::lock_guard lock(m);
|
||||
total.frames += t.frames;
|
||||
total.indexed += t.indexed;
|
||||
total.spots += t.spots;
|
||||
total.on_lattice += t.on_lattice;
|
||||
for (size_t h = 0; h < t.size(); h++) {
|
||||
total[h].frames += t[h].frames;
|
||||
total[h].indexed += t[h].indexed;
|
||||
total[h].spots += t[h].spots;
|
||||
total[h].on_lattice += t[h].on_lattice;
|
||||
}
|
||||
};
|
||||
std::vector<std::future<void>> futures;
|
||||
for (int i = 0; i < config_.nthreads; ++i)
|
||||
futures.push_back(std::async(std::launch::async, worker));
|
||||
for (auto &f : futures)
|
||||
f.get();
|
||||
for (const auto &indexer : indexers)
|
||||
indexer->RethrowIndexerError();
|
||||
return total;
|
||||
}
|
||||
|
||||
@@ -785,34 +796,61 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
scan_frames.assign(frames.begin(), frames.begin() + std::min<size_t>(frames.size(), scan_count));
|
||||
};
|
||||
find_sample();
|
||||
IndexerThreadPool pool(experiment_.GetIndexingSettings(), IndexerConstruction::OnFirstUse, gpu_work_beside_);
|
||||
// As many indexer threads as the image pass has (RunPipeline): the hypotheses are indexed side by side, and
|
||||
// with the default four the workers waited on the pool.
|
||||
IndexingSettings pool_settings = experiment_.GetIndexingSettings();
|
||||
pool_settings.IndexingThreads(std::clamp(config_.nthreads, 1, 8));
|
||||
IndexerThreadPool pool(pool_settings, IndexerConstruction::OnFirstUse, gpu_work_beside_);
|
||||
|
||||
// A hypothesis is indexed once: the searches below meet the same one more than once - the decided centre
|
||||
// is the file's whenever the file's is kept - and it gives the same count every time.
|
||||
std::map<std::tuple<float, float, float, const std::vector<DataMessage> *>, int64_t> tried;
|
||||
const auto trial = [&](float x, float y, float distance, const std::vector<DataMessage> &on) {
|
||||
const auto key = std::make_tuple(x, y, distance, &on);
|
||||
if (const auto it = tried.find(key); it != tried.end())
|
||||
return it->second;
|
||||
DiffractionExperiment e = experiment_;
|
||||
place(e, x, y, distance);
|
||||
if (stills_user_cell_)
|
||||
e.SetUnitCell(*stills_user_cell_); // the grid asks the user's cell, not the one refined at the file's distance
|
||||
// The distance scan's frames are indexed with a wider acceptance: the scan only has to find the peak, and
|
||||
// it looks at centres a few pixels off and at distances up to half a step off, either of which leaves the
|
||||
// strict tolerance with nothing on a large cell at a long distance (a 92 A cell at 199 mm indexed on no
|
||||
// centre 3 px off). Everything after the scan is counted at the run's own tolerance.
|
||||
if (&on == &scan_frames) {
|
||||
IndexingSettings wide = e.GetIndexingSettings();
|
||||
wide.Tolerance(SCAN_TOLERANCE);
|
||||
e.ImportIndexingSettings(wide);
|
||||
// is the file's whenever the file's is kept - and it gives the same count every time. The hypotheses a
|
||||
// search asks are indexed together (try_all) - a hypothesis' count does not depend on what is indexed
|
||||
// beside it - and then read one by one, in the order the search compares them.
|
||||
using TrialKey = std::tuple<float, float, float, const std::vector<DataMessage> *>;
|
||||
std::map<TrialKey, int64_t> tried;
|
||||
// Hypotheses indexed in one go: enough to keep every worker and the indexer pool busy on 8 frames each,
|
||||
// few enough that their experiments and indexers stay small.
|
||||
constexpr size_t HYPOTHESES_AT_ONCE = 64;
|
||||
const auto try_all = [&](const std::vector<TrialKey> &keys) {
|
||||
std::vector<TrialKey> todo;
|
||||
for (const auto &key : keys)
|
||||
if (!tried.contains(key) && std::find(todo.begin(), todo.end(), key) == todo.end())
|
||||
todo.push_back(key);
|
||||
for (size_t c0 = 0; c0 < todo.size(); c0 += HYPOTHESES_AT_ONCE) {
|
||||
const size_t c1 = std::min(todo.size(), c0 + HYPOTHESES_AT_ONCE);
|
||||
std::vector<StillsHypothesis> hypotheses;
|
||||
for (size_t k = c0; k < c1; k++) {
|
||||
const auto &[x, y, distance, on] = todo[k];
|
||||
DiffractionExperiment e = experiment_;
|
||||
place(e, x, y, distance);
|
||||
if (stills_user_cell_)
|
||||
e.SetUnitCell(*stills_user_cell_); // the grid asks the user's cell, not the one refined at the file's distance
|
||||
// The distance scan's frames are indexed with a wider acceptance: the scan only has to find the peak, and
|
||||
// it looks at centres a few pixels off and at distances up to half a step off, either of which leaves the
|
||||
// strict tolerance with nothing on a large cell at a long distance (a 92 A cell at 199 mm indexed on no
|
||||
// centre 3 px off). Everything after the scan is counted at the run's own tolerance.
|
||||
if (on == &scan_frames) {
|
||||
IndexingSettings wide = e.GetIndexingSettings();
|
||||
wide.Tolerance(SCAN_TOLERANCE);
|
||||
e.ImportIndexingSettings(wide);
|
||||
}
|
||||
hypotheses.push_back(StillsHypothesis{std::move(e), on});
|
||||
}
|
||||
const std::vector<StillsTrial> results = IndexStillsSample(hypotheses, pool);
|
||||
for (size_t k = c0; k < c1; k++) {
|
||||
const auto &[x, y, distance, on] = todo[k];
|
||||
const StillsTrial &t = results[k - c0];
|
||||
logger.Debug("Stills geometry trial: centre ({:.2f},{:.2f}) distance {:.2f} mm - {}/{} frames indexed, "
|
||||
"{} of their {} spots on the lattice", x, y, distance, t.indexed, t.frames,
|
||||
t.on_lattice, t.spots);
|
||||
tried[todo[k]] = t.on_lattice;
|
||||
}
|
||||
}
|
||||
const StillsTrial t = IndexStillsSample(e, pool, on);
|
||||
logger.Debug("Stills geometry trial: centre ({:.2f},{:.2f}) distance {:.2f} mm - {}/{} frames indexed, "
|
||||
"{} of their {} spots on the lattice", x, y, distance, t.indexed, t.frames,
|
||||
t.on_lattice, t.spots);
|
||||
tried[key] = t.on_lattice;
|
||||
return t.on_lattice;
|
||||
};
|
||||
const auto trial = [&](float x, float y, float distance, const std::vector<DataMessage> &on) {
|
||||
const TrialKey key{x, y, distance, &on};
|
||||
try_all({key});
|
||||
return tried.at(key);
|
||||
};
|
||||
|
||||
// The centres: the file's, the measured one and the point-symmetry capture's - two estimators of the
|
||||
@@ -826,6 +864,10 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
float x = file_x, y = file_y;
|
||||
int64_t score_here = 0;
|
||||
const auto decide_centre = [&](float distance) {
|
||||
std::vector<TrialKey> keys{{file_x, file_y, distance, &frames}};
|
||||
for (const auto &[cx, cy] : candidates)
|
||||
keys.emplace_back(cx, cy, distance, &frames);
|
||||
try_all(keys);
|
||||
const int64_t file_score = trial(file_x, file_y, distance, frames);
|
||||
int64_t best = file_score;
|
||||
float best_x = file_x, best_y = file_y;
|
||||
@@ -867,6 +909,10 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
float best_distance = file_distance, best_x = x, best_y = y;
|
||||
int64_t best = score_here;
|
||||
const auto fine = [&](float about, float cx, float cy) {
|
||||
std::vector<TrialKey> keys;
|
||||
for (const float f : {0.98f, 0.99f, 1.0f, 1.01f, 1.02f})
|
||||
keys.emplace_back(cx, cy, about * f, &frames);
|
||||
try_all(keys);
|
||||
for (const float f : {0.98f, 0.99f, 1.0f, 1.01f, 1.02f}) {
|
||||
const int64_t score = trial(cx, cy, about * f, frames);
|
||||
if (score > best) {
|
||||
@@ -893,6 +939,11 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
centres.emplace_back(file_x, file_y);
|
||||
// The local maxima of every centre's scan, strongest first; a distance is looked at once.
|
||||
std::vector<std::pair<int64_t, size_t>> maxima;
|
||||
std::vector<TrialKey> keys;
|
||||
for (const auto &[cx, cy] : centres)
|
||||
for (const float d : grid)
|
||||
keys.emplace_back(cx, cy, d, &scan_frames);
|
||||
try_all(keys);
|
||||
for (const auto &[cx, cy] : centres) {
|
||||
std::vector<int64_t> scan(grid.size());
|
||||
for (size_t i = 0; i < grid.size(); i++)
|
||||
@@ -909,6 +960,12 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
|
||||
float grid_best = file_distance, grid_x = x, grid_y = y;
|
||||
int64_t grid_score = -1;
|
||||
keys.clear();
|
||||
for (const size_t p : peaks)
|
||||
for (size_t i = (p > 0 ? p - 1 : 0); i <= std::min(grid.size() - 1, p + 1); i++)
|
||||
for (const auto &[cx, cy] : centres)
|
||||
keys.emplace_back(cx, cy, grid[i], &strongest);
|
||||
try_all(keys);
|
||||
for (const size_t p : peaks)
|
||||
for (size_t i = (p > 0 ? p - 1 : 0); i <= std::min(grid.size() - 1, p + 1); i++)
|
||||
for (const auto &[cx, cy] : centres) {
|
||||
@@ -977,6 +1034,12 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
int64_t g_best = trial(x, y, distance, strongest);
|
||||
for (const auto &[step, reach] : {std::pair{3.0f, 4}, std::pair{1.0f, 2}}) {
|
||||
const float cx0 = gx, cy0 = gy;
|
||||
std::vector<TrialKey> keys;
|
||||
for (int i = -reach; i <= reach; i++)
|
||||
for (int j = -reach; j <= reach; j++)
|
||||
if (i != 0 || j != 0)
|
||||
keys.emplace_back(cx0 + step * i, cy0 + step * j, distance, &strongest);
|
||||
try_all(keys);
|
||||
for (int i = -reach; i <= reach; i++)
|
||||
for (int j = -reach; j <= reach; j++) {
|
||||
if (i == 0 && j == 0)
|
||||
@@ -989,6 +1052,7 @@ bool Rugnux::DecideStillsGeometry(int start_image, int images_to_process) {
|
||||
}
|
||||
}
|
||||
}
|
||||
try_all({{x, y, distance, &frames}, {gx, gy, distance, &frames}});
|
||||
const int64_t here = trial(x, y, distance, frames), there = trial(gx, gy, distance, frames);
|
||||
logger.Info("Stills beam centre grid at {:.2f} mm: ({:.2f},{:.2f}) puts {} sampled spots on the lattice, "
|
||||
"({:.2f},{:.2f}) {}", distance, gx, gy, there, x, y, here);
|
||||
|
||||
Reference in new issue
Block a user