FFBIDX on the GPU: the same spots index to the same cells, whatever order the thread blocks finish in
Build Packages / Create release (push) Successful in 8s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m27s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 10m19s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 19m23s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 23m8s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 23m47s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 24m24s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 27m33s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 29m33s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 29m21s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 30m3s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 23m43s
Build Packages / Generate python client (push) Successful in 27s
Build Packages / Build documentation (push) Successful in 1m19s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 24m49s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 30m37s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 31m42s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 28m43s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 32m54s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 35m41s
Build Packages / Unit tests (push) Successful in 2h7m57s
Build Packages / Validation Battery (push) Successful in 38m34s
Build Packages / Create release (push) Successful in 8s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m27s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 10m19s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 19m23s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 23m8s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 23m47s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 24m24s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 27m33s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 29m33s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 29m21s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 30m3s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 23m43s
Build Packages / Generate python client (push) Successful in 27s
Build Packages / Build documentation (push) Successful in 1m19s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 24m49s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 30m37s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 31m42s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 28m43s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 32m54s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 35m41s
Build Packages / Unit tests (push) Successful in 2h7m57s
Build Packages / Validation Battery (push) Successful in 38m34s
Each block of the candidate-vector and the cell search sorts its own candidates and merges them into one shared top list under a lock, in the order the blocks happen to get there. The merge compared values only, so where two candidates scored the same - common on the 30-spot seed the stills path tries first, since the score is a sum of trimmed terms - the place went to whichever block merged first. The same frame then indexed to a slightly different lattice, or not at all, from one run to the next. Ties are now broken by the candidate's identity - the sample index for a vector; the cell vector (with its flip), candidate and rotation sample for a cell - which makes the order total, so the merged lists are the top-N of everything whatever the merge order. The blocks' own lists are already in that order: the block radix sort is stable and its samples ascend with the thread index. Measured on a 100k-frame serial set: 4-21 of ~33k indexed frames came out with a different lattice in every pair of runs, which the stills scaling and partiality chain turned into 0.97-0.99 CC1/2 and thousands of observations in or out of the merge. With the fix, three runs at the default thread count and one at -N 8 give the md5-identical merged .hkl; a per-frame dump (spot list, indexer answer, refined lattice) of 20k frames is identical across three runs where it differed in 4-6 frames before. FFBIDXIndexerCPU_MatchesGPU and Portable_FFBIDXIndexerCPU_SyntheticStill pass. No regression test: a repeat-and-compare test on the library's spot lists, threaded, passed before the fix too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu (cherry picked from commit 9cff12d652088851b81f6a353ac52aa8318799f2)
This commit is contained in:
1 parent
64bbc44e11
commit
c982ba4f65
1 file changed
+52
-12
@@ -1228,10 +1228,18 @@ namespace {
|
||||
// A[a] = B[b0]; b0 += 1
|
||||
// a += 1
|
||||
// ----------------------------
|
||||
if (top_val[n_cand-1] <= cand[0].value)
|
||||
// Candidates are ordered by value, equal values by sample index: a total order, so the global top
|
||||
// list is the same whatever order the blocks merge in (a value tie would otherwise go to the block
|
||||
// that happened to merge first). The block-local lists are in this order already: the block's
|
||||
// radix sort is stable and its samples ascend with the thread index.
|
||||
auto less = [](float_type va, unsigned sa, float_type vb, unsigned sb) {
|
||||
return va < vb || (va == vb && sa < sb);
|
||||
};
|
||||
|
||||
if (!less(cand[0].value, cand[0].sample, top_val[n_cand-1], top_sample[n_cand-1]))
|
||||
return;
|
||||
|
||||
if (cand[n_cand-1].value <= top_val[0]) {
|
||||
if (!less(top_val[0], top_sample[0], cand[n_cand-1].value, cand[n_cand-1].sample)) {
|
||||
for (unsigned i=0; i<n_cand; ++i) {
|
||||
top_val[i] = cand[i].value;
|
||||
top_sample[i] = cand[i].sample;
|
||||
@@ -1246,13 +1254,14 @@ namespace {
|
||||
|
||||
while (top < n_cand) {
|
||||
auto val = top_val[top];
|
||||
const unsigned val_sample = top_sample[top];
|
||||
if (cand1 < cand2) {
|
||||
if (val < cand[cand1].value) {
|
||||
if (less(val, val_sample, cand[cand1].value, cand[cand1].sample)) {
|
||||
cand[cand1].value = val;
|
||||
cand[cand1].sample = top_sample[top];
|
||||
--cand1;
|
||||
}
|
||||
if (cand[cand0].value < cand[cand2].value) {
|
||||
if (less(cand[cand0].value, cand[cand0].sample, cand[cand2].value, cand[cand2].sample)) {
|
||||
top_val[top] = cand[cand0].value;
|
||||
top_sample[top] = cand[cand0].sample;
|
||||
++cand0;
|
||||
@@ -1261,8 +1270,8 @@ namespace {
|
||||
top_sample[top] = cand[cand2].sample;
|
||||
--cand2;
|
||||
}
|
||||
} else if (val > cand[cand0].value) {
|
||||
if (val < cand[cand1].value) {
|
||||
} else if (less(cand[cand0].value, cand[cand0].sample, val, val_sample)) {
|
||||
if (less(val, val_sample, cand[cand1].value, cand[cand1].sample)) {
|
||||
cand[cand1].value = val;
|
||||
cand[cand1].sample = top_sample[top];
|
||||
--cand1;
|
||||
@@ -1289,10 +1298,41 @@ namespace {
|
||||
cell_cand_t<float_type>* __restrict__ cand,
|
||||
const unsigned n_cand) noexcept
|
||||
{
|
||||
if (out.score[n_cand-1] <= cand[0].value)
|
||||
// A total order, as in merge_top_vecs: equal values by cell vector (with its flip bit), candidate
|
||||
// vector and rotation sample, so a tie does not go to whichever block merged first. Within a block
|
||||
// only the rotation sample varies, and the block's stable radix sort leaves it ascending on ties.
|
||||
auto key_less = [](unsigned ca, unsigned va, unsigned ra, unsigned cb, unsigned vb, unsigned rb) {
|
||||
return ca != cb ? ca < cb : (va != vb ? va < vb : ra < rb);
|
||||
};
|
||||
auto out_key = [&](unsigned i, unsigned& c, unsigned& v, unsigned& r) {
|
||||
c = *reinterpret_cast<unsigned*>(&out.z[3u * i]);
|
||||
v = *reinterpret_cast<unsigned*>(&out.x[3u * i]);
|
||||
r = *reinterpret_cast<unsigned*>(&out.y[3u * i]);
|
||||
};
|
||||
auto out_less_cand = [&](unsigned i, const cell_cand_t<float_type>& b) {
|
||||
if (out.score[i] != b.value)
|
||||
return out.score[i] < b.value;
|
||||
unsigned c, v, r;
|
||||
out_key(i, c, v, r);
|
||||
return key_less(c, v, r, b.cell_vec, b.vsample, b.rsample);
|
||||
};
|
||||
auto cand_less_out = [&](const cell_cand_t<float_type>& a, unsigned i) {
|
||||
if (a.value != out.score[i])
|
||||
return a.value < out.score[i];
|
||||
unsigned c, v, r;
|
||||
out_key(i, c, v, r);
|
||||
return key_less(a.cell_vec, a.vsample, a.rsample, c, v, r);
|
||||
};
|
||||
auto cand_less = [&](const cell_cand_t<float_type>& a, const cell_cand_t<float_type>& b) {
|
||||
if (a.value != b.value)
|
||||
return a.value < b.value;
|
||||
return key_less(a.cell_vec, a.vsample, a.rsample, b.cell_vec, b.vsample, b.rsample);
|
||||
};
|
||||
|
||||
if (!cand_less_out(cand[0], n_cand-1))
|
||||
return;
|
||||
|
||||
if (cand[n_cand-1].value <= out.score[0]) {
|
||||
if (!out_less_cand(0, cand[n_cand-1])) {
|
||||
for (unsigned i=0u; i<n_cand; i++) {
|
||||
out.score[i] = cand[i].value;
|
||||
*reinterpret_cast<unsigned*>(&out.x[3u * i]) = cand[i].vsample;
|
||||
@@ -1310,14 +1350,14 @@ namespace {
|
||||
while (top < n_cand) {
|
||||
auto val = out.score[top];
|
||||
if (cand1 < cand2) {
|
||||
if (val < cand[cand1].value) {
|
||||
if (out_less_cand(top, cand[cand1])) {
|
||||
cand[cand1].value = val;
|
||||
cand[cand1].vsample = *reinterpret_cast<unsigned*>(&out.x[3u * top]);
|
||||
cand[cand1].rsample = *reinterpret_cast<unsigned*>(&out.y[3u * top]);
|
||||
cand[cand1].cell_vec = *reinterpret_cast<unsigned*>(&out.z[3u * top]);
|
||||
--cand1;
|
||||
}
|
||||
if (cand[cand0].value < cand[cand2].value) {
|
||||
if (cand_less(cand[cand0], cand[cand2])) {
|
||||
out.score[top] = cand[cand0].value;
|
||||
*reinterpret_cast<unsigned*>(&out.x[3u * top]) = cand[cand0].vsample;
|
||||
*reinterpret_cast<unsigned*>(&out.y[3u * top]) = cand[cand0].rsample;
|
||||
@@ -1330,8 +1370,8 @@ namespace {
|
||||
*reinterpret_cast<unsigned*>(&out.z[3u * top]) = cand[cand2].cell_vec;
|
||||
--cand2;
|
||||
}
|
||||
} else if (val > cand[cand0].value) {
|
||||
if (val < cand[cand1].value) {
|
||||
} else if (cand_less_out(cand[cand0], top)) {
|
||||
if (out_less_cand(top, cand[cand1])) {
|
||||
cand[cand1].value = val;
|
||||
cand[cand1].vsample = *reinterpret_cast<unsigned*>(&out.x[3u * top]);
|
||||
cand[cand1].rsample = *reinterpret_cast<unsigned*>(&out.y[3u * top]);
|
||||
|
||||
Reference in new issue
Block a user