leonarski_fandClaude Opus 4.8
4c95a289b6
The R-free flags were drawn with a per-reflection std::bernoulli_distribution off a
shared std::mt19937(12345). Two problems: in anomalous mode I(+) and I(-) are separate
merged rows, so a Bijvoet pair could be split across the work and free sets (biasing
R-free); and the draw order depended on merge/call order, so it was not reproducible or
parallel-safe. The resolution-shell grouping was cosmetic — it still drew an independent
Bernoulli per row, so it neither tied mates together nor guaranteed a per-shell quota.
New AssignRfreeFlags() (own file RfreeFlags.{h,cpp}) replaces both call sites (stills
MergeOnTheFly, rotation RotationScaleMerge). It:
- keys each reflection by its Friedel-merged (Laue) ASU index, so all symmetry- and
Friedel-equivalent reflections (including a Bijvoet pair) share one flag and are never
split;
- selects the free set by a deterministic splitmix64 hash of that key (same idiom as the
CC1/2 half-set split), so it is reproducible run-to-run and order-independent;
- stratifies by resolution: exactly round(fraction * n) of the distinct reflections in
each of 20 shells is flagged free.
Verified on an anomalous lyso merge (29536 refl, 5% free): 13548 Bijvoet pairs, 0 split;
free fraction 4.6-5.4% in every resolution shell.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.3 KiB
C++
80 lines
3.3 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "RfreeFlags.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <limits>
|
|
#include <unordered_set>
|
|
|
|
#include "HKLKey.h"
|
|
#include "../../common/ResolutionShells.h"
|
|
|
|
namespace {
|
|
// splitmix64 bit-mix of a key -> uniform double in [0, 1). Same key -> same value, so all
|
|
// mates of a reflection (which share the Laue-ASU key) get the same draw. Same idiom as the
|
|
// CC1/2 half-set split (HalfForImage in Merge.cpp).
|
|
double UniformFromKey(uint64_t key) {
|
|
uint64_t z = key + 0x9e3779b97f4a7c15ULL;
|
|
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL;
|
|
z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL;
|
|
z = z ^ (z >> 31);
|
|
return static_cast<double>(z >> 11) * (1.0 / 9007199254740992.0);
|
|
}
|
|
}
|
|
|
|
void AssignRfreeFlags(std::vector<MergedReflection> &merged, int32_t space_group_number,
|
|
double rfree_fraction) {
|
|
for (auto &r : merged)
|
|
r.rfree_flag = false;
|
|
if (rfree_fraction <= 0.0 || merged.empty())
|
|
return;
|
|
|
|
// Friedel-merged (Laue) ASU key: I(+)/I(-) and every symmetry mate collapse to one key, so a
|
|
// reflection's free/work status is a pure function of this key -> mates never split.
|
|
const HKLKeyGenerator laue_key(/*merge_friedel=*/true, space_group_number);
|
|
|
|
float d_min = std::numeric_limits<float>::max(), d_max = 0.0f;
|
|
for (const auto &r : merged)
|
|
if (std::isfinite(r.d) && r.d > 0.0f) {
|
|
d_min = std::min(d_min, r.d);
|
|
d_max = std::max(d_max, r.d);
|
|
}
|
|
|
|
// No usable resolution: fall back to a global deterministic hash (still mate-consistent).
|
|
if (!(d_min < d_max && d_min > 0.0f)) {
|
|
for (auto &r : merged)
|
|
r.rfree_flag = UniformFromKey(laue_key(r).pack()) < rfree_fraction;
|
|
return;
|
|
}
|
|
|
|
// Stratify by resolution: within each shell take exactly round(fraction * n) of the distinct
|
|
// Laue reflections as the free set, chosen by hash rank. This guarantees ~fraction of the data
|
|
// is free in every shell, deterministically and without splitting any Bijvoet/symmetry set.
|
|
constexpr int n_shells = 20;
|
|
ResolutionShells shells(d_min * 0.999f, d_max * 1.001f, n_shells);
|
|
std::vector<std::vector<std::pair<double, uint64_t>>> shell_keys(n_shells); // (hash, key)
|
|
std::unordered_set<uint64_t> seen;
|
|
seen.reserve(merged.size());
|
|
for (const auto &r : merged) {
|
|
const uint64_t key = laue_key(r).pack();
|
|
if (!seen.insert(key).second) // count each Laue set once (Bijvoet mates share a key)
|
|
continue;
|
|
const auto shell = shells.GetShell(r.d);
|
|
if (shell)
|
|
shell_keys[*shell].push_back({UniformFromKey(key), key});
|
|
}
|
|
|
|
std::unordered_set<uint64_t> free_keys;
|
|
free_keys.reserve(static_cast<size_t>(rfree_fraction * seen.size()) + 1);
|
|
for (auto &keys : shell_keys) {
|
|
std::sort(keys.begin(), keys.end());
|
|
const auto n_free = static_cast<size_t>(std::llround(rfree_fraction * keys.size()));
|
|
for (size_t i = 0; i < n_free; ++i)
|
|
free_keys.insert(keys[i].second);
|
|
}
|
|
for (auto &r : merged)
|
|
r.rfree_flag = free_keys.count(laue_key(r).pack()) > 0;
|
|
}
|