Merge pool-B into rc172
Build Packages / Create release (push) Successful in 48s
Build Packages / build:viewer-tgz:cpu (push) Successful in 11m20s
Build Packages / build:rugnux-tgz (x86_64) (push) Successful in 10m9s
Build Packages / build:viewer-tgz:cuda (push) Successful in 12m7s
Build Packages / build:rugnux:aarch64 (cross) (push) Successful in 6m26s
Build Packages / build:windows:nocuda (push) Successful in 16m59s
Build Packages / build:windows:cuda (push) Successful in 19m33s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 24m3s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 15m28s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 15m5s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 16m9s
Build Packages / build:rugnux:windows (push) Successful in 11m3s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 15m2s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 15m15s
Build Packages / Generate python client (push) Successful in 50s
Build Packages / Build documentation (push) Successful in 44s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 16m59s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 16m13s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 17m4s
Build Packages / build:rpm (rocky8) (push) Successful in 17m52s
Build Packages / build:rpm (rocky9) (push) Successful in 18m6s
Build Packages / Unit tests (push) Successful in 1h14m48s
Build Packages / Create release (push) Successful in 48s
Build Packages / build:viewer-tgz:cpu (push) Successful in 11m20s
Build Packages / build:rugnux-tgz (x86_64) (push) Successful in 10m9s
Build Packages / build:viewer-tgz:cuda (push) Successful in 12m7s
Build Packages / build:rugnux:aarch64 (cross) (push) Successful in 6m26s
Build Packages / build:windows:nocuda (push) Successful in 16m59s
Build Packages / build:windows:cuda (push) Successful in 19m33s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 24m3s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 15m28s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 15m5s
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 16m9s
Build Packages / build:rugnux:windows (push) Successful in 11m3s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 15m2s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 15m15s
Build Packages / Generate python client (push) Successful in 50s
Build Packages / Build documentation (push) Successful in 44s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 16m59s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 16m13s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 17m4s
Build Packages / build:rpm (rocky8) (push) Successful in 17m52s
Build Packages / build:rpm (rocky9) (push) Successful in 18m6s
Build Packages / Unit tests (push) Successful in 1h14m48s
Pooled run B: twin report-only evidence and free-R on the lattice holohedry, information-weighted resolution cutoff with weighted outlier median and ISa rework, sparse-lattice integration and beam-centre arbiter, GPU/CPU collapsed scale upload, geometry walk on the realised residual, --model speed-ups with FFTW structure factors, memory reductions for large-cell sweeps, -A decisions Friedel-merged. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT
This commit is contained in:
commit
c802e7c32c
51 files changed
+2259
-517
No files matched your search
@@ -6,19 +6,20 @@
|
||||
|
||||
#include "CorrelationCoefficient.h"
|
||||
|
||||
void CorrelationCoefficient::Add(double x, double y) {
|
||||
sum_x += x;
|
||||
sum_y += y;
|
||||
sum_x2 += x * x;
|
||||
sum_y2 += y * y;
|
||||
sum_xy += x * y;
|
||||
void CorrelationCoefficient::Add(double x, double y, double w) {
|
||||
sum_x += w * x;
|
||||
sum_y += w * y;
|
||||
sum_x2 += w * x * x;
|
||||
sum_y2 += w * y * y;
|
||||
sum_xy += w * x * y;
|
||||
sum_w += w;
|
||||
n_cc++;
|
||||
}
|
||||
|
||||
double CorrelationCoefficient::GetCC() const {
|
||||
if (n_cc < 2)
|
||||
if (n_cc < 2 || !(sum_w > 0.0))
|
||||
return NAN;
|
||||
const double n = static_cast<double>(n_cc);
|
||||
const double n = sum_w;
|
||||
const double mean_x = sum_x / n;
|
||||
const double mean_y = sum_y / n;
|
||||
|
||||
|
||||
@@ -9,9 +9,12 @@ class CorrelationCoefficient {
|
||||
double sum_x2 = 0.0;
|
||||
double sum_y2 = 0.0;
|
||||
double sum_xy = 0.0;
|
||||
double sum_w = 0.0;
|
||||
int n_cc = 0;
|
||||
public:
|
||||
void Add(double x, double y);
|
||||
void Add(double x, double y) { Add(x, y, 1.0); }
|
||||
// Weighted pair: a plain Pearson when every weight is the same.
|
||||
void Add(double x, double y, double w);
|
||||
[[nodiscard]] double GetCC() const;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
|
||||
// FFTW's planner - every fftwf_plan_* and fftwf_destroy_plan call - shares global state and is not
|
||||
// thread-safe; executing a plan is. So every piece of code that makes or destroys a plan takes this
|
||||
// one lock, not a lock of its own: two private locks do not stop two planners running at once.
|
||||
inline std::mutex &FFTWPlannerMutex() {
|
||||
static std::mutex m;
|
||||
return m;
|
||||
}
|
||||
@@ -64,6 +64,11 @@ struct MergedReflection {
|
||||
float sigma = NAN;
|
||||
float I_half[2] = {NAN, NAN};
|
||||
float sigma_half[2] = {NAN, NAN};
|
||||
// Weight of this reflection in a CC1/2: the precision its half-sets would have had with every
|
||||
// observation at the run's typical frame scale, over the precision they have. 1 on a sweep
|
||||
// without a weak stretch; small for a reflection measured only where the crystal barely
|
||||
// diffracted, whose scaled-up noise would otherwise count as much as a well-measured pair.
|
||||
float cc_weight = 1.0f;
|
||||
float d = 0.0;
|
||||
// Any observation of this reflection sat on an ice ring. The intensity is still merged and
|
||||
// written - the ring contaminates it, it does not make it absent - and this only marks it so
|
||||
|
||||
@@ -52,7 +52,9 @@ Both search merges also drop the **frames whose fitted per-frame scale came out
|
||||
|
||||
### 13.2 Twinning check, and translational pseudo-symmetry
|
||||
|
||||
A Padilla–Yeates $L$-test ($\langle|L|\rangle$, $\langle L^2\rangle$ — 0.500 and 0.333 untwinned, 0.375 and 0.200 for a perfect twin) and the second moment $\langle I^2\rangle/\langle I\rangle^2$ (2.0 for untwinned acentric data, 1.5 for a perfect twin; taken per resolution shell with noise-only shells skipped and Wilson outliers rejected, so a single strong reflection in a collapsed-mean shell cannot skew it) are written to the merged mmCIF as a twinning diagnostic. A merohedral twin law exists only where the Laue class is a proper subgroup of the lattice holohedry, so in the high-symmetry holohedral classes ($4/mmm$, $6/mmm$, $m\bar{3}m$, and $\bar{3}m$ on a rhombohedral lattice) twinning is never flagged, and a low $\langle|L|\rangle$ there is reported as a statistical artefact instead. The low-symmetry holohedral classes ($\bar{1}$, $2/m$, $mmm$) also admit no strictly merohedral law, but they stay eligible for the flag on purpose: *pseudo*-merohedral twinning through an accidentally special metric cannot be ruled out from the symmetry alone, and those are the classes it happens in.
|
||||
A Padilla–Yeates $L$-test ($\langle|L|\rangle$, $\langle L^2\rangle$ — 0.500 and 0.333 untwinned, 0.375 and 0.200 for a perfect twin) and the second moment $\langle I^2\rangle/\langle I\rangle^2$ (2.0 for untwinned acentric data, 1.5 for a perfect twin) are written to the merged mmCIF as a twinning diagnostic. Both are taken on intensities divided by their resolution-shell mean, over the shells whose $\langle I/\sigma\rangle$ reaches 1, with Wilson outliers rejected; the $L$-test pairs are therefore compared on one scale even where two index steps span a steep fall-off, as in phenix.xtriage and ctruncate, and the selection is by shell, never by the individual reflection's $I/\sigma$, which would cut the weak tail and bias $\langle|L|\rangle$ down. The twin fraction is quoted from the statistic that carries the verdict — the $L$-test unless the call rests on the second moment alone — and a second moment is not turned into a fraction under a detected pseudo-translation, which inflates it. A merohedral twin law exists only where the Laue class is a proper subgroup of the lattice holohedry, so in the high-symmetry holohedral classes ($4/mmm$, $6/mmm$, $m\bar{3}m$, and $\bar{3}m$ on a rhombohedral lattice) no twin is called. The $L$-test is still read there, for a different question: merging $I(h)$ with $I(Th)$ under a false operator $T$ gives $(I(h)+I(Th))/2$ whatever the twin fraction, which has the perfect-twin distribution, so $\langle|L|\rangle$ below 0.42 in a holohedral class is reported as **an adopted operator averaging unequal intensities** — the space group is too high, or a twin law was absorbed into the point group — a warning, never a change to the space group. A genuine operator leaves the untwinned 0.5. Reflections that overlap along a very long axis narrow the distribution the same way, which is one reason it stays a warning. The same numbers measured on the P1 merge the space-group search was given, before any point group was adopted, are reported beside them (`_BEFORE_SEARCH`), with that merge's own pseudo-translation declared and the reflections its lattice centring extinguishes left out. The low-symmetry holohedral classes ($\bar{1}$, $2/m$, $mmm$) also admit no strictly merohedral law, but they stay eligible for the twin call on purpose: *pseudo*-merohedral twinning through an accidentally special metric cannot be ruled out from the symmetry alone, and those are the classes it happens in.
|
||||
|
||||
Beside them, on rotation data, the run reports **twin-immune zone evidence** for the operators the adopted point group adds over each of its index-2 subgroups, read on the P1 cross-check merge. Under a twin law $T$, $I_\mathrm{obs}(h)=(1-\alpha)I(h)+\alpha I(Th)$; a reflection whose twin mate is itself up to the subgroup and Friedel is untouched at every $\alpha$, and those are exactly the reflections centric in the group but acentric in the subgroup. They read centric ($\langle|E^2-1|\rangle = 0.968$) if the added operators are real and acentric (0.736) if they are a twin law or a pseudo-symmetry — the one intensity statistic that still separates the two at $\alpha = 0.5$, where every operator statistic reads "real". Each zone is normalised against its own mean in resolution bins (and within the two phase classes of a detected pseudo-translation), read only in shells with $\langle I/\sigma\rangle \ge 5$ because noise inflates every class towards centric, and reported with $n$, a standard error and the centric-over-acentric Wilson log-likelihood ratio in nats (both densities convolved with each reflection's measurement error), beside the acentric control. It is read absolutely, never as the difference to the control: a perfect twin's control (0.541) plus noise inflation would otherwise look like true symmetry. It is **report-only** — nothing in the space-group decision reads it. It is an in-house method: published practice (Yeates' $H$-test, xtriage) excludes these reflections as uninformative about the twin fraction, and no published test was found that uses them positively; the one precedent here is a trigonal crystal whose twin-immune zone read centric and whose higher group was then confirmed by refinement.
|
||||
|
||||
**Translational pseudo-symmetry** — two copies of the contents of the asymmetric unit related by a pure translation that is not a lattice vector — is looked for on every merging run, because it is the classic predictor of a failed molecular replacement, and because it raises the second moment where twinning lowers it, so each can mask the other's test. The detection is the native-Patterson route of Read, Adams & McCoy (see the [references](CPU_DATA_ANALYSIS.md#references)): the largest off-origin peak of a Patterson computed from the merged intensities, as a fraction of the origin peak, with the peak vector then refined against the intensity modulation it should produce — the ratio of the strongest to the weakest bin mean of $\langle E^2\rangle$ over the phase $\mathrm{frac}(\mathbf{h}\cdot\mathbf{u})$. Both halves are scored against a null computed for the crystal at hand rather than against a fixed bound — both against the same intensities **permuted within resolution shells** — the peak against that map, the modulation against the same measurement made on the shuffled intensities and started where the measurement starts — because the noise floor of the peak statistic spans an order of magnitude across data sets, so no fixed percentage means the same thing twice. The shuffle leaves the reflection count, the $E^2$ distribution and the phase-bin populations untouched and removes only the correlation between a reflection's intensity and $\mathbf{h}\cdot\mathbf{u}$, which is the thing the modulation claims to see. Re-running the greedy search from random starting vectors over the *same* intensities does not work as a control: a greedy search started anywhere walks into a real modulation's basin, so it measures the search **and** whatever modulation the crystal carries — measured over 110 merged datasets, single draws of that control reached 199× and 6162× on crystals whose own modulation reads 4.0× and 67×, and on 28 of the 110 it came out at or above the signal it is subtracted from. Requiring both halves is what keeps the false-positive rate down; either alone over-calls by about a factor of two. A translation the merged data are **exactly** invariant under is reported as an undeclared lattice translation instead — a translation the data are exactly invariant under is a lattice vector by definition, so the centring or the cell is wrong, not the packing — and the pseudo-symmetry search continues underneath it, so a real pseudo-translation sitting under an undeclared centring is still found. The finding itself is report-only (the `TNCS_*` keys of `<prefix>_report.txt`): it gates nothing and changes no reflection, no scale and no group — but the axial-absence test (§13.1) and the $L$-test here both correct themselves against the modulation it measures.
|
||||
|
||||
|
||||
@@ -382,9 +382,10 @@ Each batch's $B$ is fitted on **resolution-shell means**, not on single observat
|
||||
|
||||
### 10.7 R-free test-set flags
|
||||
|
||||
A fraction of the unique reflections (`rfree_fraction`, default 0.05) is flagged as a **free (test) set**, written to the output (MTZ `FreeR_flag`, mmCIF `_refln.status` = `f`, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection's **Friedel-merged (Laue) ASU index**, which gives three properties:
|
||||
A fraction of the unique reflections (`rfree_fraction`, default 0.05) is flagged as a **free (test) set**, written to the output (MTZ `FreeR_flag`, mmCIF `_refln.status` = `f`, a text-HKL column) for model validation (§14) and for downstream refinement. The flag is a pure function of the reflection's orbit under the **lattice holohedry** — the point group of the cell's metric, found as twin laws are (Le Page two-folds within 3° obliquity, taking the lattice of the cell's own basis vectors), which contains the merging group — together with its Friedel mate. Where the cell does not carry the merging group (a space group forced on a metric without it), the Friedel-merged (Laue) ASU index of the merging group is used instead. That gives four properties:
|
||||
|
||||
- all symmetry- and Friedel-equivalent reflections share one flag — in particular a Bijvoet pair $I(+)/I(-)$, kept as two separate merged rows in anomalous mode, is **never split** across the work and free sets (which would bias R-free);
|
||||
- **twin-law mates share one flag too**, since a twin law is a lattice symmetry the crystal lacks. Keyed on the merging group instead, nearly every free reflection's twin mate lands in the working set, and in twin refinement the free reflection's calculated intensity then carries the working-set fit. The same set is what phenix.refine generates by default (`use_lattice_symmetry`). It also makes the free set independent of the space group a file is merged in: the merged file, the P1 cross-check and a re-merge in any subgroup carry one free set (nested, where the small-data floor below lifts the fraction of one of them more than another's);
|
||||
- the free/work decision is a deterministic hash of that key, so the same reflection always lands in the same set — reproducible run-to-run and independent of the order in which observations were merged;
|
||||
- the hash depends only on the reflection index, **not** on this dataset's resolution range or which reflections it happens to contain, so a uniform draw takes ~`rfree_fraction` of the distinct reflections free and — crucially — **every dataset of one crystal form gets the same free set**. That cross-dataset consistency is what a multi-dataset campaign (ensemble refinement, PanDDA) requires; a per-shell stratification tied to each dataset's own $d_\mathrm{min}$ would break it.
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ and a run that produced a textbook data set read identically for their first thr
|
||||
`SYMMETRY_AMBIGUITY`, `CENTERING_UNTESTED`, `UNUSABLE_MERGE`, `LOW_COMPLETENESS`, `SWEEP_GAPS`,
|
||||
`GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`,
|
||||
`LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `CANCELLED`, `RESOLUTION_FIT`,
|
||||
`FLIGHT_PATH`. `NONE` when nothing fired. A code appears if and only if its
|
||||
`FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`. `NONE` when nothing fired. A code appears if and only if its
|
||||
warning fired, so the flags and the `WARNING:` lines are two renderings of one list — the closed
|
||||
type for machinery, the open sentence for a person.
|
||||
- **`WARNING_COUNT=`** and the **`WARNING:`** lines follow, in the same section. They are what they
|
||||
|
||||
@@ -632,7 +632,7 @@ IndexAndRefine::DetermineRefineAnalyze(DataMessage &msg, const SpotFindingSettin
|
||||
// AnalyzeIndexing answers "is this frame worth integrating"; msg.indexing_result carries the
|
||||
// stricter "does this frame index on its own", which on rotation is not the same question.
|
||||
if (!AnalyzeIndexing(msg, outcome.experiment, *outcome.lattice_candidate, outcome.extra_lattice_candidates,
|
||||
spots_on_lattice))
|
||||
spots_on_lattice, integrate_every_frame_))
|
||||
return std::nullopt;
|
||||
|
||||
{
|
||||
|
||||
@@ -85,6 +85,8 @@ class IndexAndRefine {
|
||||
// the retained outcome instead - and it is a copy of every reflection of every image, so a caller
|
||||
// that writes no per-image file switches it off.
|
||||
bool keep_reflections_in_message_ = true;
|
||||
// Integrate every rotation frame from the sweep's lattice, whatever fraction of its spots lie on it.
|
||||
bool integrate_every_frame_ = false;
|
||||
std::vector<float> scale_cc;
|
||||
std::vector<std::optional<UnitCell> > unit_cells;
|
||||
|
||||
@@ -129,6 +131,8 @@ public:
|
||||
char GetPredictionCentring() const { return prediction_centring_; }
|
||||
// Whether the outgoing message keeps its own copy of the reflections - see the member.
|
||||
void KeepReflectionsInMessage(bool on) { keep_reflections_in_message_ = on; }
|
||||
// See the member.
|
||||
void IntegrateEveryFrame(bool on) { integrate_every_frame_ = on; }
|
||||
|
||||
// Returns whether the frame indexed (a lattice was found and refined). Integration, when it runs,
|
||||
// is a further step gated on quick_integration.
|
||||
|
||||
@@ -215,8 +215,14 @@ std::vector<Reflection> BraggIntegrationEngine::Finalize(const std::vector<Refle
|
||||
size_t npredicted,
|
||||
const std::vector<BraggFitResult> &results,
|
||||
int64_t image_number) const {
|
||||
// Sized to what is kept, not to what was predicted: this vector is the one every image holds to the
|
||||
// end of the pass, and a background ring lost to the neighbours drops a reflection here - on a
|
||||
// dense pattern more than half of them, which a reserve of npredicted would carry as dead capacity.
|
||||
size_t n_ok = 0;
|
||||
for (size_t i = 0; i < npredicted; ++i)
|
||||
n_ok += results[i].ok ? 1 : 0;
|
||||
std::vector<Reflection> out;
|
||||
out.reserve(npredicted);
|
||||
out.reserve(n_ok);
|
||||
for (size_t i = 0; i < npredicted; ++i) {
|
||||
const auto &fr = results[i];
|
||||
if (!fr.ok)
|
||||
|
||||
@@ -11,13 +11,9 @@
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
namespace {
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
|
||||
// FFTW planning is not thread-safe (execution is); same idiom as FFTIndexerCPU.
|
||||
std::mutex &FftwPlanMutex() {
|
||||
static std::mutex m;
|
||||
return m;
|
||||
}
|
||||
namespace {
|
||||
|
||||
// One real-to-complex 2D FFT workspace of fixed padded size, reused for every forward transform.
|
||||
class Rfft2 {
|
||||
@@ -32,7 +28,7 @@ public:
|
||||
Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) {
|
||||
real.resize(static_cast<size_t>(ny * nx));
|
||||
spectrum.resize(static_cast<size_t>(ny * nxc));
|
||||
std::unique_lock lock(FftwPlanMutex());
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
fwd = fftwf_plan_dft_r2c_2d(static_cast<int>(ny), static_cast<int>(nx), real.data(),
|
||||
reinterpret_cast<fftwf_complex *>(spectrum.data()),
|
||||
FFTW_ESTIMATE);
|
||||
@@ -45,7 +41,7 @@ public:
|
||||
Rfft2(const Rfft2 &) = delete;
|
||||
Rfft2 &operator=(const Rfft2 &) = delete;
|
||||
~Rfft2() {
|
||||
std::unique_lock lock(FftwPlanMutex());
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
if (fwd)
|
||||
fftwf_destroy_plan(fwd);
|
||||
if (bwd)
|
||||
@@ -136,7 +132,7 @@ BeamCenterConvSurfaces1D BeamCenterFFTCPU::LineSurfaces(const std::vector<float>
|
||||
|
||||
fftwf_plan fwd, bwd;
|
||||
{
|
||||
std::unique_lock lock(FftwPlanMutex());
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
fwd = fftwf_plan_dft_r2c_1d(static_cast<int>(nfft), in.data(),
|
||||
reinterpret_cast<fftwf_complex *>(A.data()), FFTW_ESTIMATE);
|
||||
bwd = fftwf_plan_dft_c2r_1d(static_cast<int>(nfft),
|
||||
@@ -190,7 +186,7 @@ BeamCenterConvSurfaces1D BeamCenterFFTCPU::LineSurfaces(const std::vector<float>
|
||||
out.Q = inverse(accQ);
|
||||
out.D = inverse(accD);
|
||||
{
|
||||
std::unique_lock lock(FftwPlanMutex());
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(fwd);
|
||||
fftwf_destroy_plan(bwd);
|
||||
}
|
||||
|
||||
@@ -497,10 +497,12 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
// Cost of a whole geometry over one subset, the two residual families kept apart so the log
|
||||
// can say which of them moved. Each is a mean per residual VALUE, and the number they are
|
||||
// summed over below is the same weighting the solver itself applies.
|
||||
// se_out, where asked for: the standard error of the returned mean over the residual values
|
||||
// it averages.
|
||||
auto joint_cost = [&](Subset s, const double bm[2], const double ds[1], const double rv[3],
|
||||
const double q0[3], const double q1[3], const double q2[3],
|
||||
double &pos_out, double &exc_out) {
|
||||
double cp = 0.0, ce = 0.0;
|
||||
double &pos_out, double &exc_out, double *se_out = nullptr) {
|
||||
double cp = 0.0, ce = 0.0, c4 = 0.0;
|
||||
size_t np = 0, ne = 0;
|
||||
for (size_t oi = 0; oi < n_obs; ++oi) {
|
||||
const Partial *pp = obs[oi];
|
||||
@@ -509,7 +511,10 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
angle_rad(pp->img), pp->h, pp->k, pp->l, sys, orientation);
|
||||
double res[3] = {0, 0, 0};
|
||||
r(bm, ds, det_rot, rv, q0, q1, q2, res);
|
||||
cp += res[0] * res[0] + res[1] * res[1] + res[2] * res[2];
|
||||
for (int j = 0; j < 3; ++j) {
|
||||
cp += res[j] * res[j];
|
||||
c4 += res[j] * res[j] * res[j] * res[j];
|
||||
}
|
||||
np += 3;
|
||||
}
|
||||
for (const int32_t i : selected) {
|
||||
@@ -520,11 +525,17 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
double res = 0.0;
|
||||
r(rv, q0, q1, q2, &res);
|
||||
ce += res * res;
|
||||
c4 += res * res * res * res;
|
||||
++ne;
|
||||
}
|
||||
pos_out = np ? cp / np : 0.0;
|
||||
exc_out = ne ? ce / ne : 0.0;
|
||||
return (np + ne) ? (cp + ce) / static_cast<double>(np + ne) : 0.0;
|
||||
const double n = static_cast<double>(np + ne);
|
||||
const double mean = (np + ne) ? (cp + ce) / n : 0.0;
|
||||
if (se_out)
|
||||
*se_out = (np + ne) > 1 ? std::sqrt(std::max(0.0, c4 / n - mean * mean) / (n - 1.0))
|
||||
: std::numeric_limits<double>::quiet_NaN();
|
||||
return mean;
|
||||
};
|
||||
|
||||
// One problem, both residual families, every block seeded at the geometry it is handed -
|
||||
@@ -596,7 +607,11 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
bool commit = false;
|
||||
if (n_obs >= static_cast<size_t>(settings.min_events)) {
|
||||
double pos_nom = 0.0, exc_nom = 0.0, pos_ref = 0.0, exc_ref = 0.0;
|
||||
const double cv_nom = joint_cost(VAL, beam, dist, axv, p0, p1, p2, pos_nom, exc_nom);
|
||||
double cv_nom_se = 0.0;
|
||||
const double cv_nom = joint_cost(VAL, beam, dist, axv, p0, p1, p2, pos_nom, exc_nom,
|
||||
&cv_nom_se);
|
||||
result.held_out_before = cv_nom;
|
||||
result.held_out_before_se = cv_nom_se;
|
||||
|
||||
// How far a geometry has moved from nominal, in units of one step.
|
||||
const auto beyond_a_step = [&](const double ds[1], const double q1c[3]) {
|
||||
@@ -747,6 +762,11 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
double len_nom[3], ang_nom[3], len_fit[3], ang_fit[3];
|
||||
EffectiveCellFromParams(sys, p1_0, p2_0, len_nom, ang_nom);
|
||||
EffectiveCellFromParams(sys, q1_log, q2_log, len_fit, ang_fit);
|
||||
result.cell_before = UnitCell{static_cast<float>(len_nom[0]), static_cast<float>(len_nom[1]),
|
||||
static_cast<float>(len_nom[2]),
|
||||
static_cast<float>(ang_nom[0] * 180.0 / PI),
|
||||
static_cast<float>(ang_nom[1] * 180.0 / PI),
|
||||
static_cast<float>(ang_nom[2] * 180.0 / PI)};
|
||||
logger.Info("Post-refine GEOM (joint crystal + detector): dist {:.3f} -> {:.3f} mm, beam "
|
||||
"({:.2f},{:.2f}) -> ({:.2f},{:.2f}), cell {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} "
|
||||
"{:.2f} -> {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}, held-out positional "
|
||||
@@ -1016,3 +1036,30 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool HeldOutResidualFell(const PostRefineResult &before, const PostRefineResult &after) {
|
||||
const double noise = std::hypot(before.held_out_before_se, after.held_out_before_se);
|
||||
return std::isfinite(noise) && after.held_out_before < before.held_out_before - noise;
|
||||
}
|
||||
|
||||
int ReindexPushesCellBack(const PostRefineResult &fit, const PostRefineResult &next) {
|
||||
if (!fit.cell_refined)
|
||||
return 0;
|
||||
const double from[3] = {fit.cell_before.a, fit.cell_before.b, fit.cell_before.c};
|
||||
const double to[3] = {fit.cell.a, fit.cell.b, fit.cell.c};
|
||||
const double again[3] = {next.cell_before.a, next.cell_before.b, next.cell_before.c};
|
||||
int axis = -1;
|
||||
double largest = 0.0;
|
||||
for (int j = 0; j < 3; ++j)
|
||||
if (from[j] > 0.0 && std::fabs(to[j] - from[j]) / from[j] > largest) {
|
||||
largest = std::fabs(to[j] - from[j]) / from[j];
|
||||
axis = j;
|
||||
}
|
||||
if (axis < 0 || !(again[axis] > 0.0))
|
||||
return 0;
|
||||
const double move = to[axis] - from[axis];
|
||||
const double back = again[axis] - to[axis];
|
||||
if (!(move * back < 0.0))
|
||||
return 0;
|
||||
return move > 0.0 ? axis + 1 : -(axis + 1);
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
@@ -48,6 +49,17 @@ struct PostRefineResult {
|
||||
// from. Such a move is not refused for its size - it is RATIFIED by re-integrating and re-indexing
|
||||
// at it, which a second lattice does not survive and a wrong header does (see RunAllPasses).
|
||||
bool large_move = false;
|
||||
// The held-out residual at the geometry the fit STARTED from - what the integration this fit was
|
||||
// handed actually realises there, before the fit moves anything - and its standard error over the
|
||||
// held-out values. Two passes at two geometries are compared on this, not on the fit's own
|
||||
// prediction of where it is going (see the geometry walk in RunAllPasses). NaN where the fit had
|
||||
// too few observations to measure it.
|
||||
double held_out_before = std::numeric_limits<double>::quiet_NaN();
|
||||
double held_out_before_se = std::numeric_limits<double>::quiet_NaN();
|
||||
// The cell the fit started from: the one this pass's own indexing refined. Beside `cell` (what the
|
||||
// fit committed), and against the next pass's cell_before, it says whether re-indexing at the
|
||||
// committed geometry ratifies the committed cell.
|
||||
UnitCell cell_before{};
|
||||
// Which test refused the fit and what it wanted, where it was refused; empty where it committed.
|
||||
// Nothing else says it - the geometry that comes out of a refusal is the nominal one whatever
|
||||
// refused it, so a silent refusal cannot be told from a fit that never wanted to move.
|
||||
@@ -119,3 +131,14 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations obs,
|
||||
const CrystalLattice &reference_latt,
|
||||
const PostRefineSettings &settings,
|
||||
Logger &logger);
|
||||
|
||||
// Whether the held-out residual `after` realises at its geometry is below the one `before` realises at
|
||||
// its own by more than the standard error of the difference - the cross-validation noise of the two
|
||||
// measurements. False where either was not measured.
|
||||
bool HeldOutResidualFell(const PostRefineResult &before, const PostRefineResult &after);
|
||||
|
||||
// Whether re-indexing at the geometry `fit` committed pushed the committed cell back: on the cell length
|
||||
// the fit moved most, the cell `next` (the fit of the pass run at that geometry) started from lies on the
|
||||
// side the fit moved away from. Returns +-(axis + 1), signed as the fit's move, where it did; 0 where the
|
||||
// re-indexing followed the fit, or the fit committed nothing.
|
||||
int ReindexPushesCellBack(const PostRefineResult &fit, const PostRefineResult &next);
|
||||
@@ -342,7 +342,8 @@ bool AnalyzeIndexing(DataMessage &message,
|
||||
const DiffractionExperiment &experiment,
|
||||
const CrystalLattice &latt,
|
||||
const std::vector<CrystalLattice> &extra_lattices,
|
||||
int64_t *spots_on_lattice) {
|
||||
int64_t *spots_on_lattice,
|
||||
bool integrate_every_frame) {
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
|
||||
std::vector<uint8_t> indexed_spots(message.spots.size());
|
||||
@@ -441,7 +442,8 @@ bool AnalyzeIndexing(DataMessage &message,
|
||||
// why XDS's INTEGRATE and dials.integrate predict on every image of the sweep. A frame whose spots
|
||||
// largely MISS the lattice is a different matter and is still refused.
|
||||
const bool frame_indexes = lattice_fits && nspots_indexed >= viable_cell_min_spots;
|
||||
const bool integrate_frame = experiment.IsRotationIndexing() ? lattice_fits : frame_indexes;
|
||||
const bool integrate_frame = experiment.IsRotationIndexing() ? (lattice_fits || integrate_every_frame)
|
||||
: frame_indexes;
|
||||
if (integrate_frame) {
|
||||
auto uc = latt.GetUnitCell();
|
||||
if (ok(uc.a) && ok(uc.b) && ok(uc.c) && ok(uc.alpha) && ok(uc.beta) && ok(uc.gamma)) {
|
||||
|
||||
@@ -46,6 +46,9 @@ constexpr float SPOT_BUDGET_SIGNIFICANCE_Z = 3.29f; // 2(1 - Phi(z)) = 0.001
|
||||
// same rate all the way down, and the case a bare argmax gets wrong.
|
||||
int64_t SpotBudgetFromEvidence(const std::vector<int64_t> &indexed, const std::vector<int64_t> &counted);
|
||||
|
||||
// integrate_every_frame: on rotation, integrate the frame from the sweep's lattice even where fewer than
|
||||
// LATTICE_MIN_INDEXED_FRACTION of its spots lie on it - for a sweep whose lattice was accepted on the pooled
|
||||
// spots although its frames, one at a time, cannot clear that floor (see Rugnux).
|
||||
// spots_on_lattice, when given, receives how many of this frame's non-ice spots lie on latt - the
|
||||
// numerator of the frame gate below, reported whatever the gate then decides. The rotation first
|
||||
// pass's acceptance test pools that count over its validation frames, so it needs it from the sparse
|
||||
@@ -54,6 +57,7 @@ bool AnalyzeIndexing(DataMessage &message,
|
||||
const DiffractionExperiment &experiment,
|
||||
const CrystalLattice &latt,
|
||||
const std::vector<CrystalLattice> &extra_lattices = {},
|
||||
int64_t *spots_on_lattice = nullptr);
|
||||
int64_t *spots_on_lattice = nullptr,
|
||||
bool integrate_every_frame = false);
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
static std::mutex fftw_plan_mutex;
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
|
||||
static inline double dot_abs(const Coord& a, const Coord& b) {
|
||||
return std::fabs(a.x * b.x + a.y * b.y + a.z * b.z);
|
||||
@@ -35,7 +35,7 @@ FFTIndexerCPU::FFTIndexerCPU(const IndexingSettings& settings)
|
||||
int n[1] = { H };
|
||||
|
||||
{
|
||||
std::unique_lock ul(fftw_plan_mutex);
|
||||
std::unique_lock ul(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_many_dft_r2c(
|
||||
1, n, nDirections,
|
||||
h_input_fft.data(), nullptr, 1, H,
|
||||
@@ -49,7 +49,7 @@ FFTIndexerCPU::FFTIndexerCPU(const IndexingSettings& settings)
|
||||
|
||||
|
||||
FFTIndexerCPU::~FFTIndexerCPU() {
|
||||
std::unique_lock ul(fftw_plan_mutex);
|
||||
std::unique_lock ul(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
}
|
||||
|
||||
|
||||
@@ -389,7 +389,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportReflections() {
|
||||
out.emplace_back(mr);
|
||||
}
|
||||
|
||||
AssignRfreeFlags(out, space_group, scaling_settings.GetRfreeFraction());
|
||||
AssignRfreeFlags(out, space_group, scaling_settings.GetRfreeFraction(), 500, reference_cell);
|
||||
ApplyFrenchWilson(out, space_group);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,26 @@ namespace {
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<double> CCHalfFrameFactors(const std::vector<double> &frame_scale,
|
||||
const std::vector<int64_t> &n_obs) {
|
||||
auto g_of = [&](size_t f) {
|
||||
return std::isfinite(frame_scale[f]) && frame_scale[f] > 0.0 ? frame_scale[f] : 1.0;
|
||||
};
|
||||
double s3 = 0.0, s2 = 0.0;
|
||||
for (size_t f = 0; f < frame_scale.size(); ++f) {
|
||||
const double g = g_of(f);
|
||||
s3 += n_obs[f] * g * g * g;
|
||||
s2 += n_obs[f] * g * g;
|
||||
}
|
||||
const double g_ref = s2 > 0.0 ? s3 / s2 : 1.0;
|
||||
std::vector<double> factor(frame_scale.size());
|
||||
for (size_t f = 0; f < frame_scale.size(); ++f) {
|
||||
const double r = g_ref / g_of(f);
|
||||
factor[f] = std::max(1.0, r * r);
|
||||
}
|
||||
return factor;
|
||||
}
|
||||
|
||||
ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedReflection> &merged,
|
||||
double cc_target, Logger &logger) {
|
||||
ResolutionCutoffResult result;
|
||||
@@ -82,7 +102,7 @@ ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedRefle
|
||||
const double s = 1.0 / (static_cast<double>(m.d) * m.d);
|
||||
int b = static_cast<int>((s - s_lo) / bin_w);
|
||||
b = std::clamp(b, 0, N_FIT_BINS - 1);
|
||||
bin_cc[b].Add(m.I_half[0], m.I_half[1]);
|
||||
bin_cc[b].Add(m.I_half[0], m.I_half[1], m.cc_weight);
|
||||
++bin_n[b];
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,14 @@ struct ResolutionCutoffResult {
|
||||
std::string note; // human-readable description of the decision (for logging)
|
||||
};
|
||||
|
||||
// The per-frame factor of the CC1/2 weight (MergedReflection::cc_weight): max(1, (G_ref/G)^2), the
|
||||
// variance an observation from a frame at scale G carries over the same observation at the typical
|
||||
// scale G_ref = sum G^3 / sum G^2 over the observations (n_obs[f] of them on frame f). Precision-
|
||||
// weighted, so frames the crystal barely diffracted on cannot drag the typical scale down however many
|
||||
// of them there are. A frame without a finite positive scale counts as G = 1.
|
||||
std::vector<double> CCHalfFrameFactors(const std::vector<double> &frame_scale,
|
||||
const std::vector<int64_t> &n_obs);
|
||||
|
||||
ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedReflection> &merged,
|
||||
double cc_target, Logger &logger);
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include <unordered_set>
|
||||
|
||||
#include "HKLKey.h"
|
||||
#include "gemmi/twin.hpp"
|
||||
|
||||
namespace {
|
||||
// splitmix64 bit-mix of a key -> uniform double in [0, 1). Same key -> same value, so all
|
||||
@@ -20,10 +21,36 @@ namespace {
|
||||
z = z ^ (z >> 31);
|
||||
return static_cast<double>(z >> 11) * (1.0 / 9007199254740992.0);
|
||||
}
|
||||
|
||||
|
||||
// The rotations of the lattice's own point group - its holohedry - found from the metric as for
|
||||
// twin laws (Le Page two-folds, with phenix.xtriage's default obliquity of 3 degrees, so a
|
||||
// pseudo-merohedral lattice counts as its higher metric; grouping too much only clusters the
|
||||
// free set, grouping too little splits twin mates). The centring is taken as P: the lattice of
|
||||
// the cell's own basis vectors, the same whatever group a merge is in, so the merged file, the P1
|
||||
// cross-check and any re-merge get one holohedry. On an R lattice in its hexagonal cell that is
|
||||
// 6/mmm rather than -3m, which groups each present reflection with exactly its -3m mates, the
|
||||
// extra ones being centring-absent. Empty when the cell does not carry the merging group itself
|
||||
// (a space group forced on a metric that does not have it): there is no holohedry to key on.
|
||||
std::vector<gemmi::Op> LatticeHolohedry(const UnitCell &cell, const gemmi::SpaceGroup &space_group) {
|
||||
std::vector<gemmi::Op> rots;
|
||||
for (const auto &op : gemmi::find_lattice_symmetry(static_cast<gemmi::UnitCell>(cell), 'P', 3.0).sym_ops)
|
||||
rots.push_back(gemmi::Op{op.rot, {0, 0, 0}, 'x'});
|
||||
for (const auto &op : space_group.operations().sym_ops)
|
||||
if (std::none_of(rots.begin(), rots.end(), [&](const gemmi::Op &r) { return r.rot == op.rot; }))
|
||||
return {};
|
||||
return rots;
|
||||
}
|
||||
|
||||
uint64_t PackIndex(const gemmi::Op::Miller &h) {
|
||||
constexpr int64_t bias = 1 << 20;
|
||||
return (static_cast<uint64_t>(h[0] + bias) << 42) | (static_cast<uint64_t>(h[1] + bias) << 21)
|
||||
| static_cast<uint64_t>(h[2] + bias);
|
||||
}
|
||||
}
|
||||
|
||||
void AssignRfreeFlags(std::vector<MergedReflection> &merged, const gemmi::SpaceGroup &space_group,
|
||||
double rfree_fraction, int min_free_reflections) {
|
||||
double rfree_fraction, int min_free_reflections, const std::optional<UnitCell> &cell) {
|
||||
for (auto &r : merged)
|
||||
r.rfree_flag = false;
|
||||
if (rfree_fraction <= 0.0 || merged.empty())
|
||||
@@ -36,10 +63,34 @@ void AssignRfreeFlags(std::vector<MergedReflection> &merged, const gemmi::SpaceG
|
||||
// dataset of one crystal form gets the SAME free set, which is what a multi-dataset campaign
|
||||
// (ensemble refinement, PanDDA) needs. A uniform hash draws ~rfree_fraction of the distinct
|
||||
// reflections free; a stratified per-shell draw would be tied to the dataset and break that.
|
||||
//
|
||||
// The key is the orbit of the reflection under the LATTICE HOLOHEDRY, not under the merging group.
|
||||
// A twin law is a lattice symmetry that the crystal lacks, so keying on the merging group puts a
|
||||
// free reflection's twin mate in the working set almost every time (measured: 97-98% of the free
|
||||
// reflections that have a mate), and a twin-refined R-free then reads the working set through
|
||||
// I_calc. Keyed on the holohedry, twin mates share a flag, and the free set no longer depends on
|
||||
// the space group a file is merged in - the merged MTZ, the P1 cross-check and a re-merge in any
|
||||
// subgroup carry one free set (where the small-data floor below lifts the fraction differently in
|
||||
// two of them, the smaller set is contained in the larger: each orbit has one draw). As
|
||||
// phenix.refine does by default (use_lattice_symmetry).
|
||||
const HKLKeyGenerator laue_key(/*merge_friedel=*/true, space_group);
|
||||
const std::vector<gemmi::Op> holohedry = cell ? LatticeHolohedry(*cell, space_group) : std::vector<gemmi::Op>{};
|
||||
auto key_of = [&](const MergedReflection &r) -> uint64_t {
|
||||
if (holohedry.empty())
|
||||
return laue_key(r).pack();
|
||||
// The largest index of the orbit, Friedel mates included, names it.
|
||||
gemmi::Op::Miller best{{r.h, r.k, r.l}};
|
||||
for (const auto &op : holohedry) {
|
||||
const gemmi::Op::Miller h = op.apply_to_hkl({{r.h, r.k, r.l}});
|
||||
best = std::max({best, h, gemmi::Op::Miller{{-h[0], -h[1], -h[2]}}});
|
||||
}
|
||||
return PackIndex(best);
|
||||
};
|
||||
|
||||
// Count the distinct test-eligible reflections (distinct Laue-ASU keys; mates collapse to one) so
|
||||
// the fraction can be floored to a usable test-set size on small data.
|
||||
// Count the distinct test-eligible reflections (distinct Laue-ASU keys of the merging group; mates
|
||||
// collapse to one) so the fraction can be floored to a usable test-set size on small data. Counted
|
||||
// in the merging group, not in holohedral orbits: R-free is a sum over the reflections of this file,
|
||||
// and counting orbits would lift the fraction on every dataset below its holohedry.
|
||||
std::unordered_set<uint64_t> distinct;
|
||||
distinct.reserve(merged.size());
|
||||
for (const auto &r : merged)
|
||||
@@ -54,7 +105,7 @@ void AssignRfreeFlags(std::vector<MergedReflection> &merged, const gemmi::SpaceG
|
||||
const double eff_fraction = std::max(rfree_fraction, floor_fraction);
|
||||
|
||||
for (auto &r : merged)
|
||||
r.rfree_flag = UniformFromKey(laue_key(r).pack()) < eff_fraction;
|
||||
r.rfree_flag = UniformFromKey(key_of(r)) < eff_fraction;
|
||||
}
|
||||
|
||||
size_t ApplyReferenceFreeFlags(std::vector<MergedReflection> &merged, const gemmi::SpaceGroup &space_group,
|
||||
|
||||
@@ -5,15 +5,21 @@
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "../../common/Reflection.h"
|
||||
#include "../../common/UnitCell.h"
|
||||
#include "gemmi/symmetry.hpp"
|
||||
|
||||
// Assign R-free (test-set) flags to merged reflections. The flag is a pure function of the
|
||||
// Friedel-merged (Laue) ASU key, which gives three properties:
|
||||
// Assign R-free (test-set) flags to merged reflections. With the cell given, the flag is a pure
|
||||
// function of the reflection's orbit under the LATTICE HOLOHEDRY (the metric symmetry of the cell,
|
||||
// which contains the merging group); without one, or when the cell does not carry the merging group,
|
||||
// of its Friedel-merged (Laue) ASU key in space_group. That gives four properties:
|
||||
// - all symmetry- and Friedel-equivalent reflections share one flag, so a Bijvoet pair I(+)/I(-)
|
||||
// is never split across the work and free sets;
|
||||
// - with the cell, twin-law mates share one too, and every file of one crystal - whatever space
|
||||
// group it is merged in - carries the same free set;
|
||||
// - the assignment is reproducible run-to-run and independent of merge/call order;
|
||||
// - it depends only on the reflection index, not on the dataset's resolution range or contents, so
|
||||
// every dataset of one crystal form gets the SAME free set (needed for multi-dataset campaigns:
|
||||
@@ -28,7 +34,8 @@
|
||||
// matters more than cross-dataset identity (and a shared reference/ApplyReferenceFreeFlags is the way
|
||||
// to keep exact identity there). Pass 0 to disable the floor.
|
||||
void AssignRfreeFlags(std::vector<MergedReflection> &merged, const gemmi::SpaceGroup &space_group,
|
||||
double rfree_fraction, int min_free_reflections = 500);
|
||||
double rfree_fraction, int min_free_reflections = 500,
|
||||
const std::optional<UnitCell> &cell = std::nullopt);
|
||||
|
||||
// Override the R-free flags on `merged` from a reference set (e.g. a reference MTZ's FreeR_flag
|
||||
// column), so an entire campaign shares one test set. A merged reflection whose Friedel-merged
|
||||
|
||||
@@ -343,7 +343,6 @@ void RotationScaleMerge::Ingest() {
|
||||
// pure waste. Only the survivors are built, from the source reflections, once the runs are known.
|
||||
constexpr int INT_LO = std::numeric_limits<int>::min();
|
||||
constexpr int INT_HI = std::numeric_limits<int>::max();
|
||||
std::vector<SortKey> unsorted(total);
|
||||
std::vector<int> h_lo(n_frames, INT_HI), h_hi(n_frames, INT_LO);
|
||||
ParallelFor(n_frames, nthreads, [&](int o) {
|
||||
if (reference_cell) {
|
||||
@@ -353,13 +352,10 @@ void RotationScaleMerge::Ingest() {
|
||||
double bkg_sum = 0.0;
|
||||
int bkg_n = 0;
|
||||
int lmin = INT_HI, lmax = INT_LO;
|
||||
int32_t at = src_start[o];
|
||||
for (const auto &r : partials_out[o].reflections) {
|
||||
if (std::isfinite(r.bkg) && r.bkg > 0.0f) { bkg_sum += r.bkg; ++bkg_n; }
|
||||
unsorted[at] = SortKey{r.h, r.k, r.l, r.image_number, at, r.d};
|
||||
lmin = std::min(lmin, r.h);
|
||||
lmax = std::max(lmax, r.h);
|
||||
++at;
|
||||
}
|
||||
if (bkg_n > 0) mean_bkg[o] = bkg_sum / bkg_n;
|
||||
h_lo[o] = lmin; h_hi[o] = lmax;
|
||||
@@ -395,30 +391,30 @@ void RotationScaleMerge::Ingest() {
|
||||
if (a.image_number != b.image_number) return a.image_number < b.image_number;
|
||||
return a.idx < b.idx;
|
||||
};
|
||||
// Two key arrays for a moment: the sweep above wrote them in index order, and where a key belongs
|
||||
// in the bucketed order is not known until the whole histogram is. Both together are less than half
|
||||
// of what the observation array they replace used to occupy on its own.
|
||||
// One key array, scattered into its h bucket straight from the source reflections. The buckets are
|
||||
// counted per chunk of FRAMES first; a frame's reflections are consecutive in the key numbering, so
|
||||
// the chunks are consecutive index ranges and the scatter lands each bucket in index order - the
|
||||
// same array an index-order copy scattered afterwards gave, without that copy (24 bytes a partial,
|
||||
// gigabytes on a long axis, at the top of the pass's memory).
|
||||
std::vector<SortKey> keys(total);
|
||||
{
|
||||
// Bucket by h first, then sort each bucket. h is the comparator's leading key, so the sorted
|
||||
// array is exactly the buckets laid end to end - and because the order is total (see above)
|
||||
// the permutation is the one a single std::sort produces, whichever algorithm gets there.
|
||||
// That is what makes this checkable: the output is byte-identical, not merely equivalent.
|
||||
const int n = static_cast<int>(total);
|
||||
const int nt = static_cast<int>(std::clamp<size_t>(nthreads, 1, std::max(1, n)));
|
||||
const int chunk = (n + nt - 1) / nt;
|
||||
const int nt = static_cast<int>(std::clamp<size_t>(nthreads, 1, std::max(1, n_frames)));
|
||||
const int chunk = (n_frames + nt - 1) / nt; // the split ParallelChunks makes below
|
||||
const int H = h_max - h_min + 1;
|
||||
|
||||
std::vector<std::vector<int32_t>> hist(nt, std::vector<int32_t>(H, 0));
|
||||
ParallelChunks(nt, nthreads, [&](int tlo, int thi) {
|
||||
for (int t = tlo; t < thi; ++t) {
|
||||
const int lo = t * chunk, hi = std::min(n, lo + chunk);
|
||||
for (int i = lo; i < hi; ++i) hist[t][unsorted[i].h - h_min]++;
|
||||
}
|
||||
ParallelChunks(n_frames, nthreads, [&](int lo, int hi) {
|
||||
std::vector<int32_t> &h_count = hist[lo / chunk];
|
||||
for (int o = lo; o < hi; ++o)
|
||||
for (const auto &r : partials_out[o].reflections) h_count[r.h - h_min]++;
|
||||
});
|
||||
|
||||
// Where each chunk writes each bucket: buckets in order, and within a bucket the chunks in
|
||||
// index order, so the scatter below is stable and lands in original order inside a bucket.
|
||||
// frame order, so the scatter below is stable and lands in original order inside a bucket.
|
||||
std::vector<int32_t> bstart(H + 1, 0);
|
||||
int32_t acc = 0;
|
||||
for (int b = 0; b < H; ++b) {
|
||||
@@ -431,15 +427,16 @@ void RotationScaleMerge::Ingest() {
|
||||
}
|
||||
bstart[H] = acc;
|
||||
|
||||
ParallelChunks(nt, nthreads, [&](int tlo, int thi) {
|
||||
for (int t = tlo; t < thi; ++t) {
|
||||
std::vector<int32_t> fill = hist[t];
|
||||
const int lo = t * chunk, hi = std::min(n, lo + chunk);
|
||||
for (int i = lo; i < hi; ++i) keys[fill[unsorted[i].h - h_min]++] = unsorted[i];
|
||||
ParallelChunks(n_frames, nthreads, [&](int lo, int hi) {
|
||||
std::vector<int32_t> fill = hist[lo / chunk];
|
||||
for (int o = lo; o < hi; ++o) {
|
||||
int32_t at = src_start[o];
|
||||
for (const auto &r : partials_out[o].reflections) {
|
||||
keys[fill[r.h - h_min]++] = SortKey{r.h, r.k, r.l, r.image_number, at, r.d};
|
||||
++at;
|
||||
}
|
||||
}
|
||||
});
|
||||
unsorted.clear();
|
||||
unsorted.shrink_to_fit();
|
||||
|
||||
// Largest bucket first: the tail of this is one bucket, so it should be the big one.
|
||||
std::vector<int> order(H);
|
||||
@@ -687,6 +684,13 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
|
||||
rawrun_start.swap(ns); rawrun_count.swap(nc);
|
||||
rawrun_h.swap(nh); rawrun_k.swap(nk); rawrun_l.swap(nl); rawrun_d.swap(nd);
|
||||
std::vector<SortKey>().swap(keys); // consumed (see the header)
|
||||
// Everything above that only served the remap - the old runs now in ns..nd, the per-run maps and
|
||||
// new_idx - is handed back before the observation array below is built, not at the end of the
|
||||
// function: on a long axis it is a gigabyte beside the largest array of the ingest.
|
||||
for (auto *v : {&ns, &nc, &nh, &nk, &nl, &dst_run, &dst_start, &new_idx})
|
||||
std::vector<int32_t>().swap(*v);
|
||||
std::vector<float>().swap(nd);
|
||||
std::vector<uint8_t>().swap(keep_run);
|
||||
|
||||
// The observations themselves, one frame per thread. finite_ok is the per-obs AcceptReflection
|
||||
// finiteness (immutable) - it lets ComputeAsuGroups stamp the ASU-group id per obs from a flat
|
||||
@@ -708,7 +712,9 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
|
||||
if (!keep[src]) { ++src; continue; }
|
||||
++src;
|
||||
ObsIngest &obs = partials_ingest[at];
|
||||
obs.h = r.h; obs.k = r.k; obs.l = r.l;
|
||||
obs.h = static_cast<int16_t>(r.h);
|
||||
obs.k = static_cast<int16_t>(r.k);
|
||||
obs.l = static_cast<int16_t>(r.l);
|
||||
obs.frame = o;
|
||||
obs.image_number = r.image_number;
|
||||
obs.d = r.d;
|
||||
@@ -730,7 +736,10 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
|
||||
|
||||
gpu_->SetPartialsLayout(n_keep, n_frames, frame_start.data(), frame_count.data());
|
||||
using F = RotationScaleMergeGPU::ObsField;
|
||||
constexpr int STAGE_SLICE = 8 << 20;
|
||||
// Two million observations a slice: the thirteen staging arrays are then under a hundred
|
||||
// megabytes together, where eight million put four hundred on top of the ingest's peak, and a
|
||||
// slice still spans tens of frames for the threads to share.
|
||||
constexpr int STAGE_SLICE = 2 << 20;
|
||||
constexpr int NFLOAT = 11;
|
||||
int max_frame_obs = 0;
|
||||
for (int o = 0; o < n_frames; ++o) max_frame_obs = std::max(max_frame_obs, static_cast<int>(frame_count[o]));
|
||||
@@ -3548,6 +3557,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
const std::vector<Sample> *chi2_pool = nullptr;
|
||||
double chi2_a = 0.0, chi2_b2 = 0.0;
|
||||
bool error_model_b_unmeasured = false; // b had no leverage; ISa is not reported
|
||||
bool error_model_b_resolved = true; // b^2 stood clear of its own standard error
|
||||
std::vector<Sample> &samples = em_samples;
|
||||
samples.clear();
|
||||
constexpr int n_bins = 16;
|
||||
@@ -3659,10 +3669,29 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
chi2_pool = ∈ chi2_a = error_model_a; chi2_b2 = 0.0;
|
||||
}
|
||||
} else if (det > 1e-10 * Ass * AII) {
|
||||
error_model_a = std::clamp((Bs * AII - BI * AsI) / det, 0.25, 100.0);
|
||||
const double b2 = std::max((Ass * BI - AsI * Bs) / det, 0.0);
|
||||
const double a_fit = (Bs * AII - BI * AsI) / det;
|
||||
const double b2_fit = (Ass * BI - AsI * Bs) / det;
|
||||
error_model_a = std::clamp(a_fit, 0.25, 100.0);
|
||||
const double b2 = std::max(b2_fit, 0.0);
|
||||
error_model_b = std::sqrt(b2);
|
||||
error_model_b_unmeasured = false;
|
||||
// Leverage is not enough: b^2 can have it and still come out at zero within its own error -
|
||||
// the strong bins scatter no more than counting says, or there are too few samples for a
|
||||
// bin median to mean anything. 1/b then reads the noise in b as an I/sigma, any value from
|
||||
// ~30 to infinity between merges of the same data. The standard error of b^2 is taken from
|
||||
// the bins' own scatter about the fitted line; the fit and the sigmas are left as they are,
|
||||
// and so is the fitted ISa every decision reads - this only decides what is printed.
|
||||
// Measured: z = 13.6 and 22 on two crystals whose b is plainly visible, 0.1-0.5 on a
|
||||
// 3000-sample low-multiplicity set and -1.2 to -6 on one whose strongest bins scatter
|
||||
// LESS than counting says (both printed ISa 96-130).
|
||||
double chi = 0.0;
|
||||
for (int bin = 0; bin < n_bins; ++bin) {
|
||||
const double d2w = std::max(bd2[bin], dev2_floor);
|
||||
const double r = bd2[bin] - a_fit * bs2[bin] - b2_fit * bI2[bin];
|
||||
chi += r * r / (d2w * d2w);
|
||||
}
|
||||
const double b2_se = std::sqrt(chi / (n_bins - 2) * Ass / det);
|
||||
error_model_b_resolved = b2_fit >= 2.0 * b2_se;
|
||||
error_model_active = true;
|
||||
chi2_pool = ∈ chi2_a = error_model_a; chi2_b2 = b2;
|
||||
}
|
||||
@@ -3749,26 +3778,81 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
// threshold is not the problem - 12 sigma rejects 1.35% of observations and gives the
|
||||
// same answer as 6 sigma - the rule is, and it only has a defined median from three.
|
||||
// Groups below the cut keep a NaN median and the test below skips them.
|
||||
std::vector<int32_t> start(n_groups + 1, 0);
|
||||
for (int g = 0; g < n_groups; ++g)
|
||||
start[g + 1] = start[g] + (cnt[g] >= 3 ? cnt[g] : 0);
|
||||
std::vector<float> iv(start[n_groups]);
|
||||
//
|
||||
// With the Friedel mates merged separately (-A) a hand below three borrows its MATE's
|
||||
// observations: I(+) and I(-) differ by the anomalous signal, a few percent of I and far
|
||||
// inside a six-sigma test, so together they are a median the hand alone does not have. Without
|
||||
// it a P1 anomalous merge sits at about two observations a hand and nothing is tested: one
|
||||
// observation a hundred times its partner's intensity (300 sigma apart) then survived into
|
||||
// the merge and took a whole CC1/2 bin, and the cut with it, from 0.80 to 0.18. A hand with
|
||||
// three of its own keeps its own median, as before. Median set g is group g's own; median
|
||||
// set n_groups + p is Friedel pair p's, filled only where a hand of p needs it.
|
||||
std::vector<int32_t> pair_of_group;
|
||||
std::vector<uint8_t> pair_needed;
|
||||
if (!merge_friedel) {
|
||||
const HKLKeyGenerator anom_keygen(/*merge_friedel=*/false, x.GetSpaceGroupOrP1());
|
||||
std::unordered_map<uint64_t, int32_t> pair_id;
|
||||
pair_id.reserve(static_cast<size_t>(n_groups) + 1);
|
||||
pair_of_group.resize(n_groups);
|
||||
for (int g = 0; g < n_groups; ++g) {
|
||||
const HKLKey gk = anom_keygen(group_h[g], group_k[g], group_l[g]);
|
||||
const uint64_t key = HKLKey{gk.h, gk.k, gk.l, true}.pack();
|
||||
pair_of_group[g] = pair_id.emplace(key, static_cast<int32_t>(pair_id.size())).first->second;
|
||||
}
|
||||
std::vector<int32_t> pair_cnt(pair_id.size(), 0);
|
||||
for (int g = 0; g < n_groups; ++g) pair_cnt[pair_of_group[g]] += cnt[g];
|
||||
pair_needed.assign(pair_id.size(), 0);
|
||||
for (int g = 0; g < n_groups; ++g)
|
||||
if (cnt[g] > 0 && cnt[g] < 3 && pair_cnt[pair_of_group[g]] >= 3)
|
||||
pair_needed[pair_of_group[g]] = 1;
|
||||
}
|
||||
const int n_sets = n_groups + static_cast<int>(pair_needed.size());
|
||||
std::vector<int32_t> set_cnt(n_sets, 0);
|
||||
for (int g = 0; g < n_groups; ++g) {
|
||||
if (cnt[g] >= 3) set_cnt[g] = cnt[g];
|
||||
if (!pair_needed.empty() && pair_needed[pair_of_group[g]])
|
||||
set_cnt[n_groups + pair_of_group[g]] += cnt[g];
|
||||
}
|
||||
std::vector<int32_t> start(n_sets + 1, 0);
|
||||
for (int t = 0; t < n_sets; ++t)
|
||||
start[t + 1] = start[t] + set_cnt[t];
|
||||
// Inverse-variance WEIGHTED median: a frame the crystal barely diffracted on is scaled up by
|
||||
// 1/G together with its sigma, so a plain median lets two such observations outvote one
|
||||
// well-measured one - and the test below then rejects the well-measured one against its
|
||||
// own small sigma.
|
||||
std::vector<std::pair<float, float>> iv(start[n_sets]); // (I*corr, 1/(sigma*corr)^2)
|
||||
{
|
||||
std::vector<int32_t> fill(start.begin(), start.end() - 1);
|
||||
for (int i = 0; i < n_full; ++i) {
|
||||
const int g = mf.group[i];
|
||||
if (g >= 0 && cnt[g] >= 3) iv[fill[g]++] = mf.I[i] * mf.corr[i];
|
||||
if (g < 0) continue;
|
||||
const float sc = mf.sigma[i] * mf.corr[i];
|
||||
const std::pair<float, float> v{mf.I[i] * mf.corr[i], sc > 0.0f ? 1.0f / (sc * sc) : 0.0f};
|
||||
if (cnt[g] >= 3) iv[fill[g]++] = v;
|
||||
if (!pair_needed.empty() && pair_needed[pair_of_group[g]])
|
||||
iv[fill[n_groups + pair_of_group[g]]++] = v;
|
||||
}
|
||||
}
|
||||
ParallelChunks(n_groups, ThreadsForWork(iv.size(), nthreads), [&](int glo, int ghi) {
|
||||
for (int g = glo; g < ghi; ++g) {
|
||||
const int lo = start[g], hi = start[g + 1];
|
||||
std::vector<float> set_median(n_sets, NAN);
|
||||
ParallelChunks(n_sets, ThreadsForWork(iv.size(), nthreads), [&](int tlo, int thi) {
|
||||
for (int t = tlo; t < thi; ++t) {
|
||||
const int lo = start[t], hi = start[t + 1];
|
||||
if (lo == hi) continue;
|
||||
const auto mid = iv.begin() + lo + (hi - lo) / 2;
|
||||
std::nth_element(iv.begin() + lo, mid, iv.begin() + hi);
|
||||
reject_median[g] = *mid;
|
||||
std::sort(iv.begin() + lo, iv.begin() + hi);
|
||||
double total = 0.0;
|
||||
for (int j = lo; j < hi; ++j) total += iv[j].second;
|
||||
double run = 0.0;
|
||||
int j = lo;
|
||||
for (; j < hi - 1; ++j) {
|
||||
run += iv[j].second;
|
||||
if (run >= 0.5 * total) break;
|
||||
}
|
||||
set_median[t] = iv[j].first;
|
||||
}
|
||||
});
|
||||
for (int g = 0; g < n_groups; ++g)
|
||||
reject_median[g] = cnt[g] >= 3 ? set_median[g]
|
||||
: !pair_needed.empty() ? set_median[n_groups + pair_of_group[g]] : NAN;
|
||||
}
|
||||
fit_error_model(samples);
|
||||
}
|
||||
@@ -3805,6 +3889,22 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
// above, on all threads, so a second serial walk over the fat records only rebuilds it.
|
||||
AssignHalvesByRank(obs_key, mf.group, n_groups, merge_half, nthreads);
|
||||
|
||||
// ---- CC1/2 weight (MergedReflection::cc_weight): the precision each half-set WOULD have had with
|
||||
// every observation at the run's typical frame scale, over the precision it has. An observation
|
||||
// from a frame the crystal barely diffracted on was scaled up by G_ref/G, so it carries
|
||||
// (G_ref/G)^2 more counting variance than the same observation at the typical scale; that
|
||||
// factor, and nothing intensity-dependent, is what the pair is discounted by. G_ref =
|
||||
// sum G^3 / sum G^2 over the usable observations: the precision-weighted typical scale, which
|
||||
// the dead frames cannot drag down however many of them there are. A frame brighter than
|
||||
// typical earns nothing (the factor is floored at 1), so on a sweep without a weak stretch
|
||||
// every weight is 1 and the CC1/2 is the plain Pearson it was. G is the frame's scale taken
|
||||
// before the correction surfaces (frame_scale_fulls), which are per-observation and
|
||||
// resolution-dependent and are not a property of the frame. ----
|
||||
std::vector<int64_t> n_obs_frame(frame_scale_fulls.size(), 0);
|
||||
for (int i = 0; i < n_full; ++i)
|
||||
if (mf.group[i] >= 0) ++n_obs_frame[fulls[i].frame];
|
||||
const std::vector<double> frame_cc_factor = CCHalfFrameFactors(frame_scale_fulls, n_obs_frame);
|
||||
|
||||
// ---- Merge: per-group inverse-variance sums with corrected sigma + deterministic half sets, then
|
||||
// export. In a lambda because the automatic resolution cutoff below re-runs it once the error
|
||||
// model has been refitted on the reflections that survive the cut. ----
|
||||
@@ -3838,8 +3938,10 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
ms.Resize(n_groups);
|
||||
gpu_->MergeAccum(error_model_a, error_model_b, error_model_active,
|
||||
reject_outliers, reject_nsigma, reject_median.data(), merge_half.data(),
|
||||
frame_cc_factor.data(),
|
||||
ms.swI.data(), ms.sw.data(), ms.swIh0.data(), ms.swIh1.data(),
|
||||
ms.swh0.data(), ms.swh1.data(), ms.nh0.data(), ms.nh1.data(),
|
||||
ms.swh0.data(), ms.swh1.data(), ms.swh_typ0.data(), ms.swh_typ1.data(),
|
||||
ms.nh0.data(), ms.nh1.data(),
|
||||
ms.d.data(), ms.rej.data(), ms.on_ice.data(), rejected_obs.data());
|
||||
// Per-group and independent, so the unpack splits; the rejected count is an integer sum and
|
||||
// does not care in which order the chunks add theirs in.
|
||||
@@ -3850,6 +3952,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
Accum &a = acc[g];
|
||||
a.swI = ms.swI[g]; a.sw = ms.sw[g]; a.swIh[0] = ms.swIh0[g]; a.swIh[1] = ms.swIh1[g];
|
||||
a.swh[0] = ms.swh0[g]; a.swh[1] = ms.swh1[g];
|
||||
a.swh_typ[0] = ms.swh_typ0[g]; a.swh_typ[1] = ms.swh_typ1[g];
|
||||
a.nh[0] = static_cast<size_t>(ms.nh0[g]); a.nh[1] = static_cast<size_t>(ms.nh1[g]);
|
||||
a.d = static_cast<float>(ms.d[g]);
|
||||
a.on_ice = ms.on_ice[g] != 0;
|
||||
@@ -3879,6 +3982,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
const int half = merge_half[&o - fulls.data()];
|
||||
a.swI += wI; a.sw += w;
|
||||
a.swIh[half] += wI; a.swh[half] += w; a.nh[half]++;
|
||||
a.swh_typ[half] += w * frame_cc_factor[o.frame];
|
||||
// Any contributing observation on an ice ring marks the merged reflection (as
|
||||
// MergeOnTheFly does on the stills path): the intensity is still merged and written,
|
||||
// the flag only lets a consumer that must not read ring signal as crystal signal -
|
||||
@@ -3908,6 +4012,8 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
mr.I_half[i] = static_cast<float>(a.swIh[i] / a.swh[i]);
|
||||
mr.sigma_half[i] = static_cast<float>(1.0 / std::sqrt(a.swh[i]));
|
||||
}
|
||||
mr.cc_weight = static_cast<float>((1.0 / a.swh_typ[0] + 1.0 / a.swh_typ[1])
|
||||
/ (1.0 / a.swh[0] + 1.0 / a.swh[1]));
|
||||
}
|
||||
if (!std::isfinite(a.d) || a.d <= 0.0f) continue;
|
||||
merged_I[g] = mr.I;
|
||||
@@ -4033,6 +4139,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
{
|
||||
const auto em = ToXdsErrorModel(error_model_a, error_model_b);
|
||||
result.isa = em.isa;
|
||||
result.isa_resolved = error_model_b_resolved;
|
||||
result.error_model_a = em.a;
|
||||
result.error_model_b = em.b;
|
||||
}
|
||||
@@ -4063,6 +4170,10 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
"not reach far enough for a systematic error to be seen, not that there is "
|
||||
"none; a resolution range matched to the signal would measure it",
|
||||
em.a, error_model_chi2);
|
||||
else if (!error_model_b_resolved)
|
||||
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} - b is not resolved from zero "
|
||||
"(under two standard errors), so ISa is undetermined (1/b = {:.1f})",
|
||||
em.a, em.b, em.isa);
|
||||
else if (!full_stats)
|
||||
// Neither the asymptote nor the chi2 median was measured on this merge, so neither is
|
||||
// reported for it.
|
||||
@@ -4077,7 +4188,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
// is not written needs neither. French-Wilson is deferred until after the anomalous accumulator
|
||||
// below has attached I(+)/I(-), so the two hands get their amplitudes in one pass.
|
||||
if (full_stats)
|
||||
AssignRfreeFlags(result.merged, x.GetSpaceGroupOrP1(), rfree_fraction);
|
||||
AssignRfreeFlags(result.merged, x.GetSpaceGroupOrP1(), rfree_fraction, 500, reference_cell);
|
||||
|
||||
if (reject_count > 0)
|
||||
logger.Info("Merge outlier rejection: dropped {} observations", reject_count);
|
||||
@@ -4128,8 +4239,8 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
s.unique++;
|
||||
s.sum_i_over_sigma += m.I / m.sigma; s.n_i_over_sigma++;
|
||||
if (std::isfinite(m.I_half[0]) && std::isfinite(m.I_half[1])) {
|
||||
s.cc_half.Add(m.I_half[0], m.I_half[1]);
|
||||
cc_half_overall.Add(m.I_half[0], m.I_half[1]);
|
||||
s.cc_half.Add(m.I_half[0], m.I_half[1], m.cc_weight);
|
||||
cc_half_overall.Add(m.I_half[0], m.I_half[1], m.cc_weight);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4778,8 +4889,34 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st
|
||||
}
|
||||
logger.Info("Scaled fulls (XDS order, Unity model)");
|
||||
}
|
||||
// The total scale each frame's fulls were amplified by - partial scale, incident flux and the fulls'
|
||||
// own G (corr = 1/G here) - taken BEFORE the correction surfaces fold per-observation terms into
|
||||
// corr, and before a collapsed frame's fulls are zeroed (a zeroed corr says nothing about the
|
||||
// frame's scale). Read by MergeAndStats for the CC1/2 weight.
|
||||
frame_scale_fulls.assign(n_frames, NAN);
|
||||
for (const auto &o : fulls)
|
||||
if (std::isfinite(o.corr) && o.corr > 0.0f && !std::isfinite(frame_scale_fulls[o.frame])) {
|
||||
const double gp = g_partial[o.frame] * frame_flux[o.frame];
|
||||
frame_scale_fulls[o.frame] = (std::isfinite(gp) && gp > 0.0 ? gp : 1.0) / o.corr;
|
||||
}
|
||||
const bool rejected_full_scales =
|
||||
scale_fulls && DropCollapsedFullScales(combined_on_gpu && scaled_fulls_on_gpu);
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
// The host fulls' corr is the truth from here on; a merge on the resident (GPU) fulls reads the
|
||||
// device copy, so every change to it is pushed back before the next device merge.
|
||||
auto upload_fulls_corr = [&]() {
|
||||
std::vector<float> fcorr(fulls.size());
|
||||
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads),
|
||||
[&](int lo, int hi) {
|
||||
for (int i = lo; i < hi; ++i) fcorr[i] = fulls[i].corr;
|
||||
});
|
||||
gpu_->SetFullsCorr(fcorr.data());
|
||||
};
|
||||
// Right away, not with the corrections below: the merge just before the surfaces must see the
|
||||
// collapsed frames' fulls gone on the device exactly as the host merge does.
|
||||
if (rejected_full_scales && combined_on_gpu && scaled_fulls_on_gpu)
|
||||
upload_fulls_corr();
|
||||
#endif
|
||||
|
||||
// --- 4b. Optional correction surfaces (decay = resolution x time; absorption = goniometer-frame
|
||||
// diffracted-beam direction), each an alternating multiplicative fit of the fulls' corr against
|
||||
@@ -4845,16 +4982,10 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st
|
||||
CountSweepDisposition();
|
||||
}
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
// The corrections mutate the host fulls' corr; when the merge runs on the resident (GPU) fulls, push
|
||||
// the corrected corr back to the device so the merge reads it.
|
||||
if ((corrections || rejected_full_scales || rejected_frames) && combined_on_gpu && scaled_fulls_on_gpu) {
|
||||
std::vector<float> fcorr(fulls.size());
|
||||
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads),
|
||||
[&](int lo, int hi) {
|
||||
for (int i = lo; i < hi; ++i) fcorr[i] = fulls[i].corr;
|
||||
});
|
||||
gpu_->SetFullsCorr(fcorr.data());
|
||||
}
|
||||
// The corrections and the frame rejection mutate the host fulls' corr; push it to the device so the
|
||||
// merge reads it. (The collapsed-scale drop was pushed where it happened.)
|
||||
if ((corrections || rejected_frames) && combined_on_gpu && scaled_fulls_on_gpu)
|
||||
upload_fulls_corr();
|
||||
#endif
|
||||
|
||||
// --- 5. Error model + merge + statistics. ---
|
||||
|
||||
@@ -49,6 +49,10 @@ public:
|
||||
// only ever be the more optimistic of the two. Both 0 if the model stayed at identity.
|
||||
double isa = 0.0;
|
||||
double isa_asymptotic = 0.0;
|
||||
// b^2 stood at least two standard errors clear of zero. When it did not, `isa` is still the
|
||||
// fitted 1/b and every decision reads it as before; only what is PRINTED says "undetermined",
|
||||
// since 1/b of a b that is zero within its error is the noise in b, not an I/sigma.
|
||||
bool isa_resolved = true;
|
||||
double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2)
|
||||
double error_model_b = 0.0;
|
||||
// The overall CC1/2 as it stood BEFORE the correction surfaces were folded in. That is what the
|
||||
@@ -140,16 +144,20 @@ private:
|
||||
// observation dump): every immutable per-obs field is uploaded straight from the source
|
||||
// reflections inside BuildInRangeObservations, and the host keeps only what its own remaining
|
||||
// stages touch - the geometry smoothing (delta_phi, then the recomputed partiality), plus the
|
||||
// rocking-event walk's inputs. Forty bytes against the Obs's eighty, and on that path the fat
|
||||
// rocking-event walk's inputs. Thirty-two bytes against the Obs's eighty, and on that path the fat
|
||||
// record is never materialised at all: on a fine-sliced long axis it alone is many gigabytes,
|
||||
// held beside the source reflections it was copied from.
|
||||
struct ObsIngest {
|
||||
int32_t h, k, l, frame;
|
||||
float image_number, d, zeta, delta_phi, partiality;
|
||||
// Sixteen bits are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the
|
||||
// longest axis at atomic resolution - and on a long axis this array sets the ingest's memory
|
||||
// high-water mark, so each index costs two bytes here, not four.
|
||||
int16_t h, k, l;
|
||||
// The rocking-event usability test (RockingEventFrames), evaluated on the ingest-time
|
||||
// values - which is what that walk reads on the resident path, where the host corr is
|
||||
// never refreshed (see partials_released).
|
||||
// never refreshed (see partials_released). Beside the indices, where it fills their padding.
|
||||
uint8_t rock_ok;
|
||||
int32_t frame;
|
||||
float image_number, d, zeta, delta_phi, partiality;
|
||||
};
|
||||
|
||||
// The narrow per-observation record the ingest sort orders: the raw hkl the runs are cut on, the
|
||||
@@ -273,6 +281,7 @@ private:
|
||||
std::vector<Obs> fulls; // combined fulls (rebuilt each Run), sorted by frame
|
||||
std::vector<int32_t> fulls_frame_start, fulls_frame_count; // CSR ranges of `fulls` per frame
|
||||
std::vector<double> g_full; // per-frame scale on the fulls
|
||||
std::vector<double> frame_scale_fulls; // per-frame total scale of the fulls before the surfaces (cc_weight)
|
||||
|
||||
// The five fields the host-side walks over the fulls actually read, pulled out of the 80-byte record
|
||||
// once per merge (see MergeAndStats). `group` carries the usability decision: -1 means the full is
|
||||
@@ -306,15 +315,18 @@ private:
|
||||
// The merge accumulators (see MergeAndStats' run_merge): one entry per ASU group, plus the arrays
|
||||
// the device kernel fills that the host unpacks into them. Members for the same reason as
|
||||
// FullsStaging - a merge runs several times per run and this is a hundred megabytes between them.
|
||||
struct Accum { double swI = 0, sw = 0, swIh[2] = {0, 0}, swh[2] = {0, 0}; size_t nh[2] = {0, 0}; float d = NAN;
|
||||
bool on_ice = false; };
|
||||
// swh_typ: each half-set's precision had every observation been at the run's typical frame scale
|
||||
// (see MergeAndStats, cc_weight).
|
||||
struct Accum { double swI = 0, sw = 0, swIh[2] = {0, 0}, swh[2] = {0, 0}, swh_typ[2] = {0, 0};
|
||||
size_t nh[2] = {0, 0}; float d = NAN; bool on_ice = false; };
|
||||
std::vector<Accum> merge_acc;
|
||||
struct MergeAccumStaging {
|
||||
std::vector<double> swI, sw, swIh0, swIh1, swh0, swh1, d;
|
||||
std::vector<double> swI, sw, swIh0, swIh1, swh0, swh1, swh_typ0, swh_typ1, d;
|
||||
std::vector<int32_t> nh0, nh1, rej;
|
||||
std::vector<uint8_t> on_ice;
|
||||
void Resize(int n) {
|
||||
swI.resize(n); sw.resize(n); swIh0.resize(n); swIh1.resize(n); swh0.resize(n); swh1.resize(n);
|
||||
swh_typ0.resize(n); swh_typ1.resize(n);
|
||||
d.resize(n); nh0.resize(n); nh1.resize(n); rej.resize(n); on_ice.resize(n);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -464,13 +464,13 @@ namespace {
|
||||
const int32_t *group, *frame;
|
||||
const uint8_t *on_ice, *frame_cell_ok, *half;
|
||||
const int32_t *gperm, *gstart, *gcount;
|
||||
const double *em_mean, *merged_I;
|
||||
const double *em_mean, *merged_I, *cc_factor;
|
||||
// outputs
|
||||
double *sw, *swI, *em_mean_out;
|
||||
int32_t *cnt;
|
||||
double *s2, *I2, *dev2;
|
||||
uint8_t *valid;
|
||||
double *a_swI, *a_sw, *a_swIh0, *a_swIh1, *a_swh0, *a_swh1, *a_d;
|
||||
double *a_swI, *a_sw, *a_swIh0, *a_swIh1, *a_swh0, *a_swh1, *a_swht0, *a_swht1, *a_d;
|
||||
int32_t *a_nh0, *a_nh1, *a_rejected;
|
||||
uint8_t *a_on_ice;
|
||||
double *r_absdev, *r_sumI;
|
||||
@@ -544,7 +544,7 @@ namespace {
|
||||
__global__ void MergeAccumKernel(MergeParams p) {
|
||||
for (int g = blockIdx.x * blockDim.x + threadIdx.x; g < p.n_groups; g += gridDim.x * blockDim.x) {
|
||||
const int lo = p.gstart[g], hi = lo + p.gcount[g];
|
||||
double swI = 0, sw = 0, swIh0 = 0, swIh1 = 0, swh0 = 0, swh1 = 0, dd = NAN;
|
||||
double swI = 0, sw = 0, swIh0 = 0, swIh1 = 0, swh0 = 0, swh1 = 0, swht0 = 0, swht1 = 0, dd = NAN;
|
||||
int nh0 = 0, nh1 = 0, rejected = 0, onice = 0;
|
||||
const float rmed = p.reject_median[g];
|
||||
for (int q = lo; q < hi; ++q) {
|
||||
@@ -574,13 +574,16 @@ namespace {
|
||||
// a rank within the reflection cannot be read off a running count, and computing it
|
||||
// in one place is what makes this kernel and the host merge loop agree exactly.
|
||||
const int half = p.half[i];
|
||||
const double wt = w * p.cc_factor[p.frame[i]]; // see the host merge loop (cc_weight)
|
||||
swI += wI; sw += w;
|
||||
if (half) { swIh1 += wI; swh1 += w; ++nh1; } else { swIh0 += wI; swh0 += w; ++nh0; }
|
||||
if (half) { swIh1 += wI; swh1 += w; swht1 += wt; ++nh1; }
|
||||
else { swIh0 += wI; swh0 += w; swht0 += wt; ++nh0; }
|
||||
if (p.on_ice[i]) onice = 1; // see the host merge loop
|
||||
if (!isfinite(dd) && isfinite(p.d[i]) && p.d[i] > 0.0f) dd = p.d[i];
|
||||
}
|
||||
p.a_swI[g] = swI; p.a_sw[g] = sw; p.a_swIh0[g] = swIh0; p.a_swIh1[g] = swIh1;
|
||||
p.a_swh0[g] = swh0; p.a_swh1[g] = swh1; p.a_nh0[g] = nh0; p.a_nh1[g] = nh1; p.a_d[g] = dd;
|
||||
p.a_swh0[g] = swh0; p.a_swh1[g] = swh1; p.a_swht0[g] = swht0; p.a_swht1[g] = swht1;
|
||||
p.a_nh0[g] = nh0; p.a_nh1[g] = nh1; p.a_d[g] = dd;
|
||||
p.a_rejected[g] = rejected; p.a_on_ice[g] = uint8_t(onice);
|
||||
}
|
||||
}
|
||||
@@ -657,7 +660,8 @@ struct RotationScaleMergeGPU::Impl {
|
||||
CudaDevicePtr<uint8_t> m_valid;
|
||||
CudaDevicePtr<uint8_t> m_rejected; // per-full outlier-rejected flag (MergeAccum -> MergeRmeas)
|
||||
CudaDevicePtr<uint8_t> m_half; // per-full CC1/2 half-set, assigned on the host
|
||||
CudaDevicePtr<double> a_swI, a_sw, a_swIh0, a_swIh1, a_swh0, a_swh1, a_d; // merge accum per group
|
||||
CudaDevicePtr<double> a_swI, a_sw, a_swIh0, a_swIh1, a_swh0, a_swh1, a_swht0, a_swht1, a_d; // merge accum per group
|
||||
CudaDevicePtr<double> cc_factor; // per-frame CC1/2 weight factor (length n_frames)
|
||||
CudaDevicePtr<int32_t> a_nh0, a_nh1, a_rejected;
|
||||
CudaDevicePtr<uint8_t> a_on_ice; // per group: any contributing full on an ice ring
|
||||
CudaDevicePtr<float> reject_median;
|
||||
@@ -902,9 +906,10 @@ void RotationScaleMergeGPU::MergeEmSamples(bool for_search, double min_partialit
|
||||
|
||||
void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_b, bool error_model_active,
|
||||
bool reject_outliers, double reject_nsigma, const float *reject_median,
|
||||
const uint8_t *half,
|
||||
const uint8_t *half, const double *frame_cc_factor,
|
||||
double *swI, double *sw, double *swIh0, double *swIh1,
|
||||
double *swh0, double *swh1, int32_t *nh0, int32_t *nh1, double *d_out,
|
||||
double *swh0, double *swh1, double *swh_typ0, double *swh_typ1,
|
||||
int32_t *nh0, int32_t *nh1, double *d_out,
|
||||
int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs) {
|
||||
DeviceGuard guard(impl_->device, impl_->available);
|
||||
auto &d = *impl_;
|
||||
@@ -912,6 +917,7 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
|
||||
d.a_swI = CudaDevicePtr<double>(std::max(1, ng)); d.a_sw = CudaDevicePtr<double>(std::max(1, ng));
|
||||
d.a_swIh0 = CudaDevicePtr<double>(std::max(1, ng)); d.a_swIh1 = CudaDevicePtr<double>(std::max(1, ng));
|
||||
d.a_swh0 = CudaDevicePtr<double>(std::max(1, ng)); d.a_swh1 = CudaDevicePtr<double>(std::max(1, ng));
|
||||
d.a_swht0 = CudaDevicePtr<double>(std::max(1, ng)); d.a_swht1 = CudaDevicePtr<double>(std::max(1, ng));
|
||||
d.a_nh0 = CudaDevicePtr<int32_t>(std::max(1, ng)); d.a_nh1 = CudaDevicePtr<int32_t>(std::max(1, ng));
|
||||
d.a_d = CudaDevicePtr<double>(std::max(1, ng)); d.a_rejected = CudaDevicePtr<int32_t>(std::max(1, ng));
|
||||
d.a_on_ice = CudaDevicePtr<uint8_t>(std::max(1, ng));
|
||||
@@ -920,6 +926,7 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
|
||||
"zero m_rejected");
|
||||
Upload(d.reject_median, reject_median, ng);
|
||||
Upload(d.m_half, half, d.n_fulls);
|
||||
Upload(d.cc_factor, frame_cc_factor, d.n_frames);
|
||||
|
||||
MergeParams p{};
|
||||
p.n_groups = ng; p.min_partiality = d.merge_min_part;
|
||||
@@ -934,9 +941,10 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
|
||||
p.half = d.m_half.get();
|
||||
p.var_bkg = d.f_var_bkg.get(); p.var_per_I = d.f_var_per_I.get();
|
||||
p.gperm = d.f_gperm.get(); p.gstart = d.f_gstart.get(); p.gcount = d.f_gcount.get();
|
||||
p.em_mean = d.m_em_mean.get();
|
||||
p.em_mean = d.m_em_mean.get(); p.cc_factor = d.cc_factor.get();
|
||||
p.a_swI = d.a_swI.get(); p.a_sw = d.a_sw.get(); p.a_swIh0 = d.a_swIh0.get(); p.a_swIh1 = d.a_swIh1.get();
|
||||
p.a_swh0 = d.a_swh0.get(); p.a_swh1 = d.a_swh1.get(); p.a_nh0 = d.a_nh0.get(); p.a_nh1 = d.a_nh1.get();
|
||||
p.a_swh0 = d.a_swh0.get(); p.a_swh1 = d.a_swh1.get();
|
||||
p.a_swht0 = d.a_swht0.get(); p.a_swht1 = d.a_swht1.get(); p.a_nh0 = d.a_nh0.get(); p.a_nh1 = d.a_nh1.get();
|
||||
p.a_d = d.a_d.get(); p.a_rejected = d.a_rejected.get(); p.a_on_ice = d.a_on_ice.get();
|
||||
p.rejected_obs = d.m_rejected.get();
|
||||
|
||||
@@ -947,7 +955,8 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
|
||||
auto dl = [&](void *h, const CudaDevicePtr<double> &s) {
|
||||
CudaCheck(cudaMemcpy(h, s.get(), size_t(ng) * sizeof(double), cudaMemcpyDeviceToHost), "dl accum"); };
|
||||
dl(swI, d.a_swI); dl(sw, d.a_sw); dl(swIh0, d.a_swIh0); dl(swIh1, d.a_swIh1);
|
||||
dl(swh0, d.a_swh0); dl(swh1, d.a_swh1); dl(d_out, d.a_d);
|
||||
dl(swh0, d.a_swh0); dl(swh1, d.a_swh1); dl(swh_typ0, d.a_swht0); dl(swh_typ1, d.a_swht1);
|
||||
dl(d_out, d.a_d);
|
||||
CudaCheck(cudaMemcpy(nh0, d.a_nh0.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh0");
|
||||
CudaCheck(cudaMemcpy(nh1, d.a_nh1.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh1");
|
||||
CudaCheck(cudaMemcpy(rejected, d.a_rejected.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl rej");
|
||||
|
||||
@@ -93,13 +93,15 @@ public:
|
||||
// kernel and the host merge loop cannot disagree about it.
|
||||
// Outputs length n_groups; rejected[g] counts outliers dropped (reject_median uploaded, NAN
|
||||
// where none). rejected_obs is the per-full flag (length n_fulls): the host needs it for the reductions
|
||||
// it still does itself, above all the anomalous I(+)/I(-) split.
|
||||
// it still does itself, above all the anomalous I(+)/I(-) split. frame_cc_factor (length n_frames) is
|
||||
// each frame's (G_ref/G)^2 floored at 1; swh_typ0/1 are the half-set weights multiplied by it.
|
||||
// Requires MergeEmSamples first (em_mean resident).
|
||||
void MergeAccum(double error_model_a, double error_model_b, bool error_model_active,
|
||||
bool reject_outliers, double reject_nsigma, const float *reject_median,
|
||||
const uint8_t *half,
|
||||
const uint8_t *half, const double *frame_cc_factor,
|
||||
double *swI, double *sw, double *swIh0, double *swIh1,
|
||||
double *swh0, double *swh1, int32_t *nh0, int32_t *nh1, double *d_out,
|
||||
double *swh0, double *swh1, double *swh_typ0, double *swh_typ1,
|
||||
int32_t *nh0, int32_t *nh1, double *d_out,
|
||||
int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs);
|
||||
|
||||
// Per-group R_meas accumulators (sum|I_corr-merged_I|, sum_I, n, and the count this looser walk
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
|
||||
namespace {
|
||||
// The Patterson is measured in a band, not over the whole dataset: below ~20 A the bulk solvent
|
||||
// and the beam stop dominate, above ~5 A the map answers a question about atoms rather than about
|
||||
@@ -231,8 +233,12 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
|
||||
result.refusal = "could not allocate the Patterson grid";
|
||||
return result;
|
||||
}
|
||||
fftwf_plan plan = fftwf_plan_dft_c2r_3d(nx, ny, nz, reinterpret_cast<fftwf_complex *>(coeff), map,
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_plan plan;
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_dft_c2r_3d(nx, ny, nz, reinterpret_cast<fftwf_complex *>(coeff), map,
|
||||
FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
// The near-origin exclusion, as a set of grid points rather than a per-point distance test: only
|
||||
// the points inside a 15 A ball around each origin-equivalent point can be excluded, and there
|
||||
@@ -368,7 +374,10 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
|
||||
}
|
||||
}
|
||||
|
||||
fftwf_destroy_plan(plan);
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
}
|
||||
fftwf_free(coeff);
|
||||
fftwf_free(map);
|
||||
result.measurable = true;
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include <cstdint>
|
||||
#include <iomanip>
|
||||
#include <limits>
|
||||
#include <numbers>
|
||||
#include <sstream>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
@@ -29,6 +30,33 @@ namespace {
|
||||
// admit no such operation, so twinning is geometrically impossible and the intensity statistics
|
||||
// cannot be indicating it. Low-symmetry classes stay eligible because pseudo-merohedral twinning
|
||||
// through an accidental metric specialisation cannot be excluded from the symmetry alone.
|
||||
// <|L|> of a partial twin with fraction a: I = (1-a) X + a Y for independent Wilson X, Y, and the
|
||||
// expectation of |I1 - I2|/(I1 + I2) over two independent such intensities, integrated in closed
|
||||
// form (0.500 at a = 0, 0.440 at 0.1, 0.408 at 0.2, 0.375 at 0.5; checked by simulation, and
|
||||
// the relation ctruncate tabulates).
|
||||
double MeanAbsLForTwinFraction(double a) {
|
||||
const double p = 1.0 - a, q = a;
|
||||
const double A = 1.0 / q, B = 1.0 / p - 1.0 / q;
|
||||
auto F = [&](double t) { return 2.0 / (B * B) * std::log(A + B * t) + (2.0 * A / B + 1.0) / (B * (A + B * t)); };
|
||||
const double J = F(1.0) + F(0.0) - 2.0 * F(0.5);
|
||||
return ((p * p + q * q) / 2.0 - 2.0 * J) / ((p - q) * (p - q));
|
||||
}
|
||||
|
||||
// The inverse, by bisection: the relation is monotonic, and flat near a = 0.5, where <|L|> can no
|
||||
// longer tell 0.4 from 0.5.
|
||||
double TwinFractionFromMeanAbsL(double mean_abs_l) {
|
||||
double lo = 1e-4, hi = 0.4999;
|
||||
if (mean_abs_l >= MeanAbsLForTwinFraction(lo))
|
||||
return 0.0;
|
||||
if (mean_abs_l <= MeanAbsLForTwinFraction(hi))
|
||||
return 0.5;
|
||||
for (int i = 0; i < 50; ++i) {
|
||||
const double mid = 0.5 * (lo + hi);
|
||||
(MeanAbsLForTwinFraction(mid) > mean_abs_l ? lo : hi) = mid;
|
||||
}
|
||||
return 0.5 * (lo + hi);
|
||||
}
|
||||
|
||||
bool MerohedralTwinningPossible(const gemmi::SpaceGroup* sg) {
|
||||
if (!sg)
|
||||
return true; // P1 / unknown symmetry: cannot rule twinning out
|
||||
@@ -47,7 +75,7 @@ namespace {
|
||||
|
||||
TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merged,
|
||||
const gemmi::SpaceGroup* space_group, int resolution_shells,
|
||||
const std::array<double, 3>* tncs_vector) {
|
||||
const std::array<double, 3>* tncs_vector, char lattice_centring) {
|
||||
TwinningAnalysisResult result;
|
||||
if (merged.empty())
|
||||
return result;
|
||||
@@ -58,6 +86,108 @@ TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merg
|
||||
auto acentric = [&](const MergedReflection& r) {
|
||||
return !space_group || !gops.is_reflection_centric(gemmi::Op::Miller{{r.h, r.k, r.l}});
|
||||
};
|
||||
// A merge in P1 of a centred lattice still holds the reflections the centring extinguishes (an
|
||||
// R lattice indexed in its hexagonal cell keeps the two thirds with -h+k+l != 3n). They are
|
||||
// noise, not intensities, and an L-test partner step that crosses into them pairs a real
|
||||
// reflection with an absent one. So they are left out of both statistics.
|
||||
gemmi::GroupOps centring;
|
||||
centring.sym_ops = {gemmi::Op::identity()};
|
||||
centring.cen_ops = gemmi::centring_vectors(lattice_centring);
|
||||
auto used = [&](const MergedReflection& r) {
|
||||
return UsableIntensity(r) && acentric(r)
|
||||
&& !centring.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}});
|
||||
};
|
||||
|
||||
// --- Resolution shells: the normalising mean of both statistics ---
|
||||
// Binning by 1/d^2 removes the resolution fall-off, so the second moment is 2.0 (untwinned) or 1.5
|
||||
// (perfect twin) regardless of the overall B-factor, and an L-test pair compares two intensities
|
||||
// on the same scale. The moment divides by the *square* of the shell-mean intensity, so it is not
|
||||
// robust: on weak or mis-integrated data a shell mean can collapse to the noise floor and one
|
||||
// outlier reflection then dominates (I/mean)^2 (a single I=158 in a mean~1 shell contributed 78% of
|
||||
// a whole dataset's value). To keep these twinning indicators rather than data-quality artefacts -
|
||||
// as phenix.xtriage does - we skip noise-only shells (<I/sigma> below 1) and reject Wilson outliers
|
||||
// (E^2 above 8, ~exp(-8) upper tail) with one shell-mean re-iteration so the outlier does not
|
||||
// corrupt the normalising mean either.
|
||||
//
|
||||
// The shells are equal in width in 1/d^2. Equal-COUNT shells were measured as the alternative
|
||||
// (the first equal-width shell spans a wide range of d, and its fall-off inflates the moment):
|
||||
// they move the moment the wrong way on the same data, widening its gap to phenix.xtriage by
|
||||
// 0.01-0.04 rather than closing it, so the shells were kept as they are.
|
||||
constexpr double min_shell_isig = 1.0; // shells below this are noise, not signal
|
||||
constexpr double wilson_outlier_e2 = 8.0; // reject improbably strong reflections (P ~ e^-8)
|
||||
const int n_shells = std::max(1, resolution_shells);
|
||||
double min_s = std::numeric_limits<double>::infinity();
|
||||
double max_s = -std::numeric_limits<double>::infinity();
|
||||
for (const auto& r : merged) {
|
||||
if (!used(r))
|
||||
continue;
|
||||
const double s = 1.0 / (r.d * r.d);
|
||||
min_s = std::min(min_s, s);
|
||||
max_s = std::max(max_s, s);
|
||||
}
|
||||
if (!std::isfinite(min_s) || !(max_s > min_s))
|
||||
return result;
|
||||
auto shell_of = [&](double d) {
|
||||
const double t = (1.0 / (d * d) - min_s) / (max_s - min_s);
|
||||
return std::min(n_shells - 1, std::max(0, static_cast<int>(t * n_shells)));
|
||||
};
|
||||
// Group intensities by shell, and accumulate <I/sigma> to gauge each shell's signal.
|
||||
std::vector<std::vector<double>> shell_I(n_shells);
|
||||
std::vector<double> shell_isig_sum(n_shells, 0.0);
|
||||
std::vector<int> shell_isig_n(n_shells, 0);
|
||||
for (const auto& r : merged) {
|
||||
if (!used(r))
|
||||
continue;
|
||||
const int b = shell_of(r.d);
|
||||
shell_I[b].push_back(r.I);
|
||||
if (std::isfinite(r.sigma) && r.sigma > 0.0) {
|
||||
shell_isig_sum[b] += r.I / r.sigma;
|
||||
shell_isig_n[b] += 1;
|
||||
}
|
||||
}
|
||||
// The mean intensity of each shell that carries signal; 0 marks a shell that does not.
|
||||
std::vector<double> shell_mean(n_shells, 0.0);
|
||||
for (int b = 0; b < n_shells; ++b) {
|
||||
const auto& intensities = shell_I[b];
|
||||
if (intensities.empty() || shell_isig_n[b] == 0
|
||||
|| shell_isig_sum[b] / shell_isig_n[b] < min_shell_isig)
|
||||
continue;
|
||||
double sum = 0.0;
|
||||
for (double I : intensities)
|
||||
sum += I;
|
||||
const double mean = sum / intensities.size();
|
||||
if (mean <= 0.0)
|
||||
continue;
|
||||
// Re-fit the mean over the reflections that pass the outlier cut, so the outlier does not
|
||||
// inflate the very mean it is measured against.
|
||||
sum = 0.0;
|
||||
int n_kept = 0;
|
||||
for (double I : intensities)
|
||||
if (I / mean <= wilson_outlier_e2) {
|
||||
sum += I;
|
||||
++n_kept;
|
||||
}
|
||||
if (n_kept > 0 && sum > 0.0)
|
||||
shell_mean[b] = sum / n_kept;
|
||||
}
|
||||
|
||||
// --- Second moment <I^2>/<I>^2 of acentric intensities, normalised per resolution shell ---
|
||||
double sum_e4 = 0.0;
|
||||
int n_moment = 0;
|
||||
for (int b = 0; b < n_shells; ++b) {
|
||||
if (shell_mean[b] <= 0.0)
|
||||
continue;
|
||||
for (double I : shell_I[b]) {
|
||||
const double e2 = I / shell_mean[b];
|
||||
if (e2 > wilson_outlier_e2)
|
||||
continue;
|
||||
sum_e4 += e2 * e2;
|
||||
++n_moment;
|
||||
}
|
||||
}
|
||||
result.moment_reflections = n_moment;
|
||||
if (n_moment > 0)
|
||||
result.second_moment = sum_e4 / n_moment;
|
||||
|
||||
// --- L-test ---
|
||||
// Following Padilla & Yeates (2003) Acta Cryst. D59, 1124-1130
|
||||
@@ -66,11 +196,24 @@ TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merg
|
||||
// merged reflections are unique in the asymmetric unit, so any other merged reflection is
|
||||
// genuinely non-equivalent - exactly the pairing the L-test wants. Only acentric reflections
|
||||
// with positive intensity enter, which also keeps L = (I1-I2)/(I1+I2) bounded in [-1, 1].
|
||||
//
|
||||
// The test assumes the two members of a pair have the same EXPECTED intensity, and two index
|
||||
// steps are not the same resolution: on a small cell with a large B the pair's expected
|
||||
// intensities differ by up to a factor ~3 at 2 A, which pushes <|L|> up by as much as 0.08 (an
|
||||
// untwinned crystal read 0.568, a partial twin 0.427 where phenix.xtriage reads 0.486 and 0.360).
|
||||
// So each intensity is divided by its shell mean first, as xtriage and ctruncate do, and only the
|
||||
// shells the second moment reads enter. The selection stays on the SHELL: cutting individual
|
||||
// reflections on their own I/sigma removes the weak tail of every shell and biases <|L|> down
|
||||
// (the same untwinned crystal reads 0.423 with I/sigma > 2).
|
||||
std::unordered_map<int64_t, double> intensity;
|
||||
intensity.reserve(merged.size() * 2);
|
||||
for (const auto& r : merged)
|
||||
if (UsableIntensity(r) && r.I > 0.0 && acentric(r))
|
||||
intensity.emplace(PackHKL(r.h, r.k, r.l), r.I);
|
||||
for (const auto& r : merged) {
|
||||
if (!used(r) || r.I <= 0.0)
|
||||
continue;
|
||||
const double mean = shell_mean[shell_of(r.d)];
|
||||
if (mean > 0.0)
|
||||
intensity.emplace(PackHKL(r.h, r.k, r.l), r.I / mean);
|
||||
}
|
||||
|
||||
// Step to a nearby, symmetry-independent partner. The axis step of 2 is load-bearing for TWO
|
||||
// reasons, and only the first was ever written down.
|
||||
@@ -165,98 +308,20 @@ TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merg
|
||||
run_l_test(all_steps);
|
||||
}
|
||||
|
||||
// --- Second moment <I^2>/<I>^2 of acentric intensities, normalised per resolution shell ---
|
||||
// Binning by 1/d^2 removes the resolution fall-off, so the moment is 2.0 (untwinned) or 1.5
|
||||
// (perfect twin) regardless of the overall B-factor. The moment divides by the *square* of the
|
||||
// shell-mean intensity, so it is not robust: on weak or mis-integrated data a shell mean can
|
||||
// collapse to the noise floor and one outlier reflection then dominates (I/mean)^2 (a single
|
||||
// I=158 in a mean~1 shell contributed 78% of a whole dataset's value). To keep this a twinning
|
||||
// indicator rather than a data-quality artefact - as phenix.xtriage does - we skip noise-only
|
||||
// shells (<I/sigma> below 1) and reject Wilson outliers (E^2 above 8, ~exp(-8) upper tail) with
|
||||
// one shell-mean re-iteration so the outlier does not corrupt the normalising mean either.
|
||||
constexpr double min_shell_isig = 1.0; // shells below this are noise, not signal
|
||||
constexpr double wilson_outlier_e2 = 8.0; // reject improbably strong reflections (P ~ e^-8)
|
||||
int n_shells = std::max(1, resolution_shells);
|
||||
double min_s = std::numeric_limits<double>::infinity();
|
||||
double max_s = -std::numeric_limits<double>::infinity();
|
||||
for (const auto& r : merged) {
|
||||
if (!UsableIntensity(r) || !acentric(r))
|
||||
continue;
|
||||
const double s = 1.0 / (r.d * r.d);
|
||||
min_s = std::min(min_s, s);
|
||||
max_s = std::max(max_s, s);
|
||||
}
|
||||
if (std::isfinite(min_s) && max_s > min_s) {
|
||||
auto shell_of = [&](double d) {
|
||||
const double t = (1.0 / (d * d) - min_s) / (max_s - min_s);
|
||||
return std::min(n_shells - 1, std::max(0, static_cast<int>(t * n_shells)));
|
||||
};
|
||||
// Group acentric intensities by shell, and accumulate <I/sigma> to gauge each shell's signal.
|
||||
std::vector<std::vector<double>> shell_I(n_shells);
|
||||
std::vector<double> shell_isig_sum(n_shells, 0.0);
|
||||
std::vector<int> shell_isig_n(n_shells, 0);
|
||||
for (const auto& r : merged) {
|
||||
if (!UsableIntensity(r) || !acentric(r))
|
||||
continue;
|
||||
const int b = shell_of(r.d);
|
||||
shell_I[b].push_back(r.I);
|
||||
if (std::isfinite(r.sigma) && r.sigma > 0.0) {
|
||||
shell_isig_sum[b] += r.I / r.sigma;
|
||||
shell_isig_n[b] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
double sum_e4 = 0.0;
|
||||
int n_moment = 0;
|
||||
for (int b = 0; b < n_shells; ++b) {
|
||||
const auto& intensities = shell_I[b];
|
||||
if (intensities.empty() || shell_isig_n[b] == 0
|
||||
|| shell_isig_sum[b] / shell_isig_n[b] < min_shell_isig)
|
||||
continue;
|
||||
|
||||
double sum = 0.0;
|
||||
for (double I : intensities)
|
||||
sum += I;
|
||||
double mean = sum / intensities.size();
|
||||
if (mean <= 0.0)
|
||||
continue;
|
||||
// Re-fit the mean over the reflections that pass the outlier cut, so the outlier does not
|
||||
// inflate the very mean it is measured against.
|
||||
sum = 0.0;
|
||||
int n_kept = 0;
|
||||
for (double I : intensities)
|
||||
if (I / mean <= wilson_outlier_e2) {
|
||||
sum += I;
|
||||
++n_kept;
|
||||
}
|
||||
if (n_kept == 0)
|
||||
continue;
|
||||
mean = sum / n_kept;
|
||||
if (mean <= 0.0)
|
||||
continue;
|
||||
|
||||
for (double I : intensities) {
|
||||
const double e2 = I / mean;
|
||||
if (e2 > wilson_outlier_e2)
|
||||
continue;
|
||||
sum_e4 += e2 * e2;
|
||||
++n_moment;
|
||||
}
|
||||
}
|
||||
result.moment_reflections = n_moment;
|
||||
if (n_moment > 0)
|
||||
result.second_moment = sum_e4 / n_moment;
|
||||
}
|
||||
|
||||
// Twin fraction from the second moment M = 2(1 - a + a^2): a = (1 - sqrt(2M-3))/2.
|
||||
if (result.second_moment > 0.0) {
|
||||
const double m = std::clamp(result.second_moment, 1.5, 2.0);
|
||||
result.estimated_twin_fraction = (1.0 - std::sqrt(std::max(0.0, 2.0 * m - 3.0))) / 2.0;
|
||||
}
|
||||
// Either indicator dropping clearly below its untwinned value is suspicious - but only where a twin
|
||||
// law can actually exist. In a holohedral Laue class (e.g. 422) no merohedral twinning is
|
||||
// possible, so a low <|L|> is a statistical artefact (correlated near-neighbours) rather than a
|
||||
// twin, and must not be flagged.
|
||||
// Either indicator dropping clearly below its untwinned value is suspicious - in every Laue
|
||||
// class, holohedral ones included. A holohedral class admits no twin law WITHIN it, so there a low
|
||||
// reading does not mean "twinned in this group"; it means an operator of the group averages
|
||||
// intensities that are not equal. Merging I(h) with I(Th) under a false operator T gives
|
||||
// (I(h) + I(Th))/2 for every twin fraction - the fraction cancels - so a merge under a false
|
||||
// operator has the perfect-twin distribution by construction, whether the crystal is an untwinned
|
||||
// pseudo-symmetric one, a partial twin or a perfect twin, and only a genuine operator leaves it
|
||||
// at the untwinned 0.5. Noise cannot fake it (with the I > 0 selection it pushes <|L|> towards
|
||||
// ~0.43, not 0.375) and a pseudo-translation pushes the second moment UP. Measured over 42
|
||||
// holohedral merges of the corpus, the genuine ones read <|L|> 0.436-0.513 and the one
|
||||
// over-promotion (an H3 twin merged in R32) 0.374. One more reads low, 0.365: a crystal with a
|
||||
// ~490 A axis, whose neighbouring reflections overlap and share counts - averaging of another
|
||||
// kind, which narrows the distribution the same way. That is why this is a warning and never a
|
||||
// veto on the space group.
|
||||
result.merohedral_twinning_possible = MerohedralTwinningPossible(space_group);
|
||||
// The two indicators can only move one way under twinning: <|L|> down from 0.500 towards 0.375,
|
||||
// the second moment down from 2.0 towards 1.5. A narrow intensity distribution sitting next to an
|
||||
@@ -282,9 +347,39 @@ TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merg
|
||||
// alone, and the text says the L-test could not be read. This REMOVES a veto; it never adds one.
|
||||
const bool l_test_usable = result.l_test_pairs > 0 && !result.l_test_contaminated_by_tncs;
|
||||
const bool l_test_contradicts_twin = l_test_usable && result.mean_abs_l >= 0.50;
|
||||
result.twinning_suspected = result.merohedral_twinning_possible && !l_test_contradicts_twin &&
|
||||
((l_test_usable && result.mean_abs_l < 0.44) ||
|
||||
(result.moment_reflections > 0 && result.second_moment < 1.85));
|
||||
const bool l_test_low = l_test_usable && result.mean_abs_l < 0.44;
|
||||
const bool moment_low = result.moment_reflections > 0 && result.second_moment < 1.85;
|
||||
if (result.merohedral_twinning_possible)
|
||||
result.twinning_suspected = !l_test_contradicts_twin && (l_test_low || moment_low);
|
||||
else {
|
||||
// In a holohedral class the alternative is not a partial twin anywhere between 0.5 and 0.375
|
||||
// but a false operator, which puts <|L|> at the perfect-twin 0.375 itself - so the bar sits
|
||||
// nearer that value. Measured on the corpus (normalised L-test): the weakest genuine
|
||||
// holohedral crystal reads 0.436, the over-promoted twin 0.374. The second moment does not
|
||||
// enter here: it is the statistic a pseudo-translation inflates (that same twin read 2.08,
|
||||
// the untwinned value), and it is the one a pseudo-cubic lattice in a 4/mmm group, whose twin
|
||||
// law lies OUTSIDE the Laue class, pulls down (1.66 beside an untwinned <|L|>).
|
||||
result.adopted_operators_suspect = l_test_usable && result.mean_abs_l < 0.42;
|
||||
result.twinning_suspected = result.adopted_operators_suspect;
|
||||
}
|
||||
|
||||
// The twin fraction, from the statistic that carries the verdict. Merged under a false operator
|
||||
// the fraction is lost (see above), so none is quoted there. Otherwise the L-test, which is local
|
||||
// and so indifferent to anisotropy and to a pseudo-translation it has been repaired for, unless
|
||||
// the call rests on the second moment alone. The second moment is not quoted under a detected
|
||||
// pseudo-translation, which inflates it: an H3 twin with tNCS read 2.08, the untwinned value,
|
||||
// beside a perfect-twin <|L|>.
|
||||
if (!result.adopted_operators_suspect) {
|
||||
if (l_test_usable && (l_test_low || !moment_low)) {
|
||||
result.estimated_twin_fraction = TwinFractionFromMeanAbsL(result.mean_abs_l);
|
||||
result.twin_fraction_source = TwinFractionSource::LTest;
|
||||
} else if (result.moment_reflections > 0 && tncs_vector == nullptr) {
|
||||
// M = 2(1 - a + a^2): a = (1 - sqrt(2M-3))/2.
|
||||
const double m = std::clamp(result.second_moment, 1.5, 2.0);
|
||||
result.estimated_twin_fraction = (1.0 - std::sqrt(std::max(0.0, 2.0 * m - 3.0))) / 2.0;
|
||||
result.twin_fraction_source = TwinFractionSource::SecondMoment;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -310,34 +405,46 @@ std::string TwinningAnalysisToText(const TwinningAnalysisResult& result) {
|
||||
if (result.moment_reflections > 0)
|
||||
os << " Second moment <I^2>/<I>^2 = " << result.second_moment
|
||||
<< " [untwinned 2.00, perfect twin 1.50]\n";
|
||||
if (result.twinning_suspected && result.estimated_twin_fraction > 0.01)
|
||||
os << " => Twinning suspected (estimated twin fraction ~"
|
||||
<< result.estimated_twin_fraction << "). Statistics flag the presence of twinning, not\n"
|
||||
<< " the twin law; the law itself is a question for a dedicated twin-law analysis.\n";
|
||||
else if (result.twinning_suspected)
|
||||
// The fraction is derived from the second moment alone, so a twin called by the L-test on a
|
||||
// crystal whose second moment sits at or above its untwinned value has no fraction to quote -
|
||||
// printing the 0.000 the formula returns would contradict the sentence it sits in. The
|
||||
// disagreement is worth stating, and where a pseudo-symmetry is present it has a named cause.
|
||||
os << " => Twinning suspected on the L-test. The second moment does not agree ("
|
||||
<< result.second_moment << ", at or\n"
|
||||
<< " above its untwinned 2.00), so no twin fraction is quoted"
|
||||
<< (result.l_test_tncs_step_restricted
|
||||
? " - and this\n"
|
||||
" crystal's translational pseudo-symmetry is a known reason for the second\n"
|
||||
" moment to run high, so the L-test carries the verdict here.\n"
|
||||
: ".\n")
|
||||
<< " Statistics flag the presence of twinning, not the twin law; the law itself is\n"
|
||||
<< " a question for a dedicated twin-law analysis.\n";
|
||||
else if (!result.merohedral_twinning_possible && result.laue_class_was_chosen_by_promotion)
|
||||
os << " => Cannot rule out twinning from these numbers: the Laue class is holohedral, so no\n"
|
||||
<< " merohedral twin law exists WITHIN it - but this Laue class was chosen by the\n"
|
||||
<< " space-group search itself, and promoting into a twin's holohedry is precisely what\n"
|
||||
<< " a merohedral twin looks like. The subgroup statistics reported by the search,\n"
|
||||
<< " not these numbers, are where the twinning is decided.\n";
|
||||
const bool l_test_read = result.l_test_pairs > 0 && !result.l_test_contaminated_by_tncs;
|
||||
if (result.adopted_operators_suspect)
|
||||
os << " => An adopted operator relates UNEQUAL intensities. The Laue class is holohedral, so no\n"
|
||||
<< " twin law exists within it - yet the intensities read like a twin's, and merging under\n"
|
||||
<< " a false operator produces exactly that distribution, whatever the twin fraction.\n"
|
||||
<< (result.laue_class_was_chosen_by_promotion
|
||||
? " This class was chosen by the space-group search: the promotion is suspect.\n"
|
||||
: " The space group in use is higher than these intensities support.\n")
|
||||
<< " Either the crystal has lower symmetry, or it is a twin whose law the point group has\n"
|
||||
<< " absorbed; either way it has to be refined in a subgroup. The twin fraction cannot be\n"
|
||||
<< " measured on this merge.\n";
|
||||
else if (result.twinning_suspected) {
|
||||
os << " => Twinning suspected";
|
||||
if (result.twin_fraction_source == TwinFractionSource::LTest)
|
||||
os << " (estimated twin fraction ~" << result.estimated_twin_fraction << ", from the L-test)";
|
||||
else if (result.twin_fraction_source == TwinFractionSource::SecondMoment)
|
||||
os << " (estimated twin fraction ~" << result.estimated_twin_fraction
|
||||
<< ", from the second moment)";
|
||||
else
|
||||
os << " on the second moment; no twin fraction is quoted, because this crystal's\n"
|
||||
<< " pseudo-translation inflates the second moment and <|L|> does not agree";
|
||||
os << ".\n";
|
||||
if (l_test_read && result.mean_abs_l < 0.44 && result.moment_reflections > 0
|
||||
&& result.second_moment >= 1.85)
|
||||
os << " The second moment does not agree (" << result.second_moment << ")"
|
||||
<< (result.l_test_tncs_step_restricted
|
||||
? " - this crystal's translational pseudo-symmetry is a known reason for it\n"
|
||||
" to run high, so the L-test carries the verdict here.\n"
|
||||
: ".\n");
|
||||
os << " Statistics flag the presence of twinning, not the twin law; the law itself is a\n"
|
||||
<< " question for a dedicated twin-law analysis.\n";
|
||||
} else if (!result.merohedral_twinning_possible && result.laue_class_was_chosen_by_promotion)
|
||||
os << " => No twinning indicated. The Laue class is holohedral, so no twin law exists within it,\n"
|
||||
<< " and it was chosen by the space-group search - but its operators read as genuine: a\n"
|
||||
<< " merge under a false operator has the perfect-twin distribution, not the untwinned one\n"
|
||||
<< " measured here. An operator that is only APPROXIMATE (non-crystallographic symmetry\n"
|
||||
<< " along it) also passes; the subgroup statistics reported by the search speak to that.\n";
|
||||
else if (!result.merohedral_twinning_possible)
|
||||
os << " => No twinning: the Laue class is holohedral, so no merohedral twin law exists\n"
|
||||
<< " (any <|L|> below 0.5 here is a statistical artefact, not twinning).\n";
|
||||
os << " => No twinning: the Laue class is holohedral, so no merohedral twin law exists, and no\n"
|
||||
<< " operator of it averages unequal intensities.\n";
|
||||
else if (result.l_test_contaminated_by_tncs)
|
||||
os << " => No twinning indicated by the second moment. The L-test, which is normally the\n"
|
||||
<< " stronger of the two, could not be read on this crystal (see above), so this is a\n"
|
||||
@@ -347,3 +454,262 @@ std::string TwinningAnalysisToText(const TwinningAnalysisResult& result) {
|
||||
os << " => No twinning indicated.\n";
|
||||
return os.str();
|
||||
}
|
||||
|
||||
std::string TwinningVerdictLine(const TwinningAnalysisResult& result) {
|
||||
std::ostringstream os;
|
||||
os << std::fixed << std::setprecision(3);
|
||||
const bool l_test_read = result.l_test_pairs > 0 && !result.l_test_contaminated_by_tncs;
|
||||
if (result.adopted_operators_suspect)
|
||||
os << "SYMMETRY SUSPECT (<|L|> " << result.mean_abs_l << ", second moment "
|
||||
<< std::setprecision(2) << result.second_moment
|
||||
<< ": an adopted operator averages unequal intensities)";
|
||||
else if (result.twinning_suspected) {
|
||||
os << "INDICATED (<|L|> " << result.mean_abs_l;
|
||||
if (result.twin_fraction_source != TwinFractionSource::None)
|
||||
os << std::setprecision(2) << ", twin fraction ~" << result.estimated_twin_fraction << " from the "
|
||||
<< (result.twin_fraction_source == TwinFractionSource::LTest ? "L-test" : "second moment");
|
||||
os << ")";
|
||||
} else if (l_test_read)
|
||||
os << "no indication (<|L|> " << result.mean_abs_l << ")";
|
||||
else
|
||||
os << "no indication (second moment " << std::setprecision(2) << result.second_moment
|
||||
<< "; <|L|> not readable)";
|
||||
return os.str();
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Measurement noise inflates <|E^2-1|> towards centric in every class (by +0.44 at a shell
|
||||
// <I/sigma> of 2.5 and ~0 at 16, measured), so only shells above this are read. At 5 the acentric
|
||||
// control of four reference merges reads 0.65-0.74.
|
||||
constexpr double ZONE_MIN_SHELL_ISIG = 5.0;
|
||||
// Each class is normalised against its own mean in resolution bins of this many reflections, so a
|
||||
// zone that is a plane through reciprocal space is not judged on another direction's fall-off.
|
||||
constexpr size_t ZONE_BIN = 100;
|
||||
// A reflection this far above its bin mean is not a Wilson draw (centric P ~ 1e-5): one such
|
||||
// reflection read 2.33 for a whole zone.
|
||||
constexpr double ZONE_OUTLIER_E2 = 20.0;
|
||||
|
||||
// ln p_centric(E^2) - ln p_acentric(E^2) for a measured E^2 with error s: both Wilson densities
|
||||
// convolved with the measurement error. Without the convolution the centric density, infinite at
|
||||
// zero, reads every noisy weak reflection as decisive; flooring E^2 at s instead turns the centric
|
||||
// excess of weak reflections into moderate values and read a genuine centric zone of a 1.2 A
|
||||
// crystal as acentric. Integrated over u = sqrt(E^2_true), where neither density is singular,
|
||||
// across +-6 s of the measurement (trapezoid).
|
||||
double CentricOverAcentric(double e2, double s) {
|
||||
constexpr int n = 200;
|
||||
const double u_lo = std::sqrt(std::max(0.0, e2 - 6.0 * s));
|
||||
const double u_hi = std::sqrt(std::max(0.0, e2 + 6.0 * s));
|
||||
if (!(u_hi > u_lo))
|
||||
return 0.0;
|
||||
const double du = (u_hi - u_lo) / n;
|
||||
double p_centric = 0.0, p_acentric = 0.0;
|
||||
for (int i = 0; i <= n; ++i) {
|
||||
const double u = u_lo + i * du;
|
||||
const double w = (i == 0 || i == n ? 0.5 : 1.0) * std::exp(-0.5 * std::pow((e2 - u * u) / s, 2));
|
||||
p_centric += w * std::sqrt(2.0 / std::numbers::pi) * std::exp(-0.5 * u * u);
|
||||
p_acentric += w * 2.0 * u * std::exp(-u * u);
|
||||
}
|
||||
return p_centric > 0.0 && p_acentric > 0.0 ? std::log(p_centric / p_acentric) : 0.0;
|
||||
}
|
||||
|
||||
TwinImmuneZone ReadZone(std::vector<const MergedReflection*> refl, const std::array<double, 3>* tncs) {
|
||||
TwinImmuneZone z;
|
||||
// A pseudo-translation u multiplies the intensity by 1 + f cos(2 pi h.u): normalising within
|
||||
// each sign of the cosine keeps the two populations from reading as one broad distribution.
|
||||
std::vector<const MergedReflection*> classes[2];
|
||||
for (const auto* r : refl) {
|
||||
const double c = tncs == nullptr ? 1.0
|
||||
: std::cos(2.0 * std::numbers::pi * (r->h * (*tncs)[0] + r->k * (*tncs)[1] + r->l * (*tncs)[2]));
|
||||
classes[c >= 0.0 ? 0 : 1].push_back(r);
|
||||
}
|
||||
double sum = 0.0, sum2 = 0.0;
|
||||
for (auto& cls : classes) {
|
||||
std::sort(cls.begin(), cls.end(), [](const auto* a, const auto* b) { return a->d > b->d; });
|
||||
const size_t n_bins = std::max<size_t>(1, cls.size() / ZONE_BIN);
|
||||
for (size_t b = 0; b < n_bins; ++b) {
|
||||
const size_t first = b * cls.size() / n_bins, last = (b + 1) * cls.size() / n_bins;
|
||||
double mean = 0.0;
|
||||
for (size_t i = first; i < last; ++i)
|
||||
mean += cls[i]->I;
|
||||
mean /= static_cast<double>(last - first);
|
||||
if (mean <= 0.0)
|
||||
continue;
|
||||
double kept = 0.0;
|
||||
int n_kept = 0;
|
||||
for (size_t i = first; i < last; ++i)
|
||||
if (cls[i]->I / mean <= ZONE_OUTLIER_E2) {
|
||||
kept += cls[i]->I;
|
||||
++n_kept;
|
||||
}
|
||||
if (n_kept == 0 || kept <= 0.0)
|
||||
continue;
|
||||
mean = kept / n_kept;
|
||||
for (size_t i = first; i < last; ++i) {
|
||||
const double e2 = cls[i]->I / mean;
|
||||
if (e2 > ZONE_OUTLIER_E2)
|
||||
continue;
|
||||
const double v = std::fabs(e2 - 1.0);
|
||||
sum += v;
|
||||
sum2 += v * v;
|
||||
++z.n;
|
||||
z.evidence_nats += CentricOverAcentric(e2, cls[i]->sigma / mean);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (z.n > 0) {
|
||||
z.mean_abs_e2_minus_1 = sum / z.n;
|
||||
const double var = std::max(0.0, sum2 / z.n - z.mean_abs_e2_minus_1 * z.mean_abs_e2_minus_1);
|
||||
z.standard_error = std::sqrt(var / z.n);
|
||||
}
|
||||
return z;
|
||||
}
|
||||
}
|
||||
|
||||
TwinImmuneZoneResult AnalyzeTwinImmuneZones(const std::vector<MergedReflection>& p1_merged,
|
||||
const gemmi::SpaceGroup& group,
|
||||
const std::array<double, 3>* tncs_vector) {
|
||||
TwinImmuneZoneResult result;
|
||||
result.tncs_normalised = tncs_vector != nullptr;
|
||||
const gemmi::GroupOps gops = group.operations();
|
||||
|
||||
// The index-2 subgroups H of the group's rotations: each is a hypothesis "the crystal is H, and the
|
||||
// rest of the group (G minus H, one coset) is a twin law or a pseudo-symmetry". H holds every square
|
||||
// of G, so it is the subgroup the squares generate, or that plus one more rotation; for a proper
|
||||
// point group that finds them all (G over its squares is at most a four-group).
|
||||
auto same = [](const gemmi::Op& a, const gemmi::Op& b) { return a.rot == b.rot; };
|
||||
auto contains = [&](const std::vector<gemmi::Op>& set, const gemmi::Op& op) {
|
||||
return std::any_of(set.begin(), set.end(), [&](const gemmi::Op& o) { return same(o, op); });
|
||||
};
|
||||
auto closure = [&](const std::vector<gemmi::Op>& generators) {
|
||||
std::vector<gemmi::Op> ops;
|
||||
for (const auto& g : generators)
|
||||
if (!contains(ops, g))
|
||||
ops.push_back(g);
|
||||
for (size_t i = 0; i < ops.size(); ++i)
|
||||
for (size_t j = 0; j <= i; ++j)
|
||||
for (const gemmi::Op& p : {ops[i] * ops[j], ops[j] * ops[i]})
|
||||
if (!contains(ops, p))
|
||||
ops.push_back(p);
|
||||
return ops;
|
||||
};
|
||||
std::vector<gemmi::Op> rotations;
|
||||
for (const auto& op : gops.derive_symmorphic().sym_ops)
|
||||
if (op.det_rot() > 0)
|
||||
rotations.push_back(gemmi::Op{op.rot, {0, 0, 0}, 'x'});
|
||||
std::vector<gemmi::Op> squares{gemmi::Op::identity()};
|
||||
for (const auto& g : rotations)
|
||||
squares.push_back(g * g);
|
||||
const auto square_group = closure(squares);
|
||||
std::vector<std::vector<gemmi::Op>> subgroups;
|
||||
auto add_subgroup = [&](const std::vector<gemmi::Op>& h) {
|
||||
if (2 * h.size() != rotations.size())
|
||||
return;
|
||||
for (const auto& known : subgroups)
|
||||
if (known.size() == h.size()
|
||||
&& std::all_of(h.begin(), h.end(), [&](const gemmi::Op& o) { return contains(known, o); }))
|
||||
return;
|
||||
subgroups.push_back(h);
|
||||
};
|
||||
add_subgroup(square_group);
|
||||
for (const auto& g : rotations) {
|
||||
auto gen = square_group;
|
||||
gen.push_back(g);
|
||||
add_subgroup(closure(gen));
|
||||
}
|
||||
if (subgroups.empty())
|
||||
return result;
|
||||
|
||||
// Reflections present in the group and in general position (epsilon 1): an axial row is fixed by
|
||||
// more than one operator and follows neither distribution.
|
||||
std::vector<const MergedReflection*> usable;
|
||||
for (const auto& r : p1_merged) {
|
||||
const gemmi::Op::Miller h{{r.h, r.k, r.l}};
|
||||
if (UsableIntensity(r) && std::isfinite(r.sigma) && r.sigma > 0.0
|
||||
&& !gops.is_systematically_absent(h) && gops.epsilon_factor_without_centering(h) == 1)
|
||||
usable.push_back(&r);
|
||||
}
|
||||
// The shells with signal enough to read (ZONE_MIN_SHELL_ISIG), 20 equal in 1/d^2.
|
||||
constexpr int n_shells = 20;
|
||||
double min_s = std::numeric_limits<double>::infinity(), max_s = 0.0;
|
||||
for (const auto* r : usable) {
|
||||
min_s = std::min(min_s, 1.0 / (r->d * r->d));
|
||||
max_s = std::max(max_s, 1.0 / (r->d * r->d));
|
||||
}
|
||||
if (!(max_s > min_s))
|
||||
return result;
|
||||
auto shell_of = [&](const MergedReflection* r) {
|
||||
const double t = (1.0 / (r->d * r->d) - min_s) / (max_s - min_s);
|
||||
return std::min(n_shells - 1, std::max(0, static_cast<int>(t * n_shells)));
|
||||
};
|
||||
std::vector<double> isig_sum(n_shells, 0.0);
|
||||
std::vector<int> isig_n(n_shells, 0);
|
||||
for (const auto* r : usable) {
|
||||
isig_sum[shell_of(r)] += r->I / r->sigma;
|
||||
isig_n[shell_of(r)] += 1;
|
||||
}
|
||||
std::vector<const MergedReflection*> strong;
|
||||
for (const auto* r : usable) {
|
||||
const int b = shell_of(r);
|
||||
if (isig_n[b] > 0 && isig_sum[b] / isig_n[b] >= ZONE_MIN_SHELL_ISIG)
|
||||
strong.push_back(r);
|
||||
}
|
||||
if (strong.empty())
|
||||
return result;
|
||||
result.d_max_A = 0.0;
|
||||
result.d_min_A = std::numeric_limits<double>::infinity();
|
||||
for (const auto* r : strong) {
|
||||
result.d_max_A = std::max<double>(result.d_max_A, r->d);
|
||||
result.d_min_A = std::min<double>(result.d_min_A, r->d);
|
||||
}
|
||||
|
||||
std::vector<const MergedReflection*> control;
|
||||
for (const auto* r : strong)
|
||||
if (!gops.is_reflection_centric(gemmi::Op::Miller{{r->h, r->k, r->l}}))
|
||||
control.push_back(r);
|
||||
result.control = ReadZone(control, tncs_vector);
|
||||
result.control.operators = "acentric";
|
||||
|
||||
// Per hypothesis H, the zone: centric in the group, acentric in H - the reflections some added
|
||||
// operator sends to -h, i.e. the self-mates of that coset read as a twin law.
|
||||
for (const auto& h_ops : subgroups) {
|
||||
std::vector<const MergedReflection*> zone;
|
||||
for (const auto* r : strong) {
|
||||
const gemmi::Op::Miller h{{r->h, r->k, r->l}};
|
||||
const gemmi::Op::Miller minus{{-gemmi::Op::DEN * r->h, -gemmi::Op::DEN * r->k, -gemmi::Op::DEN * r->l}};
|
||||
const bool centric_in_h = std::any_of(h_ops.begin(), h_ops.end(), [&](const gemmi::Op& op) {
|
||||
return op.apply_to_hkl_without_division(h) == minus;
|
||||
});
|
||||
if (gops.is_reflection_centric(h) && !centric_in_h)
|
||||
zone.push_back(r);
|
||||
}
|
||||
TwinImmuneZone z = ReadZone(zone, tncs_vector);
|
||||
for (const auto& op : rotations)
|
||||
if (!contains(h_ops, op))
|
||||
z.operators += (z.operators.empty() ? "" : " ") + op.as_hkl().triplet();
|
||||
result.zones.push_back(z);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string TwinImmuneZonesToText(const TwinImmuneZoneResult& result) {
|
||||
std::ostringstream os;
|
||||
os << std::fixed;
|
||||
os << "Twin-immune zones of the operators the adopted group adds over each index-2 subgroup (P1 merge, " << std::setprecision(2) << result.d_max_A
|
||||
<< "-" << result.d_min_A << " A, shells with <I/sigma> >= " << std::setprecision(0)
|
||||
<< ZONE_MIN_SHELL_ISIG << (result.tncs_normalised ? ", normalised per pseudo-translation class" : "")
|
||||
<< ")\n"
|
||||
<< " Reflections centric in the group but not in the subgroup are their own twin mates: centric if\n"
|
||||
<< " the added operators are real, acentric if they are a twin law or a pseudo-symmetry. <|E^2-1|>\n"
|
||||
<< " reads 0.968 centric, 0.736 acentric; evidence is the centric/acentric log-likelihood ratio\n"
|
||||
<< " (> 0: the added operators are real).\n";
|
||||
auto row = [&](const TwinImmuneZone& z) {
|
||||
os << " " << std::left << std::setw(34) << z.operators << std::right << " n " << std::setw(6) << z.n
|
||||
<< " <|E^2-1|> " << std::setprecision(3) << z.mean_abs_e2_minus_1 << " +- " << z.standard_error
|
||||
<< " evidence " << std::showpos << std::setprecision(1) << z.evidence_nats << std::noshowpos
|
||||
<< " nats\n";
|
||||
};
|
||||
for (const auto& z : result.zones)
|
||||
row(z);
|
||||
row(result.control);
|
||||
return os.str();
|
||||
}
|
||||
@@ -10,18 +10,23 @@
|
||||
#include "../../common/Reflection.h"
|
||||
#include "gemmi/symmetry.hpp"
|
||||
|
||||
// Intensity-statistics test for merohedral twinning, in the spirit of phenix.xtriage. Two robust,
|
||||
// normalisation-free indicators on the merged intensities:
|
||||
// Intensity-statistics test for merohedral twinning, in the spirit of phenix.xtriage. Two
|
||||
// indicators on the merged intensities, both normalised per resolution shell:
|
||||
//
|
||||
// - Padilla & Yeates L-test (Acta Cryst D59, 2003): for pairs of nearby but symmetry-independent
|
||||
// reflections, L = (I1 - I2)/(I1 + I2). Untwinned acentric data give <|L|> = 0.5, <L^2> = 1/3;
|
||||
// a perfect merohedral twin narrows this to <|L|> = 0.375, <L^2> = 0.2. Insensitive to
|
||||
// anisotropy and overall scale because it is a local ratio.
|
||||
// anisotropy and overall scale because it is a local ratio of shell-normalised intensities.
|
||||
//
|
||||
// - Second moment <I^2>/<I>^2 of acentric intensities, normalised per resolution shell so it is
|
||||
// 2.0 for untwinned data and 1.5 for a perfect twin (= 2(1 - a + a^2) for twin fraction a).
|
||||
//
|
||||
// These flag the *presence* of twinning; they do not by themselves identify the twin law.
|
||||
// These flag the *presence* of twinning; they do not by themselves identify the twin law. In a
|
||||
// holohedral Laue class, where no twin law exists, the same low reading says instead that an operator
|
||||
// of the group averages unequal intensities - the group is too high, or a twin law was absorbed into
|
||||
// it (adopted_operators_suspect).
|
||||
|
||||
enum class TwinFractionSource { None, LTest, SecondMoment };
|
||||
|
||||
struct TwinningAnalysisResult {
|
||||
int l_test_pairs = 0;
|
||||
@@ -31,14 +36,25 @@ struct TwinningAnalysisResult {
|
||||
int moment_reflections = 0;
|
||||
double second_moment = 0.0; // <I^2>/<I>^2 (shell-normalised): 2.0 untwinned, 1.5 perfect twin
|
||||
|
||||
// Twin fraction estimated from the second moment, a = (1 - sqrt(2M-3))/2, clamped to [0, 0.5].
|
||||
// Twin fraction in [0, 0.5], from the statistic named by twin_fraction_source: the L-test unless
|
||||
// the verdict rests on the second moment alone, a = (1 - sqrt(2M-3))/2. None where it cannot be
|
||||
// measured - merged under a false operator, or a second moment under a pseudo-translation.
|
||||
double estimated_twin_fraction = 0.0;
|
||||
TwinFractionSource twin_fraction_source = TwinFractionSource::None;
|
||||
|
||||
// An indicator reads twin-like (<|L|> < 0.44 or second moment < 1.85, and <|L|> not at or above
|
||||
// 0.5). Set in every Laue class; what it means depends on the next two fields.
|
||||
bool twinning_suspected = false;
|
||||
|
||||
// False when the Laue class is holohedral (4/mmm, 6/mmm, m-3m, rhombohedral -3m): no merohedral
|
||||
// twin law can exist, so a low <|L|> / second moment there is a statistical artefact, not twinning.
|
||||
// twin law can exist WITHIN it.
|
||||
bool merohedral_twinning_possible = true;
|
||||
|
||||
// twinning_suspected in a holohedral class: an operator of the group averages intensities that
|
||||
// are not equal, so the group is too high or it has absorbed a twin law. A warning about the
|
||||
// symmetry, not a twin call - and the twin fraction is not measurable on such a merge.
|
||||
bool adopted_operators_suspect = false;
|
||||
|
||||
// How the L-test coped with a translational pseudo-symmetry, when one was declared (see the
|
||||
// partner-step note in the .cpp). Exactly one of these can be set:
|
||||
// - `l_test_tncs_step_restricted`: partners were taken only along steps that preserve the
|
||||
@@ -58,6 +74,8 @@ struct TwinningAnalysisResult {
|
||||
|
||||
// The space group (when known) is used to drop centric reflections, which follow different
|
||||
// statistics (<|L|>, <I^2>/<I>^2 = 3 not 2); pass nullptr for P1 data, where none are centric.
|
||||
// `lattice_centring` drops the reflections the lattice centring extinguishes, for a P1 merge of a
|
||||
// centred lattice (whose space_group is nullptr and so says nothing about them).
|
||||
// `tncs_vector` is the fractional pseudo-translation of a DETECTED translational pseudo-symmetry on
|
||||
// this same merge (TranslationalNCSVectorForLTest), or nullptr. It changes which partner reflections
|
||||
// the L-test pairs with, so it has to be an argument rather than a flag set on the result afterwards:
|
||||
@@ -65,6 +83,49 @@ struct TwinningAnalysisResult {
|
||||
TwinningAnalysisResult AnalyzeTwinning(const std::vector<MergedReflection>& merged,
|
||||
const gemmi::SpaceGroup* space_group = nullptr,
|
||||
int resolution_shells = 20,
|
||||
const std::array<double, 3>* tncs_vector = nullptr);
|
||||
const std::array<double, 3>* tncs_vector = nullptr,
|
||||
char lattice_centring = 'P');
|
||||
|
||||
std::string TwinningAnalysisToText(const TwinningAnalysisResult& result);
|
||||
|
||||
// The verdict in one line - "INDICATED (...)", "SYMMETRY SUSPECT (...)" or "no indication (...)" -
|
||||
// so the report's summary row, its prose and the viewer all say the same thing.
|
||||
std::string TwinningVerdictLine(const TwinningAnalysisResult& result);
|
||||
|
||||
// Twin-immune zone evidence for the operators a point group adds over each of its index-2 subgroups,
|
||||
// read on a P1 merge. REPORT-ONLY.
|
||||
//
|
||||
// Under a twin law T, I_obs(h) = (1-a) I(h) + a I(Th); a reflection whose twin mate Th is h itself, up
|
||||
// to the true group and Friedel, is untouched by twinning at every fraction a. For an index-2 subgroup
|
||||
// H of the adopted group G those are the reflections centric in G but acentric in H: centric if the
|
||||
// operators G adds over H are real, acentric if they are a twin law or a pseudo-symmetry the promotion
|
||||
// took for real. So their intensity distribution says which, and does so where every operator
|
||||
// statistic is blind - at a = 0.5, where I(h) = I(Th) exactly. Read ABSOLUTELY, <|E^2-1|> against
|
||||
// 0.968 (centric) and 0.736 (acentric); never as the difference to the control, because noise, a
|
||||
// pseudo-translation and anisotropy all inflate every class towards centric, and a perfect twin's
|
||||
// control (0.541) plus that inflation looks like true symmetry's.
|
||||
struct TwinImmuneZone {
|
||||
std::string operators; // the operators G adds over H, as h,k,l triplets
|
||||
int n = 0;
|
||||
double mean_abs_e2_minus_1 = 0.0; // 0.968 centric (operator real), 0.736 acentric (not)
|
||||
double standard_error = 0.0;
|
||||
// Log-likelihood ratio, centric over acentric Wilson (each convolved with the reflection's error),
|
||||
// summed over the zone: > 0 favours the operators being real symmetry, < 0 a twin law or
|
||||
// pseudo-symmetry. About +0.2 nats per reflection for a centric zone and -0.13 for an acentric one
|
||||
// on error-free data; less either way as the errors grow.
|
||||
double evidence_nats = 0.0;
|
||||
};
|
||||
|
||||
struct TwinImmuneZoneResult {
|
||||
std::vector<TwinImmuneZone> zones; // one per index-2 subgroup of the group
|
||||
TwinImmuneZone control; // the reflections acentric in the group
|
||||
double d_max_A = 0.0; // resolution range of the shells read
|
||||
double d_min_A = 0.0;
|
||||
bool tncs_normalised = false; // normalised within the pseudo-translation's two phase classes
|
||||
};
|
||||
|
||||
TwinImmuneZoneResult AnalyzeTwinImmuneZones(const std::vector<MergedReflection>& p1_merged,
|
||||
const gemmi::SpaceGroup& group,
|
||||
const std::array<double, 3>* tncs_vector = nullptr);
|
||||
|
||||
std::string TwinImmuneZonesToText(const TwinImmuneZoneResult& result);
|
||||
@@ -8,6 +8,8 @@ ADD_LIBRARY(Rugnux STATIC
|
||||
Rugnux.h
|
||||
RugnuxCommandLine.cpp
|
||||
RugnuxCommandLine.h
|
||||
ModelFFT.cpp
|
||||
ModelFFT.h
|
||||
ModelScaling.cpp
|
||||
ModelScaling.h
|
||||
ModelValidation.cpp
|
||||
@@ -27,7 +29,7 @@ ADD_LIBRARY(Rugnux STATIC
|
||||
WriteModel.h
|
||||
)
|
||||
|
||||
TARGET_LINK_LIBRARIES(Rugnux JFJochReader JFJochImageAnalysis JFJochWriter gemmi)
|
||||
TARGET_LINK_LIBRARIES(Rugnux JFJochReader JFJochImageAnalysis JFJochWriter gemmi fftw3f)
|
||||
|
||||
# rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the
|
||||
# default), azint, scale (re-scale/merge stored reflections) or calibration (powder-ring geometry).
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "ModelFFT.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <complex>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "gemmi/fail.hpp"
|
||||
|
||||
#include "../common/FFTWPlannerLock.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// The r2c plan for an (nu, nv, nw) map stored u fastest, halving w - the layout gemmi's own transform
|
||||
// reads and writes. FFTW's planner is not thread-safe, so plans are made under the process-wide planner
|
||||
// lock (which also guards this cache); executing one on new arrays (fftwf_execute_dft_r2c) is, which is
|
||||
// how every caller uses it.
|
||||
fftwf_plan PlanFor(int nu, int nv, int nw) {
|
||||
static std::map<std::array<int, 3>, fftwf_plan> plans;
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
const std::array<int, 3> key{nu, nv, nw};
|
||||
if (const auto it = plans.find(key); it != plans.end())
|
||||
return it->second;
|
||||
float *in = fftwf_alloc_real(static_cast<size_t>(nu) * nv * nw);
|
||||
fftwf_complex *out = fftwf_alloc_complex(static_cast<size_t>(nu) * nv * (nw / 2 + 1));
|
||||
// The last dimension is the one r2c halves; the strides put u fastest on both sides.
|
||||
fftwf_iodim dims[3] = {{nu, 1, 1}, {nv, nu, nu}, {nw, nu * nv, nu * nv}};
|
||||
fftwf_plan plan = (in != nullptr && out != nullptr)
|
||||
? fftwf_plan_guru_dft_r2c(3, dims, 0, nullptr, in, out, FFTW_ESTIMATE)
|
||||
: nullptr;
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
if (plan == nullptr)
|
||||
gemmi::fail("MapToFPhi(): no FFTW plan for the grid");
|
||||
plans.emplace(key, plan);
|
||||
return plan;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map) {
|
||||
if (map.axis_order == gemmi::AxisOrder::ZYX)
|
||||
gemmi::fail("MapToFPhi(): ZYX order is not supported");
|
||||
gemmi::FPhiGrid<float> hkl;
|
||||
hkl.unit_cell = map.unit_cell;
|
||||
hkl.spacegroup = map.spacegroup;
|
||||
hkl.axis_order = map.axis_order;
|
||||
hkl.half_l = true;
|
||||
hkl.set_size_without_checking(map.nu, map.nv, map.nw / 2 + 1);
|
||||
const float norm = static_cast<float>(map.unit_cell.volume / map.point_count());
|
||||
|
||||
const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw);
|
||||
// Buffers from fftwf_malloc: a plan may only be executed on arrays aligned as the ones it was made
|
||||
// on were, which the vectors' own allocations do not promise.
|
||||
float *in = fftwf_alloc_real(map.data.size());
|
||||
fftwf_complex *out = fftwf_alloc_complex(hkl.data.size());
|
||||
if (in == nullptr || out == nullptr) {
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
gemmi::fail("MapToFPhi(): cannot allocate the FFT buffers");
|
||||
}
|
||||
std::copy(map.data.begin(), map.data.end(), in);
|
||||
fftwf_execute_dft_r2c(plan, in, out);
|
||||
// Conjugated, as gemmi does: FFTW's forward transform carries exp(-2 pi i h.x), and a structure
|
||||
// factor is the sum over exp(+2 pi i h.x).
|
||||
for (size_t i = 0; i < hkl.data.size(); ++i)
|
||||
hkl.data[i] = std::complex<float>(out[i][0] * norm, -out[i][1] * norm);
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
return hkl;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "gemmi/grid.hpp"
|
||||
#include "gemmi/recgrid.hpp"
|
||||
|
||||
// The structure factors of a real-space map on the model path (Fcalc from the model density, Fmask
|
||||
// from the bulk-solvent mask): what gemmi::transform_map_to_f_phi(map, true) returns - the
|
||||
// conjugated forward transform (F = sum rho exp(+2 pi i h.x)) scaled by V/N, kept for l >= 0 on an (nu, nv, nw/2+1) grid in the map's own XYZ order,
|
||||
// ready for prepare_asu_data() - computed with FFTW rather than gemmi's single-threaded pocketfft.
|
||||
// The same transform, not the same bits: the two libraries order their arithmetic differently.
|
||||
//
|
||||
// Plans are FFTW_ESTIMATE only - MEASURE would time candidate algorithms and could pick a different
|
||||
// one from run to run - and made once per grid size and kept, so the rigid body's hundred-odd
|
||||
// evaluations on one grid plan once. Safe to call from several threads at once.
|
||||
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map);
|
||||
+55
-17
@@ -5,6 +5,9 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
#include "../common/ParallelFor.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -34,7 +37,15 @@ double RFactor(const gemmi::Scaling<float> &scaling) {
|
||||
// pair is held fixed at each grid point and only the overall scale and the symmetry-constrained
|
||||
// anisotropic B are refined, which is the well-conditioned half of the problem and is left to
|
||||
// gemmi's solver rather than reimplemented.
|
||||
ModelScaleReport FitModelScale(gemmi::Scaling<float> &scaling, ModelScaleBox box) {
|
||||
//
|
||||
// Every grid point is its own fit: fit_isotropic_b_approximately() sets k_overall and b_star from the
|
||||
// data and the point's solvent pair alone, so a point does not depend on the one fitted before it, and
|
||||
// the points run in parallel, each chunk on its own copy of `scaling`. The winner is then read off in
|
||||
// grid order with the serial rule (lowest finite R, the first on a tie), so the answer is the serial
|
||||
// loop's bit for bit. The one exception is fit_isotropic_b_approximately() finding five or fewer
|
||||
// reflections to fit on - it then returns without setting anything and a point WOULD start from where
|
||||
// the previous one ended - so there the grid is walked in order, on `scaling` itself, as it always was.
|
||||
ModelScaleReport FitModelScale(gemmi::Scaling<float> &scaling, ModelScaleBox box, size_t nthreads) {
|
||||
ModelScaleReport report;
|
||||
report.n_points = static_cast<int>(scaling.points.size());
|
||||
if (scaling.points.size() < 20)
|
||||
@@ -44,36 +55,63 @@ ModelScaleReport FitModelScale(gemmi::Scaling<float> &scaling, ModelScaleBox box
|
||||
scaling.fix_k_sol = true; // the grid owns the solvent pair; the solver never sees it
|
||||
scaling.fix_b_sol = true;
|
||||
|
||||
// The reflections fit_isotropic_b_approximately() fits on (its own filter).
|
||||
int n_isotropic = 0;
|
||||
for (const auto &p : scaling.points)
|
||||
if (!(p.fobs < 1 || p.fobs < p.sigma))
|
||||
++n_isotropic;
|
||||
const bool independent = n_isotropic > 5;
|
||||
|
||||
double best_r = -1, best_k_sol = 0.35, best_b_sol = 46.0, best_k_overall = 1.0;
|
||||
gemmi::SMat33<double> best_b_star{0, 0, 0, 0, 0, 0};
|
||||
|
||||
auto try_point = [&](double k_sol, double b_sol) {
|
||||
scaling.k_sol = k_sol;
|
||||
scaling.b_sol = b_sol;
|
||||
scaling.fit_isotropic_b_approximately(); // a fresh starting point for this solvent pair
|
||||
scaling.fit_parameters(); // k_overall + anisotropic B only
|
||||
++report.n_grid;
|
||||
const double r = RFactor(scaling);
|
||||
// A diverged fit gives r = NaN; latched as best_r it wins every later r < best_r.
|
||||
if (std::isfinite(r) && (best_r < 0 || r < best_r)) {
|
||||
best_r = r;
|
||||
best_k_sol = k_sol;
|
||||
best_b_sol = b_sol;
|
||||
best_k_overall = scaling.k_overall;
|
||||
best_b_star = scaling.b_star;
|
||||
struct PointFit { double k_sol, b_sol, r = NAN, k_overall = 1.0; gemmi::SMat33<double> b_star{0, 0, 0, 0, 0, 0}; };
|
||||
auto fit_point = [](gemmi::Scaling<float> &s, PointFit &pf) {
|
||||
s.k_sol = pf.k_sol;
|
||||
s.b_sol = pf.b_sol;
|
||||
s.fit_isotropic_b_approximately(); // a fresh starting point for this solvent pair
|
||||
s.fit_parameters(); // k_overall + anisotropic B only
|
||||
pf.r = RFactor(s);
|
||||
pf.k_overall = s.k_overall;
|
||||
pf.b_star = s.b_star;
|
||||
};
|
||||
auto try_points = [&](std::vector<PointFit> &pts) {
|
||||
if (independent)
|
||||
ParallelChunks(static_cast<int>(pts.size()), nthreads, [&](int lo, int hi) {
|
||||
gemmi::Scaling<float> local = scaling;
|
||||
for (int i = lo; i < hi; ++i)
|
||||
fit_point(local, pts[i]);
|
||||
});
|
||||
else
|
||||
for (auto &pf : pts)
|
||||
fit_point(scaling, pf);
|
||||
for (const auto &pf : pts) {
|
||||
++report.n_grid;
|
||||
// A diverged fit gives r = NaN; latched as best_r it wins every later r < best_r.
|
||||
if (std::isfinite(pf.r) && (best_r < 0 || pf.r < best_r)) {
|
||||
best_r = pf.r;
|
||||
best_k_sol = pf.k_sol;
|
||||
best_b_sol = pf.b_sol;
|
||||
best_k_overall = pf.k_overall;
|
||||
best_b_star = pf.b_star;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Coarse pass over the whole box, then one refinement pass around the winner.
|
||||
std::vector<PointFit> coarse;
|
||||
for (double ks = box.k_lo; ks <= box.k_hi + 1e-9; ks += 0.05)
|
||||
for (double bs = box.b_lo; bs <= box.b_hi + 1e-9; bs += 10.0)
|
||||
try_point(ks, bs);
|
||||
coarse.push_back(PointFit{ks, bs});
|
||||
try_points(coarse);
|
||||
const double k0 = best_k_sol, b0 = best_b_sol;
|
||||
const double k_hi2 = std::min(box.k_hi, k0 + 0.05);
|
||||
const double b_hi2 = std::min(box.b_hi, b0 + 10.0);
|
||||
std::vector<PointFit> fine;
|
||||
for (double ks = std::max(box.k_lo, k0 - 0.05); ks <= k_hi2 + 1e-9; ks += 0.025)
|
||||
for (double bs = std::max(box.b_lo, b0 - 10.0); bs <= b_hi2 + 1e-9; bs += 5.0)
|
||||
try_point(ks, bs);
|
||||
fine.push_back(PointFit{ks, bs});
|
||||
try_points(fine);
|
||||
|
||||
scaling.k_sol = best_k_sol;
|
||||
scaling.b_sol = best_b_sol;
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "gemmi/scaling.hpp"
|
||||
|
||||
// Fit the overall scale, the anisotropic B and the flat bulk solvent of a model to observed
|
||||
@@ -31,4 +33,6 @@ struct ModelScaleReport {
|
||||
double r_work_fit = 1.0; // R on the fitted reflections, at the chosen solution
|
||||
};
|
||||
|
||||
ModelScaleReport FitModelScale(gemmi::Scaling<float> &scaling, ModelScaleBox box = {});
|
||||
// nthreads: the grid points are fitted in parallel on up to this many threads; the answer does not
|
||||
// depend on it.
|
||||
ModelScaleReport FitModelScale(gemmi::Scaling<float> &scaling, ModelScaleBox box = {}, size_t nthreads = 1);
|
||||
+41
-18
@@ -9,6 +9,7 @@
|
||||
#include <complex>
|
||||
#include <array>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
@@ -19,7 +20,7 @@
|
||||
#include <gemmi/gz.hpp> // MaybeGzipped
|
||||
#include <gemmi/it92.hpp> // IT92 x-ray form factors
|
||||
#include <gemmi/dencalc.hpp> // DensityCalculator
|
||||
#include <gemmi/fourier.hpp> // transform_map_to_f_phi, get_f_phi_on_grid, transform_f_phi_grid_to_map
|
||||
#include <gemmi/fourier.hpp> // get_f_phi_on_grid, transform_f_phi_grid_to_map (the maps)
|
||||
#include <gemmi/solmask.hpp> // SolventMasker
|
||||
#include <gemmi/scaling.hpp> // Scaling (bulk solvent + anisotropic B)
|
||||
#include <gemmi/ccp4.hpp> // Ccp4 map I/O
|
||||
@@ -30,6 +31,7 @@
|
||||
#include "../common/Logger.h"
|
||||
#include "../common/ParallelFor.h" // ParallelFor
|
||||
#include "../image_analysis/scale_merge/ReindexAmbiguity.h" // ReindexReflections
|
||||
#include "ModelFFT.h"
|
||||
#include "RigidBodyRefine.h"
|
||||
#include "SigmaA.h"
|
||||
|
||||
@@ -283,7 +285,7 @@ double frame_probe_r(const gemmi::Structure &st, const gemmi::SpaceGroup *sg,
|
||||
dc.set_refmac_compatible_blur(st.models[0]);
|
||||
dc.put_model_density_on_grid(st.models[0]);
|
||||
gemmi::AsuData<std::complex<float>> fcalc =
|
||||
gemmi::transform_map_to_f_phi(dc.grid, true)
|
||||
MapToFPhi(dc.grid)
|
||||
.prepare_asu_data(probe_d_min, dc.blur, false, false, false);
|
||||
|
||||
gemmi::GroupOps gops = sg->operations();
|
||||
@@ -476,7 +478,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
dc.set_grid_cell_and_spacegroup(ms.st);
|
||||
dc.set_refmac_compatible_blur(ms.st.models[0]);
|
||||
dc.put_model_density_on_grid(ms.st.models[0]);
|
||||
ms.fcalc = gemmi::transform_map_to_f_phi(dc.grid, true)
|
||||
ms.fcalc = MapToFPhi(dc.grid)
|
||||
.prepare_asu_data(dc.d_min, dc.blur, false, false, false);
|
||||
|
||||
// Refmac radii give a slightly lower R than the Cctbx set on our test cases, at no cost.
|
||||
@@ -486,7 +488,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
mask_grid.spacegroup = dc.grid.spacegroup;
|
||||
mask_grid.set_size_from_spacing(dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
|
||||
masker.put_mask_on_grid(mask_grid, ms.st.models[0]);
|
||||
ms.fmask = gemmi::transform_map_to_f_phi(mask_grid, true).prepare_asu_data(dc.d_min, 0);
|
||||
ms.fmask = MapToFPhi(mask_grid).prepare_asu_data(dc.d_min, 0);
|
||||
};
|
||||
compute_model_factors(mdl);
|
||||
|
||||
@@ -544,7 +546,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
gemmi::Scaling<float> scaling(ucell, sg);
|
||||
scaling.use_solvent = true;
|
||||
scaling.prepare_points(out.fmodel, fobs_work, &ms.fmask);
|
||||
FitModelScale(scaling);
|
||||
FitModelScale(scaling, {}, nthreads);
|
||||
scaling.scale_data(out.fmodel, &ms.fmask); // out.fmodel now holds the scaled, solvent-corrected Fmodel
|
||||
out.k_sol = scaling.k_sol;
|
||||
out.b_sol = scaling.b_sol;
|
||||
@@ -579,14 +581,24 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
// The real model and every null replicate below go through this same lambda, so the two are
|
||||
// comparable - a null that was not placed would be the null of a weaker procedure than the one it
|
||||
// is there to judge. It leaves the model where it ends up, and the caller puts it back.
|
||||
//
|
||||
// `fitted`: fit_model(ms, obs_in) where the caller already has it - the indexing probe fits the
|
||||
// identity labelling first, on the same structure factors and the same reflections - so the same
|
||||
// fit is not made twice in a row.
|
||||
//
|
||||
// A rejected placement puts the atoms back but not ms.fcalc/ms.fmask, which are left at the
|
||||
// rejected placement: nothing reads them afterwards - every later fit_model on this state is
|
||||
// preceded by its own compute_model_factors - and recomputing them was a full Fcalc and mask
|
||||
// for nothing.
|
||||
struct Placement {
|
||||
Fit fit;
|
||||
bool rb_applied = false;
|
||||
double rb_angle_deg = 0, rb_shift_A = 0, r_free_before_rb = 0;
|
||||
};
|
||||
auto place_and_fit = [&](ModelState &ms, const std::vector<MergedReflection> &obs_in) -> Placement {
|
||||
auto place_and_fit = [&](ModelState &ms, const std::vector<MergedReflection> &obs_in,
|
||||
std::optional<Fit> fitted) -> Placement {
|
||||
Placement out;
|
||||
out.fit = fit_model(ms, obs_in);
|
||||
out.fit = fitted ? std::move(*fitted) : fit_model(ms, obs_in);
|
||||
|
||||
// --- rigid-body placement of the model in the data cell ---
|
||||
// Re-fractionalizing the model above puts it in the right box but not in the right place: a
|
||||
@@ -598,7 +610,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
// exactly where it was read.
|
||||
const std::vector<gemmi::Position> before = ModelPositions(ms.st.models[0]);
|
||||
const RigidBodyRefineResult rb =
|
||||
RefineRigidBody(ms.st.models[0], ucell, *sg, out.fit.fobs_work, d_min, logger);
|
||||
RefineRigidBody(ms.st.models[0], ucell, *sg, out.fit.fobs_work, d_min, logger, nthreads);
|
||||
if (!rb.converged) {
|
||||
// RefineRigidBody leaves the model wherever the solver left it, usable answer or not, so
|
||||
// the restore cannot be conditional on the same flag the re-fit is.
|
||||
@@ -619,7 +631,6 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
out.fit = std::move(moved);
|
||||
} else {
|
||||
SetModelPositions(ms.st.models[0], before);
|
||||
compute_model_factors(ms);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
@@ -646,10 +657,12 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
struct IndexingProbe {
|
||||
gemmi::Op op = gemmi::Op::identity(); // the lowest-R-free relabelling
|
||||
double margin = 0; // by how much in R-free it leads the runner-up
|
||||
std::optional<Fit> identity_fit; // fit_model(ms, obs), for place_and_fit to reuse
|
||||
};
|
||||
auto probe_indexing = [&](ModelState &ms) {
|
||||
IndexingProbe out;
|
||||
std::vector<double> r_free{fit_model(ms, obs).r_free}; // identity first, then the twin laws
|
||||
out.identity_fit = fit_model(ms, obs);
|
||||
std::vector<double> r_free{out.identity_fit->r_free}; // identity first, then the twin laws
|
||||
double best_r_free = r_free.front();
|
||||
for (const auto &op : reindex_ops) {
|
||||
const double cand = fit_model(ms, ReindexReflections(obs, op)).r_free;
|
||||
@@ -678,7 +691,13 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
obs_model = &reindexed;
|
||||
}
|
||||
}
|
||||
Placement real = place_and_fit(mdl, *obs_model);
|
||||
// The probe's identity fit is this fit exactly when the identity won: same structure factors
|
||||
// (nothing has moved the model since), same reflections.
|
||||
std::optional<Fit> identity_fit;
|
||||
if (obs_model == &obs)
|
||||
identity_fit = std::move(indexing.identity_fit);
|
||||
indexing.identity_fit.reset();
|
||||
Placement real = place_and_fit(mdl, *obs_model, std::move(identity_fit));
|
||||
|
||||
// --- is there anything for the model to decide? ---
|
||||
// There are only two: the space-group label, where the model asserts the other enantiomorph, and
|
||||
@@ -736,9 +755,13 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
SetModelPositions(rep.st.models[0], as_read);
|
||||
reorient_about_centroid(rep.st.models[0], null_rotation[i]);
|
||||
compute_model_factors(rep);
|
||||
if (!reindex_ops.empty())
|
||||
null_margin[i] = probe_indexing(rep).margin;
|
||||
null_r_work[i] = place_and_fit(rep, obs).fit.r_work;
|
||||
std::optional<Fit> identity_fit;
|
||||
if (!reindex_ops.empty()) {
|
||||
IndexingProbe probe = probe_indexing(rep);
|
||||
null_margin[i] = probe.margin;
|
||||
identity_fit = std::move(probe.identity_fit); // replicates are placed against obs
|
||||
}
|
||||
null_r_work[i] = place_and_fit(rep, obs, std::move(identity_fit)).fit.r_work;
|
||||
});
|
||||
const auto [null_mean, null_sd] = mean_sd(null_r_work);
|
||||
result.fit_tested = true;
|
||||
@@ -793,7 +816,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
// are now the ones the merge produced, so the fit is remade on them.
|
||||
SetModelPositions(st.models[0], as_read);
|
||||
compute_model_factors(mdl);
|
||||
real = place_and_fit(mdl, obs);
|
||||
real = place_and_fit(mdl, obs, std::nullopt);
|
||||
record_fit(real);
|
||||
}
|
||||
}
|
||||
@@ -823,7 +846,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
|
||||
result.r_work_sigma = 0.0;
|
||||
SetModelPositions(st.models[0], as_read);
|
||||
compute_model_factors(mdl);
|
||||
real = place_and_fit(mdl, obs);
|
||||
real = place_and_fit(mdl, obs, std::nullopt);
|
||||
}
|
||||
|
||||
Fit &best = real.fit;
|
||||
@@ -1246,7 +1269,7 @@ std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model
|
||||
dc.set_refmac_compatible_blur(st.models[0]);
|
||||
dc.put_model_density_on_grid(st.models[0]);
|
||||
gemmi::AsuData<std::complex<float>> fcalc =
|
||||
gemmi::transform_map_to_f_phi(dc.grid, true).prepare_asu_data(dc.d_min, dc.blur, false, false, false);
|
||||
MapToFPhi(dc.grid).prepare_asu_data(dc.d_min, dc.blur, false, false, false);
|
||||
|
||||
// Flat bulk solvent at the standard constants. Nothing here is fitted - there are no observations
|
||||
// yet - but without it the few lowest-resolution reflections are the largest and the most wrong,
|
||||
@@ -1260,7 +1283,7 @@ std::vector<MergedReflection> ModelReferenceIntensities(const std::string &model
|
||||
mask_grid.set_size_from_spacing(dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
|
||||
masker.put_mask_on_grid(mask_grid, st.models[0]);
|
||||
gemmi::AsuData<std::complex<float>> fmask =
|
||||
gemmi::transform_map_to_f_phi(mask_grid, true).prepare_asu_data(dc.d_min, 0);
|
||||
MapToFPhi(mask_grid).prepare_asu_data(dc.d_min, 0);
|
||||
|
||||
std::unordered_map<long, std::complex<float>> mask_by_hkl;
|
||||
mask_by_hkl.reserve(fmask.v.size());
|
||||
|
||||
@@ -43,6 +43,7 @@ namespace PathologyCode {
|
||||
// in how exact it is. ANISOTROPY and RESOLUTION_FIT are already shared this way.
|
||||
constexpr const char *LATTICE_TRANSLATION = "LATTICE_TRANSLATION"; // a translation the cell does not declare
|
||||
constexpr const char *HARMONIC_CONTAMINATION = "HARMONIC_CONTAMINATION"; // the beam carries a higher harmonic
|
||||
constexpr const char *GEOMETRY_NOT_CONVERGED = "GEOMETRY_NOT_CONVERGED"; // the geometry walk ran out of rounds
|
||||
}
|
||||
|
||||
// A closed type plus an open description. The type is what machinery switches on; the text is what a
|
||||
|
||||
+139
-59
@@ -204,9 +204,8 @@ namespace {
|
||||
return o.str();
|
||||
}
|
||||
|
||||
// The finest shell whose CC1/2 still reaches the cut's own target. A fitted resolution finer than
|
||||
// this was read off a curve that never behaved like a fall-off, and asserting it states a
|
||||
// resolution the shell table on the same page refutes.
|
||||
// The finest shell whose CC1/2 still reaches the cut's own target: where the table says the signal
|
||||
// is, for the completeness quoted beside the written range.
|
||||
double FinestSupportedShell(const MergeStatistics &ms, double target) {
|
||||
double finest = 0.0;
|
||||
for (const auto &sh : ms.shells)
|
||||
@@ -215,6 +214,25 @@ namespace {
|
||||
return finest; // 0.0 = no shell reaches the target at all
|
||||
}
|
||||
|
||||
// The finest resolution the shell table supports a fitted crossing at. A shell's CC1/2 is an
|
||||
// average over the shell, so the crossing sits INSIDE the first shell whose average falls below the
|
||||
// target, between its d_max and its d_min: a fit anywhere down to that shell's d_min is what the
|
||||
// table shows, and only a fit finer than it is one the table refutes. Comparing against the last
|
||||
// shell ABOVE the target instead withheld the number on 68 of 100 battery datasets whose fit sat
|
||||
// 0.01-0.02 A inside the first sub-target shell. 0.0 when no shell reaches the target; the finest
|
||||
// shell when none falls below it.
|
||||
double CrossingShellLimit(const MergeStatistics &ms, double target) {
|
||||
double finest = 0.0;
|
||||
for (const auto &sh : ms.shells) {
|
||||
if (!std::isfinite(sh.cc_half))
|
||||
continue;
|
||||
if (sh.cc_half < target)
|
||||
return finest == 0.0 ? 0.0 : sh.d_min;
|
||||
finest = sh.d_min;
|
||||
}
|
||||
return finest;
|
||||
}
|
||||
|
||||
// Where CC1/2 climbs back above the cut's target after it has already fallen below it. Below the
|
||||
// target a shell carries no signal the cut would keep, so a shell past that point which
|
||||
// correlates again is noise correlating with noise: the curve is not a fall-off, and no single
|
||||
@@ -367,6 +385,11 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
if (result.pass_count > 1) {
|
||||
Add(s, KeyText("PASS", fmt::format("{} of {}", result.pass_number, result.pass_count)));
|
||||
Add(s, KeyText("PASS_DECISION", result.pass_decision, true));
|
||||
if (result.geometry_not_converged)
|
||||
Warn(doc, PathologyCode::GEOMETRY_NOT_CONVERGED,
|
||||
"the detector geometry post-refinement was still moving when the run stopped walking "
|
||||
"it: every round lowered the held-out residual, so the geometry this result was "
|
||||
"integrated at is the best one reached, not a settled one (see PASS_DECISION)");
|
||||
}
|
||||
|
||||
Add(s, Blank());
|
||||
@@ -528,9 +551,21 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
// operator and candidate tables that justify it.
|
||||
std::string line = fmt::format(" Point group {} from the intensity correlations",
|
||||
search.point_group_hm);
|
||||
// A bound the adopted group EXCEEDS is a gate that fired and was overridden - the point group
|
||||
// was taken from the other search arm - and must not read as the margin of a pass.
|
||||
if (std::isfinite(search.h_ratio))
|
||||
line += fmt::format(";\n twin-law H ratio {:.2f} against a bound of {:.2f}",
|
||||
search.h_ratio, search.h_ratio_bound);
|
||||
line += search.h_ratio > search.h_ratio_bound
|
||||
? fmt::format(";\n twin-law H ratio {:.2f} EXCEEDS its bound of {:.2f} - that gate fired "
|
||||
"and was overridden", search.h_ratio, search.h_ratio_bound)
|
||||
: fmt::format(";\n twin-law H ratio {:.2f} against a bound of {:.2f}",
|
||||
search.h_ratio, search.h_ratio_bound);
|
||||
// The best-operator R gate is the one that applies once two operators are confirmed, i.e.
|
||||
// above order 2 (the first step out of P1 is judged on the noise floor instead).
|
||||
if (search.point_group_order > 2 && std::isfinite(search.r_over_best)
|
||||
&& search.r_over_best_bound > 0.0 && search.r_over_best > search.r_over_best_bound)
|
||||
line += fmt::format(";\n added-operator R {:.2f}x the best operator EXCEEDS its bound of {:.2f}x - "
|
||||
"that gate fired and was overridden", search.r_over_best,
|
||||
search.r_over_best_bound);
|
||||
line += ".\n The space group follows from the systematic absences.";
|
||||
if (!alts.empty())
|
||||
line += fmt::format("\n These data cannot separate it from {} - both are consistent with"
|
||||
@@ -541,6 +576,11 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
" cannot name. Molecular replacement settles it: one of the two refines and the\n"
|
||||
" other does not - the normal outcome for a chiral group, not a failure of the\n"
|
||||
" run."));
|
||||
if (!result.space_group_arm_override.empty()) {
|
||||
Add(s, KeyText("SPACE_GROUP_ARM_OVERRIDE", result.space_group_arm_override));
|
||||
Add(s, Prose(fmt::format(" The two search merges disagreed and the higher point group was taken:\n {}.",
|
||||
result.space_group_arm_override)));
|
||||
}
|
||||
if (refused)
|
||||
Add(s, Prose(fmt::format(
|
||||
" A promotion to {} was refused: {}.\n"
|
||||
@@ -602,6 +642,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
// than the finest shell that still reaches the target asserts a resolution the shell
|
||||
// table on this very page refutes.
|
||||
const double finest_supported = FinestSupportedShell(ms, cc_target);
|
||||
const double crossing_limit = CrossingShellLimit(ms, cc_target);
|
||||
const double climbs_back = CCHalfClimbsBackAboveTarget(ms, cc_target);
|
||||
std::string fit_suppressed;
|
||||
// Only where there are shells to judge the fit against; with no table there is no
|
||||
@@ -613,15 +654,15 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
" at {:.2f} A, so this curve is not a fall-off and no one resolution can be"
|
||||
" read off it", cc_target, climbs_back);
|
||||
else if (result.resolution_fit_A) {
|
||||
if (finest_supported == 0.0)
|
||||
if (crossing_limit == 0.0)
|
||||
fit_suppressed = fmt::format(
|
||||
"CC1/2 never reaches the cut's target of {:.2f} in any shell, so there is"
|
||||
" no fall-off to fit", cc_target);
|
||||
else if (*result.resolution_fit_A < finest_supported - 1e-6)
|
||||
else if (*result.resolution_fit_A < crossing_limit - 1e-6)
|
||||
fit_suppressed = fmt::format(
|
||||
"the fit reads {:.2f} A, finer than the finest shell whose CC1/2 still"
|
||||
" reaches {:.2f} ({:.2f} A), so the curve it was read off is not a"
|
||||
" fall-off", *result.resolution_fit_A, cc_target, finest_supported);
|
||||
"the fit reads {:.2f} A, finer than the shell where CC1/2 falls below"
|
||||
" {:.2f} ({:.2f} A), so the curve it was read off is not a fall-off",
|
||||
*result.resolution_fit_A, cc_target, crossing_limit);
|
||||
else
|
||||
fitted_resolution = *result.resolution_fit_A;
|
||||
}
|
||||
@@ -659,8 +700,12 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
// CORRECT.LP. Kept in the default report because beamline monitoring scripts read them.
|
||||
Add(s, KeyReal("ERROR_MODEL_A", result.error_model_a, "{:.4f}"));
|
||||
Add(s, KeyReal("ERROR_MODEL_B", result.error_model_b, "{:.4e}"));
|
||||
Add(s, KeyReal("ISA", result.error_model_isa, "{:.2f}"));
|
||||
if (result.error_model_isa_asymptotic > 0.0)
|
||||
// A b that is zero within its error gives a 1/b that is the noise in b, not an I/sigma.
|
||||
if (result.error_model_isa_resolved)
|
||||
Add(s, KeyReal("ISA", result.error_model_isa, "{:.2f}"));
|
||||
else
|
||||
Add(s, KeyText("ISA", "undetermined"));
|
||||
if (result.error_model_isa_resolved && result.error_model_isa_asymptotic > 0.0)
|
||||
Add(s, KeyReal("ISA_ASYMPTOTIC", result.error_model_isa_asymptotic, "{:.2f}", true));
|
||||
// Only when TRUE: FALSE is the ordinary case and says nothing.
|
||||
if (result.has_reference)
|
||||
@@ -1054,59 +1099,94 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
: "UNAFFECTED",
|
||||
{"UNAFFECTED", "REPAIRED", "UNREADABLE"}));
|
||||
Add(s, KeyReal("SECOND_MOMENT_I", tw.second_moment, "{:.4f}"));
|
||||
Add(s, KeyReal("ESTIMATED_TWIN_FRACTION", tw.estimated_twin_fraction, "{:.3f}"));
|
||||
// Which statistic the fraction came from. The L-test in the same block can imply a much
|
||||
// larger fraction, and printing the two side by side with no label made the headline key
|
||||
// look like a reconciliation of both when it is derived from one of them.
|
||||
Add(s, KeyEnum("ESTIMATED_TWIN_FRACTION_FROM", "SECOND_MOMENT",
|
||||
{"SECOND_MOMENT", "L_TEST"}));
|
||||
// One verdict, in the words every other place uses (TwinningVerdictLine): INDICATED is a twin
|
||||
// in a class that admits one; SYMMETRY_SUSPECT is the same low reading in a holohedral class,
|
||||
// where it says an adopted operator averages unequal intensities.
|
||||
Add(s, KeyEnum("TWINNING_VERDICT",
|
||||
tw.adopted_operators_suspect ? "SYMMETRY_SUSPECT"
|
||||
: tw.twinning_suspected ? "INDICATED"
|
||||
: "NO_INDICATION",
|
||||
{"NO_INDICATION", "INDICATED", "SYMMETRY_SUSPECT"}));
|
||||
// The fraction comes from the statistic that carries the verdict, and the key beside it says
|
||||
// which. None is quoted where it cannot be measured: merged under a false operator, or from a
|
||||
// second moment a pseudo-translation inflates.
|
||||
if (tw.twin_fraction_source != TwinFractionSource::None)
|
||||
Add(s, KeyReal("ESTIMATED_TWIN_FRACTION", tw.estimated_twin_fraction, "{:.3f}"));
|
||||
Add(s, KeyEnum("ESTIMATED_TWIN_FRACTION_FROM",
|
||||
tw.twin_fraction_source == TwinFractionSource::LTest ? "L_TEST"
|
||||
: tw.twin_fraction_source == TwinFractionSource::SecondMoment ? "SECOND_MOMENT"
|
||||
: "NONE",
|
||||
{"SECOND_MOMENT", "L_TEST", "NONE"}));
|
||||
Add(s, KeyReal("L_TEST_MEAN_L_SQUARED", tw.mean_l_squared, "{:.4f}", true));
|
||||
if (result.pre_promotion_twinning) {
|
||||
Add(s, KeyReal("L_TEST_MEAN_ABS_L_BEFORE_SEARCH",
|
||||
result.pre_promotion_twinning->mean_abs_l, "{:.4f}", true));
|
||||
Add(s, KeyReal("SECOND_MOMENT_I_BEFORE_SEARCH",
|
||||
result.pre_promotion_twinning->second_moment, "{:.4f}", true));
|
||||
const auto &pre = *result.pre_promotion_twinning;
|
||||
Add(s, KeyReal("L_TEST_MEAN_ABS_L_BEFORE_SEARCH", pre.mean_abs_l, "{:.4f}"));
|
||||
Add(s, KeyReal("SECOND_MOMENT_I_BEFORE_SEARCH", pre.second_moment, "{:.4f}"));
|
||||
Add(s, KeyEnum("L_TEST_BEFORE_SEARCH_VS_TNCS",
|
||||
pre.l_test_contaminated_by_tncs ? "UNREADABLE"
|
||||
: pre.l_test_tncs_step_restricted ? "REPAIRED"
|
||||
: result.pre_promotion_tncs_measured ? "UNAFFECTED"
|
||||
: "NOT_MEASURED",
|
||||
{"UNAFFECTED", "REPAIRED", "UNREADABLE", "NOT_MEASURED"}));
|
||||
}
|
||||
Add(s, Blank());
|
||||
// One calm sentence, in the same terms in both branches. The old prose opened "Cannot rule
|
||||
// out twinning from these numbers" on runs whose own key said FALSE - a property of the
|
||||
// symmetry stated as if it were a finding about these data.
|
||||
if (tw.twinning_suspected)
|
||||
Add(s, Prose(" Twinning: " + TwinningVerdictLine(tw) + "."));
|
||||
if (tw.adopted_operators_suspect)
|
||||
Add(s, Prose(fmt::format(
|
||||
" Twinning: INDICATED. <|L|> is {:.3f}, where an untwinned crystal reads about 0.5 and\n"
|
||||
" a perfect twin about 0.375. The estimated fraction {:.2f} is derived from the second\n"
|
||||
" moment alone and the L-test may imply a larger one. Refinement against these data\n"
|
||||
" needs a twin law.",
|
||||
tw.mean_abs_l, tw.estimated_twin_fraction)));
|
||||
" The Laue class is holohedral, so no twin law exists within it - but merging under a\n"
|
||||
" false operator gives exactly this distribution, whatever the twin fraction. {} Either\n"
|
||||
" the crystal has lower symmetry or it is a twin whose law the point group absorbed;\n"
|
||||
" refinement belongs in a subgroup.",
|
||||
tw.laue_class_was_chosen_by_promotion
|
||||
? "The class was chosen by the space-group search, so that promotion is suspect."
|
||||
: "The space group in use is higher than these intensities support.")));
|
||||
else if (tw.twinning_suspected)
|
||||
Add(s, Prose(" Refinement against these data needs a twin law."));
|
||||
else if (tw.l_test_contaminated_by_tncs)
|
||||
// <|L|> is the number this sentence would otherwise rest on, and on this crystal it is
|
||||
// biased by the pseudo-translation above. Saying "no indication (<|L|> 0.529)" here
|
||||
// would be the same silent veto in a different place.
|
||||
Add(s, Prose(fmt::format(
|
||||
" Twinning: no indication from the second moment ({:.3f}, against 2.00 for an untwinned\n"
|
||||
" crystal and 1.50 for a perfect twin). The L-test, normally the stronger of the two, could\n"
|
||||
" not be read here: this crystal's translational pseudo-symmetry biases <|L|> and no partner\n"
|
||||
" reflection available to the test escapes that bias. A weaker statement than usual.",
|
||||
tw.second_moment)));
|
||||
else
|
||||
Add(s, Prose(fmt::format(
|
||||
" Twinning: no indication (<|L|> {:.3f}, against about 0.5 for an untwinned crystal\n"
|
||||
" and 0.375 for a perfect twin).", tw.mean_abs_l)));
|
||||
// biased by the pseudo-translation above.
|
||||
Add(s, Prose(" The L-test, normally the stronger of the two statistics, could not be read here:\n"
|
||||
" this crystal's translational pseudo-symmetry biases <|L|> and no partner reflection\n"
|
||||
" available to the test escapes that bias. A weaker statement than usual."));
|
||||
Add(s, Prose("\n" + TwinningAnalysisToText(tw), true));
|
||||
if (result.pre_promotion_twinning)
|
||||
if (result.pre_promotion_twinning) {
|
||||
const auto &pre = *result.pre_promotion_twinning;
|
||||
Add(s, Prose(fmt::format(
|
||||
" Before the space-group search (P1 merge{}): {}.",
|
||||
result.pre_promotion_tncs_measured ? "" : ", not corrected for a pseudo-translation",
|
||||
TwinningVerdictLine(pre))));
|
||||
Add(s, Prose(" The _BEFORE_SEARCH pair was measured on the merge the space-group search was given,\n"
|
||||
" before any point group was adopted. That ordering is the whole point: the statistics\n"
|
||||
" above are computed in the Laue class this run ADOPTED, so if the search promoted the\n"
|
||||
" point group they can only report that no twin law exists inside the class it chose -\n"
|
||||
" and a twin is precisely what would have caused that promotion. Where the two pairs\n"
|
||||
" disagree, the _BEFORE_SEARCH pair is the one that answers whether the crystal is\n"
|
||||
" twinned.", true));
|
||||
if (tw.twinning_suspected)
|
||||
" before any point group was adopted. The statistics above are computed in the Laue\n"
|
||||
" class this run ADOPTED; where the search promoted the point group, a twin law it\n"
|
||||
" absorbed can only show there as an operator averaging unequal intensities. These\n"
|
||||
" numbers are not circular in that way.", true));
|
||||
}
|
||||
if (result.twin_immune_zones && !result.twin_immune_zones->zones.empty()) {
|
||||
// Report-only evidence, one row per class of 2-folds of the adopted group. Not a decision:
|
||||
// the space group above was chosen without it.
|
||||
const auto &tz = *result.twin_immune_zones;
|
||||
for (size_t i = 0; i < tz.zones.size(); ++i) {
|
||||
const auto &z = tz.zones[i];
|
||||
Add(s, KeyText(fmt::format("TWIN_ZONE_{}", i + 1).c_str(),
|
||||
fmt::format("{}; n={} <|E2-1|>={:.3f}+-{:.3f} evidence={:+.1f} nats", z.operators,
|
||||
z.n, z.mean_abs_e2_minus_1, z.standard_error, z.evidence_nats)));
|
||||
}
|
||||
Add(s, KeyText("TWIN_ZONE_CONTROL",
|
||||
fmt::format("n={} <|E2-1|>={:.3f}+-{:.3f}", tz.control.n,
|
||||
tz.control.mean_abs_e2_minus_1, tz.control.standard_error)));
|
||||
Add(s, Prose("\n" + TwinImmuneZonesToText(tz)));
|
||||
}
|
||||
if (tw.adopted_operators_suspect)
|
||||
Warn(doc, PathologyCode::TWINNING, fmt::format(
|
||||
"Twinning is indicated (<|L|> = {:.3f}, <I^2>/<I>^2 = {:.3f}, estimated twin "
|
||||
"fraction {:.2f} from the second moment) - refinement against the merged data "
|
||||
"needs a twin law",
|
||||
tw.mean_abs_l, tw.second_moment, tw.estimated_twin_fraction));
|
||||
"An adopted symmetry operator relates unequal intensities (<|L|> = {:.3f} in a holohedral "
|
||||
"Laue class, where a genuine operator reads about 0.5 and a false one 0.375): the "
|
||||
"space group may be too high, or the crystal is a twin whose law the point group "
|
||||
"absorbed - refine in a subgroup",
|
||||
tw.mean_abs_l));
|
||||
else if (tw.twinning_suspected)
|
||||
Warn(doc, PathologyCode::TWINNING, fmt::format(
|
||||
"Twinning is indicated ({}) - refinement against the merged data needs a twin law",
|
||||
TwinningVerdictLine(tw)));
|
||||
}
|
||||
|
||||
// ---- radiation damage
|
||||
@@ -1692,7 +1772,9 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
sig += fmt::format(" R_meas {:.1f} %", 100.0 * o.r_meas);
|
||||
if (std::isfinite(o.cc_half))
|
||||
sig += fmt::format(" CC1/2 {:.3f}", o.cc_half);
|
||||
if (result.error_model_isa > 0.0)
|
||||
if (!result.error_model_isa_resolved)
|
||||
sig += " ISa undetermined";
|
||||
else if (result.error_model_isa > 0.0)
|
||||
sig += fmt::format(" ISa {:.1f}", result.error_model_isa);
|
||||
row("Signal", sig);
|
||||
if (std::isfinite(o.cc_anom))
|
||||
@@ -1725,9 +1807,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
result.tncs.peak_percent, result.tncs.vector_length_A)
|
||||
: std::string("no indication"));
|
||||
if (result.twinning.l_test_pairs > 0)
|
||||
row("Twinning", result.twinning.twinning_suspected
|
||||
? fmt::format("INDICATED (<|L|> {:.3f})", result.twinning.mean_abs_l)
|
||||
: std::string("no indication"));
|
||||
row("Twinning", TwinningVerdictLine(result.twinning));
|
||||
const double db = result.merge_statistics.radiation_damage_delta_b;
|
||||
if (std::isfinite(db))
|
||||
row("Radiation damage", fmt::format("relative B {:+.2f} A^2 over the sweep", db));
|
||||
|
||||
+54
-15
@@ -4,6 +4,7 @@
|
||||
#include "RigidBodyRefine.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <complex>
|
||||
@@ -13,14 +14,15 @@
|
||||
#include <ceres/rotation.h>
|
||||
|
||||
#include "gemmi/dencalc.hpp" // DensityCalculator
|
||||
#include "gemmi/fourier.hpp" // transform_map_to_f_phi
|
||||
#include "gemmi/it92.hpp" // IT92 x-ray form factors
|
||||
#include "gemmi/scaling.hpp" // Scaling (bulk solvent + anisotropic B)
|
||||
#include "gemmi/solmask.hpp" // SolventMasker
|
||||
|
||||
#include "ModelFFT.h" // MapToFPhi
|
||||
#include "ModelScaling.h" // FitModelScale
|
||||
#include "../common/JFJochMath.h" // PI
|
||||
#include "../common/Logger.h"
|
||||
#include "../common/ParallelFor.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -72,8 +74,8 @@ struct Placement {
|
||||
class Evaluator {
|
||||
public:
|
||||
Evaluator(gemmi::Model &model, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg,
|
||||
const std::vector<gemmi::Position> &base, const Placement &placement)
|
||||
: model_(model), cell_(cell), sg_(sg), base_(base), placement_(placement) {}
|
||||
const std::vector<gemmi::Position> &base, const Placement &placement, size_t nthreads)
|
||||
: model_(model), cell_(cell), sg_(sg), base_(base), placement_(placement), nthreads_(nthreads) {}
|
||||
|
||||
// The zone's observations, and the scale the residuals are expressed in.
|
||||
void SetZone(const gemmi::AsuData<gemmi::ValueSigma<float>> &fobs, double d_min) {
|
||||
@@ -86,6 +88,14 @@ public:
|
||||
solvent_fitted_ = false;
|
||||
}
|
||||
|
||||
// Hold the zone's bulk solvent at a pair another evaluator already fitted, as if this one had
|
||||
// fitted it itself (the Jacobian's column evaluators take the central evaluation's).
|
||||
void UseSolvent(double k, double b) {
|
||||
k_sol = k;
|
||||
b_sol = b;
|
||||
solvent_fitted_ = true;
|
||||
}
|
||||
|
||||
size_t NumObservations() const { return fobs_.v.size(); }
|
||||
double JacobianStep() const { return JACOBIAN_STEP_FRACTION * d_min_; }
|
||||
int evaluations = 0;
|
||||
@@ -104,7 +114,7 @@ public:
|
||||
dc.set_refmac_compatible_blur(model_);
|
||||
dc.put_model_density_on_grid(model_);
|
||||
gemmi::AsuData<std::complex<float>> fcalc =
|
||||
gemmi::transform_map_to_f_phi(dc.grid, true).prepare_asu_data(dc.d_min, dc.blur, false, false, false);
|
||||
MapToFPhi(dc.grid).prepare_asu_data(dc.d_min, dc.blur, false, false, false);
|
||||
|
||||
gemmi::SolventMasker masker(gemmi::AtomicRadiiSet::Refmac);
|
||||
gemmi::Grid<float> mask_grid;
|
||||
@@ -113,7 +123,7 @@ public:
|
||||
mask_grid.set_size_from_spacing(dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
|
||||
masker.put_mask_on_grid(mask_grid, model_);
|
||||
gemmi::AsuData<std::complex<float>> fmask =
|
||||
gemmi::transform_map_to_f_phi(mask_grid, true).prepare_asu_data(dc.d_min, 0);
|
||||
MapToFPhi(mask_grid).prepare_asu_data(dc.d_min, 0);
|
||||
if (fmask.size() != fcalc.size())
|
||||
return false;
|
||||
|
||||
@@ -135,7 +145,7 @@ public:
|
||||
if (scaling.points.empty())
|
||||
return false;
|
||||
if (!solvent_fitted_) {
|
||||
FitModelScale(scaling);
|
||||
FitModelScale(scaling, {}, nthreads_);
|
||||
k_sol = scaling.k_sol;
|
||||
b_sol = scaling.b_sol;
|
||||
solvent_fitted_ = true;
|
||||
@@ -179,6 +189,7 @@ private:
|
||||
double d_min_ = 0;
|
||||
double f_mean_ = 1;
|
||||
bool solvent_fitted_ = false;
|
||||
size_t nthreads_ = 1;
|
||||
};
|
||||
|
||||
// Ceres' own numeric differentiation steps by |x| * relative_step_size, which is zero at the start of
|
||||
@@ -186,9 +197,17 @@ private:
|
||||
// forward differences at a step chosen in the parameters' units. Analytic dF/dp would need
|
||||
// derivatives GEMMI's structure-factor path does not have, and at six parameters it is not worth it:
|
||||
// a Jacobian costs seven evaluations, and the evaluations at 6-3.5 A are cheap.
|
||||
//
|
||||
// The six shifted evaluations are independent of each other, so they run in parallel, column j on
|
||||
// `columns[j]`: an evaluator over its own copy of the model (an evaluation moves every atom to the
|
||||
// placement it is asked about, so two cannot share one), set to the same zone and handed the solvent
|
||||
// pair the central evaluation fitted - which is the pair the serial loop's shifted evaluations used,
|
||||
// since the zone's solvent is fitted by the first evaluation of the zone and every Evaluate starts
|
||||
// with the central one. Each column's arithmetic is the serial one, so the Jacobian is too.
|
||||
class RigidBodyCost : public ceres::CostFunction {
|
||||
public:
|
||||
explicit RigidBodyCost(Evaluator &ev) : ev_(ev) {
|
||||
RigidBodyCost(Evaluator &ev, std::vector<Evaluator> &columns, size_t nthreads)
|
||||
: ev_(ev), columns_(columns), nthreads_(nthreads) {
|
||||
set_num_residuals(static_cast<int>(ev.NumObservations()));
|
||||
mutable_parameter_block_sizes()->push_back(6);
|
||||
}
|
||||
@@ -198,16 +217,24 @@ public:
|
||||
if (!ev_.Residuals(parameters[0], residuals))
|
||||
return false;
|
||||
if (jacobians != nullptr && jacobians[0] != nullptr) {
|
||||
std::vector<double> shifted(n);
|
||||
for (int j = 0; j < 6; j++) {
|
||||
const double step = ev_.JacobianStep();
|
||||
std::vector<std::vector<double>> shifted(6, std::vector<double>(n));
|
||||
std::array<bool, 6> ok{};
|
||||
ParallelFor(6, nthreads_, [&](int j) {
|
||||
double q[6];
|
||||
std::copy(parameters[0], parameters[0] + 6, q);
|
||||
const double step = ev_.JacobianStep();
|
||||
q[j] += step;
|
||||
if (!ev_.Residuals(q, shifted.data()))
|
||||
columns_[j].UseSolvent(ev_.k_sol, ev_.b_sol);
|
||||
ok[j] = columns_[j].Residuals(q, shifted[j].data());
|
||||
});
|
||||
// Counted as the serial loop counted them: it stopped at the first column that failed.
|
||||
for (int j = 0; j < 6; j++) {
|
||||
++ev_.evaluations;
|
||||
ev_.unmatched = columns_[j].unmatched;
|
||||
if (!ok[j])
|
||||
return false;
|
||||
for (int i = 0; i < n; i++)
|
||||
jacobians[0][i * 6 + j] = (shifted[i] - residuals[i]) / step;
|
||||
jacobians[0][i * 6 + j] = (shifted[j][i] - residuals[i]) / step;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
@@ -215,6 +242,8 @@ public:
|
||||
|
||||
private:
|
||||
Evaluator &ev_;
|
||||
std::vector<Evaluator> &columns_;
|
||||
size_t nthreads_;
|
||||
};
|
||||
|
||||
// The directions in which this space group's origin is free. Translating the whole cell content along
|
||||
@@ -274,7 +303,8 @@ RigidBodyRefineResult RefineRigidBody(gemmi::Model &model,
|
||||
const gemmi::SpaceGroup &sg,
|
||||
const gemmi::AsuData<gemmi::ValueSigma<float>> &fobs,
|
||||
double d_min,
|
||||
Logger &logger) {
|
||||
Logger &logger,
|
||||
size_t nthreads) {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
RigidBodyRefineResult result;
|
||||
|
||||
@@ -301,7 +331,14 @@ RigidBodyRefineResult RefineRigidBody(gemmi::Model &model,
|
||||
ladder.push_back(d_min);
|
||||
|
||||
const gemmi::Mat33 gauge = GaugeProjector(sg, cell);
|
||||
Evaluator ev(model, cell, sg, base, placement);
|
||||
Evaluator ev(model, cell, sg, base, placement, nthreads);
|
||||
// The Jacobian's six column evaluators, each over its own copy of the model (see RigidBodyCost).
|
||||
// The copies only ever hold probe placements; `model` is left at the answer below.
|
||||
std::vector<gemmi::Model> column_models(6, model);
|
||||
std::vector<Evaluator> columns;
|
||||
columns.reserve(6);
|
||||
for (int j = 0; j < 6; j++)
|
||||
columns.emplace_back(column_models[j], cell, sg, base, placement, 1);
|
||||
double q[6] = {0, 0, 0, 0, 0, 0};
|
||||
bool any_zone_solved = false;
|
||||
for (double zone : ladder) {
|
||||
@@ -315,9 +352,11 @@ RigidBodyRefineResult RefineRigidBody(gemmi::Model &model,
|
||||
continue;
|
||||
result.zones.push_back(zone); // the ladder WALKED, which a thin zone drops out of
|
||||
ev.SetZone(zone_obs, zone);
|
||||
for (Evaluator &c : columns)
|
||||
c.SetZone(zone_obs, zone);
|
||||
|
||||
ceres::Problem problem;
|
||||
problem.AddResidualBlock(new RigidBodyCost(ev), nullptr, q);
|
||||
problem.AddResidualBlock(new RigidBodyCost(ev, columns, nthreads), nullptr, q);
|
||||
ceres::Solver::Options options;
|
||||
options.linear_solver_type = ceres::DENSE_QR;
|
||||
options.max_num_iterations = 15;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <vector>
|
||||
|
||||
#include "gemmi/asudata.hpp"
|
||||
@@ -41,4 +42,5 @@ RigidBodyRefineResult RefineRigidBody(gemmi::Model &model,
|
||||
const gemmi::SpaceGroup &sg,
|
||||
const gemmi::AsuData<gemmi::ValueSigma<float>> &fobs,
|
||||
double d_min,
|
||||
Logger &logger);
|
||||
Logger &logger,
|
||||
size_t nthreads = 1);
|
||||
+336
-127
@@ -497,6 +497,23 @@ Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
|
||||
|
||||
LogHeaderBeamCenter(experiment_, pixel_mask_);
|
||||
|
||||
// Every decision on a rotation run - per-frame scaling, the correction surfaces, the outlier test,
|
||||
// the error model, the CC1/2 half-sets and the resolution cut, the frame ledger, the space-group
|
||||
// search - is taken with Friedel mates merged, -A or not. Splitting them halves the multiplicity
|
||||
// every one of those gates was calibrated at, which is a noisier instrument rather than a more
|
||||
// careful one: it flipped symmetry, resolution and frame-rejection calls against the bare run on
|
||||
// several sets, none of them through the anomalous signal. It is also what XDS, AIMLESS, POINTLESS
|
||||
// and DIALS decide on. The anomalous signal is not lost: the merge always keeps the Bijvoet split,
|
||||
// and I(+)/I(-), SigAno and CCanom come from it in either mode. Stills keep -A as it was - their
|
||||
// merge has no separate split.
|
||||
if (experiment_.IsRotationIndexing() && !experiment_.GetScalingSettings().GetMergeFriedel()) {
|
||||
ScalingSettings s = experiment_.GetScalingSettings();
|
||||
experiment_.ImportScalingSettings(s.MergeFriedel(true));
|
||||
Logger("Rugnux").Info("-A on rotation data: every processing decision is made with Friedel mates "
|
||||
"merged; I(+)/I(-), SigAno and CCanom are reported from the Bijvoet split "
|
||||
"the merge keeps, exactly as without -A");
|
||||
}
|
||||
|
||||
// A manually asserted stage calibration applies to everything, before anything reads an angle.
|
||||
if (config_.rotation_scale.has_value())
|
||||
if (const auto g = experiment_.GetGoniometer()) {
|
||||
@@ -1987,50 +2004,124 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
|
||||
// What starts the walk is a move of more than one step: that is the fit saying the geometry is
|
||||
// somewhere else entirely rather than a fraction of a percent away, and it leaves every run
|
||||
// whose fit settles beside its header at the two passes it always had. Once the run IS walking
|
||||
// it keeps walking while the fit still commits, because the steps do not get smaller in
|
||||
// proportion to what is left: measured on a header 3.9 % long, the first round took a third of
|
||||
// the error and each of the next took a third of the rest. The walk is bounded rather than
|
||||
// stopped at a size, because there is no size that separates the last step of a real walk from
|
||||
// the first step of a runaway - and it does not run away: started BELOW the optimum the same fit
|
||||
// turns back up, so it settles around the answer instead of past it.
|
||||
constexpr int MAX_GEOMETRY_ROUNDS = 4;
|
||||
std::optional<std::array<float, 5>> adopted_geometry;
|
||||
int geometry_rounds = 0;
|
||||
while (!cancelled_ && geometry_rounds < MAX_GEOMETRY_ROUNDS && prepass_detector_geometry_
|
||||
&& pass2.post_refine && pass2.post_refine->detector_refined
|
||||
&& (pass2.post_refine->large_move || geometry_rounds > 0)) {
|
||||
adopted_geometry = *prepass_detector_geometry_;
|
||||
const auto &g = *adopted_geometry;
|
||||
logger.Info("Two-pass: the post-refinement at the adopted geometry still moves it "
|
||||
"(distance {:.3f} -> {:.3f} mm) - re-integrating and re-indexing at what it "
|
||||
"asks for (round {} of at most {})", pass2.post_refine->distance_before_mm,
|
||||
pass2.post_refine->distance_after_mm, geometry_rounds + 1,
|
||||
MAX_GEOMETRY_ROUNDS);
|
||||
// it keeps walking because the steps do not get smaller in proportion to what is left: measured
|
||||
// on a header 3.9 % long, the first round took a third of the error and each of the next took a
|
||||
// third of the rest.
|
||||
//
|
||||
// But a round is kept only for what it REALISES, not for what the fit predicts: the held-out
|
||||
// residual of the pass re-integrated and re-indexed at the new geometry has to fall below the
|
||||
// best round's by more than the cross-validation noise of the two (HeldOutResidualFell). With
|
||||
// the walk allowed twelve rounds, every round's fit predicted a gain the re-integration did not
|
||||
// show - the realised residual stayed flat, or passed a minimum and rose again, while the
|
||||
// distance crept on along the distance/cell-scale degeneracy - so "the fit still commits" never
|
||||
// ended a walk, and the cap decided where on the drift the run stopped. A round that does not
|
||||
// pay ends the walk and the run goes back to the best round it reached. So does a fit whose
|
||||
// committed cell the re-indexing pushes back the same way twice running (ReindexPushesCellBack):
|
||||
// the fit keeps asking for a cell the indexer at the fit's own geometry does not return, and
|
||||
// nothing ratifies it. A walk still paying when the rounds run out keeps its best round - the
|
||||
// last - and says it did not converge; the quality guard below judges it against the header
|
||||
// pass as it judges every refined pass.
|
||||
constexpr int MAX_GEOMETRY_ROUNDS = 8;
|
||||
const auto current_geometry = [&] {
|
||||
return std::array<float, 5>{experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(),
|
||||
experiment_.GetDetectorDistance_mm(),
|
||||
experiment_.GetPoniRot1_rad(), experiment_.GetPoniRot2_rad()};
|
||||
};
|
||||
const auto set_geometry = [&](const std::array<float, 5> &g) {
|
||||
experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2])
|
||||
.PoniRot1_rad(g[3]).PoniRot2_rad(g[4]);
|
||||
};
|
||||
int geometry_rounds = 0; // passes run at a geometry the walk moved to
|
||||
int walk_passes = 0; // every pass the walk ran, the return to its best round included
|
||||
int best_round = 0;
|
||||
std::array<float, 5> best_geometry = current_geometry();
|
||||
PostRefineResult best_fit; // the fit measured in the best round's pass: its realised residual
|
||||
int last_pushback = 0;
|
||||
bool geometry_not_converged = false;
|
||||
std::string walk_stop; // why the walk ended, where anything but the fit settling ended it
|
||||
std::string walk_rounds; // distance / realised residual per round, for the pass decision
|
||||
if (pass2.post_refine) {
|
||||
best_fit = *pass2.post_refine;
|
||||
walk_rounds = fmt::format("{:.3f} mm / {:.3e}", best_geometry[2], best_fit.held_out_before);
|
||||
}
|
||||
while (!cancelled_ && prepass_detector_geometry_
|
||||
&& pass2.post_refine && pass2.post_refine->detector_refined
|
||||
&& (pass2.post_refine->large_move || geometry_rounds > 0)) {
|
||||
if (geometry_rounds == MAX_GEOMETRY_ROUNDS) {
|
||||
geometry_not_converged = true;
|
||||
walk_stop = fmt::format("the fit still moved the geometry after {} rounds, every one of "
|
||||
"which lowered the realised residual", MAX_GEOMETRY_ROUNDS);
|
||||
logger.Warning("Two-pass: {} - the walk did not converge; the run keeps its last round "
|
||||
"and the quality guard judges it against the header pass", walk_stop);
|
||||
break;
|
||||
}
|
||||
const PostRefineResult fit = *pass2.post_refine;
|
||||
const std::array<float, 5> g = *prepass_detector_geometry_;
|
||||
logger.Info("Two-pass: the post-refinement at the adopted geometry still moves it "
|
||||
"(distance {:.3f} -> {:.3f} mm) - re-integrating and re-indexing at what it "
|
||||
"asks for (round {} of at most {})", fit.distance_before_mm,
|
||||
fit.distance_after_mm, geometry_rounds + 1, MAX_GEOMETRY_ROUNDS);
|
||||
set_geometry(g);
|
||||
pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
||||
++geometry_rounds;
|
||||
++walk_passes;
|
||||
const double realised = pass2.post_refine ? pass2.post_refine->held_out_before : NAN;
|
||||
walk_rounds += fmt::format(", {:.3f} mm / {:.3e}", g[2], realised);
|
||||
if (!pass2.post_refine || !HeldOutResidualFell(best_fit, *pass2.post_refine)) {
|
||||
walk_stop = fmt::format(
|
||||
"round {} did not lower the realised held-out residual by more than its noise "
|
||||
"({:.3e} against {:.3e}, standard errors {:.1e} and {:.1e})", geometry_rounds,
|
||||
realised, best_fit.held_out_before,
|
||||
pass2.post_refine ? pass2.post_refine->held_out_before_se : NAN,
|
||||
best_fit.held_out_before_se);
|
||||
logger.Info("Two-pass: {}", walk_stop);
|
||||
break;
|
||||
}
|
||||
best_round = geometry_rounds;
|
||||
best_geometry = g;
|
||||
best_fit = *pass2.post_refine;
|
||||
const int pushback = ReindexPushesCellBack(fit, *pass2.post_refine);
|
||||
if (pushback != 0 && pushback == last_pushback) {
|
||||
walk_stop = fmt::format(
|
||||
"re-indexing pushed the cell length {} back against the fit for the second "
|
||||
"round running, so nothing ratifies the cell the fit asks for",
|
||||
"abc"[std::abs(pushback) - 1]);
|
||||
logger.Info("Two-pass: {} - the walk stops at round {}", walk_stop, geometry_rounds);
|
||||
break;
|
||||
}
|
||||
last_pushback = pushback;
|
||||
}
|
||||
postrefine_probe_ = false;
|
||||
// The canonical files were written by the last round; where that is not the best one, the best
|
||||
// one is run again to write them.
|
||||
if (!cancelled_ && geometry_rounds > best_round) {
|
||||
logger.Info("Two-pass: going back to round {} of the geometry walk (distance {:.3f} mm), the "
|
||||
"last one that lowered the realised residual", best_round, best_geometry[2]);
|
||||
set_geometry(best_geometry);
|
||||
pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
||||
++walk_passes;
|
||||
}
|
||||
|
||||
// The post-refinement is measured by pass 1 and consumed by pass 2, so carry it onto whichever
|
||||
// result is returned - it is a result of the run, not of the pass that happened to fit it. Where
|
||||
// the walk took rounds they are ONE geometry decision, so what the report names is where the
|
||||
// geometry started and where the run left it, not the first step of the way there.
|
||||
if (geometry_rounds > 0 && pass1.post_refine && adopted_geometry) {
|
||||
pass1.post_refine->beam_x_after_px = (*adopted_geometry)[0];
|
||||
pass1.post_refine->beam_y_after_px = (*adopted_geometry)[1];
|
||||
pass1.post_refine->distance_after_mm = (*adopted_geometry)[2];
|
||||
if (best_round > 0 && pass1.post_refine) {
|
||||
pass1.post_refine->beam_x_after_px = best_geometry[0];
|
||||
pass1.post_refine->beam_y_after_px = best_geometry[1];
|
||||
pass1.post_refine->distance_after_mm = best_geometry[2];
|
||||
}
|
||||
pass2.post_refine = pass1.post_refine;
|
||||
pass2.pass_number = 2 + geometry_rounds;
|
||||
pass2.pass_count = 2 + geometry_rounds;
|
||||
pass2.pass_number = 2 + walk_passes;
|
||||
pass2.pass_count = 2 + walk_passes;
|
||||
if (geometry_rounds > 0)
|
||||
pass2.pass_decision = fmt::format(
|
||||
"post-refined geometry adopted over {} rounds: the fit was still walking at each of "
|
||||
"them, and the detector distance ended at {:.3f} mm against the header's {:.3f}",
|
||||
geometry_rounds + 1, (*adopted_geometry)[2],
|
||||
pass1.post_refine ? pass1.post_refine->distance_before_mm : 0.0);
|
||||
"post-refined geometry walked {} round{} and kept round {}, detector distance {:.3f} mm "
|
||||
"against the header's {:.3f}: {}; distance / realised held-out residual by round: {}",
|
||||
geometry_rounds, geometry_rounds == 1 ? "" : "s", best_round, best_geometry[2],
|
||||
pass1.post_refine ? pass1.post_refine->distance_before_mm : 0.0,
|
||||
walk_stop.empty() ? "the fit at the kept geometry commits no further move" : walk_stop,
|
||||
walk_rounds);
|
||||
|
||||
// r2 = r1 + 2 is also the inner edge of the background ring, so widening the signal disk pushes
|
||||
// that ring out into the neighbours; a reflection whose ring is left with five or fewer clean
|
||||
@@ -2157,6 +2248,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
|
||||
pass2.pass_decision = (prepass_detector_geometry_ || prepass_rotation_scale_)
|
||||
? "post-refined geometry adopted"
|
||||
: "the post-refinement committed no geometry change, so this pass reproduces the first";
|
||||
pass2.geometry_not_converged = geometry_not_converged;
|
||||
return pass2;
|
||||
}
|
||||
return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
||||
@@ -3589,12 +3681,50 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// around the value the run started with rather than around a fit that has just
|
||||
// been tested and found no better - and so a run that fails for some other
|
||||
// reason fails at the geometry it was given.
|
||||
//
|
||||
// Unless the frame counts are both at the floor and the POOLED spots do decide:
|
||||
// a crystal whose spots are a minority of every frame indexes 0/60 at any centre,
|
||||
// and then only the pooled excess over chance - each lattice measured against its
|
||||
// own wrong-spindle null, so a larger cell is not credited for fitting more by
|
||||
// chance - says which centre sees it.
|
||||
PooledEvidence at_measured, at_header;
|
||||
if (alt.result.has_value())
|
||||
at_measured = pooled_evidence(*indexer, *alt.result);
|
||||
restore_beam_center(header_x, header_y);
|
||||
logger.Info("Beam centre check: neither the file's centre ({}/{} frames) nor "
|
||||
"the measured one {:.2f} px away ({}/{}) indexes this run - the "
|
||||
"file's centre is kept and this is not a beam-centre failure",
|
||||
best.score, static_cast<int>(validation.size()), moved, alt.score,
|
||||
static_cast<int>(validation.size()));
|
||||
if (best.result.has_value())
|
||||
at_header = pooled_evidence(*indexer, *best.result);
|
||||
auto excess = [](const PooledEvidence &e) {
|
||||
return static_cast<double>(e.on_lattice - e.by_chance)
|
||||
/ static_cast<double>(std::max<int64_t>(1, e.spots));
|
||||
};
|
||||
auto rate_var = [](const PooledEvidence &e) {
|
||||
const double n = static_cast<double>(std::max<int64_t>(1, e.spots));
|
||||
const double p = static_cast<double>(e.on_lattice) / n;
|
||||
return p * (1.0 - p) / n;
|
||||
};
|
||||
const double margin = excess(at_measured) - excess(at_header);
|
||||
const double margin_sigma = std::sqrt(rate_var(at_measured) + rate_var(at_header));
|
||||
if (alt.result.has_value() && beats_chance(at_measured)
|
||||
&& margin > SPOT_BUDGET_SIGNIFICANCE_Z * margin_sigma) {
|
||||
try_beam_center(measured_x, measured_y);
|
||||
logger.Warning("Beam centre from the background: neither centre indexes a "
|
||||
"validation frame on its own, but the pooled spots put "
|
||||
"{:.1f}% on the lattice at the measured ({:.2f},{:.2f}) against "
|
||||
"{:.1f}% at the file's (chance {:.1f}% and {:.1f}%) - the run "
|
||||
"adopts the measured centre",
|
||||
100.0 * at_measured.on_lattice / std::max<int64_t>(1, at_measured.spots),
|
||||
measured_x, measured_y,
|
||||
100.0 * at_header.on_lattice / std::max<int64_t>(1, at_header.spots),
|
||||
100.0 * at_measured.by_chance / std::max<int64_t>(1, at_measured.spots),
|
||||
100.0 * at_header.by_chance / std::max<int64_t>(1, at_header.spots));
|
||||
best = alt;
|
||||
} else {
|
||||
logger.Info("Beam centre check: neither the file's centre ({}/{} frames) nor "
|
||||
"the measured one {:.2f} px away ({}/{}) indexes this run - the "
|
||||
"file's centre is kept and this is not a beam-centre failure",
|
||||
best.score, static_cast<int>(validation.size()), moved, alt.score,
|
||||
static_cast<int>(validation.size()));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
restore_beam_center(header_x, header_y);
|
||||
@@ -4196,6 +4326,22 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
}
|
||||
indexer->ForceRotationIndexerResult(*best.result);
|
||||
// A lattice the pooled spots accept while hardly a validation frame clears the per-frame
|
||||
// floor (LATTICE_MIN_INDEXED_FRACTION) on its own, because the sweep as a whole puts fewer of
|
||||
// its spots on the lattice than that floor asks of each frame: the crystal is real but its
|
||||
// spots are a minority of every frame - a weak crystal inside a powder of its own
|
||||
// microcrystals and ice. The floor cannot tell its frames from anything else, so integrate
|
||||
// every frame from the sweep's lattice, as XDS and DIALS do, and leave the frames to the
|
||||
// scaling. A sweep that is sparse only in spots per frame (a small molecule) keeps the floor.
|
||||
const bool sparse_lattice = best.score < static_cast<int>(validation.size()) / 6
|
||||
&& pooled < LATTICE_MIN_INDEXED_FRACTION;
|
||||
indexer->IntegrateEveryFrame(sparse_lattice);
|
||||
if (sparse_lattice)
|
||||
logger.Warning("Sparse lattice: {}/{} validation frames clear the per-frame floor, but the "
|
||||
"pooled spots put {:.1f}% on the lattice against {:.1f}% by chance - every "
|
||||
"frame is integrated from the sweep's lattice",
|
||||
best.score, static_cast<int>(validation.size()), 100.0 * pooled,
|
||||
100.0 * pooled_chance);
|
||||
logger.Info("First-pass spot finding: {} frames in {} batches, {:.2f} s",
|
||||
spot_cache.size(), prefetch_batches, prefetch_time_s);
|
||||
logger.Info("Two-pass rotation indexing found lattice (scheme '{}': {}/{} validation frames, "
|
||||
@@ -4638,7 +4784,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
|
||||
// Scaling and merging (full analysis only).
|
||||
if (full && !cancelled_ && result.indexing_rate.has_value() && result.indexing_rate > 0
|
||||
if (full && !cancelled_ && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found)
|
||||
&& (config_.run_scaling || !config_.reference_data.empty())) {
|
||||
// Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
|
||||
// bar reflects what is happening instead of freezing on one label.
|
||||
@@ -4877,6 +5023,99 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
const auto &rot_ss = experiment_.GetScalingSettings();
|
||||
const bool is_rotation = experiment_.IsRotationIndexing(); // rotation indexing -> rotation scaling/merge
|
||||
std::optional<RotationScaleMerge> rsm;
|
||||
std::optional<PostRefineObservations> prepass_postrefine_obs;
|
||||
// The rotation geometry post-refinement (see its call sites below for what it is for).
|
||||
const auto post_refine_geometry = [&] {
|
||||
const auto &outcomes = indexer->GetIntegrationOutcome();
|
||||
if (geometry_prepass) {
|
||||
prepass_mosaicity_.assign(outcomes.size(), NAN);
|
||||
for (size_t o = 0; o < outcomes.size(); ++o)
|
||||
if (outcomes[o].mosaicity_deg) prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg;
|
||||
}
|
||||
if (result.consensus_cell.has_value()) {
|
||||
if (const auto rot = indexer->FinalizeRotationIndexing(); rot && rot->axis) {
|
||||
if (geometry_prepass)
|
||||
prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback
|
||||
PostRefineSettings prs;
|
||||
prs.crystal_system = rot->search_result.system;
|
||||
prs.num_threads = static_cast<int>(config_.nthreads);
|
||||
prs.refine_geometry = true;
|
||||
// The centre the pre-scan measured, where the fit was precise enough to be used as
|
||||
// a centre at all (the same ceiling --estimate-beam-center adopts one on). It does
|
||||
// not move the geometry - it only tells the bound on step B's beam move where else
|
||||
// the centre is independently believed to be.
|
||||
if (measured_beam_center_
|
||||
&& measured_beam_center_->sigma_pxl <= std::max(MAX_BEAM_CENTER_SIGMA_PXL,
|
||||
BeamCenterNeed_pxl(experiment_)))
|
||||
prs.measured_beam_px = std::array<double, 2>{
|
||||
static_cast<double>(measured_beam_center_->beam_x_pxl),
|
||||
static_cast<double>(measured_beam_center_->beam_y_pxl)};
|
||||
// The first pass's own geometry, not the header's. The outcomes being post-refined
|
||||
// were integrated at it, and step B holds the tilt fixed while it fits the beam and
|
||||
// the distance - so handing it the header tilt fits a beam that has to absorb a tilt
|
||||
// error the images do not have, and the answer describes neither geometry.
|
||||
//
|
||||
// On the pre-pass the observations were gathered right after Ingest (see there) and
|
||||
// the scaling engine is finished - nothing after this point reads it on that pass
|
||||
// (the search and the reference re-merge run only on the canonical one) - so it is
|
||||
// freed BEFORE the fit: the engine holds about twice the partial payload again,
|
||||
// and with it gone the fit no longer sits on the pass's memory peak.
|
||||
if (prepass_postrefine_obs)
|
||||
rsm.reset();
|
||||
auto pr_obs = prepass_postrefine_obs
|
||||
? std::move(*prepass_postrefine_obs)
|
||||
: GatherPostRefineObservations(indexer->GetIntegrationOutcome(),
|
||||
config_.nthreads,
|
||||
/*release_reflections=*/false, logger);
|
||||
prepass_postrefine_obs.reset();
|
||||
const auto pr = PostRefineRotationGeometry(std::move(pr_obs), *rot->axis,
|
||||
rot->geom, rot->lattice, prs, logger);
|
||||
result.post_refine = pr;
|
||||
if (pr.ok && pr.detector_refined) {
|
||||
// Store, don't apply: this pass continues to write its own (header-geometry) output;
|
||||
// Run() applies the refined geometry before the second pass.
|
||||
prepass_detector_geometry_ = std::array<float, 5>{
|
||||
static_cast<float>(pr.beam_x_after_px),
|
||||
static_cast<float>(pr.beam_y_after_px),
|
||||
static_cast<float>(pr.distance_after_mm),
|
||||
rot->geom.GetPoniRot1_rad(),
|
||||
rot->geom.GetPoniRot2_rad()};
|
||||
logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, "
|
||||
"beam {:.2f},{:.2f} px, tilt {:.4f},{:.4f} deg) - {}",
|
||||
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px,
|
||||
rot->geom.GetPoniRot1_rad() * 180.0 / PI,
|
||||
rot->geom.GetPoniRot2_rad() * 180.0 / PI,
|
||||
geometry_prepass ? "it will drive the second integration pass"
|
||||
: "what the fit still asks for at the geometry "
|
||||
"this pass ran at");
|
||||
} else {
|
||||
logger.Info("Two-pass: geometry post-refine committed no detector change{}",
|
||||
geometry_prepass ? " - the second pass reproduces the first" : "");
|
||||
}
|
||||
// DECISION POINT for the goniometer rotation scale. It does not ride on pr.ok - the
|
||||
// scale is its own cross-validated fit and the crystals that have a stage fault are
|
||||
// exactly the ones whose cell and detector steps do NOT pass, because the angle error
|
||||
// is what their residual is made of. A calibration fault is rare (36 of 37 rotation
|
||||
// datasets sit at 1.0000) and applying a 1 % angle correction to a healthy dataset
|
||||
// would silently damage it, so the asymmetry is deliberate: committed only when the
|
||||
// fit is both cross-validated and outside the tolerance. A manual --rotation-scale is
|
||||
// already on the goniometer and is left alone.
|
||||
// Only the pre-pass fits the rotation scale. It is applied from the goniometer
|
||||
// the file came with, so a scale measured again on angles that have already been
|
||||
// corrected once is not a correction that can be applied on top of that one.
|
||||
if (pr.rotation_scale_suspect && !config_.rotation_scale.has_value() && geometry_prepass)
|
||||
prepass_rotation_scale_ = static_cast<float>(pr.rotation_scale);
|
||||
}
|
||||
}
|
||||
};
|
||||
// On a CANONICAL pass the post-refinement only measures (see below), and it reads nothing the
|
||||
// scaling engine produces: the reflections' hkl, I, sigma and positions are final once the
|
||||
// images are integrated, and the engine writes back per-frame fields only. So measure it
|
||||
// before the engine is built rather than beside it - the gathered observations and their
|
||||
// rocking events are gigabytes on a long axis, and beside the engine's own arrays they set
|
||||
// the pass's memory high-water mark.
|
||||
if (postrefine_probe_ && !geometry_prepass && is_rotation)
|
||||
post_refine_geometry();
|
||||
if (is_rotation) {
|
||||
if (experiment_.GetRefineRotationWedgeInScaling()
|
||||
|| rot_ss.GetRotationWedgeForScaling().has_value())
|
||||
@@ -4900,7 +5139,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// it is read: on a fine-sliced long axis the vectors are gigabytes, held to the end of the
|
||||
// pass for those 32 bytes a partial. A run with a reference MTZ keeps them: resolving the
|
||||
// indexing ambiguity re-reads them (and re-ingests).
|
||||
std::optional<PostRefineObservations> prepass_postrefine_obs;
|
||||
if (geometry_prepass && rsm && config_.reference_data.empty())
|
||||
prepass_postrefine_obs = GatherPostRefineObservations(
|
||||
indexer->GetIntegrationOutcome(), config_.nthreads,
|
||||
@@ -4916,6 +5154,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
/*measure_cc_before_corrections=*/true);
|
||||
result.error_model_isa = r.isa;
|
||||
result.error_model_isa_asymptotic = r.isa_asymptotic;
|
||||
result.error_model_isa_resolved = r.isa_resolved;
|
||||
if (r.resolution_fit_A) result.resolution_fit_A = r.resolution_fit_A;
|
||||
result.error_model_a = r.error_model_a;
|
||||
result.error_model_b = r.error_model_b;
|
||||
@@ -5081,89 +5320,8 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// reflections integrated at the geometry after it. The probe measures - it applies nothing and
|
||||
// leaves the smoothed mosaicity and the pre-pass lattice alone, which stay pass 1's for every
|
||||
// pass that consumes them; RunAllPasses decides what to do with the answer.
|
||||
if ((geometry_prepass || postrefine_probe_) && rsm && experiment_.IsRotationIndexing()) {
|
||||
const auto &outcomes = indexer->GetIntegrationOutcome();
|
||||
if (geometry_prepass) {
|
||||
prepass_mosaicity_.assign(outcomes.size(), NAN);
|
||||
for (size_t o = 0; o < outcomes.size(); ++o)
|
||||
if (outcomes[o].mosaicity_deg) prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg;
|
||||
}
|
||||
if (result.consensus_cell.has_value()) {
|
||||
if (const auto rot = indexer->FinalizeRotationIndexing(); rot && rot->axis) {
|
||||
if (geometry_prepass)
|
||||
prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback
|
||||
PostRefineSettings prs;
|
||||
prs.crystal_system = rot->search_result.system;
|
||||
prs.num_threads = static_cast<int>(config_.nthreads);
|
||||
prs.refine_geometry = true;
|
||||
// The centre the pre-scan measured, where the fit was precise enough to be used as
|
||||
// a centre at all (the same ceiling --estimate-beam-center adopts one on). It does
|
||||
// not move the geometry - it only tells the bound on step B's beam move where else
|
||||
// the centre is independently believed to be.
|
||||
if (measured_beam_center_
|
||||
&& measured_beam_center_->sigma_pxl <= std::max(MAX_BEAM_CENTER_SIGMA_PXL,
|
||||
BeamCenterNeed_pxl(experiment_)))
|
||||
prs.measured_beam_px = std::array<double, 2>{
|
||||
static_cast<double>(measured_beam_center_->beam_x_pxl),
|
||||
static_cast<double>(measured_beam_center_->beam_y_pxl)};
|
||||
// The first pass's own geometry, not the header's. The outcomes being post-refined
|
||||
// were integrated at it, and step B holds the tilt fixed while it fits the beam and
|
||||
// the distance - so handing it the header tilt fits a beam that has to absorb a tilt
|
||||
// error the images do not have, and the answer describes neither geometry.
|
||||
//
|
||||
// On the pre-pass the observations were gathered right after Ingest (see there) and
|
||||
// the scaling engine is finished - nothing after this point reads it on that pass
|
||||
// (the search and the reference re-merge run only on the canonical one) - so it is
|
||||
// freed BEFORE the fit: the engine holds about twice the partial payload again,
|
||||
// and with it gone the fit no longer sits on the pass's memory peak.
|
||||
if (prepass_postrefine_obs)
|
||||
rsm.reset();
|
||||
auto pr_obs = prepass_postrefine_obs
|
||||
? std::move(*prepass_postrefine_obs)
|
||||
: GatherPostRefineObservations(indexer->GetIntegrationOutcome(),
|
||||
config_.nthreads,
|
||||
/*release_reflections=*/false, logger);
|
||||
prepass_postrefine_obs.reset();
|
||||
const auto pr = PostRefineRotationGeometry(std::move(pr_obs), *rot->axis,
|
||||
rot->geom, rot->lattice, prs, logger);
|
||||
result.post_refine = pr;
|
||||
if (pr.ok && pr.detector_refined) {
|
||||
// Store, don't apply: this pass continues to write its own (header-geometry) output;
|
||||
// Run() applies the refined geometry before the second pass.
|
||||
prepass_detector_geometry_ = std::array<float, 5>{
|
||||
static_cast<float>(pr.beam_x_after_px),
|
||||
static_cast<float>(pr.beam_y_after_px),
|
||||
static_cast<float>(pr.distance_after_mm),
|
||||
rot->geom.GetPoniRot1_rad(),
|
||||
rot->geom.GetPoniRot2_rad()};
|
||||
logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, "
|
||||
"beam {:.2f},{:.2f} px, tilt {:.4f},{:.4f} deg) - {}",
|
||||
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px,
|
||||
rot->geom.GetPoniRot1_rad() * 180.0 / PI,
|
||||
rot->geom.GetPoniRot2_rad() * 180.0 / PI,
|
||||
geometry_prepass ? "it will drive the second integration pass"
|
||||
: "what the fit still asks for at the geometry "
|
||||
"this pass ran at");
|
||||
} else {
|
||||
logger.Info("Two-pass: geometry post-refine committed no detector change{}",
|
||||
geometry_prepass ? " - the second pass reproduces the first" : "");
|
||||
}
|
||||
// DECISION POINT for the goniometer rotation scale. It does not ride on pr.ok - the
|
||||
// scale is its own cross-validated fit and the crystals that have a stage fault are
|
||||
// exactly the ones whose cell and detector steps do NOT pass, because the angle error
|
||||
// is what their residual is made of. A calibration fault is rare (36 of 37 rotation
|
||||
// datasets sit at 1.0000) and applying a 1 % angle correction to a healthy dataset
|
||||
// would silently damage it, so the asymmetry is deliberate: committed only when the
|
||||
// fit is both cross-validated and outside the tolerance. A manual --rotation-scale is
|
||||
// already on the goniometer and is left alone.
|
||||
// Only the pre-pass fits the rotation scale. It is applied from the goniometer
|
||||
// the file came with, so a scale measured again on angles that have already been
|
||||
// corrected once is not a correction that can be applied on top of that one.
|
||||
if (pr.rotation_scale_suspect && !config_.rotation_scale.has_value() && geometry_prepass)
|
||||
prepass_rotation_scale_ = static_cast<float>(pr.rotation_scale);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (geometry_prepass && rsm && experiment_.IsRotationIndexing())
|
||||
post_refine_geometry();
|
||||
|
||||
// For a de-novo search, gate the P1 scaling+merge (the pass that feeds the space-group search) at
|
||||
// <I/sigma> >= 1 over thin resolution shells: noise-dominated high-res shells otherwise mislead
|
||||
@@ -5280,9 +5438,12 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// rotations are real, because a promotion is refused by the most damning statistic it can
|
||||
// be shown and only one arm has to be starved for that to happen, while the ABSENCES are
|
||||
// always judged on the all-observation merge, which is the only one holding the weak
|
||||
// reflections a systematic absence is made of. Both searches keep their vetoes, so a merohedral twin
|
||||
// stays refused in each (verified on a real twin at the three frame ranges where it
|
||||
// over-promotes: refused in both arms every time).
|
||||
// reflections a systematic absence is made of. The vetoes are NOT kept per arm: a promotion
|
||||
// one arm refused on its twin gates (H ratio, added-operator R) is adopted when the other arm
|
||||
// confirms it, because the higher order wins. That held on one real twin at three frame
|
||||
// ranges, where both arms refused, but an H3 partial twin has been seen refused by the
|
||||
// all-observation arm on both twin gates and promoted to R32 by the filtered one. So the
|
||||
// losing arm's refusal is logged and carried to the report below; the rule is unchanged.
|
||||
if (rsm && rsm->GetSearchMinZeta() > 0.0 && !sg_search.point_group_hm.empty()) {
|
||||
const double zeta = rsm->GetSearchMinZeta();
|
||||
rsm->SetSearchMinZeta(0.0);
|
||||
@@ -5341,10 +5502,29 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
stats_text << " !! " << msg << "\n\n";
|
||||
result.warnings.push_back({PathologyCode::SYMMETRY_AMBIGUITY, msg});
|
||||
}
|
||||
if (order_of(alt) > order_of(sg_search))
|
||||
// The arm whose lower answer is about to be overridden, and why it stopped there. A
|
||||
// refusal by a consistency gate is the case that matters: the higher group is adopted
|
||||
// anyway, and without this nothing in the output says a gate ever fired.
|
||||
const auto overridden = [&](const SearchSpaceGroupResult &lower, const char *lower_name,
|
||||
const char *higher_name) {
|
||||
std::string msg;
|
||||
if (!lower.refused_point_group_hm.empty())
|
||||
msg = fmt::format("the {} merge REFUSED {} ({}); overridden, because the {} merge "
|
||||
"confirmed the higher point group", lower_name,
|
||||
lower.refused_point_group_hm, lower.refused_reason, higher_name);
|
||||
else
|
||||
msg = fmt::format("the {} merge did not confirm the operators of the higher point "
|
||||
"group (no gate refused it); the {} merge did", lower_name,
|
||||
higher_name);
|
||||
logger.Warning("Space-group search: {}", msg);
|
||||
result.space_group_arm_override = msg;
|
||||
};
|
||||
if (order_of(alt) > order_of(sg_search)) {
|
||||
logger.Info("Space-group search: all-observation merge supports {} where the "
|
||||
"Lorentz-filtered one supports {} - taking the higher symmetry",
|
||||
alt.point_group_hm, sg_search.point_group_hm);
|
||||
overridden(sg_search, "Lorentz-filtered", "all-observation");
|
||||
}
|
||||
if (order_of(alt) < order_of(sg_search)) {
|
||||
// The filtered arm found symmetry the all-observation arm did not. Discarding it
|
||||
// was costing real point groups: over the corpus the two arms disagree on eight
|
||||
@@ -5359,6 +5539,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// that found it and the absences are re-judged here, on every observation. That
|
||||
// is what fixed_point_group is for: Stage A still runs and still reports what it
|
||||
// would have refused, only the choice is overridden.
|
||||
overridden(alt, "all-observation", "Lorentz-filtered");
|
||||
sg_opts.fixed_point_group = sg_search.point_group_representative;
|
||||
auto split = SearchSpaceGroup(sm_all.merged, sg_opts);
|
||||
sg_opts.fixed_point_group.reset();
|
||||
@@ -5383,7 +5564,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// given. Reported alongside the post-adoption analysis: once a higher Laue class has been
|
||||
// adopted, its own twinning test can only say "no twin law exists within this class", which
|
||||
// is circular - the promotion is what a twin would have caused. These numbers are not.
|
||||
const auto pre_promotion_twinning = AnalyzeTwinning(sm.merged, nullptr);
|
||||
//
|
||||
// Measured as the post-adoption test is: with the pseudo-translation of this same merge
|
||||
// declared, so the L-test can be repaired for it, and - the merge being in P1 - without the
|
||||
// reflections the lattice centring extinguishes, which an R lattice in its hexagonal cell
|
||||
// still carries (two thirds of them, and every axis step lands on one).
|
||||
TranslationalNCSResult pre_tncs;
|
||||
if (result.consensus_cell) {
|
||||
pre_tncs = AnalyzeTranslationalNCS(sm.merged, gemmi::UnitCell(*result.consensus_cell));
|
||||
result.pre_promotion_tncs_measured = pre_tncs.measurable;
|
||||
}
|
||||
const char lattice_centring = end_msg.rotation_lattice_type.has_value()
|
||||
? end_msg.rotation_lattice_type->centering : 'P';
|
||||
const auto pre_promotion_twinning = AnalyzeTwinning(
|
||||
sm.merged, nullptr, 20, TranslationalNCSVectorForLTest(pre_tncs), lattice_centring);
|
||||
result.pre_promotion_twinning = pre_promotion_twinning;
|
||||
if (pre_promotion_twinning.twinning_suspected)
|
||||
logger.Warning("Twinning indicators BEFORE the space-group decision (subgroup merge): "
|
||||
@@ -6099,6 +6293,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// comparison merges are not the run's answer, so it goes back if they lose.
|
||||
const double em_isa = result.error_model_isa;
|
||||
const double em_isa_asymptotic = result.error_model_isa_asymptotic;
|
||||
const bool em_isa_resolved = result.error_model_isa_resolved;
|
||||
const double em_a = result.error_model_a;
|
||||
const double em_b = result.error_model_b;
|
||||
const auto res_fit = result.resolution_fit_A;
|
||||
@@ -6165,6 +6360,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
experiment_.SetSpaceGroup(sg);
|
||||
result.error_model_isa = em_isa;
|
||||
result.error_model_isa_asymptotic = em_isa_asymptotic;
|
||||
result.error_model_isa_resolved = em_isa_resolved;
|
||||
result.error_model_a = em_a;
|
||||
result.error_model_b = em_b;
|
||||
result.resolution_fit_A = res_fit;
|
||||
@@ -6661,9 +6857,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
&& !superseded) {
|
||||
phase("Writing reflections");
|
||||
const ErrorModelReport em_report{
|
||||
result.error_model_isa > 0 ? fmt::format("{:.2f}", result.error_model_isa) : "?",
|
||||
result.error_model_isa_asymptotic > 0 ? fmt::format("{:.2f}", result.error_model_isa_asymptotic)
|
||||
: std::string(),
|
||||
result.error_model_isa > 0 && result.error_model_isa_resolved
|
||||
? fmt::format("{:.2f}", result.error_model_isa) : "?",
|
||||
result.error_model_isa_asymptotic > 0 && result.error_model_isa_resolved
|
||||
? fmt::format("{:.2f}", result.error_model_isa_asymptotic) : std::string(),
|
||||
result.error_model_a > 0 ? fmt::format("{:.3f}", result.error_model_a) : std::string(),
|
||||
result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string()};
|
||||
WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics,
|
||||
@@ -6740,6 +6937,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// multiplicity, so its fall-off is not this crystal's.
|
||||
const double em_isa = result.error_model_isa;
|
||||
const double em_isa_asymptotic = result.error_model_isa_asymptotic;
|
||||
const bool em_isa_resolved = result.error_model_isa_resolved;
|
||||
const double em_a = result.error_model_a;
|
||||
const double em_b = result.error_model_b;
|
||||
const auto res_fit = result.resolution_fit_A;
|
||||
@@ -6748,6 +6946,16 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
experiment_.SpaceGroupNumber(1);
|
||||
const auto p1 = scale_and_merge("P1 cross-check", false);
|
||||
|
||||
// Twin-immune zone evidence for each 2-fold the adopted group holds, read on this merge
|
||||
// because it is folded under no symmetry, at full resolution and production-scaled.
|
||||
// Report-only: nothing reads it back.
|
||||
if (determined != nullptr && determined->number > 1) {
|
||||
result.twin_immune_zones = AnalyzeTwinImmuneZones(
|
||||
p1.merged, *determined, TranslationalNCSVectorForLTest(result.tncs));
|
||||
if (!result.twin_immune_zones->zones.empty())
|
||||
logger.Info("{}", TwinImmuneZonesToText(*result.twin_immune_zones));
|
||||
}
|
||||
|
||||
// FINALIST LEDGER (--finalist-ledger), report-only and deliberately behind a flag: these
|
||||
// numbers are an instrument, and reading them as a decision before they are calibrated
|
||||
// is the one way this can do harm.
|
||||
@@ -6779,6 +6987,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
experiment_.SetSpaceGroup(determined_group);
|
||||
result.error_model_isa = em_isa;
|
||||
result.error_model_isa_asymptotic = em_isa_asymptotic;
|
||||
result.error_model_isa_resolved = em_isa_resolved;
|
||||
result.error_model_a = em_a;
|
||||
result.error_model_b = em_b;
|
||||
result.resolution_fit_A = res_fit;
|
||||
|
||||
@@ -239,6 +239,7 @@ struct ProcessResult {
|
||||
MergeStatistics merge_statistics;
|
||||
double error_model_isa = 0.0; // whole-range 1/sqrt(a*b), XDS's meaning
|
||||
double error_model_isa_asymptotic = 0.0; // strong-reflection tier; rotation path only, 0 on stills
|
||||
bool error_model_isa_resolved = true; // false: b is zero within its error, ISa printed "undetermined"
|
||||
double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2)
|
||||
double error_model_b = 0.0;
|
||||
bool has_reference = false;
|
||||
@@ -313,6 +314,13 @@ struct ProcessResult {
|
||||
// class is adopted, its own twinning test can say no more than "no twin law exists inside this
|
||||
// class", and the promotion is what a twin would have caused. Empty when no de-novo search ran.
|
||||
std::optional<TwinningAnalysisResult> pre_promotion_twinning;
|
||||
// Whether a pseudo-translation was looked for on that same merge, so its L-test could be repaired
|
||||
// for one. False when there was no cell to run the Patterson on.
|
||||
bool pre_promotion_tncs_measured = false;
|
||||
|
||||
// Twin-immune zone evidence for the 2-folds of the adopted group, on the P1 cross-check merge
|
||||
// (AnalyzeTwinImmuneZones). Report-only. Unset when no P1 cross-check merge was built.
|
||||
std::optional<TwinImmuneZoneResult> twin_immune_zones;
|
||||
|
||||
// What --model found, when one was given. Set even when the validation could not run - its
|
||||
// failure_reason then says why - so the report can report the failure rather than leave a run
|
||||
@@ -331,6 +339,10 @@ struct ProcessResult {
|
||||
// populated only when a search ran (no space group was fixed). The library no longer renders it to
|
||||
// text - the CLI formats it for stdout, the viewer draws it as a table.
|
||||
std::optional<SearchSpaceGroupResult> space_group_search;
|
||||
// When the two search arms (all observations / Lorentz-filtered) disagreed and the higher point
|
||||
// group was taken from one of them: what the other arm said - which gate refused the promotion,
|
||||
// or that it never confirmed the operators. Empty when the arms agreed.
|
||||
std::string space_group_arm_override;
|
||||
|
||||
// ---- Material for the results report (<prefix>_report.txt, see rugnux/ResultReport.h) ----
|
||||
|
||||
@@ -349,6 +361,9 @@ struct ProcessResult {
|
||||
int pass_number = 1;
|
||||
int pass_count = 1;
|
||||
std::string pass_decision;
|
||||
// The rotation geometry walk used every round it was given and its fit still asked for a move: the
|
||||
// geometry this result was integrated at is the best round the walk reached, not a settled one.
|
||||
bool geometry_not_converged = false;
|
||||
|
||||
// The radiation-damage report (rotation), the same text the log prints. Empty when not measured.
|
||||
std::string radiation_damage_text;
|
||||
|
||||
@@ -1577,7 +1577,8 @@ static int RunRugnux(int argc, char **argv) {
|
||||
if (report_resolution_arg)
|
||||
scaling_settings.ReportResolutionRange_A(
|
||||
ParseReportResolution(*report_resolution_arg, scaling_settings, logger));
|
||||
scaling_settings.MergeFriedel(!anomalous_mode);
|
||||
// Rotation merges decide with Friedel mates merged, -A or not (see the Rugnux constructor).
|
||||
scaling_settings.MergeFriedel(!anomalous_mode || rot);
|
||||
scaling_settings.MinPartiality(min_partiality);
|
||||
scaling_settings.MinCapturedFraction(
|
||||
min_captured_fraction_arg.value_or(scaling_settings.GetMinCapturedFraction()));
|
||||
@@ -1727,6 +1728,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
MergeStatistics merged_statistics;
|
||||
double error_model_isa = 0.0;
|
||||
double error_model_isa_asymptotic = 0.0;
|
||||
bool error_model_isa_resolved = true;
|
||||
std::optional<double> resolution_fit_A; // CC1/2 crossing, without the one-shell extension
|
||||
double error_model_a = 0.0;
|
||||
double error_model_b = 0.0;
|
||||
@@ -1755,6 +1757,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
merged_statistics = std::move(r.statistics);
|
||||
error_model_isa = r.isa;
|
||||
error_model_isa_asymptotic = r.isa_asymptotic;
|
||||
error_model_isa_resolved = r.isa_resolved;
|
||||
error_model_a = r.error_model_a;
|
||||
error_model_b = r.error_model_b;
|
||||
resolution_fit_A = r.resolution_fit_A;
|
||||
@@ -1952,9 +1955,10 @@ static int RunRugnux(int argc, char **argv) {
|
||||
|
||||
if (!output_prefix.empty()) {
|
||||
const ErrorModelReport em_report{
|
||||
error_model_isa > 0 ? fmt::format("{:.2f}", error_model_isa) : "?",
|
||||
error_model_isa_asymptotic > 0 ? fmt::format("{:.2f}", error_model_isa_asymptotic)
|
||||
: std::string(),
|
||||
error_model_isa > 0 && error_model_isa_resolved
|
||||
? fmt::format("{:.2f}", error_model_isa) : "?",
|
||||
error_model_isa_asymptotic > 0 && error_model_isa_resolved
|
||||
? fmt::format("{:.2f}", error_model_isa_asymptotic) : std::string(),
|
||||
error_model_a > 0 ? fmt::format("{:.3f}", error_model_a) : std::string(),
|
||||
error_model_b > 0 ? fmt::format("{:.4e}", error_model_b) : std::string()};
|
||||
WriteReflections(merged_reflections, *experiment.GetUnitCell(), experiment, merged_statistics,
|
||||
@@ -1990,6 +1994,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
{ std::ostringstream s; s << merged_statistics; scale_result.merge_statistics_text = s.str(); }
|
||||
scale_result.error_model_isa = error_model_isa;
|
||||
scale_result.error_model_isa_asymptotic = error_model_isa_asymptotic;
|
||||
scale_result.error_model_isa_resolved = error_model_isa_resolved;
|
||||
scale_result.error_model_a = error_model_a;
|
||||
scale_result.error_model_b = error_model_b;
|
||||
scale_result.resolution_fit_A = resolution_fit_A;
|
||||
|
||||
@@ -54,6 +54,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
ParallelForTest.cpp
|
||||
RugnuxTest.cpp
|
||||
ResultReportTest.cpp
|
||||
PostRefineTest.cpp
|
||||
RugnuxLargeTest.cpp
|
||||
TestData.h
|
||||
MovingAverageTest.cpp
|
||||
|
||||
@@ -306,14 +306,17 @@ namespace {
|
||||
// other (signal) or drawn independently (noise, CC1/2 ~ 0), so a whole CC1/2 curve can be built
|
||||
// band by band.
|
||||
void AddBand(std::vector<MergedReflection> &v, std::mt19937 &rng,
|
||||
double s_from, double s_to, int n, bool correlated) {
|
||||
std::normal_distribution<double> g(0.0, 1.0);
|
||||
double s_from, double s_to, int n, bool correlated, double sigma = 1.0) {
|
||||
std::normal_distribution<double> g(0.0, sigma);
|
||||
for (int j = 0; j < n; ++j) {
|
||||
MergedReflection m;
|
||||
m.d = static_cast<float>(1.0 / std::sqrt(s_from + (j + 0.5) * (s_to - s_from) / n));
|
||||
const double a = g(rng), b = g(rng);
|
||||
m.I_half[0] = static_cast<float>(a);
|
||||
m.I_half[1] = static_cast<float>(correlated ? a : b);
|
||||
// The pair's weight in a CC1/2: the precision it would have had at the typical frame scale
|
||||
// over the precision it has - 1/sigma^2 for a band scaled up by sigma from dead frames.
|
||||
m.cc_weight = static_cast<float>(1.0 / (sigma * sigma));
|
||||
v.push_back(m);
|
||||
}
|
||||
}
|
||||
@@ -336,6 +339,54 @@ TEST_CASE("ResolutionCutoff_CleanFallOff") {
|
||||
CHECK(*rc.d_cut == Catch::Approx(1.92).margin(0.15));
|
||||
}
|
||||
|
||||
// A sweep with a long stretch where the crystal barely diffracted: the reflections measured only there
|
||||
// are noise scaled up by 1/G, sigmas with them, and there are many of them at every resolution. Counted
|
||||
// by the information they carry they must not hide where the well-measured reflections stop.
|
||||
TEST_CASE("ResolutionCutoff_ScaledUpNoiseDoesNotHideTheFallOff") {
|
||||
Logger logger("test");
|
||||
std::mt19937 rng(12345);
|
||||
std::vector<MergedReflection> merged;
|
||||
AddBand(merged, rng, 0.01, 0.25, 480, true);
|
||||
AddBand(merged, rng, 0.25, 0.51, 520, false);
|
||||
AddBand(merged, rng, 0.01, 0.51, 300, false, 50.0);
|
||||
|
||||
const auto rc = ComputeCCHalfLogisticCutoff(merged, 0.30, logger);
|
||||
REQUIRE(rc.d_fit);
|
||||
CHECK(*rc.d_fit == Catch::Approx(2.0).margin(0.15));
|
||||
|
||||
// Counted as equals, the same reflections hide it: the curve reads noise from the first bin on.
|
||||
for (auto &m : merged) m.cc_weight = 1.0f;
|
||||
const auto unweighted = ComputeCCHalfLogisticCutoff(merged, 0.30, logger);
|
||||
CHECK((!unweighted.d_fit || *unweighted.d_fit > 3.0));
|
||||
}
|
||||
|
||||
// The frame factor of the CC1/2 weight is 1 on a sweep whose frames all sit at one scale, however that
|
||||
// scale is spread over the frames, and 1 for any frame brighter than typical.
|
||||
TEST_CASE("CCHalfFrameFactors_UniformScaleIsOne") {
|
||||
const auto f = CCHalfFrameFactors({1.7, 1.7, 1.7, 1.7}, {100, 3, 250, 0});
|
||||
for (double x : f) CHECK(x == 1.0);
|
||||
const auto g = CCHalfFrameFactors({1.0, 2.0}, {100, 100});
|
||||
CHECK(g[1] == 1.0);
|
||||
CHECK(g[0] == Catch::Approx(9.0 / 5.0 * 9.0 / 5.0)); // G_ref = (1 + 8) / (1 + 4)
|
||||
}
|
||||
|
||||
// Most of the sweep at 2% of the good frames' scale: the typical scale is still the good frames', not
|
||||
// the run median, and an observation from a dead frame carries 2500x the variance of a good one.
|
||||
TEST_CASE("CCHalfFrameFactors_DeadStretchDoesNotSetTheTypicalScale") {
|
||||
std::vector<double> scale;
|
||||
std::vector<int64_t> n;
|
||||
for (int f = 0; f < 100; ++f) {
|
||||
scale.push_back(f < 60 ? 0.02 : 1.0);
|
||||
n.push_back(50);
|
||||
}
|
||||
scale.push_back(NAN); // a frame without a scale counts as G = 1
|
||||
n.push_back(50);
|
||||
const auto factor = CCHalfFrameFactors(scale, n);
|
||||
CHECK(factor[0] == Catch::Approx(2500.0).epsilon(0.01));
|
||||
CHECK(factor[99] == Catch::Approx(1.0).epsilon(1e-3));
|
||||
CHECK(factor[100] == Catch::Approx(1.0).epsilon(1e-3));
|
||||
}
|
||||
|
||||
// A fall-off region a logistic cannot follow: CC1/2 drops through the target and comes straight back
|
||||
// up. The fitted crossing is then an extrapolation far past the bins it was made over, and reading
|
||||
// the cut off it writes the data deep into the noise; the crossing the bins themselves show is where
|
||||
|
||||
@@ -141,3 +141,26 @@ TEST_CASE("ModelScaling adds nothing when there is no solvent to model") {
|
||||
CHECK(scaling.b_sol >= 10.0);
|
||||
CHECK(scaling.b_sol <= 80.0);
|
||||
}
|
||||
|
||||
// The grid points are fitted in parallel; the answer must be the serial one, bit for bit.
|
||||
TEST_CASE("ModelScaling is the same on any number of threads") {
|
||||
const gemmi::UnitCell cell = Cell();
|
||||
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 21 21 21");
|
||||
REQUIRE(sg != nullptr);
|
||||
gemmi::Scaling<float> serial(cell, sg);
|
||||
serial.use_solvent = true;
|
||||
MakePoints(serial, cell, /*k_overall=*/0.5, /*b_iso=*/20.0,
|
||||
/*k_sol=*/0.35, /*b_sol=*/45.0, /*d_min=*/2.0, /*d_max=*/50.0);
|
||||
gemmi::Scaling<float> parallel = serial;
|
||||
|
||||
const ModelScaleReport r1 = FitModelScale(serial, {}, 1);
|
||||
const ModelScaleReport r8 = FitModelScale(parallel, {}, 8);
|
||||
CHECK(r1.n_grid == r8.n_grid);
|
||||
CHECK(r1.r_work_fit == r8.r_work_fit);
|
||||
CHECK(serial.k_sol == parallel.k_sol);
|
||||
CHECK(serial.b_sol == parallel.b_sol);
|
||||
CHECK(serial.k_overall == parallel.k_overall);
|
||||
CHECK(serial.b_star.u11 == parallel.b_star.u11);
|
||||
CHECK(serial.b_star.u22 == parallel.b_star.u22);
|
||||
CHECK(serial.b_star.u33 == parallel.b_star.u33);
|
||||
}
|
||||
@@ -11,8 +11,10 @@
|
||||
#include <sstream>
|
||||
|
||||
#include <gemmi/mmread_gz.hpp>
|
||||
#include <gemmi/fourier.hpp>
|
||||
|
||||
#include "../common/Logger.h"
|
||||
#include "../rugnux/ModelFFT.h"
|
||||
#include "../rugnux/ModelValidation.h"
|
||||
#include "../rugnux/RigidBodyRefine.h"
|
||||
#include "../rugnux/SigmaA.h"
|
||||
@@ -587,3 +589,76 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
|
||||
std::filesystem::remove(prefix + suffix);
|
||||
std::filesystem::remove(path);
|
||||
}
|
||||
|
||||
// The model path's structure factors come from FFTW; they must be gemmi's own transform to float
|
||||
// precision, in the same layout, so prepare_asu_data() reads the same reflections from either.
|
||||
TEST_CASE("ModelValidation_MapToFPhiMatchesGemmi", "[ModelValidation]") {
|
||||
gemmi::Grid<float> map;
|
||||
map.unit_cell.set(40.0, 50.0, 60.0, 90.0, 95.0, 90.0);
|
||||
map.spacegroup = gemmi::find_spacegroup_by_name("P 1");
|
||||
map.set_size(20, 24, 30);
|
||||
std::mt19937 rng(7);
|
||||
std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
|
||||
for (auto &x : map.data)
|
||||
x = dist(rng);
|
||||
|
||||
gemmi::FPhiGrid<float> ref = gemmi::transform_map_to_f_phi(map, true);
|
||||
gemmi::FPhiGrid<float> ours = MapToFPhi(map);
|
||||
REQUIRE(ours.nu == ref.nu);
|
||||
REQUIRE(ours.nv == ref.nv);
|
||||
REQUIRE(ours.nw == ref.nw);
|
||||
REQUIRE(ours.half_l == ref.half_l);
|
||||
REQUIRE(ours.data.size() == ref.data.size());
|
||||
double largest = 0, worst = 0;
|
||||
for (size_t i = 0; i < ref.data.size(); i++) {
|
||||
largest = std::max(largest, static_cast<double>(std::abs(ref.data[i])));
|
||||
worst = std::max(worst, static_cast<double>(std::abs(ours.data[i] - ref.data[i])));
|
||||
}
|
||||
CHECK(worst <= 1e-5 * largest);
|
||||
|
||||
const auto a = ref.prepare_asu_data(4.0, 0, false, false, false);
|
||||
const auto b = ours.prepare_asu_data(4.0, 0, false, false, false);
|
||||
REQUIRE(a.v.size() == b.v.size());
|
||||
for (size_t i = 0; i < a.v.size(); i++)
|
||||
CHECK(a.v[i].hkl == b.v[i].hkl);
|
||||
}
|
||||
|
||||
// The Jacobian's six columns are evaluated in parallel; the placement must be the serial one, bit for
|
||||
// bit.
|
||||
TEST_CASE("ModelValidation_RigidBodySameOnAnyNumberOfThreads", "[ModelValidation]") {
|
||||
Logger logger("ModelValidation_RigidBodySameOnAnyNumberOfThreads");
|
||||
|
||||
const auto path = WriteTemp("rigid_body_threads_test.pdb", ClusterPdb().c_str());
|
||||
gemmi::Structure st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect);
|
||||
const gemmi::SpaceGroup *sg = st.find_spacegroup();
|
||||
REQUIRE(sg != nullptr);
|
||||
st.setup_cell_images();
|
||||
|
||||
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
|
||||
REQUIRE_FALSE(ref.empty());
|
||||
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
|
||||
fobs.unit_cell_ = st.cell;
|
||||
fobs.spacegroup_ = sg;
|
||||
for (const auto &r : ref)
|
||||
fobs.v.push_back({{{r.h, r.k, r.l}}, {std::sqrt(r.I), 1.0f}});
|
||||
fobs.ensure_sorted();
|
||||
|
||||
std::vector<gemmi::Position> displaced;
|
||||
for (const gemmi::Position &p : ModelPositions(st.models[0]))
|
||||
displaced.emplace_back(p.x + 0.40, p.y - 0.30, p.z + 0.20);
|
||||
|
||||
gemmi::Model serial = st.models[0], parallel = st.models[0];
|
||||
SetModelPositions(serial, displaced);
|
||||
SetModelPositions(parallel, displaced);
|
||||
const RigidBodyRefineResult r1 = RefineRigidBody(serial, st.cell, *sg, fobs, 3.0, logger, 1);
|
||||
const RigidBodyRefineResult r6 = RefineRigidBody(parallel, st.cell, *sg, fobs, 3.0, logger, 6);
|
||||
CHECK(r1.evaluations == r6.evaluations);
|
||||
CHECK(r1.angle_deg == r6.angle_deg);
|
||||
CHECK(r1.shift_A == r6.shift_A);
|
||||
const auto p1 = ModelPositions(serial), p6 = ModelPositions(parallel);
|
||||
REQUIRE(p1.size() == p6.size());
|
||||
for (size_t i = 0; i < p1.size(); i++)
|
||||
CHECK((p1[i].x == p6[i].x && p1[i].y == p6[i].y && p1[i].z == p6[i].z));
|
||||
|
||||
std::filesystem::remove(path);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include "../image_analysis/geom_refinement/PostRefine.h"
|
||||
|
||||
namespace {
|
||||
PostRefineResult Measured(double residual, double se) {
|
||||
PostRefineResult r;
|
||||
r.held_out_before = residual;
|
||||
r.held_out_before_se = se;
|
||||
return r;
|
||||
}
|
||||
|
||||
PostRefineResult Fit(UnitCell before, UnitCell after) {
|
||||
PostRefineResult r;
|
||||
r.cell_refined = true;
|
||||
r.cell_before = before;
|
||||
r.cell = after;
|
||||
return r;
|
||||
}
|
||||
|
||||
UnitCell Cell(float a, float b, float c) {
|
||||
return UnitCell{.a = a, .b = b, .c = c, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("PostRefine_HeldOutResidualFell", "[PostRefine]") {
|
||||
// Down by more than the standard error of the difference, sqrt(3^2 + 4^2) = 5.
|
||||
CHECK(HeldOutResidualFell(Measured(100.0, 3.0), Measured(94.0, 4.0)));
|
||||
// Down, but within that noise.
|
||||
CHECK_FALSE(HeldOutResidualFell(Measured(100.0, 3.0), Measured(96.0, 4.0)));
|
||||
// Up.
|
||||
CHECK_FALSE(HeldOutResidualFell(Measured(100.0, 3.0), Measured(110.0, 4.0)));
|
||||
// Not measured on either side.
|
||||
CHECK_FALSE(HeldOutResidualFell(PostRefineResult{}, Measured(10.0, 1.0)));
|
||||
CHECK_FALSE(HeldOutResidualFell(Measured(100.0, 3.0), PostRefineResult{}));
|
||||
}
|
||||
|
||||
TEST_CASE("PostRefine_ReindexPushesCellBack", "[PostRefine]") {
|
||||
// The fit shortens b the most; re-indexing at its geometry returns a longer b: pushed back.
|
||||
const auto fit = Fit(Cell(96.9f, 107.9f, 112.9f), Cell(96.8f, 106.6f, 112.9f));
|
||||
CHECK(ReindexPushesCellBack(fit, Fit(Cell(96.7f, 107.6f, 112.6f), Cell(96.6f, 106.4f, 112.8f))) == -2);
|
||||
// A walk that re-indexing follows (and overshoots a little) is not pushed back.
|
||||
const auto walk = Fit(Cell(60.48f, 60.48f, 196.7f), Cell(60.05f, 60.05f, 195.2f));
|
||||
CHECK(ReindexPushesCellBack(walk, Fit(Cell(60.00f, 60.00f, 195.1f), Cell(59.8f, 59.8f, 194.2f))) == 0);
|
||||
// Nothing committed, or no cell measured by the next pass: no evidence either way.
|
||||
PostRefineResult refused = fit;
|
||||
refused.cell_refined = false;
|
||||
CHECK(ReindexPushesCellBack(refused, Fit(Cell(96.7f, 107.6f, 112.6f), Cell(96.6f, 106.4f, 112.8f))) == 0);
|
||||
CHECK(ReindexPushesCellBack(fit, PostRefineResult{}) == 0);
|
||||
}
|
||||
@@ -628,6 +628,13 @@ TEST_CASE("ResultReport_FittedResolutionSuppressed", "[Diagnostics]") {
|
||||
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nFITTED_RESOLUTION= 0.95\n")
|
||||
!= std::string::npos);
|
||||
|
||||
// A shell's CC1/2 is its average, so the crossing lies inside the first shell below the target:
|
||||
// a fit there is what the table shows, not something it refutes.
|
||||
result.resolution_fit_A = 0.85;
|
||||
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nFITTED_RESOLUTION= 0.85\n")
|
||||
!= std::string::npos);
|
||||
result.resolution_fit_A = 0.95;
|
||||
|
||||
// CC1/2 that drops through the target and climbs back above it is not a fall-off either, and the
|
||||
// finest shell reaching the target is then noise correlating with noise - so a fit the test above
|
||||
// accepts must still be suppressed, and the user told so in the default report.
|
||||
@@ -644,6 +651,30 @@ TEST_CASE("ResultReport_FittedResolutionSuppressed", "[Diagnostics]") {
|
||||
CHECK(climbs_text.find("RESOLUTION_FIT") != std::string::npos);
|
||||
}
|
||||
|
||||
// ISa of a b that is zero within its error is the noise in b; the report says undetermined, not a number.
|
||||
TEST_CASE("ResultReport_IsaUndetermined", "[Diagnostics]") {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
ProcessResult result;
|
||||
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
|
||||
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
|
||||
result.has_merge_statistics = true;
|
||||
result.merge_statistics.overall.d_max = 50.0f;
|
||||
result.merge_statistics.overall.d_min = 1.50f;
|
||||
result.merge_statistics.overall.cc_half = 0.99;
|
||||
result.error_model_a = 1.1;
|
||||
result.error_model_b = 1e-4;
|
||||
result.error_model_isa = 95.0;
|
||||
result.error_model_isa_asymptotic = 120.0;
|
||||
|
||||
const auto resolved = RenderResultReport("p", "in.h5", x, result);
|
||||
CHECK(resolved.find("\nISA= 95.00\n") != std::string::npos);
|
||||
|
||||
result.error_model_isa_resolved = false;
|
||||
const auto text = RenderResultReport("p", "in.h5", x, result);
|
||||
CHECK(text.find("\nISA= undetermined\n") != std::string::npos);
|
||||
CHECK(text.find("ISA_ASYMPTOTIC") == std::string::npos);
|
||||
}
|
||||
|
||||
// The completeness of the range where the signal is, beside the completeness of the range that was
|
||||
// written. Users compare the single overall figure against another program's and conclude the run
|
||||
// lost data, when it is the deliberately generous cut diluting the denominator - so the report says
|
||||
@@ -762,3 +793,22 @@ TEST_CASE("ResultReport_TranslationalNCS", "[Diagnostics]") {
|
||||
CHECK(lat.find("\nPATHOLOGY_FLAGS= LATTICE_TRANSLATION\n") != std::string::npos);
|
||||
CHECK(lat.find("\nTNCS_SUBLATTICE=") == std::string::npos);
|
||||
}
|
||||
|
||||
TEST_CASE("ResultReport_GeometryNotConverged", "[Diagnostics]") {
|
||||
DiffractionExperiment x(DetJF(1));
|
||||
ProcessResult result;
|
||||
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
|
||||
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
|
||||
result.pass_number = 10;
|
||||
result.pass_count = 10;
|
||||
result.pass_decision = "post-refined geometry walked 8 rounds and kept round 8";
|
||||
|
||||
CHECK(RenderResultReport("p", "in.h5", x, result).find("GEOMETRY_NOT_CONVERGED") == std::string::npos);
|
||||
|
||||
result.geometry_not_converged = true;
|
||||
const auto text = RenderResultReport("p", "in.h5", x, result);
|
||||
const auto flags = text.find("\nPATHOLOGY_FLAGS= ");
|
||||
REQUIRE(flags != std::string::npos);
|
||||
CHECK(text.substr(flags, text.find('\n', flags + 1) - flags).find("GEOMETRY_NOT_CONVERGED")
|
||||
!= std::string::npos);
|
||||
}
|
||||
@@ -182,3 +182,35 @@ TEST_CASE("R-free fraction of zero flags nothing", "[rfree]") {
|
||||
AssignRfreeFlags(v, SG(1), 0.0);
|
||||
for (const auto& r : v) CHECK(!r.rfree_flag);
|
||||
}
|
||||
|
||||
TEST_CASE("R-free flags keyed on the lattice holohedry keep twin mates together", "[rfree]") {
|
||||
// A P4 crystal on a tetragonal cell: the lattice is 4/mmm, and (h,k,l) -> (k,h,-l) is a twin law.
|
||||
// Keyed on the holohedry, a reflection and its twin mate share a flag, and the P1 merge of the same
|
||||
// crystal flags every reflection exactly as the P4 merge does. Synthetic cell.
|
||||
const std::optional<UnitCell> cell = UnitCell{47.0f, 47.0f, 63.0f, 90.0f, 90.0f, 90.0f};
|
||||
std::vector<MergedReflection> p4, p1;
|
||||
for (int h = -10; h <= 10; ++h)
|
||||
for (int k = -10; k <= 10; ++k)
|
||||
for (int l = -10; l <= 10; ++l) {
|
||||
if (h == 0 && k == 0 && l == 0) continue;
|
||||
const float d = 60.0f / (1 + h * h + k * k + l * l);
|
||||
p1.push_back(Refl(h, k, l, d));
|
||||
p4.push_back(Refl(h, k, l, d));
|
||||
}
|
||||
AssignRfreeFlags(p4, SG(75), 0.05, /*min_free=*/0, cell);
|
||||
AssignRfreeFlags(p1, SG(1), 0.05, /*min_free=*/0, cell);
|
||||
|
||||
std::map<std::tuple<int, int, int>, bool> flag;
|
||||
for (const auto& r : p4) flag[{r.h, r.k, r.l}] = r.rfree_flag;
|
||||
int twin_pairs = 0;
|
||||
for (const auto& r : p4) {
|
||||
CHECK(flag[{r.k, r.h, -r.l}] == r.rfree_flag); // twin mate
|
||||
CHECK(flag[{-r.k, r.h, r.l}] == r.rfree_flag); // P4 mate
|
||||
++twin_pairs;
|
||||
}
|
||||
CHECK(twin_pairs > 0);
|
||||
for (const auto& r : p1)
|
||||
CHECK(flag[{r.h, r.k, r.l}] == r.rfree_flag); // one free set whatever the merging group
|
||||
CHECK(FreeFraction(p4) > 0.03);
|
||||
CHECK(FreeFraction(p4) < 0.08);
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <map>
|
||||
#include <tuple>
|
||||
#include <numbers>
|
||||
#include <vector>
|
||||
|
||||
@@ -118,17 +120,171 @@ TEST_CASE("A pseudo-translation can hide a twin, and declaring it restores the c
|
||||
CHECK(declared.twinning_suspected); // and restored
|
||||
}
|
||||
|
||||
// Standing behaviour that had no test at all. A holohedral Laue class admits no merohedral twin law,
|
||||
// so whatever the intensity statistics do there they must not be reported as twinning.
|
||||
TEST_CASE("No twin is called in a holohedral Laue class", "[twinning]") {
|
||||
namespace {
|
||||
// The merge of a synthetic crystal in `group`: one row per reflection of that group's asymmetric
|
||||
// unit, the average of the P1 intensities of its orbit - which is what merging under the group
|
||||
// does, whether or not its operators are real.
|
||||
std::vector<MergedReflection> MergedIn(const std::vector<MergedReflection> &p1, const char *group) {
|
||||
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name(group);
|
||||
const gemmi::GroupOps gops = sg->operations();
|
||||
const gemmi::ReciprocalAsu asu(sg);
|
||||
std::map<std::array<int, 3>, std::pair<MergedReflection, int>> sum;
|
||||
for (const auto &r : p1) {
|
||||
const auto key = asu.to_asu(gemmi::Op::Miller{{r.h, r.k, r.l}}, gops).first;
|
||||
auto [it, fresh] = sum.try_emplace({key[0], key[1], key[2]}, r, 0);
|
||||
if (!fresh)
|
||||
it->second.first.I += r.I;
|
||||
it->second.second += 1;
|
||||
}
|
||||
std::vector<MergedReflection> out;
|
||||
for (auto &[key, v] : sum) {
|
||||
MergedReflection r = v.first;
|
||||
r.h = key[0];
|
||||
r.k = key[1];
|
||||
r.l = key[2];
|
||||
r.I /= static_cast<float>(v.second);
|
||||
out.push_back(r);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<MergedReflection> Tetragonal(const char *true_group, double twin_fraction) {
|
||||
jfjoch_test::SyntheticMergeParams p;
|
||||
p.true_space_group = true_group;
|
||||
p.twin_supergroup = "P 4 2 2";
|
||||
p.twin_fraction = twin_fraction;
|
||||
p.d_min_A = 2.5;
|
||||
return jfjoch_test::GenerateSyntheticMerged(p);
|
||||
}
|
||||
}
|
||||
|
||||
// The L-test compares two intensities as if they had the same expected value, and two index steps are
|
||||
// not the same resolution: on a small cell with a steep fall-off the raw pair differs systematically,
|
||||
// which reads as an untwinned crystal being "more untwinned" than 0.5. The shell normalisation removes
|
||||
// that, and the fraction quoted is the L-test's own.
|
||||
TEST_CASE("L-test on shell-normalised intensities", "[twinning]") {
|
||||
jfjoch_test::SyntheticMergeParams p;
|
||||
p.true_space_group = "P 4 21 2";
|
||||
p.twin_supergroup = "P 4 21 2";
|
||||
p.true_space_group = "P 1";
|
||||
p.twin_supergroup = "P 1";
|
||||
p.wilson_b_A2 = 60.0;
|
||||
p.d_min_A = 2.5;
|
||||
const auto r = AnalyzeTwinning(jfjoch_test::GenerateSyntheticMerged(p), nullptr);
|
||||
INFO("<|L|> " << r.mean_abs_l);
|
||||
CHECK(r.mean_abs_l == Catch::Approx(0.5).margin(0.02));
|
||||
CHECK_FALSE(r.twinning_suspected);
|
||||
|
||||
const auto twin = AnalyzeTwinning(Crystal(0.2), nullptr);
|
||||
CHECK(twin.twin_fraction_source == TwinFractionSource::LTest);
|
||||
CHECK(twin.estimated_twin_fraction == Catch::Approx(0.2).margin(0.05));
|
||||
}
|
||||
|
||||
// A P1 merge of a centred lattice holds the reflections the centring extinguishes. Told the centring,
|
||||
// the test leaves them out and reads the crystal; not told, it pairs present with absent ones.
|
||||
TEST_CASE("L-test on a P1 merge of a centred lattice", "[twinning]") {
|
||||
jfjoch_test::SyntheticMergeParams p;
|
||||
p.true_space_group = "R 3 :H";
|
||||
p.twin_supergroup = "R 3 :H";
|
||||
p.d_min_A = 3.0;
|
||||
const auto merged = jfjoch_test::GenerateSyntheticMerged(p);
|
||||
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 4 21 2");
|
||||
REQUIRE(sg != nullptr);
|
||||
const auto r = AnalyzeTwinning(merged, sg);
|
||||
const auto told = AnalyzeTwinning(merged, nullptr, 20, nullptr, 'R');
|
||||
CHECK(told.mean_abs_l == Catch::Approx(0.5).margin(0.02));
|
||||
const auto not_told = AnalyzeTwinning(merged, nullptr);
|
||||
CHECK(std::fabs(not_told.mean_abs_l - 0.5) > 0.05);
|
||||
}
|
||||
|
||||
// A genuine holohedral crystal: no twin law exists, and its operators read as real.
|
||||
TEST_CASE("No twin is called in a genuine holohedral Laue class", "[twinning]") {
|
||||
const auto merged = MergedIn(Tetragonal("P 4 2 2", 0.0), "P 4 2 2");
|
||||
const auto r = AnalyzeTwinning(merged, gemmi::find_spacegroup_by_name("P 4 2 2"));
|
||||
CHECK_FALSE(r.merohedral_twinning_possible);
|
||||
CHECK(r.mean_abs_l == Catch::Approx(0.5).margin(0.03));
|
||||
CHECK_FALSE(r.twinning_suspected);
|
||||
CHECK_FALSE(r.adopted_operators_suspect);
|
||||
}
|
||||
|
||||
// Merging under a false operator averages I(h) with I(Th), and (I(h) + I(Th))/2 has the perfect-twin
|
||||
// distribution whatever the twin fraction - so a subgroup crystal merged in its lattice's holohedry
|
||||
// reads <|L|> ~0.375 at every fraction, and the holohedral class must say so rather than "no twin
|
||||
// law exists".
|
||||
TEST_CASE("A holohedral merge under a false operator is called suspect", "[twinning]") {
|
||||
const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 4 2 2");
|
||||
for (double alpha : {0.0, 0.2, 0.5}) {
|
||||
const auto r = AnalyzeTwinning(MergedIn(Tetragonal("P 4", alpha), "P 4 2 2"), sg);
|
||||
CAPTURE(alpha);
|
||||
CHECK(r.mean_abs_l == Catch::Approx(0.375).margin(0.02));
|
||||
CHECK(r.adopted_operators_suspect);
|
||||
CHECK(r.twin_fraction_source == TwinFractionSource::None); // lost in the merge
|
||||
CHECK(TwinningVerdictLine(r).rfind("SYMMETRY SUSPECT", 0) == 0);
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
// A P1 merge whose centric reflections are centric: SyntheticMergedReflections draws every
|
||||
// intensity from the acentric distribution, which is all the searches it was written for need, but
|
||||
// the zone test reads exactly the difference. E^2 is chi^2 with one degree of freedom for a
|
||||
// reflection centric in the true group (a squared Box-Muller normal), exponential otherwise, keyed
|
||||
// on the asymmetric-unit index so the true group's symmetry is exact.
|
||||
std::vector<MergedReflection> WilsonTetragonal(const char *true_group, double twin_fraction) {
|
||||
const gemmi::SpaceGroup &sub = gemmi::get_spacegroup_by_name(true_group);
|
||||
const gemmi::SpaceGroup &super = gemmi::get_spacegroup_by_name("P 4 2 2");
|
||||
const gemmi::Op twin = jfjoch_test::TwinLaw(sub, super);
|
||||
const gemmi::GroupOps gops = sub.operations();
|
||||
const gemmi::ReciprocalAsu rasu(&sub);
|
||||
const gemmi::UnitCell cell(47, 47, 63, 90, 90, 90);
|
||||
auto true_intensity = [&](const gemmi::Op::Miller &hkl) {
|
||||
const auto asu = rasu.to_asu(hkl, gops).first;
|
||||
const double u1 = jfjoch_test::detail::UniformFromHkl(asu);
|
||||
const double u2 = jfjoch_test::detail::UniformFromHkl({{asu[0] + 300, asu[1] + 300, asu[2] + 300}});
|
||||
const double c = std::cos(2.0 * std::numbers::pi * u2);
|
||||
const double e2 = gops.is_reflection_centric(hkl) ? -2.0 * std::log(u1) * c * c : -std::log(u1);
|
||||
return 1000.0 * e2 * std::exp(-20.0 / (2.0 * std::pow(cell.calculate_d(hkl), 2)));
|
||||
};
|
||||
std::vector<MergedReflection> out;
|
||||
for (int h = -19; h <= 19; ++h)
|
||||
for (int k = -19; k <= 19; ++k)
|
||||
for (int l = -26; l <= 26; ++l) {
|
||||
if (std::make_tuple(h, k, l) <= std::make_tuple(-h, -k, -l))
|
||||
continue;
|
||||
const gemmi::Op::Miller hkl{{h, k, l}};
|
||||
const double d = cell.calculate_d(hkl);
|
||||
if (d < 2.5)
|
||||
continue;
|
||||
MergedReflection r;
|
||||
r.h = h;
|
||||
r.k = k;
|
||||
r.l = l;
|
||||
r.d = static_cast<float>(d);
|
||||
r.I = static_cast<float>((1.0 - twin_fraction) * true_intensity(hkl)
|
||||
+ twin_fraction * true_intensity(twin.apply_to_hkl(hkl)));
|
||||
r.sigma = static_cast<float>(0.01 * r.I + 1.0);
|
||||
out.push_back(r);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
// Reflections centric in the adopted group but acentric in a subgroup are their own twin mates under
|
||||
// the operators the group adds. A perfect twin makes every operator statistic read "real", but these
|
||||
// stay acentric under it and are centric only where the added operators are real.
|
||||
TEST_CASE("Twin-immune zones tell a perfect twin from real symmetry", "[twinning]") {
|
||||
const gemmi::SpaceGroup &p422 = gemmi::get_spacegroup_by_name("P 4 2 2");
|
||||
|
||||
const auto real = AnalyzeTwinImmuneZones(WilsonTetragonal("P 4 2 2", 0.0), p422);
|
||||
REQUIRE(real.zones.size() == 3); // over 4, over 222, over the diagonal 222
|
||||
for (const auto &z : real.zones) {
|
||||
CAPTURE(z.operators);
|
||||
CHECK(z.mean_abs_e2_minus_1 == Catch::Approx(0.968).margin(0.08));
|
||||
CHECK(z.evidence_nats > 0.0);
|
||||
}
|
||||
CHECK(real.control.mean_abs_e2_minus_1 == Catch::Approx(0.736).margin(0.05));
|
||||
|
||||
// A perfect P4 twin: whichever subgroup is taken as the parent, the operators added over it are
|
||||
// not all real, and the zone reads acentric.
|
||||
const auto twin = AnalyzeTwinImmuneZones(WilsonTetragonal("P 4", 0.5), p422);
|
||||
REQUIRE(twin.zones.size() == 3);
|
||||
for (const auto &z : twin.zones) {
|
||||
CAPTURE(z.operators);
|
||||
CHECK(z.mean_abs_e2_minus_1 == Catch::Approx(0.736).margin(0.08));
|
||||
CHECK(z.evidence_nats < 0.0);
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <complex>
|
||||
|
||||
#include "JFJochDatasetInfoChartView.h"
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
|
||||
namespace {
|
||||
// Qt Charts draws its curves 2 px wide, which is hard to follow across a desk. Take the weight
|
||||
@@ -966,18 +967,25 @@ void JFJochDatasetInfoChartView::buildFFTChart() {
|
||||
|
||||
std::vector<std::complex<float>> out(static_cast<size_t>(nComplex));
|
||||
|
||||
fftwf_plan plan = fftwf_plan_dft_r2c_1d(
|
||||
n,
|
||||
in.data(),
|
||||
reinterpret_cast<fftwf_complex *>(out.data()),
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_plan plan;
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_dft_r2c_1d(
|
||||
n,
|
||||
in.data(),
|
||||
reinterpret_cast<fftwf_complex *>(out.data()),
|
||||
FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
if (!plan) {
|
||||
return;
|
||||
}
|
||||
|
||||
fftwf_execute(plan);
|
||||
fftwf_destroy_plan(plan);
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
}
|
||||
|
||||
// Compute amplitude spectrum and frequencies (0 .. Nyquist)
|
||||
m_fftMagnitudes.resize(static_cast<size_t>(nComplex));
|
||||
|
||||
@@ -165,11 +165,7 @@ JFJochMergeStatsWindow::JFJochMergeStatsWindow(const QString &title, const Merge
|
||||
QString text = QString("Twinning: ⟨|L|⟩ = %1 ⟨I²⟩/⟨I⟩² = %2")
|
||||
.arg(twinning.mean_abs_l, 0, 'f', 3)
|
||||
.arg(twinning.second_moment, 0, 'f', 2);
|
||||
if (twinning.twinning_suspected)
|
||||
text += QString(" — twinning suspected (estimated fraction ≈ %1)")
|
||||
.arg(twinning.estimated_twin_fraction, 0, 'f', 2);
|
||||
else
|
||||
text += " — no twinning indicated";
|
||||
text += " — " + QString::fromStdString(TwinningVerdictLine(twinning));
|
||||
|
||||
auto *banner = new QLabel(text, this);
|
||||
banner->setContentsMargins(10, 6, 10, 6);
|
||||
|
||||
Reference in new issue
Block a user