mx/Jungfraujoch
3
2
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 81 Wiki Activity

rugnux --af-model: hand check reads only the anomalous map, on MR's placement and solvent

Browse Source 
The ENANTIOMORPH hand check ran a whole ValidateAgainstModel on the placed model: model read and frame
probe, a rigid body, a 9-replicate null (~0.5 s of the ~0.6-0.9 s), sigma_A, three maps written. The
decision needs only --model's rule: the model fits its null (the MR choice has already required its
own null of the same construction) and the anomalous map at the model's atoms is not inverted.

ReadModelAnomalousMap (ModelValidation) computes F_calc and F_mask to the data's resolution (GPU engine
where there is one), carries them through the bulk solvent MR scored the placement with (the overall
scale and anisotropic B do not move phases), and reads the anomalous map with the same code as the
validation: its coefficient construction and its reading at the atoms are factored out of Validate
(anomalous_coefficients, read_anomalous_map) and used by both, so plain --model is unchanged.

Hand check 0.6/0.6/0.9 s -> 0.027/0.028/0.040 s on 11if/7ris/9zmu; same ADOPTED/VETOED on all seven
cases tried (the three demos, two anomalous lysozyme sets and their mirrored-detector runs), with the
scatterer means and deepest troughs within 0.1 sigma of the full validation's.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
leonarski_f
2026-10-11 18:11:01 +02:00
co-authored by Claude Opus 5.5
parent 2a3d60fb82
commit 50e3b59fb7
7 changed files with 205 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image_analysis/structure_refinement/ModelValidation.cpp
+165 -75
View File
@@ -31,6 +31,7 @@
#include <gemmi/scaling.hpp> // Scaling (bulk solvent + anisotropic B)
#include <gemmi/ccp4.hpp> // Ccp4 map I/O
#include <gemmi/mtz.hpp> // Mtz (map-coefficient output)
#include <gemmi/stats.hpp> // calculate_data_statistics
#include "../../common/CorrelationCoefficient.h"
#include "../../common/JFJochMath.h" // PI (M_PI is not standard, and MSVC does not define it)
@@ -374,6 +375,93 @@ double frame_probe_r(const gemmi::Structure &st, const gemmi::SpaceGroup *sg,
return scaling.calculate_r_factor();
}
// The anomalous difference map's coefficients on the reflections of `fmodel`: F(+) - F(-) carried on the
// model phase turned back by 90 degrees.
//
// Read from the merged reflections as they came in, and carry each one into the model's frame here: the
// hand each Bijvoet difference belongs to is a property of the frame the merge was made in, and both
// operators can change it. F(+) and F(-) are attached to the + index of the Friedel ASU of that frame, so
// an anomalous merge - which keeps each mate as a row of its own, both carrying the same pair - is read on
// its + rows only. Taking the - rows as well would give one reflection both signs of its difference, and
// the last row written would decide.
gemmi::AsuData<std::complex<float>> anomalous_coefficients(const std::vector<MergedReflection> &merged,
const gemmi::SpaceGroup *data_sg,
const gemmi::Op &indexing_op,
const gemmi::AsuData<std::complex<float>> &fmodel) {
const gemmi::ReciprocalAsu data_asu(data_sg);
const gemmi::GroupOps data_gops = data_sg->operations();
const gemmi::ReciprocalAsu asu(fmodel.spacegroup_);
const gemmi::GroupOps gops = fmodel.spacegroup_->operations();
std::unordered_map<long, float> danom_by_hkl;
for (const MergedReflection &r : merged) {
if (!std::isfinite(r.F_plus) || !std::isfinite(r.F_minus))
continue;
gemmi::Op::Miller h{{r.h, r.k, r.l}};
if (data_gops.is_reflection_centric(h)) // a centric reflection has no anomalous difference
continue;
if (!data_asu.to_asu_sign(h, data_gops).second)
continue;
if (!(indexing_op == gemmi::Op::identity()))
h = indexing_op.apply_to_hkl(h);
const auto [hasu, plus] = asu.to_asu_sign(h, gops);
danom_by_hkl[hkl_key(hasu)] = plus ? r.F_plus - r.F_minus : r.F_minus - r.F_plus;
}
gemmi::AsuData<std::complex<float>> mapanom;
mapanom.unit_cell_ = fmodel.unit_cell_;
mapanom.spacegroup_ = fmodel.spacegroup_;
for (const auto &hv : fmodel.v) {
const auto it = danom_by_hkl.find(hkl_key(hv.hkl));
if (it == danom_by_hkl.end())
continue;
const auto phi = static_cast<float>(std::arg(hv.value) - PI / 2);
mapanom.v.push_back({hv.hkl, it->second * std::polar(1.0f, phi)});
}
return mapanom;
}
// The anomalous map read at every atom of `st`, in units of its RMS: the strongest sites, the mean at the
// anomalous scatterers, and whether the map is inverted at the model's atoms.
void read_anomalous_map(const gemmi::Grid<float> &grid, double rms, gemmi::Structure &st,
ModelValidationResult &result) {
std::vector<ModelValidationResult::AnomalousSite> sites;
double scatterer_sum = 0;
for (gemmi::Model &m : st.models)
for (gemmi::Chain &ch : m.chains)
for (gemmi::Residue &r : ch.residues)
for (gemmi::Atom &a : r.atoms) {
if (a.is_hydrogen()) // hydrogen scatters no anomalous signal
continue;
sites.push_back({fmt::format("{} {} {}{}", a.name, r.name, ch.name, r.seqid.str()),
rms > 0 ? grid.interpolate_value(a.pos, MAP_INTERPOLATION_ORDER) / rms : 0.0});
if (a.element.atomic_number() >= ANOMALOUS_SCATTERER_MIN_Z) {
scatterer_sum += sites.back().sigma;
result.anomalous_scatterers++;
}
}
if (result.anomalous_scatterers > 0)
result.anomalous_scatterer_mean_sigma = scatterer_sum / result.anomalous_scatterers;
std::sort(sites.begin(), sites.end(), [](const auto &x, const auto &y) { return x.sigma > y.sigma; });
// A model and a dataset in opposite hands turn every anomalous peak into a trough, so a map whose
// deepest hole at an atom is both deep and deeper than its highest peak says the two disagree about
// the hand. That is worth reporting: it is real evidence about the crystal, and the alternative -
// reindexing until the two agree - would erase it.
if (!sites.empty()) {
const auto &deepest = sites.back();
if (deepest.sigma < -ANOMALOUS_INVERSION_SIGMA && -deepest.sigma > sites.front().sigma) {
result.anomalous_hands_disagree = true;
result.anomalous_deepest_site = deepest.label;
result.anomalous_deepest_sigma = deepest.sigma;
}
}
if (sites.size() > MAX_ANOMALOUS_SITES)
sites.resize(MAX_ANOMALOUS_SITES);
result.anomalous_sites = std::move(sites);
}
#ifdef JFJOCH_USE_CUDA
// The model's structure factors to the data's resolution, and the maps, on the GPU - or not, decided here,
// once, before any of them is computed, and never revisited: nothing is moved to the CPU part way through.
@@ -1500,84 +1588,13 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
// rather than leaving a list of coordinates for someone to look up.
// Following ANODE, Thorn & Sheldrick (2011) J. Appl. Cryst. 44, 1285-1287
std::future<void> anomalous_map = std::async(std::launch::async, [&] {
// Read from the merged reflections as they came in, and carry each one into the model's frame
// here: the hand each Bijvoet difference belongs to is a property of the frame the merge was
// made in, and both operators can change it. F(+) and F(-) are attached to the + index of the
// Friedel ASU of that frame, so an anomalous merge - which keeps each mate as a row of its own,
// both carrying the same pair - is read on its + rows only. Taking the - rows as well would
// give one reflection both signs of its difference, and the last row written would decide.
const gemmi::SpaceGroup *data_sg = data_space_group != nullptr ? data_space_group : sg;
const gemmi::ReciprocalAsu data_asu(data_sg);
const gemmi::GroupOps data_gops = data_sg->operations();
std::unordered_map<long, float> danom_by_hkl;
for (const MergedReflection &r : merged) {
if (!std::isfinite(r.F_plus) || !std::isfinite(r.F_minus))
continue;
gemmi::Op::Miller h{{r.h, r.k, r.l}};
if (data_gops.is_reflection_centric(h)) // a centric reflection has no anomalous difference
continue;
if (!data_asu.to_asu_sign(h, data_gops).second)
continue;
if (!(result.indexing_op == gemmi::Op::identity()))
h = result.indexing_op.apply_to_hkl(h);
const auto [hasu, plus] = asu.to_asu_sign(h, gops);
danom_by_hkl[hkl_key(hasu)] = plus ? r.F_plus - r.F_minus : r.F_minus - r.F_plus;
}
gemmi::AsuData<std::complex<float>> mapanom;
mapanom.unit_cell_ = ucell;
mapanom.spacegroup_ = sg;
for (const auto &hv : fmodel.v) {
const auto it = danom_by_hkl.find(hkl_key(hv.hkl));
if (it == danom_by_hkl.end())
continue;
const auto phi = static_cast<float>(std::arg(hv.value) - PI / 2);
mapanom.v.push_back({hv.hkl, it->second * std::polar(1.0f, phi)});
}
gemmi::AsuData<std::complex<float>> mapanom =
anomalous_coefficients(merged, data_space_group != nullptr ? data_space_group : sg,
result.indexing_op, fmodel);
result.anomalous_pairs = static_cast<int>(mapanom.v.size());
if (!mapanom.v.empty()) {
const gemmi::Grid<float> grid = map_from_coefficients(mapanom, sf_gpu);
const double rms = write_ccp4(grid, output_prefix + "_anom.ccp4");
std::vector<ModelValidationResult::AnomalousSite> sites;
double scatterer_sum = 0;
for (gemmi::Model &m : st.models)
for (gemmi::Chain &ch : m.chains)
for (gemmi::Residue &r : ch.residues)
for (gemmi::Atom &a : r.atoms) {
if (a.is_hydrogen()) // hydrogen scatters no anomalous signal
continue;
sites.push_back({fmt::format("{} {} {}{}", a.name, r.name, ch.name,
r.seqid.str()),
rms > 0 ? grid.interpolate_value(a.pos, MAP_INTERPOLATION_ORDER) / rms
: 0.0});
if (a.element.atomic_number() >= ANOMALOUS_SCATTERER_MIN_Z) {
scatterer_sum += sites.back().sigma;
result.anomalous_scatterers++;
}
}
if (result.anomalous_scatterers > 0)
result.anomalous_scatterer_mean_sigma = scatterer_sum / result.anomalous_scatterers;
std::sort(sites.begin(), sites.end(),
[](const auto &x, const auto &y) { return x.sigma > y.sigma; });
// A model and a dataset in opposite hands turn every anomalous peak into a trough, so a
// map whose deepest hole at an atom is both deep and deeper than its highest peak says
// the two disagree about the hand. That is worth reporting: it is real evidence about
// the crystal, and the alternative - reindexing until the two agree - would erase it.
if (!sites.empty()) {
const auto &deepest = sites.back();
if (deepest.sigma < -ANOMALOUS_INVERSION_SIGMA && -deepest.sigma > sites.front().sigma) {
result.anomalous_hands_disagree = true;
result.anomalous_deepest_site = deepest.label;
result.anomalous_deepest_sigma = deepest.sigma;
}
}
if (sites.size() > MAX_ANOMALOUS_SITES)
sites.resize(MAX_ANOMALOUS_SITES);
result.anomalous_sites = std::move(sites);
read_anomalous_map(grid, write_ccp4(grid, output_prefix + "_anom.ccp4"), st, result);
}
});
@@ -1794,6 +1811,79 @@ const gemmi::SpaceGroup *AdoptModelFrame(const ModelValidationResult &validation
return space_group;
}
ModelValidationResult ReadModelAnomalousMap(const std::vector<MergedReflection> &merged,
const gemmi::Structure &placed, const gemmi::SpaceGroup &data_space_group,
double k_sol, double b_sol, Logger &logger, size_t nthreads) {
ModelValidationResult result;
gemmi::Structure st = placed;
st.setup_cell_images();
const gemmi::SpaceGroup *sg = st.find_spacegroup();
double d_min = 0.0;
for (const MergedReflection &r : merged)
if (r.d > 0 && (d_min == 0.0 || r.d < d_min))
d_min = r.d;
if (sg == nullptr || d_min <= 0.0) {
result.failure_reason = "no space group or no resolution";
return result;
}
// F_calc and F_mask to the data's resolution, as the validation computes them (compute_model_factors).
gemmi::AsuData<std::complex<float>> fcalc, fmask;
ModelStructureFactorsGPU *gpu = nullptr;
#ifdef JFJOCH_USE_CUDA
std::unique_ptr<ModelStructureFactorsGPU> engine;
if (get_gpu_count() > 0)
engine = StructureFactorsGPU(RigidBodyGPUEngine::CurrentDevice(), st.models[0], st.cell, *sg, d_min, 0, logger);
gpu = engine.get();
if (gpu != nullptr) {
try {
gpu->Compute(st.models[0], fcalc, fmask);
} catch (const JFJochException &e) {
throw RigidBodyGPUFailure(e.what());
}
}
#else
(void) logger;
#endif
if (gpu == nullptr) {
gemmi::DensityCalculator<Table, float> dc;
dc.d_min = d_min;
dc.rate = 1.5;
dc.set_grid_cell_and_spacegroup(st);
dc.set_refmac_compatible_blur(st.models[0]);
gemmi::Grid<float> mask_grid;
mask_grid.unit_cell = dc.grid.unit_cell;
mask_grid.spacegroup = dc.grid.spacegroup;
mask_grid.set_size_from_spacing(dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
const std::vector<size_t> orbit_leaders = OrbitLeaders(mask_grid, nthreads);
PutMaskOnGrid(mask_grid, st.models[0], orbit_leaders, nthreads);
fmask = MapToFPhi(mask_grid).prepare_asu_data(d_min, 0);
PutModelDensityOnGrid(dc, st.models[0], orbit_leaders, nthreads);
fcalc = MapToFPhi(dc.grid).prepare_asu_data(dc.d_min, dc.blur, false, false, false);
}
// With the bulk solvent the placement was scored with: the overall scale and the anisotropic B are
// real and positive and leave the phases where they are, so the solvent is all of fit_model's scaling
// that the map's phases see.
gemmi::AsuData<std::complex<float>> fmodel = fcalc;
gemmi::Scaling<float> scaling(st.cell, sg);
scaling.use_solvent = true;
scaling.k_sol = k_sol;
scaling.b_sol = b_sol;
scaling.scale_data(fmodel, &fmask);
gemmi::AsuData<std::complex<float>> mapanom =
anomalous_coefficients(merged, &data_space_group, gemmi::Op::identity(), fmodel);
result.anomalous_pairs = static_cast<int>(mapanom.v.size());
if (!mapanom.v.empty()) {
const gemmi::Grid<float> grid = map_from_coefficients(mapanom, gpu);
// The RMS write_ccp4 reads the map in.
read_anomalous_map(grid, gemmi::calculate_data_statistics(grid.data).rms, st, result);
}
result.ok = true;
return result;
}
void KeepModelVerdict(ModelValidationResult &remade, const ModelValidationResult &decided) {
remade.fit_tested = decided.fit_tested;
remade.model_fits = decided.model_fits;