mx/Jungfraujoch
3
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 81 Wiki Activity

Merged mmCIF: deposition keys, R_merge/R_pim/CC*, experiment metadata, scaled unmerged block

Browse Source 
The .cif now follows the PDBx/mmCIF dictionary's keys: _entry.id, the
entry_id of _cell/_symmetry/_reflns, pdbx_ordinal and pdbx_diffrn_id on
_reflns and _reflns_shell, diffrn_id on _diffrn_source/_diffrn_detector/
_diffrn_radiation, and _diffrn_radiation_wavelength.id. _software is a
loop naming Rugnux for 'data reduction' and for 'data scaling'.

The experiment as the input states it: _diffrn.ambient_temp from the
sample temperature, the source's general class (from NXmx
/entry/source/type, which the HDF5 reader now reads), facility and
beamline, the detector make/model in _diffrn_detector.type (where
.detector used to carry it) and its class in .detector where the name
settles it, and the collection date from /entry/start_time.

R_merge and R_pim are formed beside R_meas from the same per-reflection
sums (stills and rotation merges) and written with CC* overall and per
shell (_reflns.pdbx_Rmerge_I_obs/pdbx_Rpim_I_all/pdbx_CC_star and the
_reflns_shell counterparts), as report keys R_MERGE/R_PIM/CC_STAR and as
three new columns of the shell tables. R_pim following Weiss (2001),
CC* following Karplus & Diederichs (2012).

A second data block, data_sample_unmerged, carries the scaled unmerged
observations of a rotation merge as _diffrn_refln (measured index,
frame, scaled I and sigma, pdbx_scale_value) - the same records as the
.hkl, which the wwPDB accepts as scaled unmerged data. ScaledFull now
carries the scale and the frame. The unmerged MTZ stays unscaled.

Reflection data are unchanged: the merged _refln rows, the .mtz columns
and the .hkl are written as before.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
This commit is contained in:
leonarski_f
2026-10-09 17:50:11 +02:00
co-authored by Claude Opus 5.5
parent 341c347a3f
commit fc6b91555f
16 changed files with 395 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image_analysis/WriteReflections.cpp
+188 -22
View File
@@ -10,6 +10,7 @@
#include <algorithm>
#include <cmath>
#include <cctype>
#include <cstring>
#include <numeric>
#include <cstdio>
@@ -68,6 +69,66 @@ std::string CifStr(const std::string& s) {
return s;
}
std::string Upper(std::string s) {
for (char &c : s)
c = static_cast<char>(std::toupper(static_cast<unsigned char>(c)));
return s;
}
// The general class of the detector, _diffrn_detector.detector in the PDBx enumeration, where its make
// and model name it; empty where they do not.
std::string DetectorClass(const std::string &description) {
const std::string d = Upper(description);
if (d.find("CCD") != std::string::npos)
return "CCD";
if (d.find("IMAGE PLATE") != std::string::npos)
return "IMAGE PLATE";
for (const char *pixel : {"EIGER", "PILATUS", "JUNGFRAU"})
if (d.find(pixel) != std::string::npos)
return "PIXEL";
return "";
}
// The general class of the source, _diffrn_source.source in the PDBx enumeration, from the NXmx
// /entry/source/type the file states; empty where it states none or one PDBx has no class for.
std::string SourceClass(const std::string &nxmx_type) {
const std::string t = Upper(nxmx_type);
if (t.find("SYNCHROTRON") != std::string::npos) return "SYNCHROTRON";
if (t.find("FREE-ELECTRON") != std::string::npos || t.find("FREE ELECTRON") != std::string::npos)
return "FREE ELECTRON LASER";
if (t.find("ROTATING ANODE") != std::string::npos) return "ROTATING ANODE";
if (t.find("FIXED TUBE") != std::string::npos) return "SEALED TUBE";
if (t.find("METAL JET") != std::string::npos) return "LIQUID ANODE";
return "";
}
// The date part of an ISO-8601 time stamp (YYYY-MM-DD...), which is all the PDBx collection date
// takes; empty for anything else.
std::string IsoDate(const std::string &timestamp) {
if (timestamp.size() < 10 || timestamp[4] != '-' || timestamp[7] != '-')
return "";
return timestamp.substr(0, 10);
}
// _entry, _cell and _symmetry: what every data block of the file restates, so each one stands alone.
void WriteBlockHeader(std::ostream &out, const std::string &entry_id, const UnitCell &unitCell,
const gemmi::SpaceGroup &sg) {
out << "_entry.id " << entry_id << "\n";
out << "#\n";
out << "_cell.entry_id " << entry_id << "\n";
out << "_cell.length_a " << Fmt(unitCell.a, 3) << "\n";
out << "_cell.length_b " << Fmt(unitCell.b, 3) << "\n";
out << "_cell.length_c " << Fmt(unitCell.c, 3) << "\n";
out << "_cell.angle_alpha " << Fmt(unitCell.alpha, 2) << "\n";
out << "_cell.angle_beta " << Fmt(unitCell.beta, 2) << "\n";
out << "_cell.angle_gamma " << Fmt(unitCell.gamma, 2) << "\n";
out << "#\n";
out << "_symmetry.entry_id " << entry_id << "\n";
out << "_symmetry.space_group_name_H-M " << CifStr(sg.hm) << "\n";
out << "_symmetry.Int_Tables_number " << sg.number << "\n";
out << "#\n";
}
// One output row per reflection in the standard CCP4 anomalous layout: the merged mean (IMEAN / F)
// plus the two Bijvoet mates (I(+)/I(-), F(+)/F(-)). The merge keeps the two mates as separate rows
// (I+ under the ASU representative hkl, I- under -hkl); this collapses them into one row so the MTZ
@@ -162,6 +223,8 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
const MergeStatistics &statistics,
const ErrorModelReport &error_model,
const TwinningAnalysisResult &twinning,
const std::vector<ScaledFull> &scaled_fulls,
const OutputProvenance &provenance,
const std::string &filename,
size_t nthreads) {
@@ -171,9 +234,16 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out << std::fixed;
// The keys PDBx asks for are written with the one value this file has for each: one entry, one
// crystal, one diffraction experiment (diffrn_id 1, which _pdbx_reflns_twin below also points at)
// and one wavelength. The merged data come first, as the wwPDB expects of a structure-factor file.
const std::string entry_id = "sample";
const gemmi::SpaceGroup &sg = experiment.GetSpaceGroupOrP1();
// ---------- data block ----------
out << "data_sample" << "\n";
out << "#\n";
WriteBlockHeader(out, entry_id, unitCell, sg);
// ---------- _audit ----------
out << "_audit.revision_id 1\n";
@@ -182,35 +252,54 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out << "#\n";
// ---------- _software ----------
out << "_software.name 'Rugnux'\n";
out << "_software.version " << CifStr(jfjoch_version()) << "\n";
out << "_software.classification 'data reduction'\n";
// Rugnux integrated the images (data reduction) and scaled and merged them (data scaling), the
// two rows the wwPDB asks a data-processing program to be named under.
out << "loop_\n";
out << "_software.pdbx_ordinal\n";
out << "_software.name\n";
out << "_software.version\n";
out << "_software.classification\n";
out << "1 Rugnux " << CifStr(jfjoch_version()) << " 'data reduction'\n";
out << "2 Rugnux " << CifStr(jfjoch_version()) << " 'data scaling'\n";
out << "#\n";
// ---------- _cell ----------
out << "_cell.length_a " << Fmt(unitCell.a, 3) << "\n";
out << "_cell.length_b " << Fmt(unitCell.b, 3) << "\n";
out << "_cell.length_c " << Fmt(unitCell.c, 3) << "\n";
out << "_cell.angle_alpha " << Fmt(unitCell.alpha, 2) << "\n";
out << "_cell.angle_beta " << Fmt(unitCell.beta, 2) << "\n";
out << "_cell.angle_gamma " << Fmt(unitCell.gamma, 2) << "\n";
const gemmi::SpaceGroup &sg = experiment.GetSpaceGroupOrP1();
// ---------- _symmetry ----------
out << "_symmetry.space_group_name_H-M " << CifStr(sg.hm) << "\n";
out << "_symmetry.Int_Tables_number " << sg.number << "\n";
// ---------- _exptl_crystal / _diffrn ----------
out << "_exptl_crystal.id 1\n";
out << "#\n";
out << "_diffrn.id 1\n";
out << "_diffrn.crystal_id 1\n";
if (const auto t = experiment.GetSampleTemperature_K())
out << "_diffrn.ambient_temp " << Fmt(*t, 1) << "\n";
out << "#\n";
// ---------- _diffrn_source / _diffrn_detector ----------
// ---------- _diffrn_radiation / _diffrn_source / _diffrn_detector ----------
out << "_diffrn_radiation.diffrn_id 1\n";
out << "_diffrn_radiation.wavelength_id 1\n";
out << "_diffrn_radiation.pdbx_diffrn_protocol 'SINGLE WAVELENGTH'\n";
out << "#\n";
out << "_diffrn_radiation_wavelength.id 1\n";
out << "_diffrn_radiation_wavelength.wavelength " << Fmt(experiment.GetWavelength_A(), 5) << "\n";
out << "#\n";
// The source and the detector as the input file names them: the facility and the beamline (NXmx
// /entry/source/name and /entry/instrument/name) and the detector's make and model. The general
// classes PDBx enumerates are written only where those names settle them.
out << "_diffrn_source.diffrn_id 1\n";
if (const std::string source = SourceClass(experiment.GetSourceType()); !source.empty())
out << "_diffrn_source.source " << CifStr(source) << "\n";
if (!experiment.GetSourceName().empty())
out << "_diffrn_source.pdbx_synchrotron_site " << CifStr(experiment.GetSourceName()) << "\n";
if (!experiment.GetInstrumentName().empty())
out << "_diffrn_source.pdbx_synchrotron_beamline " << CifStr(experiment.GetInstrumentName()) << "\n";
out << "_diffrn_source.pdbx_wavelength_list " << Fmt(experiment.GetWavelength_A(), 5) << "\n";
out << "#\n";
out << "_diffrn_radiation_wavelength.wavelength " << Fmt(experiment.GetWavelength_A(), 5) << "\n";
out << "_diffrn_detector.detector " << CifStr(experiment.GetDetectorDescription()) << "\n";
out << "_diffrn_detector.diffrn_id 1\n";
if (const std::string cls = DetectorClass(experiment.GetDetectorDescription()); !cls.empty())
out << "_diffrn_detector.detector " << CifStr(cls) << "\n";
out << "_diffrn_detector.type " << CifStr(experiment.GetDetectorDescription()) << "\n";
if (const std::string date = IsoDate(provenance.collection_date); !date.empty())
out << "_diffrn_detector.pdbx_collection_date " << date << "\n";
// Detector geometry actually used for integration (refined, when geometry refinement ran - the rotation
// two-pass or the stills global refinement update it on experiment_ before the written pass). jfjoch_
// local-data-name items: feedback of the distance / beam centre the data was reduced with.
@@ -236,6 +325,9 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
// Anomalous signal-to-noise (SigAno) is written only when an anomalous split was made, so a
// non-anomalous merge keeps its previous stats block / shell-loop columns unchanged.
const bool has_anom = std::isfinite(ov.abs_diff_over_sigma_anomalous);
out << "_reflns.entry_id " << entry_id << "\n";
out << "_reflns.pdbx_ordinal 1\n";
out << "_reflns.pdbx_diffrn_id 1\n";
out << "_reflns.d_resolution_high " << Fmt(ov.d_min, 2) << "\n";
out << "_reflns.d_resolution_low " << Fmt(ov.d_max, 2) << "\n";
out << "_reflns.number_obs " << ov.unique_reflections << "\n";
@@ -243,8 +335,11 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out << "_reflns.pdbx_redundancy " << Fmt(mult(ov), 2) << "\n";
out << "_reflns.percent_possible_obs " << Fmt(compl_pct(ov), 1) << "\n";
out << "_reflns.pdbx_netI_over_sigmaI " << Fmt(ov.mean_i_over_sigma, 2) << "\n";
out << "_reflns.pdbx_Rmerge_I_obs " << Fmt(ov.r_merge, 4) << "\n";
out << "_reflns.pdbx_Rrim_I_all " << Fmt(ov.r_meas, 4) << "\n";
out << "_reflns.pdbx_Rpim_I_all " << Fmt(ov.r_pim, 4) << "\n";
out << "_reflns.pdbx_CC_half " << Fmt(ov.cc_half, 4) << "\n";
out << "_reflns.pdbx_CC_star " << Fmt(CCStar(ov.cc_half), 4) << "\n";
if (has_anom)
out << "_reflns.pdbx_absDiff_over_sigma_anomalous " << Fmt(ov.abs_diff_over_sigma_anomalous, 3)
<< " # SigAno = <|dano|>/<sigma(dano)>\n";
@@ -360,6 +455,8 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
}
out << "loop_\n";
out << "_reflns_shell.pdbx_ordinal\n";
out << "_reflns_shell.pdbx_diffrn_id\n";
out << "_reflns_shell.d_res_high\n";
out << "_reflns_shell.d_res_low\n";
out << "_reflns_shell.number_measured_obs\n";
@@ -367,17 +464,23 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out << "_reflns_shell.pdbx_redundancy\n";
out << "_reflns_shell.percent_possible_obs\n";
out << "_reflns_shell.meanI_over_sigI_obs\n";
out << "_reflns_shell.Rmerge_I_obs\n";
out << "_reflns_shell.pdbx_Rrim_I_all\n";
out << "_reflns_shell.pdbx_Rpim_I_all\n";
out << "_reflns_shell.pdbx_CC_half\n";
out << "_reflns_shell.pdbx_CC_star\n";
if (has_anom)
out << "_reflns_shell.pdbx_absDiff_over_sigma_anomalous\n";
int ordinal = 0;
for (const auto &s : statistics.shells) {
if (s.unique_reflections == 0)
continue;
out << Fmt(s.d_min, 2) << " " << Fmt(s.d_max, 2) << " "
out << ++ordinal << " 1 "
<< Fmt(s.d_min, 2) << " " << Fmt(s.d_max, 2) << " "
<< s.total_observations << " " << s.unique_reflections << " "
<< Fmt(mult(s), 2) << " " << Fmt(compl_pct(s), 1) << " "
<< Fmt(s.mean_i_over_sigma, 2) << " " << Fmt(s.r_meas, 4) << " " << Fmt(s.cc_half, 4);
<< Fmt(s.mean_i_over_sigma, 2) << " " << Fmt(s.r_merge, 4) << " " << Fmt(s.r_meas, 4) << " "
<< Fmt(s.r_pim, 4) << " " << Fmt(s.cc_half, 4) << " " << Fmt(CCStar(s.cc_half), 4);
if (has_anom)
out << " " << Fmt(s.abs_diff_over_sigma_anomalous, 3);
out << "\n";
@@ -454,6 +557,67 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out << "#\n";
out << "# End of reflections\n";
// ---------- the scaled unmerged observations, as a second data block ----------
// The observations the merge above averaged - each full with the scale and the error-model sigma
// the merge gave it, at the index it was measured at - which is what the wwPDB accepts as scaled
// unmerged data. The same records as the .hkl: an observation without a usable sigma is in
// neither. pdbx_image_id is the frame the full peaks on, numbered from 1 as the unmerged MTZ's
// BATCH is. (The unmerged MTZ itself stays unscaled, for the programs that fit their own scales.)
if (!scaled_fulls.empty()) {
out << "data_sample_unmerged\n";
out << "#\n";
WriteBlockHeader(out, entry_id, unitCell, sg);
out << "_exptl_crystal.id 1\n";
out << "#\n";
out << "_diffrn.id 1\n";
out << "_diffrn.crystal_id 1\n";
out << "#\n";
out << "_diffrn_radiation_wavelength.id 1\n";
out << "_diffrn_radiation_wavelength.wavelength " << Fmt(experiment.GetWavelength_A(), 5) << "\n";
out << "#\n";
out << "loop_\n";
out << "_diffrn_refln.diffrn_id\n";
out << "_diffrn_refln.id\n";
out << "_diffrn_refln.pdbx_image_id\n";
out << "_diffrn_refln.index_h\n";
out << "_diffrn_refln.index_k\n";
out << "_diffrn_refln.index_l\n";
out << "_diffrn_refln.intensity_net\n";
out << "_diffrn_refln.intensity_sigma\n";
out << "_diffrn_refln.pdbx_scale_value\n";
// Numbered in order over the usable fulls, so the id of each is known before the rows are
// formatted on several workers.
std::vector<int64_t> id(scaled_fulls.size() + 1, 0);
for (size_t i = 0; i < scaled_fulls.size(); ++i) {
const auto &f = scaled_fulls[i];
id[i + 1] = id[i] + ((std::isfinite(f.I) && std::isfinite(f.sigma) && f.sigma > 0.0f) ? 1 : 0);
}
const size_t nrow = scaled_fulls.size();
const size_t nw = std::max<size_t>(nthreads, 1);
const int nch = static_cast<int>(ThreadsForWork(nrow, nw, 4096));
std::vector<std::string> block(nch);
ParallelChunks(nch, nw, [&](int tlo, int thi) {
for (int t = tlo; t < thi; ++t) {
const size_t lo = nrow * t / nch, hi = nrow * (t + 1) / nch;
std::string &s = block[t];
s.reserve((hi - lo) * 64);
for (size_t i = lo; i < hi; ++i) {
if (id[i + 1] == id[i])
continue;
const auto &f = scaled_fulls[i];
const long frame = std::isfinite(f.image_number) ? 1 + std::lround(f.image_number) : 0;
s += "1 " + std::to_string(id[i + 1]) + " " + (frame > 0 ? std::to_string(frame) : "?") + " "
+ std::to_string(f.h) + " " + std::to_string(f.k) + " " + std::to_string(f.l) + " "
+ Fmt(f.I, 3) + " " + Fmt(f.sigma, 3) + " " + Fmt(f.scale, 5) + "\n";
}
}
});
for (const std::string &s : block)
out.write(s.data(), static_cast<std::streamsize>(s.size()));
out << "#\n";
}
out.close();
}
@@ -1072,6 +1236,7 @@ void WriteReflections(const std::vector<MergedReflection> &reflections,
const MergeStatistics &statistics,
const ErrorModelReport &error_model,
const TwinningAnalysisResult &twinning,
const OutputProvenance &provenance,
const std::string &filename,
size_t nthreads,
const std::vector<ScaledFull> *scaled_fulls) {
@@ -1081,6 +1246,7 @@ void WriteReflections(const std::vector<MergedReflection> &reflections,
// fulls where the merge provides them (rotation), the merged reflections otherwise (stills).
WriteMtzReflections(reflections, unitCell, experiment, filename + ".mtz");
WriteMmcifReflections(reflections, unitCell, experiment, statistics, error_model, twinning,
scaled_fulls ? *scaled_fulls : std::vector<ScaledFull>{}, provenance,
filename + ".cif", nthreads);
if (scaled_fulls && !scaled_fulls->empty())
WriteShelxHklReflections(*scaled_fulls, filename + ".hkl", nthreads);