mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 84 Wiki Activity

jfjoch_writer writes rugnux's .mtz, _maps.mtz and _model.cif from the END blocks

Browse Source 
New gemmi-only library JFJochRugnuxFiles (writer/), linked by JFJochWriter, so
the writer needs no analysis library and builds with JFJOCH_WRITER_ONLY:
- WriteMergedMtz / WriteMapCoefficientsMtz / PlacedModelStructure make the
  files from the structured blocks; WriteRugnuxResultFiles writes whichever
  blocks END carries beside the master, each under a temporary name and renamed
  into place, an existing file replaced only when the run's overwrite is set.
  A file that fails costs only itself (logged), not the run.
- FillRugnuxResultBlocks reads the blocks back from the files rugnux wrote at
  a prefix (p.mtz, p_maps.mtz, p_model.cif), for the broker to fill END.

WriteMtzReflections now goes through WriteMergedMtz (MergedReflectionsForWriter
builds the block); its output is byte-identical to before, checked by md5 on a
synthetic set in both the Friedel-merged and the anomalous layout.

StreamWriter writes the files after the master and before the END ACK, so the
ACK means they are in place; END receipt and each rename are logged at INFO
with the wall-clock time, for last-frame-to-files latency. The broker's direct
HDF5 pusher writes them too.

Test: rugnux's three files -> blocks -> CBOR -> writer reproduces all of them
(and _model.pdb) byte for byte; a ~25k-reflection END is 2.7 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-17 12:19:51 +02:00
1 parent 91f6563953
commit aca27be862
14 files changed
+706 -85
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

image_analysis/WriteReflections.cpp
+44 -75
View File
@@ -7,6 +7,7 @@
#include "scale_merge/TwinningAnalysis.h"
#include "bragg_integration/SystematicAbsence.h"
#include "../common/ParallelFor.h"
#include "../writer/WriteRugnuxFiles.h"
#include <algorithm>
#include <cmath>
@@ -121,16 +122,16 @@ std::string GitDescription() {
// (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each
// continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is
// never cut just after a space: the space starts the next one, behind its '+'.
void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) {
const auto add = [&mtz](const std::string &label, const std::string &value) {
void AddProvenanceHistory(std::vector<std::string> &history, const OutputProvenance &provenance) {
const auto add = [&history](const std::string &label, const std::string &value) {
if (value.empty())
return;
std::string rest = label + value;
while (!rest.empty() && mtz.history.size() < 30) {
while (!rest.empty() && history.size() < 30) {
size_t n = std::min<size_t>(80, rest.size());
while (n > 1 && n < rest.size() && rest[n - 1] == ' ')
--n;
mtz.history.push_back(rest.substr(0, n));
history.push_back(rest.substr(0, n));
rest = n < rest.size() ? "+" + rest.substr(n) : std::string();
}
};
@@ -606,81 +607,49 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
out.close();
}
MergedReflectionsMessage MergedReflectionsForWriter(const std::vector<MergedReflection> &reflections,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
const OutputProvenance &provenance) {
MergedReflectionsMessage m;
m.unit_cell = unitCell;
m.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
m.wavelength_A = experiment.GetWavelength_A();
m.crystal_name = experiment.GetSampleName();
m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
AddProvenanceHistory(m.history, provenance);
bool has_anom = true;
const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
for (const auto& row : out_rows) {
m.h.push_back(row.h);
m.k.push_back(row.k);
m.l.push_back(row.l);
m.i_mean.push_back(row.Imean);
m.sigma_i_mean.push_back(row.sImean);
m.f.push_back(row.Fmean);
m.sigma_f.push_back(row.sFmean);
if (has_anom) {
m.i_plus.push_back(row.Ip);
m.sigma_i_plus.push_back(row.sIp);
m.i_minus.push_back(row.Im);
m.sigma_i_minus.push_back(row.sIm);
m.f_plus.push_back(row.Fp);
m.sigma_f_plus.push_back(row.sFp);
m.f_minus.push_back(row.Fm);
m.sigma_f_minus.push_back(row.sFm);
}
m.free_flag.push_back(static_cast<uint8_t>(row.rfree));
}
return m;
}
void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
const OutputProvenance &provenance,
const std::string &filename) {
gemmi::Mtz mtz;
// Optional but recommended metadata
mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
mtz.set_cell_for_all(unitCell);
// Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention).
mtz.title = "Rugnux merged reflections";
mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
AddProvenanceHistory(mtz, provenance);
// Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the
// reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is
// dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that
// takes f'/f'' from the file would phase on.
mtz.add_base(); // the HKL_base dataset and the H K L columns
gemmi::Mtz::Dataset& ds = mtz.add_dataset("native");
ds.crystal_name = experiment.GetSampleName();
ds.wavelength = experiment.GetWavelength_A();
const int dataset_id = ds.id;
// One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for
// the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly.
bool has_anom = true;
const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
mtz.add_column("IMEAN", 'J', dataset_id, -1, false);
mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false);
if (has_anom) {
mtz.add_column("I(+)", 'K', dataset_id, -1, false);
mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false);
mtz.add_column("I(-)", 'K', dataset_id, -1, false);
mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false);
}
mtz.add_column("F", 'F', dataset_id, -1, false); // French-Wilson amplitude
mtz.add_column("SIGF", 'Q', dataset_id, -1, false);
if (has_anom) {
mtz.add_column("F(+)", 'G', dataset_id, -1, false);
mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false);
mtz.add_column("F(-)", 'G', dataset_id, -1, false);
mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false);
}
mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false);
mtz.nreflections = static_cast<int>(out_rows.size());
mtz.data.reserve(out_rows.size() * 16);
for (const auto& row : out_rows) {
mtz.data.push_back(static_cast<float>(row.h));
mtz.data.push_back(static_cast<float>(row.k));
mtz.data.push_back(static_cast<float>(row.l));
mtz.data.push_back(row.Imean);
mtz.data.push_back(row.sImean);
if (has_anom) {
mtz.data.push_back(row.Ip);
mtz.data.push_back(row.sIp);
mtz.data.push_back(row.Im);
mtz.data.push_back(row.sIm);
}
mtz.data.push_back(row.Fmean);
mtz.data.push_back(row.sFmean);
if (has_anom) {
mtz.data.push_back(row.Fp);
mtz.data.push_back(row.sFp);
mtz.data.push_back(row.Fm);
mtz.data.push_back(row.sFm);
}
mtz.data.push_back(static_cast<float>(row.rfree));
}
mtz.write_to_file(filename);
WriteMergedMtz(MergedReflectionsForWriter(reflections, unitCell, experiment, provenance), filename);
}
namespace {
@@ -948,7 +917,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
mtz.set_cell_for_all(unitCell);
mtz.title = "Rugnux unmerged observations";
mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
AddProvenanceHistory(mtz, provenance);
AddProvenanceHistory(mtz.history, provenance);
mtz.add_base(); // the HKL_base dataset and the H K L columns
gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged");
ds.crystal_name = experiment.GetSampleName();