From 91f656395392c38eeb942b6a64c18bdb8223aa5a Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 12:10:35 +0200 Subject: [PATCH 01/19] END message: optional merged / map_coefficients / placed_model blocks Structured results of an analysis run inside the broker, for jfjoch_writer to turn into .mtz, _maps.mtz and _model.cif itself: the merged reflections and the map coefficients one typed array per MTZ column, the placed model one array per atom field (coordinates as float64). Each block is a nested CBOR map under its own optional END key; a reader that does not know a key skips it, as for every other END key, and unknown keys inside a block are skipped the same way. Space groups travel as gemmi's extended H-M name, which round-trips to the same table entry for all 564 settings. Co-Authored-By: Claude Opus 5.5 --- common/JFJochMessages.h | 59 ++++++++++ docs/CBOR.md | 3 + frame_serialize/CBORStream2Deserializer.cpp | 118 ++++++++++++++++++++ frame_serialize/CBORStream2Serializer.cpp | 86 ++++++++++++++ 4 files changed, 266 insertions(+) diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index dd68afaee..bcb4dcd1e 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -368,6 +368,59 @@ struct StartMessage { std::optional smargon_position; }; +// The merged reflections as rugnux writes them to .mtz (WriteMergedMtz, +// writer/WriteRugnuxFiles.h): one row per unique reflection in the CCP4 anomalous layout, one vector +// per MTZ column. The anomalous columns (i_plus ... sigma_f_minus) are all empty where the merge kept +// no Bijvoet split, and the file then has none. free_flag is the FreeR_flag value, 0 for the test set. +struct MergedReflectionsMessage { + UnitCell unit_cell; + std::string space_group_name; // gemmi xhm(): the name keeps the setting, a number does not + float wavelength_A = 0; + std::string crystal_name; + std::vector history; // the MTZ HISTORY records after the one naming the program + std::vector h, k, l; + std::vector i_mean, sigma_i_mean; + std::vector i_plus, sigma_i_plus, i_minus, sigma_i_minus; + std::vector f, sigma_f; + std::vector f_plus, sigma_f_plus, f_minus, sigma_f_minus; + std::vector free_flag; +}; + +// The sigma_A-weighted map coefficients rugnux writes to _maps.mtz (WriteMapCoefficientsMtz), +// one vector per MTZ column. Phases in degrees, [0, 360). free_flag is FREE: 0 for the test set. +struct MapCoefficientsMessage { + UnitCell unit_cell; + std::string space_group_name; + float wavelength_A = 0; + std::vector h, k, l; + std::vector fp, fc, phic, fwt, phwt, delfwt, phdelwt, fom; + std::vector free_flag; +}; + +// The atomic model as placed against the data, one vector per atom field in file order, for +// WritePlacedModel to write _model.cif. A new residue starts wherever the model, chain or +// residue key changes. unit_cell and space_group_name are what WritePlacedModel is given (the data's); +// model_space_group_name is the group the coordinates were built in. +struct PlacedModelMessage { + std::string name; + UnitCell unit_cell; + std::string space_group_name; + std::string model_space_group_name; + std::vector model_number; + std::vector chain; + std::vector residue_name; + std::vector seq_num; + std::vector icode; // ' ' where there is none + std::vector het_flag; // 'A', 'H', or 0 where unknown + std::vector atom_name; + std::vector element; + std::vector altloc; // 0 where there is none + std::vector charge; + std::vector x, y, z; + std::vector b_iso, occupancy; + std::vector aniso; // 6 per atom (u11 u22 u33 u12 u13 u23), empty when none has one +}; + struct EndMessage { // The sample transformation chain, base first (see DetectorTransformation). Optional: when it is // empty the writer builds the same chain itself from the start message, so a producer that does @@ -481,6 +534,12 @@ struct EndMessage { // this says what was DONE about it. Offline (rugnux) only; the broker does not merge. std::vector frame_disposition; std::vector frame_disposition_codes; + + // The merged result of an analysis run inside the broker, for the writer to put beside the master + // file as .mtz, _maps.mtz and _model.cif. Absent where nothing was merged. + std::optional merged; + std::optional map_coefficients; + std::optional placed_model; }; struct MetadataMessage { diff --git a/docs/CBOR.md b/docs/CBOR.md index 9bcb8c441..b3e2e4b33 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -342,6 +342,9 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | pixel_sum | Array(int64) | Per-image sum of all valid pixels, excluding error/saturated pixels | | | image_scale_mosaicity | Array(float) | Scaling result: Image scale mosaicity \[deg\] | | | image_scale_cc | Array(float) | Scaling result: Image scale CC | | +| merged | object (optional) | Merged reflections of an in-broker analysis run, written by the writer as `.mtz`. `unit_cell`, `space_group_name` (gemmi extended H-M), `wavelength`, `crystal_name`, `history` (Array(string), MTZ HISTORY records), then one typed array per MTZ column: `h`, `k`, `l` (int32), `i_mean`, `sigma_i_mean`, `i_plus`, `sigma_i_plus`, `i_minus`, `sigma_i_minus`, `f`, `sigma_f`, `f_plus`, `sigma_f_plus`, `f_minus`, `sigma_f_minus` (float; the eight anomalous arrays empty where there is no Bijvoet split), `free_flag` (uint8, 0 = test set) | | +| map_coefficients | object (optional) | Map coefficients, written as `_maps.mtz`. `unit_cell`, `space_group_name`, `wavelength`, then `h`, `k`, `l` (int32), `fp`, `fc`, `phic`, `fwt`, `phwt`, `delfwt`, `phdelwt` (phases in degrees), `fom` (float), `free_flag` (uint8, 0 = test set) | | +| placed_model | object (optional) | Atomic model placed against the data, written as `_model.cif` (and `_model.pdb`). `name`, `unit_cell`, `space_group_name` (the data's), `model_space_group_name` (the coordinates'), then one array per atom: `model_number`, `seq_num`, `charge` (int32), `chain`, `residue_name`, `atom_name`, `element` (Array(string)), `icode`, `het_flag`, `altloc` (uint8), `x`, `y`, `z` (double, Angstrom), `b_iso`, `occupancy` (float), `aniso` (float, 6 per atom: U11 U22 U33 U12 U13 U23; empty where no atom has one) | | End-message vector fields are optional. When present, they provide master-file summary data so readers can inspect scan-level and per-image analysis results without opening every linked data file. Missing optional per-image values are encoded by the producer as zero unless otherwise noted. diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index a1f5ea29d..15d07c0e3 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -320,6 +320,19 @@ namespace { memcpy(v.data(), ptr, len); } + void GetCBORDoubleArray(CborValue &value, std::vector &v) { + if (GetCBORTag(value) != TagDoubleLE) + throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag"); + + auto [ptr, len] = GetCBORByteString(value); + + if (len % sizeof(double)) + throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch"); + + v.resize(len / sizeof(double)); + memcpy(v.data(), ptr, len); + } + void GetCBORStringArray(CborValue &value, std::vector &v) { if (!cbor_value_is_array(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); @@ -1490,6 +1503,105 @@ namespace { cborErr(cbor_value_leave_container(&value, &map_value)); } + // A key this build does not know: a tagged item takes two advances (see ProcessEndMessageElement). + void SkipCBORValue(CborValue &value) { + if (cbor_value_is_tag(&value)) + cbor_value_advance(&value); + cbor_value_advance(&value); + } + + MergedReflectionsMessage ProcessMergedElement(CborValue &value) { + MergedReflectionsMessage m; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value); + else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value); + else if (key == "wavelength") m.wavelength_A = GetCBORFloat(map_value); + else if (key == "crystal_name") m.crystal_name = GetCBORString(map_value); + else if (key == "history") GetCBORStringArray(map_value, m.history); + else if (key == "h") GetCBORInt32Array(map_value, m.h); + else if (key == "k") GetCBORInt32Array(map_value, m.k); + else if (key == "l") GetCBORInt32Array(map_value, m.l); + else if (key == "i_mean") GetCBORFloatArray(map_value, m.i_mean); + else if (key == "sigma_i_mean") GetCBORFloatArray(map_value, m.sigma_i_mean); + else if (key == "i_plus") GetCBORFloatArray(map_value, m.i_plus); + else if (key == "sigma_i_plus") GetCBORFloatArray(map_value, m.sigma_i_plus); + else if (key == "i_minus") GetCBORFloatArray(map_value, m.i_minus); + else if (key == "sigma_i_minus") GetCBORFloatArray(map_value, m.sigma_i_minus); + else if (key == "f") GetCBORFloatArray(map_value, m.f); + else if (key == "sigma_f") GetCBORFloatArray(map_value, m.sigma_f); + else if (key == "f_plus") GetCBORFloatArray(map_value, m.f_plus); + else if (key == "sigma_f_plus") GetCBORFloatArray(map_value, m.sigma_f_plus); + else if (key == "f_minus") GetCBORFloatArray(map_value, m.f_minus); + else if (key == "sigma_f_minus") GetCBORFloatArray(map_value, m.sigma_f_minus); + else if (key == "free_flag") GetCBORUInt8Array(map_value, m.free_flag); + else SkipCBORValue(map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return m; + } + + MapCoefficientsMessage ProcessMapCoefficientsElement(CborValue &value) { + MapCoefficientsMessage m; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value); + else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value); + else if (key == "wavelength") m.wavelength_A = GetCBORFloat(map_value); + else if (key == "h") GetCBORInt32Array(map_value, m.h); + else if (key == "k") GetCBORInt32Array(map_value, m.k); + else if (key == "l") GetCBORInt32Array(map_value, m.l); + else if (key == "fp") GetCBORFloatArray(map_value, m.fp); + else if (key == "fc") GetCBORFloatArray(map_value, m.fc); + else if (key == "phic") GetCBORFloatArray(map_value, m.phic); + else if (key == "fwt") GetCBORFloatArray(map_value, m.fwt); + else if (key == "phwt") GetCBORFloatArray(map_value, m.phwt); + else if (key == "delfwt") GetCBORFloatArray(map_value, m.delfwt); + else if (key == "phdelwt") GetCBORFloatArray(map_value, m.phdelwt); + else if (key == "fom") GetCBORFloatArray(map_value, m.fom); + else if (key == "free_flag") GetCBORUInt8Array(map_value, m.free_flag); + else SkipCBORValue(map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return m; + } + + PlacedModelMessage ProcessPlacedModelElement(CborValue &value) { + PlacedModelMessage m; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "name") m.name = GetCBORString(map_value); + else if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value); + else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value); + else if (key == "model_space_group_name") m.model_space_group_name = GetCBORString(map_value); + else if (key == "model_number") GetCBORInt32Array(map_value, m.model_number); + else if (key == "chain") GetCBORStringArray(map_value, m.chain); + else if (key == "residue_name") GetCBORStringArray(map_value, m.residue_name); + else if (key == "seq_num") GetCBORInt32Array(map_value, m.seq_num); + else if (key == "icode") GetCBORUInt8Array(map_value, m.icode); + else if (key == "het_flag") GetCBORUInt8Array(map_value, m.het_flag); + else if (key == "atom_name") GetCBORStringArray(map_value, m.atom_name); + else if (key == "element") GetCBORStringArray(map_value, m.element); + else if (key == "altloc") GetCBORUInt8Array(map_value, m.altloc); + else if (key == "charge") GetCBORInt32Array(map_value, m.charge); + else if (key == "x") GetCBORDoubleArray(map_value, m.x); + else if (key == "y") GetCBORDoubleArray(map_value, m.y); + else if (key == "z") GetCBORDoubleArray(map_value, m.z); + else if (key == "b_iso") GetCBORFloatArray(map_value, m.b_iso); + else if (key == "occupancy") GetCBORFloatArray(map_value, m.occupancy); + else if (key == "aniso") GetCBORFloatArray(map_value, m.aniso); + else SkipCBORValue(map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return m; + } + bool ProcessEndMessageElement(EndMessage &message, CborValue &value) { if (cbor_value_at_end(&value)) return false; @@ -1586,6 +1698,12 @@ namespace { message.space_group_number = GetCBORUInt(value); else if (key == "pixel_sum") GetCBORInt64Array(value, message.pixel_sum); + else if (key == "merged") + message.merged = ProcessMergedElement(value); + else if (key == "map_coefficients") + message.map_coefficients = ProcessMapCoefficientsElement(value); + else if (key == "placed_model") + message.placed_model = ProcessPlacedModelElement(value); else if (key == "rotation_lattice") { std::vector tmp; GetCBORFloatArray(value, tmp); diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 385723e54..82ee03e51 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -515,6 +515,89 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const UnitCell &val) cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); } +inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector& v) { + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encode_tag(&encoder, TagDoubleLE)); + cborErr(cbor_encode_byte_string(&encoder, reinterpret_cast(v.data()), v.size() * sizeof(double))); +} + +inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MergedReflectionsMessage &m) { + CborEncoder mapEncoder; + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength)); + CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell); + CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name); + CBOR_ENC(mapEncoder, "wavelength", m.wavelength_A); + CBOR_ENC(mapEncoder, "crystal_name", m.crystal_name); + CBOR_ENC(mapEncoder, "history", m.history); + CBOR_ENC(mapEncoder, "h", m.h); + CBOR_ENC(mapEncoder, "k", m.k); + CBOR_ENC(mapEncoder, "l", m.l); + CBOR_ENC(mapEncoder, "i_mean", m.i_mean); + CBOR_ENC(mapEncoder, "sigma_i_mean", m.sigma_i_mean); + CBOR_ENC(mapEncoder, "i_plus", m.i_plus); + CBOR_ENC(mapEncoder, "sigma_i_plus", m.sigma_i_plus); + CBOR_ENC(mapEncoder, "i_minus", m.i_minus); + CBOR_ENC(mapEncoder, "sigma_i_minus", m.sigma_i_minus); + CBOR_ENC(mapEncoder, "f", m.f); + CBOR_ENC(mapEncoder, "sigma_f", m.sigma_f); + CBOR_ENC(mapEncoder, "f_plus", m.f_plus); + CBOR_ENC(mapEncoder, "sigma_f_plus", m.sigma_f_plus); + CBOR_ENC(mapEncoder, "f_minus", m.f_minus); + CBOR_ENC(mapEncoder, "sigma_f_minus", m.sigma_f_minus); + CBOR_ENC(mapEncoder, "free_flag", m.free_flag); + cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); +} + +inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MapCoefficientsMessage &m) { + CborEncoder mapEncoder; + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength)); + CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell); + CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name); + CBOR_ENC(mapEncoder, "wavelength", m.wavelength_A); + CBOR_ENC(mapEncoder, "h", m.h); + CBOR_ENC(mapEncoder, "k", m.k); + CBOR_ENC(mapEncoder, "l", m.l); + CBOR_ENC(mapEncoder, "fp", m.fp); + CBOR_ENC(mapEncoder, "fc", m.fc); + CBOR_ENC(mapEncoder, "phic", m.phic); + CBOR_ENC(mapEncoder, "fwt", m.fwt); + CBOR_ENC(mapEncoder, "phwt", m.phwt); + CBOR_ENC(mapEncoder, "delfwt", m.delfwt); + CBOR_ENC(mapEncoder, "phdelwt", m.phdelwt); + CBOR_ENC(mapEncoder, "fom", m.fom); + CBOR_ENC(mapEncoder, "free_flag", m.free_flag); + cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); +} + +inline void CBOR_ENC(CborEncoder &encoder, const char* key, const PlacedModelMessage &m) { + CborEncoder mapEncoder; + cborErr(cbor_encode_text_stringz(&encoder, key)); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength)); + CBOR_ENC(mapEncoder, "name", m.name); + CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell); + CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name); + CBOR_ENC(mapEncoder, "model_space_group_name", m.model_space_group_name); + CBOR_ENC(mapEncoder, "model_number", m.model_number); + CBOR_ENC(mapEncoder, "chain", m.chain); + CBOR_ENC(mapEncoder, "residue_name", m.residue_name); + CBOR_ENC(mapEncoder, "seq_num", m.seq_num); + CBOR_ENC(mapEncoder, "icode", m.icode); + CBOR_ENC(mapEncoder, "het_flag", m.het_flag); + CBOR_ENC(mapEncoder, "atom_name", m.atom_name); + CBOR_ENC(mapEncoder, "element", m.element); + CBOR_ENC(mapEncoder, "altloc", m.altloc); + CBOR_ENC(mapEncoder, "charge", m.charge); + CBOR_ENC(mapEncoder, "x", m.x); + CBOR_ENC(mapEncoder, "y", m.y); + CBOR_ENC(mapEncoder, "z", m.z); + CBOR_ENC(mapEncoder, "b_iso", m.b_iso); + CBOR_ENC(mapEncoder, "occupancy", m.occupancy); + CBOR_ENC(mapEncoder, "aniso", m.aniso); + cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); +} + // Add encoder for LatticeMessage inline void CBOR_ENC(CborEncoder &encoder, const char* key, const LatticeMessage &val) { CborEncoder mapEncoder; @@ -857,6 +940,9 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) { // should take; the number is beside it for a reader written before the name existed. CBOR_ENC(mapEncoder, "space_group_name", message.space_group_name); CBOR_ENC(mapEncoder, "space_group_number", message.space_group_number); + CBOR_ENC(mapEncoder, "merged", message.merged); + CBOR_ENC(mapEncoder, "map_coefficients", message.map_coefficients); + CBOR_ENC(mapEncoder, "placed_model", message.placed_model); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); From aca27be862ca50dcb08ceddf44005ae5f98f15d7 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 12:19:51 +0200 Subject: [PATCH 02/19] jfjoch_writer writes rugnux's .mtz, _maps.mtz and _model.cif from the END blocks New gemmi-only library JFJochRugnuxFiles (writer/), linked by JFJochWriter, so the writer needs no analysis library and builds with JFJOCH_WRITER_ONLY: - WriteMergedMtz / WriteMapCoefficientsMtz / PlacedModelStructure make the files from the structured blocks; WriteRugnuxResultFiles writes whichever blocks END carries beside the master, each under a temporary name and renamed into place, an existing file replaced only when the run's overwrite is set. A file that fails costs only itself (logged), not the run. - FillRugnuxResultBlocks reads the blocks back from the files rugnux wrote at a prefix (p.mtz, p_maps.mtz, p_model.cif), for the broker to fill END. WriteMtzReflections now goes through WriteMergedMtz (MergedReflectionsForWriter builds the block); its output is byte-identical to before, checked by md5 on a synthetic set in both the Friedel-merged and the anomalous layout. StreamWriter writes the files after the master and before the END ACK, so the ACK means they are in place; END receipt and each rename are logged at INFO with the wall-clock time, for last-frame-to-files latency. The broker's direct HDF5 pusher writes them too. Test: rugnux's three files -> blocks -> CBOR -> writer reproduces all of them (and _model.pdb) byte for byte; a ~25k-reflection END is 2.7 MB. Co-Authored-By: Claude Opus 5.5 --- image_analysis/CMakeLists.txt | 2 +- image_analysis/WriteReflections.cpp | 119 ++++++--------- image_analysis/WriteReflections.h | 9 ++ image_pusher/HDF5FilePusher.cpp | 1 + tests/CMakeLists.txt | 1 + tests/RugnuxResultFilesTest.cpp | 219 ++++++++++++++++++++++++++++ writer/CMakeLists.txt | 18 ++- writer/FileWriter.cpp | 6 + writer/FileWriter.h | 4 + writer/RugnuxResultBlocks.cpp | 140 ++++++++++++++++++ writer/RugnuxResultBlocks.h | 22 +++ writer/StreamWriter.cpp | 7 +- writer/WriteRugnuxFiles.cpp | 210 ++++++++++++++++++++++++++ writer/WriteRugnuxFiles.h | 33 +++++ 14 files changed, 706 insertions(+), 85 deletions(-) create mode 100644 tests/RugnuxResultFilesTest.cpp create mode 100644 writer/RugnuxResultBlocks.cpp create mode 100644 writer/RugnuxResultBlocks.h create mode 100644 writer/WriteRugnuxFiles.cpp create mode 100644 writer/WriteRugnuxFiles.h diff --git a/image_analysis/CMakeLists.txt b/image_analysis/CMakeLists.txt index 098307e75..916ebb0e2 100644 --- a/image_analysis/CMakeLists.txt +++ b/image_analysis/CMakeLists.txt @@ -131,4 +131,4 @@ ADD_SUBDIRECTORY(azint) ADD_SUBDIRECTORY(roi) ADD_SUBDIRECTORY(structure_refinement) -TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochROIIntegration JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochStructureRefinement gemmi) +TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochROIIntegration JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochStructureRefinement JFJochRugnuxFiles gemmi) diff --git a/image_analysis/WriteReflections.cpp b/image_analysis/WriteReflections.cpp index faf3b5075..8cf24c2a5 100644 --- a/image_analysis/WriteReflections.cpp +++ b/image_analysis/WriteReflections.cpp @@ -7,6 +7,7 @@ #include "scale_merge/TwinningAnalysis.h" #include "bragg_integration/SystematicAbsence.h" #include "../common/ParallelFor.h" +#include "../writer/WriteRugnuxFiles.h" #include #include @@ -121,16 +122,16 @@ std::string GitDescription() { // (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each // continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is // never cut just after a space: the space starts the next one, behind its '+'. -void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) { - const auto add = [&mtz](const std::string &label, const std::string &value) { +void AddProvenanceHistory(std::vector &history, const OutputProvenance &provenance) { + const auto add = [&history](const std::string &label, const std::string &value) { if (value.empty()) return; std::string rest = label + value; - while (!rest.empty() && mtz.history.size() < 30) { + while (!rest.empty() && history.size() < 30) { size_t n = std::min(80, rest.size()); while (n > 1 && n < rest.size() && rest[n - 1] == ' ') --n; - mtz.history.push_back(rest.substr(0, n)); + history.push_back(rest.substr(0, n)); rest = n < rest.size() ? "+" + rest.substr(n) : std::string(); } }; @@ -606,81 +607,49 @@ void WriteMmcifReflections(const std::vector &reflections, out.close(); } +MergedReflectionsMessage MergedReflectionsForWriter(const std::vector &reflections, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + const OutputProvenance &provenance) { + MergedReflectionsMessage m; + m.unit_cell = unitCell; + m.space_group_name = experiment.GetSpaceGroupOrP1().xhm(); + m.wavelength_A = experiment.GetWavelength_A(); + m.crystal_name = experiment.GetSampleName(); + m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); + AddProvenanceHistory(m.history, provenance); + + bool has_anom = true; + const std::vector out_rows = BuildMergedRows(reflections, experiment, has_anom); + for (const auto& row : out_rows) { + m.h.push_back(row.h); + m.k.push_back(row.k); + m.l.push_back(row.l); + m.i_mean.push_back(row.Imean); + m.sigma_i_mean.push_back(row.sImean); + m.f.push_back(row.Fmean); + m.sigma_f.push_back(row.sFmean); + if (has_anom) { + m.i_plus.push_back(row.Ip); + m.sigma_i_plus.push_back(row.sIp); + m.i_minus.push_back(row.Im); + m.sigma_i_minus.push_back(row.sIm); + m.f_plus.push_back(row.Fp); + m.sigma_f_plus.push_back(row.sFp); + m.f_minus.push_back(row.Fm); + m.sigma_f_minus.push_back(row.sFm); + } + m.free_flag.push_back(static_cast(row.rfree)); + } + return m; +} + void WriteMtzReflections(const std::vector &reflections, const UnitCell &unitCell, const DiffractionExperiment &experiment, const OutputProvenance &provenance, const std::string &filename) { - gemmi::Mtz mtz; - - // Optional but recommended metadata - mtz.spacegroup = &experiment.GetSpaceGroupOrP1(); - mtz.set_cell_for_all(unitCell); - - // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention). - mtz.title = "Rugnux merged reflections"; - mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); - AddProvenanceHistory(mtz, provenance); - - // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the - // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is - // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that - // takes f'/f'' from the file would phase on. - mtz.add_base(); // the HKL_base dataset and the H K L columns - gemmi::Mtz::Dataset& ds = mtz.add_dataset("native"); - ds.crystal_name = experiment.GetSampleName(); - ds.wavelength = experiment.GetWavelength_A(); - - const int dataset_id = ds.id; - - // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for - // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly. - bool has_anom = true; - const std::vector out_rows = BuildMergedRows(reflections, experiment, has_anom); - - mtz.add_column("IMEAN", 'J', dataset_id, -1, false); - mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false); - if (has_anom) { - mtz.add_column("I(+)", 'K', dataset_id, -1, false); - mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false); - mtz.add_column("I(-)", 'K', dataset_id, -1, false); - mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false); - } - mtz.add_column("F", 'F', dataset_id, -1, false); // French-Wilson amplitude - mtz.add_column("SIGF", 'Q', dataset_id, -1, false); - if (has_anom) { - mtz.add_column("F(+)", 'G', dataset_id, -1, false); - mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false); - mtz.add_column("F(-)", 'G', dataset_id, -1, false); - mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false); - } - mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false); - - mtz.nreflections = static_cast(out_rows.size()); - mtz.data.reserve(out_rows.size() * 16); - for (const auto& row : out_rows) { - mtz.data.push_back(static_cast(row.h)); - mtz.data.push_back(static_cast(row.k)); - mtz.data.push_back(static_cast(row.l)); - mtz.data.push_back(row.Imean); - mtz.data.push_back(row.sImean); - if (has_anom) { - mtz.data.push_back(row.Ip); - mtz.data.push_back(row.sIp); - mtz.data.push_back(row.Im); - mtz.data.push_back(row.sIm); - } - mtz.data.push_back(row.Fmean); - mtz.data.push_back(row.sFmean); - if (has_anom) { - mtz.data.push_back(row.Fp); - mtz.data.push_back(row.sFp); - mtz.data.push_back(row.Fm); - mtz.data.push_back(row.sFm); - } - mtz.data.push_back(static_cast(row.rfree)); - } - mtz.write_to_file(filename); + WriteMergedMtz(MergedReflectionsForWriter(reflections, unitCell, experiment, provenance), filename); } namespace { @@ -948,7 +917,7 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, mtz.set_cell_for_all(unitCell); mtz.title = "Rugnux unmerged observations"; mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); - AddProvenanceHistory(mtz, provenance); + AddProvenanceHistory(mtz.history, provenance); mtz.add_base(); // the HKL_base dataset and the H K L columns gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged"); ds.crystal_name = experiment.GetSampleName(); diff --git a/image_analysis/WriteReflections.h b/image_analysis/WriteReflections.h index 559b3e662..68d2ab778 100644 --- a/image_analysis/WriteReflections.h +++ b/image_analysis/WriteReflections.h @@ -9,6 +9,7 @@ #include "../common/Reflection.h" #include "../common/UnitCell.h" #include "../common/DiffractionExperiment.h" +#include "../common/JFJochMessages.h" #include "IntegrationOutcome.h" #include @@ -49,6 +50,14 @@ void WriteMmcifReflections(const std::vector &reflections, const std::string &filename, size_t nthreads); +// The merged MTZ as a structured END-message block: the columns, metadata and HISTORY records that +// WriteMtzReflections writes, for jfjoch_writer to write the same file (WriteMergedMtz, +// writer/WriteRugnuxFiles.h). WriteMtzReflections is exactly WriteMergedMtz of this. +MergedReflectionsMessage MergedReflectionsForWriter(const std::vector &reflections, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + const OutputProvenance &provenance); + void WriteMtzReflections(const std::vector &reflections, const UnitCell &unitCell, const DiffractionExperiment &experiment, diff --git a/image_pusher/HDF5FilePusher.cpp b/image_pusher/HDF5FilePusher.cpp index bec85f54f..ac43e1661 100644 --- a/image_pusher/HDF5FilePusher.cpp +++ b/image_pusher/HDF5FilePusher.cpp @@ -66,6 +66,7 @@ bool HDF5FilePusher::EndDataCollection(const EndMessage &message) { if (!writer) throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Image pusher not ready for writing images"); writer->WriteHDF5(message); + writer->WriteResultFiles(message, logger); writer->Finalize(); } catch (...) { writer.reset(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c6dd46391..5f5c4b207 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -172,6 +172,7 @@ ADD_EXECUTABLE(jfjoch_test LoadReferenceMtzFreeFlagsTest.cpp LoadReferenceSfCifTest.cpp WriteReflectionsTest.cpp + RugnuxResultFilesTest.cpp UnitCellTest.cpp CCTest.cpp MultiLatticeSearchTest.cpp diff --git a/tests/RugnuxResultFilesTest.cpp b/tests/RugnuxResultFilesTest.cpp new file mode 100644 index 000000000..d0910ab98 --- /dev/null +++ b/tests/RugnuxResultFilesTest.cpp @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include +#include +#include +#include + +#include +#include +#include // count_atom_sites + +#include "../common/Definitions.h" +#include "../common/DiffractionExperiment.h" +#include "../common/Logger.h" +#include "../image_analysis/WriteReflections.h" +#include "../frame_serialize/CBORStream2Serializer.h" +#include "../frame_serialize/CBORStream2Deserializer.h" +#include "../writer/WriteModel.h" +#include "../writer/WriteRugnuxFiles.h" +#include "../writer/RugnuxResultBlocks.h" +#include "SyntheticMergedReflections.h" + +namespace { + constexpr UnitCell TEST_CELL{47.0f, 47.0f, 63.0f, 90.0f, 90.0f, 90.0f}; + + std::string ReadFile(const std::string &path) { + std::ifstream f(path, std::ios::binary); + return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; + } + + std::vector Labels(const gemmi::Mtz &mtz) { + std::vector v; + for (const auto &c : mtz.columns) + v.push_back(c.label); + return v; + } + + // A small model with what a placed model carries: two chains, a ligand, a water, an alternate + // conformation, an anisotropic atom, an insertion code and a charge. + gemmi::Structure TestModel() { + gemmi::Structure st; + st.name = "test"; + st.cell = TEST_CELL; + st.spacegroup_hm = "P 43 21 2"; + gemmi::Model &m = st.models.emplace_back(1); + double x = 1.0; + auto add_atom = [&x](gemmi::Residue &r, const std::string &name, const char *el, char altloc = 0) { + gemmi::Atom &a = r.atoms.emplace_back(); + a.name = name; + a.element = gemmi::Element(el); + a.altloc = altloc; + a.pos = gemmi::Position(x, 2.0 * x + 0.123, 3.0 * x - 0.456); + a.b_iso = 15.0f + static_cast(x); + a.occ = altloc ? 0.5f : 1.0f; + x += 1.37; + return &a; + }; + for (const std::string chain : {"A", "B"}) { + gemmi::Chain &ch = m.chains.emplace_back(chain); + for (int i = 1; i <= 3; i++) { + gemmi::Residue &r = ch.residues.emplace_back(gemmi::ResidueId{gemmi::SeqId(i, i == 3 ? 'A' : ' '), {}, "ALA"}); + r.het_flag = 'A'; + add_atom(r, "N", "N"); + add_atom(r, "CA", "C"); + add_atom(r, "C", "C"); + add_atom(r, "O", "O"); + add_atom(r, "CB", "C", 'A'); + add_atom(r, "CB", "C", 'B'); + } + } + gemmi::Residue &zn = m.chains[0].residues.emplace_back(gemmi::ResidueId{gemmi::SeqId(101, ' '), {}, "ZN"}); + zn.het_flag = 'H'; + gemmi::Atom *a = add_atom(zn, "ZN", "Zn"); + a->charge = 2; + a->aniso = {0.2f, 0.25f, 0.3f, 0.01f, -0.02f, 0.03f}; + gemmi::Residue &w = m.chains[1].residues.emplace_back(gemmi::ResidueId{gemmi::SeqId(201, ' '), {}, "HOH"}); + w.het_flag = 'H'; + add_atom(w, "O", "O"); + st.setup_cell_images(); + return st; + } +} + +TEST_CASE("RugnuxResultFiles: rugnux's files -> END blocks -> CBOR -> writer, byte for byte", "[rugnux_result_files]") { + Logger logger("RugnuxResultFiles"); + const std::filesystem::path dir = std::filesystem::temp_directory_path() / "rugnux_result_files_test"; + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + const std::string original = (dir / "rugnux").string(); + const std::string written = (dir / "writer").string(); + + // About the size of a standard-crystal run (~25k reflections), with an anomalous split. + jfjoch_test::SyntheticMergeParams params; + params.true_space_group = "P 43 21 2"; + params.twin_supergroup = "P 43 21 2"; + params.d_min_A = 2.25; + auto reflections = jfjoch_test::GenerateSyntheticMerged(params); + for (size_t i = 0; i < reflections.size(); i++) { + auto &r = reflections[i]; + r.I_plus = r.I * 1.01f; r.sigma_plus = r.sigma * 1.4f; + r.F_plus = std::sqrt(std::max(r.I, 1.0f)); r.sigmaF_plus = 0.1f; + if (i % 7 != 0) { + r.I_minus = r.I * 0.99f; r.sigma_minus = r.sigma * 1.4f; + r.F_minus = std::sqrt(std::max(r.I, 1.0f)); r.sigmaF_minus = 0.12f; + } + r.rfree_flag = (i % 20 == 0); + } + DiffractionExperiment experiment; + experiment.IncidentEnergy_keV(12.7f); + experiment.SpaceGroupNumber(96); + experiment.SetUnitCell(TEST_CELL); + OutputProvenance provenance; + provenance.command_line = "rugnux -N 8 input_master.h5"; + provenance.input_file = "/some/where/input_master.h5"; + WriteMtzReflections(reflections, TEST_CELL, experiment, provenance, original + ".mtz"); + + MapCoefficientsMessage maps; + maps.unit_cell = TEST_CELL; + maps.space_group_name = "P 43 21 2"; + maps.wavelength_A = experiment.GetWavelength_A(); + for (size_t i = 0; i < reflections.size(); i++) { + const float phi = static_cast((i * 37) % 360); + maps.h.push_back(reflections[i].h); + maps.k.push_back(reflections[i].k); + maps.l.push_back(reflections[i].l); + maps.fp.push_back(reflections[i].F); + maps.fc.push_back(reflections[i].F * 0.9f); + maps.phic.push_back(phi); + maps.fwt.push_back(reflections[i].F * 1.1f); + maps.phwt.push_back(phi); + maps.delfwt.push_back(reflections[i].F * 0.1f); + maps.phdelwt.push_back(phi); + maps.fom.push_back(0.8f); + maps.free_flag.push_back(reflections[i].rfree_flag ? 0 : 1); + } + WriteMapCoefficientsMtz(maps, original + "_maps.mtz"); + + WritePlacedModel(TestModel(), TEST_CELL, experiment.GetSpaceGroupOrP1(), original, logger); + + EndMessage end{}; + end.max_image_number = 100; + FillRugnuxResultBlocks(original, end); + REQUIRE(end.merged); + REQUIRE(end.map_coefficients); + REQUIRE(end.placed_model); + CHECK(end.merged->h.size() == reflections.size()); + CHECK(end.merged->i_plus.size() == reflections.size()); + CHECK(end.placed_model->atom_name.size() == 2 * 3 * 6 + 2); + + std::vector buffer(MESSAGE_SIZE_FOR_START_END); + CBORStream2Serializer serializer(buffer.data(), buffer.size()); + REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(end)); + std::cout << "END message with " << reflections.size() << " merged reflections, as many map coefficients and " + << end.placed_model->atom_name.size() << " atoms: " << serializer.GetBufferSize() << " bytes" << std::endl; + auto deserialized = CBORStream2Deserialize(buffer.data(), serializer.GetBufferSize()); + REQUIRE(deserialized); + REQUIRE(deserialized->end_message); + const EndMessage &out = *deserialized->end_message; + REQUIRE(out.merged); + REQUIRE(out.map_coefficients); + REQUIRE(out.placed_model); + CHECK(out.merged->history == end.merged->history); + CHECK(out.placed_model->x == end.placed_model->x); + + WriteRugnuxResultFiles(out, written, false, logger); + + CHECK(ReadFile(written + ".mtz") == ReadFile(original + ".mtz")); + CHECK(ReadFile(written + "_maps.mtz") == ReadFile(original + "_maps.mtz")); + CHECK(ReadFile(written + "_model.cif") == ReadFile(original + "_model.cif")); + CHECK(ReadFile(written + "_model.pdb") == ReadFile(original + "_model.pdb")); + + const gemmi::Mtz mtz = gemmi::read_mtz_file(written + ".mtz"); + CHECK(Labels(mtz) == std::vector{"H", "K", "L", "IMEAN", "SIGIMEAN", "I(+)", "SIGI(+)", "I(-)", + "SIGI(-)", "F", "SIGF", "F(+)", "SIGF(+)", "F(-)", "SIGF(-)", + "FreeR_flag"}); + CHECK(mtz.nreflections == static_cast(reflections.size())); + CHECK(mtz.datasets.back().wavelength == Catch::Approx(experiment.GetWavelength_A()).epsilon(1e-5)); + const gemmi::Mtz maps_mtz = gemmi::read_mtz_file(written + "_maps.mtz"); + CHECK(Labels(maps_mtz) == std::vector{"H", "K", "L", "FP", "FC", "PHIC", "FWT", "PHWT", "DELFWT", + "PHDELWT", "FOM", "FREE"}); + CHECK(maps_mtz.get_column_with_label("K")[5] == static_cast(maps.k[5])); + CHECK(maps_mtz.get_column_with_label("PHWT")[5] == maps.phwt[5]); + const gemmi::Structure st = gemmi::read_structure_gz(written + "_model.cif"); + CHECK(gemmi::count_atom_sites(st) == 2 * 3 * 6 + 2); + + std::filesystem::remove_all(dir); +} + +TEST_CASE("RugnuxResultFiles: an existing file is kept when overwrite is off", "[rugnux_result_files]") { + Logger logger("RugnuxResultFiles"); + const std::filesystem::path dir = std::filesystem::temp_directory_path() / "rugnux_result_files_overwrite"; + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + const std::string prefix = (dir / "run").string(); + + EndMessage end{}; + end.merged = MergedReflectionsMessage{}; + end.merged->unit_cell = TEST_CELL; + end.merged->space_group_name = "P 43 21 2"; + end.merged->h = {1}; end.merged->k = {2}; end.merged->l = {3}; + end.merged->i_mean = {10}; end.merged->sigma_i_mean = {1}; + end.merged->f = {3}; end.merged->sigma_f = {0.2}; + end.merged->free_flag = {1}; + + { std::ofstream f(prefix + ".mtz"); f << "keep"; } + WriteRugnuxResultFiles(end, prefix, false, logger); + CHECK(ReadFile(prefix + ".mtz") == "keep"); + CHECK(!std::filesystem::exists(prefix + ".mtz.tmp")); + + WriteRugnuxResultFiles(end, prefix, true, logger); + const gemmi::Mtz mtz = gemmi::read_mtz_file(prefix + ".mtz"); + CHECK(mtz.nreflections == 1); + CHECK(Labels(mtz) == std::vector{"H", "K", "L", "IMEAN", "SIGIMEAN", "F", "SIGF", "FreeR_flag"}); + + std::filesystem::remove_all(dir); +} diff --git a/writer/CMakeLists.txt b/writer/CMakeLists.txt index 2f6f43738..c96b830b7 100644 --- a/writer/CMakeLists.txt +++ b/writer/CMakeLists.txt @@ -31,14 +31,18 @@ ADD_LIBRARY(JFJochWriter STATIC HDF5DataFilePluginPerformance.h ) -TARGET_LINK_LIBRARIES(JFJochWriter JFJochPreview JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize) +# The placed atomic model (mmCIF/PDB) that rugnux writes beside its merged output. +ADD_LIBRARY(JFJochModelWriter STATIC WriteModel.cpp WriteModel.h) +TARGET_LINK_LIBRARIES(JFJochModelWriter JFJochLogger gemmi) -# The placed atomic model (mmCIF/PDB) that rugnux writes beside its merged output. Its own target, -# because it needs gemmi, which JFJochWriter does not link and a writer-only build does not want. -IF (NOT JFJOCH_WRITER_ONLY) - ADD_LIBRARY(JFJochModelWriter STATIC WriteModel.cpp WriteModel.h) - TARGET_LINK_LIBRARIES(JFJochModelWriter JFJochLogger gemmi) -ENDIF() +# rugnux's merged MTZ, map-coefficient MTZ and placed model, to and from the structured END-message +# blocks, so that jfjoch_writer writes them with gemmi alone and no analysis library. +ADD_LIBRARY(JFJochRugnuxFiles STATIC WriteRugnuxFiles.cpp WriteRugnuxFiles.h + RugnuxResultBlocks.cpp RugnuxResultBlocks.h) +TARGET_LINK_LIBRARIES(JFJochRugnuxFiles JFJochModelWriter JFJochLogger gemmi) + +TARGET_LINK_LIBRARIES(JFJochWriter JFJochPreview JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize + JFJochRugnuxFiles) # The streaming writer and the jfjoch_writer service belong to the Linux server stack, not the # portable viewer subset: JFJochStreamWriter pulls in JFJochImagePuller (not built in a viewer-only diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index 5d6b42432..92d9a3804 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -9,6 +9,7 @@ #include "../common/Logger.h" #include "../common/JFJochException.h" #include "JFJochDecompress.h" +#include "WriteRugnuxFiles.h" FileWriter::FileWriter(const StartMessage &request, bool check_overwrite_at_start, bool trusted_path) : start_message(request) { @@ -339,3 +340,8 @@ void FileWriter::WriteHDF5(const EndMessage &msg) { std::rethrow_exception(first_exception); } } + +void FileWriter::WriteResultFiles(const EndMessage &msg, Logger &logger) { + if (master_file) + WriteRugnuxResultFiles(msg, start_message.file_prefix, start_message.overwrite.value_or(false), logger); +} diff --git a/writer/FileWriter.h b/writer/FileWriter.h index c4ba5494e..df96a3e1c 100644 --- a/writer/FileWriter.h +++ b/writer/FileWriter.h @@ -9,6 +9,7 @@ #include "../common/JFJochMessages.h" #include "../common/ZMQWrappers.h" #include "HDF5NXmx.h" +#include "../common/Logger.h" #include #include #include @@ -59,6 +60,9 @@ public: void WriteHDF5(const DataMessage& msg); void WriteHDF5(const CompressedImage& msg); void WriteHDF5(const EndMessage& msg); + // The merged result files the END message carries (WriteRugnuxResultFiles), beside the master + // file, by the writer that owns it. Call after WriteHDF5(EndMessage). + void WriteResultFiles(const EndMessage& msg, Logger &logger); std::vector Finalize(); void SetupFinalizedFileSocket(const std::string &addr); // Called for every data file once it is closed and renamed into place. diff --git a/writer/RugnuxResultBlocks.cpp b/writer/RugnuxResultBlocks.cpp new file mode 100644 index 000000000..c7ac1d621 --- /dev/null +++ b/writer/RugnuxResultBlocks.cpp @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxResultBlocks.h" + +#include + +#include +#include + +namespace { + +UnitCell ToUnitCell(const gemmi::UnitCell &c) { + return UnitCell{static_cast(c.a), static_cast(c.b), static_cast(c.c), + static_cast(c.alpha), static_cast(c.beta), static_cast(c.gamma)}; +} + +// The column with this label, empty where the file has none. +std::vector Column(const gemmi::Mtz &mtz, const std::string &label) { + std::vector v; + if (const gemmi::Mtz::Column *col = mtz.column_with_label(label)) + for (int i = 0; i < col->size(); i++) + v.push_back((*col)[i]); + return v; +} + +std::vector IntColumn(const gemmi::Mtz &mtz, const std::string &label) { + std::vector v; + for (float x : Column(mtz, label)) + v.push_back(static_cast(x)); + return v; +} + +std::vector FlagColumn(const gemmi::Mtz &mtz, const std::string &label) { + std::vector v; + for (float x : Column(mtz, label)) + v.push_back(static_cast(x)); + return v; +} + +std::string SpaceGroupName(const gemmi::Mtz &mtz) { + return mtz.spacegroup != nullptr ? mtz.spacegroup->xhm() : std::string(); +} + +} // namespace + +MergedReflectionsMessage MergedReflectionsFromMtz(const std::string &path) { + const gemmi::Mtz mtz = gemmi::read_mtz_file(path); + MergedReflectionsMessage m; + m.unit_cell = ToUnitCell(mtz.cell); + m.space_group_name = SpaceGroupName(mtz); + m.wavelength_A = static_cast(mtz.datasets.back().wavelength); + m.crystal_name = mtz.datasets.back().crystal_name; + m.history = mtz.history; + m.h = IntColumn(mtz, "H"); + m.k = IntColumn(mtz, "K"); + m.l = IntColumn(mtz, "L"); + m.i_mean = Column(mtz, "IMEAN"); + m.sigma_i_mean = Column(mtz, "SIGIMEAN"); + m.i_plus = Column(mtz, "I(+)"); + m.sigma_i_plus = Column(mtz, "SIGI(+)"); + m.i_minus = Column(mtz, "I(-)"); + m.sigma_i_minus = Column(mtz, "SIGI(-)"); + m.f = Column(mtz, "F"); + m.sigma_f = Column(mtz, "SIGF"); + m.f_plus = Column(mtz, "F(+)"); + m.sigma_f_plus = Column(mtz, "SIGF(+)"); + m.f_minus = Column(mtz, "F(-)"); + m.sigma_f_minus = Column(mtz, "SIGF(-)"); + m.free_flag = FlagColumn(mtz, "FreeR_flag"); + return m; +} + +MapCoefficientsMessage MapCoefficientsFromMtz(const std::string &path) { + const gemmi::Mtz mtz = gemmi::read_mtz_file(path); + MapCoefficientsMessage c; + c.unit_cell = ToUnitCell(mtz.cell); + c.space_group_name = SpaceGroupName(mtz); + c.wavelength_A = static_cast(mtz.datasets.back().wavelength); + c.h = IntColumn(mtz, "H"); + c.k = IntColumn(mtz, "K"); + c.l = IntColumn(mtz, "L"); + c.fp = Column(mtz, "FP"); + c.fc = Column(mtz, "FC"); + c.phic = Column(mtz, "PHIC"); + c.fwt = Column(mtz, "FWT"); + c.phwt = Column(mtz, "PHWT"); + c.delfwt = Column(mtz, "DELFWT"); + c.phdelwt = Column(mtz, "PHDELWT"); + c.fom = Column(mtz, "FOM"); + c.free_flag = FlagColumn(mtz, "FREE"); + return c; +} + +PlacedModelMessage PlacedModelFromStructure(const gemmi::Structure &st, const std::string &data_space_group_name) { + PlacedModelMessage p; + p.name = st.name; + p.unit_cell = ToUnitCell(st.cell); + p.space_group_name = data_space_group_name; + p.model_space_group_name = st.spacegroup_hm; + bool has_aniso = false; + for (const gemmi::Model &m : st.models) + for (const gemmi::Chain &ch : m.chains) + for (const gemmi::Residue &r : ch.residues) + for (const gemmi::Atom &a : r.atoms) { + p.model_number.push_back(m.num); + p.chain.push_back(ch.name); + p.residue_name.push_back(r.name); + p.seq_num.push_back(*r.seqid.num); + p.icode.push_back(static_cast(r.seqid.icode)); + p.het_flag.push_back(static_cast(r.het_flag)); + p.atom_name.push_back(a.name); + p.element.push_back(a.element.name()); + p.altloc.push_back(static_cast(a.altloc)); + p.charge.push_back(a.charge); + p.x.push_back(a.pos.x); + p.y.push_back(a.pos.y); + p.z.push_back(a.pos.z); + p.b_iso.push_back(a.b_iso); + p.occupancy.push_back(a.occ); + p.aniso.insert(p.aniso.end(), {a.aniso.u11, a.aniso.u22, a.aniso.u33, + a.aniso.u12, a.aniso.u13, a.aniso.u23}); + has_aniso = has_aniso || a.aniso.nonzero(); + } + if (!has_aniso) + p.aniso.clear(); + return p; +} + +void FillRugnuxResultBlocks(const std::string &prefix, EndMessage &end) { + if (std::filesystem::exists(prefix + ".mtz")) + end.merged = MergedReflectionsFromMtz(prefix + ".mtz"); + if (std::filesystem::exists(prefix + "_maps.mtz")) + end.map_coefficients = MapCoefficientsFromMtz(prefix + "_maps.mtz"); + if (std::filesystem::exists(prefix + "_model.cif")) { + const gemmi::Structure st = gemmi::read_structure_gz(prefix + "_model.cif"); + end.placed_model = PlacedModelFromStructure(st, end.merged ? end.merged->space_group_name + : st.spacegroup_hm); + } +} diff --git a/writer/RugnuxResultBlocks.h b/writer/RugnuxResultBlocks.h new file mode 100644 index 000000000..f0198e46f --- /dev/null +++ b/writer/RugnuxResultBlocks.h @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "gemmi/model.hpp" + +#include "../common/JFJochMessages.h" + +// The END-message blocks (common/JFJochMessages.h) read back from the files rugnux wrote at +// `prefix`: .mtz into `merged`, _maps.mtz into `map_coefficients` and +// _model.cif into `placed_model`, each only where its file exists. The writer makes the same +// files again from them (WriteRugnuxResultFiles, writer/WriteRugnuxFiles.h). +void FillRugnuxResultBlocks(const std::string &prefix, EndMessage &end); + +MergedReflectionsMessage MergedReflectionsFromMtz(const std::string &path); +MapCoefficientsMessage MapCoefficientsFromMtz(const std::string &path); +// data_space_group_name: the group of the reflection files beside the model, which is what +// WritePlacedModel is to be given; the model's own group is taken from `st`. +PlacedModelMessage PlacedModelFromStructure(const gemmi::Structure &st, const std::string &data_space_group_name); diff --git a/writer/StreamWriter.cpp b/writer/StreamWriter.cpp index efac1d632..616af4955 100644 --- a/writer/StreamWriter.cpp +++ b/writer/StreamWriter.cpp @@ -4,6 +4,7 @@ #include "../common/JFJochException.h" #include "StreamWriter.h" +#include #include #include "FileWriter.h" @@ -203,14 +204,16 @@ void StreamWriter::ProcessEndMessage() { if (state == StreamWriterState::Idle || state == StreamWriterState::Finalized) return; - if (verbose) - logger.Info("Received end message"); + logger.Info("Received end message at wall clock {:.3f} s", + std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count()); if (state != StreamWriterState::Error) { try { if ((image_puller_output.cbor->end_message->max_image_number == 0) && (max_image_number > 0)) image_puller_output.cbor->end_message->max_image_number = max_image_number; file_writer->WriteHDF5(*image_puller_output.cbor->end_message); + // Before the END ACK below, so that the ACK says these files are in place too. + file_writer->WriteResultFiles(*image_puller_output.cbor->end_message, logger); } catch (const std::exception &e) { // std::exception for the same reason as in ProcessDataImage. Here the cost of a leak is the // highest: the END ACK and the notification below are never sent, the state never reaches diff --git a/writer/WriteRugnuxFiles.cpp b/writer/WriteRugnuxFiles.cpp new file mode 100644 index 000000000..2f9449485 --- /dev/null +++ b/writer/WriteRugnuxFiles.cpp @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "WriteRugnuxFiles.h" + +#include +#include + +#include + +#include "WriteModel.h" +#include "../common/Logger.h" + +void WriteMergedMtz(const MergedReflectionsMessage &merged, const std::string &filename) { + gemmi::Mtz mtz; + + // Optional but recommended metadata + mtz.spacegroup = gemmi::find_spacegroup_by_name(merged.space_group_name); + mtz.set_cell_for_all(merged.unit_cell); + + // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention). + mtz.title = "Rugnux merged reflections"; + mtz.history = merged.history; + + // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the + // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is + // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that + // takes f'/f'' from the file would phase on. + mtz.add_base(); // the HKL_base dataset and the H K L columns + gemmi::Mtz::Dataset& ds = mtz.add_dataset("native"); + ds.crystal_name = merged.crystal_name; + ds.wavelength = merged.wavelength_A; + + const int dataset_id = ds.id; + + // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for + // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly. + const bool has_anom = !merged.i_plus.empty(); + + mtz.add_column("IMEAN", 'J', dataset_id, -1, false); + mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false); + if (has_anom) { + mtz.add_column("I(+)", 'K', dataset_id, -1, false); + mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false); + mtz.add_column("I(-)", 'K', dataset_id, -1, false); + mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false); + } + mtz.add_column("F", 'F', dataset_id, -1, false); // French-Wilson amplitude + mtz.add_column("SIGF", 'Q', dataset_id, -1, false); + if (has_anom) { + mtz.add_column("F(+)", 'G', dataset_id, -1, false); + mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false); + mtz.add_column("F(-)", 'G', dataset_id, -1, false); + mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false); + } + mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false); + + const size_t n = merged.h.size(); + mtz.nreflections = static_cast(n); + mtz.data.reserve(n * 16); + for (size_t i = 0; i < n; i++) { + mtz.data.push_back(static_cast(merged.h[i])); + mtz.data.push_back(static_cast(merged.k[i])); + mtz.data.push_back(static_cast(merged.l[i])); + mtz.data.push_back(merged.i_mean[i]); + mtz.data.push_back(merged.sigma_i_mean[i]); + if (has_anom) { + mtz.data.push_back(merged.i_plus[i]); + mtz.data.push_back(merged.sigma_i_plus[i]); + mtz.data.push_back(merged.i_minus[i]); + mtz.data.push_back(merged.sigma_i_minus[i]); + } + mtz.data.push_back(merged.f[i]); + mtz.data.push_back(merged.sigma_f[i]); + if (has_anom) { + mtz.data.push_back(merged.f_plus[i]); + mtz.data.push_back(merged.sigma_f_plus[i]); + mtz.data.push_back(merged.f_minus[i]); + mtz.data.push_back(merged.sigma_f_minus[i]); + } + mtz.data.push_back(static_cast(merged.free_flag[i])); + } + mtz.write_to_file(filename); +} + +void WriteMapCoefficientsMtz(const MapCoefficientsMessage &coefficients, const std::string &filename) { + gemmi::Mtz mtz(true); + mtz.spacegroup = gemmi::find_spacegroup_by_name(coefficients.space_group_name); + mtz.set_cell_for_all(coefficients.unit_cell); + mtz.add_dataset("model_validation"); + mtz.datasets.back().wavelength = coefficients.wavelength_A; + mtz.add_column("FP", 'F', -1, -1, false); + mtz.add_column("FC", 'F', -1, -1, false); + mtz.add_column("PHIC", 'P', -1, -1, false); + mtz.add_column("FWT", 'F', -1, -1, false); + mtz.add_column("PHWT", 'P', -1, -1, false); + mtz.add_column("DELFWT", 'F', -1, -1, false); + mtz.add_column("PHDELWT", 'P', -1, -1, false); + // The figure of merit the coefficients carry, so the weighting can be read off - and undone - + // from the file rather than having to be taken on trust. + mtz.add_column("FOM", 'W', -1, -1, false); + mtz.add_column("FREE", 'I', -1, -1, false); + const size_t n = coefficients.h.size(); + mtz.data.reserve(n * 12); + for (size_t i = 0; i < n; i++) + mtz.data.insert(mtz.data.end(), {static_cast(coefficients.h[i]), + static_cast(coefficients.k[i]), + static_cast(coefficients.l[i]), + coefficients.fp[i], coefficients.fc[i], coefficients.phic[i], + coefficients.fwt[i], coefficients.phwt[i], + coefficients.delfwt[i], coefficients.phdelwt[i], + coefficients.fom[i], + static_cast(coefficients.free_flag[i])}); + mtz.nreflections = static_cast(n); + mtz.write_to_file(filename); +} + +gemmi::Structure PlacedModelStructure(const PlacedModelMessage &model) { + gemmi::Structure st; + st.name = model.name; + st.cell = model.unit_cell; + st.spacegroup_hm = model.model_space_group_name; + const bool has_aniso = !model.aniso.empty(); + gemmi::Model *m = nullptr; + gemmi::Chain *ch = nullptr; + gemmi::Residue *res = nullptr; + for (size_t i = 0; i < model.atom_name.size(); i++) { + if (m == nullptr || m->num != model.model_number[i]) { + m = &st.models.emplace_back(model.model_number[i]); + ch = nullptr; + } + if (ch == nullptr || ch->name != model.chain[i]) { + ch = &m->chains.emplace_back(model.chain[i]); + res = nullptr; + } + const gemmi::SeqId seqid(model.seq_num[i], static_cast(model.icode[i])); + if (res == nullptr || res->name != model.residue_name[i] || res->seqid != seqid + || res->het_flag != static_cast(model.het_flag[i])) { + res = &ch->residues.emplace_back(gemmi::ResidueId{seqid, {}, model.residue_name[i]}); + res->het_flag = static_cast(model.het_flag[i]); + } + gemmi::Atom &a = res->atoms.emplace_back(); + a.name = model.atom_name[i]; + a.element = gemmi::Element(model.element[i]); + a.altloc = static_cast(model.altloc[i]); + a.charge = static_cast(model.charge[i]); + a.pos = gemmi::Position(model.x[i], model.y[i], model.z[i]); + a.b_iso = model.b_iso[i]; + a.occ = model.occupancy[i]; + if (has_aniso) + a.aniso = {model.aniso[6 * i], model.aniso[6 * i + 1], model.aniso[6 * i + 2], + model.aniso[6 * i + 3], model.aniso[6 * i + 4], model.aniso[6 * i + 5]}; + } + st.setup_cell_images(); + return st; +} + +namespace { + +double WallClockSeconds() { + return std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(); +} + +// Rename tmp onto final, unless final is there already and may not be overwritten. +void RenameIntoPlace(const std::string &tmp, const std::string &final_name, bool overwrite, Logger &logger) { + if (!overwrite && std::filesystem::exists(final_name)) { + std::filesystem::remove(tmp); + logger.Error("{} already exists and overwrite is off - not written", final_name); + return; + } + std::filesystem::rename(tmp, final_name); + logger.Info("{} written, renamed into place at wall clock {:.3f} s", final_name, WallClockSeconds()); +} + +} // namespace + +void WriteRugnuxResultFiles(const EndMessage &end, const std::string &file_prefix, bool overwrite, + Logger &logger) { + // A file that cannot be written costs only itself: the master is already in place. + try { + if (end.merged) { + WriteMergedMtz(*end.merged, file_prefix + ".mtz.tmp"); + RenameIntoPlace(file_prefix + ".mtz.tmp", file_prefix + ".mtz", overwrite, logger); + } + } catch (const std::exception &e) { + logger.Error("Cannot write {}.mtz: {}", file_prefix, e.what()); + } + try { + if (end.map_coefficients) { + WriteMapCoefficientsMtz(*end.map_coefficients, file_prefix + "_maps.mtz.tmp"); + RenameIntoPlace(file_prefix + "_maps.mtz.tmp", file_prefix + "_maps.mtz", overwrite, logger); + } + } catch (const std::exception &e) { + logger.Error("Cannot write {}_maps.mtz: {}", file_prefix, e.what()); + } + try { + if (end.placed_model) { + const PlacedModelMessage &model = *end.placed_model; + // WritePlacedModel appends _model.cif and _model.pdb to the prefix it is given. + const std::string tmp_prefix = file_prefix + ".tmp"; + WritePlacedModel(PlacedModelStructure(model), model.unit_cell, + *gemmi::find_spacegroup_by_name(model.space_group_name), tmp_prefix, logger); + for (const std::string suffix : {"_model.cif", "_model.pdb"}) + if (std::filesystem::exists(tmp_prefix + suffix)) + RenameIntoPlace(tmp_prefix + suffix, file_prefix + suffix, overwrite, logger); + } + } catch (const std::exception &e) { + logger.Error("Cannot write {}_model.cif: {}", file_prefix, e.what()); + } +} diff --git a/writer/WriteRugnuxFiles.h b/writer/WriteRugnuxFiles.h new file mode 100644 index 000000000..f2fc6fe5f --- /dev/null +++ b/writer/WriteRugnuxFiles.h @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "gemmi/model.hpp" + +#include "../common/JFJochMessages.h" + +class Logger; + +// The files rugnux writes beside its merged output, made from the structured END-message blocks +// (common/JFJochMessages.h), so that jfjoch_writer can write them with no analysis library. rugnux +// writes its own .mtz and _maps.mtz through the same two functions. + +// .mtz: H K L IMEAN SIGIMEAN [I(+) SIGI(+) I(-) SIGI(-)] F SIGF [F(+) SIGF(+) F(-) SIGF(-)] +// FreeR_flag, the bracketed columns only where the block has them. +void WriteMergedMtz(const MergedReflectionsMessage &merged, const std::string &filename); + +// _maps.mtz: H K L FP FC PHIC FWT PHWT DELFWT PHDELWT FOM FREE. +void WriteMapCoefficientsMtz(const MapCoefficientsMessage &coefficients, const std::string &filename); + +// The model the block describes, for WritePlacedModel (writer/WriteModel.h). +gemmi::Structure PlacedModelStructure(const PlacedModelMessage &model); + +// The three files above, from whichever blocks `end` carries, at .mtz, _maps.mtz and +// _model.cif (WritePlacedModel adds _model.pdb where a PDB can hold the cell). Each is written under a +// temporary name and renamed into place. An existing file is replaced only with `overwrite`; otherwise +// it is kept and the new one is not written. +void WriteRugnuxResultFiles(const EndMessage &end, const std::string &file_prefix, bool overwrite, + Logger &logger); From 3a44779d798565895755b2c4e5da193821bbe4c2 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 12:20:38 +0200 Subject: [PATCH 03/19] broker: rugnux live - rugnux runs inside the broker while a DECTRIS rotation collection arrives (demo prototype) A DECTRIS rotation collection, with the mode switched on (PUT /config/rugnux), goes to JFJochReceiverRugnux instead of JFJochReceiverLite: the images are forwarded as before, without the per-image analysis, and one rugnux thread reads them back from the image buffer (JFJochImageBufferReader) as they arrive. END waits for rugnux and carries its lattice, refined geometry and per-image results. A small second run on the first 60 degrees reports early whether there is a lattice. The live state, steps and result are at GET /rugnux/status and on a new "Rugnux live" frontend page. rugnux: ProcessConfig::fast_live takes every decision where it is first made and does nothing again once the sweep is in (near-tie re-checks and the run made again, rotation-scale walk, metric-symmetry arms, geometry walk, the supercell and quality guards, the doubled cell), so the run ends a fixed time after the last image. A cut (/cancel, or deadline_s after the last image) hands over what is established. Off by default: the CLI and the battery are unchanged (--fast-live is hidden, with --simulate-live). With the mode off the broker is unchanged. Not built yet. Co-Authored-By: Claude Opus 5.5 --- broker/JFJochBrokerHttp.cpp | 57 ++- broker/JFJochBrokerHttp.h | 8 + broker/JFJochBrokerParser.cpp | 9 + broker/JFJochBrokerParser.h | 2 + broker/JFJochServices.cpp | 49 ++ broker/JFJochServices.h | 16 + broker/JFJochStateMachine.cpp | 29 ++ broker/JFJochStateMachine.h | 6 + broker/OpenAPIConvert.cpp | 54 ++ broker/OpenAPIConvert.h | 3 + broker/gen/model/Jfjoch_settings.cpp | 32 +- broker/gen/model/Jfjoch_settings.h | 10 + broker/gen/model/Rugnux_live_status.cpp | 484 ++++++++++++++++++ broker/gen/model/Rugnux_live_status.h | 204 ++++++++ broker/gen/model/Rugnux_live_step.cpp | 168 ++++++ broker/gen/model/Rugnux_live_step.h | 101 ++++ broker/gen/model/Rugnux_settings.cpp | 192 +++++++ broker/gen/model/Rugnux_settings.h | 96 ++++ broker/gen/model/Rugnux_status.cpp | 177 +++++++ broker/gen/model/Rugnux_status.h | 108 ++++ broker/gen/model/Rugnux_summary_row.cpp | 105 ++++ broker/gen/model/Rugnux_summary_row.h | 84 +++ broker/jfjoch_api.yaml | 293 +++++++++++ broker/jfjoch_broker.cpp | 1 + common/ParallelFor.h | 27 + docs/JFJOCH_BROKER.md | 21 + docs/python_client/README.md | 11 + docs/python_client/docs/DefaultApi.md | 483 +++++++++++++++++ docs/python_client/docs/JfjochSettings.md | 1 + docs/python_client/docs/RugnuxLiveStatus.md | 43 ++ docs/python_client/docs/RugnuxLiveStep.md | 32 ++ docs/python_client/docs/RugnuxSettings.md | 31 ++ docs/python_client/docs/RugnuxStatus.md | 32 ++ docs/python_client/docs/RugnuxSummaryRow.md | 30 ++ frontend/src/App.tsx | 11 +- .../src/client/@tanstack/react-query.gen.ts | 112 +++- frontend/src/client/index.ts | 4 +- frontend/src/client/sdk.gen.ts | 104 +++- frontend/src/client/types.gen.ts | 295 +++++++++++ frontend/src/client/zod.gen.ts | 83 ++- frontend/src/components/RugnuxPage.tsx | 166 ++++++ frontend/src/components/RugnuxParts.tsx | 88 ++++ frontend/src/components/RugnuxSettings.tsx | 94 ++++ frontend/src/components/RugnuxStatus.tsx | 62 +++ frontend/src/components/RugnuxTimeline.tsx | 174 +++++++ reader/CMakeLists.txt | 2 + reader/JFJochImageBufferReader.cpp | 69 +++ reader/JFJochImageBufferReader.h | 40 ++ reader/JFJochStartMessageReader.cpp | 131 +++-- reader/JFJochStartMessageReader.h | 5 + receiver/CMakeLists.txt | 4 +- receiver/JFJochReceiver.cpp | 1 + receiver/JFJochReceiver.h | 3 + receiver/JFJochReceiverRugnux.cpp | 482 +++++++++++++++++ receiver/JFJochReceiverRugnux.h | 87 ++++ receiver/JFJochReceiverService.cpp | 28 + receiver/JFJochReceiverService.h | 13 + receiver/RugnuxLiveStatus.cpp | 95 ++++ receiver/RugnuxLiveStatus.h | 71 +++ rugnux/CMakeLists.txt | 2 + rugnux/Rugnux.cpp | 17 +- rugnux/Rugnux.h | 15 + rugnux/RugnuxImagePass.cpp | 16 +- rugnux/RugnuxLive.cpp | 57 +++ rugnux/RugnuxLive.h | 21 + rugnux/RugnuxPasses.cpp | 23 +- rugnux/RugnuxScaleMerge.cpp | 12 +- rugnux/rugnux_cli.cpp | 9 + 68 files changed, 5295 insertions(+), 100 deletions(-) create mode 100644 broker/gen/model/Rugnux_live_status.cpp create mode 100644 broker/gen/model/Rugnux_live_status.h create mode 100644 broker/gen/model/Rugnux_live_step.cpp create mode 100644 broker/gen/model/Rugnux_live_step.h create mode 100644 broker/gen/model/Rugnux_settings.cpp create mode 100644 broker/gen/model/Rugnux_settings.h create mode 100644 broker/gen/model/Rugnux_status.cpp create mode 100644 broker/gen/model/Rugnux_status.h create mode 100644 broker/gen/model/Rugnux_summary_row.cpp create mode 100644 broker/gen/model/Rugnux_summary_row.h create mode 100644 docs/python_client/docs/RugnuxLiveStatus.md create mode 100644 docs/python_client/docs/RugnuxLiveStep.md create mode 100644 docs/python_client/docs/RugnuxSettings.md create mode 100644 docs/python_client/docs/RugnuxStatus.md create mode 100644 docs/python_client/docs/RugnuxSummaryRow.md create mode 100644 frontend/src/components/RugnuxPage.tsx create mode 100644 frontend/src/components/RugnuxParts.tsx create mode 100644 frontend/src/components/RugnuxSettings.tsx create mode 100644 frontend/src/components/RugnuxStatus.tsx create mode 100644 frontend/src/components/RugnuxTimeline.tsx create mode 100644 reader/JFJochImageBufferReader.cpp create mode 100644 reader/JFJochImageBufferReader.h create mode 100644 receiver/JFJochReceiverRugnux.cpp create mode 100644 receiver/JFJochReceiverRugnux.h create mode 100644 receiver/RugnuxLiveStatus.cpp create mode 100644 receiver/RugnuxLiveStatus.h create mode 100644 rugnux/RugnuxLive.cpp create mode 100644 rugnux/RugnuxLive.h diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index b722a6600..88cd13b6c 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -152,7 +153,10 @@ static const std::set PROTECTED_PATHS{ "/image_buffer/image.tiff", "/preview/plot", "/preview/plot.bin", - "/live/events" + "/live/events", + "/config/rugnux", + "/config/rugnux/model", + "/rugnux/status" }; std::optional JFJochBrokerHttp::AuthorizationHeader(const httplib::Request &req) { @@ -251,6 +255,11 @@ void JFJochBrokerHttp::register_routes(httplib::Server &server) { server.Post("/config/image_format/raw", bind_noarg(&JFJochBrokerHttp::config_image_format_raw_post)); server.Get("/config/indexing", bind_noarg(&JFJochBrokerHttp::config_indexing_get)); server.Put("/config/indexing", bind_json(&JFJochBrokerHttp::config_indexing_put, Indexing_settings{})); + server.Get("/config/rugnux", bind_noarg(&JFJochBrokerHttp::config_rugnux_get)); + server.Put("/config/rugnux", bind_json(&JFJochBrokerHttp::config_rugnux_put, Rugnux_settings{})); + server.Put("/config/rugnux/model", bind_req(&JFJochBrokerHttp::config_rugnux_model_put)); + server.Delete("/config/rugnux/model", bind_noarg(&JFJochBrokerHttp::config_rugnux_model_delete)); + server.Get("/rugnux/status", bind_noarg(&JFJochBrokerHttp::rugnux_status_get)); server.Get("/config/bragg_integration", bind_noarg(&JFJochBrokerHttp::config_bragg_integration_get)); server.Put("/config/bragg_integration", bind_json(&JFJochBrokerHttp::config_bragg_integration_put, Bragg_integration_settings{})); server.Get("/config/instrument", bind_noarg(&JFJochBrokerHttp::config_instrument_get)); @@ -963,6 +972,52 @@ void JFJochBrokerHttp::config_indexing_put(const Indexing_settings &indexingSett response.status = 200; } +void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) { + const RugnuxLiveSettings s = state_machine.GetRugnuxLive(); + Rugnux_status ret; + ret.setEnable(s.enable); + ret.setReservedCpus(s.reserved_cpus); + ret.setDeadlineS(s.deadline_s); + ret.setModelLoaded(!s.model_path.empty()); + if (!s.model_path.empty()) + ret.setModelSizeBytes(static_cast(std::filesystem::file_size(s.model_path))); + ProcessOutput(ret, response); +} + +void JFJochBrokerHttp::config_rugnux_put(const Rugnux_settings &rugnuxSettings, + httplib::Response &response) { + state_machine.SetRugnuxLive(rugnuxSettings.isEnable(), rugnuxSettings.getReservedCpus(), + rugnuxSettings.getDeadlineS()); + response.status = 200; +} + +void JFJochBrokerHttp::config_rugnux_model_put(const httplib::Request &request, + httplib::Response &response) { + if (request.body.empty()) { + response.status = 400; + response.set_content("Request body cannot be empty", "text/plain"); + return; + } + if (request.body.size() > 64 * 1024 * 1024) { + response.status = 413; + response.set_content("Model cannot be larger than 64 MiB", "text/plain"); + return; + } + state_machine.SetRugnuxModel(request.body); + logger.Info("Rugnux model uploaded ({} bytes)", request.body.size()); + response.status = 200; +} + +void JFJochBrokerHttp::config_rugnux_model_delete(httplib::Response &response) { + state_machine.SetRugnuxModel(""); + logger.Info("Rugnux model removed"); + response.status = 200; +} + +void JFJochBrokerHttp::rugnux_status_get(httplib::Response &response) { + ProcessOutput(Convert(state_machine.GetRugnuxStatus()), response); +} + void JFJochBrokerHttp::config_bragg_integration_get(httplib::Response &response) { ProcessOutput(Convert(state_machine.GetBraggIntegrationSettings()), response); } diff --git a/broker/JFJochBrokerHttp.h b/broker/JFJochBrokerHttp.h index 8d42a4593..3528ab5d3 100644 --- a/broker/JFJochBrokerHttp.h +++ b/broker/JFJochBrokerHttp.h @@ -34,6 +34,8 @@ #include "gen/model/Jfjoch_statistics.h" #include "gen/model/Plots.h" #include "gen/model/Roi_definitions.h" +#include "gen/model/Rugnux_settings.h" +#include "gen/model/Rugnux_status.h" #include "gen/model/Scan_result.h" #include "gen/model/Spot_finding_settings.h" #include "gen/model/Zeromq_metadata_settings.h" @@ -198,6 +200,12 @@ class JFJochBrokerHttp { void config_indexing_get(httplib::Response &response); void config_indexing_put(const org::openapitools::server::model::Indexing_settings &indexingSettings, httplib::Response &response); + void config_rugnux_get(httplib::Response &response); + void config_rugnux_put(const org::openapitools::server::model::Rugnux_settings &rugnuxSettings, + httplib::Response &response); + void config_rugnux_model_put(const httplib::Request &request, httplib::Response &response); + void config_rugnux_model_delete(httplib::Response &response); + void rugnux_status_get(httplib::Response &response); void config_bragg_integration_get(httplib::Response &response); void config_bragg_integration_put(const org::openapitools::server::model::Bragg_integration_settings &braggIntegrationSettings, httplib::Response &response); diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 7f4efcfbc..45409ca9c 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -276,6 +276,15 @@ void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settin } } +void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochServices &services) { + if (input.rugnuxIsSet()) + services.SetRugnuxLive(RugnuxLiveSettings{ + .enable = input.getRugnux().isEnable(), + .reserved_cpus = input.getRugnux().getReservedCpus(), + .deadline_s = input.getRugnux().getDeadlineS() + }); +} + SpotFindingSettings ParseSpotFindingSettings(const org::openapitools::server::model::Jfjoch_settings &input) { if (input.spotFindingIsSet()) return Convert(input.getSpotFinding()); diff --git a/broker/JFJochBrokerParser.h b/broker/JFJochBrokerParser.h index f5f2f8399..aff60a211 100644 --- a/broker/JFJochBrokerParser.h +++ b/broker/JFJochBrokerParser.h @@ -7,6 +7,7 @@ #include "../acquisition_device/AcquisitionDeviceGroup.h" #include "../image_pusher/ImagePusher.h" #include "../receiver/JFJochReceiverService.h" +#include "JFJochServices.h" #include "gen/model/Jfjoch_settings.h" DetectorGeometryModular ParseStandardDetectorGeometry(const org::openapitools::server::model::Detector &j); @@ -20,4 +21,5 @@ std::unique_ptr ParseImagePusher(const org::openapitools::server::m void ParseAcquisitionDeviceGroup(const org::openapitools::server::model::Jfjoch_settings &input, AcquisitionDeviceGroup &aq_devices); void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochReceiverService &service); +void ParseRugnuxSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochServices &services); SpotFindingSettings ParseSpotFindingSettings(const org::openapitools::server::model::Jfjoch_settings &input); diff --git a/broker/JFJochServices.cpp b/broker/JFJochServices.cpp index 8eb9edb96..8bd4ff15c 100644 --- a/broker/JFJochServices.cpp +++ b/broker/JFJochServices.cpp @@ -2,6 +2,10 @@ // SPDX-License-Identifier: GPL-3.0-only #include "JFJochServices.h" + +#include +#include +#include #include "../common/JFJochException.h" #include "../detector_control/SLSDetectorWrapper.h" #include "../detector_control/DectrisDetectorWrapper.h" @@ -370,6 +374,51 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) { receiver->Indexing(input); } +void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) { + if (receiver) + receiver->RugnuxLive(input.enable, input.model_path, input.reserved_cpus, input.deadline_s); + rugnux = input; +} + +RugnuxLiveSettings JFJochServices::GetRugnuxLive() const { + return rugnux; +} + +RugnuxLiveSnapshot JFJochServices::GetRugnuxStatus() const { + if (receiver) + return receiver->GetRugnuxStatus(); + return {}; +} + +void JFJochServices::SetRugnuxModel(const std::string &model) { + RugnuxLiveSettings s = rugnux; + if (!s.model_path.empty()) + std::filesystem::remove(s.model_path); + s.model_path.clear(); + + if (!model.empty()) { + if (rugnux_model_dir.empty()) { + char dir[] = "/tmp/jfjoch_rugnux_XXXXXX"; // mkdtemp creates it 0700 + if (mkdtemp(dir) == nullptr) + throw JFJochException(JFJochExceptionCategory::FileWriteError, + "Cannot create directory for the rugnux model"); + rugnux_model_dir = dir; + } + // gemmi detects PDB/mmCIF from the content, but gzip only from the extension + const bool gzipped = model.size() >= 2 && model[0] == '\x1f' && model[1] == '\x8b'; + s.model_path = rugnux_model_dir + (gzipped ? "/model.gz" : "/model"); + { + std::ofstream f(s.model_path, std::ios::binary); + f.write(model.data(), static_cast(model.size())); + if (!f) + throw JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot write the rugnux model"); + } + std::filesystem::permissions(s.model_path, + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); + } + SetRugnuxLive(s); +} + LiveCollection *JFJochServices::GetLiveCollection() { return receiver ? receiver->GetLiveCollection() : nullptr; } diff --git a/broker/JFJochServices.h b/broker/JFJochServices.h index 01f051cc0..3a732c20d 100644 --- a/broker/JFJochServices.h +++ b/broker/JFJochServices.h @@ -14,6 +14,13 @@ struct JFJochServicesOutput { JFJochReceiverOutput receiver_output; }; +struct RugnuxLiveSettings { + bool enable = false; + int64_t reserved_cpus = 4; + double deadline_s = 0; // the analysis is cut this long after the last image (0 = never) + std::string model_path; // empty = no model +}; + class JFJochServices { mutable std::shared_timed_mutex detector_mutex; @@ -22,6 +29,9 @@ class JFJochServices { std::shared_ptr image_puller; std::atomic cannot_stop_detector = false; + + RugnuxLiveSettings rugnux; + std::string rugnux_model_dir; // private directory, created at the first model upload Logger &logger; public: explicit JFJochServices(Logger &in_logger); @@ -76,6 +86,12 @@ public: void SetupIndexing(const IndexingSettings& input); + void SetRugnuxLive(const RugnuxLiveSettings &input); + RugnuxLiveSettings GetRugnuxLive() const; + // The model file is written 0600 into a private directory; an empty model removes it + void SetRugnuxModel(const std::string &model); + RugnuxLiveSnapshot GetRugnuxStatus() const; + ImagePusherStatus GetImagePusherStatus() const; LiveCollection *GetLiveCollection(); }; diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index 1b93ad2a7..47dacea7f 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -1209,6 +1209,35 @@ IndexingSettings JFJochStateMachine::GetIndexingSettings() const { return experiment.GetIndexingSettings(); } +void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s) { + std::unique_lock ul(m); + if (IsRunning()) + throw WrongDAQStateException("Cannot change rugnux settings during data collection"); + std::unique_lock ul2(rugnux_mutex); + RugnuxLiveSettings s = services.GetRugnuxLive(); + s.enable = enable; + s.reserved_cpus = reserved_cpus; + s.deadline_s = deadline_s; + services.SetRugnuxLive(s); +} + +void JFJochStateMachine::SetRugnuxModel(const std::string &model) { + std::unique_lock ul(m); + if (IsRunning()) + throw WrongDAQStateException("Cannot change rugnux model during data collection"); + std::unique_lock ul2(rugnux_mutex); + services.SetRugnuxModel(model); +} + +RugnuxLiveSettings JFJochStateMachine::GetRugnuxLive() const { + std::unique_lock ul(rugnux_mutex); + return services.GetRugnuxLive(); +} + +RugnuxLiveSnapshot JFJochStateMachine::GetRugnuxStatus() const { + return services.GetRugnuxStatus(); +} + void JFJochStateMachine::SetIndexingSettings(const IndexingSettings &input) { std::unique_lock ul(m); if (IsRunning()) diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index a28d9cecc..ca053658a 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -90,6 +90,7 @@ class JFJochStateMachine { mutable std::mutex experiment_file_writer_settings_mutex; mutable std::mutex experiment_indexing_settings_mutex; mutable std::mutex experiment_dark_mask_settings_mutex; + mutable std::mutex rugnux_mutex; DiffractionExperiment experiment; // mutex m is protecting: @@ -247,6 +248,11 @@ public: void SetIndexingSettings(const IndexingSettings &input); IndexingSettings GetIndexingSettings() const; + void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s); + void SetRugnuxModel(const std::string &model); // empty = remove the model + RugnuxLiveSettings GetRugnuxLive() const; + RugnuxLiveSnapshot GetRugnuxStatus() const; + void SetBraggIntegrationSettings(const BraggIntegrationSettings &input); BraggIntegrationSettings GetBraggIntegrationSettings() const; PixelMaskStatistics GetPixelMaskStatistics() const; diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index 8cf6f8e6d..5f8a85376 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -1221,6 +1221,60 @@ DarkMaskSettings Convert(const org::openapitools::server::model::Dark_mask_setti return ret; } +org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSnapshot& input) { + org::openapitools::server::model::Rugnux_live_status ret; + switch (input.state) { + case RugnuxLiveState::Idle: ret.setState("Idle"); break; + case RugnuxLiveState::Running: ret.setState("Running"); break; + case RugnuxLiveState::Done: ret.setState("Done"); break; + case RugnuxLiveState::Failed: ret.setState("Failed"); break; + case RugnuxLiveState::Cancelled: ret.setState("Cancelled"); break; + case RugnuxLiveState::Skipped: ret.setState("Skipped"); break; + } + ret.setMessage(input.message); + ret.setImagesExpected(input.images_expected); + ret.setImagesReceived(input.images_received); + ret.setElapsedS(input.elapsed_s); + ret.setProbe(input.probe); + if (input.probe_s) + ret.setProbeS(input.probe_s.value()); + if (input.step_progress) + ret.setStepProgress(input.step_progress.value()); + + std::vector steps; + for (const auto &s: input.steps) { + org::openapitools::server::model::Rugnux_live_step step; + step.setTitle(s.title); + step.setStartedS(s.started_s); + if (s.duration_s) + step.setDurationS(s.duration_s.value()); + step.setNotes(s.notes); + steps.push_back(step); + } + ret.setSteps(steps); + + if (input.last_image_s) + ret.setLastImageS(input.last_image_s.value()); + ret.setCut(input.cut); + if (input.cut_s) + ret.setCutS(input.cut_s.value()); + if (input.last_image_to_written_s) + ret.setLastImageToWrittenS(input.last_image_to_written_s.value()); + ret.setVerdict(input.verdict); + ret.setVerdictText(input.verdict_text); + + std::vector summary; + for (const auto &[label, value]: input.summary) { + org::openapitools::server::model::Rugnux_summary_row row; + row.setLabel(label); + row.setValue(value); + summary.push_back(row); + } + ret.setSummary(summary); + ret.setWarnings(input.warnings); + return ret; +} + org::openapitools::server::model::Image_pusher_status Convert(const ImagePusherStatus& input) { org::openapitools::server::model::Image_pusher_status ret; ret.setAddr(input.address); diff --git a/broker/OpenAPIConvert.h b/broker/OpenAPIConvert.h index fd36ea1da..6556cc0bc 100644 --- a/broker/OpenAPIConvert.h +++ b/broker/OpenAPIConvert.h @@ -30,6 +30,7 @@ #include "gen/model/Indexing_settings.h" #include "gen/model/Bragg_integration_settings.h" #include "gen/model/Scan_result.h" +#include "gen/model/Rugnux_live_status.h" #include "../common/JFJochMessages.h" #include "../common/DatasetSettings.h" @@ -41,6 +42,7 @@ #include "../common/InstrumentMetadata.h" #include "../common/ScanResult.h" #include "../image_pusher/ImagePusher.h" +#include "../receiver/RugnuxLiveStatus.h" SpotFindingSettings Convert(const org::openapitools::server::model::Spot_finding_settings &input); org::openapitools::server::model::Spot_finding_settings Convert(const SpotFindingSettings &input); @@ -99,4 +101,5 @@ org::openapitools::server::model::Dark_mask_settings Convert(const DarkMaskSetti DarkMaskSettings Convert(const org::openapitools::server::model::Dark_mask_settings& input); org::openapitools::server::model::Image_pusher_status Convert(const ImagePusherStatus& input); +org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSnapshot& input); diff --git a/broker/gen/model/Jfjoch_settings.cpp b/broker/gen/model/Jfjoch_settings.cpp index fb16b7494..9ea19e055 100644 --- a/broker/gen/model/Jfjoch_settings.cpp +++ b/broker/gen/model/Jfjoch_settings.cpp @@ -44,6 +44,7 @@ Jfjoch_settings::Jfjoch_settings() m_Zeromq_previewIsSet = false; m_Zeromq_metadataIsSet = false; m_Dark_maskIsSet = false; + m_RugnuxIsSet = false; } @@ -141,7 +142,7 @@ bool Jfjoch_settings::validate(std::stringstream& msg, const std::string& pathPr } } - + return success; } @@ -211,7 +212,10 @@ bool Jfjoch_settings::operator==(const Jfjoch_settings& rhs) const ((!zeromqMetadataIsSet() && !rhs.zeromqMetadataIsSet()) || (zeromqMetadataIsSet() && rhs.zeromqMetadataIsSet() && getZeromqMetadata() == rhs.getZeromqMetadata())) && - ((!darkMaskIsSet() && !rhs.darkMaskIsSet()) || (darkMaskIsSet() && rhs.darkMaskIsSet() && getDarkMask() == rhs.getDarkMask())) + ((!darkMaskIsSet() && !rhs.darkMaskIsSet()) || (darkMaskIsSet() && rhs.darkMaskIsSet() && getDarkMask() == rhs.getDarkMask())) && + + + ((!rugnuxIsSet() && !rhs.rugnuxIsSet()) || (rugnuxIsSet() && rhs.rugnuxIsSet() && getRugnux() == rhs.getRugnux())) ; } @@ -263,6 +267,8 @@ void to_json(nlohmann::json& j, const Jfjoch_settings& o) j["zeromq_metadata"] = o.m_Zeromq_metadata; if(o.darkMaskIsSet()) j["dark_mask"] = o.m_Dark_mask; + if(o.rugnuxIsSet()) + j["rugnux"] = o.m_Rugnux; } @@ -361,6 +367,11 @@ void from_json(const nlohmann::json& j, Jfjoch_settings& o) j.at("dark_mask").get_to(o.m_Dark_mask); o.m_Dark_maskIsSet = true; } + if(j.find("rugnux") != j.end()) + { + j.at("rugnux").get_to(o.m_Rugnux); + o.m_RugnuxIsSet = true; + } } @@ -694,6 +705,23 @@ void Jfjoch_settings::unsetDark_mask() { m_Dark_maskIsSet = false; } +org::openapitools::server::model::Rugnux_settings Jfjoch_settings::getRugnux() const +{ + return m_Rugnux; +} +void Jfjoch_settings::setRugnux(org::openapitools::server::model::Rugnux_settings const& value) +{ + m_Rugnux = value; + m_RugnuxIsSet = true; +} +bool Jfjoch_settings::rugnuxIsSet() const +{ + return m_RugnuxIsSet; +} +void Jfjoch_settings::unsetRugnux() +{ + m_RugnuxIsSet = false; +} } // namespace org::openapitools::server::model diff --git a/broker/gen/model/Jfjoch_settings.h b/broker/gen/model/Jfjoch_settings.h index 434ebd2c4..f9033a435 100644 --- a/broker/gen/model/Jfjoch_settings.h +++ b/broker/gen/model/Jfjoch_settings.h @@ -31,6 +31,7 @@ #include "Image_format_settings.h" #include "Zeromq_metadata_settings.h" #include "Detector_settings.h" +#include "Rugnux_settings.h" #include #include "Indexing_settings.h" #include "Detector.h" @@ -216,6 +217,13 @@ public: void setDarkMask(org::openapitools::server::model::Dark_mask_settings const& value); bool darkMaskIsSet() const; void unsetDark_mask(); + /// + /// + /// + org::openapitools::server::model::Rugnux_settings getRugnux() const; + void setRugnux(org::openapitools::server::model::Rugnux_settings const& value); + bool rugnuxIsSet() const; + void unsetRugnux(); friend void to_json(nlohmann::json& j, const Jfjoch_settings& o); friend void from_json(const nlohmann::json& j, Jfjoch_settings& o); @@ -262,6 +270,8 @@ protected: bool m_Zeromq_metadataIsSet; org::openapitools::server::model::Dark_mask_settings m_Dark_mask; bool m_Dark_maskIsSet; + org::openapitools::server::model::Rugnux_settings m_Rugnux; + bool m_RugnuxIsSet; }; diff --git a/broker/gen/model/Rugnux_live_status.cpp b/broker/gen/model/Rugnux_live_status.cpp new file mode 100644 index 000000000..765004d94 --- /dev/null +++ b/broker/gen/model/Rugnux_live_status.cpp @@ -0,0 +1,484 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_live_status.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_live_status::Rugnux_live_status() +{ + m_State = ""; + m_Probe = ""; + m_Probe_s = 0.0; + m_Probe_sIsSet = false; + m_Message = ""; + m_Images_expected = 0L; + m_Images_received = 0L; + m_Elapsed_s = 0.0; + m_Step_progress = 0.0f; + m_Step_progressIsSet = false; + m_Last_image_s = 0.0; + m_Last_image_sIsSet = false; + m_Cut = ""; + m_CutIsSet = false; + m_Cut_s = 0.0; + m_Cut_sIsSet = false; + m_Last_image_to_written_s = 0.0; + m_Last_image_to_written_sIsSet = false; + m_Verdict = ""; + m_Verdict_text = ""; + +} + +void Rugnux_live_status::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_live_status::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_live_status::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_status" : pathPrefix; + + + if (stepProgressIsSet()) + { + const float& value = m_Step_progress; + const std::string currentValuePath = _pathPrefix + ".stepProgress"; + + + if (value < static_cast(0.0)) + { + success = false; + msg << currentValuePath << ": must be greater than or equal to 0.0;"; + } + if (value > static_cast(1.0)) + { + success = false; + msg << currentValuePath << ": must be less than or equal to 1.0;"; + } + + } + + + /* Steps */ { + const std::vector& value = m_Steps; + const std::string currentValuePath = _pathPrefix + ".steps"; + + + { // Recursive validation of array elements + const std::string oldValuePath = currentValuePath; + int i = 0; + for (const org::openapitools::server::model::Rugnux_live_step& value : value) + { + const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]"; + + success = value.validate(msg, currentValuePath + ".steps") && success; + + i++; + } + } + + } + + + /* Summary */ { + const std::vector& value = m_Summary; + const std::string currentValuePath = _pathPrefix + ".summary"; + + + { // Recursive validation of array elements + const std::string oldValuePath = currentValuePath; + int i = 0; + for (const org::openapitools::server::model::Rugnux_summary_row& value : value) + { + const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]"; + + success = value.validate(msg, currentValuePath + ".summary") && success; + + i++; + } + } + + } + + + /* Warnings */ { + const std::vector& value = m_Warnings; + const std::string currentValuePath = _pathPrefix + ".warnings"; + + + { // Recursive validation of array elements + const std::string oldValuePath = currentValuePath; + int i = 0; + for (const std::string& value : value) + { + const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]"; + + + + i++; + } + } + + } + + return success; +} + +bool Rugnux_live_status::operator==(const Rugnux_live_status& rhs) const +{ + return + + + (getState() == rhs.getState()) + && + + (getProbe() == rhs.getProbe()) + && + + + ((!probeSIsSet() && !rhs.probeSIsSet()) || (probeSIsSet() && rhs.probeSIsSet() && getProbeS() == rhs.getProbeS())) && + + (getMessage() == rhs.getMessage()) + && + + (getImagesExpected() == rhs.getImagesExpected()) + && + + (getImagesReceived() == rhs.getImagesReceived()) + && + + (getElapsedS() == rhs.getElapsedS()) + && + + + ((!stepProgressIsSet() && !rhs.stepProgressIsSet()) || (stepProgressIsSet() && rhs.stepProgressIsSet() && getStepProgress() == rhs.getStepProgress())) && + + (getSteps() == rhs.getSteps()) + && + + + ((!lastImageSIsSet() && !rhs.lastImageSIsSet()) || (lastImageSIsSet() && rhs.lastImageSIsSet() && getLastImageS() == rhs.getLastImageS())) && + + + ((!cutIsSet() && !rhs.cutIsSet()) || (cutIsSet() && rhs.cutIsSet() && getCut() == rhs.getCut())) && + + + ((!cutSIsSet() && !rhs.cutSIsSet()) || (cutSIsSet() && rhs.cutSIsSet() && getCutS() == rhs.getCutS())) && + + + ((!lastImageToWrittenSIsSet() && !rhs.lastImageToWrittenSIsSet()) || (lastImageToWrittenSIsSet() && rhs.lastImageToWrittenSIsSet() && getLastImageToWrittenS() == rhs.getLastImageToWrittenS())) && + + (getVerdict() == rhs.getVerdict()) + && + + (getVerdictText() == rhs.getVerdictText()) + && + + (getSummary() == rhs.getSummary()) + && + + (getWarnings() == rhs.getWarnings()) + + + ; +} + +bool Rugnux_live_status::operator!=(const Rugnux_live_status& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_live_status& o) +{ + j = nlohmann::json::object(); + j["state"] = o.m_State; + j["probe"] = o.m_Probe; + if(o.probeSIsSet()) + j["probe_s"] = o.m_Probe_s; + j["message"] = o.m_Message; + j["images_expected"] = o.m_Images_expected; + j["images_received"] = o.m_Images_received; + j["elapsed_s"] = o.m_Elapsed_s; + if(o.stepProgressIsSet()) + j["step_progress"] = o.m_Step_progress; + j["steps"] = o.m_Steps; + if(o.lastImageSIsSet()) + j["last_image_s"] = o.m_Last_image_s; + if(o.cutIsSet()) + j["cut"] = o.m_Cut; + if(o.cutSIsSet()) + j["cut_s"] = o.m_Cut_s; + if(o.lastImageToWrittenSIsSet()) + j["last_image_to_written_s"] = o.m_Last_image_to_written_s; + j["verdict"] = o.m_Verdict; + j["verdict_text"] = o.m_Verdict_text; + j["summary"] = o.m_Summary; + j["warnings"] = o.m_Warnings; + +} + +void from_json(const nlohmann::json& j, Rugnux_live_status& o) +{ + j.at("state").get_to(o.m_State); + j.at("probe").get_to(o.m_Probe); + if(j.find("probe_s") != j.end()) + { + j.at("probe_s").get_to(o.m_Probe_s); + o.m_Probe_sIsSet = true; + } + j.at("message").get_to(o.m_Message); + j.at("images_expected").get_to(o.m_Images_expected); + j.at("images_received").get_to(o.m_Images_received); + j.at("elapsed_s").get_to(o.m_Elapsed_s); + if(j.find("step_progress") != j.end()) + { + j.at("step_progress").get_to(o.m_Step_progress); + o.m_Step_progressIsSet = true; + } + j.at("steps").get_to(o.m_Steps); + if(j.find("last_image_s") != j.end()) + { + j.at("last_image_s").get_to(o.m_Last_image_s); + o.m_Last_image_sIsSet = true; + } + if(j.find("cut") != j.end()) + { + j.at("cut").get_to(o.m_Cut); + o.m_CutIsSet = true; + } + if(j.find("cut_s") != j.end()) + { + j.at("cut_s").get_to(o.m_Cut_s); + o.m_Cut_sIsSet = true; + } + if(j.find("last_image_to_written_s") != j.end()) + { + j.at("last_image_to_written_s").get_to(o.m_Last_image_to_written_s); + o.m_Last_image_to_written_sIsSet = true; + } + j.at("verdict").get_to(o.m_Verdict); + j.at("verdict_text").get_to(o.m_Verdict_text); + j.at("summary").get_to(o.m_Summary); + j.at("warnings").get_to(o.m_Warnings); + +} + +std::string Rugnux_live_status::getState() const +{ + return m_State; +} +void Rugnux_live_status::setState(std::string const& value) +{ + m_State = value; +} +std::string Rugnux_live_status::getProbe() const +{ + return m_Probe; +} +void Rugnux_live_status::setProbe(std::string const& value) +{ + m_Probe = value; +} +double Rugnux_live_status::getProbeS() const +{ + return m_Probe_s; +} +void Rugnux_live_status::setProbeS(double const value) +{ + m_Probe_s = value; + m_Probe_sIsSet = true; +} +bool Rugnux_live_status::probeSIsSet() const +{ + return m_Probe_sIsSet; +} +void Rugnux_live_status::unsetProbe_s() +{ + m_Probe_sIsSet = false; +} +std::string Rugnux_live_status::getMessage() const +{ + return m_Message; +} +void Rugnux_live_status::setMessage(std::string const& value) +{ + m_Message = value; +} +int64_t Rugnux_live_status::getImagesExpected() const +{ + return m_Images_expected; +} +void Rugnux_live_status::setImagesExpected(int64_t const value) +{ + m_Images_expected = value; +} +int64_t Rugnux_live_status::getImagesReceived() const +{ + return m_Images_received; +} +void Rugnux_live_status::setImagesReceived(int64_t const value) +{ + m_Images_received = value; +} +double Rugnux_live_status::getElapsedS() const +{ + return m_Elapsed_s; +} +void Rugnux_live_status::setElapsedS(double const value) +{ + m_Elapsed_s = value; +} +float Rugnux_live_status::getStepProgress() const +{ + return m_Step_progress; +} +void Rugnux_live_status::setStepProgress(float const value) +{ + m_Step_progress = value; + m_Step_progressIsSet = true; +} +bool Rugnux_live_status::stepProgressIsSet() const +{ + return m_Step_progressIsSet; +} +void Rugnux_live_status::unsetStep_progress() +{ + m_Step_progressIsSet = false; +} +std::vector Rugnux_live_status::getSteps() const +{ + return m_Steps; +} +void Rugnux_live_status::setSteps(std::vector const& value) +{ + m_Steps = value; +} +double Rugnux_live_status::getLastImageS() const +{ + return m_Last_image_s; +} +void Rugnux_live_status::setLastImageS(double const value) +{ + m_Last_image_s = value; + m_Last_image_sIsSet = true; +} +bool Rugnux_live_status::lastImageSIsSet() const +{ + return m_Last_image_sIsSet; +} +void Rugnux_live_status::unsetLast_image_s() +{ + m_Last_image_sIsSet = false; +} +std::string Rugnux_live_status::getCut() const +{ + return m_Cut; +} +void Rugnux_live_status::setCut(std::string const& value) +{ + m_Cut = value; + m_CutIsSet = true; +} +bool Rugnux_live_status::cutIsSet() const +{ + return m_CutIsSet; +} +void Rugnux_live_status::unsetCut() +{ + m_CutIsSet = false; +} +double Rugnux_live_status::getCutS() const +{ + return m_Cut_s; +} +void Rugnux_live_status::setCutS(double const value) +{ + m_Cut_s = value; + m_Cut_sIsSet = true; +} +bool Rugnux_live_status::cutSIsSet() const +{ + return m_Cut_sIsSet; +} +void Rugnux_live_status::unsetCut_s() +{ + m_Cut_sIsSet = false; +} +double Rugnux_live_status::getLastImageToWrittenS() const +{ + return m_Last_image_to_written_s; +} +void Rugnux_live_status::setLastImageToWrittenS(double const value) +{ + m_Last_image_to_written_s = value; + m_Last_image_to_written_sIsSet = true; +} +bool Rugnux_live_status::lastImageToWrittenSIsSet() const +{ + return m_Last_image_to_written_sIsSet; +} +void Rugnux_live_status::unsetLast_image_to_written_s() +{ + m_Last_image_to_written_sIsSet = false; +} +std::string Rugnux_live_status::getVerdict() const +{ + return m_Verdict; +} +void Rugnux_live_status::setVerdict(std::string const& value) +{ + m_Verdict = value; +} +std::string Rugnux_live_status::getVerdictText() const +{ + return m_Verdict_text; +} +void Rugnux_live_status::setVerdictText(std::string const& value) +{ + m_Verdict_text = value; +} +std::vector Rugnux_live_status::getSummary() const +{ + return m_Summary; +} +void Rugnux_live_status::setSummary(std::vector const& value) +{ + m_Summary = value; +} +std::vector Rugnux_live_status::getWarnings() const +{ + return m_Warnings; +} +void Rugnux_live_status::setWarnings(std::vector const& value) +{ + m_Warnings = value; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_live_status.h b/broker/gen/model/Rugnux_live_status.h new file mode 100644 index 000000000..d2faceeed --- /dev/null +++ b/broker/gen/model/Rugnux_live_status.h @@ -0,0 +1,204 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_live_status.h + * + * Live rugnux processing of the last collection; kept until the next collection starts + */ + +#ifndef Rugnux_live_status_H_ +#define Rugnux_live_status_H_ + + +#include "Rugnux_summary_row.h" +#include +#include "Rugnux_live_step.h" +#include +#include + +namespace org::openapitools::server::model +{ + +/// +/// Live rugnux processing of the last collection; kept until the next collection starts +/// +class Rugnux_live_status +{ +public: + Rugnux_live_status(); + virtual ~Rugnux_live_status() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_live_status& rhs) const; + bool operator!=(const Rugnux_live_status& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_live_status members + + /// + /// + /// + std::string getState() const; + void setState(std::string const& value); + /// + /// Answer of the early lattice probe on the first 60 degrees; empty until it answers + /// + std::string getProbe() const; + void setProbe(std::string const& value); + /// + /// When the probe answered, since the collection started + /// + double getProbeS() const; + void setProbeS(double const value); + bool probeSIsSet() const; + void unsetProbe_s(); + /// + /// Why the run failed or was skipped + /// + std::string getMessage() const; + void setMessage(std::string const& value); + /// + /// + /// + int64_t getImagesExpected() const; + void setImagesExpected(int64_t const value); + /// + /// + /// + int64_t getImagesReceived() const; + void setImagesReceived(int64_t const value); + /// + /// + /// + double getElapsedS() const; + void setElapsedS(double const value); + /// + /// Fraction of the current pass's images + /// + float getStepProgress() const; + void setStepProgress(float const value); + bool stepProgressIsSet() const; + void unsetStep_progress(); + /// + /// + /// + std::vector getSteps() const; + void setSteps(std::vector const& value); + /// + /// When the last image arrived (and the cutoff timer started), since the collection started + /// + double getLastImageS() const; + void setLastImageS(double const value); + bool lastImageSIsSet() const; + void unsetLast_image_s(); + /// + /// Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel` (`POST /cancel`). The end message then carried what was established at the cut. + /// + std::string getCut() const; + void setCut(std::string const& value); + bool cutIsSet() const; + void unsetCut(); + /// + /// When the analysis was cut, since the collection started + /// + double getCutS() const; + void setCutS(double const value); + bool cutSIsSet() const; + void unsetCut_s(); + /// + /// From the last image received to all results written + /// + double getLastImageToWrittenS() const; + void setLastImageToWrittenS(double const value); + bool lastImageToWrittenSIsSet() const; + void unsetLast_image_to_written_s(); + /// + /// OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over + /// + std::string getVerdict() const; + void setVerdict(std::string const& value); + /// + /// + /// + std::string getVerdictText() const; + void setVerdictText(std::string const& value); + /// + /// + /// + std::vector getSummary() const; + void setSummary(std::vector const& value); + /// + /// + /// + std::vector getWarnings() const; + void setWarnings(std::vector const& value); + + friend void to_json(nlohmann::json& j, const Rugnux_live_status& o); + friend void from_json(const nlohmann::json& j, Rugnux_live_status& o); +protected: + std::string m_State; + + std::string m_Probe; + + double m_Probe_s; + bool m_Probe_sIsSet; + std::string m_Message; + + int64_t m_Images_expected; + + int64_t m_Images_received; + + double m_Elapsed_s; + + float m_Step_progress; + bool m_Step_progressIsSet; + std::vector m_Steps; + + double m_Last_image_s; + bool m_Last_image_sIsSet; + std::string m_Cut; + bool m_CutIsSet; + double m_Cut_s; + bool m_Cut_sIsSet; + double m_Last_image_to_written_s; + bool m_Last_image_to_written_sIsSet; + std::string m_Verdict; + + std::string m_Verdict_text; + + std::vector m_Summary; + + std::vector m_Warnings; + + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_live_status_H_ */ diff --git a/broker/gen/model/Rugnux_live_step.cpp b/broker/gen/model/Rugnux_live_step.cpp new file mode 100644 index 000000000..4b05aabc2 --- /dev/null +++ b/broker/gen/model/Rugnux_live_step.cpp @@ -0,0 +1,168 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_live_step.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_live_step::Rugnux_live_step() +{ + m_Title = ""; + m_Started_s = 0.0; + m_Duration_s = 0.0; + m_Duration_sIsSet = false; + +} + +void Rugnux_live_step::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_live_step::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_live_step::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_live_step" : pathPrefix; + + + + /* Notes */ { + const std::vector& value = m_Notes; + const std::string currentValuePath = _pathPrefix + ".notes"; + + + { // Recursive validation of array elements + const std::string oldValuePath = currentValuePath; + int i = 0; + for (const std::string& value : value) + { + const std::string currentValuePath = oldValuePath + "[" + std::to_string(i) + "]"; + + + + i++; + } + } + + } + + return success; +} + +bool Rugnux_live_step::operator==(const Rugnux_live_step& rhs) const +{ + return + + + (getTitle() == rhs.getTitle()) + && + + (getStartedS() == rhs.getStartedS()) + && + + + ((!durationSIsSet() && !rhs.durationSIsSet()) || (durationSIsSet() && rhs.durationSIsSet() && getDurationS() == rhs.getDurationS())) && + + (getNotes() == rhs.getNotes()) + + + ; +} + +bool Rugnux_live_step::operator!=(const Rugnux_live_step& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_live_step& o) +{ + j = nlohmann::json::object(); + j["title"] = o.m_Title; + j["started_s"] = o.m_Started_s; + if(o.durationSIsSet()) + j["duration_s"] = o.m_Duration_s; + j["notes"] = o.m_Notes; + +} + +void from_json(const nlohmann::json& j, Rugnux_live_step& o) +{ + j.at("title").get_to(o.m_Title); + j.at("started_s").get_to(o.m_Started_s); + if(j.find("duration_s") != j.end()) + { + j.at("duration_s").get_to(o.m_Duration_s); + o.m_Duration_sIsSet = true; + } + j.at("notes").get_to(o.m_Notes); + +} + +std::string Rugnux_live_step::getTitle() const +{ + return m_Title; +} +void Rugnux_live_step::setTitle(std::string const& value) +{ + m_Title = value; +} +double Rugnux_live_step::getStartedS() const +{ + return m_Started_s; +} +void Rugnux_live_step::setStartedS(double const value) +{ + m_Started_s = value; +} +double Rugnux_live_step::getDurationS() const +{ + return m_Duration_s; +} +void Rugnux_live_step::setDurationS(double const value) +{ + m_Duration_s = value; + m_Duration_sIsSet = true; +} +bool Rugnux_live_step::durationSIsSet() const +{ + return m_Duration_sIsSet; +} +void Rugnux_live_step::unsetDuration_s() +{ + m_Duration_sIsSet = false; +} +std::vector Rugnux_live_step::getNotes() const +{ + return m_Notes; +} +void Rugnux_live_step::setNotes(std::vector const& value) +{ + m_Notes = value; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_live_step.h b/broker/gen/model/Rugnux_live_step.h new file mode 100644 index 000000000..ed908d260 --- /dev/null +++ b/broker/gen/model/Rugnux_live_step.h @@ -0,0 +1,101 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_live_step.h + * + * + */ + +#ifndef Rugnux_live_step_H_ +#define Rugnux_live_step_H_ + + +#include +#include +#include + +namespace org::openapitools::server::model +{ + +/// +/// +/// +class Rugnux_live_step +{ +public: + Rugnux_live_step(); + virtual ~Rugnux_live_step() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_live_step& rhs) const; + bool operator!=(const Rugnux_live_step& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_live_step members + + /// + /// + /// + std::string getTitle() const; + void setTitle(std::string const& value); + /// + /// Start of the step, since the collection started + /// + double getStartedS() const; + void setStartedS(double const value); + /// + /// Set once the next step starts or the run ends + /// + double getDurationS() const; + void setDurationS(double const value); + bool durationSIsSet() const; + void unsetDuration_s(); + /// + /// What the step found (e.g. lattice, space group) + /// + std::vector getNotes() const; + void setNotes(std::vector const& value); + + friend void to_json(nlohmann::json& j, const Rugnux_live_step& o); + friend void from_json(const nlohmann::json& j, Rugnux_live_step& o); +protected: + std::string m_Title; + + double m_Started_s; + + double m_Duration_s; + bool m_Duration_sIsSet; + std::vector m_Notes; + + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_live_step_H_ */ diff --git a/broker/gen/model/Rugnux_settings.cpp b/broker/gen/model/Rugnux_settings.cpp new file mode 100644 index 000000000..de476f436 --- /dev/null +++ b/broker/gen/model/Rugnux_settings.cpp @@ -0,0 +1,192 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_settings.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_settings::Rugnux_settings() +{ + m_Enable = false; + m_EnableIsSet = false; + m_Reserved_cpus = 4L; + m_Reserved_cpusIsSet = false; + m_Deadline_s = 0.0; + m_Deadline_sIsSet = false; + +} + +void Rugnux_settings::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_settings::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_settings::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_settings" : pathPrefix; + + + if (reservedCpusIsSet()) + { + const int64_t& value = m_Reserved_cpus; + const std::string currentValuePath = _pathPrefix + ".reservedCpus"; + + + if (value < 0ll) + { + success = false; + msg << currentValuePath << ": must be greater than or equal to 0;"; + } + + } + + if (deadlineSIsSet()) + { + const double& value = m_Deadline_s; + const std::string currentValuePath = _pathPrefix + ".deadlineS"; + + + if (value < 0.0) + { + success = false; + msg << currentValuePath << ": must be greater than or equal to 0.0;"; + } + + } + + return success; +} + +bool Rugnux_settings::operator==(const Rugnux_settings& rhs) const +{ + return + + + + ((!enableIsSet() && !rhs.enableIsSet()) || (enableIsSet() && rhs.enableIsSet() && isEnable() == rhs.isEnable())) && + + + ((!reservedCpusIsSet() && !rhs.reservedCpusIsSet()) || (reservedCpusIsSet() && rhs.reservedCpusIsSet() && getReservedCpus() == rhs.getReservedCpus())) && + + + ((!deadlineSIsSet() && !rhs.deadlineSIsSet()) || (deadlineSIsSet() && rhs.deadlineSIsSet() && getDeadlineS() == rhs.getDeadlineS())) + + ; +} + +bool Rugnux_settings::operator!=(const Rugnux_settings& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_settings& o) +{ + j = nlohmann::json::object(); + if(o.enableIsSet()) + j["enable"] = o.m_Enable; + if(o.reservedCpusIsSet()) + j["reserved_cpus"] = o.m_Reserved_cpus; + if(o.deadlineSIsSet()) + j["deadline_s"] = o.m_Deadline_s; + +} + +void from_json(const nlohmann::json& j, Rugnux_settings& o) +{ + if(j.find("enable") != j.end()) + { + j.at("enable").get_to(o.m_Enable); + o.m_EnableIsSet = true; + } + if(j.find("reserved_cpus") != j.end()) + { + j.at("reserved_cpus").get_to(o.m_Reserved_cpus); + o.m_Reserved_cpusIsSet = true; + } + if(j.find("deadline_s") != j.end()) + { + j.at("deadline_s").get_to(o.m_Deadline_s); + o.m_Deadline_sIsSet = true; + } + +} + +bool Rugnux_settings::isEnable() const +{ + return m_Enable; +} +void Rugnux_settings::setEnable(bool const value) +{ + m_Enable = value; + m_EnableIsSet = true; +} +bool Rugnux_settings::enableIsSet() const +{ + return m_EnableIsSet; +} +void Rugnux_settings::unsetEnable() +{ + m_EnableIsSet = false; +} +int64_t Rugnux_settings::getReservedCpus() const +{ + return m_Reserved_cpus; +} +void Rugnux_settings::setReservedCpus(int64_t const value) +{ + m_Reserved_cpus = value; + m_Reserved_cpusIsSet = true; +} +bool Rugnux_settings::reservedCpusIsSet() const +{ + return m_Reserved_cpusIsSet; +} +void Rugnux_settings::unsetReserved_cpus() +{ + m_Reserved_cpusIsSet = false; +} +double Rugnux_settings::getDeadlineS() const +{ + return m_Deadline_s; +} +void Rugnux_settings::setDeadlineS(double const value) +{ + m_Deadline_s = value; + m_Deadline_sIsSet = true; +} +bool Rugnux_settings::deadlineSIsSet() const +{ + return m_Deadline_sIsSet; +} +void Rugnux_settings::unsetDeadline_s() +{ + m_Deadline_sIsSet = false; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_settings.h b/broker/gen/model/Rugnux_settings.h new file mode 100644 index 000000000..8f4c60b5b --- /dev/null +++ b/broker/gen/model/Rugnux_settings.h @@ -0,0 +1,96 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_settings.h + * + * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. + */ + +#ifndef Rugnux_settings_H_ +#define Rugnux_settings_H_ + + +#include + +namespace org::openapitools::server::model +{ + +/// +/// Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. +/// +class Rugnux_settings +{ +public: + Rugnux_settings(); + virtual ~Rugnux_settings() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_settings& rhs) const; + bool operator!=(const Rugnux_settings& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_settings members + + /// + /// Process DECTRIS rotation collections with rugnux live + /// + bool isEnable() const; + void setEnable(bool const value); + bool enableIsSet() const; + void unsetEnable(); + /// + /// CPU threads left to the receiver and the image pusher while rugnux runs + /// + int64_t getReservedCpus() const; + void setReservedCpus(int64_t const value); + bool reservedCpusIsSet() const; + void unsetReserved_cpus(); + /// + /// Fixed cutoff: this long after the last image the analysis is cut, and the collection ends with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff. + /// + double getDeadlineS() const; + void setDeadlineS(double const value); + bool deadlineSIsSet() const; + void unsetDeadline_s(); + + friend void to_json(nlohmann::json& j, const Rugnux_settings& o); + friend void from_json(const nlohmann::json& j, Rugnux_settings& o); +protected: + bool m_Enable; + bool m_EnableIsSet; + int64_t m_Reserved_cpus; + bool m_Reserved_cpusIsSet; + double m_Deadline_s; + bool m_Deadline_sIsSet; + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_settings_H_ */ diff --git a/broker/gen/model/Rugnux_status.cpp b/broker/gen/model/Rugnux_status.cpp new file mode 100644 index 000000000..da879400f --- /dev/null +++ b/broker/gen/model/Rugnux_status.cpp @@ -0,0 +1,177 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_status.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_status::Rugnux_status() +{ + m_Enable = false; + m_Reserved_cpus = 0L; + m_Deadline_s = 0.0; + m_Deadline_sIsSet = false; + m_Model_loaded = false; + m_Model_size_bytes = 0L; + m_Model_size_bytesIsSet = false; + +} + +void Rugnux_status::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_status::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_status::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_status" : pathPrefix; + + + return success; +} + +bool Rugnux_status::operator==(const Rugnux_status& rhs) const +{ + return + + + (isEnable() == rhs.isEnable()) + && + + (getReservedCpus() == rhs.getReservedCpus()) + && + + + ((!deadlineSIsSet() && !rhs.deadlineSIsSet()) || (deadlineSIsSet() && rhs.deadlineSIsSet() && getDeadlineS() == rhs.getDeadlineS())) && + + (isModelLoaded() == rhs.isModelLoaded()) + && + + + ((!modelSizeBytesIsSet() && !rhs.modelSizeBytesIsSet()) || (modelSizeBytesIsSet() && rhs.modelSizeBytesIsSet() && getModelSizeBytes() == rhs.getModelSizeBytes())) + + ; +} + +bool Rugnux_status::operator!=(const Rugnux_status& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_status& o) +{ + j = nlohmann::json::object(); + j["enable"] = o.m_Enable; + j["reserved_cpus"] = o.m_Reserved_cpus; + if(o.deadlineSIsSet()) + j["deadline_s"] = o.m_Deadline_s; + j["model_loaded"] = o.m_Model_loaded; + if(o.modelSizeBytesIsSet()) + j["model_size_bytes"] = o.m_Model_size_bytes; + +} + +void from_json(const nlohmann::json& j, Rugnux_status& o) +{ + j.at("enable").get_to(o.m_Enable); + j.at("reserved_cpus").get_to(o.m_Reserved_cpus); + if(j.find("deadline_s") != j.end()) + { + j.at("deadline_s").get_to(o.m_Deadline_s); + o.m_Deadline_sIsSet = true; + } + j.at("model_loaded").get_to(o.m_Model_loaded); + if(j.find("model_size_bytes") != j.end()) + { + j.at("model_size_bytes").get_to(o.m_Model_size_bytes); + o.m_Model_size_bytesIsSet = true; + } + +} + +bool Rugnux_status::isEnable() const +{ + return m_Enable; +} +void Rugnux_status::setEnable(bool const value) +{ + m_Enable = value; +} +int64_t Rugnux_status::getReservedCpus() const +{ + return m_Reserved_cpus; +} +void Rugnux_status::setReservedCpus(int64_t const value) +{ + m_Reserved_cpus = value; +} +double Rugnux_status::getDeadlineS() const +{ + return m_Deadline_s; +} +void Rugnux_status::setDeadlineS(double const value) +{ + m_Deadline_s = value; + m_Deadline_sIsSet = true; +} +bool Rugnux_status::deadlineSIsSet() const +{ + return m_Deadline_sIsSet; +} +void Rugnux_status::unsetDeadline_s() +{ + m_Deadline_sIsSet = false; +} +bool Rugnux_status::isModelLoaded() const +{ + return m_Model_loaded; +} +void Rugnux_status::setModelLoaded(bool const value) +{ + m_Model_loaded = value; +} +int64_t Rugnux_status::getModelSizeBytes() const +{ + return m_Model_size_bytes; +} +void Rugnux_status::setModelSizeBytes(int64_t const value) +{ + m_Model_size_bytes = value; + m_Model_size_bytesIsSet = true; +} +bool Rugnux_status::modelSizeBytesIsSet() const +{ + return m_Model_size_bytesIsSet; +} +void Rugnux_status::unsetModel_size_bytes() +{ + m_Model_size_bytesIsSet = false; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_status.h b/broker/gen/model/Rugnux_status.h new file mode 100644 index 000000000..f5bf36405 --- /dev/null +++ b/broker/gen/model/Rugnux_status.h @@ -0,0 +1,108 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_status.h + * + * + */ + +#ifndef Rugnux_status_H_ +#define Rugnux_status_H_ + + +#include + +namespace org::openapitools::server::model +{ + +/// +/// +/// +class Rugnux_status +{ +public: + Rugnux_status(); + virtual ~Rugnux_status() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_status& rhs) const; + bool operator!=(const Rugnux_status& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_status members + + /// + /// Rugnux live mode is enabled + /// + bool isEnable() const; + void setEnable(bool const value); + /// + /// CPU threads left to the receiver and the image pusher while rugnux runs + /// + int64_t getReservedCpus() const; + void setReservedCpus(int64_t const value); + /// + /// Seconds after the last image at which the analysis is cut; 0 means no cutoff + /// + double getDeadlineS() const; + void setDeadlineS(double const value); + bool deadlineSIsSet() const; + void unsetDeadline_s(); + /// + /// A model was uploaded with `PUT /config/rugnux/model` + /// + bool isModelLoaded() const; + void setModelLoaded(bool const value); + /// + /// Size of the uploaded model file + /// + int64_t getModelSizeBytes() const; + void setModelSizeBytes(int64_t const value); + bool modelSizeBytesIsSet() const; + void unsetModel_size_bytes(); + + friend void to_json(nlohmann::json& j, const Rugnux_status& o); + friend void from_json(const nlohmann::json& j, Rugnux_status& o); +protected: + bool m_Enable; + + int64_t m_Reserved_cpus; + + double m_Deadline_s; + bool m_Deadline_sIsSet; + bool m_Model_loaded; + + int64_t m_Model_size_bytes; + bool m_Model_size_bytesIsSet; + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_status_H_ */ diff --git a/broker/gen/model/Rugnux_summary_row.cpp b/broker/gen/model/Rugnux_summary_row.cpp new file mode 100644 index 000000000..5aa9514b3 --- /dev/null +++ b/broker/gen/model/Rugnux_summary_row.cpp @@ -0,0 +1,105 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ + + +#include "Rugnux_summary_row.h" +#include "Helpers.h" + +#include + +namespace org::openapitools::server::model +{ + +Rugnux_summary_row::Rugnux_summary_row() +{ + m_Label = ""; + m_Value = ""; + +} + +void Rugnux_summary_row::validate() const +{ + std::stringstream msg; + if (!validate(msg)) + { + throw org::openapitools::server::helpers::ValidationException(msg.str()); + } +} + +bool Rugnux_summary_row::validate(std::stringstream& msg) const +{ + return validate(msg, ""); +} + +bool Rugnux_summary_row::validate(std::stringstream& msg, const std::string& pathPrefix) const +{ + bool success = true; + const std::string _pathPrefix = pathPrefix.empty() ? "Rugnux_summary_row" : pathPrefix; + + + return success; +} + +bool Rugnux_summary_row::operator==(const Rugnux_summary_row& rhs) const +{ + return + + + (getLabel() == rhs.getLabel()) + && + + (getValue() == rhs.getValue()) + + + ; +} + +bool Rugnux_summary_row::operator!=(const Rugnux_summary_row& rhs) const +{ + return !(*this == rhs); +} + +void to_json(nlohmann::json& j, const Rugnux_summary_row& o) +{ + j = nlohmann::json::object(); + j["label"] = o.m_Label; + j["value"] = o.m_Value; + +} + +void from_json(const nlohmann::json& j, Rugnux_summary_row& o) +{ + j.at("label").get_to(o.m_Label); + j.at("value").get_to(o.m_Value); + +} + +std::string Rugnux_summary_row::getLabel() const +{ + return m_Label; +} +void Rugnux_summary_row::setLabel(std::string const& value) +{ + m_Label = value; +} +std::string Rugnux_summary_row::getValue() const +{ + return m_Value; +} +void Rugnux_summary_row::setValue(std::string const& value) +{ + m_Value = value; +} + + +} // namespace org::openapitools::server::model + diff --git a/broker/gen/model/Rugnux_summary_row.h b/broker/gen/model/Rugnux_summary_row.h new file mode 100644 index 000000000..c868fe387 --- /dev/null +++ b/broker/gen/model/Rugnux_summary_row.h @@ -0,0 +1,84 @@ +/** +* Jungfraujoch +* API to control Jungfraujoch developed by the Paul Scherrer Institute (Switzerland). Jungfraujoch is a data acquisition and analysis system for pixel array detectors, primarly PSI JUNGFRAU. Jungfraujoch uses FPGA boards to acquire data at high data rates. # License Clarification While this API definition is licensed under GPL-3.0, **the GPL copyleft provisions do not apply** when this file is used solely to generate OpenAPI clients or when implementing applications that interact with the API. Generated client code and applications using this API definition are not subject to the GPL license requirements and may be distributed under terms of your choosing. This exception is similar in spirit to the Linux Kernel's approach to userspace API headers and the GCC Runtime Library Exception. The Linux Kernel developers have explicitly stated that user programs that merely use the kernel interfaces (syscalls, ioctl definitions, etc.) are not derivative works of the kernel and are not subject to the terms of the GPL. This exception is intended to allow wider use of this API specification without imposing GPL requirements on applications that merely interact with the API, regardless of whether they communicate through network calls or other mechanisms. +* +* The version of the OpenAPI document: 1.0.0-rc.175 +* Contact: filip.leonarski@psi.ch +* +* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). +* https://openapi-generator.tech +* Do not edit the class manually. +*/ +/* + * Rugnux_summary_row.h + * + * + */ + +#ifndef Rugnux_summary_row_H_ +#define Rugnux_summary_row_H_ + + +#include +#include + +namespace org::openapitools::server::model +{ + +/// +/// +/// +class Rugnux_summary_row +{ +public: + Rugnux_summary_row(); + virtual ~Rugnux_summary_row() = default; + + + /// + /// Validate the current data in the model. Throws a ValidationException on failure. + /// + void validate() const; + + /// + /// Validate the current data in the model. Returns false on error and writes an error + /// message into the given stringstream. + /// + bool validate(std::stringstream& msg) const; + + /// + /// Helper overload for validate. Used when one model stores another model and calls it's validate. + /// Not meant to be called outside that case. + /// + bool validate(std::stringstream& msg, const std::string& pathPrefix) const; + + bool operator==(const Rugnux_summary_row& rhs) const; + bool operator!=(const Rugnux_summary_row& rhs) const; + + ///////////////////////////////////////////// + /// Rugnux_summary_row members + + /// + /// + /// + std::string getLabel() const; + void setLabel(std::string const& value); + /// + /// + /// + std::string getValue() const; + void setValue(std::string const& value); + + friend void to_json(nlohmann::json& j, const Rugnux_summary_row& o); + friend void from_json(const nlohmann::json& j, Rugnux_summary_row& o); +protected: + std::string m_Label; + + std::string m_Value; + + +}; + +} // namespace org::openapitools::server::model + +#endif /* Rugnux_summary_row_H_ */ diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 0370f3029..1f3bef805 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -2684,6 +2684,167 @@ components: $ref: '#/components/schemas/zeromq_metadata_settings' dark_mask: $ref: '#/components/schemas/dark_mask_settings' + rugnux: + $ref: '#/components/schemas/rugnux_settings' + rugnux_settings: + type: object + description: | + Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux + inside the broker while they arrive, and the merged result is sent to the writer. + properties: + enable: + type: boolean + default: false + description: Process DECTRIS rotation collections with rugnux live + reserved_cpus: + type: integer + format: int64 + minimum: 0 + default: 4 + description: CPU threads left to the receiver and the image pusher while rugnux runs + deadline_s: + type: number + format: double + minimum: 0.0 + default: 0.0 + description: | + Fixed cutoff: this long after the last image the analysis is cut, and the collection ends + with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff. + rugnux_status: + type: object + required: + - enable + - reserved_cpus + - model_loaded + properties: + enable: + type: boolean + description: Rugnux live mode is enabled + reserved_cpus: + type: integer + format: int64 + description: CPU threads left to the receiver and the image pusher while rugnux runs + deadline_s: + type: number + format: double + description: Seconds after the last image at which the analysis is cut; 0 means no cutoff + model_loaded: + type: boolean + description: A model was uploaded with `PUT /config/rugnux/model` + model_size_bytes: + type: integer + format: int64 + description: Size of the uploaded model file + rugnux_live_step: + type: object + required: + - title + - started_s + - notes + properties: + title: + type: string + started_s: + type: number + format: double + description: Start of the step, since the collection started + duration_s: + type: number + format: double + description: Set once the next step starts or the run ends + notes: + type: array + description: What the step found (e.g. lattice, space group) + items: + type: string + rugnux_summary_row: + type: object + required: + - label + - value + properties: + label: + type: string + value: + type: string + rugnux_live_status: + type: object + description: Live rugnux processing of the last collection; kept until the next collection starts + required: + - state + - message + - images_expected + - images_received + - elapsed_s + - steps + - verdict + - verdict_text + - summary + - warnings + - probe + properties: + state: + type: string + enum: [Idle, Running, Done, Failed, Cancelled, Skipped] + probe: + type: string + description: Answer of the early lattice probe on the first 60 degrees; empty until it answers + probe_s: + type: number + format: double + description: When the probe answered, since the collection started + message: + type: string + description: Why the run failed or was skipped + images_expected: + type: integer + format: int64 + images_received: + type: integer + format: int64 + elapsed_s: + type: number + format: double + step_progress: + type: number + format: float + minimum: 0.0 + maximum: 1.0 + description: Fraction of the current pass's images + steps: + type: array + items: + $ref: '#/components/schemas/rugnux_live_step' + last_image_s: + type: number + format: double + description: When the last image arrived (and the cutoff timer started), since the collection started + cut: + type: string + description: | + Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel` + (`POST /cancel`). The end message then carried what was established at the cut. + cut_s: + type: number + format: double + description: When the analysis was cut, since the collection started + last_image_to_written_s: + type: number + format: double + description: From the last image received to all results written + verdict: + type: string + description: OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over + verdict_text: + type: string + summary: + type: array + items: + $ref: '#/components/schemas/rugnux_summary_row' + warnings: + type: array + items: + type: string zeromq_metadata_settings: type: object required: @@ -2895,6 +3056,11 @@ paths: Any frame that is currently being processed by CPU will be finished and sent to writer. Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state. + With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection + or after its last image alike: the collection ends with what the analysis has established + (see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not + marked as cancelled. + If data collection is not running, the command has no effect. responses: "200": @@ -3331,6 +3497,133 @@ paths: application/json: schema: $ref: '#/components/schemas/detector_list' + /config/rugnux: + put: + summary: Change rugnux live mode settings + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + security: + - bearerAuth: [] + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/rugnux_settings' + responses: + "401": + description: The current dataset is protected and no valid bearer token was presented. + content: + text/plain: + schema: + type: string + "200": + description: Everything OK + "400": + description: Input parsing or validation error + content: + text/plain: + schema: + type: string + description: Exception error + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + get: + summary: Get rugnux live mode settings and model status + description: Can be done anytime + security: + - bearerAuth: [] + responses: + "401": + description: The current dataset is protected and no valid bearer token was presented. + content: + text/plain: + schema: + type: string + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/rugnux_status' + /config/rugnux/model: + put: + summary: Upload the model used by rugnux live mode + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement + and validation of every following rugnux live collection, until it is removed with + `DELETE /config/rugnux/model` or replaced by another upload. + security: + - bearerAuth: [] + requestBody: + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + "401": + description: The current dataset is protected and no valid bearer token was presented. + content: + text/plain: + schema: + type: string + "200": + description: All good + "400": + description: Empty body + "413": + description: Model larger than 64 MiB + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + delete: + summary: Remove the model used by rugnux live mode + description: | + This can only be done when detector is `Idle`, `Error` or `Inactive` states. + security: + - bearerAuth: [] + responses: + "401": + description: The current dataset is protected and no valid bearer token was presented. + content: + text/plain: + schema: + type: string + "200": + description: All good + "500": + description: Error within Jungfraujoch code - see output message. + content: + application/json: + schema: + $ref: '#/components/schemas/error_message' + /rugnux/status: + get: + summary: Get live rugnux processing status of the last collection + description: Can be done anytime + security: + - bearerAuth: [] + responses: + "401": + description: The current dataset is protected and no valid bearer token was presented. + content: + text/plain: + schema: + type: string + "200": + description: Everything OK + content: + application/json: + schema: + $ref: '#/components/schemas/rugnux_live_status' /config/zeromq_preview: put: summary: Set ZeroMQ preview settings diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index 7457e5a3f..d26812935 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -116,6 +116,7 @@ int main(int argc, char **argv) { if (receiver) broker.Services().Receiver(receiver.get()); + ParseRugnuxSettings(settings, broker.Services()); httplib::Server server; httpServer = &server; diff --git a/common/ParallelFor.h b/common/ParallelFor.h index 65b9e262c..e8dbb4e4d 100644 --- a/common/ParallelFor.h +++ b/common/ParallelFor.h @@ -16,6 +16,12 @@ #include "ThreadAffinity.h" +#ifdef __linux__ +#include +#include +#include +#endif + // Two shapes of "run this over a range on several threads", used by the analysis code. Both take the // worker count from the caller rather than asking the hardware, so a run that was told how many // threads to use keeps to it. @@ -48,6 +54,20 @@ namespace parallel_detail { size_t WorkerCount() const { return workers.size(); } + // The scheduling priority of every pool thread (Linux; elsewhere nothing). The pool is made once + // per process, so a thread that lowers its own priority does not lower the work it hands to the + // pool: the broker running rugnux beside its receiving threads sets it here as well. Raising + // it back needs the privilege to do so; returns false where that was refused. + bool SetNice(int nice) { + bool ok = true; +#ifdef __linux__ + std::lock_guard lock(m); + for (const int tid: tids) + ok = (setpriority(PRIO_PROCESS, static_cast(tid), nice) == 0) && ok; +#endif + return ok; + } + void Submit(std::function job) { { std::lock_guard lock(m); @@ -66,6 +86,12 @@ namespace parallel_detail { // It inherited whatever CPUs the thread that first used the pool was kept to. RestoreThreadAffinity(); InWorkerFlag() = true; +#ifdef __linux__ + { + std::lock_guard lock(m); + tids.push_back(static_cast(syscall(SYS_gettid))); + } +#endif Loop(); }); } catch (...) { @@ -106,6 +132,7 @@ namespace parallel_detail { std::condition_variable cv; std::deque > queue; std::vector workers; + std::vector tids; // the workers' kernel thread ids, for SetNice bool stop = false; // A function-local thread_local rather than an inline static member: Apple's linker rejects the diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index 739df1d68..4293a5147 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -170,6 +170,27 @@ The writers report each closed data file to the broker (TCP protocol version 5, for `FILE_CLOSED`); the in-process HDF5 writer reports to it directly. The ZeroMQ writer has no way back to the broker, so with it the route answers 409. +## Rugnux live mode + +With rugnux live mode on, rotation collections from a DECTRIS detector are processed by rugnux inside +the broker while they arrive. It is off by default; the configuration file switches it on with +`"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads +left to the receiver and the image pusher. + +The same settings are read with `GET /config/rugnux` and changed with `PUT /config/rugnux`. A model +for molecular replacement and validation (PDB or mmCIF, optionally gzipped, at most 64 MiB) is +uploaded as the raw file with `PUT /config/rugnux/model` (`application/octet-stream`; a larger body +gets `413`). The broker keeps it, readable only by its own user, for every following collection until +`DELETE /config/rugnux/model` removes it or another upload replaces it. `GET /config/rugnux` reports +whether a model is loaded and its size, never its contents. Changes are refused during a collection, +and while the current dataset was started with tokens, all of these routes need its bearer token. + +`GET /rugnux/status` reports the live run of the last collection, kept until the next one starts: its +state, the images received, the step it is on and how far into it, the steps so far with what each +found, and once it is over the verdict, the summary, warnings and the time from the last image to +every result written. The web frontend shows it on the dashboard. It needs the dataset's bearer token +like the routes above. + ## Setting up a local test for Jungfraujoch For development, it is possible to set up a local installation of Jungfraujoch. This will work without FPGA installed in the computer and allows testing the Jungfraujoch software layer, including diff --git a/docs/python_client/README.md b/docs/python_client/README.md index 8ca6a35c5..f4ec077d9 100644 --- a/docs/python_client/README.md +++ b/docs/python_client/README.md @@ -126,6 +126,10 @@ Class | Method | HTTP request | Description *DefaultApi* | [**config_mask_tiff_get**](docs/DefaultApi.md#config_mask_tiff_get) | **GET** /config/mask.tiff | Get mask of the detector (TIFF) *DefaultApi* | [**config_roi_get**](docs/DefaultApi.md#config_roi_get) | **GET** /config/roi | Get ROI definitions *DefaultApi* | [**config_roi_put**](docs/DefaultApi.md#config_roi_put) | **PUT** /config/roi | Upload ROI definitions +*DefaultApi* | [**config_rugnux_get**](docs/DefaultApi.md#config_rugnux_get) | **GET** /config/rugnux | Get rugnux live mode settings and model status +*DefaultApi* | [**config_rugnux_model_delete**](docs/DefaultApi.md#config_rugnux_model_delete) | **DELETE** /config/rugnux/model | Remove the model used by rugnux live mode +*DefaultApi* | [**config_rugnux_model_put**](docs/DefaultApi.md#config_rugnux_model_put) | **PUT** /config/rugnux/model | Upload the model used by rugnux live mode +*DefaultApi* | [**config_rugnux_put**](docs/DefaultApi.md#config_rugnux_put) | **PUT** /config/rugnux | Change rugnux live mode settings *DefaultApi* | [**config_select_detector_get**](docs/DefaultApi.md#config_select_detector_get) | **GET** /config/select_detector | List available detectors *DefaultApi* | [**config_select_detector_put**](docs/DefaultApi.md#config_select_detector_put) | **PUT** /config/select_detector | Select detector *DefaultApi* | [**config_spot_finding_get**](docs/DefaultApi.md#config_spot_finding_get) | **GET** /config/spot_finding | Get data processing configuration @@ -149,11 +153,13 @@ Class | Method | HTTP request | Description *DefaultApi* | [**image_buffer_status_get**](docs/DefaultApi.md#image_buffer_status_get) | **GET** /image_buffer/status | Get status of the image buffers *DefaultApi* | [**image_pusher_status_get**](docs/DefaultApi.md#image_pusher_status_get) | **GET** /image_pusher/status | Get status of image pusher *DefaultApi* | [**initialize_post**](docs/DefaultApi.md#initialize_post) | **POST** /initialize | Initialize detector and data acquisition +*DefaultApi* | [**live_events_get**](docs/DefaultApi.md#live_events_get) | **GET** /live/events | Event stream of the collection, for a reader following it *DefaultApi* | [**pedestal_post**](docs/DefaultApi.md#pedestal_post) | **POST** /pedestal | Collect dark current for the detector *DefaultApi* | [**preview_pedestal_tiff_get**](docs/DefaultApi.md#preview_pedestal_tiff_get) | **GET** /preview/pedestal.tiff | Get pedestal in TIFF format *DefaultApi* | [**preview_plot_bin_get**](docs/DefaultApi.md#preview_plot_bin_get) | **GET** /preview/plot.bin | Generate 1D plot from Jungfraujoch and send in raw binary format. Data are provided as (32-bit) float binary array. This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. *DefaultApi* | [**preview_plot_get**](docs/DefaultApi.md#preview_plot_get) | **GET** /preview/plot | Generate 1D plot from Jungfraujoch *DefaultApi* | [**result_scan_get**](docs/DefaultApi.md#result_scan_get) | **GET** /result/scan | Get full scan result +*DefaultApi* | [**rugnux_status_get**](docs/DefaultApi.md#rugnux_status_get) | **GET** /rugnux/status | Get live rugnux processing status of the last collection *DefaultApi* | [**start_post**](docs/DefaultApi.md#start_post) | **POST** /start | Start detector *DefaultApi* | [**statistics_calibration_get**](docs/DefaultApi.md#statistics_calibration_get) | **GET** /statistics/calibration | Get calibration statistics *DefaultApi* | [**statistics_data_collection_get**](docs/DefaultApi.md#statistics_data_collection_get) | **GET** /statistics/data_collection | Get data collection statistics @@ -223,6 +229,11 @@ Class | Method | HTTP request | Description - [RoiCircleList](docs/RoiCircleList.md) - [RoiDefinitions](docs/RoiDefinitions.md) - [RotationAxis](docs/RotationAxis.md) + - [RugnuxLiveStatus](docs/RugnuxLiveStatus.md) + - [RugnuxLiveStep](docs/RugnuxLiveStep.md) + - [RugnuxSettings](docs/RugnuxSettings.md) + - [RugnuxStatus](docs/RugnuxStatus.md) + - [RugnuxSummaryRow](docs/RugnuxSummaryRow.md) - [ScanResult](docs/ScanResult.md) - [ScanResultImagesInner](docs/ScanResultImagesInner.md) - [SpotFindingSettings](docs/SpotFindingSettings.md) diff --git a/docs/python_client/docs/DefaultApi.md b/docs/python_client/docs/DefaultApi.md index beb8542ed..67c3b3a53 100644 --- a/docs/python_client/docs/DefaultApi.md +++ b/docs/python_client/docs/DefaultApi.md @@ -29,6 +29,10 @@ Method | HTTP request | Description [**config_mask_tiff_get**](DefaultApi.md#config_mask_tiff_get) | **GET** /config/mask.tiff | Get mask of the detector (TIFF) [**config_roi_get**](DefaultApi.md#config_roi_get) | **GET** /config/roi | Get ROI definitions [**config_roi_put**](DefaultApi.md#config_roi_put) | **PUT** /config/roi | Upload ROI definitions +[**config_rugnux_get**](DefaultApi.md#config_rugnux_get) | **GET** /config/rugnux | Get rugnux live mode settings and model status +[**config_rugnux_model_delete**](DefaultApi.md#config_rugnux_model_delete) | **DELETE** /config/rugnux/model | Remove the model used by rugnux live mode +[**config_rugnux_model_put**](DefaultApi.md#config_rugnux_model_put) | **PUT** /config/rugnux/model | Upload the model used by rugnux live mode +[**config_rugnux_put**](DefaultApi.md#config_rugnux_put) | **PUT** /config/rugnux | Change rugnux live mode settings [**config_select_detector_get**](DefaultApi.md#config_select_detector_get) | **GET** /config/select_detector | List available detectors [**config_select_detector_put**](DefaultApi.md#config_select_detector_put) | **PUT** /config/select_detector | Select detector [**config_spot_finding_get**](DefaultApi.md#config_spot_finding_get) | **GET** /config/spot_finding | Get data processing configuration @@ -52,11 +56,13 @@ Method | HTTP request | Description [**image_buffer_status_get**](DefaultApi.md#image_buffer_status_get) | **GET** /image_buffer/status | Get status of the image buffers [**image_pusher_status_get**](DefaultApi.md#image_pusher_status_get) | **GET** /image_pusher/status | Get status of image pusher [**initialize_post**](DefaultApi.md#initialize_post) | **POST** /initialize | Initialize detector and data acquisition +[**live_events_get**](DefaultApi.md#live_events_get) | **GET** /live/events | Event stream of the collection, for a reader following it [**pedestal_post**](DefaultApi.md#pedestal_post) | **POST** /pedestal | Collect dark current for the detector [**preview_pedestal_tiff_get**](DefaultApi.md#preview_pedestal_tiff_get) | **GET** /preview/pedestal.tiff | Get pedestal in TIFF format [**preview_plot_bin_get**](DefaultApi.md#preview_plot_bin_get) | **GET** /preview/plot.bin | Generate 1D plot from Jungfraujoch and send in raw binary format. Data are provided as (32-bit) float binary array. This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. [**preview_plot_get**](DefaultApi.md#preview_plot_get) | **GET** /preview/plot | Generate 1D plot from Jungfraujoch [**result_scan_get**](DefaultApi.md#result_scan_get) | **GET** /result/scan | Get full scan result +[**rugnux_status_get**](DefaultApi.md#rugnux_status_get) | **GET** /rugnux/status | Get live rugnux processing status of the last collection [**start_post**](DefaultApi.md#start_post) | **POST** /start | Start detector [**statistics_calibration_get**](DefaultApi.md#statistics_calibration_get) | **GET** /statistics/calibration | Get calibration statistics [**statistics_data_collection_get**](DefaultApi.md#statistics_data_collection_get) | **GET** /statistics/data_collection | Get data collection statistics @@ -1729,6 +1735,318 @@ No authorization required [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) +# **config_rugnux_get** +> RugnuxStatus config_rugnux_get() + +Get rugnux live mode settings and model status + +Can be done anytime + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.models.rugnux_status import RugnuxStatus +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + + try: + # Get rugnux live mode settings and model status + api_response = api_instance.config_rugnux_get() + print("The response of DefaultApi->config_rugnux_get:\n") + pprint(api_response) + except Exception as e: + print("Exception when calling DefaultApi->config_rugnux_get: %s\n" % e) +``` + + + +### Parameters + +This endpoint does not need any parameter. + +### Return type + +[**RugnuxStatus**](RugnuxStatus.md) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: text/plain, application/json + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**200** | Everything OK | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + +# **config_rugnux_model_delete** +> config_rugnux_model_delete() + +Remove the model used by rugnux live mode + +This can only be done when detector is `Idle`, `Error` or `Inactive` states. + + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + + try: + # Remove the model used by rugnux live mode + api_instance.config_rugnux_model_delete() + except Exception as e: + print("Exception when calling DefaultApi->config_rugnux_model_delete: %s\n" % e) +``` + + + +### Parameters + +This endpoint does not need any parameter. + +### Return type + +void (empty response body) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: text/plain, application/json + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**200** | All good | - | +**500** | Error within Jungfraujoch code - see output message. | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + +# **config_rugnux_model_put** +> config_rugnux_model_put(body=body) + +Upload the model used by rugnux live mode + +This can only be done when detector is `Idle`, `Error` or `Inactive` states. +The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement +and validation of every following rugnux live collection, until it is removed with +`DELETE /config/rugnux/model` or replaced by another upload. + + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + body = None # bytearray | (optional) + + try: + # Upload the model used by rugnux live mode + api_instance.config_rugnux_model_put(body=body) + except Exception as e: + print("Exception when calling DefaultApi->config_rugnux_model_put: %s\n" % e) +``` + + + +### Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **body** | **bytearray**| | [optional] + +### Return type + +void (empty response body) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: application/octet-stream + - **Accept**: text/plain, application/json + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**200** | All good | - | +**400** | Empty body | - | +**413** | Model larger than 64 MiB | - | +**500** | Error within Jungfraujoch code - see output message. | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + +# **config_rugnux_put** +> config_rugnux_put(rugnux_settings=rugnux_settings) + +Change rugnux live mode settings + +This can only be done when detector is `Idle`, `Error` or `Inactive` states. + + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.models.rugnux_settings import RugnuxSettings +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + rugnux_settings = jfjoch_client.RugnuxSettings() # RugnuxSettings | (optional) + + try: + # Change rugnux live mode settings + api_instance.config_rugnux_put(rugnux_settings=rugnux_settings) + except Exception as e: + print("Exception when calling DefaultApi->config_rugnux_put: %s\n" % e) +``` + + + +### Parameters + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **rugnux_settings** | [**RugnuxSettings**](RugnuxSettings.md)| | [optional] + +### Return type + +void (empty response body) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: application/json + - **Accept**: text/plain, application/json + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**200** | Everything OK | - | +**400** | Input parsing or validation error | - | +**500** | Error within Jungfraujoch code - see output message. | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + # **config_select_detector_get** > DetectorList config_select_detector_get() @@ -3326,6 +3644,96 @@ No authorization required [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) +# **live_events_get** +> str live_events_get() + +Event stream of the collection, for a reader following it + +Server-sent events (text/event-stream) for the current collection, from the moment /start has +accepted it: first its whole history, then every change, until it ends or a new collection +begins. Only one reader at a time. Events, each with a JSON `data` line: +- `start` {run_number, run_name, file_prefix, number_of_images, images_per_file, cbor} - first, + once the start message is known: at once for a JUNGFRAU, when the detector sends its own for + a DECTRIS. `cbor` is the start message the writer of the master file received, CBOR in base64; + a reader builds the dataset from it; +- `file` {file_number, file, first_image, image_count} - a data file closed and renamed into + place; `file` is relative to the collection's root, as the writer named it; +- `end` {images, error?} - every writer acknowledged the end: the data files and the master + file are in place; `images` is the number of images collected. Also sent, with `error`, when + the measurement ends any other way - before the start message, for one. The stream then closes; +- `superseded` {run_number} - a new collection began. The stream then closes. +A comment line (`: keepalive`) every 10 s when nothing happens. + + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + + try: + # Event stream of the collection, for a reader following it + api_response = api_instance.live_events_get() + print("The response of DefaultApi->live_events_get:\n") + pprint(api_response) + except Exception as e: + print("Exception when calling DefaultApi->live_events_get: %s\n" % e) +``` + + + +### Parameters + +This endpoint does not need any parameter. + +### Return type + +**str** + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: text/event-stream, text/plain + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**200** | Event stream | - | +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**404** | No collection has started | - | +**409** | Another reader is following the collection, or the image pusher does not report closed data files | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + # **pedestal_post** > pedestal_post() @@ -3702,6 +4110,81 @@ This endpoint does not need any parameter. [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) +# **rugnux_status_get** +> RugnuxLiveStatus rugnux_status_get() + +Get live rugnux processing status of the last collection + +Can be done anytime + +### Example + +* Bearer Authentication (bearerAuth): + +```python +import jfjoch_client +from jfjoch_client.models.rugnux_live_status import RugnuxLiveStatus +from jfjoch_client.rest import ApiException +from pprint import pprint + +# Defining the host is optional and defaults to http://localhost:5232 +# See configuration.py for a list of all supported configuration parameters. +configuration = jfjoch_client.Configuration( + host = "http://localhost:5232" +) + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +# Configure Bearer authorization: bearerAuth +configuration = jfjoch_client.Configuration( + access_token = os.environ["BEARER_TOKEN"] +) + +# Enter a context with an instance of the API client +with jfjoch_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = jfjoch_client.DefaultApi(api_client) + + try: + # Get live rugnux processing status of the last collection + api_response = api_instance.rugnux_status_get() + print("The response of DefaultApi->rugnux_status_get:\n") + pprint(api_response) + except Exception as e: + print("Exception when calling DefaultApi->rugnux_status_get: %s\n" % e) +``` + + + +### Parameters + +This endpoint does not need any parameter. + +### Return type + +[**RugnuxLiveStatus**](RugnuxLiveStatus.md) + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: text/plain, application/json + +### HTTP response details + +| Status code | Description | Response headers | +|-------------|-------------|------------------| +**401** | The current dataset is protected and no valid bearer token was presented. | - | +**200** | Everything OK | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + # **start_post** > start_post(dataset_settings=dataset_settings) diff --git a/docs/python_client/docs/JfjochSettings.md b/docs/python_client/docs/JfjochSettings.md index b783f858d..f2ebba9f6 100644 --- a/docs/python_client/docs/JfjochSettings.md +++ b/docs/python_client/docs/JfjochSettings.md @@ -27,6 +27,7 @@ Name | Type | Description | Notes **zeromq_preview** | [**ZeromqPreviewSettings**](ZeromqPreviewSettings.md) | | [optional] **zeromq_metadata** | [**ZeromqMetadataSettings**](ZeromqMetadataSettings.md) | | [optional] **dark_mask** | [**DarkMaskSettings**](DarkMaskSettings.md) | | [optional] +**rugnux** | [**RugnuxSettings**](RugnuxSettings.md) | | [optional] ## Example diff --git a/docs/python_client/docs/RugnuxLiveStatus.md b/docs/python_client/docs/RugnuxLiveStatus.md new file mode 100644 index 000000000..631dae4d0 --- /dev/null +++ b/docs/python_client/docs/RugnuxLiveStatus.md @@ -0,0 +1,43 @@ +# RugnuxLiveStatus + +Live rugnux processing of the last collection; kept until the next collection starts + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**state** | **str** | | +**probe** | **str** | Answer of the early lattice probe on the first 60 degrees; empty until it answers | +**probe_s** | **float** | When the probe answered, since the collection started | [optional] +**message** | **str** | Why the run failed or was skipped | +**images_expected** | **int** | | +**images_received** | **int** | | +**elapsed_s** | **float** | | +**step_progress** | **float** | Fraction of the current pass's images | [optional] +**steps** | [**List[RugnuxLiveStep]**](RugnuxLiveStep.md) | | +**last_image_to_written_s** | **float** | From the last image received to all results written | [optional] +**verdict** | **str** | OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over | +**verdict_text** | **str** | | +**summary** | [**List[RugnuxSummaryRow]**](RugnuxSummaryRow.md) | | +**warnings** | **List[str]** | | + +## Example + +```python +from jfjoch_client.models.rugnux_live_status import RugnuxLiveStatus + +# TODO update the JSON string below +json = "{}" +# create an instance of RugnuxLiveStatus from a JSON string +rugnux_live_status_instance = RugnuxLiveStatus.from_json(json) +# print the JSON string representation of the object +print(RugnuxLiveStatus.to_json()) + +# convert the object into a dict +rugnux_live_status_dict = rugnux_live_status_instance.to_dict() +# create an instance of RugnuxLiveStatus from a dict +rugnux_live_status_from_dict = RugnuxLiveStatus.from_dict(rugnux_live_status_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/docs/python_client/docs/RugnuxLiveStep.md b/docs/python_client/docs/RugnuxLiveStep.md new file mode 100644 index 000000000..546a6c19c --- /dev/null +++ b/docs/python_client/docs/RugnuxLiveStep.md @@ -0,0 +1,32 @@ +# RugnuxLiveStep + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**title** | **str** | | +**started_s** | **float** | Start of the step, since the collection started | +**duration_s** | **float** | Set once the next step starts or the run ends | [optional] +**notes** | **List[str]** | What the step found (e.g. lattice, space group) | + +## Example + +```python +from jfjoch_client.models.rugnux_live_step import RugnuxLiveStep + +# TODO update the JSON string below +json = "{}" +# create an instance of RugnuxLiveStep from a JSON string +rugnux_live_step_instance = RugnuxLiveStep.from_json(json) +# print the JSON string representation of the object +print(RugnuxLiveStep.to_json()) + +# convert the object into a dict +rugnux_live_step_dict = rugnux_live_step_instance.to_dict() +# create an instance of RugnuxLiveStep from a dict +rugnux_live_step_from_dict = RugnuxLiveStep.from_dict(rugnux_live_step_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/docs/python_client/docs/RugnuxSettings.md b/docs/python_client/docs/RugnuxSettings.md new file mode 100644 index 000000000..830ba02ff --- /dev/null +++ b/docs/python_client/docs/RugnuxSettings.md @@ -0,0 +1,31 @@ +# RugnuxSettings + +Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux inside the broker while they arrive, and the merged result is sent to the writer. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**enable** | **bool** | Process DECTRIS rotation collections with rugnux live | [optional] [default to False] +**reserved_cpus** | **int** | CPU threads left to the receiver and the image pusher while rugnux runs | [optional] [default to 4] + +## Example + +```python +from jfjoch_client.models.rugnux_settings import RugnuxSettings + +# TODO update the JSON string below +json = "{}" +# create an instance of RugnuxSettings from a JSON string +rugnux_settings_instance = RugnuxSettings.from_json(json) +# print the JSON string representation of the object +print(RugnuxSettings.to_json()) + +# convert the object into a dict +rugnux_settings_dict = rugnux_settings_instance.to_dict() +# create an instance of RugnuxSettings from a dict +rugnux_settings_from_dict = RugnuxSettings.from_dict(rugnux_settings_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/docs/python_client/docs/RugnuxStatus.md b/docs/python_client/docs/RugnuxStatus.md new file mode 100644 index 000000000..8859133d0 --- /dev/null +++ b/docs/python_client/docs/RugnuxStatus.md @@ -0,0 +1,32 @@ +# RugnuxStatus + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**enable** | **bool** | Rugnux live mode is enabled | +**reserved_cpus** | **int** | CPU threads left to the receiver and the image pusher while rugnux runs | +**model_loaded** | **bool** | A model was uploaded with `PUT /config/rugnux/model` | +**model_size_bytes** | **int** | Size of the uploaded model file | [optional] + +## Example + +```python +from jfjoch_client.models.rugnux_status import RugnuxStatus + +# TODO update the JSON string below +json = "{}" +# create an instance of RugnuxStatus from a JSON string +rugnux_status_instance = RugnuxStatus.from_json(json) +# print the JSON string representation of the object +print(RugnuxStatus.to_json()) + +# convert the object into a dict +rugnux_status_dict = rugnux_status_instance.to_dict() +# create an instance of RugnuxStatus from a dict +rugnux_status_from_dict = RugnuxStatus.from_dict(rugnux_status_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/docs/python_client/docs/RugnuxSummaryRow.md b/docs/python_client/docs/RugnuxSummaryRow.md new file mode 100644 index 000000000..abee1e439 --- /dev/null +++ b/docs/python_client/docs/RugnuxSummaryRow.md @@ -0,0 +1,30 @@ +# RugnuxSummaryRow + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**label** | **str** | | +**value** | **str** | | + +## Example + +```python +from jfjoch_client.models.rugnux_summary_row import RugnuxSummaryRow + +# TODO update the JSON string below +json = "{}" +# create an instance of RugnuxSummaryRow from a JSON string +rugnux_summary_row_instance = RugnuxSummaryRow.from_json(json) +# print the JSON string representation of the object +print(RugnuxSummaryRow.to_json()) + +# convert the object into a dict +rugnux_summary_row_dict = rugnux_summary_row_instance.to_dict() +# create an instance of RugnuxSummaryRow from a dict +rugnux_summary_row_from_dict = RugnuxSummaryRow.from_dict(rugnux_summary_row_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c3f749f13..23c657b5d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -24,6 +24,7 @@ import MenuBookIcon from "@mui/icons-material/MenuBook"; import HistoryIcon from "@mui/icons-material/History"; import CodeIcon from "@mui/icons-material/Code"; import GavelIcon from "@mui/icons-material/Gavel"; +import AutoGraphIcon from "@mui/icons-material/AutoGraph"; import _ from "lodash"; import Markdown from "react-markdown"; import remarkGfm from "remark-gfm"; @@ -32,6 +33,8 @@ import StatusBar from "./components/StatusBar"; import ErrorMessage from "./components/ErrorMessage"; import DataProcessingPlots from "./components/DataProcessingPlots"; import MeasurementStatistics from "./components/MeasurementStatistics"; +import RugnuxStatus from "./components/RugnuxStatus"; +import RugnuxPage from "./components/RugnuxPage"; import DataProcessingSettings from "./components/DataProcessingSettings"; import PreviewImage from "./components/PreviewImage"; import DataCollection from "./components/DataCollection"; @@ -168,7 +171,10 @@ function App() { - + + + + @@ -188,6 +194,9 @@ function App() { { id: 'data-collection', label: 'Data collection', icon: , render: () => ( )}, + { id: 'rugnux', label: 'Rugnux live', icon: , render: () => ( + + )}, ]}, { subheader: 'Expert', items: [ { id: 'detector', label: 'Detector', icon: , render: () => ( diff --git a/frontend/src/client/@tanstack/react-query.gen.ts b/frontend/src/client/@tanstack/react-query.gen.ts index c75023d1d..db804e38d 100644 --- a/frontend/src/client/@tanstack/react-query.gen.ts +++ b/frontend/src/client/@tanstack/react-query.gen.ts @@ -3,8 +3,8 @@ import { type DefaultError, queryOptions, type UseMutationOptions } from '@tanstack/react-query'; import { client } from '../client.gen'; -import { getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from '../sdk.gen'; -import type { getConfigAzimIntData, getConfigAzimIntResponse, getConfigBraggIntegrationData, getConfigBraggIntegrationResponse, getConfigDarkMaskData, getConfigDarkMaskResponse, getConfigDetectorData, getConfigDetectorResponse, getConfigFileWriterData, getConfigFileWriterResponse, getConfigImageFormatData, getConfigImageFormatResponse, getConfigIndexingData, getConfigIndexingResponse, getConfigInstrumentData, getConfigInstrumentResponse, getConfigMaskData, getConfigMaskResponse, getConfigMaskTiffData, getConfigMaskTiffResponse, getConfigRoiData, getConfigRoiResponse, getConfigSelectDetectorData, getConfigSelectDetectorResponse, getConfigSpotFindingData, getConfigSpotFindingResponse, getConfigUserMaskData, getConfigUserMaskResponse, getConfigUserMaskTiffData, getConfigUserMaskTiffResponse, getConfigZeromqMetadataData, getConfigZeromqMetadataResponse, getConfigZeromqPreviewData, getConfigZeromqPreviewResponse, getDetectorStatusData, getDetectorStatusError, getDetectorStatusResponse, getFpgaStatusData, getFpgaStatusResponse, getImageBufferImageCborData, getImageBufferImageCborError, getImageBufferImageCborResponse, getImageBufferImageJpegData, getImageBufferImageJpegError, getImageBufferImageJpegResponse, getImageBufferImageTiffData, getImageBufferImageTiffError, getImageBufferImageTiffResponse, getImageBufferStartCborData, getImageBufferStartCborError, getImageBufferStartCborResponse, getImageBufferStatusData, getImageBufferStatusError, getImageBufferStatusResponse, getImagePusherStatusData, getImagePusherStatusError, getImagePusherStatusResponse, getPreviewPedestalTiffData, getPreviewPedestalTiffResponse, getPreviewPlotBinData, getPreviewPlotBinError, getPreviewPlotBinResponse, getPreviewPlotData, getPreviewPlotError, getPreviewPlotResponse, getResultScanData, getResultScanError, getResultScanResponse, getStatisticsCalibrationData, getStatisticsCalibrationResponse, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionError, getStatisticsDataCollectionResponse, getStatisticsResponse, getStatusData, getStatusResponse, getVersionData, getVersionResponse, getXfelEventCodeData, getXfelEventCodeResponse, getXfelPulseIdData, getXfelPulseIdResponse, postCancelData, postConfigImageFormatConversionData, postConfigImageFormatConversionError, postConfigImageFormatRawData, postConfigImageFormatRawError, postDeactivateData, postDeactivateError, postImageBufferClearData, postImageBufferClearError, postInitializeData, postInitializeError, postPedestalData, postPedestalError, postStartData, postStartError, postTriggerData, postWaitTillDoneData, postWaitTillDoneError, postWaitUntilRunningData, postWaitUntilRunningError, putConfigAzimIntData, putConfigAzimIntError, putConfigBraggIntegrationData, putConfigBraggIntegrationError, putConfigDarkMaskData, putConfigDarkMaskError, putConfigDetectorData, putConfigDetectorError, putConfigFileWriterData, putConfigFileWriterError, putConfigImageFormatData, putConfigImageFormatError, putConfigIndexingData, putConfigIndexingError, putConfigInstrumentData, putConfigInstrumentError, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageError, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffError, putConfigRoiData, putConfigRoiError, putConfigSelectDetectorData, putConfigSelectDetectorError, putConfigSpotFindingData, putConfigSpotFindingError, putConfigUserMaskData, putConfigUserMaskError, putConfigUserMaskTiffData, putConfigUserMaskTiffError, putConfigZeromqMetadataData, putConfigZeromqMetadataError, putConfigZeromqPreviewData, putConfigZeromqPreviewError } from '../types.gen'; +import { deleteConfigRugnuxModel, getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigRugnux, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getRugnuxStatus, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigRugnux, putConfigRugnuxModel, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from '../sdk.gen'; +import type { deleteConfigRugnuxModelData, deleteConfigRugnuxModelError, getConfigAzimIntData, getConfigAzimIntResponse, getConfigBraggIntegrationData, getConfigBraggIntegrationResponse, getConfigDarkMaskData, getConfigDarkMaskResponse, getConfigDetectorData, getConfigDetectorResponse, getConfigFileWriterData, getConfigFileWriterResponse, getConfigImageFormatData, getConfigImageFormatResponse, getConfigIndexingData, getConfigIndexingResponse, getConfigInstrumentData, getConfigInstrumentResponse, getConfigMaskData, getConfigMaskResponse, getConfigMaskTiffData, getConfigMaskTiffResponse, getConfigRoiData, getConfigRoiResponse, getConfigRugnuxData, getConfigRugnuxError, getConfigRugnuxResponse, getConfigSelectDetectorData, getConfigSelectDetectorResponse, getConfigSpotFindingData, getConfigSpotFindingResponse, getConfigUserMaskData, getConfigUserMaskResponse, getConfigUserMaskTiffData, getConfigUserMaskTiffResponse, getConfigZeromqMetadataData, getConfigZeromqMetadataResponse, getConfigZeromqPreviewData, getConfigZeromqPreviewResponse, getDetectorStatusData, getDetectorStatusError, getDetectorStatusResponse, getFpgaStatusData, getFpgaStatusResponse, getImageBufferImageCborData, getImageBufferImageCborError, getImageBufferImageCborResponse, getImageBufferImageJpegData, getImageBufferImageJpegError, getImageBufferImageJpegResponse, getImageBufferImageTiffData, getImageBufferImageTiffError, getImageBufferImageTiffResponse, getImageBufferStartCborData, getImageBufferStartCborError, getImageBufferStartCborResponse, getImageBufferStatusData, getImageBufferStatusError, getImageBufferStatusResponse, getImagePusherStatusData, getImagePusherStatusError, getImagePusherStatusResponse, getPreviewPedestalTiffData, getPreviewPedestalTiffResponse, getPreviewPlotBinData, getPreviewPlotBinError, getPreviewPlotBinResponse, getPreviewPlotData, getPreviewPlotError, getPreviewPlotResponse, getResultScanData, getResultScanError, getResultScanResponse, getRugnuxStatusData, getRugnuxStatusError, getRugnuxStatusResponse, getStatisticsCalibrationData, getStatisticsCalibrationResponse, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionError, getStatisticsDataCollectionResponse, getStatisticsResponse, getStatusData, getStatusResponse, getVersionData, getVersionResponse, getXfelEventCodeData, getXfelEventCodeResponse, getXfelPulseIdData, getXfelPulseIdResponse, postCancelData, postConfigImageFormatConversionData, postConfigImageFormatConversionError, postConfigImageFormatRawData, postConfigImageFormatRawError, postDeactivateData, postDeactivateError, postImageBufferClearData, postImageBufferClearError, postInitializeData, postInitializeError, postPedestalData, postPedestalError, postStartData, postStartError, postTriggerData, postWaitTillDoneData, postWaitTillDoneError, postWaitUntilRunningData, postWaitUntilRunningError, putConfigAzimIntData, putConfigAzimIntError, putConfigBraggIntegrationData, putConfigBraggIntegrationError, putConfigDarkMaskData, putConfigDarkMaskError, putConfigDetectorData, putConfigDetectorError, putConfigFileWriterData, putConfigFileWriterError, putConfigImageFormatData, putConfigImageFormatError, putConfigIndexingData, putConfigIndexingError, putConfigInstrumentData, putConfigInstrumentError, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageError, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffError, putConfigRoiData, putConfigRoiError, putConfigRugnuxData, putConfigRugnuxError, putConfigRugnuxModelData, putConfigRugnuxModelError, putConfigSelectDetectorData, putConfigSelectDetectorError, putConfigSpotFindingData, putConfigSpotFindingError, putConfigUserMaskData, putConfigUserMaskError, putConfigUserMaskTiffData, putConfigUserMaskTiffError, putConfigZeromqMetadataData, putConfigZeromqMetadataError, putConfigZeromqPreviewData, putConfigZeromqPreviewError } from '../types.gen'; /** * Initialize detector and data acquisition @@ -156,6 +156,11 @@ export const postTriggerMutation = (options?: Partial>) * Any frame that is currently being processed by CPU will be finished and sent to writer. * Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state. * + * With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection + * or after its last image alike: the collection ends with what the analysis has established + * (see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not + * marked as cancelled. + * * If data collection is not running, the command has no effect. * */ @@ -675,6 +680,109 @@ export const putConfigSelectDetectorMutation = (options?: Partial) => createQueryKey('getConfigRugnux', options); + +/** + * Get rugnux live mode settings and model status + * + * Can be done anytime + */ +export const getConfigRugnuxOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await getConfigRugnux({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: getConfigRugnuxQueryKey(options) +}); + +/** + * Change rugnux live mode settings + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * + */ +export const putConfigRugnuxMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await putConfigRugnux({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +/** + * Remove the model used by rugnux live mode + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * + */ +export const deleteConfigRugnuxModelMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await deleteConfigRugnuxModel({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +/** + * Upload the model used by rugnux live mode + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement + * and validation of every following rugnux live collection, until it is removed with + * `DELETE /config/rugnux/model` or replaced by another upload. + * + */ +export const putConfigRugnuxModelMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await putConfigRugnuxModel({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + +export const getRugnuxStatusQueryKey = (options?: Options) => createQueryKey('getRugnuxStatus', options); + +/** + * Get live rugnux processing status of the last collection + * + * Can be done anytime + */ +export const getRugnuxStatusOptions = (options?: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await getRugnuxStatus({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: getRugnuxStatusQueryKey(options) +}); + export const getConfigZeromqPreviewQueryKey = (options?: Options) => createQueryKey('getConfigZeromqPreview', options); /** diff --git a/frontend/src/client/index.ts b/frontend/src/client/index.ts index 12298acb4..a99adbd54 100644 --- a/frontend/src/client/index.ts +++ b/frontend/src/client/index.ts @@ -1,4 +1,4 @@ // This file is auto-generated by @hey-api/openapi-ts -export { getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from './sdk.gen'; -export { type azim_int_settings, azint_unit, type background, type binning, type bragg_integration_settings, type broker_status, type calibration_statistics, type ClientOptions, color_scale, type crystal_lattice, type dark_mask_settings, type dataset_settings, type dataset_settingsWritable, type detector, type detector_list, type detector_list_element, type detector_module, detector_module_direction, detector_power_state, type detector_selection, type detector_settings, detector_state, type detector_status, detector_timing, detector_type, type error_message, type experimental_coord, file_writer_format, type file_writer_settings, type fill_value, type fpga_status, geom_refinement_algorithm, type getConfigAzimIntData, type getConfigAzimIntResponse, type getConfigAzimIntResponses, type getConfigBraggIntegrationData, type getConfigBraggIntegrationResponse, type getConfigBraggIntegrationResponses, type getConfigDarkMaskData, type getConfigDarkMaskResponse, type getConfigDarkMaskResponses, type getConfigDetectorData, type getConfigDetectorResponse, type getConfigDetectorResponses, type getConfigFileWriterData, type getConfigFileWriterResponse, type getConfigFileWriterResponses, type getConfigImageFormatData, type getConfigImageFormatResponse, type getConfigImageFormatResponses, type getConfigIndexingData, type getConfigIndexingResponse, type getConfigIndexingResponses, type getConfigInstrumentData, type getConfigInstrumentResponse, type getConfigInstrumentResponses, type getConfigMaskData, type getConfigMaskResponse, type getConfigMaskResponses, type getConfigMaskTiffData, type getConfigMaskTiffResponse, type getConfigMaskTiffResponses, type getConfigRoiData, type getConfigRoiResponse, type getConfigRoiResponses, type getConfigSelectDetectorData, type getConfigSelectDetectorResponse, type getConfigSelectDetectorResponses, type getConfigSpotFindingData, type getConfigSpotFindingResponse, type getConfigSpotFindingResponses, type getConfigUserMaskData, type getConfigUserMaskResponse, type getConfigUserMaskResponses, type getConfigUserMaskTiffData, type getConfigUserMaskTiffResponse, type getConfigUserMaskTiffResponses, type getConfigZeromqMetadataData, type getConfigZeromqMetadataResponse, type getConfigZeromqMetadataResponses, type getConfigZeromqPreviewData, type getConfigZeromqPreviewResponse, type getConfigZeromqPreviewResponses, type getDetectorStatusData, type getDetectorStatusError, type getDetectorStatusErrors, type getDetectorStatusResponse, type getDetectorStatusResponses, type getFpgaStatusData, type getFpgaStatusResponse, type getFpgaStatusResponses, type getImageBufferImageCborData, type getImageBufferImageCborError, type getImageBufferImageCborErrors, type getImageBufferImageCborResponse, type getImageBufferImageCborResponses, type getImageBufferImageJpegData, type getImageBufferImageJpegError, type getImageBufferImageJpegErrors, type getImageBufferImageJpegResponse, type getImageBufferImageJpegResponses, type getImageBufferImageTiffData, type getImageBufferImageTiffError, type getImageBufferImageTiffErrors, type getImageBufferImageTiffResponse, type getImageBufferImageTiffResponses, type getImageBufferStartCborData, type getImageBufferStartCborError, type getImageBufferStartCborErrors, type getImageBufferStartCborResponse, type getImageBufferStartCborResponses, type getImageBufferStatusData, type getImageBufferStatusError, type getImageBufferStatusErrors, type getImageBufferStatusResponse, type getImageBufferStatusResponses, type getImagePusherStatusData, type getImagePusherStatusError, type getImagePusherStatusErrors, type getImagePusherStatusResponse, type getImagePusherStatusResponses, type getPreviewPedestalTiffData, type getPreviewPedestalTiffErrors, type getPreviewPedestalTiffResponse, type getPreviewPedestalTiffResponses, type getPreviewPlotBinData, type getPreviewPlotBinError, type getPreviewPlotBinErrors, type getPreviewPlotBinResponse, type getPreviewPlotBinResponses, type getPreviewPlotData, type getPreviewPlotError, type getPreviewPlotErrors, type getPreviewPlotResponse, type getPreviewPlotResponses, type getResultScanData, type getResultScanError, type getResultScanErrors, type getResultScanResponse, type getResultScanResponses, type getStatisticsCalibrationData, type getStatisticsCalibrationResponse, type getStatisticsCalibrationResponses, type getStatisticsData, type getStatisticsDataCollectionData, type getStatisticsDataCollectionError, type getStatisticsDataCollectionErrors, type getStatisticsDataCollectionResponse, type getStatisticsDataCollectionResponses, type getStatisticsResponse, type getStatisticsResponses, type getStatusData, type getStatusResponse, type getStatusResponses, type getVersionData, type getVersionResponse, type getVersionResponses, type getXfelEventCodeData, type getXfelEventCodeErrors, type getXfelEventCodeResponse, type getXfelEventCodeResponses, type getXfelPulseIdData, type getXfelPulseIdErrors, type getXfelPulseIdResponse, type getXfelPulseIdResponses, type grid_scan, type image_buffer_status, type image_format_settings, type image_id, type image_pusher_status, image_pusher_type, indexing_algorithm, type indexing_settings, type instrument_metadata, integration_model, type jfjoch_settings, type jfjoch_statistics, type jpeg_quality, type measurement_statistics, type pcie_devices, type pixel_mask_statistics, type plot, plot_type, plot_unit_x, type plots, type postCancelData, type postCancelResponses, type postConfigImageFormatConversionData, type postConfigImageFormatConversionError, type postConfigImageFormatConversionErrors, type postConfigImageFormatConversionResponses, type postConfigImageFormatRawData, type postConfigImageFormatRawError, type postConfigImageFormatRawErrors, type postConfigImageFormatRawResponses, type postDeactivateData, type postDeactivateError, type postDeactivateErrors, type postDeactivateResponses, type postImageBufferClearData, type postImageBufferClearError, type postImageBufferClearErrors, type postImageBufferClearResponses, type postInitializeData, type postInitializeError, type postInitializeErrors, type postInitializeResponses, type postPedestalData, type postPedestalError, type postPedestalErrors, type postPedestalResponses, type postStartData, type postStartError, type postStartErrors, type postStartResponses, type postTriggerData, type postTriggerResponses, type postWaitTillDoneData, type postWaitTillDoneError, type postWaitTillDoneErrors, type postWaitTillDoneResponses, type postWaitUntilRunningData, type postWaitUntilRunningError, type postWaitUntilRunningErrors, type postWaitUntilRunningResponses, type powder_calibration_fit_sigma, type powder_calibration_output, type powder_calibration_outputWritable, type powder_calibration_quality, type powder_calibration_spot_check, type putConfigAzimIntData, type putConfigAzimIntError, type putConfigAzimIntErrors, type putConfigAzimIntResponses, type putConfigBraggIntegrationData, type putConfigBraggIntegrationError, type putConfigBraggIntegrationErrors, type putConfigBraggIntegrationResponses, type putConfigDarkMaskData, type putConfigDarkMaskError, type putConfigDarkMaskErrors, type putConfigDarkMaskResponses, type putConfigDetectorData, type putConfigDetectorError, type putConfigDetectorErrors, type putConfigDetectorResponses, type putConfigFileWriterData, type putConfigFileWriterError, type putConfigFileWriterErrors, type putConfigFileWriterResponses, type putConfigImageFormatData, type putConfigImageFormatError, type putConfigImageFormatErrors, type putConfigImageFormatResponses, type putConfigIndexingData, type putConfigIndexingError, type putConfigIndexingErrors, type putConfigIndexingResponses, type putConfigInstrumentData, type putConfigInstrumentError, type putConfigInstrumentErrors, type putConfigInstrumentResponses, type putConfigInternalGeneratorImageData, type putConfigInternalGeneratorImageError, type putConfigInternalGeneratorImageErrors, type putConfigInternalGeneratorImageResponses, type putConfigInternalGeneratorImageTiffData, type putConfigInternalGeneratorImageTiffError, type putConfigInternalGeneratorImageTiffErrors, type putConfigInternalGeneratorImageTiffResponses, type putConfigRoiData, type putConfigRoiError, type putConfigRoiErrors, type putConfigRoiResponses, type putConfigSelectDetectorData, type putConfigSelectDetectorError, type putConfigSelectDetectorErrors, type putConfigSelectDetectorResponses, type putConfigSpotFindingData, type putConfigSpotFindingError, type putConfigSpotFindingErrors, type putConfigSpotFindingResponses, type putConfigUserMaskData, type putConfigUserMaskError, type putConfigUserMaskErrors, type putConfigUserMaskResponses, type putConfigUserMaskTiffData, type putConfigUserMaskTiffError, type putConfigUserMaskTiffErrors, type putConfigUserMaskTiffResponses, type putConfigZeromqMetadataData, type putConfigZeromqMetadataError, type putConfigZeromqMetadataErrors, type putConfigZeromqMetadataResponses, type putConfigZeromqPreviewData, type putConfigZeromqPreviewError, type putConfigZeromqPreviewErrors, type putConfigZeromqPreviewResponses, type resolution_estimate, type resolution_ring, type roi, type roi_azim_list, type roi_azimuthal, type roi_box, type roi_box_list, type roi_circle, type roi_circle_list, type roi_definitions, type rotation_axis, type saturation, type scan_result, type show_beam_center, type show_predictions, type show_roi, type show_spots, type show_user_mask, type spot_finding_settings, type standard_detector_geometry, type tcp_settings, type unit_cell, type zeromq_metadata_settings, type zeromq_preview_settings, type zeromq_settings } from './types.gen'; +export { deleteConfigRugnuxModel, getConfigAzimInt, getConfigBraggIntegration, getConfigDarkMask, getConfigDetector, getConfigFileWriter, getConfigImageFormat, getConfigIndexing, getConfigInstrument, getConfigMask, getConfigMaskTiff, getConfigRoi, getConfigRugnux, getConfigSelectDetector, getConfigSpotFinding, getConfigUserMask, getConfigUserMaskTiff, getConfigZeromqMetadata, getConfigZeromqPreview, getDetectorStatus, getFpgaStatus, getImageBufferImageCbor, getImageBufferImageJpeg, getImageBufferImageTiff, getImageBufferStartCbor, getImageBufferStatus, getImagePusherStatus, getLiveEvents, getPreviewPedestalTiff, getPreviewPlot, getPreviewPlotBin, getResultScan, getRugnuxStatus, getStatistics, getStatisticsCalibration, getStatisticsDataCollection, getStatus, getVersion, getXfelEventCode, getXfelPulseId, type Options, postCancel, postConfigImageFormatConversion, postConfigImageFormatRaw, postDeactivate, postImageBufferClear, postInitialize, postPedestal, postStart, postTrigger, postWaitTillDone, postWaitUntilRunning, putConfigAzimInt, putConfigBraggIntegration, putConfigDarkMask, putConfigDetector, putConfigFileWriter, putConfigImageFormat, putConfigIndexing, putConfigInstrument, putConfigInternalGeneratorImage, putConfigInternalGeneratorImageTiff, putConfigRoi, putConfigRugnux, putConfigRugnuxModel, putConfigSelectDetector, putConfigSpotFinding, putConfigUserMask, putConfigUserMaskTiff, putConfigZeromqMetadata, putConfigZeromqPreview } from './sdk.gen'; +export { type azim_int_settings, azint_unit, type background, type binning, type bragg_integration_settings, type broker_status, type calibration_statistics, type ClientOptions, color_scale, type crystal_lattice, type dark_mask_settings, type dataset_settings, type dataset_settingsWritable, type deleteConfigRugnuxModelData, type deleteConfigRugnuxModelError, type deleteConfigRugnuxModelErrors, type deleteConfigRugnuxModelResponses, type detector, type detector_list, type detector_list_element, type detector_module, detector_module_direction, detector_power_state, type detector_selection, type detector_settings, detector_state, type detector_status, detector_timing, detector_type, type error_message, type experimental_coord, file_writer_format, type file_writer_settings, type fill_value, type fpga_status, geom_refinement_algorithm, type getConfigAzimIntData, type getConfigAzimIntResponse, type getConfigAzimIntResponses, type getConfigBraggIntegrationData, type getConfigBraggIntegrationResponse, type getConfigBraggIntegrationResponses, type getConfigDarkMaskData, type getConfigDarkMaskResponse, type getConfigDarkMaskResponses, type getConfigDetectorData, type getConfigDetectorResponse, type getConfigDetectorResponses, type getConfigFileWriterData, type getConfigFileWriterResponse, type getConfigFileWriterResponses, type getConfigImageFormatData, type getConfigImageFormatResponse, type getConfigImageFormatResponses, type getConfigIndexingData, type getConfigIndexingResponse, type getConfigIndexingResponses, type getConfigInstrumentData, type getConfigInstrumentResponse, type getConfigInstrumentResponses, type getConfigMaskData, type getConfigMaskResponse, type getConfigMaskResponses, type getConfigMaskTiffData, type getConfigMaskTiffResponse, type getConfigMaskTiffResponses, type getConfigRoiData, type getConfigRoiResponse, type getConfigRoiResponses, type getConfigRugnuxData, type getConfigRugnuxError, type getConfigRugnuxErrors, type getConfigRugnuxResponse, type getConfigRugnuxResponses, type getConfigSelectDetectorData, type getConfigSelectDetectorResponse, type getConfigSelectDetectorResponses, type getConfigSpotFindingData, type getConfigSpotFindingResponse, type getConfigSpotFindingResponses, type getConfigUserMaskData, type getConfigUserMaskResponse, type getConfigUserMaskResponses, type getConfigUserMaskTiffData, type getConfigUserMaskTiffResponse, type getConfigUserMaskTiffResponses, type getConfigZeromqMetadataData, type getConfigZeromqMetadataResponse, type getConfigZeromqMetadataResponses, type getConfigZeromqPreviewData, type getConfigZeromqPreviewResponse, type getConfigZeromqPreviewResponses, type getDetectorStatusData, type getDetectorStatusError, type getDetectorStatusErrors, type getDetectorStatusResponse, type getDetectorStatusResponses, type getFpgaStatusData, type getFpgaStatusResponse, type getFpgaStatusResponses, type getImageBufferImageCborData, type getImageBufferImageCborError, type getImageBufferImageCborErrors, type getImageBufferImageCborResponse, type getImageBufferImageCborResponses, type getImageBufferImageJpegData, type getImageBufferImageJpegError, type getImageBufferImageJpegErrors, type getImageBufferImageJpegResponse, type getImageBufferImageJpegResponses, type getImageBufferImageTiffData, type getImageBufferImageTiffError, type getImageBufferImageTiffErrors, type getImageBufferImageTiffResponse, type getImageBufferImageTiffResponses, type getImageBufferStartCborData, type getImageBufferStartCborError, type getImageBufferStartCborErrors, type getImageBufferStartCborResponse, type getImageBufferStartCborResponses, type getImageBufferStatusData, type getImageBufferStatusError, type getImageBufferStatusErrors, type getImageBufferStatusResponse, type getImageBufferStatusResponses, type getImagePusherStatusData, type getImagePusherStatusError, type getImagePusherStatusErrors, type getImagePusherStatusResponse, type getImagePusherStatusResponses, type getLiveEventsData, type getLiveEventsError, type getLiveEventsErrors, type getLiveEventsResponse, type getLiveEventsResponses, type getPreviewPedestalTiffData, type getPreviewPedestalTiffErrors, type getPreviewPedestalTiffResponse, type getPreviewPedestalTiffResponses, type getPreviewPlotBinData, type getPreviewPlotBinError, type getPreviewPlotBinErrors, type getPreviewPlotBinResponse, type getPreviewPlotBinResponses, type getPreviewPlotData, type getPreviewPlotError, type getPreviewPlotErrors, type getPreviewPlotResponse, type getPreviewPlotResponses, type getResultScanData, type getResultScanError, type getResultScanErrors, type getResultScanResponse, type getResultScanResponses, type getRugnuxStatusData, type getRugnuxStatusError, type getRugnuxStatusErrors, type getRugnuxStatusResponse, type getRugnuxStatusResponses, type getStatisticsCalibrationData, type getStatisticsCalibrationResponse, type getStatisticsCalibrationResponses, type getStatisticsData, type getStatisticsDataCollectionData, type getStatisticsDataCollectionError, type getStatisticsDataCollectionErrors, type getStatisticsDataCollectionResponse, type getStatisticsDataCollectionResponses, type getStatisticsResponse, type getStatisticsResponses, type getStatusData, type getStatusResponse, type getStatusResponses, type getVersionData, type getVersionResponse, type getVersionResponses, type getXfelEventCodeData, type getXfelEventCodeErrors, type getXfelEventCodeResponse, type getXfelEventCodeResponses, type getXfelPulseIdData, type getXfelPulseIdErrors, type getXfelPulseIdResponse, type getXfelPulseIdResponses, type grid_scan, type image_buffer_status, type image_format_settings, type image_id, type image_pusher_status, image_pusher_type, indexing_algorithm, type indexing_settings, type instrument_metadata, integration_model, type jfjoch_settings, type jfjoch_statistics, type jpeg_quality, type measurement_statistics, type pcie_devices, type pixel_mask_statistics, type plot, plot_type, plot_unit_x, type plots, type postCancelData, type postCancelResponses, type postConfigImageFormatConversionData, type postConfigImageFormatConversionError, type postConfigImageFormatConversionErrors, type postConfigImageFormatConversionResponses, type postConfigImageFormatRawData, type postConfigImageFormatRawError, type postConfigImageFormatRawErrors, type postConfigImageFormatRawResponses, type postDeactivateData, type postDeactivateError, type postDeactivateErrors, type postDeactivateResponses, type postImageBufferClearData, type postImageBufferClearError, type postImageBufferClearErrors, type postImageBufferClearResponses, type postInitializeData, type postInitializeError, type postInitializeErrors, type postInitializeResponses, type postPedestalData, type postPedestalError, type postPedestalErrors, type postPedestalResponses, type postStartData, type postStartError, type postStartErrors, type postStartResponses, type postTriggerData, type postTriggerResponses, type postWaitTillDoneData, type postWaitTillDoneError, type postWaitTillDoneErrors, type postWaitTillDoneResponses, type postWaitUntilRunningData, type postWaitUntilRunningError, type postWaitUntilRunningErrors, type postWaitUntilRunningResponses, type powder_calibration_fit_sigma, type powder_calibration_output, type powder_calibration_outputWritable, type powder_calibration_quality, type powder_calibration_spot_check, type putConfigAzimIntData, type putConfigAzimIntError, type putConfigAzimIntErrors, type putConfigAzimIntResponses, type putConfigBraggIntegrationData, type putConfigBraggIntegrationError, type putConfigBraggIntegrationErrors, type putConfigBraggIntegrationResponses, type putConfigDarkMaskData, type putConfigDarkMaskError, type putConfigDarkMaskErrors, type putConfigDarkMaskResponses, type putConfigDetectorData, type putConfigDetectorError, type putConfigDetectorErrors, type putConfigDetectorResponses, type putConfigFileWriterData, type putConfigFileWriterError, type putConfigFileWriterErrors, type putConfigFileWriterResponses, type putConfigImageFormatData, type putConfigImageFormatError, type putConfigImageFormatErrors, type putConfigImageFormatResponses, type putConfigIndexingData, type putConfigIndexingError, type putConfigIndexingErrors, type putConfigIndexingResponses, type putConfigInstrumentData, type putConfigInstrumentError, type putConfigInstrumentErrors, type putConfigInstrumentResponses, type putConfigInternalGeneratorImageData, type putConfigInternalGeneratorImageError, type putConfigInternalGeneratorImageErrors, type putConfigInternalGeneratorImageResponses, type putConfigInternalGeneratorImageTiffData, type putConfigInternalGeneratorImageTiffError, type putConfigInternalGeneratorImageTiffErrors, type putConfigInternalGeneratorImageTiffResponses, type putConfigRoiData, type putConfigRoiError, type putConfigRoiErrors, type putConfigRoiResponses, type putConfigRugnuxData, type putConfigRugnuxError, type putConfigRugnuxErrors, type putConfigRugnuxModelData, type putConfigRugnuxModelError, type putConfigRugnuxModelErrors, type putConfigRugnuxModelResponses, type putConfigRugnuxResponses, type putConfigSelectDetectorData, type putConfigSelectDetectorError, type putConfigSelectDetectorErrors, type putConfigSelectDetectorResponses, type putConfigSpotFindingData, type putConfigSpotFindingError, type putConfigSpotFindingErrors, type putConfigSpotFindingResponses, type putConfigUserMaskData, type putConfigUserMaskError, type putConfigUserMaskErrors, type putConfigUserMaskResponses, type putConfigUserMaskTiffData, type putConfigUserMaskTiffError, type putConfigUserMaskTiffErrors, type putConfigUserMaskTiffResponses, type putConfigZeromqMetadataData, type putConfigZeromqMetadataError, type putConfigZeromqMetadataErrors, type putConfigZeromqMetadataResponses, type putConfigZeromqPreviewData, type putConfigZeromqPreviewError, type putConfigZeromqPreviewErrors, type putConfigZeromqPreviewResponses, type resolution_estimate, type resolution_ring, type roi, type roi_azim_list, type roi_azimuthal, type roi_box, type roi_box_list, type roi_circle, type roi_circle_list, type roi_definitions, type rotation_axis, type rugnux_live_status, type rugnux_live_step, type rugnux_settings, type rugnux_status, type rugnux_summary_row, type saturation, type scan_result, type show_beam_center, type show_predictions, type show_roi, type show_spots, type show_user_mask, type spot_finding_settings, type standard_detector_geometry, type tcp_settings, type unit_cell, type zeromq_metadata_settings, type zeromq_preview_settings, type zeromq_settings } from './types.gen'; diff --git a/frontend/src/client/sdk.gen.ts b/frontend/src/client/sdk.gen.ts index 56ad7f27d..9e0922bea 100644 --- a/frontend/src/client/sdk.gen.ts +++ b/frontend/src/client/sdk.gen.ts @@ -1,8 +1,8 @@ // This file is auto-generated by @hey-api/openapi-ts -import type { Client, ClientMeta, Options as Options2, RequestResult, TDataShape } from './client'; +import type { Client, ClientMeta, Options as Options2, RequestResult, ServerSentEventsResult, TDataShape } from './client'; import { client } from './client.gen'; -import type { getConfigAzimIntData, getConfigAzimIntResponses, getConfigBraggIntegrationData, getConfigBraggIntegrationResponses, getConfigDarkMaskData, getConfigDarkMaskResponses, getConfigDetectorData, getConfigDetectorResponses, getConfigFileWriterData, getConfigFileWriterResponses, getConfigImageFormatData, getConfigImageFormatResponses, getConfigIndexingData, getConfigIndexingResponses, getConfigInstrumentData, getConfigInstrumentResponses, getConfigMaskData, getConfigMaskResponses, getConfigMaskTiffData, getConfigMaskTiffResponses, getConfigRoiData, getConfigRoiResponses, getConfigSelectDetectorData, getConfigSelectDetectorResponses, getConfigSpotFindingData, getConfigSpotFindingResponses, getConfigUserMaskData, getConfigUserMaskResponses, getConfigUserMaskTiffData, getConfigUserMaskTiffResponses, getConfigZeromqMetadataData, getConfigZeromqMetadataResponses, getConfigZeromqPreviewData, getConfigZeromqPreviewResponses, getDetectorStatusData, getDetectorStatusErrors, getDetectorStatusResponses, getFpgaStatusData, getFpgaStatusResponses, getImageBufferImageCborData, getImageBufferImageCborErrors, getImageBufferImageCborResponses, getImageBufferImageJpegData, getImageBufferImageJpegErrors, getImageBufferImageJpegResponses, getImageBufferImageTiffData, getImageBufferImageTiffErrors, getImageBufferImageTiffResponses, getImageBufferStartCborData, getImageBufferStartCborErrors, getImageBufferStartCborResponses, getImageBufferStatusData, getImageBufferStatusErrors, getImageBufferStatusResponses, getImagePusherStatusData, getImagePusherStatusErrors, getImagePusherStatusResponses, getPreviewPedestalTiffData, getPreviewPedestalTiffErrors, getPreviewPedestalTiffResponses, getPreviewPlotBinData, getPreviewPlotBinErrors, getPreviewPlotBinResponses, getPreviewPlotData, getPreviewPlotErrors, getPreviewPlotResponses, getResultScanData, getResultScanErrors, getResultScanResponses, getStatisticsCalibrationData, getStatisticsCalibrationResponses, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionErrors, getStatisticsDataCollectionResponses, getStatisticsResponses, getStatusData, getStatusResponses, getVersionData, getVersionResponses, getXfelEventCodeData, getXfelEventCodeErrors, getXfelEventCodeResponses, getXfelPulseIdData, getXfelPulseIdErrors, getXfelPulseIdResponses, postCancelData, postCancelResponses, postConfigImageFormatConversionData, postConfigImageFormatConversionErrors, postConfigImageFormatConversionResponses, postConfigImageFormatRawData, postConfigImageFormatRawErrors, postConfigImageFormatRawResponses, postDeactivateData, postDeactivateErrors, postDeactivateResponses, postImageBufferClearData, postImageBufferClearErrors, postImageBufferClearResponses, postInitializeData, postInitializeErrors, postInitializeResponses, postPedestalData, postPedestalErrors, postPedestalResponses, postStartData, postStartErrors, postStartResponses, postTriggerData, postTriggerResponses, postWaitTillDoneData, postWaitTillDoneErrors, postWaitTillDoneResponses, postWaitUntilRunningData, postWaitUntilRunningErrors, postWaitUntilRunningResponses, putConfigAzimIntData, putConfigAzimIntErrors, putConfigAzimIntResponses, putConfigBraggIntegrationData, putConfigBraggIntegrationErrors, putConfigBraggIntegrationResponses, putConfigDarkMaskData, putConfigDarkMaskErrors, putConfigDarkMaskResponses, putConfigDetectorData, putConfigDetectorErrors, putConfigDetectorResponses, putConfigFileWriterData, putConfigFileWriterErrors, putConfigFileWriterResponses, putConfigImageFormatData, putConfigImageFormatErrors, putConfigImageFormatResponses, putConfigIndexingData, putConfigIndexingErrors, putConfigIndexingResponses, putConfigInstrumentData, putConfigInstrumentErrors, putConfigInstrumentResponses, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageErrors, putConfigInternalGeneratorImageResponses, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffErrors, putConfigInternalGeneratorImageTiffResponses, putConfigRoiData, putConfigRoiErrors, putConfigRoiResponses, putConfigSelectDetectorData, putConfigSelectDetectorErrors, putConfigSelectDetectorResponses, putConfigSpotFindingData, putConfigSpotFindingErrors, putConfigSpotFindingResponses, putConfigUserMaskData, putConfigUserMaskErrors, putConfigUserMaskResponses, putConfigUserMaskTiffData, putConfigUserMaskTiffErrors, putConfigUserMaskTiffResponses, putConfigZeromqMetadataData, putConfigZeromqMetadataErrors, putConfigZeromqMetadataResponses, putConfigZeromqPreviewData, putConfigZeromqPreviewErrors, putConfigZeromqPreviewResponses } from './types.gen'; +import type { deleteConfigRugnuxModelData, deleteConfigRugnuxModelErrors, deleteConfigRugnuxModelResponses, getConfigAzimIntData, getConfigAzimIntResponses, getConfigBraggIntegrationData, getConfigBraggIntegrationResponses, getConfigDarkMaskData, getConfigDarkMaskResponses, getConfigDetectorData, getConfigDetectorResponses, getConfigFileWriterData, getConfigFileWriterResponses, getConfigImageFormatData, getConfigImageFormatResponses, getConfigIndexingData, getConfigIndexingResponses, getConfigInstrumentData, getConfigInstrumentResponses, getConfigMaskData, getConfigMaskResponses, getConfigMaskTiffData, getConfigMaskTiffResponses, getConfigRoiData, getConfigRoiResponses, getConfigRugnuxData, getConfigRugnuxErrors, getConfigRugnuxResponses, getConfigSelectDetectorData, getConfigSelectDetectorResponses, getConfigSpotFindingData, getConfigSpotFindingResponses, getConfigUserMaskData, getConfigUserMaskResponses, getConfigUserMaskTiffData, getConfigUserMaskTiffResponses, getConfigZeromqMetadataData, getConfigZeromqMetadataResponses, getConfigZeromqPreviewData, getConfigZeromqPreviewResponses, getDetectorStatusData, getDetectorStatusErrors, getDetectorStatusResponses, getFpgaStatusData, getFpgaStatusResponses, getImageBufferImageCborData, getImageBufferImageCborErrors, getImageBufferImageCborResponses, getImageBufferImageJpegData, getImageBufferImageJpegErrors, getImageBufferImageJpegResponses, getImageBufferImageTiffData, getImageBufferImageTiffErrors, getImageBufferImageTiffResponses, getImageBufferStartCborData, getImageBufferStartCborErrors, getImageBufferStartCborResponses, getImageBufferStatusData, getImageBufferStatusErrors, getImageBufferStatusResponses, getImagePusherStatusData, getImagePusherStatusErrors, getImagePusherStatusResponses, getLiveEventsData, getLiveEventsErrors, getLiveEventsResponse, getLiveEventsResponses, getPreviewPedestalTiffData, getPreviewPedestalTiffErrors, getPreviewPedestalTiffResponses, getPreviewPlotBinData, getPreviewPlotBinErrors, getPreviewPlotBinResponses, getPreviewPlotData, getPreviewPlotErrors, getPreviewPlotResponses, getResultScanData, getResultScanErrors, getResultScanResponses, getRugnuxStatusData, getRugnuxStatusErrors, getRugnuxStatusResponses, getStatisticsCalibrationData, getStatisticsCalibrationResponses, getStatisticsData, getStatisticsDataCollectionData, getStatisticsDataCollectionErrors, getStatisticsDataCollectionResponses, getStatisticsResponses, getStatusData, getStatusResponses, getVersionData, getVersionResponses, getXfelEventCodeData, getXfelEventCodeErrors, getXfelEventCodeResponses, getXfelPulseIdData, getXfelPulseIdErrors, getXfelPulseIdResponses, postCancelData, postCancelResponses, postConfigImageFormatConversionData, postConfigImageFormatConversionErrors, postConfigImageFormatConversionResponses, postConfigImageFormatRawData, postConfigImageFormatRawErrors, postConfigImageFormatRawResponses, postDeactivateData, postDeactivateErrors, postDeactivateResponses, postImageBufferClearData, postImageBufferClearErrors, postImageBufferClearResponses, postInitializeData, postInitializeErrors, postInitializeResponses, postPedestalData, postPedestalErrors, postPedestalResponses, postStartData, postStartErrors, postStartResponses, postTriggerData, postTriggerResponses, postWaitTillDoneData, postWaitTillDoneErrors, postWaitTillDoneResponses, postWaitUntilRunningData, postWaitUntilRunningErrors, postWaitUntilRunningResponses, putConfigAzimIntData, putConfigAzimIntErrors, putConfigAzimIntResponses, putConfigBraggIntegrationData, putConfigBraggIntegrationErrors, putConfigBraggIntegrationResponses, putConfigDarkMaskData, putConfigDarkMaskErrors, putConfigDarkMaskResponses, putConfigDetectorData, putConfigDetectorErrors, putConfigDetectorResponses, putConfigFileWriterData, putConfigFileWriterErrors, putConfigFileWriterResponses, putConfigImageFormatData, putConfigImageFormatErrors, putConfigImageFormatResponses, putConfigIndexingData, putConfigIndexingErrors, putConfigIndexingResponses, putConfigInstrumentData, putConfigInstrumentErrors, putConfigInstrumentResponses, putConfigInternalGeneratorImageData, putConfigInternalGeneratorImageErrors, putConfigInternalGeneratorImageResponses, putConfigInternalGeneratorImageTiffData, putConfigInternalGeneratorImageTiffErrors, putConfigInternalGeneratorImageTiffResponses, putConfigRoiData, putConfigRoiErrors, putConfigRoiResponses, putConfigRugnuxData, putConfigRugnuxErrors, putConfigRugnuxModelData, putConfigRugnuxModelErrors, putConfigRugnuxModelResponses, putConfigRugnuxResponses, putConfigSelectDetectorData, putConfigSelectDetectorErrors, putConfigSelectDetectorResponses, putConfigSpotFindingData, putConfigSpotFindingErrors, putConfigSpotFindingResponses, putConfigUserMaskData, putConfigUserMaskErrors, putConfigUserMaskResponses, putConfigUserMaskTiffData, putConfigUserMaskTiffErrors, putConfigUserMaskTiffResponses, putConfigZeromqMetadataData, putConfigZeromqMetadataErrors, putConfigZeromqMetadataResponses, putConfigZeromqPreviewData, putConfigZeromqPreviewErrors, putConfigZeromqPreviewResponses } from './types.gen'; export type Options = Options2 & { /** @@ -103,6 +103,11 @@ export const postTrigger = (options?: Opti * Any frame that is currently being processed by CPU will be finished and sent to writer. * Given the command is making sure to gracefully stop data acquisition and detector, it might take some time to switch back after command finished to `Idle` state. * + * With rugnux live mode (`/config/rugnux`) the command also cuts the rugnux analysis, during the collection + * or after its last image alike: the collection ends with what the analysis has established + * (see `cut` in `/rugnux/status`). After the last image the collection itself is complete and is not + * marked as cancelled. + * * If data collection is not running, the command has no effect. * */ @@ -371,6 +376,76 @@ export const putConfigSelectDetector = (op } }); +/** + * Get rugnux live mode settings and model status + * + * Can be done anytime + */ +export const getConfigRugnux = (options?: Options): RequestResult => (options?.client ?? client).get({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/config/rugnux', + ...options +}); + +/** + * Change rugnux live mode settings + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * + */ +export const putConfigRugnux = (options?: Options): RequestResult => (options?.client ?? client).put({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/config/rugnux', + ...options, + headers: { + 'Content-Type': 'application/json', + ...options?.headers + } +}); + +/** + * Remove the model used by rugnux live mode + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * + */ +export const deleteConfigRugnuxModel = (options?: Options): RequestResult => (options?.client ?? client).delete({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/config/rugnux/model', + ...options +}); + +/** + * Upload the model used by rugnux live mode + * + * This can only be done when detector is `Idle`, `Error` or `Inactive` states. + * The model (PDB or mmCIF, optionally gzipped; at most 64 MiB) is used for molecular replacement + * and validation of every following rugnux live collection, until it is removed with + * `DELETE /config/rugnux/model` or replaced by another upload. + * + */ +export const putConfigRugnuxModel = (options?: Options): RequestResult => (options?.client ?? client).put({ + bodySerializer: null, + security: [{ scheme: 'bearer', type: 'http' }], + url: '/config/rugnux/model', + ...options, + headers: { + 'Content-Type': 'application/octet-stream', + ...options?.headers + } +}); + +/** + * Get live rugnux processing status of the last collection + * + * Can be done anytime + */ +export const getRugnuxStatus = (options?: Options): RequestResult => (options?.client ?? client).get({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/rugnux/status', + ...options +}); + /** * Get ZeroMQ preview settings */ @@ -648,6 +723,31 @@ export const getImageBufferStartCbor = (op ...options }); +/** + * Event stream of the collection, for a reader following it + * + * Server-sent events (text/event-stream) for the current collection, from the moment /start has + * accepted it: first its whole history, then every change, until it ends or a new collection + * begins. Only one reader at a time. Events, each with a JSON `data` line: + * - `start` {run_number, run_name, file_prefix, number_of_images, images_per_file, cbor} - first, + * once the start message is known: at once for a JUNGFRAU, when the detector sends its own for + * a DECTRIS. `cbor` is the start message the writer of the master file received, CBOR in base64; + * a reader builds the dataset from it; + * - `file` {file_number, file, first_image, image_count} - a data file closed and renamed into + * place; `file` is relative to the collection's root, as the writer named it; + * - `end` {images, error?} - every writer acknowledged the end: the data files and the master + * file are in place; `images` is the number of images collected. Also sent, with `error`, when + * the measurement ends any other way - before the start message, for one. The stream then closes; + * - `superseded` {run_number} - a new collection began. The stream then closes. + * A comment line (`: keepalive`) every 10 s when nothing happens. + * + */ +export const getLiveEvents = (options?: Options): Promise> => (options?.client ?? client).sse.get({ + security: [{ scheme: 'bearer', type: 'http' }], + url: '/live/events', + ...options +}); + /** * Get image message in CBOR format * diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 8338b2b5e..8b17d4515 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -1873,6 +1873,125 @@ export type jfjoch_settings = { zeromq_preview?: zeromq_preview_settings; zeromq_metadata?: zeromq_metadata_settings; dark_mask?: dark_mask_settings; + rugnux?: rugnux_settings; +}; + +/** + * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux + * inside the broker while they arrive, and the merged result is sent to the writer. + * + */ +export type rugnux_settings = { + /** + * Process DECTRIS rotation collections with rugnux live + */ + enable?: boolean; + /** + * CPU threads left to the receiver and the image pusher while rugnux runs + */ + reserved_cpus?: number; + /** + * Fixed cutoff: this long after the last image the analysis is cut, and the collection ends + * with what it has established (as `POST /cancel` after the last image does). 0 means no cutoff. + * + */ + deadline_s?: number; +}; + +export type rugnux_status = { + /** + * Rugnux live mode is enabled + */ + enable: boolean; + /** + * CPU threads left to the receiver and the image pusher while rugnux runs + */ + reserved_cpus: number; + /** + * Seconds after the last image at which the analysis is cut; 0 means no cutoff + */ + deadline_s?: number; + /** + * A model was uploaded with `PUT /config/rugnux/model` + */ + model_loaded: boolean; + /** + * Size of the uploaded model file + */ + model_size_bytes?: number; +}; + +export type rugnux_live_step = { + title: string; + /** + * Start of the step, since the collection started + */ + started_s: number; + /** + * Set once the next step starts or the run ends + */ + duration_s?: number; + /** + * What the step found (e.g. lattice, space group) + */ + notes: Array; +}; + +export type rugnux_summary_row = { + label: string; + value: string; +}; + +/** + * Live rugnux processing of the last collection; kept until the next collection starts + */ +export type rugnux_live_status = { + state: 'Idle' | 'Running' | 'Done' | 'Failed' | 'Cancelled' | 'Skipped'; + /** + * Answer of the early lattice probe on the first 60 degrees; empty until it answers + */ + probe: string; + /** + * When the probe answered, since the collection started + */ + probe_s?: number; + /** + * Why the run failed or was skipped + */ + message: string; + images_expected: number; + images_received: number; + elapsed_s: number; + /** + * Fraction of the current pass's images + */ + step_progress?: number; + steps: Array; + /** + * When the last image arrived (and the cutoff timer started), since the collection started + */ + last_image_s?: number; + /** + * Empty, or why the analysis was cut: `deadline` (`deadline_s` after the last image) or `cancel` + * (`POST /cancel`). The end message then carried what was established at the cut. + * + */ + cut?: string; + /** + * When the analysis was cut, since the collection started + */ + cut_s?: number; + /** + * From the last image received to all results written + */ + last_image_to_written_s?: number; + /** + * OK, WARNINGS, UNUSABLE or FAILED; empty until the run is over + */ + verdict: string; + verdict_text: string; + summary: Array; + warnings: Array; }; export type zeromq_metadata_settings = { @@ -3060,6 +3179,149 @@ export type putConfigSelectDetectorResponses = { 200: unknown; }; +export type getConfigRugnuxData = { + body?: never; + path?: never; + query?: never; + url: '/config/rugnux'; +}; + +export type getConfigRugnuxErrors = { + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; +}; + +export type getConfigRugnuxError = getConfigRugnuxErrors[keyof getConfigRugnuxErrors]; + +export type getConfigRugnuxResponses = { + /** + * Everything OK + */ + 200: rugnux_status; +}; + +export type getConfigRugnuxResponse = getConfigRugnuxResponses[keyof getConfigRugnuxResponses]; + +export type putConfigRugnuxData = { + body?: rugnux_settings; + path?: never; + query?: never; + url: '/config/rugnux'; +}; + +export type putConfigRugnuxErrors = { + /** + * Exception error + */ + 400: string; + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; + /** + * Error within Jungfraujoch code - see output message. + */ + 500: error_message; +}; + +export type putConfigRugnuxError = putConfigRugnuxErrors[keyof putConfigRugnuxErrors]; + +export type putConfigRugnuxResponses = { + /** + * Everything OK + */ + 200: unknown; +}; + +export type deleteConfigRugnuxModelData = { + body?: never; + path?: never; + query?: never; + url: '/config/rugnux/model'; +}; + +export type deleteConfigRugnuxModelErrors = { + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; + /** + * Error within Jungfraujoch code - see output message. + */ + 500: error_message; +}; + +export type deleteConfigRugnuxModelError = deleteConfigRugnuxModelErrors[keyof deleteConfigRugnuxModelErrors]; + +export type deleteConfigRugnuxModelResponses = { + /** + * All good + */ + 200: unknown; +}; + +export type putConfigRugnuxModelData = { + body?: Blob | File; + path?: never; + query?: never; + url: '/config/rugnux/model'; +}; + +export type putConfigRugnuxModelErrors = { + /** + * Empty body + */ + 400: unknown; + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; + /** + * Model larger than 64 MiB + */ + 413: unknown; + /** + * Error within Jungfraujoch code - see output message. + */ + 500: error_message; +}; + +export type putConfigRugnuxModelError = putConfigRugnuxModelErrors[keyof putConfigRugnuxModelErrors]; + +export type putConfigRugnuxModelResponses = { + /** + * All good + */ + 200: unknown; +}; + +export type getRugnuxStatusData = { + body?: never; + path?: never; + query?: never; + url: '/rugnux/status'; +}; + +export type getRugnuxStatusErrors = { + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; +}; + +export type getRugnuxStatusError = getRugnuxStatusErrors[keyof getRugnuxStatusErrors]; + +export type getRugnuxStatusResponses = { + /** + * Everything OK + */ + 200: rugnux_live_status; +}; + +export type getRugnuxStatusResponse = getRugnuxStatusResponses[keyof getRugnuxStatusResponses]; + export type getConfigZeromqPreviewData = { body?: never; path?: never; @@ -3738,6 +4000,39 @@ export type getImageBufferStartCborResponses = { export type getImageBufferStartCborResponse = getImageBufferStartCborResponses[keyof getImageBufferStartCborResponses]; +export type getLiveEventsData = { + body?: never; + path?: never; + query?: never; + url: '/live/events'; +}; + +export type getLiveEventsErrors = { + /** + * The current dataset is protected and no valid bearer token was presented. + */ + 401: string; + /** + * No collection has started + */ + 404: unknown; + /** + * Another reader is following the collection, or the image pusher does not report closed data files + */ + 409: unknown; +}; + +export type getLiveEventsError = getLiveEventsErrors[keyof getLiveEventsErrors]; + +export type getLiveEventsResponses = { + /** + * Event stream + */ + 200: string; +}; + +export type getLiveEventsResponse = getLiveEventsResponses[keyof getLiveEventsResponses]; + export type getImageBufferImageCborData = { body?: never; path?: never; diff --git a/frontend/src/client/zod.gen.ts b/frontend/src/client/zod.gen.ts index 2b962afbf..0b49a5242 100644 --- a/frontend/src/client/zod.gen.ts +++ b/frontend/src/client/zod.gen.ts @@ -792,6 +792,67 @@ export const zBraggIntegrationSettings = z.object({ max_hkl: z.int().gte(1).lte(511).optional().default(100) }); +/** + * Rugnux live mode: rotation collections from a DECTRIS detector are processed by rugnux + * inside the broker while they arrive, and the merged result is sent to the writer. + * + */ +export const zRugnuxSettings = z.object({ + enable: z.boolean().optional().default(false), + reserved_cpus: z.coerce.bigint().gte(BigInt(0)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(4)), + deadline_s: z.number().gte(0).optional().default(0) +}); + +export const zRugnuxStatus = z.object({ + enable: z.boolean(), + reserved_cpus: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }), + deadline_s: z.number().optional(), + model_loaded: z.boolean(), + model_size_bytes: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional() +}); + +export const zRugnuxLiveStep = z.object({ + title: z.string(), + started_s: z.number(), + duration_s: z.number().optional(), + notes: z.array(z.string()) +}); + +export const zRugnuxSummaryRow = z.object({ + label: z.string(), + value: z.string() +}); + +/** + * Live rugnux processing of the last collection; kept until the next collection starts + */ +export const zRugnuxLiveStatus = z.object({ + state: z.enum([ + 'Idle', + 'Running', + 'Done', + 'Failed', + 'Cancelled', + 'Skipped' + ]), + probe: z.string(), + probe_s: z.number().optional(), + message: z.string(), + images_expected: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }), + images_received: z.coerce.bigint().min(BigInt('-9223372036854775808'), { error: 'Invalid value: Expected int64 to be >= -9223372036854775808' }).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }), + elapsed_s: z.number(), + step_progress: z.number().gte(0).lte(1).optional(), + steps: z.array(zRugnuxLiveStep), + last_image_s: z.number().optional(), + cut: z.string().optional(), + cut_s: z.number().optional(), + last_image_to_written_s: z.number().optional(), + verdict: z.string(), + verdict_text: z.string(), + summary: z.array(zRugnuxSummaryRow), + warnings: z.array(z.string()) +}); + export const zZeromqMetadataSettings = z.object({ enabled: z.boolean().default(true), period_ms: z.coerce.bigint().gte(BigInt(1)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).default(BigInt(1000)), @@ -857,7 +918,8 @@ export const zJfjochSettings = z.object({ image_pusher: zImagePusherType, zeromq_preview: zZeromqPreviewSettings.optional(), zeromq_metadata: zZeromqMetadataSettings.optional(), - dark_mask: zDarkMaskSettings.optional() + dark_mask: zDarkMaskSettings.optional(), + rugnux: zRugnuxSettings.optional() }); export const zDatasetSettingsWritable = z.object({ @@ -1186,6 +1248,20 @@ export const zGetConfigSelectDetectorResponse = zDetectorList; export const zPutConfigSelectDetectorBody = zDetectorSelection; +/** + * Everything OK + */ +export const zGetConfigRugnuxResponse = zRugnuxStatus; + +export const zPutConfigRugnuxBody = zRugnuxSettings; + +export const zPutConfigRugnuxModelBody = z.string(); + +/** + * Everything OK + */ +export const zGetRugnuxStatusResponse = zRugnuxLiveStatus; + /** * Everything OK */ @@ -1410,6 +1486,11 @@ export const zGetResultScanResponse = zScanResult; */ export const zGetImageBufferStartCborResponse = z.string(); +/** + * Event stream + */ +export const zGetLiveEventsResponse = z.string(); + export const zGetImageBufferImageCborQuery = z.object({ id: z.coerce.bigint().gte(BigInt(-2)).max(BigInt('9223372036854775807'), { error: 'Invalid value: Expected int64 to be <= 9223372036854775807' }).optional().default(BigInt(-1)) }); diff --git a/frontend/src/components/RugnuxPage.tsx b/frontend/src/components/RugnuxPage.tsx new file mode 100644 index 000000000..6dc22e4ac --- /dev/null +++ b/frontend/src/components/RugnuxPage.tsx @@ -0,0 +1,166 @@ +import {Alert, Box, Chip, LinearProgress, Paper, Stack, Typography} from "@mui/material"; +import {indigo, lime, deepOrange, grey} from "@mui/material/colors"; +import {useQuery} from "@tanstack/react-query"; +import {getConfigRugnuxOptions, getRugnuxStatusOptions} from "../client/@tanstack/react-query.gen"; +import {rugnux_live_status} from "../client"; +import {useTokenRequired} from "../auth"; +import EmptyState from "./EmptyState"; +import RugnuxTimeline from "./RugnuxTimeline"; +import RugnuxSettings from "./RugnuxSettings"; +import {CutAlert, lastImageTime, ProbeAlert, StepTable, VerdictAlert} from "./RugnuxParts"; +import BoltIcon from "@mui/icons-material/Bolt"; + +const STATE_COLOUR: Record = { + Running: {bg: indigo[500], fg: "#fff"}, + Done: {bg: lime[600], fg: "rgba(0,0,0,.87)"}, + Failed: {bg: deepOrange[700], fg: "#fff"}, + Cancelled: {bg: grey[500], fg: "#fff"}, + Skipped: {bg: grey[400], fg: "rgba(0,0,0,.87)"}, +}; + +// The summary rows rugnux writes; the ones a crystallographer looks at first get a stronger tile. +const KEY_ROW = /cell|space group|resolution|cc1\/2|r-?free|model/i; + +function Section({title, children}: { title: string, children: React.ReactNode }) { + return ( + + + {title} + + {children} + + ); +} + +function Header({s}: { s: rugnux_live_status }) { + const colour = STATE_COLOUR[s.state] ?? STATE_COLOUR.Skipped; + const latency = s.last_image_to_written_s; + const lastImage = lastImageTime(s); + return ( + + + + Rugnux live + + + + {s.images_received} / {s.images_expected} images + {lastImage !== undefined && ` · collected in ${lastImage.toFixed(1)} s`} + {` · ${s.elapsed_s.toFixed(1)} s since the first image`} + + {s.message && {s.message}} + + + + {latency !== undefined ? "Last image → all results written" : "Processing for"} + + + {(latency ?? s.elapsed_s).toFixed(1)} s + + + + ); +} + +function CurrentStep({s}: { s: rugnux_live_status }) { + const current = s.state === 'Running' && s.steps.length > 0 ? s.steps[s.steps.length - 1] : undefined; + if (!current) + return null; + return ( + + + {current.title} + {s.step_progress !== undefined && + {(s.step_progress * 100).toFixed(0)} %} + + + + ); +} + +function SummaryTiles({s}: { s: rugnux_live_status }) { + if (s.summary.length === 0) + return null; + return ( + + {s.summary.map((row, i) => { + const key = KEY_ROW.test(row.label) || KEY_ROW.test(row.value); + const short = row.value.length <= 24; + return ( + + + {row.label} + + + {row.value} + + + ); + })} + + ); +} + +function RugnuxPage({measuring}: { measuring: boolean }) { + const {data: s, isError} = useQuery({ + ...getRugnuxStatusOptions(), + refetchInterval: 1000, + }); + const {data: settings} = useQuery({...getConfigRugnuxOptions(), refetchInterval: 2000}); + const tokenRequired = useTokenRequired(); + + let live; + if (isError && tokenRequired) + live = } + text="Dataset token required (key button in the top bar)"/>; + else if (isError || s === undefined || s.state === 'Idle') + live = } + text="No rugnux run yet - start a rotation collection with live processing enabled"/>; + else + live = ( + +
+ + +
+ +
+ +
+ + +
+
+ {s.verdict + ? + : + {s.state === 'Running' ? "Results appear when the run is over." : "No result."} + } + + 0 ? 2 : 0}}> + {s.warnings.map((w, i) => {w})} + +
+ + + ); + + return ( + + {live} + + + ); +} + +export default RugnuxPage; diff --git a/frontend/src/components/RugnuxParts.tsx b/frontend/src/components/RugnuxParts.tsx new file mode 100644 index 000000000..45d37a6a7 --- /dev/null +++ b/frontend/src/components/RugnuxParts.tsx @@ -0,0 +1,88 @@ +import {Alert, AlertTitle, Box, Paper, Table, TableBody, TableCell, TableContainer, TableRow, Typography} from "@mui/material"; +import {rugnux_live_status, rugnux_live_step} from "../client"; + +// Pieces of the rugnux live status shared by the dashboard card and the Rugnux page. + +export const verdictSeverity = (verdict: string) => { + switch (verdict) { + case "OK": + return "success"; + case "WARNINGS": + return "warning"; + default: + return "error"; + } +}; + +// When the last image arrived, in seconds since the collection started. Older brokers do not report +// it; there it follows from the latency (every result written `latency` after the last image). +export const lastImageTime = (s: rugnux_live_status) => + s.last_image_s ?? (s.last_image_to_written_s !== undefined ? s.elapsed_s - s.last_image_to_written_s : undefined); + +export function CutAlert({s}: { s: rugnux_live_status }) { + if (!s.cut) + return null; + return ( + + Analysis cut ({s.cut}){s.cut_s !== undefined && ` at ${s.cut_s.toFixed(1)} s`} - results as established + + ); +} + +export function ProbeAlert({s, large}: { s: rugnux_live_status, large?: boolean }) { + if (!s.probe) + return null; + const noLattice = s.probe.startsWith("no lattice"); + return ( + + {large && + Lattice probe{s.probe_s !== undefined && ` · ${s.probe_s.toFixed(1)} s after the first image`} + } + {s.probe}{!large && s.probe_s !== undefined && ` (at ${s.probe_s.toFixed(1)} s)`} + + ); +} + +export function VerdictAlert({s}: { s: rugnux_live_status }) { + if (!s.verdict) + return null; + return ( + + {s.verdict} + {s.verdict_text} + + ); +} + +const stepDuration = (step: rugnux_live_step) => + step.duration_s !== undefined ? step.duration_s.toFixed(1) + " s" : "running"; + +export function StepTable({steps}: { steps: rugnux_live_step[] }) { + if (steps.length === 0) + return null; + return ( + + + + {steps.map((step, i) => ( + + + {step.started_s.toFixed(1)} s + + + {step.title} + {step.notes.map((note, j) => + {note})} + + + {stepDuration(step)} + + + ))} + +
+ + ); +} diff --git a/frontend/src/components/RugnuxSettings.tsx b/frontend/src/components/RugnuxSettings.tsx new file mode 100644 index 000000000..8fd76bbca --- /dev/null +++ b/frontend/src/components/RugnuxSettings.tsx @@ -0,0 +1,94 @@ +import {ChangeEvent, useEffect, useState} from 'react'; + +import {Button, FormControlLabel, Stack, Switch, Typography} from "@mui/material"; +import UploadFileIcon from "@mui/icons-material/UploadFile"; +import DeleteIcon from "@mui/icons-material/Delete"; +import {useQuery, useQueryClient} from "@tanstack/react-query"; +import { + deleteConfigRugnuxModelMutation, getConfigRugnuxOptions, getConfigRugnuxQueryKey, + putConfigRugnuxModelMutation, putConfigRugnuxMutation +} from "../client/@tanstack/react-query.gen"; +import NumberTextField from "./NumberTextField"; +import SettingsPanel, {UploadSnackbarView} from "./SettingsPanel"; +import {useUpload} from "./useUpload"; + +const sizeText = (bytes?: number) => + bytes === undefined ? "" : bytes >= 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)} MiB` : `${(bytes / 1024).toFixed(0)} KiB`; + +// Rugnux live mode (enable, reserved CPUs) and the model used for MR, as two panels side by side. +function RugnuxSettings({measuring}: { measuring: boolean }) { + const queryClient = useQueryClient(); + const {data: s} = useQuery({...getConfigRugnuxOptions(), refetchInterval: 2000}); + const refresh = {onSuccess: () => queryClient.invalidateQueries({queryKey: getConfigRugnuxQueryKey()})}; + + const settings = useUpload({...putConfigRugnuxMutation(), ...refresh}); + const upload = useUpload({...putConfigRugnuxModelMutation(), ...refresh}); + const remove = useUpload({...deleteConfigRugnuxModelMutation(), ...refresh}); + + const [enable, setEnable] = useState(false); + const [cpus, setCpus] = useState(4); + const [cpusError, setCpusError] = useState(false); + const [deadline, setDeadline] = useState(0); + const [deadlineError, setDeadlineError] = useState(false); + const [counter, setCounter] = useState(0); + const [file, setFile] = useState(); + + useEffect(() => { + if (s !== undefined) { + setEnable(s.enable); + setCpus(s.reserved_cpus); + setDeadline(s.deadline_s ?? 0); + setCounter(c => c + 1); + } + }, [s?.enable, s?.reserved_cpus, s?.deadline_s]); + + const dirty = s !== undefined && (enable !== s.enable || cpus !== s.reserved_cpus || deadline !== (s.deadline_s ?? 0)); + + return ( + + settings.submit({enable, reserved_cpus: cpus, deadline_s: deadline})} + uploadDisabled={measuring || cpusError || deadlineError || settings.pending} + snackbar={settings.snackbar}> + setEnable(e.target.checked)}/>}/> + { setCpus(v); setCpusError(err); }}/> + { setDeadline(v); setDeadlineError(err); }}/> + + + + + {s?.model_loaded ? `Model loaded · ${sizeText(s.model_size_bytes)}` : "No model loaded"} + + + PDB or mmCIF, optionally gzipped, up to 64 MiB. Kept until removed. + + + + + + + + + + + ); +} + +export default RugnuxSettings; diff --git a/frontend/src/components/RugnuxStatus.tsx b/frontend/src/components/RugnuxStatus.tsx new file mode 100644 index 000000000..e54a3f4ab --- /dev/null +++ b/frontend/src/components/RugnuxStatus.tsx @@ -0,0 +1,62 @@ +import {memo} from 'react'; + +import {Alert, Box, LinearProgress, Typography} from "@mui/material"; +import {useQuery} from "@tanstack/react-query"; +import {getRugnuxStatusOptions} from "../client/@tanstack/react-query.gen"; +import {useTokenRequired} from "../auth"; +import SettingsPanel from "./SettingsPanel"; +import {CutAlert, ProbeAlert, StepTable, VerdictAlert} from "./RugnuxParts"; + +// Live rugnux processing of the last collection, as a dashboard card: the step it is on, the steps +// behind it with what they found, and once it is over the verdict. Nothing while rugnux has not run. +function RugnuxStatus() { + const {data: s, isError} = useQuery({ + ...getRugnuxStatusOptions(), + refetchInterval: 1000, + }); + const tokenRequired = useTokenRequired(); + + if (isError && tokenRequired) + return ( + + Dataset token required (key button in the top bar) + + ); + if (isError || s === undefined || s.state === 'Idle') + return null; + + const current = s.state === 'Running' && s.steps.length > 0 ? s.steps[s.steps.length - 1] : undefined; + + return ( + + + {s.state} · {s.images_received} / {s.images_expected} images · {s.elapsed_s.toFixed(0)} s + + {s.message && {s.message}} + + + + {current && ( + + {current.title} + + + )} + + {s.last_image_to_written_s !== undefined && ( + + Last image → all written: {s.last_image_to_written_s.toFixed(1)} s + + )} + + + + {s.warnings.map((w, i) => ( + {w} + ))} + + ); +} + +export default memo(RugnuxStatus); diff --git a/frontend/src/components/RugnuxTimeline.tsx b/frontend/src/components/RugnuxTimeline.tsx new file mode 100644 index 000000000..2eccfc1a1 --- /dev/null +++ b/frontend/src/components/RugnuxTimeline.tsx @@ -0,0 +1,174 @@ +import {Box, Tooltip, Typography} from "@mui/material"; +import {keyframes} from "@mui/system"; +import {indigo, lime, lightBlue, orange, grey, deepOrange} from "@mui/material/colors"; +import {rugnux_live_status} from "../client"; +import {lastImageTime} from "./RugnuxParts"; + +// The overlap of collection and processing on one time axis (seconds since the first image): the +// detector lane fills while images arrive, the rugnux lane shows its steps where they ran, and the +// gap between "last image" and "all written" is the headline latency. + +const stripes = keyframes` + from { background-position: 0 0; } + to { background-position: 40px 0; } +`; + +const runningSx = { + backgroundImage: "linear-gradient(135deg, rgba(255,255,255,.28) 25%, transparent 25%, transparent 50%, " + + "rgba(255,255,255,.28) 50%, rgba(255,255,255,.28) 75%, transparent 75%, transparent)", + backgroundSize: "40px 40px", + animation: `${stripes} 1s linear infinite`, +}; + +const STEP_COLOURS = [indigo[500], indigo[300], indigo[700], lightBlue[700], indigo[400], lightBlue[900]]; + +const LABEL_WIDTH = 120; +const LANE_HEIGHT = 64; +const AXIS_HEIGHT = 28; + +function niceStep(span: number) { + for (const s of [1, 2, 5, 10, 20, 30, 60, 120]) + if (span / s <= 12) + return s; + return 300; +} + +function Marker({x, colour, label, top, dashed}: { x: number, colour: string, label: string, top: number, dashed?: boolean }) { + return ( + + + + {label} + + + ); +} + +// deadline_s: the configured cutoff after the last image, 0 = none +function RugnuxTimeline({s, deadline_s}: { s: rugnux_live_status, deadline_s: number }) { + const lastImage = lastImageTime(s); + const collecting = lastImage === undefined && s.state === 'Running'; + const collectionEnd = lastImage ?? s.elapsed_s; + const rate = collectionEnd > 0 ? s.images_received / collectionEnd : 0; + const expectedEnd = rate > 0 ? s.images_expected / rate : 0; + + const deadline = lastImage !== undefined && deadline_s > 0 ? lastImage + deadline_s : undefined; + const span = Math.max(s.elapsed_s, expectedEnd, deadline ?? 0, 10) * 1.06; + const pct = (t: number) => 100 * Math.min(Math.max(t, 0), span) / span; + const tick = niceStep(span); + const ticks = Array.from({length: Math.floor(span / tick) + 1}, (_, i) => i * tick); + + const done = s.state !== 'Running'; + const latency = s.last_image_to_written_s; + + return ( + + + {["Detector", "Rugnux"].map((l, i) => ( + + {l} + + ))} + + + + {/* grid lines */} + {ticks.map(t => ( + + ))} + + {/* detector lane */} + + + + {s.images_received} / {s.images_expected} images{rate > 0 && ` · ${rate.toFixed(0)} Hz`} + + + {collecting && expectedEnd > collectionEnd && ( + + )} + + + {/* rugnux lane */} + + {s.steps.map((step, i) => { + const open = step.duration_s === undefined; + const end = open ? s.elapsed_s : step.started_s + step.duration_s!; + const width = Math.max(pct(end) - pct(step.started_s), 0.4); + return ( + + {step.title} · {open ? "running" : `${step.duration_s!.toFixed(1)} s`} + {step.notes.map((n, j) =>
{n}
)} + }> + + + {step.title} + + + + ); + })} + + + {/* axis */} + + {ticks.map(t => ( + + {t} s + + ))} + + + {/* markers */} + {s.probe_s !== undefined && ( + + )} + {lastImage !== undefined && ( + + )} + {deadline !== undefined && s.cut !== 'deadline' && ( + + )} + {s.cut && s.cut_s !== undefined && ( + + )} + {!done && } + + {/* latency bracket */} + {latency !== undefined && lastImage !== undefined && ( + + + +{latency.toFixed(1)} s → all written + + + )} + + + ); +} + +export default RugnuxTimeline; diff --git a/reader/CMakeLists.txt b/reader/CMakeLists.txt index de234d23f..0d6350514 100644 --- a/reader/CMakeLists.txt +++ b/reader/CMakeLists.txt @@ -19,6 +19,8 @@ ADD_LIBRARY(JFJochReader STATIC HDF5ImageLocator.cpp JFJochStartMessageReader.cpp JFJochStartMessageReader.h + JFJochImageBufferReader.cpp + JFJochImageBufferReader.h HDF5ImageLocator.h HDF5ImageSource.cpp HDF5ImageSource.h diff --git a/reader/JFJochImageBufferReader.cpp b/reader/JFJochImageBufferReader.cpp new file mode 100644 index 000000000..a7ca4e2cc --- /dev/null +++ b/reader/JFJochImageBufferReader.cpp @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochImageBufferReader.h" + +#include +#include + +#include "JFJochStartMessageReader.h" +#include "../common/JFJochException.h" +#include "../frame_serialize/CBORStream2Deserializer.h" + +JFJochImageBufferReader::JFJochImageBufferReader(ImageBuffer &buffer) : buffer_(buffer) {} + +void JFJochImageBufferReader::Open(const StartMessage &start) { + number_of_images_ = start.number_of_images; + ended_ = false; + SetStartMessage(DatasetFromStartMessage(start, default_experiment)); +} + +void JFJochImageBufferReader::Ended() { + ended_ = true; +} + +uint64_t JFJochImageBufferReader::GetNumberOfImages() const { + return number_of_images_; +} + +void JFJochImageBufferReader::Close() { + SetStartMessage({}); +} + +bool JFJochImageBufferReader::WaitForMessage(int64_t image_number, std::vector &cbor) { + if (image_number < 0 || static_cast(image_number) >= number_of_images_) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Image out of bounds"); + // The buffer has no notification of its own; at 100 Hz a millisecond's poll costs nothing. + while (!buffer_.GetImage(cbor, image_number)) { + if (ended_) + return buffer_.GetImage(cbor, image_number); // one that came in just before the end + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +bool JFJochImageBufferReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &ret) { + if (!WaitForMessage(image_number, ret.read_buffer)) + return false; + const auto msg = CBORStream2Deserialize(ret.read_buffer); + if (msg->msg_type != CBORImageType::IMAGE || !msg->data_message) + return false; + // Copied out of the message: read_buffer is scratch a caller may hand to the next read. + const CompressedImage &image = msg->data_message->image; + ret.image_buffer.resize(image.GetCompressedSize()); + memcpy(ret.image_buffer.data(), image.GetCompressed(), image.GetCompressedSize()); + ret.image = CompressedImage(ret.image_buffer.data(), ret.image_buffer.size(), image.GetWidth(), + image.GetHeight(), image.GetMode(), image.GetCompressionAlgorithm()); + return true; +} + +bool JFJochImageBufferReader::LoadImage_i(std::shared_ptr &dataset, DataMessage &message, + std::vector &buffer, int64_t image_number, bool) { + if (!dataset || !WaitForMessage(image_number, buffer)) + return false; + const auto msg = CBORStream2Deserialize(buffer); + if (msg->msg_type != CBORImageType::IMAGE || !msg->data_message) + return false; + message = *msg->data_message; + return true; +} diff --git a/reader/JFJochImageBufferReader.h b/reader/JFJochImageBufferReader.h new file mode 100644 index 000000000..5a7c5203d --- /dev/null +++ b/reader/JFJochImageBufferReader.h @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +#include "JFJochReader.h" +#include "../common/ImageBuffer.h" + +// Reads a collection inside the broker while it arrives: the dataset comes from its start message, +// the images from the broker's image buffer, where the receiver puts each one as it forwards it. A +// read of an image not there yet waits for it; after Ended it returns false for an image that never +// came. The buffer must hold the whole collection (slots >= images) - a slot taken again for a later +// image would hand out that image instead. +class JFJochImageBufferReader : public JFJochReader { +public: + explicit JFJochImageBufferReader(ImageBuffer &buffer); + + void Open(const StartMessage &start); + // No image comes after this: the collection is over, or abandoned. + void Ended(); + + [[nodiscard]] uint64_t GetNumberOfImages() const override; + void Close() override; + bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override; + std::vector ReadSpots(int64_t image) const override { return {}; } + +private: + bool LoadImage_i(std::shared_ptr &dataset, DataMessage &message, + std::vector &buffer, int64_t image_number, bool update_dataset) override; + // The image's CBOR message, as the receiver put it in the buffer; false when it never came. + bool WaitForMessage(int64_t image_number, std::vector &cbor); + + ImageBuffer &buffer_; + uint64_t number_of_images_ = 0; + std::atomic ended_ = false; +}; diff --git a/reader/JFJochStartMessageReader.cpp b/reader/JFJochStartMessageReader.cpp index eb8f768f4..f0c5fc14a 100644 --- a/reader/JFJochStartMessageReader.cpp +++ b/reader/JFJochStartMessageReader.cpp @@ -15,78 +15,77 @@ namespace { const std::string prefix = std::filesystem::path(start.file_prefix).filename().string(); return fmt::format("{:s}_data_{:06d}.h5", prefix, file_number + 1); } +} - // The dataset as the start message describes it - the same quantities an HDF5 master states, taken - // from the message the writer wrote it from. - std::shared_ptr Dataset(const StartMessage &start, const DiffractionExperiment &defaults) { - auto dataset = std::make_shared(); - dataset->arm_date = start.arm_date; - dataset->jfjoch_release = start.jfjoch_release; - dataset->error_value = start.error_value; - dataset->file_detect_ice_rings = start.detect_ice_rings; +std::shared_ptr DatasetFromStartMessage(const StartMessage &start, + const DiffractionExperiment &defaults) { + auto dataset = std::make_shared(); + dataset->arm_date = start.arm_date; + dataset->jfjoch_release = start.jfjoch_release; + dataset->error_value = start.error_value; + dataset->file_detect_ice_rings = start.detect_ice_rings; - auto &x = dataset->experiment; - x = defaults; - // The reader hands every image out as signed 32-bit, as JFJochHDF5Reader does. - x.BitDepthImage(32); - x.PixelSigned(true); - x.FilePrefix(std::filesystem::path(start.file_prefix).filename().string()); - x.BeamX_pxl(start.beam_center_x); - x.BeamY_pxl(start.beam_center_y); - x.DetectorDistance_mm(start.detector_distance * 1000.0f); - x.PoniRot1_rad(start.poni_rot1.value_or(0.0f)); - x.PoniRot2_rad(start.poni_rot2.value_or(0.0f)); - x.PoniRot3_rad(start.poni_rot3.value_or(0.0f)); - x.IncidentEnergy_keV(WVL_1A_IN_KEV / start.incident_wavelength); - if (start.incident_wavelength_spread) - x.BandwidthFWHM(start.incident_wavelength_spread.value() / start.incident_wavelength); - x.DetectIceRings(start.detect_ice_rings.value_or(false)); + auto &x = dataset->experiment; + x = defaults; + // The reader hands every image out as signed 32-bit, as JFJochHDF5Reader does. + x.BitDepthImage(32); + x.PixelSigned(true); + x.FilePrefix(std::filesystem::path(start.file_prefix).filename().string()); + x.BeamX_pxl(start.beam_center_x); + x.BeamY_pxl(start.beam_center_y); + x.DetectorDistance_mm(start.detector_distance * 1000.0f); + x.PoniRot1_rad(start.poni_rot1.value_or(0.0f)); + x.PoniRot2_rad(start.poni_rot2.value_or(0.0f)); + x.PoniRot3_rad(start.poni_rot3.value_or(0.0f)); + x.IncidentEnergy_keV(WVL_1A_IN_KEV / start.incident_wavelength); + if (start.incident_wavelength_spread) + x.BandwidthFWHM(start.incident_wavelength_spread.value() / start.incident_wavelength); + x.DetectIceRings(start.detect_ice_rings.value_or(false)); - DetectorSetup detector = DetDECTRIS(start.image_size_x, start.image_size_y, start.detector_description, {}); - detector.PixelSize_um(start.pixel_size_x * 1e6f); - detector.MirrorY(start.mirror_y); - detector.ImageOrientation(DetectorOrientation(start.detector_orientation_mirror_y, - start.detector_orientation_quarter_turns)); - detector.SensorThickness_um(start.sensor_thickness * 1e6f); - detector.SensorMaterial(start.sensor_material); - detector.SaturationLimit(SaturationLimitFromValue(start.saturation_value)); - detector.BitDepthImage(32); - detector.MinFrameTime(std::chrono::microseconds(0)); - detector.MinCountTime(std::chrono::microseconds(0)); - detector.ReadOutTime(std::chrono::nanoseconds(0)); - x.Detector(detector); + DetectorSetup detector = DetDECTRIS(start.image_size_x, start.image_size_y, start.detector_description, {}); + detector.PixelSize_um(start.pixel_size_x * 1e6f); + detector.MirrorY(start.mirror_y); + detector.ImageOrientation(DetectorOrientation(start.detector_orientation_mirror_y, + start.detector_orientation_quarter_turns)); + detector.SensorThickness_um(start.sensor_thickness * 1e6f); + detector.SensorMaterial(start.sensor_material); + detector.SaturationLimit(SaturationLimitFromValue(start.saturation_value)); + detector.BitDepthImage(32); + detector.MinFrameTime(std::chrono::microseconds(0)); + detector.MinCountTime(std::chrono::microseconds(0)); + detector.ReadOutTime(std::chrono::nanoseconds(0)); + x.Detector(detector); - x.FrameTime(std::chrono::duration_cast(std::chrono::duration(start.frame_time)), - std::chrono::duration_cast(std::chrono::duration(start.count_time))); + x.FrameTime(std::chrono::duration_cast(std::chrono::duration(start.frame_time)), + std::chrono::duration_cast(std::chrono::duration(start.count_time))); - x.Goniometer(start.goniometer); - x.GridScan(start.grid_scan); - x.Smargon(start.smargon_position); - x.NumTriggers(1); - x.ImagesPerTrigger(static_cast(start.number_of_images)); + x.Goniometer(start.goniometer); + x.GridScan(start.grid_scan); + x.Smargon(start.smargon_position); + x.NumTriggers(1); + x.ImagesPerTrigger(static_cast(start.number_of_images)); - InstrumentMetadata instrument; - instrument.SourceName(start.source_name); - instrument.SourceType(start.source_type); - instrument.InstrumentName(start.instrument_name); - x.ImportInstrumentMetadata(instrument); - x.SampleName(start.sample_name); - x.SampleTemperature_K(start.sample_temperature_K); - x.RingCurrent_mA(start.ring_current_mA); - x.TotalFlux(start.total_flux); - x.AttenuatorTransmission(start.attenuator_transmission); - if (start.beam_size_x && start.beam_size_y) { - x.BeamSizeX_um(start.beam_size_x.value() * 1e6f); - x.BeamSizeY_um(start.beam_size_y.value() * 1e6f); - } - - if (!start.pixel_mask.empty()) - dataset->pixel_mask = std::make_shared(start.pixel_mask.begin()->second); - else - dataset->pixel_mask = std::make_shared( - std::vector(start.image_size_x * start.image_size_y)); - return dataset; + InstrumentMetadata instrument; + instrument.SourceName(start.source_name); + instrument.SourceType(start.source_type); + instrument.InstrumentName(start.instrument_name); + x.ImportInstrumentMetadata(instrument); + x.SampleName(start.sample_name); + x.SampleTemperature_K(start.sample_temperature_K); + x.RingCurrent_mA(start.ring_current_mA); + x.TotalFlux(start.total_flux); + x.AttenuatorTransmission(start.attenuator_transmission); + if (start.beam_size_x && start.beam_size_y) { + x.BeamSizeX_um(start.beam_size_x.value() * 1e6f); + x.BeamSizeY_um(start.beam_size_y.value() * 1e6f); } + + if (!start.pixel_mask.empty()) + dataset->pixel_mask = std::make_shared(start.pixel_mask.begin()->second); + else + dataset->pixel_mask = std::make_shared( + std::vector(start.image_size_x * start.image_size_y)); + return dataset; } void JFJochStartMessageReader::Open(const StartMessage &start, const std::string &master_path) { @@ -114,7 +113,7 @@ void JFJochStartMessageReader::Open(const StartMessage &start, const std::string ended_ = false; lost_.clear(); } - SetStartMessage(Dataset(start, default_experiment)); + SetStartMessage(DatasetFromStartMessage(start, default_experiment)); } void JFJochStartMessageReader::FileClosed(uint64_t file_number, uint64_t image_count) { diff --git a/reader/JFJochStartMessageReader.h b/reader/JFJochStartMessageReader.h index 75d943f69..064c6d58b 100644 --- a/reader/JFJochStartMessageReader.h +++ b/reader/JFJochStartMessageReader.h @@ -19,6 +19,11 @@ public: explicit LiveCollectionLost(const std::string &what) : std::runtime_error(what) {} }; +// The dataset as the start message describes it - the same quantities an HDF5 master states, taken +// from the message the writer wrote it from. +std::shared_ptr DatasetFromStartMessage(const StartMessage &start, + const DiffractionExperiment &defaults); + // Reads a collection while it is being written: the dataset comes from its start message, as the // broker sends it, and the images straight from the data files - /entry/data/data in each - once the // broker reports a file closed and renamed into place (FileClosed). A read of an image whose file has diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index 228f49597..1cfd1aa69 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -11,7 +11,9 @@ ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverCurrentStatus.h JFJochReceiverOutput.h JFJochReceiverLite.h JFJochReceiverLite.cpp + JFJochReceiverRugnux.h JFJochReceiverRugnux.cpp + RugnuxLiveStatus.h RugnuxLiveStatus.cpp JFJochReceiver.cpp JFJochReceiver.h) -TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImagePuller JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview) +TARGET_LINK_LIBRARIES(JFJochReceiver ImagePusher JFJochImagePuller JFJochImageAnalysis JFJochAcquisitionDevice JFJochCommon JFJochHLSSimulation JFJochPreview JFJochRugnux) diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index abb36ea42..42910ecb1 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -197,6 +197,7 @@ void JFJochReceiver::SendEndMessage() { message.adu_histogram["module" + std::to_string(i)] = adu_histogram_module[i]->GetHistogram(); scan_result.FillEndMessage(message); + AddToEndMessage(message); if (push_images_to_writer) { if (!image_pusher.EndDataCollection(message)) diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index d65f6235c..93fb213ae 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -96,6 +96,9 @@ protected: void SendStartMessage(); void SendEndMessage(); void SaveStartMessageToImageBuffer(const StartMessage &msg); + // The last word on the end message before it is sent: a receiver whose analysis is not the + // per-image one fills in what it found. + virtual void AddToEndMessage(EndMessage &message) {} virtual void Cancel(const JFJochException &e); public: JFJochReceiver(const DiffractionExperiment& experiment, diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp new file mode 100644 index 000000000..b876253f2 --- /dev/null +++ b/receiver/JFJochReceiverRugnux.cpp @@ -0,0 +1,482 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochReceiverRugnux.h" + +#include +#include +#include +#include +#include + +#include "spdlog/fmt/fmt.h" + +#include "JFJochReceiverLite.h" +#include "../common/ParallelFor.h" +#include "../frame_serialize/CBORStream2Deserializer.h" +#include "../rugnux/RugnuxLive.h" +#include "../rugnux/ResultReport.h" + +using namespace std::chrono_literals; + +namespace { + int64_t SteadyNowNs() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); + } + + double SecondsSince(int64_t ns) { + return static_cast(SteadyNowNs() - ns) / 1e9; + } + + // A run reachable for Cut while it exists: registered in `slot` (under `m`, cancelled at once where the + // cut came first) and taken off again before it is destroyed, an exception included. + class Reachable { + std::mutex &m; + Rugnux *&slot; + public: + Reachable(std::mutex &m, Rugnux *&slot, Rugnux &run, const bool &cut, const std::atomic &cancelled) + : m(m), slot(slot) { + std::unique_lock ul(m); + slot = &run; + if (cut || cancelled) + run.Cancel(); + } + ~Reachable() { + std::unique_lock ul(m); + slot = nullptr; + } + }; + + // What rugnux reports as it goes: its per-image results into the receiver's plots and scan + // result, which a later pass over the same image overwrites - so the canonical pass, the last to + // read every image, is what stays - and its steps into the log. + class LiveObserver : public RugnuxObserver { + JFJochReceiverPlots &plots; + ScanResultGenerator &scan_result; + const AzimuthalIntegrationMapping &mapping; + RugnuxLiveStatus &status; + Logger &logger; + public: + LiveObserver(JFJochReceiverPlots &plots, ScanResultGenerator &scan_result, + const AzimuthalIntegrationMapping &mapping, RugnuxLiveStatus &status, Logger &logger) + : plots(plots), scan_result(scan_result), mapping(mapping), status(status), logger(logger) {} + + void OnImageProcessed(const DataMessage &msg) override { + plots.Add(msg, AzimuthalIntegrationProfile(mapping)); + scan_result.Add(msg); + } + void OnProgress(uint64_t done, uint64_t total) override { status.Progress(done, total); } + void OnStep(const std::string &title) override { + status.Step(title); + logger.Info("rugnux: {}", title); + } + void OnNote(const std::string &text) override { + status.Note(text); + logger.Info("rugnux: {}", text); + } + }; +} + +JFJochReceiverRugnux::JFJochReceiverRugnux(const DiffractionExperiment &in_experiment, + const PixelMask &in_pixel_mask, + ImagePuller &in_image_puller, + ImagePusher &in_image_pusher, + Logger &in_logger, + int64_t in_forward_nthreads, + int in_rugnux_nthreads, + const std::string &in_model_path, + double in_deadline_s, + RugnuxLiveStatus &in_live_status, + const SpotFindingSettings &in_spot_finding_settings, + JFJochReceiverCurrentStatus &in_current_status, + JFJochReceiverPlots &in_plots, + ImageBuffer &in_image_buffer, + ZMQPreviewSocket *in_zmq_preview_socket, + ZMQMetadataSocket *in_zmq_metadata_socket) + : JFJochReceiver(in_experiment, in_image_buffer, in_image_pusher, in_current_status, in_plots, + in_spot_finding_settings, in_logger, in_pixel_mask, in_zmq_preview_socket, + in_zmq_metadata_socket, nullptr), + image_puller(in_image_puller), + reader(in_image_buffer), + forward_nthreads(JFJochReceiverLite::NumberOfDataAnalysisThreads(in_forward_nthreads, in_experiment)), + rugnux_nthreads(in_rugnux_nthreads), + model_path(in_model_path), + deadline_s(in_deadline_s), + live_status(in_live_status), + measurement_started(1), + forward_started(forward_nthreads) { + logger.Info("Starting {} forward threads; rugnux runs on {} threads while the images arrive", + forward_nthreads, rugnux_nthreads); + for (int i = 0; i < forward_nthreads; i++) + forward_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverRugnux::ForwardThread, this, i)); + measurement = std::async(std::launch::async, &JFJochReceiverRugnux::MeasurementThread, this); + forward_started.wait(); +} + +JFJochReceiverRugnux::~JFJochReceiverRugnux() { + cancelled = true; + reader.Ended(); + { + std::unique_lock ul(rugnux_m); + if (rugnux != nullptr) + rugnux->Cancel(); + if (probe != nullptr) + probe->Cancel(); + } + try { + if (measurement.valid()) + measurement.get(); + } catch (const std::exception &e) { + logger.Error("ReceiverRugnux teardown failed: {}", e.what()); + } +} + +void JFJochReceiverRugnux::MeasurementThread() { + std::future rugnux_done, probe_done; + try { + // Waiting for the start message, as JFJochReceiverLite::MeasurementThread. + std::optional give_up_at; + auto msg = image_puller.PollImage(); + while (!cancelled && (!msg.has_value() || !msg->cbor || !msg->cbor->start_message)) { + if (detector_finished && !give_up_at) + give_up_at = std::chrono::steady_clock::now() + StartMessageGrace; + if (give_up_at && (std::chrono::steady_clock::now() > *give_up_at)) { + logger.Error("Detector finished, but no start message arrived - abandoning measurement"); + cancelled = true; + break; + } + msg = image_puller.PollImage(); + } + if (cancelled) { + current_status.SetProgress({}); + current_status.SetStatus(GetStatus()); + measurement_started.count_down(); + return; + } + + Configure(msg->cbor->start_message.value()); + image_buffer.StartMeasurement(experiment); + start_time = std::chrono::system_clock::now(); + SendStartMessage(); + + // Rugnux reads the images back from the buffer, so it needs one slot per image: a slot taken + // again would hand it a later image. Without that the collection is only forwarded. + const auto slots = image_buffer.GetStatus().total_slots; + live_status.Start(experiment.GetImageNum()); + if (slots < experiment.GetImageNum()) { + logger.Error("Image buffer holds {} images, the collection has {}: no rugnux analysis for it", + slots, experiment.GetImageNum()); + live_status.Finish(RugnuxLiveState::Skipped, "the image buffer cannot hold the whole collection"); + } else { + // The start message as the writer gets it, so rugnux sees the dataset an HDF5 master states. + std::vector start_cbor; + image_buffer.GetStartMessage(start_cbor); + reader.Open(CBORStream2Deserialize(start_cbor)->start_message.value()); + char dir_template[] = "/tmp/jfjoch_rugnux_XXXXXX"; + if (mkdtemp(dir_template) == nullptr) + throw JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot create rugnux scratch directory"); + scratch_dir = dir_template; + rugnux_done = std::async(std::launch::async, &JFJochReceiverRugnux::RugnuxThread, this); + probe_done = std::async(std::launch::async, &JFJochReceiverRugnux::ProbeThread, this); + } + measurement_started.count_down(); + } catch (const JFJochException &e) { + logger.ErrorException(e); + Cancel(e); + measurement_started.count_down(); + throw; + } + + logger.Info("Receiving started"); + for (auto &f: forward_futures) + f.get(); + reader.Ended(); + images_done = true; + live_status.LastImage(); + const int64_t last_image = last_image_ns.load(); + logger.Info("Receiving finished, {} images", images_collected.load()); + + if (rugnux_done.valid()) { + // The fixed cutoff: the analysis gets deadline_s after the last image, then gives what it has. + if (deadline_s > 0 && rugnux_done.wait_for(std::chrono::duration(deadline_s)) + == std::future_status::timeout) + Cut("deadline"); + rugnux_done.wait(); + logger.Info("rugnux live: done {:.2f} s after the last image", SecondsSince(last_image)); + } + if (probe_done.valid()) + probe_done.wait(); + + current_status.SetProgress(1.0); + current_status.SetStatus(GetStatus()); + + SendEndMessage(); + const double end_sent_s = SecondsSince(last_image); + + if (!image_buffer.CheckIfBufferReturned(SendFinalizeTimeout)) { + logger.Error("Send commands not finalized in {} seconds", SendFinalizeTimeout.count()); + throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in " + + std::to_string(SendFinalizeTimeout.count()) + " seconds"); + } + if (push_images_to_writer) + writer_error = image_pusher.Finalize(); + if (last_image > 0 && rugnux_done.valid()) { + const double written_s = SecondsSince(last_image); + live_status.Latency(written_s); + logger.Info("rugnux live: last image -> all written {:.2f} s (end message sent at {:.2f} s)", + written_s, end_sent_s); + } + + if (!scratch_dir.empty()) { + std::error_code ec; + std::filesystem::remove_all(scratch_dir, ec); + } + end_time = std::chrono::system_clock::now(); + current_status.SetProgress({}); + current_status.SetStatus(GetStatus()); +} + +void JFJochReceiverRugnux::RugnuxThread() { + // Below the forward threads: every thread rugnux starts inherits this, so an image arriving finds a + // core however busy rugnux keeps the machine. + // The process-wide ParallelFor pool is not started by this thread, so it is lowered separately. + setpriority(PRIO_PROCESS, static_cast(syscall(SYS_gettid)), 10); + parallel_detail::WorkerPool::Instance().SetNice(10); + try { + const auto dataset = reader.GetDataset(); + const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, scratch_dir + "/p", model_path); + LiveObserver observer(plots, scan_result, *az_int_mapping, live_status, logger); + ProcessResult result; + { + Rugnux process(reader, setup.experiment, *dataset->pixel_mask, setup.config); + Reachable reachable(rugnux_m, rugnux, process, cut, cancelled); + result = process.Run(&observer); + } + // The verdict and summary of the report a `rugnux` run writes, without writing it. + const ReportDocument doc = BuildReportDocument(scratch_dir + "/p", experiment.GetFilePrefix(), + setup.experiment, result); + std::vector warnings; + for (const auto &w: doc.warnings) + if (!w.developer_only) + warnings.push_back(w.text); + live_status.Result(doc.verdict, doc.verdict_text, doc.summary_rows, warnings); + live_status.Finish(result.cancelled ? RugnuxLiveState::Cancelled : RugnuxLiveState::Done); + std::unique_lock ul(rugnux_m); + rugnux_result = std::move(result); + } catch (const std::exception &e) { + logger.Error("rugnux live: {}", e.what()); + live_status.Finish(RugnuxLiveState::Failed, e.what()); + } + if (!parallel_detail::WorkerPool::Instance().SetNice(0)) + logger.Warning("rugnux live: the ParallelFor pool stays at nice 10 - raising it back needs CAP_SYS_NICE"); +} + +void JFJochReceiverRugnux::ProbeThread() { + // Rugnux waits for half the sweep before it decides on a lattice. In the meantime the machine is + // idle, so a small run of its own indexes the first 60 degrees and says at once whether there is + // a crystal - quick feedback for the person at the beamline, above all where there is none. It + // decides nothing for the run, which goes on as it would without it. + constexpr double PROBE_DEG = 60.0; + setpriority(PRIO_PROCESS, static_cast(syscall(SYS_gettid)), 10); + try { + const auto dataset = reader.GetDataset(); + const auto gonio = dataset->experiment.GetGoniometer(); + if (!gonio || gonio->GetIncrement_deg() == 0.0f) + return; + const int64_t frames = std::lround(PROBE_DEG / std::abs(gonio->GetIncrement_deg())); + if (2 * frames > static_cast(reader.GetNumberOfImages())) + return; // a sweep this short is decided on soon enough by the run itself + RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, "", ""); + setup.config.end_image = static_cast(frames); + setup.config.rotation_postrefine_geometry = false; + setup.config.run_scaling = false; + ProcessResult r; + { + Rugnux probe_run(reader, setup.experiment, *dataset->pixel_mask, setup.config); + Reachable reachable(rugnux_m, probe, probe_run, cut, cancelled); + r = probe_run.Run(nullptr); + } + if (r.consensus_cell && r.rotation_lattice_type) { + const auto &c = *r.consensus_cell; + live_status.Probe(fmt::format("lattice on the first {:.0f} deg: {}-centred {}, {:.1f} {:.1f} {:.1f} " + "{:.1f} {:.1f} {:.1f}, {:.0f}% of the images indexed", PROBE_DEG, + r.rotation_lattice_type->centering, + gemmi::crystal_system_str(r.rotation_lattice_type->crystal_system), + c.a, c.b, c.c, c.alpha, c.beta, c.gamma, + 100.0 * r.indexing_rate.value_or(0.0f))); + } else + live_status.Probe(fmt::format("no lattice on the first {:.0f} deg", PROBE_DEG)); + } catch (const NoLatticeFound &e) { + live_status.Probe(fmt::format("no lattice on the first {:.0f} deg: {}", PROBE_DEG, e.reason)); + } catch (const std::exception &e) { + logger.Warning("rugnux live: the lattice probe failed: {}", e.what()); + } + logger.Info("rugnux live: lattice probe - {}", live_status.Get().probe); +} + +void JFJochReceiverRugnux::AddToEndMessage(EndMessage &message) { + std::unique_lock ul(rugnux_m); + if (!rugnux_result) + return; + const EndMessage &r = rugnux_result->end_message; + message.indexing_rate = r.indexing_rate; + message.bkg_estimate = r.bkg_estimate; + message.spindle_blind_fraction = r.spindle_blind_fraction; + message.ice_ring_score_mean = r.ice_ring_score_mean; + message.az_int_result = r.az_int_result; + message.rotation_lattice = r.rotation_lattice; + message.rotation_lattice_type = r.rotation_lattice_type; + message.rotation_extra_lattices = r.rotation_extra_lattices; + message.unit_cell = r.unit_cell; + message.space_group_name = r.space_group_name; + message.space_group_number = r.space_group_number; + message.reindex_matrix = r.reindex_matrix; + message.refined_beam_center_x = r.refined_beam_center_x; + message.refined_beam_center_y = r.refined_beam_center_y; + message.refined_poni_rot1 = r.refined_poni_rot1; + message.refined_poni_rot2 = r.refined_poni_rot2; + message.refined_poni_rot3 = r.refined_poni_rot3; + message.refined_rotation_axis = r.refined_rotation_axis; + // TODO: merged data, map coefficients and placed model from scratch_dir (writer branch). Only here, + // with rugnux_result set: every file in scratch_dir is then whole - rugnux writes synchronously and a + // cut never stops a write half-way - while a run that threw may have left one half-written. +} + +void JFJochReceiverRugnux::Configure(const StartMessage &msg) { + // As JFJochReceiverLite::Configure. + if ((experiment.GetXPixelsNum() != msg.image_size_x) + || (experiment.GetYPixelsNum() != msg.image_size_y)) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in detector size"); + if (fabs(experiment.GetPixelSize_mm() - msg.pixel_size_x * 1e3) > 1e-7) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Mismatch in pixel size"); + + experiment.Detector().Description(msg.detector_description); + experiment.Detector().SerialNumber(msg.detector_serial_number); + experiment.Detector().SensorMaterial(msg.sensor_material); + experiment.Detector().SensorThickness_um(msg.sensor_thickness * 1e6); + experiment.Detector().SaturationLimit(SaturationLimitFromValue(msg.saturation_value)); + experiment.Detector().CountRateCorrectionApplied(msg.countrate_correction_enabled); + experiment.Detector().CountRateCorrectionLookupTable(msg.countrate_correction_lookup_table); + experiment.Detector().FlatfieldApplied(msg.flatfield_enabled); + experiment.Detector().VirtualPixelInterpolationApplied(msg.virtual_pixel_interpolation_enabled); + experiment.ApplyPixelMask(false); + experiment.Detector().BitDepthImage(msg.bit_depth_image); + experiment.PixelSigned(msg.pixel_signed); +} + +void JFJochReceiverRugnux::ForwardThread(uint32_t id) { + forward_started.count_down(); + measurement_started.wait(); + + while (!cancelled && !end_message_received) { + try { + auto msg = image_puller.PollImage(std::chrono::milliseconds(5)); + if (!msg.has_value() || !msg->cbor) + continue; + if (msg->cbor->end_message) { + end_message_received = true; + } else if (msg->cbor->calibration) { + if (push_images_to_writer) + image_pusher.SendCalibration(msg->cbor->calibration.value()); + } else if (msg->cbor->data_message) { + last_image_ns = SteadyNowNs(); + live_status.ImageReceived(); + DataMessage data_msg = msg->cbor->data_message.value(); + ++images_collected; + + const auto compressed_size_img = data_msg.image.GetCompressedSize(); + const auto uncompressed_size_img = data_msg.image.GetUncompressedSize(); + compressed_size += compressed_size_img; + uncompressed_size += uncompressed_size_img; + UpdateMaxImageReceived(data_msg.number); + + data_msg.original_number = data_msg.number; + data_msg.user_data = experiment.GetImageAppendix(); + data_msg.run_number = experiment.GetRunNumber(); + data_msg.run_name = experiment.GetRunName(); + data_msg.receiver_buf_available = image_buffer.GetAvailSlots(); + data_msg.receiver_aq_dev_delay = image_puller.GetCurrentFifoUtilization(); + if (compressed_size_img > 0) + data_msg.compression_ratio = static_cast(uncompressed_size_img) + / static_cast(compressed_size_img); + + auto loc = image_buffer.GetImageSlot(); + if (loc == nullptr) + writer_queue_full = true; + else { + auto writer_buffer = (uint8_t *) loc->GetImage(); + CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize()); + serializer.SerializeImage(data_msg); + loc->SetImageNumber(data_msg.number); + loc->SetImageSize(serializer.GetBufferSize()); + loc->SetIndexed(false); + loc->ReadyToSend(); + + if (zmq_preview_socket != nullptr) + zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize()); + if (zmq_metadata_socket != nullptr) + zmq_metadata_socket->AddDataMessage(data_msg); + if (push_images_to_writer) { + if (image_pusher.SendImage(*loc)) + ++images_sent; + } else + loc->release(); + UpdateMaxImageSent(data_msg.number); + } + } + UpdateMaxDelay(image_puller.GetCurrentFifoUtilization()); + current_status.SetProgress(GetProgress()); + current_status.SetStatus(GetStatus()); + } catch (const JFJochException &e) { + logger.ErrorException(e); + Cancel(e); + } + } +} + +void JFJochReceiverRugnux::Cancel(bool silent) { + detector_finished = true; + // The silent cancel is the detector reading idle at the end of every collection: rugnux goes on. + if (silent) + return; + // /cancel is the one way in for the beamline, which cannot tell the collection from its analysis: it + // ends the collection while the images still arrive, as JFJochReceiverLite does, and in every case + // cuts the analysis, so the end message goes out at once with what rugnux has established. + if (!images_done) + JFJochReceiver::Cancel(silent); + Cut("cancel"); +} + +void JFJochReceiverRugnux::Cut(const std::string &reason) { + std::unique_lock ul(rugnux_m); + if (cut || rugnux_result) + return; + cut = true; + reader.Ended(); // no image is waited for any more + if (rugnux != nullptr) + rugnux->Cancel(); + if (probe != nullptr) + probe->Cancel(); + live_status.Cut(reason); + logger.Info("rugnux live: analysis cut ({}) - the end message carries what is established", reason); +} + +void JFJochReceiverRugnux::StopReceiver() { + if (measurement.valid()) { + measurement.get(); + logger.Info("Receiver stopped"); + } +} + +float JFJochReceiverRugnux::GetEfficiency() const { + if (experiment.GetFrameNum() == 0) + return 0; + return static_cast(images_collected) / static_cast(experiment.GetFrameNum()); +} + +float JFJochReceiverRugnux::GetProgress() const { + if (experiment.GetFrameNum() == 0) + return 0.0; + return static_cast(max_image_number_received) / static_cast(experiment.GetFrameNum()); +} diff --git a/receiver/JFJochReceiverRugnux.h b/receiver/JFJochReceiverRugnux.h new file mode 100644 index 000000000..1e480e4df --- /dev/null +++ b/receiver/JFJochReceiverRugnux.h @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +#include "JFJochReceiver.h" +#include "../image_puller/ImagePuller.h" +#include "../reader/JFJochImageBufferReader.h" +#include "../rugnux/Rugnux.h" +#include "RugnuxLiveStatus.h" + +// A DECTRIS rotation collection forwarded to the writer as JFJochReceiverLite does, but analysed by +// rugnux while it arrives instead of image by image: the forward threads put each image into the +// image buffer and send it on, and one rugnux thread reads the collection back from that buffer. The +// end message waits for rugnux, and carries what it found - the lattice, refined geometry, per-image +// results and its merged data, map coefficients and placed model (see AddToEndMessage). +class JFJochReceiverRugnux : public JFJochReceiver { + using JFJochReceiver::Cancel; + + ImagePuller &image_puller; + JFJochImageBufferReader reader; + + std::atomic end_message_received{false}; + std::atomic detector_finished{false}; // see JFJochReceiverLite + constexpr static auto StartMessageGrace = std::chrono::seconds(5); + + const int64_t forward_nthreads; + const int rugnux_nthreads; + const std::string model_path; + const double deadline_s; // after the last image, the analysis is cut (0 = never) + std::string scratch_dir; // rugnux's own output, read back into the end message + RugnuxLiveStatus &live_status; + + std::vector> forward_futures; + std::future measurement; + std::latch measurement_started; + std::latch forward_started; + + std::mutex rugnux_m; + Rugnux *rugnux = nullptr; // while it runs, for Cut + Rugnux *probe = nullptr; // the same for the lattice probe + std::optional rugnux_result; + bool cut = false; + std::atomic images_done{false}; // the forward threads have ended + + // The latency of the demonstration: from the last image in to every result in place. + std::atomic last_image_ns{0}; + + void MeasurementThread(); + void ForwardThread(uint32_t id); + void RugnuxThread(); + void ProbeThread(); + void Configure(const StartMessage &message); + // Stops the analysis where it is: rugnux returns what it has established (see Rugnux::Cancel), and + // the end message goes out with that. Reason: "deadline" or "cancel". + void Cut(const std::string &reason); + void AddToEndMessage(EndMessage &message) override; +public: + JFJochReceiverRugnux(const DiffractionExperiment &experiment, + const PixelMask &pixel_mask, + ImagePuller &image_puller, + ImagePusher &image_pusher, + Logger &logger, + int64_t forward_nthreads, + int rugnux_nthreads, + const std::string &model_path, + double deadline_s, + RugnuxLiveStatus &live_status, + const SpotFindingSettings &spot_finding_settings, + JFJochReceiverCurrentStatus ¤t_status, + JFJochReceiverPlots &plots, + ImageBuffer &image_buffer, + ZMQPreviewSocket *zmq_preview_socket = nullptr, + ZMQMetadataSocket *zmq_metadata_socket = nullptr); + ~JFJochReceiverRugnux() override; + + void StopReceiver() override; + void Cancel(bool silent) override; + + float GetEfficiency() const override; + float GetProgress() const override; + + void SetSpotFindingSettings(const SpotFindingSettings &spot_finding_settings) override {} +}; diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index ecbab954e..ce0c633c3 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -2,6 +2,9 @@ // SPDX-License-Identifier: GPL-3.0-only #include "JFJochReceiverService.h" +#include "JFJochReceiverRugnux.h" +#include "../common/ThreadAffinity.h" +#include "../rugnux/RugnuxDefaults.h" #include "JFJochReceiverFPGA.h" #include "JFJochReceiverLite.h" #include "../preview/JFJochJPEG.h" @@ -17,6 +20,23 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic spot_finding_settings(DiffractionExperiment::DefaultDataProcessingSettings()) { } +JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, const std::string &model_path, + int reserved_cpus, double deadline_s) { + std::unique_lock ul_state(state_mutex); + if (state != ReceiverState::Idle) + throw JFJochException(JFJochExceptionCategory::WrongDAQState, + "Cannot change rugnux settings during data collection"); + rugnux_enable = enable; + rugnux_model_path = model_path; + rugnux_reserved_cpus = reserved_cpus; + rugnux_deadline_s = deadline_s; + return *this; +} + +RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const { + return rugnux_status.Get(); +} + JFJochReceiverService &JFJochReceiverService::NumThreads(int64_t input) { if (input <= 0) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Thread number must be above zero"); @@ -103,6 +123,14 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, image_puller = std::make_shared( experiment.GetDetectorSetup().GetDECTRISStream2Addr()); } + if (rugnux_enable && RugnuxRotationData(experiment)) { + receiver = std::make_unique( + experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads_local, + std::max(1, static_cast(AvailableCpus()) - rugnux_reserved_cpus), + rugnux_model_path, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots, image_buffer, + zmq_preview_socket.get(), zmq_metadata_socket.get()); + break; + } receiver = std::make_unique(experiment, pixel_mask, *image_puller, diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index cb9045b1a..57a9985a5 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -3,6 +3,8 @@ #pragma once +#include "RugnuxLiveStatus.h" + #include #include @@ -21,6 +23,14 @@ class JFJochReceiverService { std::shared_ptr image_puller; int64_t nthreads = 8; + // Rotation collections from a DECTRIS detector analysed by rugnux while they arrive + // (JFJochReceiverRugnux), instead of image by image. Off unless asked for. + bool rugnux_enable = false; + std::string rugnux_model_path; + int rugnux_reserved_cpus = 4; // left to the forward threads and the pusher + double rugnux_deadline_s = 0; // the analysis is cut this long after the last image (0 = never) + RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends + enum class ReceiverState {Idle, Running}; ReceiverState state = ReceiverState::Idle; mutable std::mutex state_mutex; @@ -43,6 +53,9 @@ public: ImagePusher &pusher, size_t send_buffer_size_MiB = 512); JFJochReceiverService& NumThreads(int64_t input); + JFJochReceiverService& RugnuxLive(bool enable, const std::string &model_path, int reserved_cpus, + double deadline_s); + RugnuxLiveSnapshot GetRugnuxStatus() const; JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); JFJochReceiverService& MetadataSocket(const std::string &addr); diff --git a/receiver/RugnuxLiveStatus.cpp b/receiver/RugnuxLiveStatus.cpp new file mode 100644 index 000000000..a0247a8c9 --- /dev/null +++ b/receiver/RugnuxLiveStatus.cpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxLiveStatus.h" + +double RugnuxLiveStatus::Now() const { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +void RugnuxLiveStatus::CloseStep() { + if (!s.steps.empty() && !s.steps.back().duration_s) + s.steps.back().duration_s = Now() - s.steps.back().started_s; +} + +void RugnuxLiveStatus::Start(int64_t images_expected) { + std::unique_lock ul(m); + s = RugnuxLiveSnapshot{}; + s.state = RugnuxLiveState::Running; + s.images_expected = images_expected; + start = std::chrono::steady_clock::now(); +} + +void RugnuxLiveStatus::ImageReceived() { + std::unique_lock ul(m); + ++s.images_received; +} + +void RugnuxLiveStatus::Step(const std::string &title) { + std::unique_lock ul(m); + CloseStep(); + s.steps.push_back(RugnuxLiveStep{.title = title, .started_s = Now()}); + s.step_progress.reset(); +} + +void RugnuxLiveStatus::Note(const std::string &text) { + std::unique_lock ul(m); + if (s.steps.empty()) + s.steps.push_back(RugnuxLiveStep{.title = "", .started_s = Now()}); + s.steps.back().notes.push_back(text); +} + +void RugnuxLiveStatus::Progress(uint64_t done, uint64_t total) { + std::unique_lock ul(m); + if (total > 0) + s.step_progress = static_cast(done) / static_cast(total); +} + +void RugnuxLiveStatus::Finish(RugnuxLiveState state, const std::string &message) { + std::unique_lock ul(m); + CloseStep(); + s.elapsed_s = Now(); + s.state = state; + s.message = message; + s.step_progress.reset(); +} + +void RugnuxLiveStatus::Result(const std::string &verdict, const std::string &verdict_text, + const std::vector> &summary, + const std::vector &warnings) { + std::unique_lock ul(m); + s.verdict = verdict; + s.verdict_text = verdict_text; + s.summary = summary; + s.warnings = warnings; +} + +void RugnuxLiveStatus::Latency(double last_image_to_written_s) { + std::unique_lock ul(m); + s.last_image_to_written_s = last_image_to_written_s; +} + +void RugnuxLiveStatus::LastImage() { + std::unique_lock ul(m); + s.last_image_s = Now(); +} + +void RugnuxLiveStatus::Cut(const std::string &reason) { + std::unique_lock ul(m); + s.cut = reason; + s.cut_s = Now(); +} + +void RugnuxLiveStatus::Probe(const std::string &text) { + std::unique_lock ul(m); + s.probe = text; + s.probe_s = Now(); +} + +RugnuxLiveSnapshot RugnuxLiveStatus::Get() const { + std::unique_lock ul(m); + RugnuxLiveSnapshot ret = s; + if (s.state == RugnuxLiveState::Running) + ret.elapsed_s = Now(); + return ret; +} diff --git a/receiver/RugnuxLiveStatus.h b/receiver/RugnuxLiveStatus.h new file mode 100644 index 000000000..04e03f00c --- /dev/null +++ b/receiver/RugnuxLiveStatus.h @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include +#include +#include + +// Where the live rugnux run of the last collection is, for the broker's status route: the steps it has +// taken with what each found ("lattice found", "space group P 21 21 21"), the step it is on and how far +// into its images, and - once it is over - its verdict and summary as the report states them, and the +// latency from the last image to every result written. +enum class RugnuxLiveState { Idle, Running, Done, Failed, Cancelled, Skipped }; + +struct RugnuxLiveStep { + std::string title; + double started_s = 0.0; // since the collection started + std::optional duration_s; // set once the next step starts or the run ends + std::vector notes; +}; + +struct RugnuxLiveSnapshot { + RugnuxLiveState state = RugnuxLiveState::Idle; + std::string message; // why it failed or was skipped + int64_t images_expected = 0; + int64_t images_received = 0; + double elapsed_s = 0.0; + std::optional step_progress; // fraction of the current pass's images + std::vector steps; + // The lattice probe on the first 60 degrees (JFJochReceiverRugnux::ProbeThread): an early answer + // to "is there a crystal", seconds into the collection; the run's own answer comes later. + std::string probe; + std::optional probe_s; // when it answered, since the collection started + std::optional last_image_s; // when the last image arrived, since the collection started + // The analysis was cut - "deadline" (the broker's deadline_s after the last image) or "cancel" - at + // cut_s since the collection started; what it had established went out, the steps after it were left out. + std::string cut; + std::optional cut_s; + std::optional last_image_to_written_s; + std::string verdict; // OK | WARNINGS | UNUSABLE | FAILED + std::string verdict_text; + std::vector> summary; + std::vector warnings; +}; + +class RugnuxLiveStatus { + mutable std::mutex m; + RugnuxLiveSnapshot s; + std::chrono::steady_clock::time_point start; + double Now() const; + void CloseStep(); +public: + void Start(int64_t images_expected); + void ImageReceived(); + void Step(const std::string &title); + void Note(const std::string &text); + void Progress(uint64_t done, uint64_t total); + void Finish(RugnuxLiveState state, const std::string &message = ""); + void Result(const std::string &verdict, const std::string &verdict_text, + const std::vector> &summary, + const std::vector &warnings); + void Latency(double last_image_to_written_s); + void LastImage(); + void Cut(const std::string &reason); + void Probe(const std::string &text); + RugnuxLiveSnapshot Get() const; +}; diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index 27eb8cfb1..83444105f 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -17,6 +17,8 @@ ADD_LIBRARY(JFJochRugnux STATIC RugnuxScaleMerge.cpp RugnuxDefaults.cpp RugnuxDefaults.h + RugnuxLive.cpp + RugnuxLive.h Rugnux.h RugnuxCommandLine.cpp RugnuxCommandLine.h diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index e6e6eb076..a7c231288 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -419,11 +419,11 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) { result = RunAllPasses(observer); } catch (const NoLatticeFound &) { // A run that found no lattice after deciding a near tie early is made again deciding late. - if (!near_tie_rechecks_->carried) + if (!near_tie_rechecks_->carried || config_.fast_live) throw; lost = true; } - if (lost || (!result.cancelled && NearTieOverturned())) { + if (lost || (!result.cancelled && !config_.fast_live && NearTieOverturned())) { JoinBeamCenterCapture(); Logger logger("Rugnux"); logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, " @@ -431,6 +431,12 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) { result = again->RunAllPasses(observer); run = again.get(); } + // Cut short in fast_live: a pass whose images were not all read has no lattice and no merge, so the + // run's answer is the last pass that ran to its end. + if (config_.fast_live && result.cancelled && established_) { + result = *established_; + result.cancelled = true; + } result.total_time_s = std::chrono::duration(std::chrono::steady_clock::now() - run_start).count(); // For the report's beam-centre warning: what the background measured, and what this crystal asks // of the centre - at the file's geometry, as the pre-scan's check asks it, but on the cell the run @@ -707,7 +713,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // skip the search that is the whole point of running again. // The run as this pass finds it, for the re-check of a near tie its first pass may decide early. std::shared_ptr recheck; - if (!decide_late_ && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing + if (!decide_late_ && !config_.fast_live && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing && config_.two_pass_rotation && !force_rotation_result_.has_value() && !config_.forced_rotation_lattice.has_value() && experiment_.IsRotationIndexing()) recheck = std::make_shared(*this); @@ -1072,7 +1078,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b image_pass_mark.Stop(); // Scaling and merging (full analysis only). - if (full && !cancelled_ && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found) + // As ImagePass's stopped(): in fast_live a pass that read all its images merges whatever comes after. + const bool images_cut = config_.fast_live ? result.cancelled : static_cast(cancelled_); + if (full && !images_cut && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found) && (config_.run_scaling || !config_.reference_data.empty())) { TimingMark mark(logger, "scale, merge and symmetry"); std::optional gpu_merge; // what the speculative probe waits for on a small card @@ -1190,6 +1198,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b writer->Finalize(); result.written_master_path = config_.output_prefix + "_process.h5"; } + result.end_message = end_msg; output_mark.Stop(); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index e5cca61d9..afbdca2aa 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -110,6 +110,13 @@ struct ProcessConfig { // are decided (Rugnux::decide_late_). bool live = false; + // Live in the broker, where the run has to end a fixed time after the last frame: every decision + // stands where it was first taken, and nothing the run would otherwise do again once the sweep is + // in - the near-tie re-checks and the run made again, the rotation-scale walk, the metric-symmetry + // arms, the geometry walk, the supercell and quality guards, the doubled cell - is done. A + // difficult crystal can come out worse; an easy one comes out the same, sooner. + bool fast_live = false; + // Rotation indexing (FullAnalysis) bool rotation_indexing = false; bool two_pass_rotation = true; @@ -326,6 +333,9 @@ struct ProcessResult { // beam-centre pre-scan that runs before the first one. processing_time_s is the last pass alone // and so understates a two-pass run by more than a factor of two. double total_time_s = 0.0; + // The end message of this pass, as the _process.h5 records it: the lattice, refined geometry, + // reindexing and the run means - what the broker puts into its own when it runs rugnux live. + EndMessage end_message; // Workers the per-image loop actually ran, which is not config.nthreads once the loop is capped. int image_loop_threads = 0; double frame_rate_hz = 0.0; @@ -813,6 +823,9 @@ class Rugnux { CancelFlag Own() const { CancelFlag f; f.run = flag; return f; } }; CancelFlag cancelled_; + // fast_live: the last pass that ran to its end. Where a cancel cuts the pass after it short, Run() + // returns this one, so the broker sends what is established (lattice, geometry) rather than nothing. + std::optional established_; // Where the steps and notes of the pass now running go: the observer on a pass that makes the // result, nullptr on a probe pass and on a copy run beside the main one. Set by RunPipeline. @@ -1357,6 +1370,8 @@ public: const PowderRings &GetPowderRings() const { return powder_; } // Request cancellation; safe to call from any thread (the worker loop checks between images). + // fast_live makes it a cut: a pass that is past its images still merges and writes its files, + // only the model is left out, and a pass cut in its images gives way to the last complete one. void Cancel() { cancelled_ = true; } bool IsCancelled() const { return cancelled_; } }; diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index dc17a0419..0510a79e9 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -409,6 +409,10 @@ void Rugnux::ImagePass(PipelineLocals &p) { std::chrono::steady_clock::now() - image_loop_start).count(); result.cancelled = cancelled_; + // Whether the pass stops here. In fast_live a cut that comes after the last image read lets the pass + // finish - its lattice, its merge and its files are what the run has established - and only the model + // after them is left out (ScaleMergeAndSymmetry). + const auto stopped = [&] { return config_.fast_live ? result.cancelled : static_cast(cancelled_); }; result.images_processed = finished_count.load(); Note(fmt::format("{} images, {:.0f} per second", result.images_processed, result.images_processed / std::max(1e-6, result.image_loop_time_s))); @@ -424,7 +428,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // The footprint of the reflection rather than of the spot: the pre-scan's widths plus where the // spots sit against their predictions (SpotFootprint.h). Like the widths it acts only where the // reflection outgrows the r1 disk, and like them it is held back for the canonical pass. - if (measure_offsets && !cancelled_) { + if (measure_offsets && !stopped()) { const SpotFootprint fp = FootprintWithOffsets(*prescan_footprint_, footprint_offsets); const BraggIntegrationSettings canonical = bragg_adaptive_.value_or(experiment_.GetBraggIntegrationSettings()); bool outgrows = false; @@ -446,7 +450,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { } } - if (full && indexer && experiment_.IsRotationIndexing() && !cancelled_) { + if (full && indexer && experiment_.IsRotationIndexing() && !stopped()) { const SupercellProbe probe = indexer->GetSupercellProbe(); if (probe[0][0].n > 0 && probe[0][1].n > 0) { std::string table; @@ -594,7 +598,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { predicted += p_i; crowded += c_i; } - if (predicted > 0 && !cancelled_) { + if (predicted > 0 && !stopped()) { const double f = static_cast(crowded) / static_cast(predicted); const double n = static_cast(crowding_frames.size()); double ss = 0.0; @@ -611,7 +615,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // Every image failing is a total failure, not a run that produced nothing: it used to be reported // only as per-image log lines while the process still exited 0 with no output file. - if (!cancelled_ && images_to_process > 0 && result.images_processed == 0) + if (!stopped() && images_to_process > 0 && result.images_processed == 0) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "No image could be analyzed - see the per-image errors above"); @@ -621,7 +625,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // Powder calibration: the whole run has now been accumulated - the summed (q x azimuth) profile in // plots, or the pooled spot list - so fit the detector geometry to it. The caller reports it and // writes the PONI file; nothing here is stored in the _process.h5. - if (calibration && !cancelled_) + if (calibration && !stopped()) FitCalibration(p, calibration_spot_list); // End message (also written to the file). @@ -642,7 +646,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { end_msg.az_int_result["dataset"] = plots.GetAzIntProfile(); end_msg.indexing_rate = result.indexing_rate; - if (full && !cancelled_) { + if (full && !stopped()) { const auto rot = indexer->FinalizeRotationIndexing(); indexer->RethrowIndexerError(); if (rot.has_value()) { diff --git a/rugnux/RugnuxLive.cpp b/rugnux/RugnuxLive.cpp new file mode 100644 index 000000000..afd1bc3f1 --- /dev/null +++ b/rugnux/RugnuxLive.cpp @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxLive.h" + +#include "RugnuxDefaults.h" + +RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, + const std::string &output_prefix, const std::string &model_path) { + RugnuxLiveSetup s; + + // The experiment as rugnux_cli.cpp sets it up for a rotation sweep with no options given. + DiffractionExperiment &x = s.experiment; + x = dataset.experiment; + ApplyRugnuxExperimentDefaults(x); + x.NumTriggers(1); + ClearStoredCrystal(x); + x.MaxSpotCount(RUGNUX_MAX_SPOT_COUNT); + + IndexingSettings indexing; + indexing.Algorithm(IndexingAlgorithmEnum::Auto); + indexing.RotationIndexing(true); + indexing.GeomRefinementAlgorithm(GeomRefinementAlgorithmEnum::BeamCenter); + indexing.IndexIceRings(false); + x.ImportIndexingSettings(indexing); + x.DetectIceRings(RugnuxDetectIceRings(dataset.file_detect_ice_rings, true)); + + ScalingSettings scaling = RugnuxDefaultScalingSettings(true); + scaling.ScaleFulls(true); + scaling.RelativeBDegrees(0.0); + scaling.MergeFriedel(true); + scaling.MinPartiality(0.02); + scaling.MinCCForImage(0.0); + x.ImportScalingSettings(scaling); + + ProcessConfig &c = s.config; + c.mode = ProcessMode::FullAnalysis; + c.nthreads = nthreads; + c.output_prefix = output_prefix; + c.write_process_h5 = false; + c.export_unmerged = false; + c.spot_finding = RugnuxDefaultSpotFinding(true); + c.rotation_indexing = true; + c.two_pass_rotation = true; + c.rotation_postrefine_geometry = true; + c.rotation_indexing_image_count = RUGNUX_ROTATION_INDEXING_IMAGES; + c.detect_beam_stop = RUGNUX_DETECT_BEAM_STOP_FRAMES; + c.beam_center_check = true; + c.measure_spot_budget = true; + c.run_scaling = true; + c.model_path = model_path; + c.live = true; + c.fast_live = true; + // Decided late, on the whole sweep, as it re-checks its radius there - which fast_live does not. + c.adaptive_integration_radius = false; + return s; +} diff --git a/rugnux/RugnuxLive.h b/rugnux/RugnuxLive.h new file mode 100644 index 000000000..c4aba1297 --- /dev/null +++ b/rugnux/RugnuxLive.h @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "Rugnux.h" +#include "../reader/JFJochReaderDataset.h" + +// A rotation collection processed inside the broker while its frames arrive: what a bare `rugnux` on +// the same file would set, with the run made fast_live. The output goes to output_prefix - a place +// private to the broker, from which it builds the END message - and the model, where one is given, +// is placed and validated as `rugnux --model` does. +struct RugnuxLiveSetup { + DiffractionExperiment experiment; + ProcessConfig config; +}; + +RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, + const std::string &output_prefix, const std::string &model_path); diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 01ae4bc01..6b41c4e23 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -387,6 +387,8 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { pass1.pass_number = 1; pass1.pass_count = 2; if (cancelled_) { config_.output_prefix = base_prefix; return pass1; } + if (config_.fast_live) + established_ = pass1; // Beam centre as a LATTICE hypothesis, decided on the merge. The first pass indexed both // centres, found the same cell with a different metric symmetry read off it, and measured that @@ -658,7 +660,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // pass off and throws it away, so the walk runs first, on its own, and the refined passes // after it. Either way the run ends where the sequential walk leaves it. constexpr double ROTATION_SCALE_STEP = 0.01; - if (!cancelled_ && gonio_snapshot && pass1.post_refine && !config_.rotation_scale) { + if (!cancelled_ && !config_.fast_live && gonio_snapshot && pass1.post_refine && !config_.rotation_scale) { const auto check = std::make_shared(); RotationScaleWalk walk; if (std::fabs(pass1.post_refine->rotation_scale - 1.0) >= ROTATION_SCALE_STEP) { @@ -940,7 +942,8 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // Let every refined pass from here on ask the quality guard's question of itself, so one that // is going to be thrown away can skip the report and the files it would have superseded. - quality_guard_pass1_ = pass1; + if (!config_.fast_live) + quality_guard_pass1_ = pass1; config_.output_prefix = base_prefix; // the refined pass is the canonical result (no _02 suffix) // The canonical pass measures the post-refinement again where the pre-pass moved the geometry // (it applies nothing), so that the walk below has something to decide on. @@ -997,7 +1000,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro double split = 0.0; std::optional demoted_arm; // run where the data resolve the equality std::shared_ptr demoted_recheck; // the demoted arm again on all the frames, for a near tie - if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) { + if (!cancelled_ && !config_.fast_live && (length_arm || angle_arm) && pass1.rotation_lattice_type) { split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0; const double dx = std::max(experiment_.GetBeamX_pxl(), static_cast(experiment_.GetXPixelsNum()) - experiment_.GetBeamX_pxl()); @@ -1101,8 +1104,8 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical) -> {}_*", base_prefix); - speculate_geometry_probe_ = true; // a walk that follows starts with an indexing probe - rerun_if_starved_ = true; + speculate_geometry_probe_ = !config_.fast_live; // a walk that follows starts with an indexing probe + rerun_if_starved_ = !config_.fast_live; const DiffractionExperiment before_pass2 = experiment_; auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); if (radius_recheck_) { @@ -1296,7 +1299,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // from here, not from where its own probe left the run. DiffractionExperiment last_round_start = experiment_; bool walk_shown = false; - while (!cancelled_ && prepass_detector_geometry_ + while (!cancelled_ && !config_.fast_live && prepass_detector_geometry_ && pass2.post_refine && pass2.post_refine->detector_refined) { if (!walk_shown) { step("Geometry check"); @@ -1437,7 +1440,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // RunPipeline) - so each is followed by it. After the first re-run the radius is the fixed one // and this never fires again. const auto rerun_if_starved = [&](ProcessResult &pass, const std::string decision) { - if (cancelled_ || !bragg_adaptive_ || !bkg_starved_fraction_ + if (cancelled_ || config_.fast_live || !bragg_adaptive_ || !bkg_starved_fraction_ || *bkg_starved_fraction_ <= spot_width::BKG_STARVED_MAX_FRACTION) return; logger.Info("Two-pass: at the widened integration radius r1={:.1f} the neighbouring " @@ -1487,7 +1490,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro }; const double v1 = primitive_volume(*pass1.consensus_cell, pass1.consensus_centering); const double v2 = primitive_volume(*pass2.consensus_cell, pass2.consensus_centering); - if (v1 > 0.0 && v2 > 1.5 * v1 && !supercell_settled_) { + if (v1 > 0.0 && v2 > 1.5 * v1 && !supercell_settled_ && !config_.fast_live) { logger.Info("Two-pass: second-pass primitive cell volume {:.0f} A^3 is {:.2f}x pass-1's " "({:.0f} A^3) - a spurious supercell; re-running the second pass with " "pass-1's result forced", @@ -1519,7 +1522,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // but it is not guaranteed to be, and the guard that decides is RefinedPassIsWorse above. It // has already been asked once, inside the pass itself, which is how a pass that is about to be // thrown away knows not to write a report; this is where the answer is acted on. - if (!cancelled_ && pass1.has_merge_statistics && pass2.has_merge_statistics) { + if (!cancelled_ && !config_.fast_live && pass1.has_merge_statistics && pass2.has_merge_statistics) { logger.Info("Two-pass quality guard: the refined pass holds {} reflections at I/sigma >= 2 and " "{} at <= -2 of {} in a {:.0f} A^3 primitive cell, the header-geometry pass {} and {} " "of {} in {:.0f} A^3; low-order axial {} against {}", @@ -1800,7 +1803,7 @@ namespace { ProcessResult Rugnux::DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half, const std::string &base_prefix) { - if (cancelled_ || user_cell_ || user_fixed_sg_ || !half.supercell || !last_rotation_result_ + if (cancelled_ || config_.fast_live || user_cell_ || user_fixed_sg_ || !half.supercell || !last_rotation_result_ || !half.has_merge_statistics || !half.consensus_cell || !DoubledCellWorthMerging(*half.supercell)) return half; Logger logger("Rugnux"); diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 658fadb27..78ea9d081 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -3992,8 +3992,16 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { gemmi::SpaceGroup space_group; }; std::optional placed_model_out; + // fast_live, cut (the broker's deadline or a cancel): the model is left out, and the files are written + // as a run without one writes them. Asked again before the search below. + const auto model_cut = [&] { + if (!config_.fast_live || !cancelled_) + return false; + Note("cut: the model is left out"); + return true; + }; const bool model_run = result.consensus_cell && write_files && !config_.model_path.empty() && !geometry_prepass - && !superseded; + && !superseded && !model_cut(); // The twin hypothesis the model is asked about: its own subgroup's merge where it has one, the data's // otherwise - and whether that merge was made only for the model, so is written only where it chooses it. std::optional &model_twin = model_twin_merge ? model_twin_file : twin_file; @@ -4307,7 +4315,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // veto - and nothing else, never an indexing: a different group or the subgroup it chooses goes to the // files of its own, _mr, from the merge it was chosen on. It never makes the rigid body's // verdict (MODEL_FIT) either; its own null is what placed it. - if (model_run && (model_predicted || config_.force_mr || !placed_by_rigid_body)) { + if (model_run && (model_predicted || config_.force_mr || !placed_by_rigid_body) && !model_cut()) { phase("Molecular replacement"); Step("Molecular replacement"); TimingMark mark(logger, "model prior"); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 33f713b9f..e218c8a70 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -318,6 +318,7 @@ enum { OPT_WRITE_PROCESS_H5, OPT_FINALIST_LEDGER, OPT_SIMULATE_LIVE, + OPT_FAST_LIVE, OPT_DEVELOPER, OPT_NO_GPU, OPT_FORCE_STILL, @@ -376,6 +377,7 @@ static option long_options[] = { {"write-process-h5", no_argument, nullptr, OPT_WRITE_PROCESS_H5}, {"finalist-ledger", no_argument, nullptr, OPT_FINALIST_LEDGER}, {"simulate-live", required_argument, nullptr, OPT_SIMULATE_LIVE}, + {"fast-live", no_argument, nullptr, OPT_FAST_LIVE}, // undocumented: the broker's live run (ProcessConfig::fast_live) {"developer", no_argument, nullptr, OPT_DEVELOPER}, {"no-gpu", no_argument, nullptr, OPT_NO_GPU}, {"smooth-g", optional_argument, nullptr, OPT_SMOOTH_G}, @@ -942,6 +944,7 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { std::optional scale_fulls_arg; // --scale-fulls / --no-scale-fulls; default on for rot3d bool write_process_h5_flag = false; // --write-process-h5; also write _process.h5 when merging bool simulate_live = false; // --simulate-live + bool fast_live = false; // --fast-live, with --simulate-live bool finalist_ledger_flag = false; // --finalist-ledger; report-only symmetry evidence table std::optional detect_ice_rings; // --detect-ice-rings[=on|off]; unset => use the dataset (file) value bool index_ice_rings = false; // --index-ice-rings[=on|off]; index on the ice-band spots too @@ -1367,6 +1370,9 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { case OPT_FINALIST_LEDGER: finalist_ledger_flag = true; break; + case OPT_FAST_LIVE: + fast_live = true; + break; case OPT_SIMULATE_LIVE: simulate_live = true; simulated_frame_time = std::chrono::microseconds( @@ -3136,6 +3142,9 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { config.write_process_h5 = run_scaling ? write_process_h5_flag : true; config.finalist_ledger = finalist_ledger_flag; config.live = broker_feed || simulate_live; + config.fast_live = fast_live && config.live; + if (config.fast_live) + config.adaptive_integration_radius = false; // as MakeRugnuxLiveSetup if (simulate_live) logger.Info("Simulated live collection: image i readable (i+1) x {:.1f} ms after the run starts", simulated_frame_time.count() / 1000.0); From 4e6d7c33b302f40b7abb59f37269c557afe07dfd Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 12:20:50 +0200 Subject: [PATCH 04/19] rugnux live: END carries the merged data, map coefficients and placed model rugnux wrote Co-Authored-By: Claude Opus 5.5 --- receiver/JFJochReceiverRugnux.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index b876253f2..2254855ca 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -16,6 +16,7 @@ #include "../frame_serialize/CBORStream2Deserializer.h" #include "../rugnux/RugnuxLive.h" #include "../rugnux/ResultReport.h" +#include "../writer/RugnuxResultBlocks.h" using namespace std::chrono_literals; @@ -338,9 +339,10 @@ void JFJochReceiverRugnux::AddToEndMessage(EndMessage &message) { message.refined_poni_rot2 = r.refined_poni_rot2; message.refined_poni_rot3 = r.refined_poni_rot3; message.refined_rotation_axis = r.refined_rotation_axis; - // TODO: merged data, map coefficients and placed model from scratch_dir (writer branch). Only here, - // with rugnux_result set: every file in scratch_dir is then whole - rugnux writes synchronously and a - // cut never stops a write half-way - while a run that threw may have left one half-written. + // Merged data, map coefficients and placed model, for the writer to write beside the master. Only + // here, with rugnux_result set: every file in scratch_dir is then whole - rugnux writes synchronously + // and a cut never stops a write half-way - while a run that threw may have left one half-written. + FillRugnuxResultBlocks(scratch_dir + "/p", message); } void JFJochReceiverRugnux::Configure(const StartMessage &msg) { From f215300a07942c95b18d2c0432a288cd22b15e23 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 14:27:10 +0200 Subject: [PATCH 05/19] ImageBuffer reader: give up on an image MAX_IMAGE_LAG (100) behind the newest, and a unit test Images reach the buffer in order give or take a few, so one still missing when the buffer holds an image 100 past it is lost, not late: the read returns false instead of waiting for the end of the collection. The newest image number comes from ImageBuffer::GetStatus, which scans every slot under the buffer's lock, so it is asked only every 100 polls (~10 Hz). The test puts CBOR images into an ImageBuffer the way the receiver does and checks read-back of the pixels, waiting for an image not yet there, the lag limit, the end of the collection and the bounds. Co-Authored-By: Claude Opus 5.5 --- reader/JFJochImageBufferReader.cpp | 7 +- reader/JFJochImageBufferReader.h | 5 +- tests/CMakeLists.txt | 1 + tests/JFJochImageBufferReaderTest.cpp | 97 +++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tests/JFJochImageBufferReaderTest.cpp diff --git a/reader/JFJochImageBufferReader.cpp b/reader/JFJochImageBufferReader.cpp index a7ca4e2cc..67d6f687e 100644 --- a/reader/JFJochImageBufferReader.cpp +++ b/reader/JFJochImageBufferReader.cpp @@ -34,9 +34,14 @@ bool JFJochImageBufferReader::WaitForMessage(int64_t image_number, std::vector(image_number) >= number_of_images_) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Image out of bounds"); // The buffer has no notification of its own; at 100 Hz a millisecond's poll costs nothing. - while (!buffer_.GetImage(cbor, image_number)) { + for (int poll = 1; !buffer_.GetImage(cbor, image_number); poll++) { if (ended_) return buffer_.GetImage(cbor, image_number); // one that came in just before the end + // Images arrive in order, give or take a few: one still missing when the buffer already holds + // an image MAX_IMAGE_LAG past it is lost, not late. GetStatus scans every slot under the + // buffer's lock, so it is asked only every 100 polls. + if (poll % 100 == 0 && buffer_.GetStatus().max_image_number >= image_number + MAX_IMAGE_LAG) + return buffer_.GetImage(cbor, image_number); std::this_thread::sleep_for(std::chrono::milliseconds(1)); } return true; diff --git a/reader/JFJochImageBufferReader.h b/reader/JFJochImageBufferReader.h index 5a7c5203d..7acf60aa8 100644 --- a/reader/JFJochImageBufferReader.h +++ b/reader/JFJochImageBufferReader.h @@ -13,10 +13,13 @@ // Reads a collection inside the broker while it arrives: the dataset comes from its start message, // the images from the broker's image buffer, where the receiver puts each one as it forwards it. A // read of an image not there yet waits for it; after Ended it returns false for an image that never -// came. The buffer must hold the whole collection (slots >= images) - a slot taken again for a later +// came, and so does a read of an image still missing when one MAX_IMAGE_LAG past it is there. The +// buffer must hold the whole collection (slots >= images) - a slot taken again for a later // image would hand out that image instead. class JFJochImageBufferReader : public JFJochReader { public: + constexpr static int64_t MAX_IMAGE_LAG = 100; + explicit JFJochImageBufferReader(ImageBuffer &buffer); void Open(const StartMessage &start); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5f5c4b207..0866355a7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,6 +87,7 @@ ADD_EXECUTABLE(jfjoch_test SpotTilesTest.cpp LossyFilterTest.cpp ImageBufferTest.cpp + JFJochImageBufferReaderTest.cpp PixelMaskTest.cpp RegressionTest.cpp FPGAHLSModulesTest.cpp diff --git a/tests/JFJochImageBufferReaderTest.cpp b/tests/JFJochImageBufferReaderTest.cpp new file mode 100644 index 000000000..41abe0b31 --- /dev/null +++ b/tests/JFJochImageBufferReaderTest.cpp @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include +#include +#include + +#include "../reader/JFJochImageBufferReader.h" +#include "../common/ZeroCopyReturnValue.h" +#include "../frame_serialize/CBORStream2Serializer.h" + +// The receiver's way of putting an image in the buffer: serialize the message into a slot, hand it out. +static void PutImage(ImageBuffer &buffer, const DiffractionExperiment &experiment, int64_t number, + const std::vector &pixels) { + DataMessage msg{}; + msg.number = number; + msg.image = CompressedImage(pixels, experiment.GetXPixelsNum(), experiment.GetYPixelsNum()); + auto loc = buffer.GetImageSlot(); + REQUIRE(loc != nullptr); + CBORStream2Serializer serializer((uint8_t *) loc->GetImage(), experiment.GetImageBufferLocationSize()); + serializer.SerializeImage(msg); + loc->SetImageNumber(number); + loc->SetImageSize(serializer.GetBufferSize()); + loc->SetIndexed(false); + loc->ReadyToSend(); + loc->release(); +} + +TEST_CASE("JFJochImageBufferReader", "[JFJochReader]") { + DiffractionExperiment experiment(DetJF(1)); + experiment.ImagesPerTrigger(200); + + ImageBuffer buffer(4 * experiment.GetImageBufferLocationSize()); + buffer.StartMeasurement(experiment); + + StartMessage start{}; + experiment.FillMessage(start); + REQUIRE(start.number_of_images == 200); + + JFJochImageBufferReader reader(buffer); + reader.Open(start); + REQUIRE(reader.GetNumberOfImages() == 200); + REQUIRE(reader.GetDataset()); + + const size_t npixel = experiment.GetXPixelsNum() * experiment.GetYPixelsNum(); + std::vector pixels(npixel, 7); + pixels[123] = 4567; + PutImage(buffer, experiment, 0, pixels); + + JFJochReaderRawImage raw; + REQUIRE(reader.ReadRawImage(0, raw)); + REQUIRE(raw.image.GetWidth() == experiment.GetXPixelsNum()); + REQUIRE(raw.image.GetHeight() == experiment.GetYPixelsNum()); + REQUIRE(raw.image_buffer.size() == npixel * sizeof(int32_t)); + int32_t value; + memcpy(&value, raw.image.GetCompressed() + 123 * sizeof(int32_t), sizeof(int32_t)); + REQUIRE(value == 4567); + + auto image = reader.LoadImage(0); + REQUIRE(image); + REQUIRE(image->ImageData().number == 0); + + REQUIRE_THROWS(reader.ReadRawImage(200, raw)); + + // An image not there yet is waited for: the read returns once the receiver puts it in. + auto pending = std::async(std::launch::async, [&] { + JFJochReaderRawImage r; + return reader.ReadRawImage(1, r); + }); + REQUIRE(pending.wait_for(std::chrono::milliseconds(50)) == std::future_status::timeout); + PutImage(buffer, experiment, 1, pixels); + REQUIRE(pending.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(pending.get()); + + // Images arrive in order, give or take a few: one still missing when an image MAX_IMAGE_LAG past + // it is there is lost, while one nearer may still come. + PutImage(buffer, experiment, 150, pixels); + auto lost = std::async(std::launch::async, [&] { + JFJochReaderRawImage r; + return reader.ReadRawImage(150 - JFJochImageBufferReader::MAX_IMAGE_LAG, r); + }); + REQUIRE(lost.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(!lost.get()); + + // After the end, an image that never came is not waited for. + auto missing = std::async(std::launch::async, [&] { + JFJochReaderRawImage r; + return reader.ReadRawImage(150 - JFJochImageBufferReader::MAX_IMAGE_LAG + 1, r); + }); + REQUIRE(missing.wait_for(std::chrono::milliseconds(300)) == std::future_status::timeout); + reader.Ended(); + REQUIRE(missing.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + REQUIRE(!missing.get()); + + REQUIRE(buffer.Finalize(std::chrono::microseconds(1))); +} From 4bf41983ed13bd71ed304029a7dbc837249b6bb2 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 16:11:58 +0200 Subject: [PATCH 06/19] rugnux: the model is parsed by the caller and handed in as RugnuxModel The analysis read the --model file itself, four to five times per run (the model kind, every validation, molecular replacement, the stills reference intensities) and located the PAE sidecar from the file name. The library is to do no file I/O, so the model now arrives in memory: - RugnuxModel (image_analysis/structure_refinement/RugnuxModel.h): the gemmi structure as read, whether it is ModelCIF, the PAE matrix where a prediction came with one, and the name the report gives it. - ParseRugnuxModel / ReadRugnuxModelFile (writer/RugnuxModelFile.h, in JFJochRugnuxFiles): bytes in, gzip by magic, PDB/mmCIF by content, _ma_qa_metric detection as before; the file reader takes the PAE JSON from the same place PaePathFor named. - ProcessConfig::model replaces model_path; PrepareSearchModel, RunMolecularReplacement, ValidateAgainstModel and ModelReferenceIntensities take the model; result fields name it by model_name. - rugnux reads the model before the run starts and stops with an error where the file holds no model (it used to run and report the model as not validated). The viewer reads it when the job is set up. - The broker parses an uploaded model at once and answers 400 where it is not one; it keeps the parsed model in memory, with no temporary directory or file. It takes no PAE. Co-Authored-By: Claude Opus 5.5 --- broker/JFJochBrokerHttp.cpp | 20 ++-- broker/JFJochServices.cpp | 33 +------ broker/JFJochServices.h | 6 +- broker/JFJochStateMachine.cpp | 4 +- broker/JFJochStateMachine.h | 2 +- .../structure_refinement/CMakeLists.txt | 1 + .../structure_refinement/ModelValidation.cpp | 53 +++++----- .../structure_refinement/ModelValidation.h | 16 +-- .../MolecularReplacement.cpp | 64 +++--------- .../MolecularReplacement.h | 20 ++-- .../structure_refinement/RugnuxModel.h | 20 ++++ receiver/JFJochReceiverRugnux.cpp | 8 +- receiver/JFJochReceiverRugnux.h | 4 +- receiver/JFJochReceiverService.cpp | 6 +- receiver/JFJochReceiverService.h | 6 +- rugnux/ResultReport.cpp | 4 +- rugnux/Rugnux.cpp | 4 +- rugnux/Rugnux.h | 8 +- rugnux/RugnuxCommandLine.cpp | 4 +- rugnux/RugnuxLive.cpp | 4 +- rugnux/RugnuxLive.h | 2 +- rugnux/RugnuxScaleMerge.cpp | 20 ++-- rugnux/rugnux_cli.cpp | 17 +++- tests/ModelScaleGPUTest.cpp | 3 +- tests/ModelValidationTest.cpp | 79 ++++++++------- tests/MolecularReplacementTest.cpp | 32 +++--- tests/ResultReportTest.cpp | 6 +- tests/RugnuxTest.cpp | 4 +- viewer/windows/JFJochProcessingJobsWindow.cpp | 15 ++- writer/CMakeLists.txt | 8 +- writer/RugnuxModelFile.cpp | 98 +++++++++++++++++++ writer/RugnuxModelFile.h | 23 +++++ 32 files changed, 352 insertions(+), 242 deletions(-) create mode 100644 image_analysis/structure_refinement/RugnuxModel.h create mode 100644 writer/RugnuxModelFile.cpp create mode 100644 writer/RugnuxModelFile.h diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index 88cd13b6c..f0b638ba3 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include "../preview/JFJochTIFF.h" #include "OpenAPIConvert.h" #include "../image_pusher/LiveCollection.h" +#include "../writer/RugnuxModelFile.h" #include #include "gen/model/Error_message.h" @@ -978,9 +978,9 @@ void JFJochBrokerHttp::config_rugnux_get(httplib::Response &response) { ret.setEnable(s.enable); ret.setReservedCpus(s.reserved_cpus); ret.setDeadlineS(s.deadline_s); - ret.setModelLoaded(!s.model_path.empty()); - if (!s.model_path.empty()) - ret.setModelSizeBytes(static_cast(std::filesystem::file_size(s.model_path))); + ret.setModelLoaded(s.model != nullptr); + if (s.model) + ret.setModelSizeBytes(s.model_size_bytes); ProcessOutput(ret, response); } @@ -1003,13 +1003,21 @@ void JFJochBrokerHttp::config_rugnux_model_put(const httplib::Request &request, response.set_content("Model cannot be larger than 64 MiB", "text/plain"); return; } - state_machine.SetRugnuxModel(request.body); + std::shared_ptr model; + try { + model = std::make_shared(ParseRugnuxModel(request.body, "model")); + } catch (const std::exception &e) { + response.status = 400; + response.set_content(std::string("Not an atomic model: ") + e.what(), "text/plain"); + return; + } + state_machine.SetRugnuxModel(std::move(model), static_cast(request.body.size())); logger.Info("Rugnux model uploaded ({} bytes)", request.body.size()); response.status = 200; } void JFJochBrokerHttp::config_rugnux_model_delete(httplib::Response &response) { - state_machine.SetRugnuxModel(""); + state_machine.SetRugnuxModel(nullptr, 0); logger.Info("Rugnux model removed"); response.status = 200; } diff --git a/broker/JFJochServices.cpp b/broker/JFJochServices.cpp index 8bd4ff15c..31dfaee5f 100644 --- a/broker/JFJochServices.cpp +++ b/broker/JFJochServices.cpp @@ -3,9 +3,6 @@ #include "JFJochServices.h" -#include -#include -#include #include "../common/JFJochException.h" #include "../detector_control/SLSDetectorWrapper.h" #include "../detector_control/DectrisDetectorWrapper.h" @@ -376,7 +373,7 @@ void JFJochServices::SetupIndexing(const IndexingSettings &input) { void JFJochServices::SetRugnuxLive(const RugnuxLiveSettings &input) { if (receiver) - receiver->RugnuxLive(input.enable, input.model_path, input.reserved_cpus, input.deadline_s); + receiver->RugnuxLive(input.enable, input.model, input.reserved_cpus, input.deadline_s); rugnux = input; } @@ -390,32 +387,10 @@ RugnuxLiveSnapshot JFJochServices::GetRugnuxStatus() const { return {}; } -void JFJochServices::SetRugnuxModel(const std::string &model) { +void JFJochServices::SetRugnuxModel(std::shared_ptr model, int64_t size_bytes) { RugnuxLiveSettings s = rugnux; - if (!s.model_path.empty()) - std::filesystem::remove(s.model_path); - s.model_path.clear(); - - if (!model.empty()) { - if (rugnux_model_dir.empty()) { - char dir[] = "/tmp/jfjoch_rugnux_XXXXXX"; // mkdtemp creates it 0700 - if (mkdtemp(dir) == nullptr) - throw JFJochException(JFJochExceptionCategory::FileWriteError, - "Cannot create directory for the rugnux model"); - rugnux_model_dir = dir; - } - // gemmi detects PDB/mmCIF from the content, but gzip only from the extension - const bool gzipped = model.size() >= 2 && model[0] == '\x1f' && model[1] == '\x8b'; - s.model_path = rugnux_model_dir + (gzipped ? "/model.gz" : "/model"); - { - std::ofstream f(s.model_path, std::ios::binary); - f.write(model.data(), static_cast(model.size())); - if (!f) - throw JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot write the rugnux model"); - } - std::filesystem::permissions(s.model_path, - std::filesystem::perms::owner_read | std::filesystem::perms::owner_write); - } + s.model = std::move(model); + s.model_size_bytes = s.model ? size_bytes : 0; SetRugnuxLive(s); } diff --git a/broker/JFJochServices.h b/broker/JFJochServices.h index 3a732c20d..c232ff5b9 100644 --- a/broker/JFJochServices.h +++ b/broker/JFJochServices.h @@ -18,7 +18,8 @@ struct RugnuxLiveSettings { bool enable = false; int64_t reserved_cpus = 4; double deadline_s = 0; // the analysis is cut this long after the last image (0 = never) - std::string model_path; // empty = no model + std::shared_ptr model; // null = no model + int64_t model_size_bytes = 0; // as uploaded }; class JFJochServices { @@ -31,7 +32,6 @@ class JFJochServices { std::atomic cannot_stop_detector = false; RugnuxLiveSettings rugnux; - std::string rugnux_model_dir; // private directory, created at the first model upload Logger &logger; public: explicit JFJochServices(Logger &in_logger); @@ -89,7 +89,7 @@ public: void SetRugnuxLive(const RugnuxLiveSettings &input); RugnuxLiveSettings GetRugnuxLive() const; // The model file is written 0600 into a private directory; an empty model removes it - void SetRugnuxModel(const std::string &model); + void SetRugnuxModel(std::shared_ptr model, int64_t size_bytes); RugnuxLiveSnapshot GetRugnuxStatus() const; ImagePusherStatus GetImagePusherStatus() const; diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index 47dacea7f..97c880ded 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -1221,12 +1221,12 @@ void JFJochStateMachine::SetRugnuxLive(bool enable, int64_t reserved_cpus, doubl services.SetRugnuxLive(s); } -void JFJochStateMachine::SetRugnuxModel(const std::string &model) { +void JFJochStateMachine::SetRugnuxModel(std::shared_ptr model, int64_t size_bytes) { std::unique_lock ul(m); if (IsRunning()) throw WrongDAQStateException("Cannot change rugnux model during data collection"); std::unique_lock ul2(rugnux_mutex); - services.SetRugnuxModel(model); + services.SetRugnuxModel(std::move(model), size_bytes); } RugnuxLiveSettings JFJochStateMachine::GetRugnuxLive() const { diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index ca053658a..ed494dcaf 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -249,7 +249,7 @@ public: IndexingSettings GetIndexingSettings() const; void SetRugnuxLive(bool enable, int64_t reserved_cpus, double deadline_s); - void SetRugnuxModel(const std::string &model); // empty = remove the model + void SetRugnuxModel(std::shared_ptr model, int64_t size_bytes); // null = remove the model RugnuxLiveSettings GetRugnuxLive() const; RugnuxLiveSnapshot GetRugnuxStatus() const; diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index aacd1211b..0644f809d 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -5,6 +5,7 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC MolecularReplacement.cpp MolecularReplacement.h $<$:MolecularReplacementGPU.cu> MolecularReplacementGPU.h ModelValidation.cpp ModelValidation.h + RugnuxModel.h RigidBodyRefine.cpp RigidBodyRefine.h $<$:RigidBodyGPU.cpp> $<$:RigidBodyGPU.cu> diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index a79dd085d..0497730c2 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -21,9 +21,7 @@ #include #include -#include // read_structure_gz #include // expand_ncs -#include // MaybeGzipped #include // IT92 x-ray form factors #include // DensityCalculator #include // get_size_for_hkl, get_f_phi_on_grid (the maps) @@ -537,7 +535,7 @@ namespace { // validation either way. ModelValidationResult Validate(const std::vector &merged, const UnitCell &cell, - const std::string &model_path, + const RugnuxModel &model, const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group, @@ -548,32 +546,29 @@ ModelValidationResult Validate(const std::vector &merged, const ModelValidationSchedule &schedule, const gemmi::Op *twin_law, bool rigid_body_gpu, - const gemmi::Structure *model) { + const gemmi::Structure *placed) { ModelValidationResult result; - result.model_path = model_path; + result.model_name = model.name; - // --- read the atomic model --- + // --- the atomic model --- ModelState mdl; gemmi::Structure &st = mdl.st; // the real model, the one the maps and the report describe try { - // Detect, not the default: without it GEMMI picks the format from the extension and only - // falls back to the content when it does not recognise one. A model arrives named however - // whoever produced it named it, so the file itself is the better authority. - st = model != nullptr ? *model : gemmi::read_structure_gz(model_path, gemmi::CoorFormat::Detect); + st = placed != nullptr ? *placed : model.structure; expand_strict_ncs(st); } catch (const std::exception &e) { - result.failure_reason = fmt::format("cannot read model {}: {}", model_path, e.what()); + result.failure_reason = fmt::format("cannot read model {}: {}", model.name, e.what()); logger.Error("Model validation: {}", result.failure_reason); return result; } if (st.models.empty() || !st.cell.is_crystal()) { - result.failure_reason = fmt::format("model {} has no atoms or no unit cell", model_path); + result.failure_reason = fmt::format("model {} has no atoms or no unit cell", model.name); logger.Error("Model validation: {}", result.failure_reason); return result; } const gemmi::SpaceGroup *sg = st.find_spacegroup(); if (!sg) { - result.failure_reason = fmt::format("model {} has no usable space group", model_path); + result.failure_reason = fmt::format("model {} has no usable space group", model.name); logger.Error("Model validation: {}", result.failure_reason); return result; } @@ -659,7 +654,7 @@ ModelValidationResult Validate(const std::vector &merged, // and their maps and reflection files can be compared directly. Moving the model would leave // each dataset in whichever indexing its own run happened to pick. // A model handed over already placed in the data's cell (molecular replacement) is on the data's axes. - if (data_cell.is_crystal() && model == nullptr) { + if (data_cell.is_crystal() && placed == nullptr) { const gemmi::GroupOps model_gops = sg->operations(); const bool same_point_group = data_space_group != nullptr && same_rotations(data_space_group->operations(), model_gops); @@ -1921,7 +1916,7 @@ ModelValidationGPUPlan PlanModelValidationGPU(const ModelValidationGPUNeed &need ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, - const std::string &model_path, + const RugnuxModel &model, const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group, @@ -1931,18 +1926,18 @@ ModelValidationResult ValidateAgainstModel(const std::vector & const std::vector &report_shell_d_min, const ModelValidationSchedule &schedule, const gemmi::Op *twin_law, - const gemmi::Structure *model) { + const gemmi::Structure *placed) { #ifdef JFJOCH_USE_CUDA // A CUDA failure ends the validation, not the run, and nothing is moved to the CPU part way through: // the validation reports why it did not finish, and a validation that did not finish decides nothing, // so the reflection files are those of a run without a model. try { - return Validate(merged, cell, model_path, output_prefix, logger, data_space_group, + return Validate(merged, cell, model, output_prefix, logger, data_space_group, probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, - true, model); + true, placed); } catch (const RigidBodyGPUFailure &e) { ModelValidationResult failed; - failed.model_path = model_path; + failed.model_name = model.name; failed.failure_reason = fmt::format("the GPU failed during the validation ({})", e.what()); logger.Error("Model validation: {}", failed.failure_reason); cuda_throw_if_context_lost(); // a lost device ends the run here, not at its next user @@ -1950,8 +1945,8 @@ ModelValidationResult ValidateAgainstModel(const std::vector & return failed; } #else - return Validate(merged, cell, model_path, output_prefix, logger, data_space_group, probe_indexing_ambiguity, - nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, false, model); + return Validate(merged, cell, model, output_prefix, logger, data_space_group, probe_indexing_ambiguity, + nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, false, placed); #endif } @@ -2107,7 +2102,7 @@ void KeepModelVerdict(ModelValidationResult &remade, const ModelValidationResult remade.model_space_group_number = decided.model_space_group_number; } -std::vector ModelReferenceIntensities(const std::string &model_path, +std::vector ModelReferenceIntensities(const RugnuxModel &model, const std::optional &cell, const gemmi::SpaceGroup *space_group, double d_min, @@ -2118,19 +2113,15 @@ std::vector ModelReferenceIntensities(const std::string &model return out; } - gemmi::Structure st; + gemmi::Structure st = model.structure; try { - // Detect, not the default: without it GEMMI picks the format from the extension and only - // falls back to the content when it does not recognise one. A model arrives named however - // whoever produced it named it, so the file itself is the better authority. - st = gemmi::read_structure_gz(model_path, gemmi::CoorFormat::Detect); expand_strict_ncs(st); } catch (const std::exception &e) { - logger.Error("Model reference: cannot read model {}: {}", model_path, e.what()); + logger.Error("Model reference: cannot read model {}: {}", model.name, e.what()); return out; } if (st.models.empty() || !st.cell.is_crystal()) { - logger.Error("Model reference: model {} has no atoms or no unit cell", model_path); + logger.Error("Model reference: model {} has no atoms or no unit cell", model.name); return out; } @@ -2153,7 +2144,7 @@ std::vector ModelReferenceIntensities(const std::string &model st.spacegroup_hm = space_group->xhm(); const gemmi::SpaceGroup *sg = st.find_spacegroup(); if (!sg) { - logger.Error("Model reference: model {} has no usable space group", model_path); + logger.Error("Model reference: model {} has no usable space group", model.name); return out; } st.setup_cell_images(); @@ -2203,6 +2194,6 @@ std::vector ModelReferenceIntensities(const std::string &model } logger.Info("Model reference: {} intensities computed from {} to {:.2f} A, space group {}", - out.size(), model_path, d_min, sg->short_name()); + out.size(), model.name, d_min, sg->short_name()); return out; } diff --git a/image_analysis/structure_refinement/ModelValidation.h b/image_analysis/structure_refinement/ModelValidation.h index 4bc05dff7..594c3fb65 100644 --- a/image_analysis/structure_refinement/ModelValidation.h +++ b/image_analysis/structure_refinement/ModelValidation.h @@ -15,6 +15,7 @@ #include "../../common/Reflection.h" // MergedReflection #include "../../common/UnitCell.h" +#include "RugnuxModel.h" class Logger; namespace gemmi { struct Structure; } @@ -22,7 +23,7 @@ namespace gemmi { struct Structure; } // Result of validating merged intensities against an atomic model. struct ModelValidationResult { bool ok = false; - std::string model_path; // the file that was asked for, so the report can name it + std::string model_name; // RugnuxModel::name, so the report can name the model that was asked for // Why it did not run, when ok is false: the file could not be read, had no atoms, no cell or no // usable space group. A caller that asked for a model must be able to say so rather than finish // quietly with no R-free and no maps, which reads exactly like a run that was never asked. @@ -208,8 +209,7 @@ struct ModelValidationResult { std::shared_ptr placed_model; }; -// Given merged intensities and an atomic model (PDB or mmCIF, gzipped or not - the format is taken -// from the file's content), scale the model structure factors (with an +// Given merged intensities and an atomic model, scale the model structure factors (with an // optimized flat bulk-solvent contribution and an overall anisotropic B) to the WORKING observed // amplitudes, then report R-work / R-free and write 2Fo-Fc and Fo-Fc electron-density maps // (CCP4) plus an MTZ of map coefficients next to output_prefix. Nothing the reported R-free is @@ -242,8 +242,8 @@ struct ModelValidationResult { // twin_law, where given, is a twin law of the data's group (a Miller-index operator in the data's // basis): the model is then also scored against the data as twinned by it (twin_target and below). // -// model, where given, is the model itself - one already placed in the data's cell, by molecular -// replacement - and model_path only names it; no other setting is then probed for it. +// placed, where given, is the model itself - one already placed in the data's cell, by molecular +// replacement - and `model` only names it; no other setting is then probed for it. // // report_shell_d_min is the merge statistics' own shell bounds, coarse to fine, and it is what // cc_model_shells is binned on. Sharing the grid is the point: a reader has to be able to put a @@ -329,7 +329,7 @@ ModelValidationGPUPlan PlanModelValidationGPU(const ModelValidationGPUNeed &need ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, - const std::string &model_path, + const RugnuxModel &model, const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group = nullptr, @@ -339,7 +339,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & const std::vector &report_shell_d_min = {}, const ModelValidationSchedule &schedule = {}, const gemmi::Op *twin_law = nullptr, - const gemmi::Structure *model = nullptr); + const gemmi::Structure *placed = nullptr); // Reindex `merged` into the frame ValidateAgainstModel reported, so the reflection files that are // written describe the same indexing as the R-factors and the maps. Returns the space group they are @@ -378,7 +378,7 @@ ModelValidationResult ReadModelAnomalousMap(const std::vector // serial-stills run: there the ambiguity has to be broken per image, at integration time, long before // there is a merge to fit a model to. `cell` and `space_group` override the model's own, where // the run already knows them. Empty on any failure (which is logged). -std::vector ModelReferenceIntensities(const std::string &model_path, +std::vector ModelReferenceIntensities(const RugnuxModel &model, const std::optional &cell, const gemmi::SpaceGroup *space_group, double d_min, diff --git a/image_analysis/structure_refinement/MolecularReplacement.cpp b/image_analysis/structure_refinement/MolecularReplacement.cpp index 47d165261..fdb9d6f15 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.cpp +++ b/image_analysis/structure_refinement/MolecularReplacement.cpp @@ -18,15 +18,11 @@ #include #include #include -#include #include -#include -#include // read_structure_gz #include // remove_hydrogens, remove_alternative_conformations #include // setup_entities, remove_ligands_and_waters -#include // read_cif_gz #include // DensityCalculator #include // IT92 x-ray form factors #include // Scaling (bulk solvent + anisotropic B) @@ -892,19 +888,10 @@ void finish_search_model(SearchModel &m) { } // namespace -SearchModel PrepareSearchModel(const std::string &path) { +SearchModel PrepareSearchModel(const RugnuxModel &model_in) { SearchModel out; - bool modelcif = false; - try { - out.st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect); - if (out.st.input_format == gemmi::CoorFormat::Mmcif) { - const gemmi::cif::Document doc = gemmi::read_cif_gz(path); - modelcif = !doc.blocks.empty() && doc.blocks[0].has_mmcif_category("_ma_qa_metric"); - } - } catch (const std::exception &e) { - out.failure_reason = std::string("cannot read the model: ") + e.what(); - return out; - } + const bool modelcif = model_in.modelcif; + out.st = model_in.structure; if (out.st.models.empty()) { out.failure_reason = "the model has no atoms"; return out; @@ -978,39 +965,15 @@ SearchModel PrepareSearchModel(const std::string &path) { // --- domains from the predicted aligned error --- -std::string PaePathFor(const std::string &model_path) { - const size_t slash = model_path.find_last_of("/\\"); - const std::string dir = slash == std::string::npos ? std::string() : model_path.substr(0, slash + 1); - const std::string name = model_path.substr(dir.size()); - const size_t at = name.rfind("-model_v"); - if (at == std::string::npos) - return {}; - const size_t dot = name.find('.', at); - const std::string version = name.substr(at + 8, dot == std::string::npos ? std::string::npos : dot - at - 8); - return dir + name.substr(0, at) + "-predicted_aligned_error_v" + version + ".json"; -} - // A prediction cut into domains by its predicted aligned error before molecular replacement, as Oeffner et // al. (2022) Acta Cryst. D78, 1303-1314 do; here by average linkage. -std::vector> PaeDomains(const std::string &pae_path, const SearchModel &model) { +std::vector> PaeDomains(const std::vector> &pae, const SearchModel &model) { // Two domains a prediction does not place relative to each other with confidence are this far apart in // mean PAE (within one domain it is a few angstroms); a domain is at least this many residues. constexpr double PAE_DOMAIN_A = 10.0; constexpr int DOMAIN_MIN_RESIDUES = 40; // The clustering starts from runs of this many consecutive residues kept. constexpr int PIECE = 5; - std::vector> pae; - try { - std::ifstream in(pae_path); - if (!in) - return {}; - const nlohmann::json j = nlohmann::json::parse(in); - const nlohmann::json &m = j.is_array() ? j.at(0).at("predicted_aligned_error") : j.at("predicted_aligned_error"); - for (const auto &row : m) - pae.push_back(row.get>()); - } catch (const std::exception &) { - return {}; - } const size_t n = pae.size(); if (n != static_cast(model.residues_read) || std::any_of(pae.begin(), pae.end(), [n](const std::vector &r) { return r.size() != n; })) @@ -3323,7 +3286,7 @@ MolecularReplacementGPUPlan PlanMolecularReplacementGPU(const MolecularReplaceme return plan; } -MolecularReplacementResult RunMolecularReplacement(const std::string &model_path, const std::vector &merges, +MolecularReplacementResult RunMolecularReplacement(const RugnuxModel &input, const std::vector &merges, Logger &logger, size_t nthreads, size_t gpu_resident_bytes) { const auto t0 = std::chrono::steady_clock::now(); // What the search may hold on the card: the budget model validation keeps to. What the stream-ordered pool @@ -3338,25 +3301,24 @@ MolecularReplacementResult RunMolecularReplacement(const std::string &model_path #else (void) gpu_resident_bytes; #endif - SearchModel model = PrepareSearchModel(model_path); + SearchModel model = PrepareSearchModel(input); if (!model.failure_reason.empty()) { MolecularReplacementResult result; - result.model_path = model_path; + result.model_name = input.name; result.residues_read = model.residues_read; result.failure_reason = model.failure_reason; return result; } model.description = "the whole model"; // Only a prediction has a predicted aligned error. - const std::string pae_path = model.predicted ? PaePathFor(model_path) : std::string(); - const std::vector> domains = pae_path.empty() ? std::vector>() - : PaeDomains(pae_path, model); + const bool pae = model.predicted && input.pae.has_value(); + const std::vector> domains = pae ? PaeDomains(*input.pae, model) : std::vector>(); const double t_model = seconds_since(t0); - if (!pae_path.empty()) - logger.Info("Molecular replacement: {} rigid domain(s) in the predicted aligned error {}", domains.size(), pae_path); + if (pae) + logger.Info("Molecular replacement: {} rigid domain(s) in the predicted aligned error", domains.size()); MolecularReplacementResult result = search_model(model, merges, logger, nthreads, gpu_budget); - result.model_path = model_path; + result.model_name = input.name; result.pae_domains = static_cast(domains.size()); result.t_model += t_model; // Where the whole model is placed nowhere and its prediction falls apart into domains whose relative @@ -3403,7 +3365,7 @@ MolecularReplacementResult RunMolecularReplacement(const std::string &model_path found[best].candidates[found[best].chosen].space_group.xhm()); } else if (best >= 0 && agree) { MolecularReplacementResult chosen = std::move(found[best]); - chosen.model_path = model_path; + chosen.model_name = input.name; chosen.pae_domains = result.pae_domains; chosen.t_model = result.t_model; chosen.t_transform = result.t_transform; diff --git a/image_analysis/structure_refinement/MolecularReplacement.h b/image_analysis/structure_refinement/MolecularReplacement.h index ea163349a..29105a330 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.h +++ b/image_analysis/structure_refinement/MolecularReplacement.h @@ -35,6 +35,7 @@ #include "../../common/Reflection.h" // MergedReflection #include "../../common/UnitCell.h" +#include "RugnuxModel.h" class Logger; @@ -55,19 +56,16 @@ struct SearchModel { }; // A prediction where the file says so - ModelCIF's _ma_qa_metric - or where it has no cell and its B column // holds one value per residue, from 0 to 100; an experimental structure otherwise. -SearchModel PrepareSearchModel(const std::string &path); +SearchModel PrepareSearchModel(const RugnuxModel &model); -// The rigid domains of a prediction from its predicted aligned error (PAE, the AlphaFold DB's JSON: a matrix +// The rigid domains of a prediction from its predicted aligned error (PAE, the AlphaFold DB's matrix // "predicted_aligned_error" over the residues of the whole prediction): the kept residues clustered by average // linkage on the symmetrised PAE until every two clusters are a mean PAE_DOMAIN_A apart; clusters of at least -// DOMAIN_MIN_RESIDUES, largest first, each a list of positions in the prediction. Empty where the file cannot -// be read or does not match the model. -std::vector> PaeDomains(const std::string &pae_path, const SearchModel &model); +// DOMAIN_MIN_RESIDUES, largest first, each a list of positions in the prediction. Empty where the matrix does +// not match the model. +std::vector> PaeDomains(const std::vector> &pae, const SearchModel &model); // The search model of one domain: the residues of `model` at those positions, recentred. SearchModel DomainModel(const SearchModel &model, const std::vector &residues); -// The PAE file the AlphaFold DB puts beside a model: AF--F1-model_v.cif -> -// AF--F1-predicted_aligned_error_v.json in the same directory; empty where the name does not follow it. -std::string PaePathFor(const std::string &model_path); // Rotations covering SO(3) with neighbours about `step_deg` apart: the identity, then shells of // rotation angle kappa = step, 2 step, ... 180 deg, each with its axes on a golden-spiral lattice of @@ -341,7 +339,7 @@ struct MRCandidate { struct MolecularReplacementResult { bool ok = false; std::string failure_reason; - std::string model_path; + std::string model_name; // RugnuxModel::name int residues_read = 0, residues_kept = 0; double mass_da = 0; std::string search_model; // the whole model, or the PAE domain the decision was made with @@ -405,7 +403,7 @@ struct MolecularReplacementGPUPlan { // to be done rather than spilling past it. MolecularReplacementGPUPlan PlanMolecularReplacementGPU(const MolecularReplacementGPUNeed &need); -// Search `merges` (the answer's first) for the predicted model at `model_path`, in every space group with +// Search `merges` (the answer's first) for the predicted `model`, in every space group with // the rotations of each merge's group - one copy, then further copies one at a time while each new one // stands out, packs and lowers R-free - and decide. A candidate is PLACED where a copy's translation // function stands out (TF Z >= 8), the copies pack, one copy fits the cell, and its R-work beats its own @@ -413,7 +411,7 @@ MolecularReplacementGPUPlan PlanMolecularReplacementGPU(const MolecularReplaceme // every other by an R-free margin; it chooses the twin hypothesis' merge only where none of the // answer's groups can hold the model at all (Matthews, packing) and a group of the subgroup can. // `gpu_resident_bytes`: what is held on the card meanwhile by what came before (a merge engine still at work). -MolecularReplacementResult RunMolecularReplacement(const std::string &model_path, const std::vector &merges, +MolecularReplacementResult RunMolecularReplacement(const RugnuxModel &model, const std::vector &merges, Logger &logger, size_t nthreads, size_t gpu_resident_bytes = 0); // The hand check of an ENANTIOMORPH decision, filled into result.hand_check: taken into the answer by diff --git a/image_analysis/structure_refinement/RugnuxModel.h b/image_analysis/structure_refinement/RugnuxModel.h new file mode 100644 index 000000000..92253b2f7 --- /dev/null +++ b/image_analysis/structure_refinement/RugnuxModel.h @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +#include "gemmi/model.hpp" + +// An atomic model handed to a run (rugnux --model / --mr, the broker's upload), as its file held it. The +// caller parses it (ParseRugnuxModel, writer/RugnuxModelFile.h); the analysis reads no file. +struct RugnuxModel { + std::string name; // what the report calls it: the file it was given as + gemmi::Structure structure; // as read, input_format kept + bool modelcif = false; // a ModelCIF file (_ma_qa_metric): a prediction, whatever its B column holds + // A prediction's predicted aligned error over the residues of the whole prediction, where it came with one. + std::optional>> pae; +}; diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index 2254855ca..ec3084643 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -86,7 +86,7 @@ JFJochReceiverRugnux::JFJochReceiverRugnux(const DiffractionExperiment &in_exper Logger &in_logger, int64_t in_forward_nthreads, int in_rugnux_nthreads, - const std::string &in_model_path, + std::shared_ptr in_model, double in_deadline_s, RugnuxLiveStatus &in_live_status, const SpotFindingSettings &in_spot_finding_settings, @@ -102,7 +102,7 @@ JFJochReceiverRugnux::JFJochReceiverRugnux(const DiffractionExperiment &in_exper reader(in_image_buffer), forward_nthreads(JFJochReceiverLite::NumberOfDataAnalysisThreads(in_forward_nthreads, in_experiment)), rugnux_nthreads(in_rugnux_nthreads), - model_path(in_model_path), + model(std::move(in_model)), deadline_s(in_deadline_s), live_status(in_live_status), measurement_started(1), @@ -246,7 +246,7 @@ void JFJochReceiverRugnux::RugnuxThread() { parallel_detail::WorkerPool::Instance().SetNice(10); try { const auto dataset = reader.GetDataset(); - const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, scratch_dir + "/p", model_path); + const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, scratch_dir + "/p", model); LiveObserver observer(plots, scan_result, *az_int_mapping, live_status, logger); ProcessResult result; { @@ -288,7 +288,7 @@ void JFJochReceiverRugnux::ProbeThread() { const int64_t frames = std::lround(PROBE_DEG / std::abs(gonio->GetIncrement_deg())); if (2 * frames > static_cast(reader.GetNumberOfImages())) return; // a sweep this short is decided on soon enough by the run itself - RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, "", ""); + RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, "", nullptr); setup.config.end_image = static_cast(frames); setup.config.rotation_postrefine_geometry = false; setup.config.run_scaling = false; diff --git a/receiver/JFJochReceiverRugnux.h b/receiver/JFJochReceiverRugnux.h index 1e480e4df..22a3e9742 100644 --- a/receiver/JFJochReceiverRugnux.h +++ b/receiver/JFJochReceiverRugnux.h @@ -29,7 +29,7 @@ class JFJochReceiverRugnux : public JFJochReceiver { const int64_t forward_nthreads; const int rugnux_nthreads; - const std::string model_path; + const std::shared_ptr model; const double deadline_s; // after the last image, the analysis is cut (0 = never) std::string scratch_dir; // rugnux's own output, read back into the end message RugnuxLiveStatus &live_status; @@ -66,7 +66,7 @@ public: Logger &logger, int64_t forward_nthreads, int rugnux_nthreads, - const std::string &model_path, + std::shared_ptr model, double deadline_s, RugnuxLiveStatus &live_status, const SpotFindingSettings &spot_finding_settings, diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index ce0c633c3..36a65245d 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -20,14 +20,14 @@ JFJochReceiverService::JFJochReceiverService(AcquisitionDeviceGroup &in_aq_devic spot_finding_settings(DiffractionExperiment::DefaultDataProcessingSettings()) { } -JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, const std::string &model_path, +JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::shared_ptr model, int reserved_cpus, double deadline_s) { std::unique_lock ul_state(state_mutex); if (state != ReceiverState::Idle) throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Cannot change rugnux settings during data collection"); rugnux_enable = enable; - rugnux_model_path = model_path; + rugnux_model = std::move(model); rugnux_reserved_cpus = reserved_cpus; rugnux_deadline_s = deadline_s; return *this; @@ -127,7 +127,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, receiver = std::make_unique( experiment, pixel_mask, *image_puller, image_pusher, logger, nthreads_local, std::max(1, static_cast(AvailableCpus()) - rugnux_reserved_cpus), - rugnux_model_path, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots, image_buffer, + rugnux_model, rugnux_deadline_s, rugnux_status, spot_finding_settings, receiver_status, plots, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get()); break; } diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index 57a9985a5..1eb6d4df6 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -14,6 +14,8 @@ #include "../preview/ZMQMetadataSocket.h" #include "../preview/ZMQPreviewSocket.h" +struct RugnuxModel; + class JFJochReceiverService { std::unique_ptr receiver; AcquisitionDeviceGroup &aq_devices; @@ -26,7 +28,7 @@ class JFJochReceiverService { // Rotation collections from a DECTRIS detector analysed by rugnux while they arrive // (JFJochReceiverRugnux), instead of image by image. Off unless asked for. bool rugnux_enable = false; - std::string rugnux_model_path; + std::shared_ptr rugnux_model; int rugnux_reserved_cpus = 4; // left to the forward threads and the pusher double rugnux_deadline_s = 0; // the analysis is cut this long after the last image (0 = never) RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends @@ -53,7 +55,7 @@ public: ImagePusher &pusher, size_t send_buffer_size_MiB = 512); JFJochReceiverService& NumThreads(int64_t input); - JFJochReceiverService& RugnuxLive(bool enable, const std::string &model_path, int reserved_cpus, + JFJochReceiverService& RugnuxLive(bool enable, std::shared_ptr model, int reserved_cpus, double deadline_s); RugnuxLiveSnapshot GetRugnuxStatus() const; JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); diff --git a/rugnux/ResultReport.cpp b/rugnux/ResultReport.cpp index bda044f24..9e56e7da6 100644 --- a/rugnux/ResultReport.cpp +++ b/rugnux/ResultReport.cpp @@ -2329,7 +2329,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, ModelValidationResult unplaced; if (!result.model_validation) { const auto &mp = *result.model_prior; - unplaced.model_path = mp.model_path; + unplaced.model_name = mp.model_name; unplaced.failure_reason = mp.ok ? "molecular replacement did not place the model" : "the model could not be searched for: " + mp.failure_reason; } @@ -2338,7 +2338,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, const bool mr_hand = by_mr && result.model_prior->hand_check.adopted; ReportSection s; s.title = "5. MODEL VALIDATION"; - Add(s, KeyText("MODEL_FILE", mv.model_path)); + Add(s, KeyText("MODEL_FILE", mv.model_name)); if (!result.model_placement.empty()) Add(s, KeyEnum("MODEL_PLACEMENT", result.model_placement, {"RIGID_BODY", "MR", "NOT_PLACED"})); if (!mv.ok) { diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index a7c231288..23c1dfedb 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -967,12 +967,12 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b std::vector model_reference; if (!config_.reference_data.empty()) { indexer->ReferenceIntensities(config_.reference_data); - } else if (!config_.model_path.empty() && !experiment_.IsRotationIndexing() + } else if (config_.model && !experiment_.IsRotationIndexing() && experiment_.GetUnitCell().has_value() && experiment_.GetGemmiSpaceGroup().has_value()) { const auto d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A() .value_or(experiment_.GetDetectorMaxResolution_A()); model_reference = ModelReferenceIntensities( - config_.model_path, experiment_.GetUnitCell(), + *config_.model, experiment_.GetUnitCell(), &experiment_.GetSpaceGroupOrP1(), d_min, logger); if (!model_reference.empty()) { indexer->ReferenceIntensities(model_reference); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index afbdca2aa..c70acc7da 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -207,10 +207,10 @@ struct ProcessConfig { // run can write both. bool export_unmerged_partials = false; - // If set, after merging, validate the merged intensities against this atomic model (PDB or - // mmCIF, gzipped or not): compute R-free with an optimized bulk solvent and write 2Fo-Fc / - // Fo-Fc maps (--model). - std::string model_path; + // If set, after merging, validate the merged intensities against this atomic model: compute + // R-free with an optimized bulk solvent and write 2Fo-Fc / Fo-Fc maps (--model). The caller parses + // it (ParseRugnuxModel, writer/RugnuxModelFile.h). + std::shared_ptr model; // --mr: the model is searched for by molecular replacement straight away, without the rigid-body // placement first. Without it a --model that is a prediction, or an experimental structure the rigid body diff --git a/rugnux/RugnuxCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp index 1de2ddb02..4dd6d6dc7 100644 --- a/rugnux/RugnuxCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -82,8 +82,8 @@ std::string RugnuxCommandLine(const ProcessConfig &config, if (!config.reference_column.empty()) add("--reference-column", config.reference_column); } - if (!config.model_path.empty()) - add(config.force_mr ? "--mr" : "--model", config.model_path); + if (config.model) + add(config.force_mr ? "--mr" : "--model", config.model->name); } if (config.start_image != 0) add("-s", std::to_string(config.start_image)); diff --git a/rugnux/RugnuxLive.cpp b/rugnux/RugnuxLive.cpp index afd1bc3f1..b05cf60b8 100644 --- a/rugnux/RugnuxLive.cpp +++ b/rugnux/RugnuxLive.cpp @@ -6,7 +6,7 @@ #include "RugnuxDefaults.h" RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, - const std::string &output_prefix, const std::string &model_path) { + const std::string &output_prefix, std::shared_ptr model) { RugnuxLiveSetup s; // The experiment as rugnux_cli.cpp sets it up for a rotation sweep with no options given. @@ -48,7 +48,7 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr c.beam_center_check = true; c.measure_spot_budget = true; c.run_scaling = true; - c.model_path = model_path; + c.model = std::move(model); c.live = true; c.fast_live = true; // Decided late, on the whole sweep, as it re-checks its radius there - which fast_live does not. diff --git a/rugnux/RugnuxLive.h b/rugnux/RugnuxLive.h index c4aba1297..c1e0820e8 100644 --- a/rugnux/RugnuxLive.h +++ b/rugnux/RugnuxLive.h @@ -18,4 +18,4 @@ struct RugnuxLiveSetup { }; RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, - const std::string &output_prefix, const std::string &model_path); + const std::string &output_prefix, std::shared_ptr model); diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 78ea9d081..6cb1ff152 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -1571,8 +1571,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { size_t model_rotations = 0; std::optional model_group; gemmi::UnitCell model_cell; - if (!config_.model_path.empty() && write_files && !geometry_prepass) { - const SearchModel m = PrepareSearchModel(config_.model_path); + if (config_.model && write_files && !geometry_prepass) { + const SearchModel m = PrepareSearchModel(*config_.model); model_predicted = m.failure_reason.empty() && m.predicted; if (const gemmi::SpaceGroup *sg = m.st.find_spacegroup(); sg != nullptr && m.st.cell.is_crystal()) { model_rotations = sg->operations().sym_ops.size(); @@ -3073,7 +3073,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // comparison), so with one the merge in the subgroup is made wherever a metric re-ask promoted from the // indexed lattice's group, or the model is in a group with fewer rotations than the adopted one. const bool model_tests_twin = - !config_.model_path.empty() + config_.model && (twin_metric_subgroup.has_value() || (!model_predicted && model_rotations > 0 && model_rotations < result.space_group->operations().sym_ops.size())); @@ -4000,7 +4000,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { Note("cut: the model is left out"); return true; }; - const bool model_run = result.consensus_cell && write_files && !config_.model_path.empty() && !geometry_prepass + const bool model_run = result.consensus_cell && write_files && config_.model && !geometry_prepass && !superseded && !model_cut(); // The twin hypothesis the model is asked about: its own subgroup's merge where it has one, the data's // otherwise - and whether that merge was made only for the model, so is written only where it chooses it. @@ -4096,7 +4096,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { schedule.gpu_resident_bytes = merge_on_card; schedule.gpu_beside_bytes = beside; ahead.merged = std::move(merged); - return ValidateAgainstModel(ahead.merged, cell, config_.model_path, config_.output_prefix, ahead.log, + return ValidateAgainstModel(ahead.merged, cell, *config_.model, config_.output_prefix, ahead.log, in, /*probe_indexing_ambiguity=*/false, static_cast(config_.nthreads), wavelength, report_shell_d_min, schedule); @@ -4115,7 +4115,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // validation decides: relabelling the outcomes and the group is what it must not see happen. // Every such relabelling reaches the P1 merge afterwards, through merge_to_written. ModelValidationResult validation = - ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, + ValidateAgainstModel(sm.merged, *result.consensus_cell, *config_.model, config_.output_prefix, logger, data_sg ? &*data_sg : nullptr, /*probe_indexing_ambiguity=*/config_.reference_data.empty(), static_cast(config_.nthreads), experiment_.GetWavelength_A(), @@ -4199,7 +4199,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { ModelValidationSchedule kept_verdict; kept_verdict.fit_test = false; kept_verdict.gpu_resident_bytes = merge_on_card; - remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, + remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, *config_.model, config_.output_prefix, logger, sg_now ? &*sg_now : nullptr, /*probe_indexing_ambiguity=*/false, @@ -4246,7 +4246,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { twin_schedule.maps_superseded = [&](const ModelValidationResult &v) { return !maps_from(v); }; phase("Validating against model (twin hypothesis)"); auto twin_validation = ValidateAgainstModel( - mt.merged, mt.cell, config_.model_path, config_.output_prefix, logger, + mt.merged, mt.cell, *config_.model, config_.output_prefix, logger, &mt.space_group, /*probe_indexing_ambiguity=*/config_.reference_data.empty(), static_cast(config_.nthreads), experiment_.GetWavelength_A(), {}, twin_schedule, &mt.law); @@ -4328,7 +4328,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { if (model_twin) merges.push_back({"TWIN_SUBGROUP_MERGE", &model_twin->merged, model_twin->cell, model_twin->space_group, model_twin->law}); - MolecularReplacementResult mr = RunMolecularReplacement(config_.model_path, merges, logger, + MolecularReplacementResult mr = RunMolecularReplacement(*config_.model, merges, logger, static_cast(config_.nthreads), merge_on_card); if (mr.chosen >= 0) { const MRCandidate &c = mr.candidates[mr.chosen]; @@ -4371,7 +4371,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { placed_elsewhere.gpu_resident_bytes = merge_on_card; phase("Validating against model (molecular replacement)"); ModelValidationResult v = ValidateAgainstModel( - own ? own->merged : sm.merged, cell, config_.model_path, + own ? own->merged : sm.merged, cell, *config_.model, own ? result.model_prior_file_prefix : config_.output_prefix, logger, &data_sg, /*probe_indexing_ambiguity=*/false, static_cast(config_.nthreads), experiment_.GetWavelength_A(), report_shell_d_min, placed_elsewhere, nullptr, c.placed.get()); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index e218c8a70..c77b05cb1 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -70,6 +70,7 @@ #include "RugnuxDefaults.h" #include "../image_analysis/structure_refinement/ModelValidation.h" #include "../writer/WriteModel.h" +#include "../writer/RugnuxModelFile.h" #include "ResultReport.h" #include "RugnuxConsole.h" #ifdef __APPLE__ @@ -1609,6 +1610,16 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { logger.Error("--model and --mr both name the model; give one"); exit(EXIT_FAILURE); } + // Read before anything runs: a file that holds no model ends the run here rather than after it. + std::shared_ptr model; + if (const std::string path = mr_model.empty() ? model_pdb : mr_model; !path.empty()) { + try { + model = std::make_shared(ReadRugnuxModelFile(path)); + } catch (const std::exception &e) { + logger.Error("Cannot read the model {}: {}", path, e.what()); + exit(EXIT_FAILURE); + } + } if ((indexing_algorithm == IndexingAlgorithmEnum::FFT || indexing_algorithm == IndexingAlgorithmEnum::FFBIDX) && get_gpu_count() == 0) { @@ -2300,7 +2311,7 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { // looking exactly like one that was never given --model. std::string model_validation_failure; std::optional model_validation; - if (!output_prefix.empty() && !model_pdb.empty()) { + if (!output_prefix.empty() && model) { console->OnStep("Model validation"); const auto data_sg = experiment.GetGemmiSpaceGroup(); // With a reference MTZ the merohedral indexing was already resolved (stills per-image @@ -2310,7 +2321,7 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { for (const auto &sh : merged_statistics.shells) report_shell_d_min.push_back(sh.d_min); const auto validation = - ValidateAgainstModel(merged_reflections, *experiment.GetUnitCell(), model_pdb, + ValidateAgainstModel(merged_reflections, *experiment.GetUnitCell(), *model, output_prefix, logger, data_sg ? &*data_sg : nullptr, /*probe_indexing_ambiguity=*/reference_data.empty(), @@ -3135,7 +3146,7 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { config.observation_dump_path = dump_observations; config.export_unmerged = export_unmerged; config.export_unmerged_partials = export_unmerged_partials; - config.model_path = mr_model.empty() ? model_pdb : mr_model; + config.model = model; config.force_mr = !mr_model.empty(); // When merging, the merged reflections (.mtz/.cif) are the wanted output; skip the large // _process.h5 unless explicitly requested. Without merging, the _process.h5 is the only output. diff --git a/tests/ModelScaleGPUTest.cpp b/tests/ModelScaleGPUTest.cpp index 86c167f83..90a3230b1 100644 --- a/tests/ModelScaleGPUTest.cpp +++ b/tests/ModelScaleGPUTest.cpp @@ -30,6 +30,7 @@ #include "../image_analysis/structure_refinement/ModelScaleGPU.h" #include "../image_analysis/structure_refinement/ModelValidation.h" #include "../image_analysis/structure_refinement/RigidBodyRefine.h" +#include "../writer/RugnuxModelFile.h" namespace { @@ -104,7 +105,7 @@ gemmi::Scaling ModelPoints(const char *cryst, double zone) { const std::string path = "model_scale_gpu_test.pdb"; std::ofstream(path) << pdb; Logger logger("ModelScaleGPUTest"); - const auto reference = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto reference = ModelReferenceIntensities(ReadRugnuxModelFile(path), {}, {}, 3.0, logger); gemmi::Structure st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect); std::filesystem::remove(path); st.setup_cell_images(); diff --git a/tests/ModelValidationTest.cpp b/tests/ModelValidationTest.cpp index cec04d31d..131ef6712 100644 --- a/tests/ModelValidationTest.cpp +++ b/tests/ModelValidationTest.cpp @@ -31,6 +31,7 @@ #endif #include "../image_analysis/structure_refinement/SigmaA.h" #include "../writer/WriteModel.h" +#include "../writer/RugnuxModelFile.h" #include "../image_analysis/scale_merge/ReindexAmbiguity.h" namespace { @@ -107,6 +108,10 @@ ATOM 2 C . CB GLY A 1 12.000 14.000 16.000 1.00 20.00 f.close(); return name; } + + RugnuxModel ReadModel(const std::string &path) { + return ReadRugnuxModelFile(path); + } } // --model used to call gemmi::read_pdb unconditionally, so a deposited model handed over as mmCIF - @@ -212,8 +217,8 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") { const auto pdb = WriteTemp("model_validation_test.pdb", kPdb); const auto cif = WriteTemp("model_validation_test.cif", kMmcif); - const auto from_pdb = ModelReferenceIntensities(pdb, {}, {}, 4.0, logger); - const auto from_cif = ModelReferenceIntensities(cif, {}, {}, 4.0, logger); + const auto from_pdb = ModelReferenceIntensities(ReadModel(pdb), {}, {}, 4.0, logger); + const auto from_cif = ModelReferenceIntensities(ReadModel(cif), {}, {}, 4.0, logger); REQUIRE_FALSE(from_pdb.empty()); REQUIRE_FALSE(from_cif.empty()); @@ -222,7 +227,7 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") { // The extension is not what decides: the same mmCIF under a .pdb name still reads. const auto misnamed = WriteTemp("model_validation_test_misnamed.pdb", kMmcif); - CHECK_FALSE(ModelReferenceIntensities(misnamed, {}, {}, 4.0, logger).empty()); + CHECK_FALSE(ModelReferenceIntensities(ReadModel(misnamed), {}, {}, 4.0, logger).empty()); std::filesystem::remove(pdb); std::filesystem::remove(cif); @@ -256,8 +261,8 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") { const auto ncs_path = WriteTemp("model_validation_test_ncs.pdb", with_ncs); const auto full_path = WriteTemp("model_validation_test_ncs_full.pdb", expanded); - const auto from_ncs = ModelReferenceIntensities(ncs_path, {}, {}, 4.0, logger); - const auto from_full = ModelReferenceIntensities(full_path, {}, {}, 4.0, logger); + const auto from_ncs = ModelReferenceIntensities(ReadModel(ncs_path), {}, {}, 4.0, logger); + const auto from_full = ModelReferenceIntensities(ReadModel(full_path), {}, {}, 4.0, logger); REQUIRE(from_ncs.size() == from_full.size()); REQUIRE_FALSE(from_ncs.empty()); for (size_t i = 0; i < from_ncs.size(); i++) { @@ -275,22 +280,22 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") { TEST_CASE("ModelValidation_UnusableModelGivesAReason", "[ModelValidation]") { Logger logger("ModelValidation_UnusableModelGivesAReason"); + // A file that holds no model is refused where it is read, before anything runs. const auto empty_file = WriteTemp("model_validation_test_bogus.pdb", "not a coordinate file\n"); + CHECK_THROWS(ReadModel(empty_file)); + CHECK_THROWS(ReadModel("model_validation_test_does_not_exist.pdb")); + std::filesystem::remove(empty_file); + // A model with atoms but no crystal cell to put them in. + RugnuxModel no_cell; + no_cell.name = "no_cell.pdb"; + no_cell.structure.models.emplace_back(1); const auto result = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60, .alpha = 90, .beta = 90, .gamma = 90}, - empty_file, "", logger); + no_cell, "", logger); CHECK_FALSE(result.ok); CHECK_FALSE(result.failure_reason.empty()); - CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring(empty_file)); - - const auto missing = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60, - .alpha = 90, .beta = 90, .gamma = 90}, - "model_validation_test_does_not_exist.pdb", "", logger); - CHECK_FALSE(missing.ok); - CHECK_FALSE(missing.failure_reason.empty()); - - std::filesystem::remove(empty_file); + CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring("no_cell.pdb")); } // A rigid-body step is only worth taking if it can find a shift it was not told about. The check is @@ -306,7 +311,7 @@ TEST_CASE("ModelValidation_RigidBodyRecoversASmallShift", "[ModelValidation]") { // "Observed" amplitudes: the model's own structure factors, so the target's minimum is exactly // where the model started. - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); REQUIRE_FALSE(ref.empty()); gemmi::AsuData> fobs; fobs.unit_cell_ = st.cell; @@ -353,7 +358,7 @@ TEST_CASE("ModelValidation_RigidBodyLeavesThePolarDirectionAlone", "[ModelValida REQUIRE(sg != nullptr); st.setup_cell_images(); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); REQUIRE_FALSE(ref.empty()); gemmi::AsuData> fobs; fobs.unit_cell_ = st.cell; @@ -400,7 +405,7 @@ TEST_CASE("ModelValidation_RigidBodySolventStaysPhysical", "[ModelValidation]") REQUIRE(sg != nullptr); st.setup_cell_images(); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); REQUIRE_FALSE(ref.empty()); gemmi::AsuData> fobs; fobs.unit_cell_ = st.cell; @@ -487,7 +492,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat // The data: the cluster's own |F|^2 in its own frame, 30 x 34 x 38, P 21 21 21. const auto data_model = WriteTemp("cob_data_test.pdb", ClusterPdb().c_str()); - auto obs = ModelReferenceIntensities(data_model, {}, {}, 2.5, logger); + auto obs = ModelReferenceIntensities(ReadModel(data_model), {}, {}, 2.5, logger); REQUIRE(obs.size() > 500); for (size_t i = 0; i < obs.size(); i++) { obs[i].F = std::sqrt(std::max(0.0f, obs[i].I)); @@ -518,7 +523,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat } const auto other_setting = WriteTemp("cob_model_test.pdb", permuted.c_str()); - const auto moved = ValidateAgainstModel(obs, data_cell, other_setting, prefix, logger, data_sg, + const auto moved = ValidateAgainstModel(obs, data_cell, ReadModel(other_setting), prefix, logger, data_sg, /*probe_indexing_ambiguity=*/false, 1, 1.0); REQUIRE(moved.ok); logger.Info("Change of basis test: took {} out of {}, R-work {:.4f}", @@ -529,7 +534,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat // The isomorphous case, which is what nearly every run is: the model already describes the // lattice the way the data do, the only operators that map its cell are its own symmetry, and // nothing is scored or moved. - const auto same = ValidateAgainstModel(obs, data_cell, data_model, prefix, logger, data_sg, + const auto same = ValidateAgainstModel(obs, data_cell, ReadModel(data_model), prefix, logger, data_sg, false, 1, 1.0); REQUIRE(same.ok); CHECK(same.change_of_basis_op == gemmi::Op::identity()); @@ -551,7 +556,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida // Point group 4 on a tetragonal lattice (4/mmm): one alternative indexing. const auto model = WriteTemp("reidx_model_test.pdb", ClusterPdb("CRYST1 34.000 34.000 38.000 90.00 90.00 90.00 P 4 4\n").c_str()); - auto own = ModelReferenceIntensities(model, {}, {}, 2.5, logger); + auto own = ModelReferenceIntensities(ReadModel(model), {}, {}, 2.5, logger); REQUIRE(own.size() > 500); for (size_t i = 0; i < own.size(); i++) { own[i].F = std::sqrt(std::max(0.0f, own[i].I)); @@ -566,7 +571,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida // The data as a run that picked the other indexing would have merged them. const auto obs = ReindexReflections(own, laws.front()); - const auto to_model = ValidateAgainstModel(obs, cell, model, prefix, logger, p4, + const auto to_model = ValidateAgainstModel(obs, cell, ReadModel(model), prefix, logger, p4, /*probe_indexing_ambiguity=*/true, 1, 1.0); REQUIRE(to_model.ok); CHECK(to_model.change_of_basis_op == gemmi::Op::identity()); @@ -575,7 +580,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida CHECK(to_model.r_work < 0.15); // A reference already fixed the data's indexing: the data stay, and the model is moved. - const auto fixed = ValidateAgainstModel(obs, cell, model, prefix, logger, p4, false, 1, 1.0); + const auto fixed = ValidateAgainstModel(obs, cell, ReadModel(model), prefix, logger, p4, false, 1, 1.0); REQUIRE(fixed.ok); CHECK_FALSE(fixed.change_of_basis_op == gemmi::Op::identity()); CHECK(fixed.indexing_op == gemmi::Op::identity()); @@ -599,7 +604,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida r.F = std::sqrt(std::max(0.0f, r.I)); } } - const auto twin = ValidateAgainstModel(twinned, cell, model, prefix, logger, p4, true, 1, 1.0); + const auto twin = ValidateAgainstModel(twinned, cell, ReadModel(model), prefix, logger, p4, true, 1, 1.0); REQUIRE(twin.ok); CHECK_FALSE(twin.indexing_decided); CHECK(twin.indexing_op == gemmi::Op::identity()); @@ -607,7 +612,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida // Data already in the model's indexing: nothing moves. The null is run all the same, as on every // validation that judges its model (ModelValidationSchedule::fit_test), and the model fits. - const auto same = ValidateAgainstModel(own, cell, model, prefix, logger, p4, true, 1, 1.0); + const auto same = ValidateAgainstModel(own, cell, ReadModel(model), prefix, logger, p4, true, 1, 1.0); REQUIRE(same.ok); CHECK(same.change_of_basis_op == gemmi::Op::identity()); CHECK(same.indexing_op == gemmi::Op::identity()); @@ -736,7 +741,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]") Logger logger("ModelValidation_CCModelFollowsTheSignalByShell"); const auto path = WriteTemp("cc_model_test.pdb", ClusterPdb().c_str()); - auto obs = ModelReferenceIntensities(path, {}, {}, 2.5, logger); + auto obs = ModelReferenceIntensities(ReadModel(path), {}, {}, 2.5, logger); REQUIRE(obs.size() > 1000); // The shells the correlation is reported on, coarse to fine, and the noise each one gets as a @@ -777,7 +782,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]") const auto result = ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90}, - path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), + ReadModel(path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), /*probe_indexing_ambiguity=*/false, 1, 1.0, shells); REQUIRE(result.ok); REQUIRE(result.cc_model_shells.size() == shells.size()); @@ -805,7 +810,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]") const auto no_shells = ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90}, - path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), + ReadModel(path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), false, 1, 1.0); CHECK(no_shells.ok); CHECK(no_shells.cc_model_shells.empty()); @@ -827,7 +832,7 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio const auto data_path = WriteTemp("off_metric_data.pdb", ClusterPdb().c_str()); const auto wrong_path = WriteTemp("off_metric_wrong.pdb", ClusterPdb("CRYST1 40.000 40.000 40.000 90.00 90.00 90.00 I 21 3 24\n", 7).c_str()); - auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger); + auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger); REQUIRE(obs.size() > 1000); for (size_t i = 0; i < obs.size(); i++) { obs[i].sigma = std::max(1.0f, 0.05f * obs[i].I); @@ -837,7 +842,7 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio const std::string prefix = (std::filesystem::temp_directory_path() / "off_metric_test").string(); const auto result = ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90}, - wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21")); + ReadModel(wrong_path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21")); CHECK(result.ok); CHECK(!result.adopted_model_enantiomorph); @@ -854,7 +859,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat Logger logger("ModelValidation_CCModelUnrelatedModelConfirmsNothing"); const auto data_path = WriteTemp("cc_null_data.pdb", ClusterPdb().c_str()); const auto wrong_path = WriteTemp("cc_null_wrong.pdb", ClusterPdb(kCryst, 7).c_str()); - auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger); + auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger); std::erase_if(obs, [](const MergedReflection &r) { return r.d > 8.0f; }); REQUIRE(obs.size() > 1000); for (size_t i = 0; i < obs.size(); i++) { @@ -870,7 +875,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat const std::string prefix = (std::filesystem::temp_directory_path() / "cc_null_test").string(); const auto result = ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90}, - wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), + ReadModel(wrong_path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"), /*probe_indexing_ambiguity=*/false, 1, 1.0, shells); REQUIRE(result.ok); REQUIRE(result.cc_model_shells.size() == shells.size()); @@ -1003,7 +1008,7 @@ TEST_CASE("ModelValidation_RigidBodySameOnAnyNumberOfThreads", "[ModelValidation REQUIRE(sg != nullptr); st.setup_cell_images(); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); REQUIRE_FALSE(ref.empty()); gemmi::AsuData> fobs; fobs.unit_cell_ = st.cell; @@ -1298,7 +1303,7 @@ namespace { gemmi::AsuData> OwnAmplitudes(const char *cryst, const gemmi::Structure &st, double d_min, Logger &logger) { const auto path = WriteTemp("rigid_body_own_amplitudes.pdb", ClusterPdb(cryst).c_str()); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); std::filesystem::remove(path); gemmi::AsuData> fobs; fobs.unit_cell_ = st.cell; @@ -1327,7 +1332,7 @@ namespace { const gemmi::SpaceGroup *sg = st.find_spacegroup(); // "Observed" amplitudes: the model's own, as placed at q = 0. const auto path = WriteTemp("rigid_body_jacobian_test.pdb", ClusterPdb(cryst).c_str()); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); std::filesystem::remove(path); REQUIRE_FALSE(ref.empty()); @@ -1455,7 +1460,7 @@ TEST_CASE("ModelValidation_RigidBodyReportsTheRotationItApplied", "[ModelValidat const gemmi::SpaceGroup *sg = st.find_spacegroup(); REQUIRE(sg != nullptr); st.setup_cell_images(); - const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger); + const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger); std::filesystem::remove(path); REQUIRE_FALSE(ref.empty()); gemmi::AsuData> fobs; diff --git a/tests/MolecularReplacementTest.cpp b/tests/MolecularReplacementTest.cpp index e03353a9f..078755225 100644 --- a/tests/MolecularReplacementTest.cpp +++ b/tests/MolecularReplacementTest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "../image_analysis/structure_refinement/MolecularReplacement.h" #include "../image_analysis/structure_refinement/RigidBodyRefine.h" #include "../common/Logger.h" +#include "../writer/RugnuxModelFile.h" #ifdef JFJOCH_USE_CUDA #include "../common/CUDAWrapper.h" #include "../image_analysis/structure_refinement/MolecularReplacementGPU.h" @@ -322,7 +324,7 @@ TEST_CASE("MolecularReplacement_ModelKind", "[MolecularReplacement]") { const float plddt[3] = {90, 85, 95}; write("MolecularReplacement_ModelKind_af.pdb", "CRYST1 1.000 1.000 1.000 90.00 90.00 90.00 P 1 1", plddt, false, false); - const SearchModel af = PrepareSearchModel("MolecularReplacement_ModelKind_af.pdb"); + const SearchModel af = PrepareSearchModel(ReadRugnuxModelFile("MolecularReplacement_ModelKind_af.pdb")); CHECK(af.failure_reason.empty()); CHECK(af.predicted); CHECK(af.residues_kept == 3); @@ -331,7 +333,7 @@ TEST_CASE("MolecularReplacement_ModelKind", "[MolecularReplacement]") { const float refined[3] = {25, 112, 40}; write("MolecularReplacement_ModelKind_xtal.pdb", "CRYST1 50.000 60.000 70.000 90.00 90.00 90.00 P 21 21 21 4", refined, true, true); - const SearchModel x = PrepareSearchModel("MolecularReplacement_ModelKind_xtal.pdb"); + const SearchModel x = PrepareSearchModel(ReadRugnuxModelFile("MolecularReplacement_ModelKind_xtal.pdb")); CHECK(x.failure_reason.empty()); CHECK(!x.predicted); CHECK(x.residues_kept == 3); @@ -375,19 +377,20 @@ TEST_CASE("MolecularReplacement_PaeDomains", "[MolecularReplacement]") { m.st.models.push_back(model); m.residues_read = 120; m.residues_kept = 110; - const std::string path = "MolecularReplacement_PaeDomains.json"; - { - std::ofstream out(path); - out << "[{\"predicted_aligned_error\":["; - for (int i = 0; i < 120; i++) { - out << (i ? "," : "") << "["; - for (int j = 0; j < 120; j++) - out << (j ? "," : "") << ((i < 55) == (j < 55) ? 2 : 25); - out << "]"; - } - out << "],\"max_predicted_aligned_error\":31.75}]"; + std::ostringstream json; + json << "[{\"predicted_aligned_error\":["; + for (int i = 0; i < 120; i++) { + json << (i ? "," : "") << "["; + for (int j = 0; j < 120; j++) + json << (j ? "," : "") << ((i < 55) == (j < 55) ? 2 : 25); + json << "]"; } - const auto domains = PaeDomains(path, m); + json << "],\"max_predicted_aligned_error\":31.75}]"; + const RugnuxModel parsed = ParseRugnuxModel( + "ATOM 1 CA ALA A 1 0.000 0.000 0.000 1.00 90.00 C\nEND\n", + "AF-P12345-F1-model_v6.pdb", json.str()); + REQUIRE(parsed.pae); + const auto domains = PaeDomains(*parsed.pae, m); REQUIRE(domains.size() == 2); CHECK(domains[0].size() == 60); CHECK(domains[0].front() == 60); @@ -395,7 +398,6 @@ TEST_CASE("MolecularReplacement_PaeDomains", "[MolecularReplacement]") { const SearchModel d = DomainModel(m, domains[1]); CHECK(d.residues_kept == 50); CHECK(d.description == "PAE domain of 50 residues (1-50)"); - std::remove(path.c_str()); } // A placement of a group carried into a subgroup's merge: in P 21 21 21 read as P 1 21 1 on the same cell each diff --git a/tests/ResultReportTest.cpp b/tests/ResultReportTest.cpp index d70ea191f..5d94e077d 100644 --- a/tests/ResultReportTest.cpp +++ b/tests/ResultReportTest.cpp @@ -383,7 +383,7 @@ TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") { ModelValidationResult good; good.ok = true; - good.model_path = "model.cif"; + good.model_name = "model.cif"; good.model_space_group_number = 96; good.r_work = 0.1823; good.r_free = 0.2145; @@ -468,7 +468,7 @@ TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") { mr_result.model_placement = "MR"; MolecularReplacementResult mr; mr.ok = true; - mr.model_path = "model.cif"; + mr.model_name = "model.cif"; mr.predicted_model = false; mr.decision = "ENANTIOMORPH"; MRCandidate c; @@ -568,7 +568,7 @@ TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") { result.model_validation = good; ModelValidationResult failed; - failed.model_path = "broken.pdb"; + failed.model_name = "broken.pdb"; failed.failure_reason = "model broken.pdb has no atoms or no unit cell"; result.model_validation = failed; diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index 74bf4c128..f6855eab4 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -274,7 +274,9 @@ TEST_CASE("RugnuxCommandLine_Full", "[process]") { CHECK(cmd.find("--model") == std::string::npos); // --mr is --model with the search forced, and is given back as itself. - config.model_path = "model.cif"; + auto model = std::make_shared(); + model->name = "model.cif"; + config.model = model; CHECK(RugnuxCommandLine(config, x, "in.h5").find("--model model.cif") != std::string::npos); config.force_mr = true; const std::string mr_cmd = RugnuxCommandLine(config, x, "in.h5"); diff --git a/viewer/windows/JFJochProcessingJobsWindow.cpp b/viewer/windows/JFJochProcessingJobsWindow.cpp index 0ba1ddbe4..b24a566e7 100644 --- a/viewer/windows/JFJochProcessingJobsWindow.cpp +++ b/viewer/windows/JFJochProcessingJobsWindow.cpp @@ -6,6 +6,7 @@ #include "JFJochMergeStatsWindow.h" #include "../widgets/ToolbarIcons.h" #include "../../rugnux/RugnuxCommandLine.h" +#include "../../writer/RugnuxModelFile.h" #include "../../common/ThreadAffinity.h" #include @@ -396,8 +397,6 @@ ProcessConfig JFJochProcessingJobsWindow::buildConfig(const JobSpec &spec, const config.reference_space_group = inputs.reference_space_group; config.reference_path = inputs.reference_path.toStdString(); config.reference_column = inputs.reference_column.toStdString(); - // Validated at the end of the job, when there is an output to write the maps beside. - config.model_path = inputs.model_path.toStdString(); // Stills geometry-refinement two-pass. The dialog only offers it for a stills-with-cell run // (Rugnux no-ops it otherwise), so honour it whenever set; unset leaves the pass off. if (spec.refine_geometry) @@ -422,7 +421,17 @@ void JFJochProcessingJobsWindow::newJob(ProcessMode mode, CalibrationSelection c if (action == 0) return; - const ProcessConfig config = buildConfig(spec, inputs); + ProcessConfig config = buildConfig(spec, inputs); + // Validated at the end of the job, against the merge. Read here, so that a file holding no model is + // said at once rather than after the run. + if (spec.mode == ProcessMode::FullAnalysis && !inputs.model_path.isEmpty()) { + try { + config.model = std::make_shared(ReadRugnuxModelFile(inputs.model_path.toStdString())); + } catch (const std::exception &e) { + QMessageBox::warning(this, "Processing", QString("Cannot read the model: %1").arg(e.what())); + return; + } + } // The experiment carries the panel's indexing settings — including RotationIndexing set by the // rotation axis (needed so IndexAndRefine builds a rotation indexer). On top of that it gets diff --git a/writer/CMakeLists.txt b/writer/CMakeLists.txt index c96b830b7..ad4a0a3fa 100644 --- a/writer/CMakeLists.txt +++ b/writer/CMakeLists.txt @@ -36,10 +36,12 @@ ADD_LIBRARY(JFJochModelWriter STATIC WriteModel.cpp WriteModel.h) TARGET_LINK_LIBRARIES(JFJochModelWriter JFJochLogger gemmi) # rugnux's merged MTZ, map-coefficient MTZ and placed model, to and from the structured END-message -# blocks, so that jfjoch_writer writes them with gemmi alone and no analysis library. +# blocks, so that jfjoch_writer writes them with gemmi alone and no analysis library; and the atomic model +# a run is given, parsed for it. ADD_LIBRARY(JFJochRugnuxFiles STATIC WriteRugnuxFiles.cpp WriteRugnuxFiles.h - RugnuxResultBlocks.cpp RugnuxResultBlocks.h) -TARGET_LINK_LIBRARIES(JFJochRugnuxFiles JFJochModelWriter JFJochLogger gemmi) + RugnuxResultBlocks.cpp RugnuxResultBlocks.h + RugnuxModelFile.cpp RugnuxModelFile.h) +TARGET_LINK_LIBRARIES(JFJochRugnuxFiles JFJochModelWriter JFJochLogger gemmi ZLIB::ZLIB) TARGET_LINK_LIBRARIES(JFJochWriter JFJochPreview JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize JFJochRugnuxFiles) diff --git a/writer/RugnuxModelFile.cpp b/writer/RugnuxModelFile.cpp new file mode 100644 index 000000000..15740b333 --- /dev/null +++ b/writer/RugnuxModelFile.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxModelFile.h" + +#include +#include +#include + +#include +#include +#include + +namespace { + +// The bytes of a gzip stream inflated, every member of it. +std::string Gunzip(const std::string &bytes) { + z_stream s{}; + if (inflateInit2(&s, 15 + 32) != Z_OK) + throw std::runtime_error("cannot start zlib"); + std::string out; + char buf[1 << 16]; + s.next_in = reinterpret_cast(const_cast(bytes.data())); + s.avail_in = static_cast(bytes.size()); + int ret = Z_OK; + while (ret != Z_STREAM_END || s.avail_in > 0) { + if (ret == Z_STREAM_END) + inflateReset(&s); + s.next_out = reinterpret_cast(buf); + s.avail_out = sizeof buf; + ret = inflate(&s, Z_NO_FLUSH); + if (ret != Z_OK && ret != Z_STREAM_END) { + inflateEnd(&s); + throw std::runtime_error("the gzip stream is damaged"); + } + out.append(buf, sizeof buf - s.avail_out); + if (ret == Z_OK && s.avail_in == 0 && s.avail_out != 0) { + inflateEnd(&s); + throw std::runtime_error("the gzip stream is cut short"); + } + } + inflateEnd(&s); + return out; +} + +std::string ReadFile(const std::string &path) { + std::ifstream f(path, std::ios::binary); + if (!f) + throw std::runtime_error("cannot open " + path); + return {std::istreambuf_iterator(f), std::istreambuf_iterator()}; +} + +} // namespace + +RugnuxModel ParseRugnuxModel(const std::string &bytes, const std::string &name, const std::string &pae_json) { + RugnuxModel m; + m.name = name; + std::string text = bytes.size() >= 2 && bytes[0] == '\x1f' && bytes[1] == '\x8b' ? Gunzip(bytes) : bytes; + gemmi::cif::Document doc; + m.structure = gemmi::read_structure_from_memory(text.data(), text.size(), name, gemmi::CoorFormat::Detect, &doc); + if (m.structure.models.empty()) + throw std::runtime_error(name + " holds no atoms"); + m.modelcif = m.structure.input_format == gemmi::CoorFormat::Mmcif && !doc.blocks.empty() + && doc.blocks[0].has_mmcif_category("_ma_qa_metric"); + if (!pae_json.empty()) { + try { + const nlohmann::json j = nlohmann::json::parse(pae_json); + const nlohmann::json &pae = j.is_array() ? j.at(0).at("predicted_aligned_error") + : j.at("predicted_aligned_error"); + std::vector> rows; + for (const auto &row : pae) + rows.push_back(row.get>()); + m.pae = std::move(rows); + } catch (const std::exception &) { + } + } + return m; +} + +RugnuxModel ReadRugnuxModelFile(const std::string &path) { + const std::string pae_path = PaePathFor(path); + std::string pae_json; + if (!pae_path.empty() && std::ifstream(pae_path)) + pae_json = ReadFile(pae_path); + return ParseRugnuxModel(ReadFile(path), path, pae_json); +} + +std::string PaePathFor(const std::string &model_path) { + const size_t slash = model_path.find_last_of("/\\"); + const std::string dir = slash == std::string::npos ? std::string() : model_path.substr(0, slash + 1); + const std::string name = model_path.substr(dir.size()); + const size_t at = name.rfind("-model_v"); + if (at == std::string::npos) + return {}; + const size_t dot = name.find('.', at); + const std::string version = name.substr(at + 8, dot == std::string::npos ? std::string::npos : dot - at - 8); + return dir + name.substr(0, at) + "-predicted_aligned_error_v" + version + ".json"; +} diff --git a/writer/RugnuxModelFile.h b/writer/RugnuxModelFile.h new file mode 100644 index 000000000..5f242e6b1 --- /dev/null +++ b/writer/RugnuxModelFile.h @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "../image_analysis/structure_refinement/RugnuxModel.h" + +// The atomic model of a run from the bytes of its file: PDB or mmCIF, gzipped or not, the format taken from +// the content, never from a name. `pae_json` is the AlphaFold DB's predicted-aligned-error JSON of a +// prediction, empty where there is none; one that cannot be read is left out, as one never given. `name` is +// what the report calls the model, and what gemmi names a PDB structure after. Throws where the bytes hold no +// atomic model. +RugnuxModel ParseRugnuxModel(const std::string &bytes, const std::string &name, const std::string &pae_json = {}); + +// The model file at `path`, with the predicted aligned error the AlphaFold DB puts beside it where it is there +// (PaePathFor). Named by `path`. +RugnuxModel ReadRugnuxModelFile(const std::string &path); + +// The PAE file the AlphaFold DB puts beside a model: AF--F1-model_v.cif -> +// AF--F1-predicted_aligned_error_v.json in the same directory; empty where the name does not follow it. +std::string PaePathFor(const std::string &model_path); From 6efc42360a0ae371241ac0e0e6b7ce95f0e8c25e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 16:51:07 +0200 Subject: [PATCH 07/19] ParseRugnuxModel: a file with no atom sites is not a model gemmi reads any text it does not recognise as PDB and can hand back a structure with an empty model in it, so 'no models' let garbage through. Count the atom sites instead. Co-Authored-By: Claude Opus 5.5 --- writer/RugnuxModelFile.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/writer/RugnuxModelFile.cpp b/writer/RugnuxModelFile.cpp index 15740b333..287872ff4 100644 --- a/writer/RugnuxModelFile.cpp +++ b/writer/RugnuxModelFile.cpp @@ -10,6 +10,7 @@ #include #include #include +#include // count_atom_sites namespace { @@ -58,7 +59,7 @@ RugnuxModel ParseRugnuxModel(const std::string &bytes, const std::string &name, std::string text = bytes.size() >= 2 && bytes[0] == '\x1f' && bytes[1] == '\x8b' ? Gunzip(bytes) : bytes; gemmi::cif::Document doc; m.structure = gemmi::read_structure_from_memory(text.data(), text.size(), name, gemmi::CoorFormat::Detect, &doc); - if (m.structure.models.empty()) + if (gemmi::count_atom_sites(m.structure) == 0) throw std::runtime_error(name + " holds no atoms"); m.modelcif = m.structure.input_format == gemmi::CoorFormat::Mmcif && !doc.blocks.empty() && doc.blocks[0].has_mmcif_category("_ma_qa_metric"); From 5f731db81dbf514c9ee6dbb44c33b49eb317d302 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 16:51:20 +0200 Subject: [PATCH 08/19] rugnux: the library returns its outputs in memory; the callers write the files The rugnux library wrote its results itself: _process.h5 through a FileWriter, the merged/twin/_mr reflection files, the unmerged MTZs, the maps and their MTZ, the placed models, _detector.jpg and _plot.txt, and on a doubled cell it renamed files on disk. It now writes none of them. - MtzTable (common/MtzTable.h): an MTZ file's header, columns and batch headers with no file behind it. It replaces MergedReflectionsMessage and MapCoefficientsMessage in the END message, with one CBOR encoding. WriteMtz (writer/WriteRugnuxFiles.h) is the one serialiser, used by rugnux and jfjoch_writer alike; it refuses a table whose columns differ in length, name a missing dataset, carry a malformed batch header or name a space group gemmi does not know. MergedMtzTable and UnmergedMtzTable build the tables the old writers built. - ProcessResult::outputs (RugnuxOutputs) holds, for the output pass only: the answer's merge (MTZ table plus what the .cif/.hkl are written from), the twin hypothesis' and the model's choice's merges, the unmerged and unmerged-partials tables, the maps (coefficients always; the grids only with ProcessConfig::keep_map_grids), the placed models, the detector JPEG and the per-image plot table, and the start message of the no-lattice _process.h5. Map RMS is taken from the grid in memory, as the CCP4 header stated it, whether or not the map is kept. - The per-image analysis goes to a caller-supplied RugnuxImageSink; rugnux and the viewer give a FileWriter-based ProcessFileSink. The geometry pre-pass no longer makes a _01_process.h5. - The doubled cell keeps both output sets: the answer's in outputs, the other's in DoubledCell::other_outputs with its suffix; the report names files by suffix after the caller's prefix. - The output prefix, write_process_h5 and the observation dump path are gone from ProcessConfig; --dump-observations is a callback. - What used to be gated on having files to write (model validation and MR, the twin-hypothesis merges, the finalist ledger, the scaled fulls, the unmerged tables) is gated on the output pass instead, so a caller with no prefix gets the same results. - RugnuxOutputFiles (new library JFJochRugnuxOutput, linked by rugnux, the viewer and the tests) writes every file from the result with the names and formats rugnux has always used. The viewer writes its job's files the same way. Co-Authored-By: Claude Opus 5.5 --- common/CMakeLists.txt | 1 + common/JFJochMessages.h | 34 +--- common/MtzTable.h | 48 +++++ frame_serialize/CBORStream2Deserializer.cpp | 108 ++++++----- frame_serialize/CBORStream2Serializer.cpp | 87 +++++---- image_analysis/WriteReflections.cpp | 167 +++++++++-------- image_analysis/WriteReflections.h | 40 ++-- .../scale_merge/RotationScaleMerge.cpp | 23 +-- .../scale_merge/RotationScaleMerge.h | 5 +- .../structure_refinement/ModelValidation.cpp | 163 +++++++--------- .../structure_refinement/ModelValidation.h | 29 ++- receiver/JFJochReceiverRugnux.cpp | 4 +- rugnux/CMakeLists.txt | 9 +- rugnux/DiagnosticOutput.cpp | 18 +- rugnux/DiagnosticOutput.h | 9 +- rugnux/ResultReport.cpp | 45 ++--- rugnux/ResultReport.h | 11 -- rugnux/Rugnux.cpp | 122 +++++------- rugnux/Rugnux.h | 129 +++++++++---- rugnux/RugnuxCalibration.cpp | 3 - rugnux/RugnuxCommandLine.cpp | 8 +- rugnux/RugnuxCommandLine.h | 4 + rugnux/RugnuxFirstPass.cpp | 3 - rugnux/RugnuxFirstPassAccept.cpp | 3 - rugnux/RugnuxFirstPassRescues.cpp | 3 - rugnux/RugnuxImagePass.cpp | 5 +- rugnux/RugnuxLive.cpp | 4 +- rugnux/RugnuxLive.h | 8 +- rugnux/RugnuxOutputFiles.cpp | 176 ++++++++++++++++++ rugnux/RugnuxOutputFiles.h | 58 ++++++ rugnux/RugnuxPasses.cpp | 64 ++----- rugnux/RugnuxPipeline.h | 23 +-- rugnux/RugnuxPreScan.cpp | 3 - rugnux/RugnuxScaleMerge.cpp | 165 ++++++++-------- rugnux/RugnuxStills.cpp | 3 - rugnux/rugnux_cli.cpp | 45 ++++- tests/CMakeLists.txt | 2 +- tests/DiagnosticOutputTest.cpp | 11 +- tests/ModelValidationTest.cpp | 41 ++-- tests/RugnuxResultFilesTest.cpp | 77 +++++--- tests/RugnuxTest.cpp | 25 ++- tests/WriteReflectionsTest.cpp | 29 +-- viewer/CMakeLists.txt | 2 +- viewer/JFJochProcessController.cpp | 23 ++- viewer/JFJochProcessController.h | 7 +- viewer/windows/JFJochProcessingJobsWindow.cpp | 23 +-- writer/RugnuxResultBlocks.cpp | 97 +++------- writer/RugnuxResultBlocks.h | 3 +- writer/WriteRugnuxFiles.cpp | 146 +++++---------- writer/WriteRugnuxFiles.h | 12 +- 50 files changed, 1163 insertions(+), 965 deletions(-) create mode 100644 common/MtzTable.h create mode 100644 rugnux/RugnuxOutputFiles.cpp create mode 100644 rugnux/RugnuxOutputFiles.h diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f8a8ff5c9..e447b5509 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -161,6 +161,7 @@ ADD_LIBRARY(JFJochCommon STATIC ScalingSettings.h UnitCell.cpp UnitCell.h + MtzTable.h CorrelationCoefficient.cpp CorrelationCoefficient.h JFJochReceiverPlots.cpp diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index bcb4dcd1e..449929d2f 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -16,6 +16,7 @@ #include "CompressedImage.h" #include "SpotToSave.h" #include "UnitCell.h" +#include "MtzTable.h" #include "GoniometerAxis.h" #include "DetectorTransformation.h" #include "GridScanSettings.h" @@ -368,35 +369,6 @@ struct StartMessage { std::optional smargon_position; }; -// The merged reflections as rugnux writes them to .mtz (WriteMergedMtz, -// writer/WriteRugnuxFiles.h): one row per unique reflection in the CCP4 anomalous layout, one vector -// per MTZ column. The anomalous columns (i_plus ... sigma_f_minus) are all empty where the merge kept -// no Bijvoet split, and the file then has none. free_flag is the FreeR_flag value, 0 for the test set. -struct MergedReflectionsMessage { - UnitCell unit_cell; - std::string space_group_name; // gemmi xhm(): the name keeps the setting, a number does not - float wavelength_A = 0; - std::string crystal_name; - std::vector history; // the MTZ HISTORY records after the one naming the program - std::vector h, k, l; - std::vector i_mean, sigma_i_mean; - std::vector i_plus, sigma_i_plus, i_minus, sigma_i_minus; - std::vector f, sigma_f; - std::vector f_plus, sigma_f_plus, f_minus, sigma_f_minus; - std::vector free_flag; -}; - -// The sigma_A-weighted map coefficients rugnux writes to _maps.mtz (WriteMapCoefficientsMtz), -// one vector per MTZ column. Phases in degrees, [0, 360). free_flag is FREE: 0 for the test set. -struct MapCoefficientsMessage { - UnitCell unit_cell; - std::string space_group_name; - float wavelength_A = 0; - std::vector h, k, l; - std::vector fp, fc, phic, fwt, phwt, delfwt, phdelwt, fom; - std::vector free_flag; -}; - // The atomic model as placed against the data, one vector per atom field in file order, for // WritePlacedModel to write _model.cif. A new residue starts wherever the model, chain or // residue key changes. unit_cell and space_group_name are what WritePlacedModel is given (the data's); @@ -537,8 +509,8 @@ struct EndMessage { // The merged result of an analysis run inside the broker, for the writer to put beside the master // file as .mtz, _maps.mtz and _model.cif. Absent where nothing was merged. - std::optional merged; - std::optional map_coefficients; + std::optional merged; // MergedMtzTable (image_analysis/WriteReflections.h) + std::optional map_coefficients; // ModelMaps::coefficients (ModelValidation.h) std::optional placed_model; }; diff --git a/common/MtzTable.h b/common/MtzTable.h new file mode 100644 index 000000000..c6dbf7b2e --- /dev/null +++ b/common/MtzTable.h @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include + +#include "UnitCell.h" + +// What an MTZ file holds - its header, its columns and its batch headers - with no file behind it: the +// merged reflections, the map coefficients and the unmerged observations rugnux makes, for whatever +// writes them (WriteMtz, writer/WriteRugnuxFiles.h) or carries them (the END message). Every dataset +// has the table's cell; a dataset's id is its position in `datasets`. +struct MtzTable { + struct Dataset { + std::string project_name; + std::string crystal_name; + std::string dataset_name; + float wavelength_A = 0.0f; // 0 = not set + }; + struct Column { + std::string label; + char type = 'R'; // the MTZ column type: H, J, Q, F, I, ... + int32_t dataset_id = 0; + std::vector values; // one per row + }; + // The header of one image of unmerged data: the integer and real arrays as the MTZ format lays them + // out (29 and 156 long), and the names of its goniostat axes. + struct Batch { + int32_t number = 0; + std::string title; + std::vector ints; + std::vector floats; + std::vector axes; + }; + + std::string title; + UnitCell unit_cell{}; + std::string space_group_name; // extended Hermann-Mauguin (xhm), which keeps the setting + std::vector history; + std::array sort_order{}; + std::vector datasets; + std::vector columns; + std::vector batches; +}; diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 15d07c0e3..2341bbc3f 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1510,64 +1510,84 @@ namespace { cbor_value_advance(&value); } - MergedReflectionsMessage ProcessMergedElement(CborValue &value) { - MergedReflectionsMessage m; + MtzTable::Dataset ProcessMtzDatasetElement(CborValue &value) { + MtzTable::Dataset d; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); - if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value); - else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value); - else if (key == "wavelength") m.wavelength_A = GetCBORFloat(map_value); - else if (key == "crystal_name") m.crystal_name = GetCBORString(map_value); - else if (key == "history") GetCBORStringArray(map_value, m.history); - else if (key == "h") GetCBORInt32Array(map_value, m.h); - else if (key == "k") GetCBORInt32Array(map_value, m.k); - else if (key == "l") GetCBORInt32Array(map_value, m.l); - else if (key == "i_mean") GetCBORFloatArray(map_value, m.i_mean); - else if (key == "sigma_i_mean") GetCBORFloatArray(map_value, m.sigma_i_mean); - else if (key == "i_plus") GetCBORFloatArray(map_value, m.i_plus); - else if (key == "sigma_i_plus") GetCBORFloatArray(map_value, m.sigma_i_plus); - else if (key == "i_minus") GetCBORFloatArray(map_value, m.i_minus); - else if (key == "sigma_i_minus") GetCBORFloatArray(map_value, m.sigma_i_minus); - else if (key == "f") GetCBORFloatArray(map_value, m.f); - else if (key == "sigma_f") GetCBORFloatArray(map_value, m.sigma_f); - else if (key == "f_plus") GetCBORFloatArray(map_value, m.f_plus); - else if (key == "sigma_f_plus") GetCBORFloatArray(map_value, m.sigma_f_plus); - else if (key == "f_minus") GetCBORFloatArray(map_value, m.f_minus); - else if (key == "sigma_f_minus") GetCBORFloatArray(map_value, m.sigma_f_minus); - else if (key == "free_flag") GetCBORUInt8Array(map_value, m.free_flag); + if (key == "project_name") d.project_name = GetCBORString(map_value); + else if (key == "crystal_name") d.crystal_name = GetCBORString(map_value); + else if (key == "dataset_name") d.dataset_name = GetCBORString(map_value); + else if (key == "wavelength") d.wavelength_A = GetCBORFloat(map_value); else SkipCBORValue(map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); - return m; + return d; } - MapCoefficientsMessage ProcessMapCoefficientsElement(CborValue &value) { - MapCoefficientsMessage m; + MtzTable::Column ProcessMtzColumnElement(CborValue &value) { + MtzTable::Column c; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); - if (key == "unit_cell") m.unit_cell = ProcessUnitCellElement(map_value); - else if (key == "space_group_name") m.space_group_name = GetCBORString(map_value); - else if (key == "wavelength") m.wavelength_A = GetCBORFloat(map_value); - else if (key == "h") GetCBORInt32Array(map_value, m.h); - else if (key == "k") GetCBORInt32Array(map_value, m.k); - else if (key == "l") GetCBORInt32Array(map_value, m.l); - else if (key == "fp") GetCBORFloatArray(map_value, m.fp); - else if (key == "fc") GetCBORFloatArray(map_value, m.fc); - else if (key == "phic") GetCBORFloatArray(map_value, m.phic); - else if (key == "fwt") GetCBORFloatArray(map_value, m.fwt); - else if (key == "phwt") GetCBORFloatArray(map_value, m.phwt); - else if (key == "delfwt") GetCBORFloatArray(map_value, m.delfwt); - else if (key == "phdelwt") GetCBORFloatArray(map_value, m.phdelwt); - else if (key == "fom") GetCBORFloatArray(map_value, m.fom); - else if (key == "free_flag") GetCBORUInt8Array(map_value, m.free_flag); + if (key == "label") c.label = GetCBORString(map_value); + else if (key == "type") c.type = GetCBORString(map_value).at(0); + else if (key == "dataset_id") c.dataset_id = static_cast(GetCBORInt(map_value)); + else if (key == "values") GetCBORFloatArray(map_value, c.values); else SkipCBORValue(map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); - return m; + return c; + } + + MtzTable::Batch ProcessMtzBatchElement(CborValue &value) { + MtzTable::Batch b; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "number") b.number = static_cast(GetCBORInt(map_value)); + else if (key == "title") b.title = GetCBORString(map_value); + else if (key == "ints") GetCBORInt32Array(map_value, b.ints); + else if (key == "floats") GetCBORFloatArray(map_value, b.floats); + else if (key == "axes") GetCBORStringArray(map_value, b.axes); + else SkipCBORValue(map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return b; + } + + MtzTable ProcessMtzTableElement(CborValue &value) { + MtzTable t; + CborValue map_value; + cborErr(cbor_value_enter_container(&value, &map_value)); + while (!cbor_value_at_end(&map_value)) { + auto key = GetCBORString(map_value); + if (key == "title") t.title = GetCBORString(map_value); + else if (key == "unit_cell") t.unit_cell = ProcessUnitCellElement(map_value); + else if (key == "space_group_name") t.space_group_name = GetCBORString(map_value); + else if (key == "history") GetCBORStringArray(map_value, t.history); + else if (key == "sort_order") { + std::vector v; + GetCBORInt32Array(map_value, v); + if (v.size() != t.sort_order.size()) + throw JFJochException(JFJochExceptionCategory::CBORError, "MTZ sort order of wrong length"); + std::copy(v.begin(), v.end(), t.sort_order.begin()); + } else if (key == "datasets" || key == "columns" || key == "batches") { + CborValue array_value; + cborErr(cbor_value_enter_container(&map_value, &array_value)); + while (!cbor_value_at_end(&array_value)) { + if (key == "datasets") t.datasets.push_back(ProcessMtzDatasetElement(array_value)); + else if (key == "columns") t.columns.push_back(ProcessMtzColumnElement(array_value)); + else t.batches.push_back(ProcessMtzBatchElement(array_value)); + } + cborErr(cbor_value_leave_container(&map_value, &array_value)); + } else SkipCBORValue(map_value); + } + cborErr(cbor_value_leave_container(&value, &map_value)); + return t; } PlacedModelMessage ProcessPlacedModelElement(CborValue &value) { @@ -1699,9 +1719,9 @@ namespace { else if (key == "pixel_sum") GetCBORInt64Array(value, message.pixel_sum); else if (key == "merged") - message.merged = ProcessMergedElement(value); + message.merged = ProcessMtzTableElement(value); else if (key == "map_coefficients") - message.map_coefficients = ProcessMapCoefficientsElement(value); + message.map_coefficients = ProcessMtzTableElement(value); else if (key == "placed_model") message.placed_model = ProcessPlacedModelElement(value); else if (key == "rotation_lattice") { diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 82ee03e51..eaf732fd1 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -521,53 +521,52 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector(v.data()), v.size() * sizeof(double))); } -inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MergedReflectionsMessage &m) { - CborEncoder mapEncoder; +inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MtzTable &t) { + CborEncoder mapEncoder, arrayEncoder, elementEncoder; cborErr(cbor_encode_text_stringz(&encoder, key)); cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength)); - CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell); - CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name); - CBOR_ENC(mapEncoder, "wavelength", m.wavelength_A); - CBOR_ENC(mapEncoder, "crystal_name", m.crystal_name); - CBOR_ENC(mapEncoder, "history", m.history); - CBOR_ENC(mapEncoder, "h", m.h); - CBOR_ENC(mapEncoder, "k", m.k); - CBOR_ENC(mapEncoder, "l", m.l); - CBOR_ENC(mapEncoder, "i_mean", m.i_mean); - CBOR_ENC(mapEncoder, "sigma_i_mean", m.sigma_i_mean); - CBOR_ENC(mapEncoder, "i_plus", m.i_plus); - CBOR_ENC(mapEncoder, "sigma_i_plus", m.sigma_i_plus); - CBOR_ENC(mapEncoder, "i_minus", m.i_minus); - CBOR_ENC(mapEncoder, "sigma_i_minus", m.sigma_i_minus); - CBOR_ENC(mapEncoder, "f", m.f); - CBOR_ENC(mapEncoder, "sigma_f", m.sigma_f); - CBOR_ENC(mapEncoder, "f_plus", m.f_plus); - CBOR_ENC(mapEncoder, "sigma_f_plus", m.sigma_f_plus); - CBOR_ENC(mapEncoder, "f_minus", m.f_minus); - CBOR_ENC(mapEncoder, "sigma_f_minus", m.sigma_f_minus); - CBOR_ENC(mapEncoder, "free_flag", m.free_flag); - cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); -} + CBOR_ENC(mapEncoder, "title", t.title); + CBOR_ENC(mapEncoder, "unit_cell", t.unit_cell); + CBOR_ENC(mapEncoder, "space_group_name", t.space_group_name); + CBOR_ENC(mapEncoder, "history", t.history); + CBOR_ENC(mapEncoder, "sort_order", std::vector(t.sort_order.begin(), t.sort_order.end())); -inline void CBOR_ENC(CborEncoder &encoder, const char* key, const MapCoefficientsMessage &m) { - CborEncoder mapEncoder; - cborErr(cbor_encode_text_stringz(&encoder, key)); - cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, CborIndefiniteLength)); - CBOR_ENC(mapEncoder, "unit_cell", m.unit_cell); - CBOR_ENC(mapEncoder, "space_group_name", m.space_group_name); - CBOR_ENC(mapEncoder, "wavelength", m.wavelength_A); - CBOR_ENC(mapEncoder, "h", m.h); - CBOR_ENC(mapEncoder, "k", m.k); - CBOR_ENC(mapEncoder, "l", m.l); - CBOR_ENC(mapEncoder, "fp", m.fp); - CBOR_ENC(mapEncoder, "fc", m.fc); - CBOR_ENC(mapEncoder, "phic", m.phic); - CBOR_ENC(mapEncoder, "fwt", m.fwt); - CBOR_ENC(mapEncoder, "phwt", m.phwt); - CBOR_ENC(mapEncoder, "delfwt", m.delfwt); - CBOR_ENC(mapEncoder, "phdelwt", m.phdelwt); - CBOR_ENC(mapEncoder, "fom", m.fom); - CBOR_ENC(mapEncoder, "free_flag", m.free_flag); + cborErr(cbor_encode_text_stringz(&mapEncoder, "datasets")); + cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.datasets.size())); + for (const auto &d : t.datasets) { + cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 4)); + CBOR_ENC(elementEncoder, "project_name", d.project_name); + CBOR_ENC(elementEncoder, "crystal_name", d.crystal_name); + CBOR_ENC(elementEncoder, "dataset_name", d.dataset_name); + CBOR_ENC(elementEncoder, "wavelength", d.wavelength_A); + cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder)); + } + cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder)); + + cborErr(cbor_encode_text_stringz(&mapEncoder, "columns")); + cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.columns.size())); + for (const auto &c : t.columns) { + cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 4)); + CBOR_ENC(elementEncoder, "label", c.label); + CBOR_ENC(elementEncoder, "type", std::string(1, c.type)); + CBOR_ENC(elementEncoder, "dataset_id", static_cast(c.dataset_id)); + CBOR_ENC(elementEncoder, "values", c.values); + cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder)); + } + cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder)); + + cborErr(cbor_encode_text_stringz(&mapEncoder, "batches")); + cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, t.batches.size())); + for (const auto &b : t.batches) { + cborErr(cbor_encoder_create_map(&arrayEncoder, &elementEncoder, 5)); + CBOR_ENC(elementEncoder, "number", static_cast(b.number)); + CBOR_ENC(elementEncoder, "title", b.title); + CBOR_ENC(elementEncoder, "ints", b.ints); + CBOR_ENC(elementEncoder, "floats", b.floats); + CBOR_ENC(elementEncoder, "axes", b.axes); + cborErr(cbor_encoder_close_container(&arrayEncoder, &elementEncoder)); + } + cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder)); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); } diff --git a/image_analysis/WriteReflections.cpp b/image_analysis/WriteReflections.cpp index 8cf24c2a5..53e4f69af 100644 --- a/image_analysis/WriteReflections.cpp +++ b/image_analysis/WriteReflections.cpp @@ -607,39 +607,48 @@ void WriteMmcifReflections(const std::vector &reflections, out.close(); } -MergedReflectionsMessage MergedReflectionsForWriter(const std::vector &reflections, - const UnitCell &unitCell, - const DiffractionExperiment &experiment, - const OutputProvenance &provenance) { - MergedReflectionsMessage m; +MtzTable MergedMtzTable(const std::vector &reflections, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + const OutputProvenance &provenance) { + MtzTable m; m.unit_cell = unitCell; m.space_group_name = experiment.GetSpaceGroupOrP1().xhm(); - m.wavelength_A = experiment.GetWavelength_A(); - m.crystal_name = experiment.GetSampleName(); + // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention). + m.title = "Rugnux merged reflections"; m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); AddProvenanceHistory(m.history, provenance); + // HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the reserved base dataset + // as far as CCP4's mtzlib is concerned, and the wavelength on it is dropped - mtzinfo then reports the + // Cu K-alpha default 1.54187 A, which is what a program that takes f'/f'' from the file would phase on. + m.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f}, + {"native", experiment.GetSampleName(), "native", experiment.GetWavelength_A()}}; + // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for + // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly. bool has_anom = true; const std::vector out_rows = BuildMergedRows(reflections, experiment, has_anom); - for (const auto& row : out_rows) { - m.h.push_back(row.h); - m.k.push_back(row.k); - m.l.push_back(row.l); - m.i_mean.push_back(row.Imean); - m.sigma_i_mean.push_back(row.sImean); - m.f.push_back(row.Fmean); - m.sigma_f.push_back(row.sFmean); - if (has_anom) { - m.i_plus.push_back(row.Ip); - m.sigma_i_plus.push_back(row.sIp); - m.i_minus.push_back(row.Im); - m.sigma_i_minus.push_back(row.sIm); - m.f_plus.push_back(row.Fp); - m.sigma_f_plus.push_back(row.sFp); - m.f_minus.push_back(row.Fm); - m.sigma_f_minus.push_back(row.sFm); - } - m.free_flag.push_back(static_cast(row.rfree)); + std::vector> columns{{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, + {"IMEAN", 'J'}, {"SIGIMEAN", 'Q'}}; + if (has_anom) + columns.insert(columns.end(), {{"I(+)", 'K'}, {"SIGI(+)", 'M'}, {"I(-)", 'K'}, {"SIGI(-)", 'M'}}); + columns.insert(columns.end(), {{"F", 'F'}, {"SIGF", 'Q'}}); // F: the French-Wilson amplitude + if (has_anom) + columns.insert(columns.end(), {{"F(+)", 'G'}, {"SIGF(+)", 'L'}, {"F(-)", 'G'}, {"SIGF(-)", 'L'}}); + columns.push_back({"FreeR_flag", 'I'}); + for (const auto &[label, type] : columns) + m.columns.push_back({label, type, type == 'H' ? 0 : 1, {}}); + for (const auto &row : out_rows) { + std::vector v{static_cast(row.h), static_cast(row.k), static_cast(row.l), + row.Imean, row.sImean}; + if (has_anom) + v.insert(v.end(), {row.Ip, row.sIp, row.Im, row.sIm}); + v.insert(v.end(), {row.Fmean, row.sFmean}); + if (has_anom) + v.insert(v.end(), {row.Fp, row.sFp, row.Fm, row.sFm}); + v.push_back(static_cast(row.rfree)); + for (size_t c = 0; c < v.size(); c++) + m.columns[c].values.push_back(v[c]); } return m; } @@ -649,7 +658,7 @@ void WriteMtzReflections(const std::vector &reflections, const DiffractionExperiment &experiment, const OutputProvenance &provenance, const std::string &filename) { - WriteMergedMtz(MergedReflectionsForWriter(reflections, unitCell, experiment, provenance), filename); + WriteMtz(MergedMtzTable(reflections, unitCell, experiment, provenance), filename); } namespace { @@ -906,22 +915,22 @@ std::vector> SumRockingEvents(const std::vector &outcomes, - const UnitCell &unitCell, - const DiffractionExperiment &experiment, - bool sum_partials, - const OutputProvenance &provenance, - size_t nthreads) { - gemmi::Mtz mtz; - mtz.spacegroup = &experiment.GetSpaceGroupOrP1(); - mtz.set_cell_for_all(unitCell); +MtzTable UnmergedMtzTable(const std::vector &outcomes, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + bool sum_partials, + const OutputProvenance &provenance, + size_t nthreads) { + const gemmi::SpaceGroup *spacegroup = &experiment.GetSpaceGroupOrP1(); + MtzTable mtz; + mtz.space_group_name = spacegroup->xhm(); + mtz.unit_cell = unitCell; mtz.title = "Rugnux unmerged observations"; mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); AddProvenanceHistory(mtz.history, provenance); - mtz.add_base(); // the HKL_base dataset and the H K L columns - gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged"); - ds.crystal_name = experiment.GetSampleName(); - ds.wavelength = experiment.GetWavelength_A(); + mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f}, + {"unmerged", experiment.GetSampleName(), "unmerged", experiment.GetWavelength_A()}}; + const int dataset_id = 1; // Every data column belongs to HKL_base and only the batches carry a dataset id, as in a // POINTLESS file. Down to FLAG this is POINTLESS's own column set, plus QE and FLIGHT; the four after it @@ -941,22 +950,11 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, // rises above 1 toward the detector edge where a longer crossing makes the sensor more // efficient. DIALS stores the un-normalised absorbed fraction instead; the two differ by the // per-dataset constant QE(0), which is an overall scale. - mtz.add_column("M/ISYM", 'Y', 0, -1, false); - mtz.add_column("BATCH", 'B', 0, -1, false); - mtz.add_column("I", 'J', 0, -1, false); - mtz.add_column("SIGI", 'Q', 0, -1, false); - mtz.add_column("FRACTIONCALC", 'R', 0, -1, false); - mtz.add_column("XDET", 'R', 0, -1, false); - mtz.add_column("YDET", 'R', 0, -1, false); - mtz.add_column("ROT", 'R', 0, -1, false); - mtz.add_column("LP", 'R', 0, -1, false); - mtz.add_column("QE", 'R', 0, -1, false); - mtz.add_column("FLIGHT", 'R', 0, -1, false); - mtz.add_column("FLAG", 'I', 0, -1, false); - mtz.add_column("DELPHI", 'R', 0, -1, false); - mtz.add_column("ZETA", 'R', 0, -1, false); - mtz.add_column("BGMEAN", 'R', 0, -1, false); - mtz.add_column("BGVAR", 'R', 0, -1, false); + for (const auto &[label, type] : std::vector>{ + {"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'}, + {"FRACTIONCALC", 'R'}, {"XDET", 'R'}, {"YDET", 'R'}, {"ROT", 'R'}, {"LP", 'R'}, {"QE", 'R'}, + {"FLIGHT", 'R'}, {"FLAG", 'I'}, {"DELPHI", 'R'}, {"ZETA", 'R'}, {"BGMEAN", 'R'}, {"BGVAR", 'R'}}) + mtz.columns.push_back({label, type, 0, {}}); const auto gon = experiment.GetGoniometer(); // A scanning goniometer means rugnux integrated a rocking curve image by image, so each @@ -985,7 +983,7 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, // None of the three is a scale, and a program that reads this file has no way to recompute them. The // partiality is NOT divided out - that IS a scale, FRACTIONCALC carries it, and every program // this file is for wants to handle it its own way. - gemmi::UnmergedHklMover hkl_mover(mtz.spacegroup); + gemmi::UnmergedHklMover hkl_mover(spacegroup); // The batch headers are written for every image the observations span, not only for the images // that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that // indexed nothing leaves a hole that cuts the sweep in two: measured on a 360 deg sweep with 67 @@ -1000,7 +998,7 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, // roughly halve AIMLESS's . Screw and glide absences ARE written: they are the evidence // the space group was chosen on, so a program reading this file can check that choice, and // removing them turns its test into an assumption. XDS and DIALS draw the line in the same place. - const char centering = mtz.spacegroup ? mtz.spacegroup->hm[0] : 'P'; + const char centering = spacegroup->hm[0]; // On stills only the crystals the merge takes are written: one whose cell is not the run's was indexed // in another lattice (on few spots, typically an axis multiple), so its indices do not name the @@ -1040,13 +1038,14 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, add_row(r); } - // Every row has its own slot, so they are filled on several workers. + // Every row has its own slot in each column, so they are filled on several workers. const size_t w = mtz.columns.size(); - mtz.data.resize(rows.size() * w); + for (auto &c : mtz.columns) + c.values.resize(rows.size()); ParallelChunks(static_cast(rows.size()), ThreadsForWork(rows.size(), nthreads), [&](int lo, int hi) { + std::vector row(w); for (int i = lo; i < hi; ++i) { const Reflection &r = *rows[i]; - float *row = &mtz.data[static_cast(i) * w]; std::array hkl{r.h, r.k, r.l}; const int isym = hkl_mover.move_to_asu(hkl); row[0] = static_cast(hkl[0]); @@ -1073,9 +1072,10 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, row[16] = r.zeta; row[17] = r.bkg; row[18] = r.var_bkg; + for (size_t c = 0; c < w; c++) + mtz.columns[c].values[i] = row[c]; } }); - mtz.nreflections = static_cast(mtz.data.size() / mtz.columns.size()); // The batch header's orientation matrix is the crystal at rotation angle zero - each batch's own // PHISTT is applied on top of it - but the lattice stored with an outcome is the crystal as it @@ -1106,7 +1106,7 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, gemmi::Mtz::Batch batch; batch.title = "Rugnux"; batch.axes.emplace_back("PHI"); - batch.set_dataset_id(ds.id); + batch.set_dataset_id(dataset_id); batch.ints[12] = 1; // ncryst batch.ints[14] = partials ? 1 : 2; // ldtype: oscillation data (2D spots) / area detector (3D) batch.ints[15] = 1; // jsaxs: the goniostat scan axis @@ -1143,38 +1143,41 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, batch.number = number; batch.floats[36] = phi_start_deg(static_cast(number - 1)); // phistt batch.floats[37] = batch.floats[36] + wedge_deg; // phiend - mtz.batches.push_back(batch); + mtz.batches.push_back({batch.number, batch.title, {batch.ints.begin(), batch.ints.end()}, batch.floats, + batch.axes}); } // By H K L M/ISYM BATCH, the order POINTLESS leaves an unmerged file in. This is Mtz::sort(5) - // a stable sort on those five columns - done on several workers: rows equal in all five keep the // order they were added in, so with the row number as the last key the order is total and the // parallel sort returns the same rows in the same order. - if (!mtz.has_data()) - mtz.sort(5); // no rows: its own "No data." failure, as before + if (rows.empty()) + gemmi::fail("No data."); // as gemmi's own sort has it { - std::vector order(mtz.nreflections); + std::vector order(rows.size()); std::iota(order.begin(), order.end(), 0); ParallelSort(order.begin(), order.end(), nthreads, [&](int i, int j) { - const float *a = &mtz.data[static_cast(i) * w], *b = &mtz.data[static_cast(j) * w]; - for (int n = 0; n < 5; ++n) - if (a[n] != b[n]) - return a[n] < b[n]; + for (int n = 0; n < 5; ++n) { + const float a = mtz.columns[n].values[i], b = mtz.columns[n].values[j]; + if (a != b) + return a < b; + } return i < j; }); - std::vector sorted(mtz.data.size()); - ParallelChunks(static_cast(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) { - for (int i = lo; i < hi; ++i) - std::memcpy(&sorted[static_cast(i) * w], &mtz.data[static_cast(order[i]) * w], - w * sizeof(float)); - }); - mtz.data.swap(sorted); - mtz.sort_order = {{1, 2, 3, 4, 5}}; + for (auto &c : mtz.columns) { + std::vector sorted(order.size()); + ParallelChunks(static_cast(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) + sorted[i] = c.values[order[i]]; + }); + c.values.swap(sorted); + } + mtz.sort_order = {1, 2, 3, 4, 5}; } return mtz; } -void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector &outcomes) { +void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector &outcomes) { // The reflecting range of the same image the orientation matrix is taken from. std::optional mosaicity_deg; for (const auto &outcome : outcomes) { @@ -1194,9 +1197,9 @@ void WriteUnmergedMtzReflections(const std::vector &outcomes const OutputProvenance &provenance, const std::string &filename, size_t nthreads) { - gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, provenance, nthreads); + MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads); SetUnmergedMtzMosaicity(mtz, outcomes); - mtz.write_to_file(filename); + WriteMtz(mtz, filename); } void WriteReflections(const std::vector &reflections, diff --git a/image_analysis/WriteReflections.h b/image_analysis/WriteReflections.h index 68d2ab778..1d9053fd8 100644 --- a/image_analysis/WriteReflections.h +++ b/image_analysis/WriteReflections.h @@ -9,11 +9,9 @@ #include "../common/Reflection.h" #include "../common/UnitCell.h" #include "../common/DiffractionExperiment.h" -#include "../common/JFJochMessages.h" +#include "../common/MtzTable.h" #include "IntegrationOutcome.h" -#include - struct MergeStatistics; struct TwinningAnalysisResult; @@ -50,13 +48,14 @@ void WriteMmcifReflections(const std::vector &reflections, const std::string &filename, size_t nthreads); -// The merged MTZ as a structured END-message block: the columns, metadata and HISTORY records that -// WriteMtzReflections writes, for jfjoch_writer to write the same file (WriteMergedMtz, -// writer/WriteRugnuxFiles.h). WriteMtzReflections is exactly WriteMergedMtz of this. -MergedReflectionsMessage MergedReflectionsForWriter(const std::vector &reflections, - const UnitCell &unitCell, - const DiffractionExperiment &experiment, - const OutputProvenance &provenance); +// The merged MTZ: H K L IMEAN SIGIMEAN [I(+) SIGI(+) I(-) SIGI(-)] F SIGF [F(+) SIGF(+) F(-) SIGF(-)] +// FreeR_flag, one row per unique reflection in the CCP4 anomalous layout, the bracketed columns only where +// the merge kept a Bijvoet split; with the HISTORY records saying where it came from. WriteMtzReflections +// writes it (WriteMtz, writer/WriteRugnuxFiles.h). +MtzTable MergedMtzTable(const std::vector &reflections, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + const OutputProvenance &provenance); void WriteMtzReflections(const std::vector &reflections, const UnitCell &unitCell, @@ -98,17 +97,16 @@ void WriteUnmergedMtzReflections(const std::vector &outcomes const std::string &filename, size_t nthreads = 1); -// The same file in two steps, for a caller that builds it while the outcomes' per-image scale fields -// are still being written: UnmergedMtz reads everything but mosaicity_deg, which -// SetUnmergedMtzMosaicity then copies into the batch headers. The Mtz points at the experiment's space -// group, so the experiment has to outlive it. -gemmi::Mtz UnmergedMtz(const std::vector &outcomes, - const UnitCell &unitCell, - const DiffractionExperiment &experiment, - bool sum_partials, - const OutputProvenance &provenance, - size_t nthreads = 1); -void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector &outcomes); +// The same file as its table, in two steps, for a caller that builds it while the outcomes' per-image +// scale fields are still being written: UnmergedMtzTable reads everything but mosaicity_deg, which +// SetUnmergedMtzMosaicity then copies into the batch headers. +MtzTable UnmergedMtzTable(const std::vector &outcomes, + const UnitCell &unitCell, + const DiffractionExperiment &experiment, + bool sum_partials, + const OutputProvenance &provenance, + size_t nthreads = 1); +void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector &outcomes); void WriteReflections(const std::vector &reflections, const UnitCell &unitCell, diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 86dc97067..b16a5e24a 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -303,10 +303,10 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment, std::optional reference_cell, int scaling_iterations, size_t nthreads, Logger &logger, - std::string observation_dump_path) + std::function observation_dump) : x(experiment), partials_out(partial_outcomes), reference_cell(std::move(reference_cell)), nthreads(nthreads == 0 ? AvailableCpus() : nthreads), logger(&logger), - observation_dump_path(std::move(observation_dump_path)) { + observation_dump(std::move(observation_dump)) { const auto s = x.GetScalingSettings(); min_partiality = s.GetMinPartiality(); d_min_limit = s.GetHighResolutionLimit_A(); @@ -401,7 +401,7 @@ void RotationScaleMerge::IngestHost() { // only the ingest arrays depends on whether the device pipeline will be active. gpu_ = std::make_unique(gpu_device); gpu_active_ = gpu_->Available() && !scale_on_cpu_; - resident_ingest = gpu_active_ && observation_dump_path.empty(); + resident_ingest = gpu_active_ && !observation_dump; #endif frame_cell_ok.assign(n_frames, 1); @@ -4403,7 +4403,7 @@ void RotationScaleMerge::Combine() { // Combine one raw-hkl run into fulls, appended to `out`. Independent per run (the events of one hkl // touch no shared state), so runs parallelise cleanly. Returns the number of usable partials seen. // `dump` (serial path only) writes each emitted full for the diagnostic observation dump. - auto process_rawrun = [&](int r, std::vector &out, std::ofstream *dump, int64_t &n_overloaded) -> size_t { + auto process_rawrun = [&](int r, std::vector &out, std::ostream *dump, int64_t &n_overloaded) -> size_t { prefetch_run(r + COMBINE_PREFETCH_RUNS); const int lo = rawrun_start[r], hi = rawrun_start[r] + rawrun_count[r]; std::vector ev; // usable perm-indices of this raw hkl, in image-number order @@ -4545,11 +4545,10 @@ void RotationScaleMerge::Combine() { size_t n_used = 0; std::vector> part; // the fulls of consecutive runs, in run order - if (!observation_dump_path.empty() || nthreads <= 1) { + if (observation_dump || nthreads <= 1) { // Serial (diagnostic dump needs a single writer). - std::ofstream dump; - if (!observation_dump_path.empty()) { - dump.open(observation_dump_path); + std::ostringstream dump; + if (observation_dump) { // Columns are enough to rescale/merge the combined fulls in an external program: peak_frame // is a per-full batch (ZD) and px/py the predicted detector position (XD/YD). dump << "# h k l I sigma d n_frames captured_fraction peak_frame px py\n"; @@ -4557,7 +4556,9 @@ void RotationScaleMerge::Combine() { part.resize(1); part[0].reserve(n_run); for (int r = 0; r < n_run; ++r) - n_used += process_rawrun(r, part[0], dump.is_open() ? &dump : nullptr, fulls_dropped_overloaded); + n_used += process_rawrun(r, part[0], observation_dump ? &dump : nullptr, fulls_dropped_overloaded); + if (observation_dump) + observation_dump(dump.str()); } else { // Parallel over contiguous rawrun chunks; the per-thread fulls are taken in run order below, so // the result is deterministic. @@ -6837,7 +6838,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool // scale-fulls loop below. Only the diagnostic dump (serial, one writer) falls back to CPU. The // decay/absorption correction surfaces run on the downloaded fulls (px/py is carried through the // combine) and their corrected corr is re-uploaded before the resident merge - so they stay on the GPU. - if (gpu_active_ && observation_dump_path.empty()) { + if (gpu_active_ && !observation_dump) { // The smoothed corr is already resident (scaling + smooth-G ran on the device, no round-trip). const int nf = gpu_->Combine(rawrun_group.data(), min_partiality, capture_uncertainty_coeff, min_captured_fraction, max_frame_gap, fulls_dropped_overloaded); diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index b1aa6d658..820d3fb7a 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -136,7 +136,7 @@ public: int scaling_iterations, size_t nthreads, Logger &logger, - std::string observation_dump_path = {}); + std::function observation_dump = {}); // Copy the per-frame partials into the flat buffers. Call once before the first Run(). void Ingest(); @@ -358,7 +358,8 @@ private: std::optional reference_cell; size_t nthreads; Logger *logger; - std::string observation_dump_path; + // Handed the text of the diagnostic observation dump each time the combine runs (CPU only). + std::function observation_dump; // The group the current Run merges in (see Run); null outside a Run that was given one. const gemmi::SpaceGroup *run_group = nullptr; const gemmi::SpaceGroup &Group() const { return run_group ? *run_group : x.GetSpaceGroupOrP1(); } diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index 0497730c2..6044222da 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -27,8 +27,6 @@ #include // get_size_for_hkl, get_f_phi_on_grid (the maps) #include // SolventMasker #include // Scaling (bulk solvent + anisotropic B) -#include // Ccp4 map I/O -#include // Mtz (map-coefficient output) #include // calculate_data_statistics #include "../../common/CorrelationCoefficient.h" @@ -79,22 +77,9 @@ gemmi::Grid map_from_coefficients(gemmi::AsuData> &co return MapFromFPhi(gemmi::get_f_phi_on_grid(coef, size, true)); } -// A file this run may already have written - the validation in the model's setting writes over the -// first one's maps - is removed before it is written again, not truncated: XFS (and ext4) flush a file -// truncated and rewritten when it is closed, and three 150 MB maps forced out to a disk that way cost -// seconds where a fresh file costs nothing. -void remove_before_rewriting(const std::string &path) { - std::remove(path.c_str()); -} - -// Write a map as CCP4; return its RMS (the sigma the map is read in). -double write_ccp4(const gemmi::Grid &map, const std::string &path) { - gemmi::Ccp4 ccp4; - ccp4.grid = map; - ccp4.update_ccp4_header(2); - remove_before_rewriting(path); - ccp4.write_ccp4_map(path); - return ccp4.hstats.rms; +// The RMS of a map, the sigma it is read in: the one its CCP4 header states (Ccp4::update_ccp4_header). +double map_rms(const gemmi::Grid &map) { + return gemmi::calculate_data_statistics(map.data).rms; } // Cubic, not the default linear, for reading a map at a point. The maps are sampled every d_min/3, @@ -536,7 +521,6 @@ namespace { ModelValidationResult Validate(const std::vector &merged, const UnitCell &cell, const RugnuxModel &model, - const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group, bool probe_indexing_ambiguity, @@ -1711,23 +1695,27 @@ ModelValidationResult Validate(const std::vector &merged, mapfofc.v.push_back({terms[i].hkl, delfwt[i] * ph}); } - // A validation started before the one it follows had decided writes nothing until that one has, - // and nothing at all where it decided otherwise. - if (schedule.write_gate.valid() && !schedule.write_gate.get()) { - result.failure_reason = "superseded before its files were written"; + // A validation started before the one it follows had decided makes no maps until that one has, + // and none at all where it decided otherwise. + if (schedule.maps_gate.valid() && !schedule.maps_gate.get()) { + result.failure_reason = "superseded before its maps were made"; return result; } - // Where the validation in the model's setting, started beside this one, is going to be kept, it - // writes the maps and the map coefficients on the axes the files are written on, and this one's would - // be overwritten unread: they are not made at all. + // Where the validation in the model's setting, started beside this one, is going to be kept, its + // maps and map coefficients are on the axes the files are written on, and this one's would be + // thrown away unread: they are not made at all. const bool maps_superseded = schedule.maps_superseded && schedule.maps_superseded(result); + std::shared_ptr maps; if (!maps_superseded) { - // --- write the maps and score the 2mFo-DFc map at atom centres (a real map peaks there) --- + maps = std::make_shared(); + // --- make the maps and score the 2mFo-DFc map at atom centres (a real map peaks there) --- // The three maps - this one, the difference map and the anomalous map - share nothing but what - // they read, so the other two are made and written on threads of their own beside this one. + // they read, so the other two are made on threads of their own beside this one. Nothing here + // reads the difference map, so it is made only where it is kept. std::future fofc_map = std::async(std::launch::async, [&] { - write_ccp4(map_from_coefficients(mapfofc, sf_gpu), output_prefix + "_fofc.ccp4"); + if (schedule.keep_map_grids) + maps->map_fofc = map_from_coefficients(mapfofc, sf_gpu); }); // --- anomalous difference map, where the merge kept the Bijvoet split --- @@ -1741,13 +1729,15 @@ ModelValidationResult Validate(const std::vector &merged, result.indexing_op, fmodel); result.anomalous_pairs = static_cast(mapanom.v.size()); if (!mapanom.v.empty()) { - const gemmi::Grid grid = map_from_coefficients(mapanom, sf_gpu); - read_anomalous_map(grid, write_ccp4(grid, output_prefix + "_anom.ccp4"), st, result); + gemmi::Grid grid = map_from_coefficients(mapanom, sf_gpu); + read_anomalous_map(grid, map_rms(grid), st, result); + if (schedule.keep_map_grids) + maps->map_anom = std::move(grid); } }); - const gemmi::Grid grid2fofc = map_from_coefficients(map2fofc, sf_gpu); - const double rms2 = write_ccp4(grid2fofc, output_prefix + "_2fofc.ccp4"); + gemmi::Grid grid2fofc = map_from_coefficients(map2fofc, sf_gpu); + const double rms2 = map_rms(grid2fofc); { double s = 0; int n = 0; for (gemmi::Model &m : st.models) @@ -1756,6 +1746,8 @@ ModelValidationResult Validate(const std::vector &merged, for (gemmi::Atom &a : r.atoms) { s += grid2fofc.interpolate_value(a.pos, MAP_INTERPOLATION_ORDER); ++n; } result.mean_atom_density_sigma = (n > 0 && rms2 > 0) ? (s / n) / rms2 : 0; } + if (schedule.keep_map_grids) + maps->map_2fofc = std::move(grid2fofc); fofc_map.get(); anomalous_map.get(); } @@ -1780,60 +1772,53 @@ ModelValidationResult Validate(const std::vector &merged, : "the model does not fit these data"); if (!maps_superseded) { - // --- MTZ of map coefficients so the maps can be re-opened / rebuilt in Coot etc. --- - try { - gemmi::Mtz mtz(true); - // The group the REFLECTIONS end up in, which is the model's only where its hand was adopted - - // AdoptModelFrame decides this the same way a few lines below. An enantiomorphic pair indexes - // identically, so the coefficients are the same numbers either way and only the label moves; - // but the two groups have different screw translations, so a reader that expands symmetry out - // of this file works in the wrong one if the label disagrees with the .mtz beside it. - mtz.spacegroup = result.adopted_model_enantiomorph - ? sg - // No data group means the caller merged in P1, and P1 is what the - // reflection files beside this one carry - not the model's group. - : (data_space_group != nullptr ? data_space_group - : gemmi::find_spacegroup_by_number(1)); - mtz.set_cell_for_all(ucell); - mtz.add_dataset("model_validation"); - mtz.datasets.back().wavelength = wavelength_A; - mtz.add_column("FP", 'F', -1, -1, false); - mtz.add_column("FC", 'F', -1, -1, false); - mtz.add_column("PHIC", 'P', -1, -1, false); - mtz.add_column("FWT", 'F', -1, -1, false); - mtz.add_column("PHWT", 'P', -1, -1, false); - mtz.add_column("DELFWT", 'F', -1, -1, false); - mtz.add_column("PHDELWT", 'P', -1, -1, false); - // The figure of merit the coefficients carry, so the weighting can be read off - and undone - - // from the file rather than having to be taken on trust. - mtz.add_column("FOM", 'W', -1, -1, false); - mtz.add_column("FREE", 'I', -1, -1, false); - std::vector data; - for (size_t i = 0; i < terms.size(); i++) { - // [0, 360), the convention the MTZ 'P' columns carried before sigma_A weighting: std::arg - // returns (-180, 180] and a P column is not supposed to. - const double phi_wrapped = terms[i].phi * 180.0 / PI; - const auto phi_deg = static_cast(phi_wrapped < 0 ? phi_wrapped + 360.0 : phi_wrapped); - data.insert(data.end(), {static_cast(terms[i].hkl[0]), static_cast(terms[i].hkl[1]), - static_cast(terms[i].hkl[2]), - static_cast(terms[i].fo), static_cast(terms[i].fc), - phi_deg, - fwt[i], phi_deg, - delfwt[i], phi_deg, - static_cast(sigma_a.weight[i].m), - terms[i].free ? 0.0f : 1.0f}); - } - mtz.nreflections = static_cast(terms.size()); - mtz.data = std::move(data); - remove_before_rewriting(output_prefix + "_maps.mtz"); - mtz.write_to_file(output_prefix + "_maps.mtz"); - } catch (const std::exception &e) { - logger.Warning("Model validation: could not write map MTZ: {}", e.what()); + // --- the map coefficients, so the maps can be re-opened / rebuilt in Coot etc. --- + MtzTable &mtz = maps->coefficients; + // The group the REFLECTIONS end up in, which is the model's only where its hand was adopted - + // AdoptModelFrame decides this the same way a few lines below. An enantiomorphic pair indexes + // identically, so the coefficients are the same numbers either way and only the label moves; + // but the two groups have different screw translations, so a reader that expands symmetry out + // of this file works in the wrong one if the label disagrees with the .mtz beside it. + mtz.space_group_name = result.adopted_model_enantiomorph + ? sg->xhm() + // No data group means the caller merged in P1, and P1 is what the + // reflection files beside this one carry - not the model's group. + : (data_space_group != nullptr ? data_space_group->xhm() + : gemmi::find_spacegroup_by_number(1)->xhm()); + mtz.unit_cell = UnitCell{static_cast(ucell.a), static_cast(ucell.b), + static_cast(ucell.c), static_cast(ucell.alpha), + static_cast(ucell.beta), static_cast(ucell.gamma)}; + mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f}, + {"model_validation", "model_validation", "model_validation", + static_cast(wavelength_A)}}; + // The figure of merit the coefficients carry, so the weighting can be read off - and undone - + // from the file rather than having to be taken on trust. + const std::pair columns[] = {{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"FP", 'F'}, + {"FC", 'F'}, {"PHIC", 'P'}, {"FWT", 'F'}, {"PHWT", 'P'}, + {"DELFWT", 'F'}, {"PHDELWT", 'P'}, {"FOM", 'W'}, + {"FREE", 'I'}}; + for (const auto &[label, type] : columns) + mtz.columns.push_back({label, type, type == 'H' ? 0 : 1, {}}); + for (size_t i = 0; i < terms.size(); i++) { + // [0, 360), the convention the MTZ 'P' columns carried before sigma_A weighting: std::arg + // returns (-180, 180] and a P column is not supposed to. + const double phi_wrapped = terms[i].phi * 180.0 / PI; + const auto phi_deg = static_cast(phi_wrapped < 0 ? phi_wrapped + 360.0 : phi_wrapped); + const float row[] = {static_cast(terms[i].hkl[0]), static_cast(terms[i].hkl[1]), + static_cast(terms[i].hkl[2]), + static_cast(terms[i].fo), static_cast(terms[i].fc), + phi_deg, + fwt[i], phi_deg, + delfwt[i], phi_deg, + static_cast(sigma_a.weight[i].m), + terms[i].free ? 0.0f : 1.0f}; + for (size_t c = 0; c < mtz.columns.size(); c++) + mtz.columns[c].values.push_back(row[c]); } } result.ok = true; - result.maps_prefix = output_prefix; + result.maps = std::move(maps); // The placed coordinates, for the caller to write out beside the maps once the frame is settled. result.placed_model = std::make_shared(st); logger.Info("Model validation: R-work={:.4f} ({} refl) R-free={:.4f} ({} refl) " @@ -1862,12 +1847,7 @@ ModelValidationResult Validate(const std::vector &merged, "data were indexed in the wrong hand, and reindexing would hide which", result.anomalous_deepest_site, result.anomalous_deepest_sigma); if (maps_superseded) - logger.Info("Model validation: no maps made here - the validation in the model's setting writes them"); - else - logger.Info("Model validation: wrote {}_2fofc.ccp4, {}_fofc.ccp4{}, {}_maps.mtz", - output_prefix, output_prefix, - result.anomalous_sites.empty() ? "" : ", " + output_prefix + "_anom.ccp4", - output_prefix); + logger.Info("Model validation: no maps made here - the validation in the model's setting makes them"); if (result.fit_tested && !result.model_fits) logger.Info("Model validation: the model was rejected, so nothing downstream moved - the " "reflection files are byte for byte the ones a run with no model would have " @@ -1917,7 +1897,6 @@ ModelValidationGPUPlan PlanModelValidationGPU(const ModelValidationGPUNeed &need ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, const RugnuxModel &model, - const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group, bool probe_indexing_ambiguity, @@ -1932,7 +1911,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & // the validation reports why it did not finish, and a validation that did not finish decides nothing, // so the reflection files are those of a run without a model. try { - return Validate(merged, cell, model, output_prefix, logger, data_space_group, + return Validate(merged, cell, model, logger, data_space_group, probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, true, placed); } catch (const RigidBodyGPUFailure &e) { @@ -1945,7 +1924,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & return failed; } #else - return Validate(merged, cell, model, output_prefix, logger, data_space_group, probe_indexing_ambiguity, + return Validate(merged, cell, model, logger, data_space_group, probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, schedule, twin_law, false, placed); #endif } diff --git a/image_analysis/structure_refinement/ModelValidation.h b/image_analysis/structure_refinement/ModelValidation.h index 594c3fb65..4dac7e8e8 100644 --- a/image_analysis/structure_refinement/ModelValidation.h +++ b/image_analysis/structure_refinement/ModelValidation.h @@ -11,15 +11,25 @@ #include #include +#include "gemmi/grid.hpp" #include "gemmi/symmetry.hpp" #include "../../common/Reflection.h" // MergedReflection #include "../../common/UnitCell.h" +#include "../../common/MtzTable.h" #include "RugnuxModel.h" class Logger; namespace gemmi { struct Structure; } +// The maps of a validation: their coefficients, as _maps.mtz holds them, and the maps themselves, +// as the .ccp4 files hold them, where the caller asked for them (ModelValidationSchedule::keep_map_grids). +// The anomalous map is there only where the merge kept the Bijvoet split. +struct ModelMaps { + MtzTable coefficients; + std::optional> map_2fofc, map_fofc, map_anom; +}; + // Result of validating merged intensities against an atomic model. struct ModelValidationResult { bool ok = false; @@ -58,7 +68,11 @@ struct ModelValidationResult { double radial_misfit = NAN; int shell_scale_bins = 0; double mean_atom_density_sigma = 0.0; // mean 2mFo-DFc value at atom centres, in sigma - std::string maps_prefix; // where the .ccp4 / _maps.mtz were written + // The maps, where this validation made them; the caller takes them over (null after that). + std::shared_ptr maps; + // Where the caller puts the maps beside the reflection files: "" for the answer's, "_mr" for the + // model's choice. Set by the caller; the report names . + std::string maps_suffix; // The rigid-body placement of the model against these data, where it was committed. The step is // taken only when it lowers R-free, so an unmoved model means the data said no, not that nothing @@ -211,8 +225,8 @@ struct ModelValidationResult { // Given merged intensities and an atomic model, scale the model structure factors (with an // optimized flat bulk-solvent contribution and an overall anisotropic B) to the WORKING observed -// amplitudes, then report R-work / R-free and write 2Fo-Fc and Fo-Fc electron-density maps -// (CCP4) plus an MTZ of map coefficients next to output_prefix. Nothing the reported R-free is +// amplitudes, then report R-work / R-free and make 2Fo-Fc and Fo-Fc electron-density maps and +// their coefficients (ModelValidationResult::maps). Nothing the reported R-free is // computed against was fitted on the free reflections: the scale, like the rigid-body placement // below, sees the working set only and is then applied to all of them. // @@ -266,10 +280,10 @@ struct ModelFrameForecast { struct ModelValidationSchedule { // Called once, before the null, with the forecast above. std::function on_forecast; - // Where valid, waited on before the first file is written: false returns without writing any. - std::shared_future write_gate; + // Where valid, waited on before the maps are made: false returns without making any. + std::shared_future maps_gate; // Asked once the verdict, the setting and the indexing are decided, before any map is made: true - // where another validation will write this one's maps and map coefficients, which are then not made. + // where another validation's maps and map coefficients are the ones kept, so this one's are not made. std::function maps_superseded; // The null that says whether the model fits. False where the verdict is another validation's // (KeepModelVerdict) or was reached elsewhere (a molecular-replacement placement, scored for its maps). @@ -280,6 +294,8 @@ struct ModelValidationSchedule { // GPU memory another validation running beside this one holds there. It limits how many engines // this one takes and never what runs on the GPU, so no number depends on whether the other runs. size_t gpu_beside_bytes = 0; + // Keep the maps themselves (ModelMaps), not only their coefficients: rugnux writes them as .ccp4. + bool keep_map_grids = false; }; // What a model validation needs of its card, part by part: the bytes of one engine of each, how many @@ -330,7 +346,6 @@ ModelValidationGPUPlan PlanModelValidationGPU(const ModelValidationGPUNeed &need ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, const RugnuxModel &model, - const std::string &output_prefix, Logger &logger, const gemmi::SpaceGroup *data_space_group = nullptr, bool probe_indexing_ambiguity = true, diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index ec3084643..b4e9daf2a 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -246,7 +246,7 @@ void JFJochReceiverRugnux::RugnuxThread() { parallel_detail::WorkerPool::Instance().SetNice(10); try { const auto dataset = reader.GetDataset(); - const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, scratch_dir + "/p", model); + const RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, model); LiveObserver observer(plots, scan_result, *az_int_mapping, live_status, logger); ProcessResult result; { @@ -288,7 +288,7 @@ void JFJochReceiverRugnux::ProbeThread() { const int64_t frames = std::lround(PROBE_DEG / std::abs(gonio->GetIncrement_deg())); if (2 * frames > static_cast(reader.GetNumberOfImages())) return; // a sweep this short is decided on soon enough by the run itself - RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, "", nullptr); + RugnuxLiveSetup setup = MakeRugnuxLiveSetup(*dataset, rugnux_nthreads, nullptr); setup.config.end_image = static_cast(frames); setup.config.rotation_postrefine_geometry = false; setup.config.run_scaling = false; diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index 83444105f..448975f01 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -31,13 +31,18 @@ ADD_LIBRARY(JFJochRugnux STATIC BrokerFeed.h ) -TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter JFJochModelWriter JFJochPreview gemmi fftw3f +TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi fftw3f httplib::httplib) +# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux +# returns: the library itself writes none. +ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h) +TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter) + # rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the # default), azint, scale (re-scale/merge stored reflections) or calibration (powder-ring geometry). ADD_EXECUTABLE(rugnux rugnux_cli.cpp RugnuxConsole.cpp RugnuxConsole.h) -TARGET_LINK_LIBRARIES(rugnux JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter) +TARGET_LINK_LIBRARIES(rugnux JFJochRugnuxOutput JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter) INSTALL(TARGETS rugnux RUNTIME COMPONENT rugnux) # macOS: sign the installed binary with the Developer ID (hardened runtime, secure timestamp) so the # .tgz can be notarized - see JFJOCH_MACOS_SIGNING_IDENTITY in the top-level CMakeLists.txt. The diff --git a/rugnux/DiagnosticOutput.cpp b/rugnux/DiagnosticOutput.cpp index 96557552d..7083bccaa 100644 --- a/rugnux/DiagnosticOutput.cpp +++ b/rugnux/DiagnosticOutput.cpp @@ -5,7 +5,6 @@ #include #include -#include #include @@ -103,17 +102,13 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment, return WriteJPEGToMem(rgb_image, 85); } -void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plot, - const std::optional &goniometer, size_t image_count) { - const std::string path = output_prefix + "_plot.txt"; - std::ofstream file(path, std::ofstream::out | std::ofstream::trunc); - if (!file) - throw JFJochException(JFJochExceptionCategory::FileWriteError, - "Cannot open " + path + " for writing"); +std::string PerImagePlotText(const PerImagePlot &plot) { + const std::optional &goniometer = plot.goniometer; + std::string file; // One comment line, its names over their own columns, and nothing else a plotting program has // to be told to skip. The '#' takes the first character of the image-number field. - file << "#" << fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}" + file += "#" + fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}" " {:>7} {:>11}\n", "image", "angle_deg", "bkg", "resolution_A", "spots", "scale_G", "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition"); @@ -125,7 +120,7 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo || std::any_of(plot.scale_g.begin(), plot.scale_g.end(), [](float g) { return std::isfinite(g); }); - for (size_t i = 0; i < image_count; i++) { + for (size_t i = 0; i < plot.image_count; i++) { const float angle = goniometer ? goniometer->GetAngle_deg(static_cast(i)) + goniometer->GetWedge_deg() / 2.0f @@ -152,11 +147,12 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo } else if (merge_ran && std::isfinite(scale_g)) disposition = 1.0f; - file << fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}" + file += fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}" " {:>12.4f} {:>8} {:>7} {:>11}\n", i, angle, Element(plot.bkg_estimate, i), Element(plot.resolution_A, i), Element(plot.spot_count, i), scale_g, Element(plot.mosaicity_deg, i), Element(plot.cc_to_merge, i), Element(plot.cc_n, i), merge_ran ? std::to_string(merged) : "nan", disposition); } + return file; } diff --git a/rugnux/DiagnosticOutput.h b/rugnux/DiagnosticOutput.h index 0566b2da5..04d41d4e2 100644 --- a/rugnux/DiagnosticOutput.h +++ b/rugnux/DiagnosticOutput.h @@ -40,9 +40,10 @@ struct PerImagePlot { std::vector cc_to_merge; std::vector cc_n; std::vector frame_disposition; + std::optional goniometer; // the angle column; none: nan + size_t image_count = 0; // the rows }; -// Writes _plot.txt: one '#' legend line naming the columns, then one whitespace- -// separated row per image, so `plot '' using 1:4` works with nothing in between. -void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plot, - const std::optional &goniometer, size_t image_count); +// The text of _plot.txt: one '#' legend line naming the columns, then one whitespace-separated +// row per image, so `plot '' using 1:4` works with nothing in between. +std::string PerImagePlotText(const PerImagePlot &plot); diff --git a/rugnux/ResultReport.cpp b/rugnux/ResultReport.cpp index 9e56e7da6..61d8a5203 100644 --- a/rugnux/ResultReport.cpp +++ b/rugnux/ResultReport.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -1459,7 +1458,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, domains ? ", or spots of the further lattice domains found land on those positions" : "", result.doubled_cell ? fmt::format("Both cells were merged; the other one is in {}.mtz - compare them in refinement", - result.doubled_cell->other_prefix) + output_prefix + result.doubled_cell->other_suffix) : fmt::format("The doubled cell {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} was not merged", sc.doubled_cell.a, sc.doubled_cell.b, sc.doubled_cell.c, sc.doubled_cell.alpha, sc.doubled_cell.beta, sc.doubled_cell.gamma))); @@ -1467,7 +1466,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, const auto &h = *result.doubled_cell; Add(s, KeyBool("DOUBLED_CELL_ADOPTED", h.adopted)); Add(s, KeyText("DOUBLED_CELL_DECISION", h.decision)); - Add(s, KeyText("DOUBLED_CELL_OTHER_FILE", h.other_prefix + ".mtz")); + Add(s, KeyText("DOUBLED_CELL_OTHER_FILE", output_prefix + h.other_suffix + ".mtz")); Add(s, KeyText("DOUBLED_CELL_OTHER_CELL", CellString(h.other_cell))); Add(s, KeyText("DOUBLED_CELL_OTHER_SPACE_GROUP", h.other_space_group)); Add(s, KeyReal("DOUBLED_CELL_OTHER_D_MIN", h.other_d_min_A, "{:.2f}")); @@ -1932,7 +1931,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, } if (result.twin_subgroup_merge) { const auto &t = *result.twin_subgroup_merge; - Add(s, KeyText("TWIN_SUBGROUP_FILE", t.file_prefix + ".mtz")); + Add(s, KeyText("TWIN_SUBGROUP_FILE", output_prefix + t.file_suffix + ".mtz")); Add(s, KeyText("TWIN_SUBGROUP_SPACE_GROUP", t.space_group)); Add(s, KeyText("TWIN_SUBGROUP_LAW", t.twin_law)); Add(s, KeyReal("TWIN_SUBGROUP_FRACTION", t.twin_fraction, "{:.3f}")); @@ -1952,7 +1951,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, " as _pdbx_reflns_twin). Refine against it with that twin law where the model prefers it.\n" " Nothing in this report or in the answer's files depends on it.{}", t.adopted_space_group, t.reason, t.space_group, t.adopted_space_group, t.space_group, - t.space_group, t.file_prefix, t.twin_law, t.twin_fraction, + t.space_group, output_prefix + t.file_suffix, t.twin_law, t.twin_fraction, t.other_subgroups.empty() ? std::string() : "\n Other subgroups the search confirmed, each a twin hypothesis of\n" " its own (re-run with -S to merge in one): see\n" @@ -2441,12 +2440,12 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, mv.indexing_margin_null_sd), true)); Add(s, KeyReal("MODEL_INDEXING_MARGIN_SIGMA", mv.indexing_margin_sigma, "{:+.2f}", true)); } - if (!mv.maps_prefix.empty()) - Add(s, KeyText("MAPS_PREFIX", mv.maps_prefix)); + if (mv.ok && !output_prefix.empty()) + Add(s, KeyText("MAPS_PREFIX", output_prefix + mv.maps_suffix)); // The model's choice of another group or of the twin hypothesis' subgroup, by either route: files of // their own, never the answer's. - if (!result.model_prior_file_prefix.empty()) { - Add(s, KeyText("MODEL_CHOICE_FILE_PREFIX", result.model_prior_file_prefix)); + if (!result.model_prior_file_suffix.empty()) { + Add(s, KeyText("MODEL_CHOICE_FILE_PREFIX", output_prefix + result.model_prior_file_suffix)); Add(s, KeyText("MODEL_CHOICE_SPACE_GROUP", result.model_prior_file_space_group)); } // The twin hypothesis (section 4): the model scored against that merge too, and which of @@ -2474,7 +2473,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, " fraction a scanned on the working set. The maps and the placed model come from {},\n" " the merge the model fits better on R-free ({:.4f} against {:.4f}). This chooses the\n" " maps only: the answer's reflection files and every number above are the answer's.", - t.space_group, t.file_prefix, t.twin_law, + t.space_group, output_prefix + t.file_suffix, t.twin_law, t.maps_from_this_merge ? "the merge in " + t.space_group : "the answer's merge", tv.twin_target ? std::min(tv.r_free, tv.r_free_twin) : tv.r_free, mv.r_free))); } @@ -2704,7 +2703,8 @@ ReportDocument BuildReportDocument(const std::string &output_prefix, " atoms that is not inverted (ENANTIOMORPH_VETOED otherwise) - and nothing else; another group it\n" " chooses is written as {}.mtz/.cif with the placed model.", mp.d_max_search, mp.d_min_search, - result.model_prior_file_prefix.empty() ? std::string("_mr") : result.model_prior_file_prefix))); + result.model_prior_file_suffix.empty() ? std::string("_mr") + : output_prefix + result.model_prior_file_suffix))); Add(s, Prose(fmt::format( " Seconds: model {:.2f}, transform {:.2f}, rotation {:.2f}, translation {:.2f}, rigid body and\n" " R-free {:.2f}, null {:.2f}, PAE domains {:.2f}, hand check {:.3f}.", mp.t_model, mp.t_transform, @@ -3136,26 +3136,3 @@ std::string RenderResultReport(const std::string &output_prefix, BuildReportDocument(output_prefix, input_file, experiment, result, provenance), provenance.developer); } - -ReportDocument WriteResultReport(const std::string &output_prefix, - const std::string &input_file, - const DiffractionExperiment &experiment, - const ProcessResult &result, - Logger &logger, - const RunProvenance &provenance) { - ReportDocument doc = BuildReportDocument(output_prefix, input_file, experiment, result, provenance); - if (output_prefix.empty()) - return doc; // "compute the statistics, persist nothing" - - const std::string filename = output_prefix + "_report.txt"; - // The report is unconditional, so it must never be the reason a run fails: a run that produced a - // good .mtz must survive an unwritable path or a full disk. Report the failure and carry on. - try { - std::ofstream file(filename); - file.exceptions(std::ios::failbit | std::ios::badbit); - file << RenderReportText(doc, provenance.developer); - } catch (const std::exception &e) { - logger.Warning("Could not write the results report {}: {}", filename, e.what()); - } - return doc; -} diff --git a/rugnux/ResultReport.h b/rugnux/ResultReport.h index d27bd3eb5..bdbbe1b72 100644 --- a/rugnux/ResultReport.h +++ b/rugnux/ResultReport.h @@ -72,14 +72,3 @@ std::string RenderResultReport(const std::string &output_prefix, const DiffractionExperiment &experiment, const ProcessResult &result, const RunProvenance &provenance = {}); - -// Renders and writes _report.txt, and returns the document it rendered (the CLI's -// closing summary reads it). Writes nothing when the prefix is empty (the "compute statistics, -// persist nothing" mode). Never throws: a run that produced good reflections must not be lost -// because a side file could not be written, so a failure is logged and swallowed. -ReportDocument WriteResultReport(const std::string &output_prefix, - const std::string &input_file, - const DiffractionExperiment &experiment, - const ProcessResult &result, - Logger &logger, - const RunProvenance &provenance = {}); diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 23c1dfedb..899a7c4de 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -40,7 +38,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" @@ -752,7 +749,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b const bool calibration = (config_.mode == ProcessMode::Calibration); const bool calibration_spots = calibration && config_.calibration_method == CalibrationMethod::Spots; const bool per_image_analysis = full || calibration; - const bool write_files = write_output && !config_.output_prefix.empty(); + // The output pass makes what the run returns to its caller (ProcessResult::outputs). + if (write_output && !geometry_prepass) + result.outputs = std::make_shared(); // Set below the first merge, once the pass can tell that RunAllPasses is going to throw it away and // run again - see there for what that turns off and why it is knowable at that point. bool superseded = false; @@ -762,9 +761,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b experiment_.BitDepthImage(32).PixelSigned(true); experiment_.Mode(DetectorMode::Standard); experiment_.OverwriteExistingFiles(true); - // Offline processing: the output prefix is a local path the operator chose (rugnux -o or the - // viewer's "next to the input file"), so use the trusted setter that permits an absolute path. - experiment_.FilePrefixTrusted(config_.output_prefix.empty() ? "output" : config_.output_prefix); experiment_.SetFileWriterFormat(FileWriterFormat::NXmxLegacy); experiment_.ImagesPerTrigger(images_to_process); experiment_.NumTriggers(1); @@ -838,7 +834,8 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b JFJochReceiverPlots plots; plots.Setup(experiment_, mapping); - // Output file (NXmxIntegrated master that links back to the original images). + // The per-image analysis' start message (an NXmxIntegrated master that links back to the original + // images); the image sink names the file. StartMessage start_message; experiment_.FillMessage(start_message); start_message.arm_date = dataset->arm_date; @@ -889,7 +886,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b } // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing - // but the process file reads them. Copy them where a writer is actually built rather than here: + // but the process file reads them. Copy them where the start message is handed on rather than here: // the default merging path writes no process file at all. auto fill_writer_maps = [&] { start_message.pixel_mask["default"] = pixel_mask_.GetMask(experiment_); @@ -897,18 +894,18 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b start_message.roi_map = experiment_.ExportROIMap(); }; - std::unique_ptr writer; + // The output pass streams its per-image analysis to the caller's image sink, where there is one. + RugnuxImageSink *const sink = write_output && !geometry_prepass ? config_.image_sink : nullptr; std::unique_ptr writer_queue; // Whether anything is in place to break the indexing ambiguity, for the warning further down: a // reference MTZ, or (serial stills) the reference computed from a model just below. bool ambiguity_reference = !config_.reference_data.empty(); - if (write_files && config_.write_process_h5) { + if (sink) { fill_writer_maps(); - writer = std::make_unique(start_message, /*check_overwrite_at_start=*/true, - /*trusted_path=*/true); + sink->Start(start_message, output_label_); // Deep enough that a worker never waits for the writer in the normal case, shallow enough that // the backlog is bounded by the worker count rather than by the length of the run. - writer_queue = std::make_unique(*writer, 4 * std::max(1, config_.nthreads)); + writer_queue = std::make_unique(*sink, 4 * std::max(1, config_.nthreads)); } const char *mode_name = full ? "full analysis" @@ -1002,26 +999,23 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image; // Declared here rather than where they are first filled, so that the phases below can share them - // through PipelineLocals: the bytes the image loop read, the end message (also written to the file), - // and the unmerged MTZ when it is built beside the reflection files (see there) with the - // experiment it is built from - which it points into for its space group, so it lives until the - // file is written. + // through PipelineLocals: the bytes the image loop read, the end message (also handed to the sink), + // and the unmerged MTZ when it is built beside the reflection files (see there). std::atomic total_uncompressed_bytes = 0; EndMessage end_msg; - std::optional unmerged_experiment; - std::future unmerged_mtz; + std::future unmerged_mtz; PipelineLocals p{ .logger = logger, .result = result, .observer = observer, .geometry_prepass = geometry_prepass, .full = full, .calibration = calibration, .calibration_spots = calibration_spots, .per_image_analysis = per_image_analysis, - .write_files = write_files, .start_image = start_image, .end_image = end_image, + .write_output = write_output, .start_image = start_image, .end_image = end_image, .images_to_process = images_to_process, .prepass_images = prepass_images, .loop_end = loop_end, .dataset = dataset, .mapping = mapping, .plots = plots, - .writer = writer, .writer_queue = writer_queue, .indexer_pool = indexer_pool, .indexer = indexer, + .sink = sink, .writer_queue = writer_queue, .indexer_pool = indexer_pool, .indexer = indexer, .indexing_bound_A = indexing_bound_A, .census_future = census_future, .superseded = superseded, .ambiguity_reference = ambiguity_reference, .start_time = start_time, .total_uncompressed_bytes = total_uncompressed_bytes, .end_msg = end_msg, - .unmerged_experiment = unmerged_experiment, .unmerged_mtz = unmerged_mtz}; + .unmerged_mtz = unmerged_mtz}; // First pass of two-pass rotation indexing (full analysis only). if (full && force_rotation_result_.has_value()) { @@ -1098,47 +1092,42 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b return result; } - TimingMark output_mark(logger, "output files"); + TimingMark output_mark(logger, "outputs"); // Unmerged observations. After the merge, because that is where the space group is settled and // the file has to be written in it, but from the integrated observations rather than the merged // ones: the partiality and the per-image scale are left for the reading program, which fits a - // scale model of its own. write_files, not write_output - without an output prefix there is - // nowhere to put it, and the name would come out as a bare "_unmerged.mtz". - if (full && !cancelled_ && write_files && !geometry_prepass && !superseded && result.consensus_cell) { + // scale model of its own. + if (full && !cancelled_ && result.outputs && !superseded && result.consensus_cell) { if (config_.export_unmerged) { - if (observer) observer->OnPhase("Writing unmerged reflections"); - const std::string path = config_.output_prefix + "_unmerged.mtz"; - if (unmerged_mtz.valid()) { - gemmi::Mtz mtz = unmerged_mtz.get(); - SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome()); - mtz.write_to_file(path); - } else - WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell, - experiment_, true, config_.provenance, path, config_.nthreads); - logger.Info("Unmerged observations written to {}", path); + if (observer) observer->OnPhase("Unmerged reflections"); + MtzTable mtz = unmerged_mtz.valid() + ? unmerged_mtz.get() + : UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell, + experiment_, true, config_.provenance, config_.nthreads); + SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome()); + result.outputs->unmerged = std::move(mtz); } if (config_.export_unmerged_partials) { - if (observer) observer->OnPhase("Writing unmerged reflections"); - const std::string path = config_.output_prefix + "_unmerged_partials.mtz"; - WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell, - experiment_, false, config_.provenance, path, config_.nthreads); - logger.Info("Unmerged partials written to {}", path); + if (observer) observer->OnPhase("Unmerged reflections"); + MtzTable mtz = UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell, + experiment_, false, config_.provenance, config_.nthreads); + SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome()); + result.outputs->unmerged_partials = std::move(mtz); } } // When merging is on, the _process.h5 is skipped because the merged reflections are the wanted // output and the per-image file is large. If nothing indexed there are no merged reflections, so - // the run would finish successfully having written no file at all. Write one now: the per-image - // messages went past while there was no writer, but the end message still carries the per-image + // the run would finish successfully having written no file at all. Hand the caller one: the + // per-image messages went past with no sink, but the end message still carries the per-image // scalars (spot counts, background, resolution estimate) and the dataset metadata - which is what // anyone asking "why did nothing index" needs, and it is small precisely because nothing indexed. - if (!writer && full && write_files && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) { - logger.Warning("No image indexed, so there are no merged reflections to write - writing " - "{}_process.h5 with the per-image analysis instead", config_.output_prefix); + if (!sink && full && result.outputs && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) { + logger.Warning("No image indexed, so there are no merged reflections - the per-image analysis is " + "kept for a _process.h5 instead"); fill_writer_maps(); - writer = std::make_unique(start_message, /*check_overwrite_at_start=*/true, - /*trusted_path=*/true); + result.outputs->process_start = start_message; } // The per-frame sigma_M the merge wrote back onto the outcomes (the _plots.txt column), for the report. @@ -1154,22 +1143,13 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b result.mosaicity_deg_p90 = mos[mos.size() * 9 / 10]; } } - // The two files a person checks the run by eye with. Not on the geometry pre-pass or a pass that - // is thrown away, which would otherwise leave a stale _01 copy beside the real one. The table is - // written on a stills run too - it is the only per-image scaling table there is, and the columns + // The two files a person checks the run by eye with. Not on a pass that is thrown away. The table + // is made on a stills run too - it is the only per-image scaling table there is, and the columns // a stills run cannot fill come out as nan; the picture is rotation-only at its source, since // only the rotation pre-scan builds the projection it draws. - if (full && write_files && !cancelled_ && !geometry_prepass && !superseded) { - const std::string jpeg = detector_diagnostic_jpeg_.valid() ? detector_diagnostic_jpeg_.get() : std::string(); - if (!jpeg.empty()) { - const std::string path = config_.output_prefix + "_detector.jpg"; - std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if (f) { - f.write(jpeg.data(), static_cast(jpeg.size())); - logger.Info("Detector diagnostic image written to {}", path); - } else - logger.Warning("Cannot open {} for writing", path); - } + if (full && result.outputs && !cancelled_ && !superseded) { + if (detector_diagnostic_jpeg_.valid()) + result.outputs->detector_jpeg = detector_diagnostic_jpeg_.get(); PerImagePlot plot; plots.GetPlotRaw(plot.bkg_estimate, PlotType::BkgEstimate, ""); @@ -1188,17 +1168,17 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b } if (result.has_merge_statistics) plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition; - WritePerImagePlot(config_.output_prefix, plot, experiment_.GetGoniometer(), - static_cast(images_to_process)); - logger.Info("Per-image table written to {}_plot.txt", config_.output_prefix); + plot.goniometer = experiment_.GetGoniometer(); + plot.image_count = static_cast(images_to_process); + result.outputs->plot = std::move(plot); } - if (writer) { - writer->WriteHDF5(end_msg); - writer->Finalize(); - result.written_master_path = config_.output_prefix + "_process.h5"; - } + if (sink) + sink->End(end_msg); result.end_message = end_msg; + // A pass that is to be run again makes nothing. + if (superseded) + result.outputs.reset(); output_mark.Stop(); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index c70acc7da..505e637ee 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -44,6 +44,7 @@ #include "../image_analysis/WriteReflections.h" // OutputProvenance #include "../image_analysis/bragg_integration/SpotWidth.h" // FluxCurve #include "../image_analysis/bragg_integration/SpotFootprint.h" // FootprintSpot +#include "DiagnosticOutput.h" // PerImagePlot class JFJochReader; class JFJochHDF5Reader; @@ -51,17 +52,31 @@ struct JFJochReaderRawImage; struct PipelineLocals; // RugnuxPipeline.h, private to the implementation // Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux (all of its --mode -// values that read images) and (later) the viewer. The full workflow lives here, not in the CLIs: +// values that read images), the viewer and the broker. The full workflow lives here, not in the CLIs: // setup, an optional two-pass rotation-indexing pre-pass, a parallel per-image loop (std::thread), -// an optional scaling/merging post-pass, and the _process.h5 output. The detector geometry and all -// algorithm settings are configured on the DiffractionExperiment by the caller; ProcessConfig only -// carries run control. Cancellable from any thread (e.g. SIGINT or a GUI button) via Cancel(). +// an optional scaling/merging post-pass. It reads and writes no file: what it makes is returned in +// ProcessResult (RugnuxOutputs) and handed to RugnuxImageSink, and the caller writes it +// (RugnuxOutputFiles.h). The detector geometry and all algorithm settings are configured on the +// DiffractionExperiment by the caller; ProcessConfig only carries run control. Cancellable from any +// thread (e.g. SIGINT or a GUI button) via Cancel(). enum class ProcessMode { AzimuthalIntegration, // preprocess + azimuthal integration only (rugnux --mode=azint) FullAnalysis, // spot finding + indexing + refinement + integration (rugnux --mode=mx) Calibration // detector geometry from powder rings (rugnux --mode=calibration) }; +// Where the per-image analysis of an output pass goes as it is made: rugnux and the viewer put it in +// _process.h5 (ProcessFileSink, RugnuxOutputFiles.h). Called from one thread at a time. +class RugnuxImageSink { +public: + virtual ~RugnuxImageSink() = default; + // A pass starts. `label` tells its files apart from the answer's: empty for every pass but the + // doubled cell's, which is "_doubled" (ProcessResult::DoubledCell). + virtual void Start(const StartMessage &start, const std::string &label) = 0; + virtual void Write(const DataMessage &msg) = 0; + virtual void End(const EndMessage &end) = 0; +}; + struct ProcessConfig { ProcessMode mode = ProcessMode::FullAnalysis; @@ -73,20 +88,23 @@ struct ProcessConfig { // The per-image loop is capped in that case (see RunPipeline); an explicit -N never is. bool nthreads_auto = false; - // Output prefix for the _process.h5 (and scaled reflections). Empty => process without writing. - std::string output_prefix; - // What the reflection files record about where they came from (WriteReflections). + // What the reflection files record about where they came from (the MTZ HISTORY, WriteReflections). OutputProvenance provenance; - // Write the per-image _process.h5. Defaults true; rugnux turns it off when merging - // (the .mtz/.cif is the wanted output and the h5 is large) unless --write-process-h5 is given. - bool write_process_h5 = true; + // Where the per-image analysis of each output pass goes (the _process.h5); none: it is not kept. + // rugnux gives one when not merging, or with --write-process-h5. + RugnuxImageSink *image_sink = nullptr; - // Write the merged reflections (.mtz/.cif). Defaults true (the CLI always writes them when - // merging); the viewer can turn them off independently of the _process.h5 (e.g. a quick run that - // only wants the ISa/R-meas numbers). Only relevant to FullAnalysis with scaling. + // Make the merged reflections' outputs (RugnuxOutputs::merged and the twin hypothesis' and finalist + // ledger's merges). Defaults true (the CLI always writes them when merging); the viewer can turn them + // off (e.g. a quick run that only wants the ISa/R-meas numbers). Only relevant to FullAnalysis with + // scaling. bool write_merged = true; + // Keep the model validation's maps themselves (ModelMaps), not only their coefficients: rugnux writes + // them as .ccp4. The broker never asks for them. + bool keep_map_grids = false; + SpotFindingSettings spot_finding; // FullAnalysis spot finding (and Calibration by spots) // Calibration: how the powder rings are measured, and the calibrant's rings as q = 2*pi/d (1/A). @@ -191,15 +209,16 @@ struct ProcessConfig { std::string reference_path; std::string reference_column; - // Diagnostic: if set, the -P rot3d combine writes the unmerged fulls here (for comparison vs XDS). - std::string observation_dump_path; + // Diagnostic: if set, handed the unmerged fulls of every -P rot3d combine as text, for comparison + // with XDS (--dump-observations). + std::function observation_dump; - // Write the integrated observations as _unmerged.mtz, for aimless / pointless / careless - // and anything else that scales unmerged data itself - each rocking event summed into one full. - // On by default (--no-export-unmerged turns it off): it is how a run's observations reach the - // rest of the field, and a user who does not know the flag exists should still get the file. - // It is the largest file a run produces, so a batch that only wants the merged numbers turns it - // off. Needs an output prefix. + // Make the integrated observations as _unmerged.mtz (RugnuxOutputs::unmerged), for aimless / + // pointless / careless and anything else that scales unmerged data itself - each rocking event summed + // into one full. On by default (--no-export-unmerged turns it off): it is how a run's observations + // reach the rest of the field, and a user who does not know the flag exists should still get the + // file. It is the largest file a run produces, so a batch that only wants the merged numbers turns + // it off. bool export_unmerged = true; // The same as _unmerged_partials.mtz, with each partial written as its own row for the @@ -322,6 +341,48 @@ struct SearchMergeSignal { int64_t inner_reflections = 0; // ... of this many reflections }; +// The reflection files of one merge, .mtz and .cif (and the answer's .hkl): the MTZ as its +// table, and what the mmCIF and the HKLF-4 file are written from (WriteReflections.h). +struct MergedOutput { + std::string suffix; // "", "_twin_" or "_mr" + MtzTable mtz; + std::vector reflections; + UnitCell cell{}; + DiffractionExperiment experiment; // in the group the files are written in + MergeStatistics statistics; + ErrorModelReport error_model; + TwinningAnalysisResult twinning; // a twin hypothesis' law and fraction, for its mmCIF + std::vector scaled_fulls; // the .hkl's records, where the merge made them (rotation) +}; + +// An atomic model as placed against the data (_model.cif, WritePlacedModel): `cell` and +// `space_group` are those of the reflection files beside it. +struct PlacedModelOutput { + std::shared_ptr model; + UnitCell cell{}; + gemmi::SpaceGroup space_group; +}; + +// What a run makes for its caller to keep - the files rugnux writes, each named here after the file it +// goes to at the caller's prefix. Made by the output pass, the one whose result the run returns. +struct RugnuxOutputs { + std::optional merged; // .mtz/.cif/.hkl + std::optional twin; // the twin hypothesis' merge + std::optional model_prior; // the model's choice, _mr.mtz/.cif + std::shared_ptr maps; // _maps.mtz, _2fofc/_fofc/_anom.ccp4 + std::shared_ptr model_prior_maps; // the same at _mr + std::optional placed_model; // _model.cif/.pdb + std::optional model_prior_placed_model; // _mr_model.cif/.pdb + std::optional unmerged; // _unmerged.mtz + std::optional unmerged_partials; // _unmerged_partials.mtz + std::string detector_jpeg; // _detector.jpg; empty: none drawn + std::optional plot; // _plot.txt + // A run that indexed nothing, with no per-image analysis streamed (ProcessConfig::image_sink): the + // start message of a _process.h5 holding the end message (ProcessResult::end_message), so + // there is a file that says why nothing indexed. + std::optional process_start; +}; + struct ProcessResult { bool cancelled = false; uint64_t images_processed = 0; @@ -336,6 +397,8 @@ struct ProcessResult { // The end message of this pass, as the _process.h5 records it: the lattice, refined geometry, // reindexing and the run means - what the broker puts into its own when it runs rugnux live. EndMessage end_message; + // What this pass made for the caller to keep - the output pass's only; null on every other pass. + std::shared_ptr outputs; // Workers the per-image loop actually ran, which is not config.nthreads once the loop is capped. int image_loop_threads = 0; double frame_rate_hz = 0.0; @@ -392,7 +455,6 @@ struct ProcessResult { // every other class, or where the free refinement is not available. std::optional lattice_angle_departure_deg; MeanProcessingTime mean_processing_time{}; - std::optional written_master_path; std::string merge_statistics_text; // populated when scaling/merging ran // Structured merge statistics (per-shell + overall), populated when scaling/merging ran. ISa is @@ -569,7 +631,7 @@ struct ProcessResult { std::string reason; // why the hypothesis was written std::string adopted_space_group; // the group of the answer, as written std::string space_group; // the subgroup, as written (standard setting) - std::string file_prefix; // _twin_ + std::string file_suffix; // _twin_, or _mr: the files are std::string twin_law; // hkl triplet, on the subgroup file's axes double twin_fraction = NAN; // EstimateTwinLawFraction: a lower bound int twin_fraction_pairs = 0; @@ -591,7 +653,8 @@ struct ProcessResult { struct DoubledCell { bool adopted = false; // the answer is the doubled cell, the other one the probed cell std::string decision; // why, in the words the run reports it in - std::string other_prefix; // _halfcell or _doubled + std::string other_suffix; // _halfcell or _doubled: the other cell's files are + std::shared_ptr other_outputs; UnitCell other_cell{}; std::string other_space_group; double other_d_min_A = NAN, other_cc_half = NAN, other_r_free = NAN; @@ -611,9 +674,9 @@ struct ProcessResult { std::string model_placement; // What molecular replacement found, where it ran; failure_reason says why where it could not. std::optional model_prior; - // The file prefix the model's choice of another group or the twin hypothesis' subgroup was written under - // (_mr), and that group; empty where nothing was. - std::string model_prior_file_prefix; + // The file suffix the model's choice of another group or the twin hypothesis' subgroup is written under + // (_mr), and that group; empty where nothing was. + std::string model_prior_file_suffix; std::string model_prior_file_space_group; // Space group used for the final re-scale/merge and written to the master file: determined by the @@ -883,8 +946,11 @@ class Rugnux { std::optional force_rotation_result_; // The indexing result the last full pass integrated on, which the doubled-cell hypothesis doubles. std::optional last_rotation_result_; - // Keep the written merged reflections on the result (ProcessResult::merged_reflections). + // Keep the merged reflections on the result (ProcessResult::merged_reflections). bool keep_merged_reflections_ = false; + // What the image sink is told the output pass is (RugnuxImageSink::Start): "_doubled" for the doubled + // cell's pass, empty otherwise. + std::string output_label_; // The space group the USER fixed (-S on the CLI, the settings panel in the viewer), captured before // anything can clear it - a pass that determines a group of its own writes it onto the experiment, @@ -1270,7 +1336,8 @@ class Rugnux { RugnuxObserver *observer); // One full run of the pipeline (index + integrate + optional scale/merge/space-group + output). - // write_output == false suppresses all file output (the geometry pre-pass writes nothing). + // write_output == false makes no outputs (ProcessResult::outputs) and streams nothing to the image sink; + // nor does the geometry pre-pass. // geometry_prepass == true stops after post-refining the detector geometry from the first integration // and applies it (+ the refined lattice) to experiment_, so the next pass re-integrates with it. ProcessResult RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass); @@ -1342,7 +1409,6 @@ class Rugnux { }; // What the refined passes start from. struct RefinedPassStart { - std::string base_prefix; std::optional gonio_snapshot; std::array header_geometry{}; RotationScaleOutcome scale_walk; // where the walk has ended before they start @@ -1351,8 +1417,7 @@ class Rugnux { // the rotation-scale walk and it overturned them. // After the refined passes: where the supercell probe's class stands clear, integrate and merge the // doubled cell too, and return whichever of the two is the answer (ProcessResult::doubled_cell). - ProcessResult DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half, - const std::string &base_prefix); + ProcessResult DoubledCellHypothesis(RugnuxObserver *observer, ProcessResult half); std::optional RefinedPasses(RugnuxObserver *observer, ProcessResult pass1, const RefinedPassStart &start); // The rotation-scale walk, its probes on copies of the run, while the refined passes run on a copy diff --git a/rugnux/RugnuxCalibration.cpp b/rugnux/RugnuxCalibration.cpp index 334d5d6e8..11d3e5cb6 100644 --- a/rugnux/RugnuxCalibration.cpp +++ b/rugnux/RugnuxCalibration.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +37,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" diff --git a/rugnux/RugnuxCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp index 4dd6d6dc7..9bdb53dd1 100644 --- a/rugnux/RugnuxCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -53,6 +53,8 @@ namespace { std::string RugnuxCommandLine(const ProcessConfig &config, const DiffractionExperiment &experiment, const std::string &input_file, + const std::string &output_prefix, + bool write_process_h5, const std::string &calibrant_name) { std::vector args; const bool azint = (config.mode == ProcessMode::AzimuthalIntegration); @@ -71,8 +73,8 @@ std::string RugnuxCommandLine(const ProcessConfig &config, args.push_back(val); }; - if (!config.output_prefix.empty()) - add("-o", config.output_prefix); + if (!output_prefix.empty()) + add("-o", output_prefix); // Every CPU the run may use is what rugnux takes without -N, on whatever machine runs the command. if (!config.nthreads_auto) add("-N", std::to_string(config.nthreads)); @@ -224,7 +226,7 @@ std::string RugnuxCommandLine(const ProcessConfig &config, // When merging, the CLI skips the large _process.h5 unless asked; emit the flag when it is // wanted so a copied command matches the GUI's "Save _process.h5" choice. (write_merged has // no CLI equivalent - the CLI always writes the .mtz/.cif when merging.) - if (config.write_process_h5) + if (write_process_h5) args.emplace_back("--write-process-h5"); // The low-resolution limit is on by default at the same value in both, so emit it only when // it differs - including 0, which is how the CLI spells "no limit". diff --git a/rugnux/RugnuxCommandLine.h b/rugnux/RugnuxCommandLine.h index f387a1686..b9f202709 100644 --- a/rugnux/RugnuxCommandLine.h +++ b/rugnux/RugnuxCommandLine.h @@ -14,7 +14,11 @@ class DiffractionExperiment; // matter for the run, not every obscure flag; geometry is taken from the input file, so geometry // overrides are not emitted. calibrant_name is only used by ProcessMode::Calibration - the config // carries the calibrant's rings, and --calibrant takes the name they were resolved from. +// output_prefix and write_process_h5 are where the run puts its files and whether a merging run writes +// its _process.h5 too. std::string RugnuxCommandLine(const ProcessConfig &config, const DiffractionExperiment &experiment, const std::string &input_file, + const std::string &output_prefix, + bool write_process_h5, const std::string &calibrant_name = ""); diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 2f4840c13..74ba2d3f2 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +37,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" diff --git a/rugnux/RugnuxFirstPassAccept.cpp b/rugnux/RugnuxFirstPassAccept.cpp index f0eb897c6..aaf31b21c 100644 --- a/rugnux/RugnuxFirstPassAccept.cpp +++ b/rugnux/RugnuxFirstPassAccept.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +37,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index ccc616dfe..554ca3fb3 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +37,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index 0510a79e9..8d31c3c00 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -4,7 +4,6 @@ #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" -#include "../writer/WriteModel.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" @@ -19,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +37,6 @@ #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" -#include "../writer/FileWriter.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" @@ -713,7 +710,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // it is written in - the pair the volume comparisons in Run() need. result.consensus_centering = rot->search_result.centering; // Log only: what the rest of the sweep says about the decisions taken on its first part. - if (!p.geometry_prepass && p.write_files && experiment_.IsRotationIndexing()) + if (!p.geometry_prepass && p.write_output && experiment_.IsRotationIndexing()) FarEndCheck(p, *rot); } result.consensus_cell = indexer->GetConsensusUnitCell(); diff --git a/rugnux/RugnuxLive.cpp b/rugnux/RugnuxLive.cpp index b05cf60b8..df46c56e6 100644 --- a/rugnux/RugnuxLive.cpp +++ b/rugnux/RugnuxLive.cpp @@ -6,7 +6,7 @@ #include "RugnuxDefaults.h" RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, - const std::string &output_prefix, std::shared_ptr model) { + std::shared_ptr model) { RugnuxLiveSetup s; // The experiment as rugnux_cli.cpp sets it up for a rotation sweep with no options given. @@ -36,8 +36,6 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr ProcessConfig &c = s.config; c.mode = ProcessMode::FullAnalysis; c.nthreads = nthreads; - c.output_prefix = output_prefix; - c.write_process_h5 = false; c.export_unmerged = false; c.spot_finding = RugnuxDefaultSpotFinding(true); c.rotation_indexing = true; diff --git a/rugnux/RugnuxLive.h b/rugnux/RugnuxLive.h index c1e0820e8..c24df0e26 100644 --- a/rugnux/RugnuxLive.h +++ b/rugnux/RugnuxLive.h @@ -9,13 +9,13 @@ #include "../reader/JFJochReaderDataset.h" // A rotation collection processed inside the broker while its frames arrive: what a bare `rugnux` on -// the same file would set, with the run made fast_live. The output goes to output_prefix - a place -// private to the broker, from which it builds the END message - and the model, where one is given, -// is placed and validated as `rugnux --model` does. +// the same file would set, with the run made fast_live. What it makes is returned in memory +// (ProcessResult::outputs), for the broker to build the END message from, and the model, where one is +// given, is placed and validated as `rugnux --model` does. struct RugnuxLiveSetup { DiffractionExperiment experiment; ProcessConfig config; }; RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthreads, - const std::string &output_prefix, std::shared_ptr model); + std::shared_ptr model); diff --git a/rugnux/RugnuxOutputFiles.cpp b/rugnux/RugnuxOutputFiles.cpp new file mode 100644 index 000000000..1570f47e0 --- /dev/null +++ b/rugnux/RugnuxOutputFiles.cpp @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxOutputFiles.h" + +#include +#include + +#include + +#include "../writer/FileWriter.h" +#include "../writer/WriteModel.h" +#include "../writer/WriteRugnuxFiles.h" + +namespace { + +void WriteCcp4(const gemmi::Grid &map, const std::string &path) { + gemmi::Ccp4 ccp4; + ccp4.grid = map; + ccp4.update_ccp4_header(2); + ccp4.write_ccp4_map(path); +} + +// The files of one output pass at `prefix`, all but its _process.h5. +void WriteOutputs(const RugnuxOutputs &o, const std::string &prefix, const OutputProvenance &provenance, + size_t nthreads, Logger &logger) { + if (o.merged) + WriteMergedOutput(*o.merged, prefix, provenance, nthreads, /*hkl=*/true); + if (o.twin) + WriteMergedOutput(*o.twin, prefix, provenance, nthreads, /*hkl=*/false); + if (o.model_prior) + WriteMergedOutput(*o.model_prior, prefix, provenance, nthreads, /*hkl=*/false); + if (o.maps) + WriteModelMaps(*o.maps, prefix, logger); + if (o.model_prior_maps) + WriteModelMaps(*o.model_prior_maps, prefix + "_mr", logger); + if (o.placed_model) + WritePlacedModel(*o.placed_model->model, o.placed_model->cell, o.placed_model->space_group, prefix, logger); + if (o.model_prior_placed_model) + WritePlacedModel(*o.model_prior_placed_model->model, o.model_prior_placed_model->cell, + o.model_prior_placed_model->space_group, prefix + "_mr", logger); + if (o.unmerged) { + WriteMtz(*o.unmerged, prefix + "_unmerged.mtz"); + logger.Info("Unmerged observations written to {}_unmerged.mtz", prefix); + } + if (o.unmerged_partials) { + WriteMtz(*o.unmerged_partials, prefix + "_unmerged_partials.mtz"); + logger.Info("Unmerged partials written to {}_unmerged_partials.mtz", prefix); + } + if (!o.detector_jpeg.empty()) { + const std::string path = prefix + "_detector.jpg"; + std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); + if (f) { + f.write(o.detector_jpeg.data(), static_cast(o.detector_jpeg.size())); + logger.Info("Detector diagnostic image written to {}", path); + } else + logger.Warning("Cannot open {} for writing", path); + } + if (o.plot) { + WritePerImagePlot(prefix, *o.plot); + logger.Info("Per-image table written to {}_plot.txt", prefix); + } +} + +} // namespace + +ProcessFileSink::ProcessFileSink(std::string prefix) : prefix_(std::move(prefix)) {} + +ProcessFileSink::~ProcessFileSink() = default; + +void ProcessFileSink::Start(const StartMessage &start, const std::string &label) { + writer_.reset(); + StartMessage s = start; + s.file_prefix = prefix_ + label; + writer_ = std::make_unique(s, /*check_overwrite_at_start=*/true, /*trusted_path=*/true); +} + +void ProcessFileSink::Write(const DataMessage &msg) { + writer_->Write(msg); +} + +void ProcessFileSink::End(const EndMessage &end) { + writer_->WriteHDF5(end); + writer_->Finalize(); + writer_.reset(); +} + +void WriteRugnuxOutputs(const ProcessResult &result, const std::string &prefix, const OutputProvenance &provenance, + size_t nthreads, Logger &logger) { + if (result.outputs) { + WriteOutputs(*result.outputs, prefix, provenance, nthreads, logger); + if (result.outputs->process_start) { + StartMessage start = *result.outputs->process_start; + start.file_prefix = prefix; + FileWriter writer(start, /*check_overwrite_at_start=*/true, /*trusted_path=*/true); + writer.WriteHDF5(result.end_message); + writer.Finalize(); + } + } + if (result.doubled_cell) { + const ProcessResult::DoubledCell &h = *result.doubled_cell; + if (h.other_outputs) + WriteOutputs(*h.other_outputs, prefix + h.other_suffix, provenance, nthreads, logger); + // The doubled cell's pass put its per-image analysis under a name of its own; where it is the answer, + // the two _process.h5 files trade names, as the other files of the two passes are named. + std::error_code ec; + if (h.adopted && std::filesystem::exists(prefix + "_doubled_process.h5", ec)) { + if (std::filesystem::exists(prefix + "_process.h5", ec)) + std::filesystem::rename(prefix + "_process.h5", prefix + h.other_suffix + "_process.h5", ec); + std::filesystem::rename(prefix + "_doubled_process.h5", prefix + "_process.h5", ec); + } + } +} + +void WriteMergedOutput(const MergedOutput &merged, const std::string &prefix, const OutputProvenance &provenance, + size_t nthreads, bool hkl) { + // An MTZ, an mmCIF and a SHELX HKLF-4 .hkl - each has its uses downstream (MTZ for the CCP4 / phenix + // reflection tools, mmCIF for deposition and as the self-describing native format, HKLF-4 for SHELXL + // refinement and SHELXC / ANODE substructure solution). The .hkl holds the unmerged scaled fulls where the + // merge provides them (rotation), the merged reflections otherwise (stills). + const std::string file = prefix + merged.suffix; + WriteMtz(merged.mtz, file + ".mtz"); + WriteMmcifReflections(merged.reflections, merged.cell, merged.experiment, merged.statistics, merged.error_model, + merged.twinning, provenance, file + ".cif", nthreads); + if (!hkl) + return; + if (!merged.scaled_fulls.empty()) + WriteShelxHklReflections(merged.scaled_fulls, file + ".hkl", nthreads); + else + WriteShelxHklReflections(merged.reflections, merged.experiment, file + ".hkl", nthreads); +} + +void WriteModelMaps(const ModelMaps &maps, const std::string &prefix, Logger &logger) { + WriteMtz(maps.coefficients, prefix + "_maps.mtz"); + if (maps.map_2fofc) + WriteCcp4(*maps.map_2fofc, prefix + "_2fofc.ccp4"); + if (maps.map_fofc) + WriteCcp4(*maps.map_fofc, prefix + "_fofc.ccp4"); + if (maps.map_anom) + WriteCcp4(*maps.map_anom, prefix + "_anom.ccp4"); + logger.Info("Model validation: wrote {}_maps.mtz{}", prefix, + maps.map_2fofc ? fmt::format(", {0}_2fofc.ccp4, {0}_fofc.ccp4{1}", prefix, + maps.map_anom ? ", " + prefix + "_anom.ccp4" : std::string()) + : std::string()); +} + +void WritePerImagePlot(const std::string &prefix, const PerImagePlot &plot) { + const std::string path = prefix + "_plot.txt"; + std::ofstream file(path, std::ofstream::out | std::ofstream::trunc); + if (!file) + throw JFJochException(JFJochExceptionCategory::FileWriteError, "Cannot open " + path + " for writing"); + file << PerImagePlotText(plot); +} + +ReportDocument WriteResultReport(const std::string &output_prefix, + const std::string &input_file, + const DiffractionExperiment &experiment, + const ProcessResult &result, + Logger &logger, + const RunProvenance &provenance) { + ReportDocument doc = BuildReportDocument(output_prefix, input_file, experiment, result, provenance); + if (output_prefix.empty()) + return doc; // "compute the statistics, persist nothing" + + const std::string filename = output_prefix + "_report.txt"; + // The report is unconditional, so it must never be the reason a run fails: a run that produced a + // good .mtz must survive an unwritable path or a full disk. Report the failure and carry on. + try { + std::ofstream file(filename); + file.exceptions(std::ios::failbit | std::ios::badbit); + file << RenderReportText(doc, provenance.developer); + } catch (const std::exception &e) { + logger.Warning("Could not write the results report {}: {}", filename, e.what()); + } + return doc; +} diff --git a/rugnux/RugnuxOutputFiles.h b/rugnux/RugnuxOutputFiles.h new file mode 100644 index 000000000..86f425576 --- /dev/null +++ b/rugnux/RugnuxOutputFiles.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +#include "Rugnux.h" +#include "ResultReport.h" + +class FileWriter; + +// The files of a run, written by the program that ran it - rugnux and the viewer - from what the analysis +// returned: the library itself writes none. + +//