// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../reader/JFJochHDF5Reader.h" #include "../reader/JFJochCBFReader.h" #include "../reader/JFJochMarCCDReader.h" #include "../reader/JFJochSMVReader.h" #include "../common/Logger.h" #include "../common/Definitions.h" #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" #include "../common/print_license.h" #include "../common/CUDAWrapper.h" #include "../common/JFJochMath.h" #include "../image_analysis/bragg_integration/CalcISigma.h" #include "../image_analysis/geom_refinement/Calibrants.h" #include "../image_analysis/indexing/IndexerThreadPool.h" #include "../image_analysis/LoadFCalcFromMtz.h" #include "../image_analysis/SensorAbsorption.h" #include "../image_analysis/UpdateReflectionResolution.h" #include "../image_analysis/WriteReflections.h" #include "../image_analysis/scale_merge/Merge.h" #include "../image_analysis/scale_merge/RfreeFlags.h" #include "../image_analysis/scale_merge/ReindexAmbiguity.h" // MatchReference #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" #include "../image_analysis/scale_merge/RotationScaleMerge.h" #include "../image_analysis/scale_merge/ResolutionCutoff.h" #include "../image_analysis/scale_merge/AnisotropyAnalysis.h" #include "../image_analysis/scale_merge/TwinningAnalysis.h" #include "../image_analysis/scale_merge/TranslationalNCS.h" #include "../image_analysis/scale_merge/SearchSpaceGroup.h" #include "Rugnux.h" #include "RugnuxDefaults.h" #include "ModelValidation.h" #include "WriteModel.h" #include "ResultReport.h" // Spots kept per image (the strongest ones) and handed to indexing. Offline reprocessing is not // bound by the online spot budget, so this is rugnux's own default rather than the 250 the // DatasetSettings constructor uses for the detector. Measured over the 37-crystal rotation battery, // lifting it to 1000 leaves low-resolution R_meas better or equal on every crystal (16 better, 0 // worse, the rest untouched because their frames never reach the cap), with R_meas 14/4 and ISa 14/6 // in its favour and no measurable cost in wall clock. It is also what jfjoch_viewer already sends. constexpr int64_t RUGNUX_MAX_SPOT_COUNT = 1000; // Default rot3d per-frame scale-G smoothing range (XDS DELPHI-like), in degrees of rotation. constexpr double SMOOTH_G_DEFAULT_DEG = 5.0; // Default rot3d per-batch relative-B batch width (bare --relative-b), in degrees of rotation. constexpr double RELATIVE_B_DEFAULT_DEG = 10.0; // What the run does (--mode). Each of these was once a flag of its own; one option keeps them // mutually exclusive by construction. enum class RugnuxMode { MX, Azint, Scale, Calibration }; // Azimuthal sectors used by --mode calibration --calibration rings when --azim-phi-bins was not given. // The default of 1 is a plain radial profile, in which a ring has been averaged over every direction so // nothing is left to say where its centre is (RingsFromAzimuthalProfile refuses below 4). Sweeping the // count on a LaB6 exposure and on a crystal's ice rings, 32 and above agree to about 0.1 px and the // residual is flat, while 8 sectors is visibly coarser (0.3 px away, twice the residual). Going higher // buys nothing and runs into the 65534-bin limit on q x azimuth, so 32 is the default. constexpr int CALIBRATION_AZIM_BINS_DEFAULT = 32; void print_usage() { std::cout << "Usage rugnux {} " << std::endl; std::cout << "Options:" << std::endl; std::cout << " -o, --output-prefix Output file prefix (default: output). mx and scale runs always write " "_report.txt, the results report; an empty prefix writes nothing at all" << std::endl; std::cout << " -N, --threads Number of threads (default: all hardware threads, of which the " "per-image loop uses at most 16 per GPU; an explicit value is used as given)" << std::endl; std::cout << " -s, --start-image Start image number (default: 0)" << std::endl; std::cout << " -e, --end-image End image number (default: all)" << std::endl; std::cout << " -t, --stride Image stride (default: 1)" << std::endl; std::cout << " -v, --verbose Verbose output" << std::endl; std::cout << std::endl; std::cout << " Mode" << std::endl; std::cout << " --mode What this run does (default: mx)" << std::endl; std::cout << " mx Full analysis - spot finding, indexing, integration and merging" << std::endl; std::cout << " azint Only azimuthal integration (no spot finding/indexing); writes _process.h5" << std::endl; std::cout << " scale Only re-scale/merge the already-integrated reflections in (no re-integration)" << std::endl; std::cout << " calibration Determine the detector geometry from powder rings; writes .poni for pyFAI and .json, whose dataset_settings member is a jfjoch_broker dataset_settings body as it stands" << std::endl; std::cout << std::endl; std::cout << " Calibration (--mode calibration)" << std::endl; std::cout << " --calibrant Powder standard: lab6|agbh|ceo2|si|ice (default: lab6, case-insensitive). Overridden by -C: a unit cell given there IS the standard, with its rings enumerated from the cell and its absences from -S where that is given, or taken as primitive where it is not. -S supplies SYMMETRY absences only: a standard whose extinctions come from its atomic basis rather than from a symmetry element - silicon, whose 222 is absent in Fd-3m for that reason - is better named with --calibrant" << std::endl; std::cout << " --calibration How the rings are measured: rings|spots (default: rings). rings sums the (q x azimuth) azimuthal profile over every processed image and fits the ring arcs in it; spots pools the found spots and fits those. -s/-e/--stride select the images; rings defaults --azim-phi-bins to 32" << std::endl; std::cout << " --no-refine-tilt Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance. For a calibration handed to a program that cannot express a tilted detector, e.g. XDS" << std::endl; std::cout << std::endl; std::cout << " Detector mask" << std::endl; std::cout << " --detect-beam-stop[=N|off] Find the beam stop and its holder in a projection of N images and add them to the pixel mask (bit 9), so nothing shadowed by them is integrated. ON by default (60 images); =off disables. Reflections behind the stop are attenuated but not flagged, so they are integrated low with a plausible sigma and no existing rejection catches them" << std::endl; std::cout << " --estimate-beam-center Place the beam centre before anything is indexed, and use it in place of the header value when it is measured precisely enough. On a sweep that reaches half a turn it comes from the symmetry of the spot positions - the frames 180 deg apart are each other's mirror image, and every reflection is recorded twice - and where the sweep does not reach that far, from the isotropy of the scattered background, which needs only a few frames. It reads frames of its own, chosen as pairs half a turn apart, so it does not change the mask --detect-beam-stop finds. Ignored when --beam-x/--beam-y are given, and when a stills geometry refinement has already placed the centre from indexed spots" << std::endl; std::cout << " --beam-center-search[=N|off] After a first pass that indexes fewer than half the validation frames, try the beam centre a pixel at a time out to N px along each detector axis and keep the first one that indexes a majority. A beam-centre error is fixed in the lab frame, so it smears the accumulated reciprocal-space cloud and the FFT takes an axis harmonic instead of the true axis; nothing downstream repairs that. ON by default (12 px); =off disables. It runs only after a pass that has already failed, so a run that indexes never pays for it" << std::endl; std::cout << " --beam-center-check[=off] Measure the beam centre from the isotropy of the scattered background on EVERY run, report how far the file's value is from it, and index a second first pass with it to see whether the two centres give the same lattice. The fit reads the projection --detect-beam-stop already builds, so it costs no extra frames. On a run that indexes, the measured centre is adopted where the file's centre indexes nothing and the measured one indexes a majority, and where the two centres return cells related by an integer volume factor, in either direction, and the measured centre's lattice carries materially more of the pooled spots against its own chance level - the axis harmonic a centre error along the spindle produces, which nothing downstream repairs. Where the indexing cannot tell the two centres apart - the same cell with a different metric symmetry, or the same lattice found at both only by reading less of each frame - the first pass is run at both and the run keeps the one that merges better (on the same lattice: the one that measures more reflections at I/sigma >= 2). ON by default; =off disables" << std::endl; std::cout << " --no-fit-spindle Take the goniometer axis from the file rather than measuring the spindle's rotation about the beam from the spots. Measuring it is the DEFAULT and the file is never right: every master writes an exact lab axis and no goniometer is one. Both mirror lines of the beam-centre estimator turn with the spindle, so an axis a few tenths of a milliradian out smears the vote until a neighbouring tooth wins. Only has an effect with --estimate-beam-center" << std::endl; std::cout << std::endl; std::cout << " Spot finding" << std::endl; std::cout << " --spot-sigma Noise sigma level for spot finding (default: 4.0)" << std::endl; std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << std::endl; std::cout << " --min-pix-per-spot Minimum connected strong pixels per spot. If omitted, min-pix is chosen PER IMAGE (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed count x indexed fraction is kept. Give an explicit value to force a fixed min-pix instead." << std::endl; std::cout << " --adaptive-spots Self-calibrating detection (DEFAULT): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed --spot-threshold, so one setting adapts across datasets (no per-dataset --spot-threshold/--spot-sigma tuning)." << std::endl; std::cout << " --no-adaptive-spots Turn adaptive detection off and use the fixed --spot-threshold / --spot-sigma finder instead" << std::endl; std::cout << " --spot-false-pixels Adaptive detection operating point: expected noise pixels tolerated per frame (default: 100; implies --adaptive-spots)" << std::endl; std::cout << " --spot-high-resolution High resolution limit for spot finding. If omitted (or 0), spot finding is not clipped in resolution and extends as far as the detector reaches" << std::endl; std::cout << " --spot-low-resolution Low resolution limit for spot finding, in A (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weakly-diffracting serial data; 0 removes the limit)" << std::endl; std::cout << " --max-spots Max spot count per image, the strongest ones, handed to indexing. Default: measured from the data on rotation (as deep into each image's spot list as its spots still lie on the lattice), capped at 1000" << std::endl; std::cout << " --detect-ice-rings[=on|off] Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's setting, or - where the file says nothing - on for rotation and off for stills" << std::endl; std::cout << std::endl; std::cout << " Indexing" << std::endl; std::cout << " (A dataset with a rotation goniometer axis is processed as rotation data by default; use --force-still to override)" << std::endl; std::cout << " --force-still Process a rotation (goniometer) dataset as independent stills (still indexing + per-image ScaleOnTheFly) instead of rotation" << std::endl; std::cout << " -R, --two-pass-rotation[=num] Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default: 100)" << std::endl; std::cout << " --single-pass-rotation[=num] Use online-like single-pass rotation indexing (optional: min angular range deg)" << std::endl; std::cout << " --min-indexed-spots Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead, and integration is not gated by it" << std::endl; std::cout << " --force-rotation-lattice Force rotation indexer with external lattice (in Angstrom) : \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, skips first pass)" << std::endl; std::cout << " --rotation-scale Goniometer rotation scale: the stage turned k times the angle stored in the file (the commanded one). Applied to both passes; overrides the fitted correction" << std::endl; std::cout << " --rotation-no-postrefine Disable the (default-on) two-pass rotation post-refine (post-refine detector distance/beam + cell/axis, then re-integrate; only the refined pass is written, as the canonical _* output)" << std::endl; std::cout << " -X, --indexing-algorithm Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl; std::cout << " -S, --space-group Space group number (96) or symbol (P43212) - for indexing and scaling" << std::endl; std::cout << " -C, --unit-cell Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl; std::cout << " --fft-min-unit-cell Shortest cell axis the FFT search accepts, in A (default: 10). A candidate with a shorter axis is discarded, but a de-novo rotation run also tries a second first-pass hypothesis with the floor at 5 A, so a small-molecule cell is indexed without this flag; -C lowers it on its own to cover the cell given" << std::endl; std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|flex); flex tries all three per image and keeps whichever indexes the most spots (alias: multi)" << std::endl; std::cout << " --refine-geometry[=N|off] Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default: 200) then re-indexes (lifts weak-stills indexing). Default ON for stills when a reference cell is given (-C / reference MTZ); =off disables" << std::endl; std::cout << " --index-ice-rings[=on|off] Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: off; no effect without --detect-ice-rings)" << std::endl; std::cout << std::endl; std::cout << " Scaling and merging (on by default)" << std::endl; std::cout << " --no-merge Skip scaling and merging; write only the per-image _process.h5" << std::endl; std::cout << " --scale-fulls rot3d: after the 3D combine, refit a per-frame scale on the fulls (XDS order, Unity model). Default ON for rot3d" << std::endl; std::cout << " --no-scale-fulls Disable the rot3d scale-fulls refit (it is on by default for rot3d)" << std::endl; std::cout << " --write-process-h5 Also write the (large) _process.h5 when merging (default: only .mtz/.cif when merging)" << std::endl; std::cout << " --finalist-ledger Report the full-resolution evidence for each space group the search considered, not only the one it adopted (report-only; the decision is unchanged)" << std::endl; std::cout << " --developer Write the full _report.txt: the pipeline-internal keys (the anisotropy gate, the space-group candidate and operator tables, the model-fit null, the sweep internals) and the long explanations, which the default report leaves out. Nothing is computed differently - the same report, rendered in full" << std::endl; std::cout << " --smooth-g[=deg] rot3d: smooth per-frame scale G over a deg-degree rotation range (XDS DELPHI-like) before the combine (default: 5 for rot3d; 0 = off)" << std::endl; std::cout << " --relative-b[=deg] rot3d: fit a per-batch relative-B (beyond the single decay slope) over deg-degree batches; cross-validated (default: 10 deg when bare; off otherwise)" << std::endl; std::cout << " --no-scaling-corrections rot3d: disable the (default-on) decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls" << std::endl; std::cout << " --no-expected-variance-merge stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting" << std::endl; std::cout << " -A, --anomalous Anomalous mode (don't merge Friedel pairs)" << std::endl; std::cout << " --scaling-high-resolution High resolution limit for scaling/merging (manual override; default: no limit)" << std::endl; std::cout << " --scaling-low-resolution Low resolution limit for scaling/merging, in A (default: 50, also XDS's own default; 0 = no limit). The beam stop suppresses air scatter well beyond the shadow the beam-stop mask covers, so reflections coarser than this are measured on a depressed background and mostly integrate negative; masking does not reach them" << std::endl; std::cout << " --resolution-cutoff Automatic high-resolution cutoff for the written reflections + reported shells: cc-logistic|off (default: cc-logistic; ignored when --scaling-high-resolution is set)" << std::endl; std::cout << " --resolution-cc-target CC1/2 target defining the cc-logistic fall-off (default: 0.30)" << std::endl; std::cout << " --resolution-shells Number of resolution shells in the reported statistics table (default: 9, as XDS)" << std::endl; std::cout << " --report-resolution [,] Also report the merging statistics over this resolution range (A, either order; dmax defaults to the run's own low-resolution limit) as a second table beside the run's own - the REFRES_* keys of the report - binned from the same merged reflections, so a run can be read against another program's table at that program's range. Report-only: the cut, the scaling, the error model and every decision stay the run's own, and shells finer than the run's own limit are empty rather than filled from data the run did not keep" << std::endl; std::cout << " --ice-min-score Ice-presence gate: measured ice score (1 = no ice) a run must reach before ANY ice handling is applied - the flagging and the exclusion from scaling (default: 1.5). The eleven hexagonal bands cover 16-26% of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. 0 = no gate (always handle ice)" << std::endl; std::cout << " --ice-min-spot-ratio Second ice-presence channel: found spots on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so --ice-min-score alone is blind to it. Default 2.0; 0 disables this channel" << std::endl; std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; std::cout << " --capture-uncertainty rot3d: systematic sigma ~num*(1-captured_fraction)*I on under-captured fulls (default: 1.0 for rot3d, 0 otherwise)" << std::endl; std::cout << " --min-captured-fraction rot3d: drop a combined full whose rocking curve was captured below this fraction (edge-of-sweep truncated fulls) (default: 0.7 for rotation, 0 otherwise; 0 = off)" << std::endl; std::cout << " --mosaicity Diagnostic: fix the scaling mosaicity (deg) instead of the per-image seed" << std::endl; std::cout << " --prediction-mosaicity Diagnostic: fix the rocking width the PREDICTION window opens to (deg), leaving the partiality on the per-image sigma_M. Prediction and partiality are one number by default, so a sigma_M that moves takes the integrated reflection population with it" << std::endl; std::cout << " --reject-outliers Per-observation merge outlier rejection, N sigma from the per-reflection median (default: 6 for rot3d, XDS/DIALS-style; 0 = off)" << std::endl; std::cout << " --search-min-zeta De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry |zeta| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law; where the two searches disagree, the merge of ALL the observations decides" << std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; std::cout << " --scaling-iterations Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle (default: 100)" << std::endl; std::cout << " -z, --reference Reference data of the same crystal form: an MTZ, or a PDB structure-factor mmCIF (e.g. 1abc-sf.cif, first merged block with a usable column; R-free set from _refln.status f/o, else pdbx_r_free_flag). Gzipped or not, recognised by content. --reference-mtz is the same option" << std::endl; std::cout << " --reference-column