Azimuthal integration: optional sigma clipping of the reported profile
The profile is the MEAN of each bin, so a few strong reflections landing in a bin lift it exactly as a smooth powder ring does. That is the wrong quantity whenever the profile is wanted as a background rather than as a measurement of what is in the bin - the ice score being the case in point, where reading a plain profile INVERTED the metric: over 37 rotation crystals the two highest-scoring crystals had no ice at all. The adaptive spot finder already computes the right thing, a sigma-clipped per-resolution-ring background, as a byproduct of its own threshold. Where it runs, the ice score uses that. Where it does not - --no-adaptive-spots, --azint-only, and anything reading the profile the broker wrote - there was no way to get it. This adds one: azim_int_settings.sigma_clip (rugnux --azim-sigma-clip), 0 = off, minimum 2 because a tighter clip rejects a large part of a clean Gaussian bin and biases the estimate low rather than removing outliers. Two clip passes follow the plain one, matching the finder's recipe - the first pass's standard deviation is itself inflated by the peaks being removed, so one pass leaves a threshold that is still too generous. A bin with fewer than eight pixels is left alone: at the detector edge and behind the beam stop there is no spread to clip on. Both engines do it. On the GPU the accept range is computed by a small kernel and stays resident, so a clip pass is one more read of the same pixels and no round trip; the two accumulation kernels take the range as a pointer that is null on the plain pass. Measured on a JUNGFRAU rotation dataset, non-adaptive path: azimuthal integration 0.02 -> 0.06 ms per image, exactly the 3x the extra passes predict, against a 0.34 ms per-image total. Note what the result IS: the smooth background under the peaks, not the bin mean. It should not be switched on where a ring's integrated intensity is wanted - the powder-ring geometry fit reads ring peaks, and those are what a clip is designed to remove. Off by default, so nothing changes unless it is asked for. Not exposed over the REST API - that needs the generated model regenerated, which is a separate step. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -119,6 +119,21 @@ AzimuthalIntegrationSettings &AzimuthalIntegrationSettings::AzimuthalBinCount(in
|
||||
return *this;
|
||||
}
|
||||
|
||||
AzimuthalIntegrationSettings &AzimuthalIntegrationSettings::SigmaClip(float input) {
|
||||
check_min("Azimuthal integration sigma clip", input, 0.0f);
|
||||
// Below ~2 sigma a clip rejects a large part of a clean Gaussian bin, which biases the estimate
|
||||
// low rather than removing outliers - so a value in (0, 2) is a mistake, not a tight setting.
|
||||
if (input > 0.0f && input < 2.0f)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"Azimuthal integration sigma clip must be 0 (off) or at least 2");
|
||||
sigma_clip_nsigma = input;
|
||||
return *this;
|
||||
}
|
||||
|
||||
float AzimuthalIntegrationSettings::GetSigmaClip() const {
|
||||
return sigma_clip_nsigma;
|
||||
}
|
||||
|
||||
AzimuthalIntegrationSettings &AzimuthalIntegrationSettings::ForceCPUinFPGAWorkflow(bool input) {
|
||||
force_cpu_in_fpga_workflow = input;
|
||||
return *this;
|
||||
|
||||
@@ -25,6 +25,16 @@ class AzimuthalIntegrationSettings {
|
||||
float bkg_estimate_low_q_recipA = 2.0f * PI / 5.0;
|
||||
float q_spacing = 0.01;
|
||||
int32_t azim_bins = 1;
|
||||
// Sigma clipping of the reported profile. 0 = off: every unmasked pixel of a bin contributes, so
|
||||
// the profile is that bin's plain MEAN and a few strong reflections landing in it lift it exactly
|
||||
// as a powder ring would. With n > 0 the accumulation is repeated, rejecting pixels further than
|
||||
// n standard deviations from their own bin's mean; a powder ring is azimuthally smooth and
|
||||
// survives, Bragg peaks do not, so what is left is the smooth background under them. This is the
|
||||
// same recipe the adaptive spot finder already uses for its per-ring threshold - it gets the
|
||||
// clipped background for free as a byproduct - and this is how the other workflows reach it.
|
||||
// NOTE the result is then a BACKGROUND estimate, not the ring mean: do not switch it on where the
|
||||
// integrated intensity of a ring is what is wanted.
|
||||
float sigma_clip_nsigma = 0.0f;
|
||||
// Compute azimuthal integration on the CPU instead of the FPGA during the FPGA
|
||||
// acquisition workflow. Lifts the FPGA bin-count limit and adds standard-deviation output.
|
||||
bool force_cpu_in_fpga_workflow = false;
|
||||
@@ -44,6 +54,7 @@ public:
|
||||
AzimuthalIntegrationSettings& BkgEstimateQRange_recipA(float low, float high);
|
||||
AzimuthalIntegrationSettings& AzimuthalBinCount(int32_t input);
|
||||
AzimuthalIntegrationSettings& ForceCPUinFPGAWorkflow(bool input);
|
||||
AzimuthalIntegrationSettings& SigmaClip(float input);
|
||||
|
||||
[[nodiscard]] bool IsSolidAngleCorrection() const;
|
||||
[[nodiscard]] bool IsPolarizationCorrection() const;
|
||||
@@ -56,6 +67,8 @@ public:
|
||||
[[nodiscard]] int32_t GetQBinCount() const;
|
||||
[[nodiscard]] int32_t GetAzimuthalBinCount() const;
|
||||
[[nodiscard]] bool IsForceCPUinFPGAWorkflow() const;
|
||||
// Sigma-clip multiplier for the reported profile; 0 = off (plain per-bin mean).
|
||||
[[nodiscard]] float GetSigmaClip() const;
|
||||
|
||||
[[nodiscard]] float GetBkgEstimateLowQ_recipA() const;
|
||||
[[nodiscard]] float GetBkgEstimateHighQ_recipA() const;
|
||||
|
||||
@@ -70,16 +70,3 @@ inline bool IsOnIceRing(float d_A, float half_width_q_recipA) {
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Index into ICE_RING_RES_A of the hexagonal-ice ring resolution d sits on (within half_width in
|
||||
// q = 2*pi/d), or -1 if none. Used to look up that ring's per-image strength for the merge gate.
|
||||
inline int IceRingIndex(float d_A, float half_width_q_recipA) {
|
||||
if (!(d_A > 0.0f))
|
||||
return -1;
|
||||
constexpr float two_pi = 6.283185307f;
|
||||
const float q = two_pi / d_A;
|
||||
for (size_t i = 0; i < ICE_RING_RES_A.size(); ++i)
|
||||
if (std::fabs(q - two_pi / ICE_RING_RES_A[i]) < half_width_q_recipA)
|
||||
return static_cast<int>(i);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -164,15 +164,6 @@ bool ScalingSettings::GetExpectedVarianceMerge() const {
|
||||
return expected_variance_merge;
|
||||
}
|
||||
|
||||
ScalingSettings &ScalingSettings::IceRingMergeMask(bool input) {
|
||||
ice_ring_merge_mask = input;
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool ScalingSettings::GetIceRingMergeMask() const {
|
||||
return ice_ring_merge_mask;
|
||||
}
|
||||
|
||||
float ScalingSettings::GetIceMinScore() const {
|
||||
return ice_min_score;
|
||||
}
|
||||
|
||||
@@ -62,33 +62,10 @@ class ScalingSettings {
|
||||
// better on weak. --no-expected-variance-merge restores the old observed-sigma weighting.
|
||||
bool expected_variance_merge = true;
|
||||
|
||||
// Merge-time ice-ring mask (FindDecorrelatedIceRings): after a first merge, drop a hexagonal-ice ring
|
||||
// whose merged half-set CC1/2 has collapsed below its resolution shoulders and merge again. Separate
|
||||
// from --detect-ice-rings, which flags ice SPOTS for indexing and keeps ice reflections out of the
|
||||
// SCALE fit - so this can be turned on or off on its own without changing how the data were indexed
|
||||
// and scaled. Only consulted when detect_ice_rings is on.
|
||||
//
|
||||
// OFF by default. Deleting reflections is not what the field does - AIMLESS, DIALS, xia2, XDS and
|
||||
// CrystFEL all keep ice-band reflections in the merge, and only autoPROC removes them - and the
|
||||
// deletion did not pay for itself when it was measured against a structure-referenced metric. On the
|
||||
// one crystal in the rotation battery where the mask both fires and the anomalous arbiter can score
|
||||
// it, dropping the ring changed the anomalous peak height by -0.001 +- 0.018 sigma (2% of the mean
|
||||
// site height), while costing 1149 unique reflections whose mean I/sigma was 3.62 against the
|
||||
// dataset's own 3.05 - i.e. it deletes better-than-average data. Overall R_meas, CC1/2 and ISa were
|
||||
// identical to three significant figures either way, and the affected shell went from 82.9% to
|
||||
// 100.0% complete without it.
|
||||
//
|
||||
// It is not useless, which is why the switch stays: over the 37-crystal rotation battery it fires
|
||||
// on 5, changes no space group, and those 5 disagree in sign - it clearly helps the two most
|
||||
// heavily iced (one gains 3.8 R_meas and 4.0 CC1/2 points, the other 23 points of high-shell
|
||||
// CC1/2), is a wash on two and costs a third. It always costs completeness where it fires. So:
|
||||
// off as a default, worth turning on by hand on a badly iced crystal.
|
||||
bool ice_ring_merge_mask = false;
|
||||
// Minimum measured ice strength (iceRingScore, 1 = no ice) before any ice-ring handling is applied
|
||||
// at all. The eleven fixed hexagonal bands cover 16-26% of the unique reflections at typical
|
||||
// resolutions REGARDLESS of whether the crystal has ice, so flagging unconditionally taxes clean
|
||||
// data for nothing - and the merge-time ring mask has been observed to fire on crystals with no
|
||||
// measurable ice. 0 disables the gate (always handle ice, the previous behaviour).
|
||||
// data for nothing. 0 disables the gate (always handle ice, the previous behaviour).
|
||||
// 1.5 is measured, not guessed: over 37 rotation crystals the score lands at 1.00-1.22 on the
|
||||
// thirty with no ice, 1.28-1.44 on four borderline ones whose ice-ring positions show no
|
||||
// azimuthally smooth elevation, and 2.08-2.37 on the three with confirmed ice - and a decoy null
|
||||
@@ -142,7 +119,6 @@ public:
|
||||
ScalingSettings& CorrectionSurfaces(bool input);
|
||||
ScalingSettings& StillsPartialityRefine(bool input);
|
||||
ScalingSettings& ExpectedVarianceMerge(bool input);
|
||||
ScalingSettings& IceRingMergeMask(bool input);
|
||||
ScalingSettings& IceMinScore(float input);
|
||||
ScalingSettings& IceMinSpotRatio(float input);
|
||||
ScalingSettings& SmoothGDegrees(double input);
|
||||
@@ -180,7 +156,6 @@ public:
|
||||
[[nodiscard]] bool GetCorrectionSurfaces() const;
|
||||
[[nodiscard]] bool GetStillsPartialityRefine() const;
|
||||
[[nodiscard]] bool GetExpectedVarianceMerge() const;
|
||||
[[nodiscard]] bool GetIceRingMergeMask() const;
|
||||
[[nodiscard]] float GetIceMinScore() const;
|
||||
[[nodiscard]] float GetIceMinSpotRatio() const;
|
||||
[[nodiscard]] double GetSmoothGDegrees() const;
|
||||
|
||||
+3
-3
@@ -11,9 +11,9 @@ This is an UNSTABLE release. It includes many experimental features, as well as
|
||||
* Bragg integration: The **uncertainty of the background estimate** is now propagated into `sigma`; both engines omitted it, which understated every reflection's `sigma` by `sqrt(1 + n_signal/n_bkg)` = 1.109 with the shipped stencil. Expect `<I/sigma>` to fall by that factor on every dataset.
|
||||
* Bragg integration: New **radial background correction** (`--background-radial[=on|off|auto]`, default `auto`) for the bias a curved radial background leaves in a flat ring mean - tens of counts on a single reflection sitting on an ice ring. `auto` applies it per image where that image's ice score shows a *smooth* powder ring: measured against a fixed atomic model it removes 43 % of the ice bands' excess amplitude on smooth ice, but on ice made of discrete crystallite spots - where a radius-only background model has nothing to fit - it makes the bias worse, and the gate separates the two.
|
||||
* rugnux: **Ice-ring handling now runs only when the crystal is measured to have ice** - the eleven fixed bands cost 16-26 % of the unique reflections whether it does or not. Detection uses two channels, the spot finder's peak-excluded radial profile (`--ice-min-score`, default 1.5) and found spots on the rings against ice-free flanks (`--ice-min-spot-ratio`, default 2.0), which catch smooth and textured ice respectively; the previous score was read off the plain azimuthal profile, where strong reflections in a ring's bin score as ice.
|
||||
* rugnux: The merge-time **ice-ring mask is now off by default** (`--ice-ring-mask[=on|off]`, its own switch rather than sharing `--detect-ice-rings` with indexing). Measured against anomalous peak height it returned -0.001 +- 0.018 sigma while deleting reflections whose mean `I/sigma` was *above* the dataset average, leaving `R_meas`, CC1/2 and ISa unchanged and the affected shell 17 completeness points short. Ice reflections are still kept out of the scale fit and the space-group search, as every established scaling program does.
|
||||
* rugnux: **Multiplicity was over-reported** whenever the ice-ring mask dropped a band: the per-shell observation counter rode on the R_meas re-walk, which deliberately ignores the mask, so observations the merge had excluded were counted against a unique count that excluded them - and on the GPU path a fully-masked group's resolution is NaN, which `ResolutionShells::GetShell` silently binned as the lowest-resolution shell rather than rejecting. Only counts were affected (intensities, sigmas, R_meas, CC1/2, completeness and ISa were always right), and only on that path.
|
||||
* rugnux: **The first-pass rotation spots are now found rather than taken from the file**, and the ice measurement is made there - on ~100 images, before anything is discarded, which is what lets the ice gate above act in the pass that produces the data. `--redo-rotation-spots` (now the behaviour) and the reuse path are removed. The stored spots were the acquisition's, found at its threshold with its ice-band entries already dropped by its spot budget, so the lattice search never saw the spot-finding settings and ice counted from them was under-read by construction. Costs ~3 % of the run; over the rotation battery `R_meas` improves on 9 of the 10 crystals that move materially.
|
||||
* rugnux: The merge-time **ice-ring mask has been removed**. Measured against anomalous peak height it returned -0.001 +- 0.018 sigma while deleting reflections whose mean `I/sigma` was *above* the dataset average, leaving `R_meas`, CC1/2 and ISa unchanged and the affected shell 17 completeness points short. Ice reflections are still kept out of the scale fit and the space-group search, as every established scaling program does, and are kept in the final merge.
|
||||
* rugnux: **Multiplicity was over-reported** by the rotation merge: the per-shell observation counter rode on the R_meas re-walk, which deliberately applies a wider filter than the merge, so observations the merge had excluded were counted against a unique count that excluded them - and a group the merge dropped entirely has a NaN resolution, which `ResolutionShells::GetShell` silently binned as the lowest-resolution shell rather than rejecting. Only counts were affected; intensities, sigmas, R_meas, CC1/2, completeness and ISa were always right.
|
||||
* rugnux: **The first-pass rotation spots are now found rather than taken from the file**. `--redo-rotation-spots` (now the behaviour) and the reuse path are removed. The stored spots were the acquisition's, found at its threshold with its ice-band entries already dropped by its spot budget, so the lattice search never saw the spot-finding settings. Costs ~3 % of the run; over the rotation battery `R_meas` improves on 9 of the 10 crystals that move materially.
|
||||
* rugnux: Rotation indexing **no longer keeps a metric symmetry that indexes almost nothing**. The Bravais class is decided from the unrefined FFT candidate against a fixed 3° tolerance, so a lattice pseudo-symmetric to a few tenths of a degree is promoted a class too far and the constraint then snaps a real angle to the ideal one - measured, one crystal's promoted cell indexed 2 of 60 validation frames where its own primitive cell indexed 39, and the run died. The first pass now drops such a promotion. Battery: 33/37 space groups matching XDS with one hard failure becomes 34/37 with none, every other crystal identical.
|
||||
* Powder calibration: the ring geometry fit can now read its rings off an **azimuthally-binned profile summed over a run** instead of a spot list from one image (`RingsFromAzimuthalProfile`). A powder ring is an arc, not a set of spots, and its roundness fixes the beam centre without reference to the calibrant's d-spacings or the detector distance - the one parameter Bragg data constrain worst. Extraction only; nothing calls it yet.
|
||||
* rugnux: De-novo **space-group search** substantially more robust - centering ranked by net absences and judged on absent-class strength, merohedral-twin over-promotion vetoed, and genuine high-symmetry groups recovered on weak data.
|
||||
|
||||
@@ -184,7 +184,15 @@ C_\mathrm{pol}(2\theta,\phi) =
|
||||
$
|
||||
applied as a divisor to intensities (i.e. scale by $1/C_\mathrm{pol}$) when enabled.
|
||||
|
||||
### 2.3 Background estimate for profiles
|
||||
### 2.3 Sigma-clipped profiles
|
||||
|
||||
The estimator above is the **mean** of each bin, so a few strong reflections landing in a bin raise it exactly as a smooth powder ring would. Where the profile is wanted as a *background* — an ice-ring measurement being the case in point — the accumulation can instead be repeated, each pass rejecting pixels further than $n$ standard deviations from their own bin's mean as measured by the pass before (`azim_int_settings.sigma_clip`, `--azim-sigma-clip`; 0 = off). A powder ring is azimuthally smooth and survives the clip; Bragg peaks do not.
|
||||
|
||||
Two clip passes follow the plain one, because the first pass's standard deviation is itself inflated by the peaks being removed, so a single pass leaves a threshold that is still too generous. A bin with fewer than eight pixels is left unclipped — at the detector edge and behind the beam stop there is no spread to speak of, and clipping on it would reject most of what is there. Each pass is one more read of the same pixels, which measures at about 3× the azimuthal-integration time and is invisible against the rest of the frame.
|
||||
|
||||
This is the same quantity the adaptive spot finder (§3.2) already computes as a byproduct of its own per-ring threshold, which is why the ice score prefers the finder's version where one ran (§3.3); the setting is how the other workflows reach it.
|
||||
|
||||
### 2.4 Background estimate for profiles
|
||||
|
||||
A background estimate is derived from the profile as its mean intensity over a fixed low-to-mid $Q$ window (default $2\pi/5$ to $2\pi/3$ Å$^{-1}$). This background is used for monitoring and diagnostics; it is **not** the same as the local Bragg-spot background used in summation integration (§9.2).
|
||||
|
||||
@@ -264,7 +272,7 @@ The profile the score is read off is the **peak-excluded** one, not the plain az
|
||||
|
||||
The radial profile sees ice only as a **smooth powder ring**. Ice in large crystallites diffracts as discrete spots, leaves the profile flat, and is invisible to the score above, so a second channel is read from the spot list itself: the spots found on the ice bands are counted against the spots found in the ice-free flanks $[w,2w)$ either side of each band, rescaled to the bands' own $q$ width (a flank landing on another ring is dropped with its width). The indicator is the ratio pooled over the run — per image the control is a handful of spots and the ratio means nothing — and it is taken before the spot-count filter, which orders ice spots last and would discard them first. The two channels barely overlap: over the rotation battery the crystals with smooth ice read 2.1–2.4 on the profile and ~1.0 on the spots, the crystals with textured ice ~1.1 and 3.8–17.6, and a clean crystal 1.04 on both. Both counts are stored per image (`spot_count_ice_rings`, `spot_count_ice_control`; HDF5 `/entry/MX/peakCountIceRingRes`, `/entry/MX/peakCountIceRingControl`).
|
||||
|
||||
Both channels are used offline as a **gate** on ice handling: unless the run reaches `--ice-min-score` (default 1.5) on the profile or `--ice-min-spot-ratio` (default 2.0) on the spots — 0 disables a channel — ice-ring flagging, the exclusion from the scale fit and the merge-time ice mask (§10.10) are all skipped. The eleven fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. Which rings are then dropped from the merge remains data-driven, from the per-ring merged CC1/2.
|
||||
Both channels are used offline as a **gate** on ice handling: unless the run reaches `--ice-min-score` (default 1.5) on the profile or `--ice-min-spot-ratio` (default 2.0) on the spots — 0 disables a channel — ice-ring flagging and the exclusion from the scale fit (§10.10) are skipped. The eleven fixed bands cover 16–26 % of the unique reflections at typical resolutions whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss.
|
||||
|
||||
A further optional safeguard removes isolated high-resolution “spur” spots by detecting large gaps in $1/d$ (or $q$) space and discarding spots beyond the gap. This is intended for macromolecular diffraction where edge-of-detector backgrounds can be extremely low.
|
||||
|
||||
@@ -761,11 +769,7 @@ A reference dataset (`--reference-mtz`) supplies known intensities for the same
|
||||
|
||||
Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q$ of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are **kept in the final merge**, which is also what the established scaling programs do by default, so the affected shells keep their completeness.
|
||||
|
||||
Nothing is deleted by default. An **optional** merge-time mask (`--ice-ring-mask[=on|off]`, default **off**, switched separately from ice detection because detection also decides how the data are indexed) will drop a band that can be shown to be dead from the merged data rather than from the presence of ice: after the final merge the half-set $\mathrm{CC}_{1/2}$ is computed inside each ice band and in the shoulders $[w,3w)$ either side of it (shoulder reflections lying on a *neighbouring* band are excluded — the three bands near 1.9 Å are only 0.05–0.06 Å$^{-1}$ apart in $q$, so an unguarded shoulder compares ice against ice), and a band falling more than a margin below its shoulders is dropped and the data re-merged. The margin is 0.10, the 99th percentile of a null measured directly — the identical statistic evaluated at **decoy bands**, $q$ positions carrying no ice ring at all — because at these populations a fixed CC margin is not a fixed significance level, and the parametric (Fisher-$z$) error understates the real scatter of heavy-tailed merged intensities several-fold.
|
||||
|
||||
It is off by default because the deletion did not pay for itself when it was measured against an external reference rather than against the merge's own statistics. On the one rotation-battery crystal where the mask fires *and* an anomalous-peak-height arbiter can score the result, dropping the band changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data. This also matches the field: established scaling programs keep ice-band reflections in the merged output and exclude them only from the model fit.
|
||||
|
||||
It is a switch rather than dead code because it does help some crystals. Over a 37-crystal rotation battery it fires on 5 and changes no space group; of those 5 it clearly helps the two most heavily iced (3.8 points of $R_\mathrm{meas}$ and 4.0 of $\mathrm{CC}_{1/2}$ on one, 23 points of *high-shell* $\mathrm{CC}_{1/2}$ on the other), is a wash on two, and costs a third. It always costs completeness where it fires — 6 to 8 points. Turning it on by hand is reasonable for a badly iced crystal; making it a default is not.
|
||||
Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge's own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged $\mathrm{CC}_{1/2}$ and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+5
-5
@@ -37,8 +37,7 @@ scale with the thread count (`-N`).
|
||||
**Input** is a single Jungfraujoch HDF5 master file (NXmx-based). Spots are always found by `rugnux`
|
||||
itself, including for the two-pass rotation first pass — the spot lists a dataset may already carry
|
||||
were found online, at the acquisition's threshold and with its ice-band spots already discarded, so
|
||||
reusing them would hide the spot-finding settings from the lattice search and stop the run measuring
|
||||
its own ice.
|
||||
reusing them would hide the spot-finding settings from the lattice search.
|
||||
|
||||
**Output** (controlled by `-o, --output-prefix`, default `output`):
|
||||
|
||||
@@ -213,7 +212,7 @@ Spot finding:
|
||||
| `--spot-low-resolution <num>` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data) |
|
||||
| `--min-pix-per-spot <num>` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is indexed at min-pix 3/2/1 and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. |
|
||||
| `--max-spots <num>` | Maximum spots kept per image (the strongest ones) and handed to indexing (default: 1000) |
|
||||
| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills**. The merge-time mask is separate, see `--ice-ring-mask` |
|
||||
| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** |
|
||||
|
||||
Azimuthal integration (the radial profile behind the per-image ice-ring score):
|
||||
|
||||
@@ -223,6 +222,7 @@ Azimuthal integration (the radial profile behind the per-image ice-ring score):
|
||||
| `--azim-min-q <num>` | Minimum Q, 1/Å |
|
||||
| `--azim-max-q <num>` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see |
|
||||
| `--azim-phi-bins <num>` | Number of azimuthal (phi) bins (default: 1) |
|
||||
| `--azim-sigma-clip <num>` | **Sigma-clip** the azimuthal profile: repeat the integration twice more, each time rejecting pixels further than *num* standard deviations from their own bin's mean (default: 0 = off; must be ≥ 2). A powder ring is azimuthally smooth and survives the clip, Bragg peaks do not, so the profile becomes the smooth **background under** the peaks rather than the bin mean — which is what an ice-ring measurement wants and what a ring's integrated intensity does not. Costs one extra pass over the image per clip, ~3× the azimuthal-integration time (measured 0.02 → 0.06 ms per image), and nothing else in the frame |
|
||||
| `--polarization-correction <on\|off>` | Enable/disable the azimuthal polarization correction |
|
||||
| `--solid-angle-correction <on\|off>` | Enable/disable the azimuthal solid-angle correction |
|
||||
|
||||
@@ -245,6 +245,7 @@ rotation explicitly and pick the pass or lattice.
|
||||
| `--force-rotation-lattice <vec>` | Force rotation lattice (9 floats, Å), skipping the first pass |
|
||||
| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see the rotation section) |
|
||||
| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), `=off` disables |
|
||||
| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) |
|
||||
|
||||
Indexer choice in brief: `ffbidx` (GPU) refines toward a **known cell** and is best for sparse
|
||||
serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the
|
||||
@@ -269,8 +270,7 @@ Scaling and merging:
|
||||
| `--resolution-cc-target <num>` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) |
|
||||
| `--resolution-shells <num>` | Number of resolution shells in the reported statistics table (default: 10) |
|
||||
| `--min-partiality <num>` | Minimum partiality to accept a reflection (default: 0.02) |
|
||||
| `--ice-ring-mask[=on\|off]` | Merge-time ice-ring mask: after a first merge, drop a hexagonal-ice ring whose merged half-set CC1/2 has collapsed below its resolution shoulders and merge again (default: **off**; only acts when `--detect-ice-rings` is on). This is **only** the merge-time mask — ice-spot flagging and the ice exclusion from scaling stay as `--detect-ice-rings` set them. It is off because deleting the band was measured not to pay for itself: it costs completeness in the affected shell and removes reflections whose mean I/σ is *above* the dataset average, while leaving the anomalous signal, R_meas, CC½ and ISa unchanged |
|
||||
| `--ice-min-score <num>` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging, the exclusion from scaling and the merge-time mask (default: 1.5; 0 = no gate). The eleven fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness |
|
||||
| `--ice-min-score <num>` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The eleven fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness |
|
||||
| `--ice-min-spot-ratio <num>` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) |
|
||||
| `--reject-outliers <num>` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for `rot3d`, off otherwise) |
|
||||
| `--min-image-cc <num>` | Per-image CC limit, percent (default: no limit) |
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
#include "AzIntEngine.h"
|
||||
|
||||
AzIntEngine::AzIntEngine(const AzimuthalIntegrationMapping &integration)
|
||||
@@ -9,4 +12,27 @@ azint_bins(integration.GetBinNumber()),
|
||||
npixel(integration.GetPixelToBin().size()),
|
||||
azint_sum(integration.GetBinNumber(), 0.0f),
|
||||
azint_sum2(integration.GetBinNumber(), 0.0f),
|
||||
azint_count(integration.GetBinNumber(), 0u){}
|
||||
azint_count(integration.GetBinNumber(), 0u),
|
||||
clip_nsigma(integration.Settings().GetSigmaClip()),
|
||||
clip_lo(clip_nsigma > 0.0f ? integration.GetBinNumber() : 0),
|
||||
clip_hi(clip_nsigma > 0.0f ? integration.GetBinNumber() : 0) {}
|
||||
|
||||
void AzIntEngine::UpdateClipLimits() {
|
||||
constexpr float inf = std::numeric_limits<float>::infinity();
|
||||
for (int i = 0; i < azint_bins; ++i) {
|
||||
// A handful of pixels have no spread worth the name, and clipping on it would reject most of
|
||||
// them. Leave such a bin alone - it is the detector edge and the beam stop, not signal.
|
||||
if (azint_count[i] < 8) {
|
||||
clip_lo[i] = -inf;
|
||||
clip_hi[i] = inf;
|
||||
continue;
|
||||
}
|
||||
const double n = azint_count[i];
|
||||
const double mean = azint_sum[i] / n;
|
||||
// sum2/n - mean^2 is the exact variance of what was accumulated; it can go slightly negative
|
||||
// through float cancellation on a bin whose pixels are all but identical, hence the floor.
|
||||
const double sd = std::sqrt(std::max(0.0, azint_sum2[i] / n - mean * mean));
|
||||
clip_lo[i] = static_cast<float>(mean - clip_nsigma * sd);
|
||||
clip_hi[i] = static_cast<float>(mean + clip_nsigma * sd);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,20 @@ protected:
|
||||
std::vector<float> azint_sum;
|
||||
std::vector<float> azint_sum2;
|
||||
std::vector<uint32_t> azint_count;
|
||||
|
||||
// Sigma clipping (AzimuthalIntegrationSettings::SigmaClip; 0 = off). Two clip passes follow the
|
||||
// plain one - the same one-plain-plus-two recipe the adaptive spot finder uses, where the second
|
||||
// matters because the first pass's standard deviation is itself inflated by the peaks being
|
||||
// removed, so one pass alone leaves a threshold that is still too generous.
|
||||
static constexpr int CLIP_PASSES = 2;
|
||||
const float clip_nsigma;
|
||||
std::vector<float> clip_lo;
|
||||
std::vector<float> clip_hi;
|
||||
|
||||
// Per-bin accept range from the accumulators of the pass just finished. A bin with too few pixels
|
||||
// to have a meaningful spread is left unclipped rather than guessed at.
|
||||
void UpdateClipLimits();
|
||||
[[nodiscard]] int PassCount() const { return clip_nsigma > 0.0f ? 1 + CLIP_PASSES : 1; }
|
||||
public:
|
||||
AzIntEngine(const AzimuthalIntegrationMapping& integration);
|
||||
virtual ~AzIntEngine() = default;
|
||||
|
||||
@@ -13,12 +13,6 @@ public:
|
||||
// image is anything that can be referenced with operator[]
|
||||
template <class T>
|
||||
void RunAzint(const T &image, AzimuthalIntegrationProfile &profile) {
|
||||
for (int i = 0; i < azint_count.size(); i++) {
|
||||
azint_sum[i] = 0.0f;
|
||||
azint_sum2[i] = 0.0f;
|
||||
azint_count[i] = 0;
|
||||
}
|
||||
|
||||
if (image.size() != npixel)
|
||||
throw std::runtime_error("ImageSpotFinder::AzimIntegration: Mismatch in size");
|
||||
|
||||
@@ -27,22 +21,38 @@ public:
|
||||
|
||||
using pixel_t = std::remove_cv_t<std::remove_reference_t<decltype(image[0])>>;
|
||||
|
||||
for (int i = 0; i < image.size(); i++) {
|
||||
const pixel_t v = image[i];
|
||||
// saturated pixels use the type's max value; for signed types
|
||||
// masked/bad pixels additionally use the min value
|
||||
if (v == std::numeric_limits<pixel_t>::max())
|
||||
continue;
|
||||
if constexpr (std::is_signed_v<pixel_t>) {
|
||||
if (v == std::numeric_limits<pixel_t>::min())
|
||||
continue;
|
||||
// Pass 0 accumulates every valid pixel; each later pass repeats it, rejecting the pixels that
|
||||
// fell outside their bin's mean +- n sigma as measured by the pass before (see UpdateClipLimits).
|
||||
// Only the last pass's accumulators reach the profile.
|
||||
const int passes = PassCount();
|
||||
for (int pass = 0; pass < passes; ++pass) {
|
||||
if (pass > 0)
|
||||
UpdateClipLimits();
|
||||
|
||||
for (int i = 0; i < azint_count.size(); i++) {
|
||||
azint_sum[i] = 0.0f;
|
||||
azint_sum2[i] = 0.0f;
|
||||
azint_count[i] = 0;
|
||||
}
|
||||
const float val = static_cast<float>(v) * corrections[i];
|
||||
const float val_sq = val * val;
|
||||
const uint16_t bin = pixel_to_bin[i];
|
||||
if (bin < azint_bins) {
|
||||
|
||||
for (int i = 0; i < image.size(); i++) {
|
||||
const pixel_t v = image[i];
|
||||
// saturated pixels use the type's max value; for signed types
|
||||
// masked/bad pixels additionally use the min value
|
||||
if (v == std::numeric_limits<pixel_t>::max())
|
||||
continue;
|
||||
if constexpr (std::is_signed_v<pixel_t>) {
|
||||
if (v == std::numeric_limits<pixel_t>::min())
|
||||
continue;
|
||||
}
|
||||
const uint16_t bin = pixel_to_bin[i];
|
||||
if (bin >= azint_bins)
|
||||
continue;
|
||||
const float val = static_cast<float>(v) * corrections[i];
|
||||
if (pass > 0 && (val < clip_lo[bin] || val > clip_hi[bin]))
|
||||
continue;
|
||||
azint_sum[bin] += val;
|
||||
azint_sum2[bin] += val_sq;
|
||||
azint_sum2[bin] += val * val;
|
||||
++azint_count[bin];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ void gpu_azim_shared(
|
||||
float *__restrict__ azint_sum,
|
||||
float *__restrict__ azint_sum2,
|
||||
uint32_t *__restrict__ azint_count,
|
||||
const float *__restrict__ clip_lo,
|
||||
const float *__restrict__ clip_hi,
|
||||
size_t num_pixels,
|
||||
int azint_bins) {
|
||||
extern __shared__ float shared[];
|
||||
@@ -43,10 +45,14 @@ void gpu_azim_shared(
|
||||
|
||||
if (bin < azint_bins && valid) {
|
||||
const float val = static_cast<float>(v) * corrections[idx];
|
||||
const float val2 = val * val;
|
||||
atomicAdd(&s_sum[bin], val);
|
||||
atomicAdd(&s_sum2[bin], val2);
|
||||
atomicAdd(&s_count[bin], 1);
|
||||
// clip_lo is null on the plain pass; on a clip pass it is the previous pass's
|
||||
// mean +- n sigma for this bin.
|
||||
if (clip_lo == nullptr || (val >= clip_lo[bin] && val <= clip_hi[bin])) {
|
||||
const float val2 = val * val;
|
||||
atomicAdd(&s_sum[bin], val);
|
||||
atomicAdd(&s_sum2[bin], val2);
|
||||
atomicAdd(&s_count[bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +74,8 @@ void gpu_azim(
|
||||
float *__restrict__ azint_sum,
|
||||
float *__restrict__ azint_sum2,
|
||||
uint32_t *__restrict__ azint_count,
|
||||
const float *__restrict__ clip_lo,
|
||||
const float *__restrict__ clip_hi,
|
||||
size_t num_pixels,
|
||||
int azint_bins) {
|
||||
for (size_t idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
@@ -80,10 +88,12 @@ void gpu_azim(
|
||||
|
||||
if (bin < azint_bins && valid) {
|
||||
const float val = static_cast<float>(v) * corrections[idx];
|
||||
const float val2 = val * val;
|
||||
atomicAdd(&azint_sum[bin], val);
|
||||
atomicAdd(&azint_sum2[bin], val2);
|
||||
atomicAdd(&azint_count[bin], 1);
|
||||
if (clip_lo == nullptr || (val >= clip_lo[bin] && val <= clip_hi[bin])) {
|
||||
const float val2 = val * val;
|
||||
atomicAdd(&azint_sum[bin], val);
|
||||
atomicAdd(&azint_sum2[bin], val2);
|
||||
atomicAdd(&azint_count[bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -94,6 +104,8 @@ AzIntEngineGPU::AzIntEngineGPU(const AzimuthalIntegrationMapping &integration, s
|
||||
gpu_sum(azint_bins),
|
||||
gpu_sum2(azint_bins),
|
||||
gpu_count(azint_bins),
|
||||
gpu_clip_lo(clip_nsigma > 0.0f ? azint_bins : 0),
|
||||
gpu_clip_hi(clip_nsigma > 0.0f ? azint_bins : 0),
|
||||
cpu_sum_reg(azint_sum),
|
||||
cpu_sum2_reg(azint_sum2),
|
||||
cpu_count_reg(azint_count) {
|
||||
@@ -116,23 +128,62 @@ AzIntEngineGPU::AzIntEngineGPU(const AzimuthalIntegrationMapping &integration, s
|
||||
integration.GetPixelToBin().data(), *stream);
|
||||
}
|
||||
|
||||
// Per-bin accept range from the accumulators of the pass just finished; mirrors
|
||||
// AzIntEngine::UpdateClipLimits, kept on the device so a clip pass costs no round trip.
|
||||
__global__
|
||||
void gpu_azim_clip_limits(
|
||||
const float *__restrict__ azint_sum,
|
||||
const float *__restrict__ azint_sum2,
|
||||
const uint32_t *__restrict__ azint_count,
|
||||
float *__restrict__ clip_lo,
|
||||
float *__restrict__ clip_hi,
|
||||
int azint_bins,
|
||||
float nsigma) {
|
||||
for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < azint_bins; i += blockDim.x * gridDim.x) {
|
||||
const uint32_t n = azint_count[i];
|
||||
if (n < 8) {
|
||||
clip_lo[i] = -INFINITY;
|
||||
clip_hi[i] = INFINITY;
|
||||
continue;
|
||||
}
|
||||
const double dn = n;
|
||||
const double mean = azint_sum[i] / dn;
|
||||
const double sd = sqrt(fmax(0.0, azint_sum2[i] / dn - mean * mean));
|
||||
clip_lo[i] = static_cast<float>(mean - nsigma * sd);
|
||||
clip_hi[i] = static_cast<float>(mean + nsigma * sd);
|
||||
}
|
||||
}
|
||||
|
||||
void AzIntEngineGPU::Run(const ImagePreprocessorBuffer &image, AzimuthalIntegrationProfile &profile) {
|
||||
if (image.size() != integration.GetPixelToBin().size())
|
||||
throw std::runtime_error("ImageSpotFinder::AzimIntegration: Mismatch in size");
|
||||
cuda_err(cudaMemsetAsync(gpu_sum, 0, sizeof(float) * azint_bins, *stream));
|
||||
cuda_err(cudaMemsetAsync(gpu_sum2, 0, sizeof(float) * azint_bins, *stream));
|
||||
cuda_err(cudaMemsetAsync(gpu_count, 0, sizeof(uint32_t) * azint_bins, *stream));
|
||||
|
||||
if (shared_needed < shared_size) {
|
||||
gpu_azim_shared<<<blocks, threads, shared_needed, *stream>>>(
|
||||
gpu_pixel_to_bin->get(),gpu_azint_correction->get(),image.getGPUBuffer(), gpu_sum, gpu_sum2,
|
||||
gpu_count, npixel, azint_bins
|
||||
);
|
||||
} else {
|
||||
gpu_azim<<<blocks, threads, 0, *stream>>>(
|
||||
// Pass 0 accumulates every valid pixel; each later pass repeats it with the accept range the pass
|
||||
// before measured, so only the last pass's accumulators reach the profile. The limits stay on the
|
||||
// device between passes - a clip pass is one more read of the same pixels and nothing else.
|
||||
const int passes = PassCount();
|
||||
for (int pass = 0; pass < passes; ++pass) {
|
||||
if (pass > 0)
|
||||
gpu_azim_clip_limits<<<blocks, threads, 0, *stream>>>(
|
||||
gpu_sum, gpu_sum2, gpu_count, gpu_clip_lo, gpu_clip_hi, azint_bins, clip_nsigma);
|
||||
const float *lo = pass > 0 ? gpu_clip_lo.get() : nullptr;
|
||||
const float *hi = pass > 0 ? gpu_clip_hi.get() : nullptr;
|
||||
|
||||
cuda_err(cudaMemsetAsync(gpu_sum, 0, sizeof(float) * azint_bins, *stream));
|
||||
cuda_err(cudaMemsetAsync(gpu_sum2, 0, sizeof(float) * azint_bins, *stream));
|
||||
cuda_err(cudaMemsetAsync(gpu_count, 0, sizeof(uint32_t) * azint_bins, *stream));
|
||||
|
||||
if (shared_needed < shared_size) {
|
||||
gpu_azim_shared<<<blocks, threads, shared_needed, *stream>>>(
|
||||
gpu_pixel_to_bin->get(),gpu_azint_correction->get(),image.getGPUBuffer(), gpu_sum, gpu_sum2,
|
||||
gpu_count, npixel, azint_bins
|
||||
);
|
||||
gpu_count, lo, hi, npixel, azint_bins
|
||||
);
|
||||
} else {
|
||||
gpu_azim<<<blocks, threads, 0, *stream>>>(
|
||||
gpu_pixel_to_bin->get(),gpu_azint_correction->get(),image.getGPUBuffer(), gpu_sum, gpu_sum2,
|
||||
gpu_count, lo, hi, npixel, azint_bins
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
cudaMemcpyAsync(azint_sum.data(), gpu_sum, sizeof(float) * azint_bins, cudaMemcpyDeviceToHost, *stream);
|
||||
|
||||
@@ -22,6 +22,9 @@ class AzIntEngineGPU : public AzIntEngine {
|
||||
CudaDevicePtr<float> gpu_sum;
|
||||
CudaDevicePtr<float> gpu_sum2;
|
||||
CudaDevicePtr<uint32_t> gpu_count;
|
||||
// Per-bin accept range for a sigma-clip pass; empty when clipping is off.
|
||||
CudaDevicePtr<float> gpu_clip_lo;
|
||||
CudaDevicePtr<float> gpu_clip_hi;
|
||||
CudaRegisteredVector<float> cpu_sum_reg;
|
||||
CudaRegisteredVector<float> cpu_sum2_reg;
|
||||
CudaRegisteredVector<uint32_t> cpu_count_reg;
|
||||
|
||||
@@ -13,8 +13,6 @@ ADD_LIBRARY(JFJochScaleMerge
|
||||
RotationScaleMerge.h
|
||||
ResolutionCutoff.cpp
|
||||
ResolutionCutoff.h
|
||||
IceRingMask.cpp
|
||||
IceRingMask.h
|
||||
HKLKey.cpp
|
||||
HKLKey.h
|
||||
RfreeFlags.cpp
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
|
||||
#include "IceRingMask.h"
|
||||
#include "../../common/Definitions.h" // ICE_RING_RES_A
|
||||
#include "../../common/CorrelationCoefficient.h"
|
||||
|
||||
std::vector<char> FindDecorrelatedIceRings(const std::vector<MergedReflection> &merged,
|
||||
float half_width_q_recipA, Logger &logger) {
|
||||
if (merged.empty())
|
||||
return {};
|
||||
|
||||
constexpr float two_pi = 6.283185307f;
|
||||
const float w = half_width_q_recipA;
|
||||
std::vector<char> mask(ICE_RING_RES_A.size(), 0);
|
||||
|
||||
for (size_t i = 0; i < ICE_RING_RES_A.size(); ++i) {
|
||||
const float q_ring = two_pi / ICE_RING_RES_A[i];
|
||||
CorrelationCoefficient ring, shoulder;
|
||||
size_t n_ring = 0, n_shoulder = 0;
|
||||
for (const auto &m : merged) {
|
||||
if (!(m.d > 0.0f) || !std::isfinite(m.I_half[0]) || !std::isfinite(m.I_half[1]))
|
||||
continue;
|
||||
const float dq = std::fabs(two_pi / m.d - q_ring);
|
||||
if (dq < w) { ring.Add(m.I_half[0], m.I_half[1]); ++n_ring; }
|
||||
// The shoulder is the control, so it has to be free of ice itself. The hexagonal rings are
|
||||
// not evenly spaced - 1.947/1.916/1.882 A sit 0.05-0.06 apart in q - so for those three the
|
||||
// [w, 3w) band around one ring lands squarely on its neighbours, and the test ends up
|
||||
// comparing ice against ice. Measured, that is the ONLY thing this exclusion changes: over
|
||||
// the battery it removes firings on those three rings and leaves every other firing's CC
|
||||
// pair identical to three decimals.
|
||||
else if (dq < 3.0f * w && !IsOnIceRing(m.d, w)) { shoulder.Add(m.I_half[0], m.I_half[1]); ++n_shoulder; }
|
||||
}
|
||||
// The 0.10 margin is the 99th percentile of this statistic measured on DECOY bands - the same
|
||||
// ring/shoulder geometry evaluated at q positions carrying no ice ring - over the 37-crystal
|
||||
// rotation battery. At the 0.05 it replaces, 4% of ice-free bands cleared the bar; at 0.10,
|
||||
// 1%. (A margin in CC is not a fixed significance: at the observed populations 0.05 ranges
|
||||
// from 1.1 to 7.3 sigma across firings, and the nominal Fisher-z error underestimates the
|
||||
// real scatter of these heavy-tailed intensities by ~2.7x, so the null was measured, not
|
||||
// derived.)
|
||||
if (n_ring >= 20 && n_shoulder >= 20 && shoulder.GetCC() > 0.5
|
||||
&& ring.GetCC() < shoulder.GetCC() - 0.10) {
|
||||
mask[i] = 1;
|
||||
logger.Info("Ice-ring mask: {:.2f} A ring CC1/2 {:.3f} << shoulders {:.3f}; masked from merge",
|
||||
ICE_RING_RES_A[i], ring.GetCC(), shoulder.GetCC());
|
||||
}
|
||||
}
|
||||
|
||||
if (std::none_of(mask.begin(), mask.end(), [](char c) { return c != 0; }))
|
||||
return {};
|
||||
return mask;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "../../common/Logger.h"
|
||||
#include "../../common/Reflection.h" // MergedReflection
|
||||
|
||||
// Hexagonal-ice rings whose merged half-set CC1/2 has collapsed well below the resolution shoulders
|
||||
// either side: ice has decorrelated them and their Bragg intensity is unrecoverable, so the caller
|
||||
// should re-merge with them dropped. A weak or absent ring tracks its neighbours and is not
|
||||
// returned, which is what keeps completeness on a clean crystal untouched.
|
||||
//
|
||||
// Returns a mask indexed like ICE_RING_RES_A, or an empty vector when no ring qualifies - so the
|
||||
// caller can test emptiness rather than scanning. Shared by the full pipeline and the offline
|
||||
// --scale path, which have to reach the same verdict on the same data.
|
||||
std::vector<char> FindDecorrelatedIceRings(const std::vector<MergedReflection> &merged,
|
||||
float half_width_q_recipA, Logger &logger);
|
||||
@@ -54,13 +54,6 @@ MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional<UnitCell> &cell)
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool MergeOnTheFly::IsMaskedRing(const Reflection &r) const {
|
||||
if (masked_ice_rings.empty())
|
||||
return false;
|
||||
const int ring = IceRingIndex(r.d, mask_ice_half_width_q);
|
||||
return ring >= 0 && ring < static_cast<int>(masked_ice_rings.size()) && masked_ice_rings[ring];
|
||||
}
|
||||
|
||||
void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id) {
|
||||
std::unique_lock ul(merged_mutex);
|
||||
|
||||
@@ -79,8 +72,6 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id
|
||||
continue;
|
||||
if (exclude_ice_rings && r.on_ice_ring)
|
||||
continue;
|
||||
if (IsMaskedRing(r))
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
|
||||
@@ -199,8 +190,6 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
|
||||
continue;
|
||||
if (exclude_ice_rings && r.on_ice_ring)
|
||||
continue;
|
||||
if (IsMaskedRing(r))
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
const float I_corr = r.I * r.image_scale_corr;
|
||||
@@ -633,11 +622,7 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
|
||||
continue;
|
||||
const int s = *shell;
|
||||
if (s >= 0 && s < n_shells) {
|
||||
// Multiplicity counts what the merge kept. This walk deliberately applies a wider
|
||||
// filter than the merge so R_meas is computed on the same reflections either way,
|
||||
// but a masked-ring reflection is not in `unique` and must not be counted against it.
|
||||
if (!IsMaskedRing(r))
|
||||
acc[s].total_obs++;
|
||||
acc[s].total_obs++;
|
||||
const auto key = generator(r).pack();
|
||||
const auto mit = merged_I.find(key);
|
||||
if (mit != merged_I.end()) {
|
||||
|
||||
@@ -109,11 +109,6 @@ class MergeOnTheFly {
|
||||
// see the ice-contaminated intensities. The final in-symmetry merge keeps them (for completeness).
|
||||
bool exclude_ice_rings = false;
|
||||
|
||||
// Ice rings (indices into ICE_RING_RES_A) to exclude from this merge because their merged CC1/2
|
||||
// collapsed relative to their resolution neighbours - ice decorrelated them. Empty = none masked.
|
||||
std::vector<char> masked_ice_rings;
|
||||
float mask_ice_half_width_q = 0.03f;
|
||||
|
||||
HKLKeyGenerator generator;
|
||||
|
||||
std::map<uint64_t, MergeAccum> accumulator;
|
||||
@@ -145,15 +140,11 @@ class MergeOnTheFly {
|
||||
size_t reject_count = 0;
|
||||
|
||||
bool Mask(const IntegrationOutcome &outcome);
|
||||
[[nodiscard]] bool IsMaskedRing(const Reflection &r) const;
|
||||
public:
|
||||
MergeOnTheFly(const DiffractionExperiment &x);
|
||||
MergeOnTheFly& ReferenceCell(const std::optional<UnitCell> &cell);
|
||||
MergeOnTheFly& ExcludeIceRings(bool input) { exclude_ice_rings = input; return *this; }
|
||||
MergeOnTheFly& FilterByImageCC(bool input) { filter_by_image_cc = input; return *this; }
|
||||
MergeOnTheFly& MaskIceRings(std::vector<char> masked, float half_width_q) {
|
||||
masked_ice_rings = std::move(masked); mask_ice_half_width_q = half_width_q; return *this;
|
||||
}
|
||||
|
||||
// Fit the global error model from the spread of symmetry-equivalent observations.
|
||||
// Call once before merging; AddImage then applies it.
|
||||
|
||||
@@ -165,7 +165,7 @@ namespace {
|
||||
RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
|
||||
std::vector<IntegrationOutcome> &partial_outcomes,
|
||||
std::optional<UnitCell> reference_cell,
|
||||
int scaling_iterations, float ice_ring_half_width_q,
|
||||
int scaling_iterations,
|
||||
size_t nthreads, Logger &logger,
|
||||
std::string observation_dump_path)
|
||||
: x(experiment), partials_out(partial_outcomes), reference_cell(std::move(reference_cell)),
|
||||
@@ -197,7 +197,6 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
|
||||
mosaicity_deg = *forced;
|
||||
else
|
||||
mosaicity_deg = s.GetDefaultMosaicity();
|
||||
ice_half_width_q = ice_ring_half_width_q;
|
||||
}
|
||||
|
||||
void RotationScaleMerge::Ingest() {
|
||||
@@ -682,7 +681,6 @@ int RotationScaleMerge::ComputeAsuGroups(const HKLKeyGenerator &keygen) {
|
||||
}
|
||||
|
||||
void RotationScaleMerge::ReduceGroupMeans(const std::vector<Obs> &obs, int n_groups,
|
||||
bool exclude_ice, const std::vector<char> &masked,
|
||||
std::vector<double> &out_mean) const {
|
||||
// Inverse-variance per-group mean of I*corr = the merge reference (a segmented reduction over the
|
||||
// groups; the CPU stand-in for a CUDA reduce_by_key). No cell mask here: the scaling reference
|
||||
@@ -691,11 +689,6 @@ void RotationScaleMerge::ReduceGroupMeans(const std::vector<Obs> &obs, int n_gro
|
||||
for (const auto &o : obs) {
|
||||
if (o.group < 0) continue;
|
||||
if (!(o.corr > 0.0f) || !std::isfinite(o.corr)) continue;
|
||||
if (exclude_ice && o.on_ice) continue;
|
||||
if (!masked.empty()) {
|
||||
const int ring = IceRingIndex(o.d, ice_half_width_q);
|
||||
if (ring >= 0 && ring < static_cast<int>(masked.size()) && masked[ring]) continue;
|
||||
}
|
||||
if (o.partiality < min_partiality) continue;
|
||||
const float I_corr = o.I * o.corr;
|
||||
const float sigma_corr = o.sigma * o.corr;
|
||||
@@ -1649,21 +1642,14 @@ namespace {
|
||||
}
|
||||
|
||||
RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool for_search,
|
||||
const std::vector<char> &masked,
|
||||
bool fulls_resident) {
|
||||
// A full is usable for the merge / error model if it passes AddImage's filters (with the current
|
||||
// ice/masked-ring context). group >= 0 already encodes "not absent and passes AcceptReflection".
|
||||
auto masked_ring = [&](const Obs &o) {
|
||||
if (masked.empty()) return false;
|
||||
const int ring = IceRingIndex(o.d, ice_half_width_q);
|
||||
return ring >= 0 && ring < static_cast<int>(masked.size()) && masked[ring];
|
||||
};
|
||||
// ice context). group >= 0 already encodes "not absent and passes AcceptReflection".
|
||||
auto usable_merge = [&](const Obs &o) {
|
||||
if (o.group < 0) return false;
|
||||
if (!frame_cell_ok[o.frame]) return false;
|
||||
if (!(o.corr > 0.0f) || !std::isfinite(o.corr)) return false;
|
||||
if (for_search && o.on_ice) return false;
|
||||
if (masked_ring(o)) return false;
|
||||
if (o.partiality < min_partiality) return false;
|
||||
const float I_corr = o.I * o.corr, sigma_corr = o.sigma * o.corr;
|
||||
return std::isfinite(I_corr) && std::isfinite(sigma_corr) && sigma_corr > 0.0f;
|
||||
@@ -1676,7 +1662,6 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
bool use_gpu_merge = false;
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
use_gpu_merge = fulls_resident && !fulls.empty();
|
||||
std::vector<uint8_t> gpu_masked(masked.begin(), masked.end());
|
||||
#endif
|
||||
|
||||
// ---- Error model: fit dev2 = a*sigma^2 + b^2*<I>^2 from symmetry-equivalent scatter. ----
|
||||
@@ -1694,8 +1679,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
const int nf = static_cast<int>(fulls.size());
|
||||
std::vector<double> gs2(nf), gI2(nf), gdev2(nf);
|
||||
std::vector<uint8_t> gvalid(nf);
|
||||
gpu_->MergeEmSamples(for_search, gpu_masked.data(), static_cast<int>(gpu_masked.size()),
|
||||
ice_half_width_q, min_partiality, em_mean.data(), cnt.data(),
|
||||
gpu_->MergeEmSamples(for_search, min_partiality, em_mean.data(), cnt.data(),
|
||||
gs2.data(), gI2.data(), gdev2.data(), gvalid.data());
|
||||
samples.reserve(nf);
|
||||
for (int i = 0; i < nf; ++i)
|
||||
@@ -2002,7 +1986,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
}
|
||||
}
|
||||
|
||||
// R_meas: re-walk the fulls (Mask = cell only; no ice / masked-ring / error-model), accumulate
|
||||
// R_meas: re-walk the fulls (Mask = cell only; no ice / error-model), accumulate
|
||||
// |I_i - <I>| per reflection.
|
||||
struct RmeasObs { double sum_abs_dev = 0, sum_I = 0; int n = 0, shell = -1; };
|
||||
std::vector<RmeasObs> rmeas(n_groups);
|
||||
@@ -2018,11 +2002,11 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
const auto shell = shells.GetShell(acc[g].d);
|
||||
if (!shell || *shell < 0 || *shell >= n_shells) continue;
|
||||
// Count the MERGED population, not the R_meas one. The R_meas re-walk deliberately
|
||||
// ignores the ring mask (and, on a search pass, the ice flag), so its count includes
|
||||
// observations that never entered `unique` - which inflates the reported multiplicity
|
||||
// of whatever shell they land in. acc[g].nh is what actually went into this group's
|
||||
// mean, and it is zero for a masked group. (acc[g].d is NaN for such a group, so
|
||||
// GetShell above already declines it; this is the same statement made where it counts.)
|
||||
// ignores the ice flag on a search pass, so its count includes observations that never
|
||||
// entered `unique` - which inflates the reported multiplicity of whatever shell they
|
||||
// land in. acc[g].nh is what actually went into this group's mean, and it is zero for a
|
||||
// group the merge dropped entirely. (acc[g].d is NaN for such a group, so GetShell above
|
||||
// already declines it; this is the same statement made where it counts.)
|
||||
sa[*shell].total_obs += static_cast<int>(acc[g].nh[0] + acc[g].nh[1]);
|
||||
if (std::isfinite(merged_I[g]) && rn[g] > 0) {
|
||||
auto &r = rmeas[g];
|
||||
@@ -2173,8 +2157,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
return result;
|
||||
}
|
||||
|
||||
RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search,
|
||||
const std::vector<char> &masked_ice_rings) {
|
||||
RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search) {
|
||||
const int sg_number = x.GetSpaceGroupNumber().value_or(1);
|
||||
HKLKeyGenerator keygen(merge_friedel, sg_number);
|
||||
|
||||
@@ -2208,7 +2191,7 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search,
|
||||
#endif
|
||||
if (!scaled_on_gpu) {
|
||||
for (int it = 0; it < scaling_iter; ++it) {
|
||||
ReduceGroupMeans(partials, n_groups, false, {}, partial_mean);
|
||||
ReduceGroupMeans(partials, n_groups, partial_mean);
|
||||
FitPerFrameG(partials, frame_start, frame_count, partial_mean, /*unity=*/false, g_partial);
|
||||
UpdateCorr(partials, g_partial, frame_scaled_scratch);
|
||||
}
|
||||
@@ -2297,7 +2280,7 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search,
|
||||
}
|
||||
#endif
|
||||
if (!cc_on_gpu) {
|
||||
ReduceGroupMeans(partials, n_groups, false, {}, partial_mean);
|
||||
ReduceGroupMeans(partials, n_groups, partial_mean);
|
||||
ComputePerFrameCC(partial_mean, cc, cc_n);
|
||||
}
|
||||
FinalizePerFrameScale(cc, cc_n, partial_scaled);
|
||||
@@ -2427,7 +2410,7 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search,
|
||||
if (scale_fulls && !scaled_fulls_on_gpu) {
|
||||
std::vector<double> full_mean;
|
||||
for (int it = 0; it < scaling_iter; ++it) {
|
||||
ReduceGroupMeans(fulls, n_groups, false, {}, full_mean);
|
||||
ReduceGroupMeans(fulls, n_groups, full_mean);
|
||||
FitPerFrameG(fulls, fulls_frame_start, fulls_frame_count, full_mean, /*unity=*/true, g_full);
|
||||
UpdateCorr(fulls, g_full, frame_scaled_scratch);
|
||||
}
|
||||
@@ -2465,6 +2448,6 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search,
|
||||
#endif
|
||||
|
||||
// --- 5. Error model + merge + statistics. ---
|
||||
auto r = MergeAndStats(n_groups, for_search, masked_ice_rings, combined_on_gpu && scaled_fulls_on_gpu);
|
||||
auto r = MergeAndStats(n_groups, for_search, combined_on_gpu && scaled_fulls_on_gpu);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -53,7 +53,6 @@ public:
|
||||
std::vector<IntegrationOutcome> &partial_outcomes,
|
||||
std::optional<UnitCell> reference_cell,
|
||||
int scaling_iterations,
|
||||
float ice_ring_half_width_q,
|
||||
size_t nthreads,
|
||||
Logger &logger,
|
||||
std::string observation_dump_path = {});
|
||||
@@ -65,8 +64,7 @@ public:
|
||||
// for the space group currently set on the experiment, reusing the ingested buffers.
|
||||
// for_search: the de-novo P1 pass whose merged intensities feed the space-group search - ice-ring
|
||||
// reflections are dropped from the merge and the error model (kept otherwise, for completeness).
|
||||
// masked_ice_rings: rings (indices into ICE_RING_RES_A) to drop from the final merge; empty = none.
|
||||
Result Run(bool for_search, const std::vector<char> &masked_ice_rings = {});
|
||||
Result Run(bool for_search);
|
||||
|
||||
// Override the high-resolution cut for the next Run() - used to gate the de-novo P1 search pass at
|
||||
// <I/sigma> >= 1 without cutting the final in-symmetry merge. Reset to the manual limit afterwards.
|
||||
@@ -131,7 +129,6 @@ private:
|
||||
int modulation_iter = 0; // >0: fit a detector-plane modulation (flat-field) surface, this many iterations
|
||||
double relative_b_deg = 0.0; // >0: fit a per-batch relative-B (batch width in deg); 0 = off
|
||||
double mosaicity_deg = 0.1;
|
||||
float ice_half_width_q = 0.0f;
|
||||
// Automatic high-resolution cutoff for the written reflections + reported shells (post-merge; the
|
||||
// scaling, combine and error model always run over the full range). Manual d_min_limit wins.
|
||||
ResolutionCutoffMethod resolution_cutoff_method = ResolutionCutoffMethod::Off;
|
||||
@@ -193,11 +190,8 @@ private:
|
||||
// rawrun_group, the group_h/k/l representative tables, and partials[].group; returns the group count.
|
||||
int ComputeAsuGroups(const HKLKeyGenerator &key_generator);
|
||||
|
||||
// Inverse-variance per-group mean of I*corr over `obs` (the merge reference). exclude_ice/masked drop
|
||||
// those reflections (used for the error-model/merge means, not the scaling reference).
|
||||
void ReduceGroupMeans(const std::vector<Obs> &obs, int n_groups,
|
||||
bool exclude_ice, const std::vector<char> &masked_ice_rings,
|
||||
std::vector<double> &out_mean) const;
|
||||
// Inverse-variance per-group mean of I*corr over `obs` (the merge reference).
|
||||
void ReduceGroupMeans(const std::vector<Obs> &obs, int n_groups, std::vector<double> &out_mean) const;
|
||||
|
||||
// Robust per-frame G fit (IRLS, Cauchy k=3), unity=false uses the rotation partiality, unity=true the
|
||||
// scale-fulls (partiality already folded in). Reads out_mean[group] as the reference intensity.
|
||||
@@ -296,6 +290,5 @@ private:
|
||||
// Error model + merge + statistics over the fulls (the last stage). n_groups is the fulls group count.
|
||||
// fulls_resident: the (scaled) fulls + their group CSR are still on the GPU, so the em-stats / samples
|
||||
// / merge-accumulate / R_meas reductions run there (only per-group + samples come back).
|
||||
Result MergeAndStats(int n_groups, bool for_search, const std::vector<char> &masked_ice_rings,
|
||||
bool fulls_resident);
|
||||
Result MergeAndStats(int n_groups, bool for_search, bool fulls_resident);
|
||||
};
|
||||
|
||||
@@ -414,18 +414,6 @@ namespace {
|
||||
|
||||
// ===== error-model + merge reductions over the resident, scaled fulls (mirror MergeAndStats) =====
|
||||
|
||||
// Device copy of common/Definitions.h IceRingIndex (only consulted when the merge has masked rings).
|
||||
__device__ __forceinline__ int IceRingIndexDev(float d, float hw) {
|
||||
if (!(d > 0.0f)) return -1;
|
||||
const float two_pi = 6.283185307f;
|
||||
const float q = two_pi / d;
|
||||
const float rings[11] = {3.895f, 3.661f, 3.438f, 2.667f, 2.249f, 2.068f,
|
||||
1.947f, 1.916f, 1.882f, 1.719f, 1.522f};
|
||||
for (int i = 0; i < 11; ++i)
|
||||
if (fabsf(q - two_pi / rings[i]) < hw) return i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Deterministic CC1/2 half from the frame index (splitmix64), matching Merge.cpp / RSM HalfForImage.
|
||||
__device__ __forceinline__ int HalfForImageDev(long long image_id) {
|
||||
unsigned long long z = (unsigned long long)image_id + 0x9e3779b97f4a7c15ULL;
|
||||
@@ -438,11 +426,10 @@ namespace {
|
||||
struct MergeParams {
|
||||
int n_groups;
|
||||
double min_partiality, error_model_a, error_model_b, reject_nsigma;
|
||||
float ice_hw;
|
||||
int for_search, n_masked, error_model_active, reject_outliers;
|
||||
int for_search, error_model_active, reject_outliers;
|
||||
const float *I, *sigma, *corr, *partiality, *d, *reject_median;
|
||||
const int32_t *group, *frame;
|
||||
const uint8_t *on_ice, *frame_cell_ok, *masked;
|
||||
const uint8_t *on_ice, *frame_cell_ok;
|
||||
const int32_t *gperm, *gstart, *gcount;
|
||||
const double *em_mean, *merged_I;
|
||||
// outputs
|
||||
@@ -465,16 +452,12 @@ namespace {
|
||||
const float c = p.corr[i];
|
||||
if (!(c > 0.0f) || !isfinite(c)) return false;
|
||||
if (p.for_search && p.on_ice[i]) return false;
|
||||
if (p.n_masked > 0) {
|
||||
const int ring = IceRingIndexDev(p.d[i], p.ice_hw);
|
||||
if (ring >= 0 && ring < p.n_masked && p.masked[ring]) return false;
|
||||
}
|
||||
if (p.partiality[i] < p.min_partiality) return false;
|
||||
const float I_corr = p.I[i] * c, sigma_corr = p.sigma[i] * c;
|
||||
return isfinite(I_corr) && isfinite(sigma_corr) && sigma_corr > 0.0f;
|
||||
}
|
||||
|
||||
// The looser R_meas filter (no ice / masked-ring / for_search - Mask = cell only).
|
||||
// The looser R_meas filter (no ice / for_search - Mask = cell only).
|
||||
__device__ __forceinline__ bool RmeasUsable(int i, const MergeParams &p) {
|
||||
const int g = p.group[i];
|
||||
if (g < 0) return false;
|
||||
@@ -561,7 +544,7 @@ namespace {
|
||||
// One thread per group: R_meas accumulators (sum|I_corr - merged_I|, sum_I, n) + the count of
|
||||
// observations this looser walk accepted. Mirrors MergeAndStats' R_meas re-walk (cell-only filter).
|
||||
// That count is NOT the per-shell total_observations - it is wider than the merge, so it would
|
||||
// over-report multiplicity on a masked ring; the host uses it only to skip empty groups.
|
||||
// over-report multiplicity; the host uses it only to skip empty groups.
|
||||
__global__ void MergeRmeasKernel(MergeParams p) {
|
||||
for (int g = blockIdx.x * blockDim.x + threadIdx.x; g < p.n_groups; g += gridDim.x * blockDim.x) {
|
||||
const int lo = p.gstart[g], hi = lo + p.gcount[g];
|
||||
@@ -620,7 +603,7 @@ struct RotationScaleMergeGPU::Impl {
|
||||
CudaDevicePtr<uint8_t> smooth_apply; // per-frame smooth-G apply flag + ratio, length n_frames
|
||||
CudaDevicePtr<double> smooth_ratio;
|
||||
// merge / error-model reductions over the resident fulls (reuse the fulls group CSR f_gperm/...)
|
||||
CudaDevicePtr<uint8_t> frame_cell_ok, merge_masked;
|
||||
CudaDevicePtr<uint8_t> frame_cell_ok;
|
||||
CudaDevicePtr<double> m_sw, m_swI, m_em_mean; // per group (n_groups)
|
||||
CudaDevicePtr<int32_t> m_cnt;
|
||||
CudaDevicePtr<double> m_s2, m_I2, m_dev2; // per full (n_fulls)
|
||||
@@ -631,8 +614,7 @@ struct RotationScaleMergeGPU::Impl {
|
||||
CudaDevicePtr<float> reject_median;
|
||||
CudaDevicePtr<double> merged_I, r_absdev, r_sumI; // R_meas per group (merged_I uploaded)
|
||||
CudaDevicePtr<int32_t> r_n, r_nusable;
|
||||
int merge_for_search = 0, merge_n_masked = 0; // filter context for one MergeAndStats call
|
||||
float merge_ice_hw = 0.0f;
|
||||
int merge_for_search = 0; // filter context for one MergeAndStats call
|
||||
double merge_min_part = 0.0;
|
||||
|
||||
// combine: extra per-obs inputs + the one-time raw-hkl run layout
|
||||
@@ -743,26 +725,23 @@ void RotationScaleMergeGPU::SetFrameCellOk(const uint8_t *frame_cell_ok) {
|
||||
|
||||
// The per-group inv-var mean (em_mean) + the per-full leverage-corrected error-model samples over the
|
||||
// resident+scaled fulls. Stashes the filter context for the later MergeAccum/MergeRmeas calls.
|
||||
void RotationScaleMergeGPU::MergeEmSamples(bool for_search, const uint8_t *masked, int n_masked,
|
||||
double ice_half_width_q, double min_partiality,
|
||||
void RotationScaleMergeGPU::MergeEmSamples(bool for_search, double min_partiality,
|
||||
double *em_mean_out, int32_t *cnt_out, double *s2_out,
|
||||
double *I2_out, double *dev2_out, uint8_t *valid_out) {
|
||||
auto &d = *impl_;
|
||||
const int ng = d.n_groups, nf = d.n_fulls;
|
||||
d.merge_for_search = for_search ? 1 : 0; d.merge_n_masked = n_masked;
|
||||
d.merge_ice_hw = float(ice_half_width_q); d.merge_min_part = min_partiality;
|
||||
d.merge_for_search = for_search ? 1 : 0; d.merge_min_part = min_partiality;
|
||||
d.m_sw = CudaDevicePtr<double>(std::max(1, ng)); d.m_swI = CudaDevicePtr<double>(std::max(1, ng));
|
||||
d.m_em_mean = CudaDevicePtr<double>(std::max(1, ng)); d.m_cnt = CudaDevicePtr<int32_t>(std::max(1, ng));
|
||||
d.m_s2 = CudaDevicePtr<double>(std::max(1, nf)); d.m_I2 = CudaDevicePtr<double>(std::max(1, nf));
|
||||
d.m_dev2 = CudaDevicePtr<double>(std::max(1, nf)); d.m_valid = CudaDevicePtr<uint8_t>(std::max(1, nf));
|
||||
Upload(d.merge_masked, masked, n_masked);
|
||||
|
||||
MergeParams p{};
|
||||
p.n_groups = ng; p.min_partiality = min_partiality; p.ice_hw = d.merge_ice_hw;
|
||||
p.for_search = d.merge_for_search; p.n_masked = n_masked;
|
||||
p.n_groups = ng; p.min_partiality = min_partiality;
|
||||
p.for_search = d.merge_for_search;
|
||||
p.I = d.f_I.get(); p.sigma = d.f_sigma.get(); p.corr = d.f_corr.get(); p.partiality = d.f_partiality.get();
|
||||
p.d = d.f_d.get(); p.group = d.f_group.get(); p.frame = d.f_frame.get();
|
||||
p.on_ice = d.f_on_ice.get(); p.frame_cell_ok = d.frame_cell_ok.get(); p.masked = d.merge_masked.get();
|
||||
p.on_ice = d.f_on_ice.get(); p.frame_cell_ok = d.frame_cell_ok.get();
|
||||
p.gperm = d.f_gperm.get(); p.gstart = d.f_gstart.get(); p.gcount = d.f_gcount.get();
|
||||
p.em_mean = d.m_em_mean.get(); p.sw = d.m_sw.get(); p.swI = d.m_swI.get();
|
||||
p.em_mean_out = d.m_em_mean.get(); p.cnt = d.m_cnt.get();
|
||||
@@ -804,15 +783,15 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
|
||||
Upload(d.reject_median, reject_median, ng);
|
||||
|
||||
MergeParams p{};
|
||||
p.n_groups = ng; p.min_partiality = d.merge_min_part; p.ice_hw = d.merge_ice_hw;
|
||||
p.for_search = d.merge_for_search; p.n_masked = d.merge_n_masked;
|
||||
p.n_groups = ng; p.min_partiality = d.merge_min_part;
|
||||
p.for_search = d.merge_for_search;
|
||||
p.error_model_a = error_model_a; p.error_model_b = error_model_b;
|
||||
p.error_model_active = error_model_active ? 1 : 0;
|
||||
p.reject_outliers = reject_outliers ? 1 : 0; p.reject_nsigma = reject_nsigma;
|
||||
p.reject_median = d.reject_median.get();
|
||||
p.I = d.f_I.get(); p.sigma = d.f_sigma.get(); p.corr = d.f_corr.get(); p.partiality = d.f_partiality.get();
|
||||
p.d = d.f_d.get(); p.group = d.f_group.get(); p.frame = d.f_frame.get();
|
||||
p.on_ice = d.f_on_ice.get(); p.frame_cell_ok = d.frame_cell_ok.get(); p.masked = d.merge_masked.get();
|
||||
p.on_ice = d.f_on_ice.get(); p.frame_cell_ok = d.frame_cell_ok.get();
|
||||
p.gperm = d.f_gperm.get(); p.gstart = d.f_gstart.get(); p.gcount = d.f_gcount.get();
|
||||
p.em_mean = d.m_em_mean.get();
|
||||
p.a_swI = d.a_swI.get(); p.a_sw = d.a_sw.get(); p.a_swIh0 = d.a_swIh0.get(); p.a_swIh1 = d.a_swIh1.get();
|
||||
|
||||
@@ -65,9 +65,8 @@ public:
|
||||
|
||||
// Per-group inv-var mean (em_mean, length n_groups) + per-full leverage-corrected error-model samples
|
||||
// (s2/I2/dev2 + valid flag, length n_fulls), mirroring MergeAndStats' first two error-model loops.
|
||||
// Stashes (for_search, masked, ice, min_partiality) for the MergeAccum/MergeRmeas calls that follow.
|
||||
void MergeEmSamples(bool for_search, const uint8_t *masked, int n_masked,
|
||||
double ice_half_width_q, double min_partiality,
|
||||
// Stashes (for_search, min_partiality) for the MergeAccum/MergeRmeas calls that follow.
|
||||
void MergeEmSamples(bool for_search, double min_partiality,
|
||||
double *em_mean_out, int32_t *cnt_out, double *s2_out, double *I2_out,
|
||||
double *dev2_out, uint8_t *valid_out);
|
||||
|
||||
|
||||
+4
-72
@@ -34,7 +34,6 @@
|
||||
#include "../image_analysis/scale_merge/RfreeFlags.h"
|
||||
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
||||
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
||||
#include "../image_analysis/scale_merge/IceRingMask.h"
|
||||
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
|
||||
#include "../image_analysis/scale_merge/ScalingResult.h"
|
||||
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
|
||||
@@ -563,11 +562,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
const auto start_time = std::chrono::steady_clock::now();
|
||||
|
||||
// First pass of two-pass rotation indexing (full analysis only).
|
||||
// Ice measured on the first-pass sample (see where they are filled): pooled here so the verdict
|
||||
// below, which runs after the first pass and before the per-image loop, can read them.
|
||||
std::vector<float> sample_ice_scores;
|
||||
double sample_ice_ring_spots = 0.0, sample_ice_control_spots = 0.0;
|
||||
|
||||
if (full && force_rotation_result_.has_value()) {
|
||||
// Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
|
||||
// search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
|
||||
@@ -617,12 +611,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
if (dataset->efficiency.size() > image_idx)
|
||||
m.image_collection_efficiency = dataset->efficiency[image_idx];
|
||||
analysis->Analyze(m, *profile, first_pass);
|
||||
if (m.ice_ring_score)
|
||||
sample_ice_scores.push_back(*m.ice_ring_score);
|
||||
if (m.spot_count_ice_rings)
|
||||
sample_ice_ring_spots += static_cast<double>(*m.spot_count_ice_rings);
|
||||
if (m.spot_count_ice_control)
|
||||
sample_ice_control_spots += static_cast<double>(*m.spot_count_ice_control);
|
||||
spots = std::move(m.spots);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
@@ -923,39 +911,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
}
|
||||
|
||||
// Ice-presence verdict, from the first-pass sample and before the pass that acts on it. The eleven
|
||||
// hexagonal bands are fixed geometry and hold 16-26 % of the unique reflections whether or not the
|
||||
// crystal has ice, so setting their spots aside on a clean crystal discards a fifth of them for
|
||||
// nothing - and de-prioritising them in the --max-spots budget throws away the strongest first.
|
||||
// Where the sample sees no ice, index on all the spots instead. Only the SAMPLE can decide this:
|
||||
// the spot channel is a ratio pooled over images, so it is meaningless on any single frame.
|
||||
if (full && !sample_ice_scores.empty() && experiment_.IsDetectIceRings()
|
||||
&& !experiment_.GetIndexingSettings().GetIndexIceRings()) {
|
||||
const float min_score = experiment_.GetScalingSettings().GetIceMinScore();
|
||||
const float min_ratio = experiment_.GetScalingSettings().GetIceMinSpotRatio();
|
||||
const float score = std::accumulate(sample_ice_scores.begin(), sample_ice_scores.end(), 0.0f)
|
||||
/ static_cast<float>(sample_ice_scores.size());
|
||||
const bool smooth = score >= min_score;
|
||||
const bool spotty = min_ratio > 0.0f && sample_ice_control_spots > 0.0
|
||||
&& sample_ice_ring_spots / sample_ice_control_spots >= min_ratio;
|
||||
if (!smooth && !spotty) {
|
||||
IndexingSettings is = experiment_.GetIndexingSettings();
|
||||
is.IndexIceRings(true);
|
||||
experiment_.ImportIndexingSettings(is);
|
||||
logger.Info("First-pass sample of {} images sees no ice (score {:.2f} < {:.2f}, spot ratio "
|
||||
"{:.2f} < {:.2f}): indexing on the ice-band spots too",
|
||||
sample_ice_scores.size(), score, min_score,
|
||||
sample_ice_control_spots > 0.0
|
||||
? sample_ice_ring_spots / sample_ice_control_spots : NAN, min_ratio);
|
||||
} else {
|
||||
logger.Info("First-pass sample of {} images sees ice (score {:.2f}, spot ratio {:.2f}): "
|
||||
"ice-band spots set aside for indexing",
|
||||
sample_ice_scores.size(), score,
|
||||
sample_ice_control_spots > 0.0
|
||||
? sample_ice_ring_spots / sample_ice_control_spots : NAN);
|
||||
}
|
||||
}
|
||||
|
||||
// Main per-image loop, spread over N worker threads pulling from a shared counter. HDF5 reads
|
||||
// are serialized by the global hdf5_mutex; the analysis runs in parallel.
|
||||
std::atomic<int> next_ordinal = 0;
|
||||
@@ -1165,10 +1120,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// them outright guts low/mid-resolution completeness on crystals that merge them fine.
|
||||
// The eleven hexagonal-ice bands cover 16-26% of the unique reflections at typical resolutions
|
||||
// whether or not the crystal has any ice, so handling ice unconditionally taxes clean data for
|
||||
// nothing. Gate the whole mechanism - this flagging and the merge-time CC1/2 ring mask below -
|
||||
// on the run's own measured ice strength (1 = no ice; the strongest ring over the smooth radial
|
||||
// background, measured on the peak-excluded profile so Bragg peaks cannot pose as ice). The
|
||||
// score is logged either way, so a run that gates off still reports why.
|
||||
// nothing. Gate the flagging on the run's own measured ice strength (1 = no ice; the strongest
|
||||
// ring over the smooth radial background, measured on the peak-excluded profile so Bragg peaks
|
||||
// cannot pose as ice). The score is logged either way, so a run that gates off still reports why.
|
||||
// Ice is looked for on TWO channels, because it arrives in two forms. Fine-grained ice is a
|
||||
// smooth powder ring and shows up in the radial profile (ice_ring_score). Ice in large
|
||||
// crystallites arrives as discrete reflections instead, leaves the profile flat, and is only
|
||||
@@ -1218,9 +1172,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// reference); the scaling loop below is skipped, so G is stable across the two passes.
|
||||
// Smoothing it more than once would compound the correction, so do it only on the first
|
||||
// pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly.
|
||||
// Ice rings masked from the merge by the CC1/2 test after the final merge; empty until then,
|
||||
// at which point a re-merge applies them.
|
||||
std::vector<char> masked_ice_rings;
|
||||
|
||||
// Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single
|
||||
// allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame
|
||||
@@ -1318,7 +1269,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// self-consistent.
|
||||
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
||||
static_cast<int>(config_.scaling_iter),
|
||||
config_.spot_finding.ice_ring_width_Q_recipA,
|
||||
config_.nthreads, logger, config_.observation_dump_path);
|
||||
rsm->Ingest();
|
||||
}
|
||||
@@ -1326,7 +1276,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
auto scale_and_merge = [&](const std::string &label, bool for_search) -> ScaleMergeResult {
|
||||
if (rsm) {
|
||||
phase("Scale/combine/merge (" + label + ")");
|
||||
auto r = rsm->Run(for_search, masked_ice_rings);
|
||||
auto r = rsm->Run(for_search);
|
||||
result.error_model_isa = r.isa;
|
||||
return ScaleMergeResult{std::move(r.merged), std::move(r.statistics)};
|
||||
}
|
||||
@@ -1367,7 +1317,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// intensities and the error model, so the space-group search sees clean data; the final
|
||||
// in-symmetry merge keeps them so completeness is not lost.
|
||||
merge_engine.ExcludeIceRings(for_search);
|
||||
merge_engine.MaskIceRings(masked_ice_rings, config_.spot_finding.ice_ring_width_Q_recipA);
|
||||
if (result.consensus_cell.has_value())
|
||||
merge_engine.ReferenceCell(*result.consensus_cell);
|
||||
// Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells)
|
||||
@@ -1728,7 +1677,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
if (rsm) {
|
||||
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
||||
static_cast<int>(config_.scaling_iter),
|
||||
config_.spot_finding.ice_ring_width_Q_recipA,
|
||||
config_.nthreads, logger, config_.observation_dump_path);
|
||||
rsm->Ingest();
|
||||
}
|
||||
@@ -1780,7 +1728,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
||||
static_cast<int>(config_.scaling_iter),
|
||||
config_.spot_finding.ice_ring_width_Q_recipA,
|
||||
config_.nthreads, logger, config_.observation_dump_path);
|
||||
rsm->Ingest();
|
||||
phase("Re-merging in the reference indexing");
|
||||
@@ -1788,21 +1735,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
}
|
||||
|
||||
// Ice-ring CC1/2 mask: a hexagonal-ice ring whose merged half-set CC1/2 collapses well below its
|
||||
// resolution shoulders has been decorrelated by ice (its Bragg is unrecoverable) - drop it and
|
||||
// re-merge. Weak/absent rings track their neighbours and stay, so completeness on clean crystals
|
||||
// is untouched. Uses the ring band (+/- ice width in q=2pi/d) vs the shoulders either side.
|
||||
if (experiment_.IsDetectIceRings() && experiment_.GetScalingSettings().GetIceRingMergeMask()
|
||||
&& ice_present && !sm.merged.empty()) {
|
||||
auto mask = FindDecorrelatedIceRings(sm.merged, config_.spot_finding.ice_ring_width_Q_recipA,
|
||||
logger);
|
||||
if (!mask.empty()) {
|
||||
masked_ice_rings = std::move(mask);
|
||||
const auto final_sg = experiment_.GetGemmiSpaceGroup();
|
||||
sm = scale_and_merge(final_sg ? final_sg->short_name() : "P1", false);
|
||||
}
|
||||
}
|
||||
|
||||
const auto twin_sg_number = experiment_.GetSpaceGroupNumber();
|
||||
const gemmi::SpaceGroup *twin_sg = twin_sg_number
|
||||
? gemmi::find_spacegroup_by_number(twin_sg_number.value()) : nullptr;
|
||||
|
||||
+52
-74
@@ -31,7 +31,6 @@
|
||||
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
|
||||
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
||||
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
||||
#include "../image_analysis/scale_merge/IceRingMask.h"
|
||||
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
|
||||
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
|
||||
#include "Rugnux.h"
|
||||
@@ -78,7 +77,7 @@ void print_usage() {
|
||||
std::cout << " --spot-high-resolution <num> High resolution limit for spot finding. If omitted (or 0), spot finding is not clipped in resolution and extends as far as the detector reaches" << std::endl;
|
||||
std::cout << " --spot-low-resolution <num> Low resolution limit for spot finding, in A (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weakly-diffracting serial data)" << std::endl;
|
||||
std::cout << " --max-spots <num> Max spot count per image, the strongest ones, handed to indexing (default: 1000)" << std::endl;
|
||||
std::cout << " --detect-ice-rings[=on|off] Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's setting, or - where the file says nothing - on for rotation and off for stills. The merge-time ice-ring mask is separate, see --ice-ring-mask" << std::endl;
|
||||
std::cout << " --detect-ice-rings[=on|off] Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's setting, or - where the file says nothing - on for rotation and off for stills" << std::endl;
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << " Indexing" << std::endl;
|
||||
@@ -93,6 +92,7 @@ void print_usage() {
|
||||
std::cout << " -C, --unit-cell <cell> Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl;
|
||||
std::cout << " -r, --refine <txt> Geometry refinement algorithm (none|orientation|beam_and_lattice|flex); flex tries all three per image and keeps whichever indexes the most spots (alias: multi)" << std::endl;
|
||||
std::cout << " --refine-geometry[=N|off] Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default: 200) then re-indexes (lifts weak-stills indexing). Default ON for stills when a reference cell is given (-C / reference MTZ); =off disables" << std::endl;
|
||||
std::cout << " --index-ice-rings[=on|off] Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: off; no effect without --detect-ice-rings)" << std::endl;
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << " Scaling and merging (on by default)" << std::endl;
|
||||
@@ -109,9 +109,8 @@ void print_usage() {
|
||||
std::cout << " --resolution-cutoff <txt> Automatic high-resolution cutoff for the written reflections + reported shells: cc-logistic|off (default: cc-logistic; ignored when --scaling-high-resolution is set)" << std::endl;
|
||||
std::cout << " --resolution-cc-target <num> CC1/2 target defining the cc-logistic fall-off (default: 0.30)" << std::endl;
|
||||
std::cout << " --resolution-shells <num> Number of resolution shells in the reported statistics table (default: 10)" << std::endl;
|
||||
std::cout << " --ice-min-score <num> Ice-presence gate: measured ice score (1 = no ice) a run must reach before ANY ice handling is applied - the flagging, the exclusion from scaling and the merge-time mask (default: 1.5). The eleven hexagonal bands cover 16-26% of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. 0 = no gate (always handle ice)" << std::endl;
|
||||
std::cout << " --ice-min-score <num> Ice-presence gate: measured ice score (1 = no ice) a run must reach before ANY ice handling is applied - the flagging and the exclusion from scaling (default: 1.5). The eleven hexagonal bands cover 16-26% of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. 0 = no gate (always handle ice)" << std::endl;
|
||||
std::cout << " --ice-min-spot-ratio <num> Second ice-presence channel: found spots on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so --ice-min-score alone is blind to it. Default 2.0; 0 disables this channel" << std::endl;
|
||||
std::cout << " --ice-ring-mask[=on|off] Drop a hexagonal-ice ring from the merge when its merged half-set CC1/2 has collapsed below its resolution shoulders, then re-merge (default: OFF - measured to delete better-than-average reflections for no gain; needs --detect-ice-rings on). This is ONLY the merge-time mask - ice-spot flagging and the ice exclusion from scaling stay as --detect-ice-rings sets them" << std::endl;
|
||||
std::cout << " --min-partiality <num> Minimum partiality to accept reflection (default: 0.02)" << std::endl;
|
||||
std::cout << " --capture-uncertainty <num> rot3d: systematic sigma ~num*(1-captured_fraction)*I on under-captured fulls (default: 1.0 for rot3d, 0 otherwise)" << std::endl;
|
||||
std::cout << " --min-captured-fraction <num> rot3d: drop a combined full whose rocking curve was captured below this fraction (edge-of-sweep truncated fulls) (default: 0.7 for rotation, 0 otherwise; 0 = off)" << std::endl;
|
||||
@@ -139,6 +138,7 @@ void print_usage() {
|
||||
std::cout << " --azim-min-q <num> Azimuthal-integration minimum Q (1/A)" << std::endl;
|
||||
std::cout << " --azim-max-q <num> Azimuthal-integration maximum Q (1/A). If omitted, integration extends to the highest Q the detector reaches." << std::endl;
|
||||
std::cout << " --azim-phi-bins <num> Number of azimuthal (phi) bins (default: 1)" << std::endl;
|
||||
std::cout << " --azim-sigma-clip <num> Sigma-clip the azimuthal profile: repeat the integration twice more, each time rejecting pixels further than <num> sigma from their own bin's mean (default: 0 = off, plain per-bin mean; must be >= 2). A powder ring is azimuthally smooth and survives, Bragg peaks do not, so the result is the smooth background UNDER the peaks - do not use it where a ring's integrated intensity is wanted" << std::endl;
|
||||
std::cout << " --polarization-correction <on|off> Enable/disable azimuthal polarization correction" << std::endl;
|
||||
std::cout << " --solid-angle-correction <on|off> Enable/disable azimuthal solid angle correction" << std::endl;
|
||||
std::cout << std::endl;
|
||||
@@ -197,7 +197,7 @@ enum {
|
||||
OPT_NO_SCALING_CORRECTIONS,
|
||||
OPT_NO_EXPECTED_VARIANCE_MERGE,
|
||||
OPT_DETECT_ICE_RINGS,
|
||||
OPT_ICE_RING_MASK,
|
||||
OPT_INDEX_ICE_RINGS,
|
||||
OPT_ICE_MIN_SCORE,
|
||||
OPT_ICE_MIN_SPOT_RATIO,
|
||||
OPT_NO_SCALE_FULLS,
|
||||
@@ -206,6 +206,7 @@ enum {
|
||||
OPT_AZIM_MIN_Q,
|
||||
OPT_AZIM_MAX_Q,
|
||||
OPT_AZIM_PHI_BINS,
|
||||
OPT_AZIM_SIGMA_CLIP,
|
||||
OPT_AZINT_ONLY,
|
||||
OPT_SCALE,
|
||||
OPT_NO_MERGE,
|
||||
@@ -254,6 +255,7 @@ static option long_options[] = {
|
||||
{"azim-min-q", required_argument, nullptr, OPT_AZIM_MIN_Q},
|
||||
{"azim-max-q", required_argument, nullptr, OPT_AZIM_MAX_Q},
|
||||
{"azim-phi-bins", required_argument, nullptr, OPT_AZIM_PHI_BINS},
|
||||
{"azim-sigma-clip", required_argument, nullptr, OPT_AZIM_SIGMA_CLIP},
|
||||
{"polarization-correction", required_argument, nullptr, OPT_POLARIZATION_CORRECTION},
|
||||
{"solid-angle-correction", required_argument, nullptr, OPT_SOLID_ANGLE_CORRECTION},
|
||||
{"beam-x", required_argument, nullptr, OPT_BEAM_X},
|
||||
@@ -298,7 +300,7 @@ static option long_options[] = {
|
||||
{"integrator", required_argument, nullptr, OPT_INTEGRATOR},
|
||||
{"simple-stills", no_argument, nullptr, OPT_SIMPLE_STILLS},
|
||||
{"detect-ice-rings", optional_argument, nullptr, OPT_DETECT_ICE_RINGS},
|
||||
{"ice-ring-mask", optional_argument, nullptr, OPT_ICE_RING_MASK},
|
||||
{"index-ice-rings", optional_argument, nullptr, OPT_INDEX_ICE_RINGS},
|
||||
{"ice-min-score", required_argument, nullptr, OPT_ICE_MIN_SCORE},
|
||||
{"ice-min-spot-ratio", required_argument, nullptr, OPT_ICE_MIN_SPOT_RATIO},
|
||||
{"reject-outliers", required_argument, nullptr, OPT_REJECT_OUTLIERS},
|
||||
@@ -531,11 +533,12 @@ static int RunRugnux(int argc, char **argv) {
|
||||
std::optional<bool> scale_fulls_arg; // --scale-fulls / --no-scale-fulls; default on for rot3d
|
||||
bool write_process_h5_flag = false; // --write-process-h5; also write _process.h5 when merging
|
||||
std::optional<bool> detect_ice_rings; // --detect-ice-rings[=on|off]; unset => use the dataset (file) value
|
||||
bool ice_ring_mask = false; // --ice-ring-mask[=on|off]; merge-time CC1/2 ice-ring mask
|
||||
bool index_ice_rings = false; // --index-ice-rings[=on|off]; index on the ice-band spots too
|
||||
std::optional<double> ice_min_score_arg; // --ice-min-score: ice-presence gate on the measured score
|
||||
std::optional<double> ice_min_spot_ratio_arg; // --ice-min-spot-ratio: the same gate on the spot channel
|
||||
std::optional<float> min_q, max_q, q_spacing; // azimuthal integration range / -q spacing (1/A)
|
||||
std::optional<int32_t> azimuthal_bins; // --azimuthal-bins
|
||||
std::optional<double> azim_sigma_clip; // --azim-sigma-clip: 0 = off
|
||||
std::optional<bool> polarization_correction; // --polarization-correction (azimuthal integration)
|
||||
std::optional<bool> solid_angle_correction; // --solid-angle-correction (azimuthal integration)
|
||||
|
||||
@@ -842,13 +845,13 @@ static int RunRugnux(int argc, char **argv) {
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
case OPT_ICE_RING_MASK:
|
||||
case OPT_INDEX_ICE_RINGS:
|
||||
if (optarg == nullptr || strcmp(optarg, "on") == 0)
|
||||
ice_ring_mask = true;
|
||||
index_ice_rings = true;
|
||||
else if (strcmp(optarg, "off") == 0)
|
||||
ice_ring_mask = false;
|
||||
index_ice_rings = false;
|
||||
else {
|
||||
logger.Error("Invalid --ice-ring-mask value: {} (expected on|off)", optarg);
|
||||
logger.Error("Invalid --index-ice-rings value: {} (expected on|off)", optarg);
|
||||
print_usage();
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
@@ -969,6 +972,9 @@ static int RunRugnux(int argc, char **argv) {
|
||||
case OPT_AZIM_MAX_Q:
|
||||
max_q = atof(optarg);
|
||||
break;
|
||||
case OPT_AZIM_SIGMA_CLIP:
|
||||
azim_sigma_clip = parse_double_arg(optarg, "--azim-sigma-clip", logger);
|
||||
break;
|
||||
case OPT_AZIM_PHI_BINS:
|
||||
azimuthal_bins = atoi(optarg);
|
||||
break;
|
||||
@@ -1190,7 +1196,6 @@ static int RunRugnux(int argc, char **argv) {
|
||||
scaling_settings.MinCCForImage(min_image_cc / 100.0); // --min-image-cc is percent; the setting is a fraction
|
||||
scaling_settings.StillsPartialityRefine(!simple_stills_flag);
|
||||
scaling_settings.ExpectedVarianceMerge(!no_expected_variance_merge);
|
||||
scaling_settings.IceRingMergeMask(ice_ring_mask);
|
||||
if (ice_min_score_arg)
|
||||
scaling_settings.IceMinScore(static_cast<float>(*ice_min_score_arg));
|
||||
if (ice_min_spot_ratio_arg)
|
||||
@@ -1262,10 +1267,6 @@ static int RunRugnux(int argc, char **argv) {
|
||||
std::vector<MergedReflection> merged_reflections;
|
||||
MergeStatistics merged_statistics;
|
||||
double error_model_isa = 0.0;
|
||||
// Ice rings dropped from the merge because their CC1/2 collapsed. Decided from a first merge
|
||||
// and applied by a second, as the full pipeline does - flagging alone only keeps the ice
|
||||
// reflections out of the SCALE fit, which on its own costs a little and buys nothing.
|
||||
std::vector<char> masked_ice_rings;
|
||||
|
||||
// Rotation (rot3d): the dedicated RotationScaleMerge does the whole self-scale -> 3D combine ->
|
||||
// merge, including the default-on decay + absorption correction surfaces. It does not support
|
||||
@@ -1279,25 +1280,13 @@ static int RunRugnux(int argc, char **argv) {
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"Rotation scaling/merging (RotationScaleMerge) does not support reference "
|
||||
"scaling or wedge refinement");
|
||||
// The ice half-width has to be the real one: it is what turns a reflection's resolution
|
||||
// into a ring index, so a zero here makes every ice test inside the merge a no-op.
|
||||
RotationScaleMerge rsm(experiment, reflections, experiment.GetUnitCell(),
|
||||
scaling_iter, ice_width, nthreads, logger);
|
||||
scaling_iter, nthreads, logger);
|
||||
rsm.Ingest();
|
||||
// Ingest() is separate from Run() precisely so the merge can be repeated; the ice-ring
|
||||
// mask below needs a first merge before it can be decided.
|
||||
auto run = [&](const std::vector<char> &masked) {
|
||||
auto r = rsm.Run(false, masked);
|
||||
merged_reflections = std::move(r.merged);
|
||||
merged_statistics = std::move(r.statistics);
|
||||
error_model_isa = r.isa;
|
||||
};
|
||||
run({});
|
||||
if (experiment.IsDetectIceRings() && experiment.GetScalingSettings().GetIceRingMergeMask() && ice_present) {
|
||||
masked_ice_rings = FindDecorrelatedIceRings(merged_reflections, ice_width, logger);
|
||||
if (!masked_ice_rings.empty())
|
||||
run(masked_ice_rings);
|
||||
}
|
||||
auto r = rsm.Run(false);
|
||||
merged_reflections = std::move(r.merged);
|
||||
merged_statistics = std::move(r.statistics);
|
||||
error_model_isa = r.isa;
|
||||
} else {
|
||||
// Scaling self-references: the reference MTZ (if any) fixes the cell/space group, reports
|
||||
// CCref and provides the R-free test set, but is NOT a scale anchor - scaling each image
|
||||
@@ -1313,48 +1302,35 @@ static int RunRugnux(int argc, char **argv) {
|
||||
const double mean_tilt = refiner.Run(reflections, nthreads);
|
||||
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
|
||||
}
|
||||
// The merge alone, repeatable for the ice-ring mask below. The scaling above is NOT redone:
|
||||
// it has already been applied to `reflections`, and running it twice would compound the
|
||||
// correction.
|
||||
auto merge = [&](const std::vector<char> &masked) {
|
||||
MergeOnTheFly merge_engine(experiment);
|
||||
merge_engine.ReferenceCell(experiment.GetUnitCell());
|
||||
// --min-image-cc has to hold for the merge itself, not only for the reported statistics.
|
||||
merge_engine.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
|
||||
if (!masked.empty())
|
||||
merge_engine.MaskIceRings(masked, ice_width);
|
||||
// Fit the (a, b) error model from symmetry-mate scatter before merging, exactly as the full
|
||||
// pipeline does (Rugnux.cpp). Without this the offline --scale merge would use the identity
|
||||
// model and produce much worse stills intensities (no (b*I)^2 systematic term, no sigma floor).
|
||||
merge_engine.RefineErrorModel(reflections);
|
||||
if (merge_engine.ErrorModelActive())
|
||||
logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f} chi2={:.2f}", merge_engine.ErrorModelA(),
|
||||
merge_engine.ErrorModelB(),
|
||||
merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0,
|
||||
merge_engine.ErrorModelChi2());
|
||||
for (size_t i = 0; i < reflections.size(); ++i)
|
||||
merge_engine.AddImage(reflections[i], static_cast<int64_t>(i));
|
||||
merged_reflections = merge_engine.ExportReflections();
|
||||
MergeOnTheFly merge_engine(experiment);
|
||||
merge_engine.ReferenceCell(experiment.GetUnitCell());
|
||||
// --min-image-cc has to hold for the merge itself, not only for the reported statistics.
|
||||
merge_engine.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
|
||||
// Fit the (a, b) error model from symmetry-mate scatter before merging, exactly as the full
|
||||
// pipeline does (Rugnux.cpp). Without this the offline --scale merge would use the identity
|
||||
// model and produce much worse stills intensities (no (b*I)^2 systematic term, no sigma floor).
|
||||
merge_engine.RefineErrorModel(reflections);
|
||||
if (merge_engine.ErrorModelActive())
|
||||
logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f} chi2={:.2f}", merge_engine.ErrorModelA(),
|
||||
merge_engine.ErrorModelB(),
|
||||
merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0,
|
||||
merge_engine.ErrorModelChi2());
|
||||
for (size_t i = 0; i < reflections.size(); ++i)
|
||||
merge_engine.AddImage(reflections[i], static_cast<int64_t>(i));
|
||||
merged_reflections = merge_engine.ExportReflections();
|
||||
|
||||
// Automatic high-resolution cutoff (post-merge), matching the full-analysis path: a manual
|
||||
// --scaling-high-resolution wins, otherwise trim the written reflections + reported shells
|
||||
// to the CC1/2 fall-off. (Rotation is cut inside RotationScaleMerge above.)
|
||||
const auto &cut_ss = experiment.GetScalingSettings();
|
||||
// The offline --scale path re-scales a stored _process.h5 and is never a P1 search merge.
|
||||
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
|
||||
merged_reflections, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
|
||||
cut_ss.GetResolutionCCTarget(), /*for_search=*/false, logger);
|
||||
// Automatic high-resolution cutoff (post-merge), matching the full-analysis path: a manual
|
||||
// --scaling-high-resolution wins, otherwise trim the written reflections + reported shells
|
||||
// to the CC1/2 fall-off. (Rotation is cut inside RotationScaleMerge above.)
|
||||
const auto &cut_ss = experiment.GetScalingSettings();
|
||||
// The offline --scale path re-scales a stored _process.h5 and is never a P1 search merge.
|
||||
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
|
||||
merged_reflections, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
|
||||
cut_ss.GetResolutionCCTarget(), /*for_search=*/false, logger);
|
||||
|
||||
merged_statistics = merge_engine.MergeStats(merged_reflections, reflections, reference_data,
|
||||
effective_d_min);
|
||||
error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
|
||||
};
|
||||
merge({});
|
||||
if (experiment.IsDetectIceRings() && experiment.GetScalingSettings().GetIceRingMergeMask() && ice_present) {
|
||||
masked_ice_rings = FindDecorrelatedIceRings(merged_reflections, ice_width, logger);
|
||||
if (!masked_ice_rings.empty())
|
||||
merge(masked_ice_rings);
|
||||
}
|
||||
merged_statistics = merge_engine.MergeStats(merged_reflections, reflections, reference_data,
|
||||
effective_d_min);
|
||||
error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
|
||||
}
|
||||
|
||||
logger.Info("Scale + merge completed in {:.2f} s ({} unique reflections)",
|
||||
@@ -1451,6 +1427,8 @@ static int RunRugnux(int argc, char **argv) {
|
||||
azint_settings.QSpacing_recipA(q_spacing.value());
|
||||
if (azimuthal_bins)
|
||||
azint_settings.AzimuthalBinCount(azimuthal_bins.value());
|
||||
if (azim_sigma_clip)
|
||||
azint_settings.SigmaClip(static_cast<float>(azim_sigma_clip.value()));
|
||||
if (polarization_correction)
|
||||
azint_settings.PolarizationCorrection(polarization_correction.value());
|
||||
if (solid_angle_correction)
|
||||
@@ -1560,6 +1538,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
if (rotation_indexing_range.has_value())
|
||||
indexing_settings.RotationIndexingMinAngularRange_deg(rotation_indexing_range.value());
|
||||
indexing_settings.GeomRefinementAlgorithm(refinement_algorithm);
|
||||
indexing_settings.IndexIceRings(index_ice_rings);
|
||||
experiment.ImportIndexingSettings(indexing_settings);
|
||||
|
||||
// --detect-ice-rings[=on|off] overrides the value carried in from the dataset (HDF5MetadataSource
|
||||
@@ -1585,7 +1564,6 @@ static int RunRugnux(int argc, char **argv) {
|
||||
scaling_settings.CorrectionSurfaces(false);
|
||||
scaling_settings.StillsPartialityRefine(!simple_stills_flag);
|
||||
scaling_settings.ExpectedVarianceMerge(!no_expected_variance_merge);
|
||||
scaling_settings.IceRingMergeMask(ice_ring_mask);
|
||||
if (ice_min_score_arg)
|
||||
scaling_settings.IceMinScore(static_cast<float>(*ice_min_score_arg));
|
||||
if (ice_min_spot_ratio_arg)
|
||||
|
||||
Reference in New Issue
Block a user