Merge commit '86f17740f' into stillsidx
This commit is contained in:
commit
cb044459ba
22 files changed
+674
-225
No files matched your search
@@ -367,9 +367,12 @@ generated API model and internal types.
|
||||
(`beam_stop/ShadowFinder` is built and runs by default in `rugnux` — it is no longer the unbuilt
|
||||
prototype an older version of this file described). Least-squares refinement uses **Ceres** (fetched in
|
||||
`image_analysis/CMakeLists.txt`, built with miniglog, no MKL, no Ceres-CUDA, CXX_THREADS).
|
||||
The indexer is chosen with `-X` (`FFBIDX|FFT|FFTW|Auto|None`, default `Auto`, which resolves to a
|
||||
The indexer is chosen with `-X` (`Auto|FFT|cuFFT|FFTW|FFBIDX|None`, default `Auto`, which resolves to a
|
||||
GPU indexer when one is present and `fftw` otherwise): `ffbidx` wants a known cell (`-C`) and suits
|
||||
sparse serial stills; `fft`/`fftw` index de novo and suit strong rotation data.
|
||||
sparse serial stills; `fft` indexes de novo and suits strong rotation data - it is the algorithm, run
|
||||
with cuFFT on a GPU and with FFTW without one (or under `--no-gpu`), while `cufft`/`fftw` name one
|
||||
implementation each. Only rugnux's CLI spells it this way: internally (and in the broker's OpenAPI)
|
||||
`FFT` is the GPU implementation, which the CLI's `cufft` maps onto.
|
||||
|
||||
**FPGA** (`fpga/`): `hls/` is the Vitis HLS source (image-analysis kernels), `hls_simulation/`
|
||||
runs that same HLS on CPU for hardware-free testing, `hdl/` is the Verilog RTL, `host_library/` is
|
||||
|
||||
+40
-25
@@ -28,50 +28,63 @@ std::string get_gpu_description() {
|
||||
}
|
||||
|
||||
namespace {
|
||||
// How many of one kind of GPU work are running, for a thread to wait until none is.
|
||||
struct GPUWorkCount {
|
||||
std::mutex m;
|
||||
std::condition_variable none;
|
||||
int n = 0;
|
||||
void Add() {
|
||||
std::lock_guard lock(m);
|
||||
++n;
|
||||
// How many probe passes (GPUWorkBeside) and main-line image passes (GPUImagePass) are running, for a
|
||||
// thread to wait until none of one kind is.
|
||||
std::mutex gpu_work_m;
|
||||
std::condition_variable gpu_work_changed;
|
||||
int gpu_work_beside_n = 0, gpu_image_passes_n = 0;
|
||||
|
||||
// A card too small for a probe pass beside the main line's image pass, or beside another probe pass:
|
||||
// measured on a 16 Mpx detector, two of them together held 8.6 GB. On such a card a probe pass waits,
|
||||
// before it starts, for every other one and for the image passes to end, and an image pass for the
|
||||
// probe passes - so they take turns, and only when each runs depends on the card, never what it
|
||||
// computes. None waits once it is running, so nothing waits in a circle.
|
||||
constexpr size_t TAKE_TURNS_BELOW_BYTES = 12'000'000'000ull;
|
||||
|
||||
bool take_turns() {
|
||||
static const bool small = get_gpu_count() > 0 && get_gpu_total_memory(0) < TAKE_TURNS_BELOW_BYTES;
|
||||
return small;
|
||||
}
|
||||
|
||||
void remove(int &n) {
|
||||
{
|
||||
std::lock_guard lock(gpu_work_m);
|
||||
--n;
|
||||
}
|
||||
void Remove() {
|
||||
{
|
||||
std::lock_guard lock(m);
|
||||
--n;
|
||||
}
|
||||
none.notify_all();
|
||||
}
|
||||
};
|
||||
GPUWorkCount gpu_work_beside, gpu_image_passes;
|
||||
gpu_work_changed.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
GPUWorkBeside::GPUWorkBeside() {
|
||||
gpu_work_beside.Add();
|
||||
std::unique_lock lock(gpu_work_m);
|
||||
if (take_turns())
|
||||
gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0 && gpu_work_beside_n == 0; });
|
||||
++gpu_work_beside_n;
|
||||
}
|
||||
|
||||
GPUWorkBeside::~GPUWorkBeside() {
|
||||
gpu_work_beside.Remove();
|
||||
remove(gpu_work_beside_n);
|
||||
}
|
||||
|
||||
bool wait_for_gpu_work_beside(std::chrono::seconds timeout) {
|
||||
std::unique_lock lock(gpu_work_beside.m);
|
||||
return gpu_work_beside.none.wait_for(lock, timeout, [] { return gpu_work_beside.n == 0; });
|
||||
std::unique_lock lock(gpu_work_m);
|
||||
return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; });
|
||||
}
|
||||
|
||||
GPUImagePass::GPUImagePass() {
|
||||
gpu_image_passes.Add();
|
||||
std::unique_lock lock(gpu_work_m);
|
||||
if (take_turns())
|
||||
gpu_work_changed.wait(lock, [] { return gpu_work_beside_n == 0; });
|
||||
++gpu_image_passes_n;
|
||||
}
|
||||
|
||||
GPUImagePass::~GPUImagePass() {
|
||||
gpu_image_passes.Remove();
|
||||
remove(gpu_image_passes_n);
|
||||
}
|
||||
|
||||
void wait_for_gpu_image_passes() {
|
||||
std::unique_lock lock(gpu_image_passes.m);
|
||||
gpu_image_passes.none.wait(lock, [] { return gpu_image_passes.n == 0; });
|
||||
std::unique_lock lock(gpu_work_m);
|
||||
gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; });
|
||||
}
|
||||
|
||||
#ifndef JFJOCH_USE_CUDA
|
||||
@@ -80,6 +93,8 @@ int32_t get_gpu_count() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void hide_gpus() {}
|
||||
|
||||
std::vector<std::string> get_gpu_names() {
|
||||
return {};
|
||||
}
|
||||
|
||||
+19
-4
@@ -18,7 +18,17 @@ inline void cuda_err(cudaError_t val) {
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::atomic<bool> gpus_hidden{false};
|
||||
}
|
||||
|
||||
void hide_gpus() {
|
||||
gpus_hidden = true;
|
||||
}
|
||||
|
||||
int32_t get_gpu_count() {
|
||||
if (gpus_hidden)
|
||||
return 0;
|
||||
int device_count;
|
||||
cudaError_t val = cudaGetDeviceCount(&device_count);
|
||||
switch (val) {
|
||||
@@ -137,11 +147,13 @@ void set_gpu_blocking_sync() {
|
||||
// cudaSetDevice creates the device's context, serially, and on four cards that was most of a
|
||||
// second before the run had started. The function is taken from the driver at run time, so the
|
||||
// program does not link against it and still starts on a machine without one.
|
||||
// Without a device nothing is called at all: even cudaGetLastError() would load the driver.
|
||||
if (get_gpu_count() == 0)
|
||||
return;
|
||||
void *fn = nullptr;
|
||||
cudaDriverEntryPointQueryResult found{};
|
||||
if (get_gpu_count() == 0
|
||||
|| cudaGetDriverEntryPointByVersion("cuDevicePrimaryCtxSetFlags", &fn, 12000, cudaEnableDefault,
|
||||
&found) != cudaSuccess
|
||||
if (cudaGetDriverEntryPointByVersion("cuDevicePrimaryCtxSetFlags", &fn, 12000, cudaEnableDefault, &found)
|
||||
!= cudaSuccess
|
||||
|| found != cudaDriverEntryPointSuccess) {
|
||||
cudaGetLastError();
|
||||
return;
|
||||
@@ -175,10 +187,13 @@ void release_gpu_contexts() {
|
||||
}
|
||||
|
||||
void cuda_clear_error() {
|
||||
cudaGetLastError();
|
||||
if (!gpus_hidden) // hidden, nothing has used CUDA, and this call would still load the driver
|
||||
cudaGetLastError();
|
||||
}
|
||||
|
||||
void cuda_throw_if_context_lost() {
|
||||
if (gpus_hidden)
|
||||
return;
|
||||
// cudaFree(nullptr) frees nothing, but a lost context fails it. The last error cannot say the same:
|
||||
// once cudaGetLastError() has returned a sticky error, it and cudaPeekAtLastError() report success.
|
||||
const cudaError_t err = cudaFree(nullptr);
|
||||
|
||||
+10
-2
@@ -11,6 +11,12 @@
|
||||
|
||||
int32_t get_gpu_count();
|
||||
|
||||
// From here on get_gpu_count() is 0 without asking CUDA, so nothing in the process creates a context or
|
||||
// touches a device and every GPU path takes its CPU path, as on a machine without a GPU: what
|
||||
// CUDA_VISIBLE_DEVICES= does, asked for by the program (rugnux --no-gpu). Call it before anything uses
|
||||
// CUDA; there is no way back.
|
||||
void hide_gpus();
|
||||
|
||||
// Names of the visible GPUs, in device order and one entry per device, so repeated cards repeat.
|
||||
// Empty without CUDA and on a machine with no device, which is also what get_gpu_count() == 0 says.
|
||||
std::vector<std::string> get_gpu_names();
|
||||
@@ -89,7 +95,8 @@ void cuda_throw_if_context_lost();
|
||||
|
||||
// GPU work that runs beside the main line of a process and gives all of its device memory back when it
|
||||
// ends: a probe pass run on a copy of the run while the run itself scales and merges. Hold one for as
|
||||
// long as that work runs. Build-independent - it only counts.
|
||||
// long as that work runs. On a small card it waits first for the main line's image passes (GPUImagePass)
|
||||
// and for other work beside to end. Build-independent - it only counts.
|
||||
class GPUWorkBeside {
|
||||
public:
|
||||
GPUWorkBeside();
|
||||
@@ -103,7 +110,8 @@ bool wait_for_gpu_work_beside(std::chrono::seconds timeout);
|
||||
|
||||
// The image pass of a run's main line - not of a probe holding a GPUWorkBeside: the stretch that holds
|
||||
// the most of the card, and that never waits for a probe, so a probe that does not fit beside it can
|
||||
// wait for it to end. Hold one for as long as the image pass runs. Build-independent - it only counts.
|
||||
// wait for it to end. Hold one for as long as the image pass runs. On a card too small for both, it
|
||||
// waits first for the work beside to end. Build-independent - it only counts.
|
||||
class GPUImagePass {
|
||||
public:
|
||||
GPUImagePass();
|
||||
|
||||
@@ -1710,7 +1710,8 @@ IndexingAlgorithmEnum DiffractionExperiment::GetIndexingAlgorithm() const {
|
||||
return IndexingAlgorithmEnum::FFT;
|
||||
return IndexingAlgorithmEnum::FFBIDX;
|
||||
case IndexingAlgorithmEnum::FFT:
|
||||
return IndexingAlgorithmEnum::FFT;
|
||||
// No silent switch to FFTW, too slow for live stills: a missing GPU turns indexing off
|
||||
return get_gpu_count() == 0 ? IndexingAlgorithmEnum::None : IndexingAlgorithmEnum::FFT;
|
||||
case IndexingAlgorithmEnum::FFTW:
|
||||
return IndexingAlgorithmEnum::FFTW;
|
||||
default:
|
||||
|
||||
@@ -521,7 +521,7 @@ Parallel Connected-Component Labelling on GPUs" (2018), IEEE Trans. Parallel Dis
|
||||
The FFT indexer's GPU peak search divides by a fixed window width with a reciprocal multiplication
|
||||
corrected once by the exact remainder, which returns the correctly rounded quotient, following
|
||||
P. W. Markstein, "Computation of elementary functions on the IBM RISC System/6000 processor"
|
||||
(1990), IBM J. Res. Dev. 34, 111-119 (DOI not verified here).
|
||||
(1990), IBM J. Res. Dev. 34, 111-119.
|
||||
|
||||
This software uses the Viridis, Magma and Inferno colormaps from Matplotlib under its
|
||||
BSD-compatible license. J. D. Hunter, "Matplotlib: A 2D graphics environment" (2007), Comput. Sci.
|
||||
|
||||
@@ -330,6 +330,7 @@ General:
|
||||
| `-e, --end-image <num>` | Last image to process (default: all) |
|
||||
| `-t, --stride <num>` | Process every *n*-th image (default: 1) |
|
||||
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
|
||||
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched and every GPU step takes its CPU path (`-X Auto` and `-X FFT` index with FFTW; `-X cuFFT` and `-X FFBIDX` are refused) |
|
||||
|
||||
Mode — `--mode <name>` (default `mx`):
|
||||
|
||||
@@ -407,7 +408,7 @@ For very long axes the direction grid's angular resolution binds well below 500
|
||||
| Option | Description |
|
||||
| --- | --- |
|
||||
| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation |
|
||||
| `-X, --indexing-algorithm <txt>` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` |
|
||||
| `-X, --indexing-algorithm <txt>` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each. `cuFFT` and `FFBIDX` (known cell, `-C`) need a GPU and are refused without one. `Auto` takes `FFBIDX` on stills with a known cell and a GPU, and `FFT` otherwise; rotation data always use the FFT search |
|
||||
| `-C, --unit-cell <cell>` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it |
|
||||
| `--fft-min-unit-cell <num>` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered |
|
||||
| `--min-indexed-spots <num>` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
|
||||
|
||||
@@ -113,8 +113,8 @@ build has no CUDA there and needs nothing installed beyond Xcode and CMake.
|
||||
## Hardware
|
||||
|
||||
As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. The CUDA build
|
||||
provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`fft`); without CUDA
|
||||
only the CPU `fftw` indexer is available. With a GPU present most of the per-image pipeline runs on
|
||||
provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`cufft`); without CUDA
|
||||
only the CPU `fftw` indexer is available, and `-X fft` runs on it. With a GPU present most of the per-image pipeline runs on
|
||||
the device — bitshuffle+LZ4 decompression, miniCBF byte-offset decoding and marCCD widening, image
|
||||
preprocessing, azimuthal integration, spot finding, prediction and Bragg integration — as do the
|
||||
pre-scan (beam-stop mask, background beam-centre fit, spot widths, defective pixels; one accumulator
|
||||
@@ -124,7 +124,7 @@ fits and rigid-body placement of the validation and the rotation and translation
|
||||
molecular replacement. The CPU implementations are the path where there is no GPU, and the two agree
|
||||
to rounding rather than bit for bit (parts of the scaling and the model fits round differently
|
||||
on the card), so a GPU and a CPU run can differ in the last digits of the merged data. The choice is automatic;
|
||||
a CUDA build runs on the CPU when the GPU is hidden from it (`CUDA_VISIBLE_DEVICES= rugnux ...`).
|
||||
a CUDA build runs on the CPU when told to (`rugnux --no-gpu ...`).
|
||||
The thread count (`-N`) governs the CPU side of all of it.
|
||||
|
||||
The released CUDA builds need only an NVIDIA **driver** on the host, no CUDA toolkit: 525.60.13 or
|
||||
@@ -189,7 +189,7 @@ a big cell. One sweep of a crystal with an axis of about 640 Å predicted 236 mi
|
||||
peaked at **17.5 GB of host memory on a GPU build and 72.5 GB on a CPU-only build**. A CPU-only build
|
||||
keeps in host RAM what a GPU build keeps on the card, so it is the one that needs the large machine.
|
||||
When such a set is too large for GPU scaling on the card, the run stops and says to run it on the
|
||||
CPU (`CUDA_VISIBLE_DEVICES= rugnux ...`, or a CPU-only build) — which then wants that much host memory.
|
||||
CPU (`rugnux --no-gpu ...`, or a CPU-only build) — which then wants that much host memory.
|
||||
|
||||
### When host memory runs out
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_
|
||||
msg += buf;
|
||||
}
|
||||
return msg + ". Either another program is using this GPU, or this data set needs more GPU memory than "
|
||||
"the card has - free the GPU and run again, or run on the CPU (CUDA_VISIBLE_DEVICES= rugnux "
|
||||
"the card has - free the GPU and run again, or run on the CPU (rugnux --no-gpu "
|
||||
"..., or a CPU-only build)";
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "../../common/CorrelationCoefficient.h"
|
||||
#include "../../common/ResolutionShells.h"
|
||||
#include "../../common/Definitions.h"
|
||||
#include "../../common/ParallelFor.h"
|
||||
#include "HKLKey.h"
|
||||
#include "RfreeFlags.h"
|
||||
#include "FrenchWilson.h"
|
||||
@@ -32,6 +33,27 @@ namespace {
|
||||
z = z ^ (z >> 31);
|
||||
return static_cast<int>(z & 1ULL);
|
||||
}
|
||||
|
||||
// Calls reduce(i, out) for the images in parallel, a block of them at a time, and then consume(i, out)
|
||||
// for each image of the block in image order - the order that every floating-point sum, every
|
||||
// half-set choice and every hash-map insertion of the merge depends on. The result is therefore the
|
||||
// serial loop's, bit for bit, at any thread count.
|
||||
template <typename T, typename Reduce, typename Consume>
|
||||
void ReduceImagesInOrder(size_t n_images, size_t nthreads, Reduce reduce, Consume consume) {
|
||||
constexpr size_t BLOCK_IMAGES = 4096;
|
||||
std::vector<std::vector<T>> out(std::min(BLOCK_IMAGES, n_images));
|
||||
for (size_t first = 0; first < n_images; first += BLOCK_IMAGES) {
|
||||
const int n = static_cast<int>(std::min(BLOCK_IMAGES, n_images - first));
|
||||
ParallelChunks(n, nthreads, [&](int lo, int hi) {
|
||||
for (int j = lo; j < hi; ++j) {
|
||||
out[j].clear();
|
||||
reduce(first + j, out[j]);
|
||||
}
|
||||
});
|
||||
for (int j = 0; j < n; ++j)
|
||||
consume(first + j, out[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x)
|
||||
@@ -57,14 +79,8 @@ MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional<UnitCell> &cell)
|
||||
return *this;
|
||||
}
|
||||
|
||||
void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id) {
|
||||
std::unique_lock ul(merged_mutex);
|
||||
|
||||
if (Mask(outcome))
|
||||
return;
|
||||
|
||||
const int prefer = HalfForImage(image_id);
|
||||
|
||||
size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vector<Contribution> &out) const {
|
||||
size_t rejected = 0;
|
||||
for (const auto &r: outcome.reflections) {
|
||||
if (generator.IsSystematicallyAbsent(r))
|
||||
continue;
|
||||
@@ -94,24 +110,36 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id
|
||||
const auto mit = reject_median_I.find(hkl_key);
|
||||
if (mit != reject_median_I.end() &&
|
||||
std::fabs(I_corr - mit->second) > reject_nsigma * sigma_corr) {
|
||||
++reject_count;
|
||||
++rejected;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
auto it = accumulator.find(hkl_key);
|
||||
if (it == accumulator.end())
|
||||
it = accumulator.emplace(hkl_key, MergeAccum{
|
||||
.h = hkl.plus ? hkl.h : -hkl.h,
|
||||
.k = hkl.plus ? hkl.k : -hkl.k,
|
||||
.l = hkl.plus ? hkl.l : -hkl.l,
|
||||
}).first;
|
||||
|
||||
const float w = 1.0f / (sigma_corr * sigma_corr);
|
||||
const float wI = w * I_corr;
|
||||
out.push_back(Contribution{
|
||||
.key = hkl_key,
|
||||
.h = hkl.plus ? hkl.h : -hkl.h,
|
||||
.k = hkl.plus ? hkl.k : -hkl.k,
|
||||
.l = hkl.plus ? hkl.l : -hkl.l,
|
||||
.w = w,
|
||||
.wI = w * I_corr,
|
||||
.d = r.d,
|
||||
.on_ice_ring = r.on_ice_ring
|
||||
});
|
||||
}
|
||||
return rejected;
|
||||
}
|
||||
|
||||
it->second.sum_wI += wI;
|
||||
it->second.sum_w += w;
|
||||
void MergeOnTheFly::Accumulate(const std::vector<Contribution> &contributions, int64_t image_id) {
|
||||
const int prefer = HalfForImage(image_id);
|
||||
|
||||
for (const auto &c: contributions) {
|
||||
auto it = accumulator.find(c.key);
|
||||
if (it == accumulator.end())
|
||||
it = accumulator.emplace(c.key, MergeAccum{.h = c.h, .k = c.k, .l = c.l}).first;
|
||||
|
||||
it->second.sum_wI += c.wI;
|
||||
it->second.sum_w += c.w;
|
||||
|
||||
// CC1/2 correlates the two half-set means, so a reflection whose observations all land in one
|
||||
// half has no second mean and drops out of the statistic entirely. Taking the image's own
|
||||
@@ -119,18 +147,44 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id
|
||||
// reflection floor(n/2) against ceil(n/2), which is the split cctbx and XDS use.
|
||||
const int half = it->second.n_half[prefer] > it->second.n_half[1 - prefer]
|
||||
? 1 - prefer : prefer;
|
||||
it->second.sum_wI_half[half] += wI;
|
||||
it->second.sum_w_half[half] += w;
|
||||
it->second.sum_wI_half[half] += c.wI;
|
||||
it->second.sum_w_half[half] += c.w;
|
||||
it->second.n_half[half]++;
|
||||
|
||||
if (r.on_ice_ring)
|
||||
if (c.on_ice_ring)
|
||||
it->second.on_ice_ring = true;
|
||||
|
||||
if (!std::isfinite(it->second.d) && std::isfinite(r.d) && r.d > 0.0f)
|
||||
it->second.d = r.d;
|
||||
if (!std::isfinite(it->second.d) && std::isfinite(c.d) && c.d > 0.0f)
|
||||
it->second.d = c.d;
|
||||
}
|
||||
}
|
||||
|
||||
void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id) {
|
||||
std::unique_lock ul(merged_mutex);
|
||||
|
||||
if (Mask(outcome))
|
||||
return;
|
||||
std::vector<Contribution> contributions;
|
||||
reject_count += Contributions(outcome, contributions);
|
||||
Accumulate(contributions, image_id);
|
||||
}
|
||||
|
||||
void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
|
||||
std::unique_lock ul(merged_mutex);
|
||||
|
||||
std::vector<size_t> rejected(outcomes.size(), 0);
|
||||
ReduceImagesInOrder<Contribution>(
|
||||
outcomes.size(), nthreads,
|
||||
[&](size_t i, std::vector<Contribution> &out) {
|
||||
if (!Mask(outcomes[i]))
|
||||
rejected[i] = Contributions(outcomes[i], out);
|
||||
},
|
||||
[&](size_t i, const std::vector<Contribution> &out) {
|
||||
reject_count += rejected[i];
|
||||
Accumulate(out, static_cast<int64_t>(i));
|
||||
});
|
||||
}
|
||||
|
||||
float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
|
||||
float var_bkg,
|
||||
uint64_t hkl_key) const {
|
||||
@@ -188,29 +242,36 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
|
||||
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl,
|
||||
// applying exactly the filters AddImage uses. ---
|
||||
struct Obs { float I, sigma; };
|
||||
struct KeyedObs { uint64_t key; Obs obs; };
|
||||
std::unordered_map<uint64_t, std::vector<Obs>> groups;
|
||||
|
||||
for (const auto &outcome: outcomes) {
|
||||
if (Mask(outcome))
|
||||
continue;
|
||||
for (const auto &r: outcome.reflections) {
|
||||
if (generator.IsSystematicallyAbsent(r))
|
||||
continue;
|
||||
if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr))
|
||||
continue;
|
||||
if (!AcceptReflection(r, high_resolution_limit, low_resolution_limit))
|
||||
continue;
|
||||
if (exclude_ice_rings && r.on_ice_ring)
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
const float I_corr = r.I * r.image_scale_corr;
|
||||
const float sigma_corr = r.sigma * r.image_scale_corr;
|
||||
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
|
||||
continue;
|
||||
groups[generator(r).pack()].push_back({I_corr, sigma_corr});
|
||||
}
|
||||
}
|
||||
ReduceImagesInOrder<KeyedObs>(
|
||||
outcomes.size(), nthreads,
|
||||
[&](size_t i, std::vector<KeyedObs> &out) {
|
||||
if (Mask(outcomes[i]))
|
||||
return;
|
||||
for (const auto &r: outcomes[i].reflections) {
|
||||
if (generator.IsSystematicallyAbsent(r))
|
||||
continue;
|
||||
if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr))
|
||||
continue;
|
||||
if (!AcceptReflection(r, high_resolution_limit, low_resolution_limit))
|
||||
continue;
|
||||
if (exclude_ice_rings && r.on_ice_ring)
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
const float I_corr = r.I * r.image_scale_corr;
|
||||
const float sigma_corr = r.sigma * r.image_scale_corr;
|
||||
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
|
||||
continue;
|
||||
out.push_back({generator(r).pack(), {I_corr, sigma_corr}});
|
||||
}
|
||||
},
|
||||
[&](size_t, const std::vector<KeyedObs> &out) {
|
||||
for (const auto &o: out)
|
||||
groups[o.key].push_back(o.obs);
|
||||
});
|
||||
|
||||
// --- 2. One global pool of (sigma^2, <I>^2, bias-corrected squared deviation). For an
|
||||
// observation in a group of n, the residual from the inverse-variance mean has
|
||||
@@ -337,7 +398,7 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
|
||||
error_model_chi2 = chi2.empty() ? 0.0 : median(chi2) / CHI2_1_MEDIAN;
|
||||
}
|
||||
|
||||
bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) {
|
||||
bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) const {
|
||||
if (reference_cell) {
|
||||
auto cell = outcome.latt.GetUnitCell();
|
||||
if (!cell.is_close(*reference_cell,
|
||||
@@ -355,12 +416,20 @@ bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<MergedReflection> MergeOnTheFly::ExportReflections() {
|
||||
std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
|
||||
std::unique_lock ul(merged_mutex);
|
||||
|
||||
std::vector<MergedReflection> out;
|
||||
std::vector<std::pair<uint64_t, const MergeAccum *>> by_key;
|
||||
by_key.reserve(accumulator.size());
|
||||
for (const auto &[key, accum]: accumulator)
|
||||
by_key.emplace_back(key, &accum);
|
||||
std::sort(by_key.begin(), by_key.end(),
|
||||
[](const auto &a, const auto &b) { return a.first < b.first; });
|
||||
|
||||
out.reserve(accumulator.size());
|
||||
for (const auto &accum: accumulator | std::views::values) {
|
||||
for (const auto &[key, accum_ptr]: by_key) {
|
||||
const MergeAccum &accum = *accum_ptr;
|
||||
if (accum.sum_w <= 0.0)
|
||||
continue;
|
||||
|
||||
@@ -388,18 +457,23 @@ std::vector<MergedReflection> MergeOnTheFly::ExportReflections() {
|
||||
|
||||
out.emplace_back(mr);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<MergedReflection> MergeOnTheFly::ExportReflections() {
|
||||
auto out = ExportIntensities();
|
||||
AssignRfreeFlags(out, space_group, scaling_settings.GetRfreeFraction(), 500, reference_cell);
|
||||
ApplyFrenchWilson(out, space_group);
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<MergedReflection> MergeAll(const DiffractionExperiment &x,
|
||||
const std::vector<IntegrationOutcome> &integration_outcome) {
|
||||
const std::vector<IntegrationOutcome> &integration_outcome,
|
||||
size_t nthreads,
|
||||
const std::optional<UnitCell> &reference_cell) {
|
||||
MergeOnTheFly merge(x);
|
||||
for (size_t i = 0; i < integration_outcome.size(); ++i)
|
||||
merge.AddImage(integration_outcome[i], static_cast<int64_t>(i));
|
||||
return merge.ExportReflections();
|
||||
merge.Threads(nthreads).ReferenceCell(reference_cell).AddImages(integration_outcome);
|
||||
return merge.ExportIntensities();
|
||||
}
|
||||
|
||||
struct ShellAccum {
|
||||
@@ -630,43 +704,51 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
|
||||
std::unordered_map<uint64_t, RmeasObs> rmeas_obs;
|
||||
rmeas_obs.reserve(merged.size());
|
||||
|
||||
for (int i = 0; i < integration_outcome.size(); ++i) {
|
||||
if (Mask(integration_outcome[i]))
|
||||
continue;
|
||||
struct ShellObs { uint64_t key; float I_corr; float merged_I; int shell; bool merged; };
|
||||
ReduceImagesInOrder<ShellObs>(
|
||||
integration_outcome.size(), nthreads,
|
||||
[&](size_t i, std::vector<ShellObs> &out) {
|
||||
if (Mask(integration_outcome[i]))
|
||||
return;
|
||||
for (const auto &r: integration_outcome[i].reflections) {
|
||||
if (generator.IsSystematicallyAbsent(r))
|
||||
continue;
|
||||
if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr))
|
||||
continue;
|
||||
if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A))
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
|
||||
for (const auto &r: integration_outcome[i].reflections) {
|
||||
if (generator.IsSystematicallyAbsent(r))
|
||||
continue;
|
||||
if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr))
|
||||
continue;
|
||||
if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A))
|
||||
continue;
|
||||
if (r.partiality < min_partiality)
|
||||
continue;
|
||||
const float I_corr = r.I * r.image_scale_corr;
|
||||
const float sigma_corr = r.sigma * r.image_scale_corr;
|
||||
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
|
||||
continue;
|
||||
|
||||
const float I_corr = r.I * r.image_scale_corr;
|
||||
const float sigma_corr = r.sigma * r.image_scale_corr;
|
||||
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
|
||||
continue;
|
||||
|
||||
const auto shell = shells.GetShell(r.d);
|
||||
if (!shell.has_value())
|
||||
continue;
|
||||
const int s = *shell;
|
||||
if (s >= 0 && s < n_shells) {
|
||||
acc[s].total_obs++;
|
||||
const auto key = generator(r).pack();
|
||||
const auto mit = merged_I.find(key);
|
||||
if (mit != merged_I.end()) {
|
||||
auto &ra = rmeas_obs[key];
|
||||
ra.sum_abs_dev += std::abs(static_cast<double>(I_corr) - mit->second);
|
||||
ra.sum_I += I_corr;
|
||||
ra.n++;
|
||||
ra.shell = s;
|
||||
const auto shell = shells.GetShell(r.d);
|
||||
if (!shell.has_value())
|
||||
continue;
|
||||
const int s = *shell;
|
||||
if (s >= 0 && s < n_shells) {
|
||||
const auto key = generator(r).pack();
|
||||
const auto mit = merged_I.find(key);
|
||||
out.push_back({key, I_corr, mit != merged_I.end() ? mit->second : NAN, s,
|
||||
mit != merged_I.end()});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[&](size_t, const std::vector<ShellObs> &out) {
|
||||
for (const auto &o: out) {
|
||||
acc[o.shell].total_obs++;
|
||||
if (o.merged) {
|
||||
auto &ra = rmeas_obs[o.key];
|
||||
ra.sum_abs_dev += std::abs(static_cast<double>(o.I_corr) - o.merged_I);
|
||||
ra.sum_I += o.I_corr;
|
||||
ra.n++;
|
||||
ra.shell = o.shell;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// R_meas per shell: sum over reflections of sqrt(n/(n-1)) * sum_i|I_i - <I>|, over sum of I_i.
|
||||
// R_merge and R_pim beside it, with the factor 1 and sqrt(1/(n-1)) - R_pim following Weiss (2001)
|
||||
|
||||
@@ -283,10 +283,11 @@ class MergeOnTheFly {
|
||||
// merged intensities feed the space-group search and the error model - those model fits must not
|
||||
// see the ice-contaminated intensities. The final in-symmetry merge keeps them (for completeness).
|
||||
bool exclude_ice_rings = false;
|
||||
size_t nthreads = 1;
|
||||
|
||||
HKLKeyGenerator generator;
|
||||
|
||||
std::map<uint64_t, MergeAccum> accumulator;
|
||||
std::unordered_map<uint64_t, MergeAccum> accumulator; // exported in key order
|
||||
|
||||
// Global error model (XDS form): sigma_corr^2 = a*sigma^2 + (b*<I>)^2. a rescales the
|
||||
// (under-estimated) counting variance; the (b*<I>)^2 term adds the intensity-
|
||||
@@ -315,12 +316,26 @@ class MergeOnTheFly {
|
||||
std::unordered_map<uint64_t, float> reject_median_I;
|
||||
size_t reject_count = 0;
|
||||
|
||||
bool Mask(const IntegrationOutcome &outcome);
|
||||
bool Mask(const IntegrationOutcome &outcome) const;
|
||||
|
||||
// One accepted observation, reduced to what the accumulator adds for it.
|
||||
struct Contribution {
|
||||
uint64_t key;
|
||||
int32_t h, k, l; // the reflection's own index, as the accumulator records it
|
||||
float w, wI, d;
|
||||
bool on_ice_ring;
|
||||
};
|
||||
// The observations of one image that reach the merge; returns how many the outlier test dropped.
|
||||
size_t Contributions(const IntegrationOutcome &outcome, std::vector<Contribution> &out) const;
|
||||
void Accumulate(const std::vector<Contribution> &contributions, int64_t image_id);
|
||||
public:
|
||||
MergeOnTheFly(const DiffractionExperiment &x);
|
||||
MergeOnTheFly& ReferenceCell(const std::optional<UnitCell> &cell);
|
||||
MergeOnTheFly& ExcludeIceRings(bool input) { exclude_ice_rings = input; return *this; }
|
||||
MergeOnTheFly& FilterByImageCC(bool input) { filter_by_image_cc = input; return *this; }
|
||||
// Workers for the passes over every observation (AddImages, RefineErrorModel, MergeStats); the
|
||||
// result does not depend on the number.
|
||||
MergeOnTheFly& Threads(size_t input) { nthreads = std::max<size_t>(1, input); return *this; }
|
||||
|
||||
// Fit the global error model from the spread of symmetry-equivalent observations.
|
||||
// Call once before merging; AddImage then applies it.
|
||||
@@ -341,6 +356,9 @@ public:
|
||||
// because it holds every observation at once; a streaming accumulator cannot, and there is no
|
||||
// second, device-side walk of the stills observations for this to disagree with.
|
||||
void AddImage(const IntegrationOutcome& outcome, int64_t image_id);
|
||||
// AddImage for every outcome, the image id being its index: the same merge, bit for bit, with the
|
||||
// per-observation work spread over the workers and the accumulation kept in image order.
|
||||
void AddImages(const std::vector<IntegrationOutcome> &outcomes);
|
||||
|
||||
// d_min_override, when set, is the effective high-resolution limit for the shell table (used for
|
||||
// the automatic resolution cutoff computed by the caller); otherwise the manual
|
||||
@@ -355,10 +373,17 @@ public:
|
||||
std::optional<ReportResolutionRange> declared_range = std::nullopt);
|
||||
|
||||
std::vector<MergedReflection> ExportReflections();
|
||||
// The merged intensities alone - no free-R flags, no French-Wilson amplitudes - which is all a
|
||||
// scaling reference reads.
|
||||
std::vector<MergedReflection> ExportIntensities();
|
||||
};
|
||||
|
||||
// The merged intensities of the images whose cell is close to reference_cell (all of them without one):
|
||||
// the reference a per-image scale is fitted against.
|
||||
std::vector<MergedReflection> MergeAll(const DiffractionExperiment &x,
|
||||
const std::vector<IntegrationOutcome> &reflections);
|
||||
const std::vector<IntegrationOutcome> &reflections,
|
||||
size_t nthreads,
|
||||
const std::optional<UnitCell> &reference_cell);
|
||||
|
||||
// Pearson CC between one image's corrected intensities (I * image_scale_corr) and a reference set of
|
||||
// full intensities, over the reflections that would enter the merge (non-ice, within the resolution
|
||||
|
||||
@@ -395,7 +395,7 @@ void RotationScaleMerge::IngestHost() {
|
||||
// Probed before the observations are built: whether the build lays down the full Obs array or
|
||||
// only the ingest arrays depends on whether the device pipeline will be active.
|
||||
gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
|
||||
gpu_active_ = gpu_->Available();
|
||||
gpu_active_ = gpu_->Available() && !scale_on_cpu_;
|
||||
resident_ingest = gpu_active_ && observation_dump_path.empty();
|
||||
#endif
|
||||
|
||||
@@ -690,6 +690,22 @@ void RotationScaleMerge::IngestOnDevice() {
|
||||
}
|
||||
#endif
|
||||
|
||||
bool RotationScaleMerge::OnGPU() const {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
return gpu_active_;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t RotationScaleMerge::GPUScalingPeakBytes() const {
|
||||
// The device scaling's peak, fitted to what nsys measured on four large data sets (33-123 M
|
||||
// observations, 4-16 M distinct hkl): within 5% on each. Unlike DeviceBytes, an estimate and not
|
||||
// an upper bound - it decides where the GPU clearly cannot take the scaling.
|
||||
constexpr size_t PEAK_BYTES_PER_OBS = 81, PEAK_BYTES_PER_HKL = 291;
|
||||
return PEAK_BYTES_PER_OBS * static_cast<size_t>(n_partials_obs) + PEAK_BYTES_PER_HKL * rawrun_start.size();
|
||||
}
|
||||
|
||||
size_t RotationScaleMerge::DeviceBytes() const {
|
||||
// What each costs at the peak, measured: an observation's arrays and its group CSR; a full's arrays
|
||||
// with the surface terms, or with the outlier test's and the error model's samples; an hkl's raw run
|
||||
@@ -734,6 +750,26 @@ void RotationScaleMerge::IngestDevice(const RotationScaleMerge *observations_fro
|
||||
gpu_->CopyObservationsFrom(*observations_from->gpu_);
|
||||
else if (resident_ingest)
|
||||
IngestOnDevice();
|
||||
// The scaling's footprint on the card is known once the ingest has counted the observations and the
|
||||
// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run
|
||||
// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the
|
||||
// card but not on the moment. The CPU path builds its observations on the host, so a resident
|
||||
// ingest is made again there.
|
||||
if (gpu_active_ && !observations_from) {
|
||||
const size_t card = get_gpu_total_memory(gpu_device);
|
||||
if (GPUScalingPeakBytes() > CPU_SCALING_ABOVE_CARD_FRACTION * card) {
|
||||
logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about "
|
||||
"{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead",
|
||||
n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9,
|
||||
100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9);
|
||||
scale_on_cpu_ = true;
|
||||
gpu_active_ = false;
|
||||
if (resident_ingest) {
|
||||
gpu_.reset();
|
||||
IngestHost();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (gpu_active_ && !resident_ingest && !observations_from) {
|
||||
const int n = static_cast<int>(partials.size());
|
||||
gpu_->SetPartialsLayout(n, n_frames, frame_start.data(), frame_count.data());
|
||||
|
||||
@@ -177,6 +177,10 @@ public:
|
||||
// engine's peak was at most 81% of this. Where the ingest on the card peaks higher, its peak.
|
||||
// Call after IngestDevice().
|
||||
[[nodiscard]] size_t DeviceBytes() const;
|
||||
// The device scaling's estimated peak (see the .cpp). Call after the ingest.
|
||||
[[nodiscard]] size_t GPUScalingPeakBytes() const;
|
||||
// Above this fraction of the card's memory the scaling runs on the CPU.
|
||||
static constexpr double CPU_SCALING_ABOVE_CARD_FRACTION = 0.8;
|
||||
|
||||
// Merge only the first n frames of the outcomes (by default all of them). Set before Ingest().
|
||||
void SetFrameLimit(int n) { frame_limit = n; }
|
||||
@@ -185,6 +189,8 @@ public:
|
||||
// the merge runs: on cards of one model the result is the same on any of them.
|
||||
void SetGpuDevice(int device) { gpu_device = device; }
|
||||
[[nodiscard]] int GetGpuDevice() const { return gpu_device; }
|
||||
// Whether this engine scales on the GPU (decided by the ingest).
|
||||
[[nodiscard]] bool OnGPU() const;
|
||||
|
||||
// Whether Run() also hands back the merge's fulls unmerged (Result::scaled_fulls). Off by default:
|
||||
// it is a copy as long as the fulls, wanted only by a caller that writes them.
|
||||
@@ -551,6 +557,8 @@ private:
|
||||
// the CPU loops as the bit-parity fallback. Built in Ingest.
|
||||
NotCopied<RotationScaleMergeGPU> gpu_;
|
||||
bool gpu_active_ = false;
|
||||
// The scaling does not fit the card (IngestDevice): this engine scales on the CPU.
|
||||
bool scale_on_cpu_ = false;
|
||||
// The fulls of this pass live on the device (the GPU combine ran), so the correction surfaces can
|
||||
// build their terms there. Set by RunArm.
|
||||
bool fulls_on_device_ = false;
|
||||
|
||||
@@ -1127,7 +1127,7 @@ namespace {
|
||||
"Scaling %lld partial observations needs more GPU memory than this card has: %.1f GB for "
|
||||
"their per-observation arrays alone (%zu bytes each), before the merge's other buffers, "
|
||||
"on a %.1f GB card. This data set is too large for GPU scaling on this card - run it on "
|
||||
"the CPU (CUDA_VISIBLE_DEVICES= rugnux ..., or a CPU-only build) on a machine with plenty of "
|
||||
"the CPU (rugnux --no-gpu ..., or a CPU-only build) on a machine with plenty of "
|
||||
"host memory: the CPU path keeps all of it in RAM, which for a set this large can be "
|
||||
"tens of GB",
|
||||
static_cast<long long>(n_obs), double(n_obs) * OBS_BYTES / 1e9, OBS_BYTES,
|
||||
@@ -1146,7 +1146,7 @@ namespace {
|
||||
"Scaling %lld partial observations ran out of GPU memory: a buffer of %.1f MB did not fit, "
|
||||
"with %.1f GB of the %.1f GB card free (the per-observation arrays take %.1f GB). Either "
|
||||
"another program is using this GPU, or this data set is too large for GPU scaling on this "
|
||||
"card - run it on the CPU (CUDA_VISIBLE_DEVICES= rugnux ..., or a CPU-only build) on a "
|
||||
"card - run it on the CPU (rugnux --no-gpu ..., or a CPU-only build) on a "
|
||||
"machine with plenty of host memory",
|
||||
static_cast<long long>(n_obs), double(bytes) / 1e6, double(free_bytes) / 1e9,
|
||||
double(total_bytes) / 1e9, double(n_obs) * OBS_BYTES / 1e9);
|
||||
@@ -1162,7 +1162,7 @@ namespace {
|
||||
snprintf(msg, sizeof(msg),
|
||||
"GPU busy: scaling %lld partial observations waited %lld minutes for GPU work running "
|
||||
"beside it in this process to give back its memory, and it did not. The data set fits "
|
||||
"this card; run it again, or on the CPU (CUDA_VISIBLE_DEVICES= rugnux ...)",
|
||||
"this card; run it again, or on the CPU (rugnux --no-gpu ...)",
|
||||
static_cast<long long>(n_obs),
|
||||
static_cast<long long>(std::chrono::duration_cast<std::chrono::minutes>(GPU_BUSY_TIMEOUT).count()));
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, msg);
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
|
||||
#include "StillsPartialityRefine.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cmath>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <ceres/ceres.h>
|
||||
#include <ceres/rotation.h>
|
||||
|
||||
#include "Merge.h"
|
||||
@@ -101,44 +102,114 @@ namespace {
|
||||
return G;
|
||||
}
|
||||
|
||||
// Ceres residual: refine the orientation tilt (psi_x, psi_y) holding the scale G fixed. The tilt
|
||||
// rotates the base reciprocal vector q; partiality follows analytically. Residual is the intensity
|
||||
// mismatch weighted by 1/sigma, exactly matching ScaleOnTheFly's intensity-space objective.
|
||||
struct PsiResidual {
|
||||
double qx, qy, qz;
|
||||
// The rotation by the angle-axis vector (psi_x, psi_y, 0) as a matrix (Rodrigues), so that one tilt
|
||||
// is applied to all the reflections of a crystal by a matrix product each.
|
||||
std::array<double, 9> TiltMatrix(double psi_x, double psi_y) {
|
||||
const double theta = std::sqrt(psi_x * psi_x + psi_y * psi_y);
|
||||
if (theta < 1e-15)
|
||||
return {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
|
||||
const double kx = psi_x / theta, ky = psi_y / theta;
|
||||
const double c = std::cos(theta), s = std::sin(theta), t = 1.0 - c;
|
||||
return {c + t * kx * kx, t * kx * ky, s * ky,
|
||||
t * kx * ky, c + t * ky * ky, -s * kx,
|
||||
-s * ky, s * kx, c};
|
||||
}
|
||||
|
||||
// ComputeP with the tilt given as its rotation matrix.
|
||||
struct TiltModel {
|
||||
double s0x, s0y, s0z, inv_lambda;
|
||||
double gamma0, gamma_e, bw;
|
||||
double G, inv_corr, Iref, Iobs, weight;
|
||||
|
||||
template<typename T>
|
||||
bool operator()(const T *const psi, T *residual) const {
|
||||
T q[3] = {T(qx), T(qy), T(qz)};
|
||||
T aa[3] = {psi[0], psi[1], T(0.0)};
|
||||
T qr[3];
|
||||
ceres::AngleAxisRotatePoint(aa, q, qr);
|
||||
const T Sx = qr[0] + T(s0x), Sy = qr[1] + T(s0y), Sz = qr[2] + T(s0z);
|
||||
const T de = ceres::sqrt(Sx * Sx + Sy * Sy + Sz * Sz) - T(inv_lambda);
|
||||
const T dstar = ceres::sqrt(qr[0] * qr[0] + qr[1] * qr[1] + qr[2] * qr[2]);
|
||||
const T sig_ang = T(gamma_e) * dstar;
|
||||
const T sbw = T(bw) * ceres::abs(qr[2]);
|
||||
const T sig2 = T(gamma0) * T(gamma0) + sig_ang * sig_ang + sbw * sbw;
|
||||
const T p = ceres::exp(T(-0.5) * de * de / sig2);
|
||||
residual[0] = T(weight) * (T(G) * p * T(inv_corr) * T(Iref) - T(Iobs));
|
||||
return true;
|
||||
double P(const std::array<double, 9> &R, const FitObs &o) const {
|
||||
const double qx = R[0] * o.qx + R[1] * o.qy + R[2] * o.qz;
|
||||
const double qy = R[3] * o.qx + R[4] * o.qy + R[5] * o.qz;
|
||||
const double qz = R[6] * o.qx + R[7] * o.qy + R[8] * o.qz;
|
||||
const double de = VecLen(qx + s0x, qy + s0y, qz + s0z) - inv_lambda;
|
||||
const double sig_ang = gamma_e * VecLen(qx, qy, qz);
|
||||
const double sbw = bw * std::fabs(qz);
|
||||
const double sig2 = gamma0 * gamma0 + sig_ang * sig_ang + sbw * sbw;
|
||||
if (!(sig2 > 0.0))
|
||||
return 1.0;
|
||||
return std::exp(-0.5 * de * de / sig2);
|
||||
}
|
||||
};
|
||||
|
||||
// Gaussian prior N(0, sigma_prior^2) on the tilt. The data residuals above are (model-obs)/sigma, a
|
||||
// proper chi^2, so the MAP prior residual is simply dpsi/sigma_prior - no scale calibration needed.
|
||||
struct PsiPrior {
|
||||
double inv_sigma;
|
||||
template<typename T>
|
||||
bool operator()(const T *const psi, T *residual) const {
|
||||
residual[0] = T(inv_sigma) * psi[0];
|
||||
residual[1] = T(inv_sigma) * psi[1];
|
||||
return true;
|
||||
// Refine the orientation tilt (psi_x, psi_y) holding the scale G fixed: minimise
|
||||
// sum_i rho(r_i^2) / 2 + |psi / sigma_prior|^2 / 2, r_i = weight_i (G p_i(psi) Iref_i / corr_i - Iobs_i),
|
||||
// with the Cauchy rho(s) = k^2 log(1 + s / k^2) - the intensity-space objective of ScaleOnTheFly, and
|
||||
// the Gaussian prior a proper MAP term since r_i is already in sigma units. Levenberg-Marquardt on the
|
||||
// normal equations reweighted by rho', the Jacobian by central differences, every trial tilt held inside
|
||||
// the bound. Returns false when the cost cannot be evaluated, leaving psi as it came in.
|
||||
bool RefineTilt(const std::vector<FitObs> &obs, const TiltModel &model, double G, double robust_k,
|
||||
double inv_sigma_prior, double max_tilt, double psi[2]) {
|
||||
constexpr int MAX_ITERATIONS = 25;
|
||||
constexpr double STEP = 1e-7; // rad, for the central differences
|
||||
const double k2 = robust_k * robust_k;
|
||||
const double prior2 = inv_sigma_prior * inv_sigma_prior;
|
||||
|
||||
auto cost = [&](const double p[2]) {
|
||||
const auto R = TiltMatrix(p[0], p[1]);
|
||||
double c = 0.5 * prior2 * (p[0] * p[0] + p[1] * p[1]);
|
||||
for (const auto &o: obs) {
|
||||
const double r = o.weight * (G * model.P(R, o) * o.inv_corr * o.Iref - o.Iobs);
|
||||
c += 0.5 * k2 * std::log1p(r * r / k2);
|
||||
}
|
||||
return c;
|
||||
};
|
||||
|
||||
double current = cost(psi);
|
||||
if (!std::isfinite(current))
|
||||
return false;
|
||||
double lambda = 1e-4;
|
||||
for (int iter = 0; iter < MAX_ITERATIONS; ++iter) {
|
||||
const auto R = TiltMatrix(psi[0], psi[1]);
|
||||
const auto Rxp = TiltMatrix(psi[0] + STEP, psi[1]), Rxm = TiltMatrix(psi[0] - STEP, psi[1]);
|
||||
const auto Ryp = TiltMatrix(psi[0], psi[1] + STEP), Rym = TiltMatrix(psi[0], psi[1] - STEP);
|
||||
double A00 = prior2, A01 = 0.0, A11 = prior2;
|
||||
double b0 = prior2 * psi[0], b1 = prior2 * psi[1];
|
||||
for (const auto &o: obs) {
|
||||
const double a = o.weight * G * o.inv_corr * o.Iref;
|
||||
const double r = a * model.P(R, o) - o.weight * o.Iobs;
|
||||
const double jx = a * (model.P(Rxp, o) - model.P(Rxm, o)) / (2.0 * STEP);
|
||||
const double jy = a * (model.P(Ryp, o) - model.P(Rym, o)) / (2.0 * STEP);
|
||||
const double w = 1.0 / (1.0 + r * r / k2); // rho'
|
||||
A00 += w * jx * jx;
|
||||
A01 += w * jx * jy;
|
||||
A11 += w * jy * jy;
|
||||
b0 += w * jx * r;
|
||||
b1 += w * jy * r;
|
||||
}
|
||||
|
||||
bool accepted = false;
|
||||
double trial[2] = {psi[0], psi[1]};
|
||||
double trial_cost = current;
|
||||
while (lambda < 1e10) {
|
||||
const double M00 = A00 * (1.0 + lambda), M11 = A11 * (1.0 + lambda);
|
||||
const double det = M00 * M11 - A01 * A01;
|
||||
if (!(det > 0.0))
|
||||
break;
|
||||
trial[0] = std::clamp(psi[0] - (M11 * b0 - A01 * b1) / det, -max_tilt, max_tilt);
|
||||
trial[1] = std::clamp(psi[1] - (M00 * b1 - A01 * b0) / det, -max_tilt, max_tilt);
|
||||
trial_cost = cost(trial);
|
||||
if (std::isfinite(trial_cost) && trial_cost < current) {
|
||||
accepted = true;
|
||||
lambda = std::max(lambda / 10.0, 1e-12);
|
||||
break;
|
||||
}
|
||||
lambda *= 10.0;
|
||||
}
|
||||
if (!accepted)
|
||||
break;
|
||||
const double step = std::hypot(trial[0] - psi[0], trial[1] - psi[1]);
|
||||
const double decrease = current - trial_cost;
|
||||
psi[0] = trial[0];
|
||||
psi[1] = trial[1];
|
||||
current = trial_cost;
|
||||
if (decrease <= 1e-6 * current || step <= 1e-10)
|
||||
break;
|
||||
}
|
||||
};
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
StillsPartialityRefine::StillsPartialityRefine(const DiffractionExperiment &x)
|
||||
@@ -238,6 +309,8 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
|
||||
double psi[2] = {0.0, 0.0};
|
||||
double G = 1.0;
|
||||
|
||||
const TiltModel model{.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda,
|
||||
.gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw};
|
||||
const bool refine_tilt = obs.size() >= settings_.min_reflections;
|
||||
const int inner = refine_tilt ? settings_.inner_iterations : 1;
|
||||
|
||||
@@ -258,42 +331,11 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
|
||||
if (!refine_tilt)
|
||||
break;
|
||||
|
||||
// (2) Refine the tilt holding G fixed.
|
||||
ceres::Problem problem;
|
||||
for (const auto &o: obs) {
|
||||
auto *cost = new ceres::AutoDiffCostFunction<PsiResidual, 1, 2>(new PsiResidual{
|
||||
.qx = o.qx, .qy = o.qy, .qz = o.qz,
|
||||
.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda,
|
||||
.gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw,
|
||||
.G = G, .inv_corr = o.inv_corr, .Iref = o.Iref, .Iobs = o.Iobs, .weight = o.weight});
|
||||
problem.AddResidualBlock(cost, new ceres::CauchyLoss(settings_.robust_k), psi);
|
||||
}
|
||||
if (settings_.prior_sigma_deg > 0.0) {
|
||||
const double inv_sigma = kRadToDeg / settings_.prior_sigma_deg; // 1 / sigma_prior (rad)
|
||||
problem.AddResidualBlock(new ceres::AutoDiffCostFunction<PsiPrior, 2, 2>(
|
||||
new PsiPrior{inv_sigma}), nullptr, psi);
|
||||
}
|
||||
problem.SetParameterLowerBound(psi, 0, -settings_.max_tilt_rad);
|
||||
problem.SetParameterUpperBound(psi, 0, settings_.max_tilt_rad);
|
||||
problem.SetParameterLowerBound(psi, 1, -settings_.max_tilt_rad);
|
||||
problem.SetParameterUpperBound(psi, 1, settings_.max_tilt_rad);
|
||||
|
||||
ceres::Solver::Options options;
|
||||
options.linear_solver_type = ceres::DENSE_QR;
|
||||
options.minimizer_progress_to_stdout = false;
|
||||
options.num_threads = 1;
|
||||
options.max_num_iterations = 25;
|
||||
ceres::Solver::Summary summary;
|
||||
const double psi_before[2] = {psi[0], psi[1]};
|
||||
ceres::Solve(options, &problem, &summary);
|
||||
// A solve that failed leaves whatever the minimiser last wrote in psi, and that tilt would go
|
||||
// straight onto every reflection's partiality below. Keep the tilt this crystal came in with
|
||||
// and stop refining it instead - G alone is still a usable model.
|
||||
if (!summary.IsSolutionUsable()) {
|
||||
psi[0] = psi_before[0];
|
||||
psi[1] = psi_before[1];
|
||||
// (2) Refine the tilt holding G fixed. A cost that cannot be evaluated keeps the tilt this crystal
|
||||
// came in with and stops refining it - G alone is still a usable model.
|
||||
const double inv_sigma_prior = settings_.prior_sigma_deg > 0.0 ? kRadToDeg / settings_.prior_sigma_deg : 0.0;
|
||||
if (!RefineTilt(obs, model, G, settings_.robust_k, inv_sigma_prior, settings_.max_tilt_rad, psi))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep what the crystal came in with. This refinement is on by default, so a crystal the tilt model
|
||||
@@ -349,7 +391,14 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
|
||||
return tilt_deg;
|
||||
}
|
||||
|
||||
double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, size_t nthreads) const {
|
||||
double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, size_t nthreads,
|
||||
const std::optional<UnitCell> &reference_cell) const {
|
||||
const auto &indexing = experiment_.GetIndexingSettings();
|
||||
auto merged_crystal = [&](const IntegrationOutcome &o) {
|
||||
return !reference_cell || o.latt.GetUnitCell().is_close(*reference_cell,
|
||||
indexing.GetUnitCellDistTolerance(),
|
||||
indexing.GetUnitCellAngleTolerance_deg());
|
||||
};
|
||||
if (nthreads == 0)
|
||||
nthreads = AvailableCpus();
|
||||
nthreads = std::max<size_t>(1, nthreads);
|
||||
@@ -358,7 +407,7 @@ double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, si
|
||||
|
||||
for (int outer = 0; outer < settings_.outer_iterations; ++outer) {
|
||||
// Reference full intensities from the current corrections.
|
||||
const std::vector<MergedReflection> merged = MergeAll(experiment_, outcomes);
|
||||
const std::vector<MergedReflection> merged = MergeAll(experiment_, outcomes, nthreads, reference_cell);
|
||||
std::map<HKLKey, double> reference;
|
||||
for (const auto &m: merged)
|
||||
reference[hkl_key_generator_(m)] = m.I;
|
||||
@@ -370,7 +419,7 @@ double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, si
|
||||
auto worker = [&]() {
|
||||
size_t i = next.fetch_add(1);
|
||||
while (i < outcomes.size()) {
|
||||
const double t = RefineOne(outcomes[i], reference);
|
||||
const double t = merged_crystal(outcomes[i]) ? RefineOne(outcomes[i], reference) : 0.0;
|
||||
if (t > 0.0) {
|
||||
double prev = tilt_sum.load();
|
||||
while (!tilt_sum.compare_exchange_weak(prev, prev + t)) {}
|
||||
|
||||
@@ -43,7 +43,10 @@ public:
|
||||
explicit StillsPartialityRefine(const DiffractionExperiment &x);
|
||||
|
||||
// Refine all crystals in place. Returns the mean |dpsi| applied (degrees), for diagnostics.
|
||||
double Run(std::vector<IntegrationOutcome> &outcomes, size_t nthreads = 0) const;
|
||||
// Only the crystals whose cell is close to reference_cell (all without one) are refined and make the
|
||||
// reference: the others are not merged.
|
||||
double Run(std::vector<IntegrationOutcome> &outcomes, size_t nthreads,
|
||||
const std::optional<UnitCell> &reference_cell) const;
|
||||
|
||||
private:
|
||||
const DiffractionExperiment experiment_;
|
||||
|
||||
@@ -25,7 +25,7 @@ namespace {
|
||||
const char *indexing_alg_flag(IndexingAlgorithmEnum a) {
|
||||
switch (a) {
|
||||
case IndexingAlgorithmEnum::FFBIDX: return "ffbidx";
|
||||
case IndexingAlgorithmEnum::FFT: return "fft";
|
||||
case IndexingAlgorithmEnum::FFT: return "cufft"; // the GPU implementation; rugnux's -X fft is the algorithm
|
||||
case IndexingAlgorithmEnum::FFTW: return "fftw";
|
||||
case IndexingAlgorithmEnum::None: return "none";
|
||||
case IndexingAlgorithmEnum::Auto:
|
||||
|
||||
@@ -368,9 +368,19 @@ void Rugnux::ImagePass(PipelineLocals &p) {
|
||||
// which is what a previous attempt at this got wrong.
|
||||
// Only the full-analysis worker drives a card; the azimuthal one preprocesses and integrates on
|
||||
// the CPU and wants every thread it can have, so the cap must not reach it.
|
||||
// A worker also holds about 24 bytes a detector pixel of its card - sixteen held 6.3 GB on a 16.8 Mpx
|
||||
// detector - and the workers keep to half of the card, so a small card gets fewer of them. How many
|
||||
// there are changes only how fast the loop runs, never what it computes.
|
||||
const int gpus = get_gpu_count();
|
||||
int workers_per_card = 16;
|
||||
if (gpus > 0) {
|
||||
constexpr int64_t WORKER_BYTES_PER_PIXEL = 24;
|
||||
const auto fit = static_cast<int64_t>(get_gpu_total_memory(0) / 2)
|
||||
/ (WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum());
|
||||
workers_per_card = static_cast<int>(std::clamp<int64_t>(fit, 1, 16));
|
||||
}
|
||||
const int image_workers = (per_image_analysis && config_.nthreads_auto && gpus > 0)
|
||||
? std::min(config_.nthreads, std::max(16, 16 * gpus))
|
||||
? std::min(config_.nthreads, workers_per_card * gpus)
|
||||
: config_.nthreads;
|
||||
if (image_workers < config_.nthreads)
|
||||
logger.Info("Per-image loop: {} of {} threads ({} GPUs) - past what a card can take queue "
|
||||
|
||||
@@ -1161,7 +1161,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
// is the data sets up to about 50 M observations at high multiplicity; the large ones merge
|
||||
// their arms one after the other.
|
||||
if (!geometry_prepass && !experiment_.GetGemmiSpaceGroup().has_value() && rsm->GetSearchMinZeta() > 0.0
|
||||
&& cards > 0) {
|
||||
&& cards > 0 && rsm->OnGPU()) {
|
||||
const int32_t device = std::min(cards - 1, 2);
|
||||
constexpr size_t CARD_BYTES_RESERVED = size_t(3) << 30;
|
||||
if (cards >= 3
|
||||
@@ -1250,7 +1250,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
|
||||
{
|
||||
phase("Scaling images (" + label + ")");
|
||||
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome());
|
||||
TimingMark scale_mark(logger, "stills scaling images " + label);
|
||||
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads,
|
||||
result.consensus_cell);
|
||||
indexer->ScaleAllImages(merge_result);
|
||||
}
|
||||
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
|
||||
@@ -1258,8 +1260,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
// correction (no re-integration). Never for the P1 search pass.
|
||||
if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) {
|
||||
phase("Partiality post-refine (" + label + ")");
|
||||
TimingMark refine_mark(logger, "stills partiality post-refine " + label);
|
||||
StillsPartialityRefine refiner(experiment_);
|
||||
const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads);
|
||||
const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads, result.consensus_cell);
|
||||
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
|
||||
// The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally
|
||||
// when the image has no prior CC - which is exactly the state the guard above leaves. So
|
||||
@@ -1272,7 +1275,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
|
||||
|
||||
phase("Merging");
|
||||
TimingMark merge_mark(logger, "stills error model and merge " + label);
|
||||
MergeOnTheFly merge_engine(experiment_);
|
||||
merge_engine.Threads(config_.nthreads);
|
||||
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
|
||||
// intensities and the error model, so the space-group search sees clean data; the final
|
||||
// in-symmetry merge keeps them so completeness is not lost.
|
||||
@@ -1296,11 +1301,11 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
|
||||
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
|
||||
}
|
||||
for (size_t i = 0; i < merge_input.size(); ++i)
|
||||
merge_engine.AddImage(merge_input[i], static_cast<int64_t>(i));
|
||||
merge_engine.AddImages(merge_input);
|
||||
|
||||
ScaleMergeResult out;
|
||||
out.merged = merge_engine.ExportReflections();
|
||||
merge_mark.Stop();
|
||||
|
||||
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
|
||||
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
|
||||
@@ -1313,6 +1318,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A);
|
||||
|
||||
phase("Computing statistics");
|
||||
TimingMark stats_mark(logger, "stills statistics " + label);
|
||||
out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min);
|
||||
// The same kept reflections binned over the reference range (--report-resolution); the
|
||||
// stills error model is fitted over the whole range already, so no ISa of its own.
|
||||
|
||||
+38
-12
@@ -118,6 +118,8 @@ void print_usage() {
|
||||
"(debug level); an empty prefix writes nothing at all" << std::endl;
|
||||
std::cout << " -N, --threads <num> Number of threads (default: every CPU the run may use - its affinity and cgroup quota - of which the "
|
||||
"per-image loop uses at most 16 per GPU; an explicit value is used as given)" << std::endl;
|
||||
std::cout << " --no-gpu Run on the CPU only, as on a machine without a GPU: CUDA is not "
|
||||
"touched and every GPU step takes its CPU path (-X Auto and FFT index with FFTW)" << std::endl;
|
||||
std::cout << " -s, --start-image <num> Start image number (default: 0)" << std::endl;
|
||||
std::cout << " -e, --end-image <num> End image number (default: all)" << std::endl;
|
||||
std::cout << " -t, --stride <num> Image stride (default: 1)" << std::endl;
|
||||
@@ -167,7 +169,9 @@ void print_usage() {
|
||||
std::cout << " --rotation-scale <k> Goniometer rotation scale: the stage turned k times the angle stored in the file (the commanded one). Applied to both passes; overrides the fitted correction" << std::endl;
|
||||
std::cout << " --rotation-no-postrefine Disable the (default-on) two-pass rotation post-refine (post-refine detector distance/beam + cell/axis, then re-integrate; only the refined pass is written, as the canonical <prefix>_* output)" << std::endl;
|
||||
std::cout << " --prepass-fraction <f> Integrate the first fraction f of the sweep, but at least 120 deg of it, in the two-pass rotation geometry pre-pass (0.05-1, default 0.5; 1 = the whole sweep); pass 2 reads every frame. The pre-scan and the first-pass indexing sample the same part of the sweep" << std::endl;
|
||||
std::cout << " -X, --indexing-algorithm <txt> Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl;
|
||||
std::cout << " -X, --indexing-algorithm <txt> Indexing algorithm (Auto|FFT|cuFFT|FFTW|FFBIDX|None, default Auto). FFT is "
|
||||
"the de-novo FFT search, on the GPU (cuFFT) where there is one and --no-gpu is not given, otherwise on the "
|
||||
"CPU (FFTW); cuFFT and FFTW name one implementation each. cuFFT and FFBIDX (known cell) need a GPU" << std::endl;
|
||||
std::cout << " -S, --space-group <num|symbol> Space group number (96) or symbol (P43212) - for indexing and scaling" << std::endl;
|
||||
std::cout << " -C, --unit-cell <cell> Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl;
|
||||
std::cout << " --fft-min-unit-cell <num> Shortest cell axis the FFT search accepts, in A (default: 10). A candidate with a shorter axis is discarded, but a de-novo rotation run also tries a second first-pass hypothesis with the floor at 5 A, so a small-molecule cell is indexed without this flag; -C lowers it on its own to cover the cell given" << std::endl;
|
||||
@@ -318,6 +322,7 @@ enum {
|
||||
OPT_FINALIST_LEDGER,
|
||||
OPT_SIMULATE_LIVE,
|
||||
OPT_DEVELOPER,
|
||||
OPT_NO_GPU,
|
||||
OPT_FORCE_STILL,
|
||||
OPT_AZIM_MIN_Q,
|
||||
OPT_AZIM_MAX_Q,
|
||||
@@ -375,6 +380,7 @@ static option long_options[] = {
|
||||
{"finalist-ledger", no_argument, nullptr, OPT_FINALIST_LEDGER},
|
||||
{"simulate-live", required_argument, nullptr, OPT_SIMULATE_LIVE},
|
||||
{"developer", no_argument, nullptr, OPT_DEVELOPER},
|
||||
{"no-gpu", no_argument, nullptr, OPT_NO_GPU},
|
||||
{"smooth-g", optional_argument, nullptr, OPT_SMOOTH_G},
|
||||
{"relative-b", optional_argument, nullptr, OPT_RELATIVE_B},
|
||||
{"no-scaling-corrections", no_argument, nullptr, OPT_NO_SCALING_CORRECTIONS},
|
||||
@@ -852,7 +858,7 @@ extern "C" void fftwf_assertion_failed(const char *s, int line, const char *file
|
||||
// cannot reject on its own (a resolution limit of zero, a distance of zero, a polarisation above one,
|
||||
// an unreadable file), so main wraps this and reports rather than letting the exception terminate the
|
||||
// process with no diagnostic and exit code 134.
|
||||
static int RunRugnux(int argc, char **argv) {
|
||||
static int RunRugnux(int argc, char **argv, bool no_gpu) {
|
||||
// The whole invocation, reported at the end and written into the results report. Started before
|
||||
// anything else so it covers reading the file and setting up as well as the processing.
|
||||
const auto invocation_start = std::chrono::steady_clock::now();
|
||||
@@ -898,8 +904,8 @@ static int RunRugnux(int argc, char **argv) {
|
||||
: fmt::format(" (git {} {})", jfjoch_git_sha1().substr(0, 6), jfjoch_git_date()), build);
|
||||
logger.Info("Build flags: {}", jfjoch_build_cxx_flags().empty() ? "none" : jfjoch_build_cxx_flags());
|
||||
logger.Info("Command line: {}", provenance.command_line);
|
||||
logger.Info("GPU: {}", provenance.gpu_count > 0
|
||||
? fmt::format("{} ({} visible)", provenance.gpu_description, provenance.gpu_count)
|
||||
logger.Info("GPU: {}", no_gpu ? std::string("not used (--no-gpu) - running on the CPU")
|
||||
: provenance.gpu_count > 0 ? fmt::format("{} ({} visible)", provenance.gpu_description, provenance.gpu_count)
|
||||
: std::string("none visible - running on the CPU"));
|
||||
RegisterHDF5Filter();
|
||||
|
||||
@@ -1129,7 +1135,9 @@ static int RunRugnux(int argc, char **argv) {
|
||||
|
||||
if (alg == "ffbidx")
|
||||
indexing_algorithm = IndexingAlgorithmEnum::FFBIDX;
|
||||
else if (alg == "fft")
|
||||
else if (alg == "fft") // the algorithm: on the GPU where there is one, else with FFTW
|
||||
indexing_algorithm = get_gpu_count() > 0 ? IndexingAlgorithmEnum::FFT : IndexingAlgorithmEnum::FFTW;
|
||||
else if (alg == "cufft")
|
||||
indexing_algorithm = IndexingAlgorithmEnum::FFT;
|
||||
else if (alg == "fftw")
|
||||
indexing_algorithm = IndexingAlgorithmEnum::FFTW;
|
||||
@@ -1203,7 +1211,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
dump_observations = optarg;
|
||||
break;
|
||||
case 'F':
|
||||
indexing_algorithm = IndexingAlgorithmEnum::FFT;
|
||||
indexing_algorithm = get_gpu_count() > 0 ? IndexingAlgorithmEnum::FFT : IndexingAlgorithmEnum::FFTW;
|
||||
break;
|
||||
case 'A':
|
||||
anomalous_mode = true;
|
||||
@@ -1361,6 +1369,8 @@ static int RunRugnux(int argc, char **argv) {
|
||||
case OPT_DEVELOPER:
|
||||
provenance.developer = true;
|
||||
break;
|
||||
case OPT_NO_GPU: // acted on in main, before anything asks CUDA for the devices
|
||||
break;
|
||||
case OPT_SMOOTH_G:
|
||||
smooth_g_deg_arg = optarg ? parse_double_arg(optarg, "--smooth-g", logger) : SMOOTH_G_DEFAULT_DEG;
|
||||
break;
|
||||
@@ -1588,6 +1598,14 @@ static int RunRugnux(int argc, char **argv) {
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if ((indexing_algorithm == IndexingAlgorithmEnum::FFT || indexing_algorithm == IndexingAlgorithmEnum::FFBIDX)
|
||||
&& get_gpu_count() == 0) {
|
||||
logger.Error("-X {} runs on the GPU only, and {}; -X FFT or Auto index on the CPU",
|
||||
indexing_algorithm == IndexingAlgorithmEnum::FFT ? "cuFFT" : "FFBIDX",
|
||||
no_gpu ? "--no-gpu is given" : "no GPU is visible");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
input_file = argv[optind];
|
||||
const std::string broker_url = (optind == argc - 2) ? argv[optind + 1] : "";
|
||||
|
||||
@@ -1639,7 +1657,8 @@ static int RunRugnux(int argc, char **argv) {
|
||||
// Said before the run rather than after it: the GPUs are what makes rugnux fast, and a machine
|
||||
// that turns out to have none - a driver mismatch, a CUDA_VISIBLE_DEVICES left over from another
|
||||
// job - is worth knowing about while there is still time to stop and fix it.
|
||||
console->Field("Compute", provenance.gpu_count > 0
|
||||
console->Field("Compute", no_gpu ? fmt::format("CPU only (--no-gpu), {} threads", nthreads)
|
||||
: provenance.gpu_count > 0
|
||||
? fmt::format("{}{}, {} threads", provenance.gpu_count > 1 ? fmt::format("{} x ", provenance.gpu_count) : "",
|
||||
provenance.gpu_description, nthreads)
|
||||
: fmt::format("no GPU visible - {} CPU threads, which is much slower", nthreads));
|
||||
@@ -2103,16 +2122,17 @@ static int RunRugnux(int argc, char **argv) {
|
||||
// against a foreign dataset injects cross-dataset systematics and is a worse reference than
|
||||
// the data's own merge. The per-image scale G is the exact one-pass solution, so one pass
|
||||
// (iterating a self-rebuilt reference only re-fits the freshly-scaled noise on weak stills).
|
||||
ScaleOnTheFly(experiment, MergeAll(experiment, reflections)).Scale(reflections, nthreads);
|
||||
ScaleOnTheFly(experiment, MergeAll(experiment, reflections, nthreads, experiment.GetUnitCell())).Scale(reflections, nthreads);
|
||||
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
|
||||
// orientation tilt against the merge and recompute each reflection's partiality + scale correction
|
||||
// (no re-integration), then merge with the improved corrections.
|
||||
if (experiment.GetScalingSettings().GetStillsPartialityRefine()) {
|
||||
StillsPartialityRefine refiner(experiment);
|
||||
const double mean_tilt = refiner.Run(reflections, nthreads);
|
||||
const double mean_tilt = refiner.Run(reflections, nthreads, experiment.GetUnitCell());
|
||||
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
|
||||
}
|
||||
MergeOnTheFly merge_engine(experiment);
|
||||
merge_engine.Threads(nthreads);
|
||||
merge_engine.ReferenceCell(experiment.GetUnitCell());
|
||||
// --min-image-cc has to hold for the merge itself, not only for the reported statistics.
|
||||
merge_engine.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
|
||||
@@ -2128,8 +2148,7 @@ static int RunRugnux(int argc, char **argv) {
|
||||
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
|
||||
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
|
||||
}
|
||||
for (size_t i = 0; i < reflections.size(); ++i)
|
||||
merge_engine.AddImage(reflections[i], static_cast<int64_t>(i));
|
||||
merge_engine.AddImages(reflections);
|
||||
merged_reflections = merge_engine.ExportReflections();
|
||||
|
||||
// Automatic high-resolution cutoff (post-merge), matching the full-analysis path: a manual
|
||||
@@ -3297,9 +3316,16 @@ int main(int argc, char **argv) {
|
||||
// Before anything creates a CUDA context, which fixes the flags for the device. A thread
|
||||
// waiting on the GPU then sleeps instead of spinning on a core - measured on a 16M rotation
|
||||
// run, a fifth of all CPU time went to that spinning, with no change in wall time either way.
|
||||
// --no-gpu is looked for here, ahead of the parser, because that already asks CUDA for the devices.
|
||||
bool no_gpu = false;
|
||||
for (int i = 1; i < argc; i++)
|
||||
if (std::strcmp(argv[i], "--no-gpu") == 0)
|
||||
no_gpu = true;
|
||||
if (no_gpu)
|
||||
hide_gpus();
|
||||
set_gpu_blocking_sync();
|
||||
enable_gpu_numa_binding();
|
||||
const int ret = RunRugnux(argc, argv);
|
||||
const int ret = RunRugnux(argc, argv, no_gpu);
|
||||
// Everything that used a card has been destroyed with the run.
|
||||
release_gpu_contexts();
|
||||
return ret;
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include "../image_analysis/bragg_integration/CalcISigma.h"
|
||||
#include "../image_analysis/scale_merge/ErrorModel.h"
|
||||
#include "../image_analysis/scale_merge/HKLKey.h"
|
||||
#include "../image_analysis/scale_merge/Merge.h"
|
||||
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
||||
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
|
||||
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
|
||||
#include "gemmi/reciproc.hpp"
|
||||
|
||||
TEST_CASE("HKLKey_NoSG_noMergeFriedel") {
|
||||
@@ -585,3 +588,161 @@ TEST_CASE("Global Wilson B over a widened window, with no upper cap", "[wilson]"
|
||||
CHECK(1.0 / (3.9 * 3.9) - 1.0 / (low_res.d_low_A * low_res.d_low_A)
|
||||
== Catch::Approx(1.0 / 9.0 - 1.0 / 16.0).epsilon(0.01));
|
||||
}
|
||||
|
||||
// The parallel AddImages is the serial AddImage loop, bit for bit: the per-observation work is spread
|
||||
// over the workers, but every sum and every half-set choice is still made in image order. More images
|
||||
// than one of its blocks, so the block boundary is crossed.
|
||||
TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") {
|
||||
DiffractionExperiment x;
|
||||
x.SetSpaceGroup(TestSpaceGroup());
|
||||
|
||||
std::mt19937 rng(7);
|
||||
std::uniform_int_distribution<int> index(-6, 6);
|
||||
std::uniform_real_distribution<float> unit(0.0f, 1.0f);
|
||||
std::vector<IntegrationOutcome> outcomes(5000);
|
||||
for (auto &o: outcomes) {
|
||||
for (int j = 0; j < 20; ++j) {
|
||||
Reflection r{};
|
||||
r.h = index(rng);
|
||||
r.k = index(rng);
|
||||
r.l = index(rng);
|
||||
if (r.h == 0 && r.k == 0 && r.l == 0)
|
||||
continue;
|
||||
r.d = 20.0f / std::sqrt(static_cast<float>(r.h * r.h + r.k * r.k + r.l * r.l));
|
||||
r.I = 100.0f * unit(rng) - 5.0f;
|
||||
r.sigma = 1.0f + 5.0f * unit(rng);
|
||||
r.var_bkg = r.sigma * r.sigma * unit(rng);
|
||||
r.prescaling_corr = 1.0f;
|
||||
r.partiality = 0.5f + 0.5f * unit(rng);
|
||||
r.image_scale_corr = 0.5f + unit(rng);
|
||||
r.on_ice_ring = unit(rng) < 0.05f;
|
||||
o.reflections.push_back(r);
|
||||
}
|
||||
}
|
||||
|
||||
MergeOnTheFly serial(x);
|
||||
serial.RefineErrorModel(outcomes);
|
||||
for (size_t i = 0; i < outcomes.size(); ++i)
|
||||
serial.AddImage(outcomes[i], static_cast<int64_t>(i));
|
||||
const auto serial_merged = serial.ExportReflections();
|
||||
|
||||
MergeOnTheFly parallel(x);
|
||||
parallel.Threads(8);
|
||||
parallel.RefineErrorModel(outcomes);
|
||||
parallel.AddImages(outcomes);
|
||||
const auto parallel_merged = parallel.ExportReflections();
|
||||
|
||||
REQUIRE(serial.ErrorModelActive());
|
||||
CHECK(serial.ErrorModelA() == parallel.ErrorModelA());
|
||||
CHECK(serial.ErrorModelB() == parallel.ErrorModelB());
|
||||
REQUIRE(serial_merged.size() == parallel_merged.size());
|
||||
for (size_t i = 0; i < serial_merged.size(); ++i) {
|
||||
CHECK(serial_merged[i].h == parallel_merged[i].h);
|
||||
CHECK(serial_merged[i].k == parallel_merged[i].k);
|
||||
CHECK(serial_merged[i].l == parallel_merged[i].l);
|
||||
CHECK(serial_merged[i].I == parallel_merged[i].I);
|
||||
CHECK(serial_merged[i].sigma == parallel_merged[i].sigma);
|
||||
for (int half = 0; half < 2; ++half) // NaN where a half is empty, on both sides
|
||||
CHECK(std::memcmp(&serial_merged[i].I_half[half], ¶llel_merged[i].I_half[half], sizeof(float)) == 0);
|
||||
}
|
||||
|
||||
const auto serial_stats = serial.MergeStats(serial_merged, outcomes);
|
||||
const auto parallel_stats = parallel.MergeStats(parallel_merged, outcomes);
|
||||
CHECK(serial_stats.overall.total_observations == parallel_stats.overall.total_observations);
|
||||
CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half);
|
||||
CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas);
|
||||
}
|
||||
|
||||
// Stills partiality post-refinement on noiseless synthetic crystals: each crystal's true orientation is
|
||||
// tilted from the stored one by up to 0.05 deg, its intensities are the true partials of a fixed set of
|
||||
// full intensities, and its stored partialities those of the stored orientation. The refinement has to
|
||||
// move the partialities towards the true ones. (The Ceres solve this replaced gave the same numbers: error
|
||||
// 0.0559 -> 0.0302, mean tilt 0.0206 deg.)
|
||||
TEST_CASE("StillsPartialityRefine_RecoversTheTilt") {
|
||||
constexpr float CELL = 50.0f, WAVELENGTH = 1.0f, D_MIN = 2.0f;
|
||||
constexpr double GAMMA0 = 0.001, ACCEPT = 1.7 * GAMMA0; // RLP width; |dist_ewald| accepted at prediction
|
||||
DiffractionExperiment x;
|
||||
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
|
||||
DiffractionGeometry geom;
|
||||
geom.Wavelength_A(WAVELENGTH);
|
||||
const Coord S0 = geom.GetScatteringVector();
|
||||
|
||||
std::mt19937 rng(11);
|
||||
std::normal_distribution<double> gauss(0.0, 1.0);
|
||||
std::uniform_real_distribution<double> unit(0.0, 1.0);
|
||||
auto full_I = [](int h, int k, int l) { // a fixed, Friedel-symmetric pseudo-random full intensity
|
||||
const uint64_t key = HKLKeyGenerator(true, gemmi::get_spacegroup_by_name("P 1"))(h, k, l).pack();
|
||||
return 10.0 + 990.0 * static_cast<double>((key * 0x9e3779b97f4a7c15ULL) >> 40) / static_cast<double>(1ULL << 24);
|
||||
};
|
||||
auto rotate = [](const double R[9], const Coord &v) {
|
||||
return Coord(static_cast<float>(R[0] * v.x + R[1] * v.y + R[2] * v.z),
|
||||
static_cast<float>(R[3] * v.x + R[4] * v.y + R[5] * v.z),
|
||||
static_cast<float>(R[6] * v.x + R[7] * v.y + R[8] * v.z));
|
||||
};
|
||||
auto rotation = [](double ax, double ay, double az, double angle, double R[9]) { // Rodrigues
|
||||
const double n = std::sqrt(ax * ax + ay * ay + az * az);
|
||||
ax /= n; ay /= n; az /= n;
|
||||
const double c = std::cos(angle), s = std::sin(angle), t = 1.0 - c;
|
||||
const double M[9] = {c + t * ax * ax, t * ax * ay - s * az, t * ax * az + s * ay,
|
||||
t * ax * ay + s * az, c + t * ay * ay, t * ay * az - s * ax,
|
||||
t * ax * az - s * ay, t * ay * az + s * ax, c + t * az * az};
|
||||
std::copy(M, M + 9, R);
|
||||
};
|
||||
auto partiality = [&](const Coord &q) {
|
||||
const double de = (q + S0).Length() - 1.0 / WAVELENGTH;
|
||||
return std::exp(-0.5 * de * de / (GAMMA0 * GAMMA0));
|
||||
};
|
||||
|
||||
std::vector<IntegrationOutcome> outcomes(150);
|
||||
std::vector<std::vector<double>> true_p(outcomes.size());
|
||||
const int hmax = static_cast<int>(CELL / D_MIN);
|
||||
for (size_t c = 0; c < outcomes.size(); ++c) {
|
||||
double R[9], T[9];
|
||||
rotation(gauss(rng), gauss(rng), gauss(rng), 2.0 * M_PI * unit(rng), R);
|
||||
const double tilt = (0.02 + 0.03 * unit(rng)) * M_PI / 180.0, phi = 2.0 * M_PI * unit(rng);
|
||||
rotation(std::cos(phi), std::sin(phi), 0.0, tilt, T); // about an axis normal to the beam
|
||||
auto &o = outcomes[c];
|
||||
o.geom = geom;
|
||||
o.latt = CrystalLattice(rotate(R, Coord(CELL, 0, 0)), rotate(R, Coord(0, CELL, 0)),
|
||||
rotate(R, Coord(0, 0, CELL)));
|
||||
const double G = 0.5 + unit(rng);
|
||||
for (int h = -hmax; h <= hmax; ++h)
|
||||
for (int k = -hmax; k <= hmax; ++k)
|
||||
for (int l = -hmax; l <= hmax; ++l) {
|
||||
const Coord q = o.latt.Astar() * h + o.latt.Bstar() * k + o.latt.Cstar() * l;
|
||||
const float dstar = q.Length();
|
||||
if (dstar == 0.0f || dstar > 1.0f / D_MIN)
|
||||
continue;
|
||||
if (std::fabs((q + S0).Length() - 1.0 / WAVELENGTH) > ACCEPT)
|
||||
continue;
|
||||
const double p_true = partiality(rotate(T, q));
|
||||
Reflection r{};
|
||||
r.h = h; r.k = k; r.l = l;
|
||||
r.d = 1.0f / dstar;
|
||||
r.I = static_cast<float>(G * p_true * full_I(h, k, l));
|
||||
r.sigma = static_cast<float>(1.0 + 0.01 * r.I);
|
||||
r.prescaling_corr = 1.0f;
|
||||
r.partiality = static_cast<float>(partiality(q));
|
||||
r.image_scale_corr = 1.0f / r.partiality;
|
||||
o.reflections.push_back(r);
|
||||
true_p[c].push_back(p_true);
|
||||
}
|
||||
REQUIRE(o.reflections.size() >= 40);
|
||||
}
|
||||
|
||||
auto mean_abs_error = [&]() {
|
||||
double sum = 0.0;
|
||||
size_t n = 0;
|
||||
for (size_t c = 0; c < outcomes.size(); ++c)
|
||||
for (size_t i = 0; i < outcomes[c].reflections.size(); ++i, ++n)
|
||||
sum += std::fabs(outcomes[c].reflections[i].partiality - true_p[c][i]);
|
||||
return sum / static_cast<double>(n);
|
||||
};
|
||||
const double before = mean_abs_error();
|
||||
ScaleOnTheFly(x, MergeAll(x, outcomes, 4, std::nullopt)).Scale(outcomes, 4);
|
||||
const double mean_tilt = StillsPartialityRefine(x).Run(outcomes, 4, std::nullopt);
|
||||
const double after = mean_abs_error();
|
||||
CHECK(mean_tilt > 0.015);
|
||||
CHECK(mean_tilt < 0.06);
|
||||
CHECK(after < 0.6 * before); // measured 0.54: the prior holds the larger tilts back
|
||||
}
|
||||
@@ -263,7 +263,7 @@ TEST_CASE("RugnuxCommandLine_Full", "[process]") {
|
||||
CHECK(cmd.find("-N 8") != std::string::npos);
|
||||
CHECK(cmd.find("-e 500") != std::string::npos);
|
||||
CHECK(cmd.find("-o run1") != std::string::npos);
|
||||
CHECK(cmd.find("-X fft") != std::string::npos);
|
||||
CHECK(cmd.find("-X cufft") != std::string::npos);
|
||||
CHECK(cmd.find("-S 96") != std::string::npos);
|
||||
// -R takes an optional argument, so its value must be attached (-R30); a separate "-R 30" token
|
||||
// would not re-parse (getopt would leave 30 as a positional and drop the count).
|
||||
|
||||
Reference in new issue
Block a user