diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8ec48905..014a35e6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,6 +27,10 @@ This is an UNSTABLE release. It includes many experimental features, as well as * `images_per_file` is chosen from the acquisition when it is not given: a rotation sweep of at most 20000 images goes into a single data file, a grid scan splits on whole fast-axis rows, and stills and serial keep 1000. * rugnux: observations outside the scaling resolution range are dropped as they are ingested, rather than scaled, combined and error-modelled first; merged results are unchanged. * rugnux: two runs of the same command on the same images now produce the same intensities - the GPU prediction, profile-learning and profile-fit steps no longer depend on the order their blocks happen to run in. +* rugnux: the offline lattice refinement is bounded by iterations rather than by a wall clock, so a loaded machine can no longer refine to a different lattice; a live acquisition keeps its real-time bound. +* rugnux: a rotation run is substantially faster - a 24-crystal set that took 7m55s takes 6m28s, with the space group unchanged on every one of them. Most of it is work that was being done and thrown away: the geometry pre-pass no longer merges, corrects, and writes a result the second pass replaces, observations outside the scaling resolution range are never built, and the detector lookup tables are built once for the run rather than once per worker. +* rugnux: the geometry pre-pass no longer writes `_01.mtz`, `_01.cif`, `_01.hkl` or its scaling table. The second pass rewrites all of it at the refined geometry seconds later, and that is the result to use. +* rugnux: with `-N` left at its default the per-image loop uses at most eight workers per GPU. Past that, workers spend their time queueing kernels rather than running them - measured, a 16 Mpx set is slower at 48 workers than at 8. An explicit `-N` is obeyed as given. * rugnux: the detector-frame modulation correction is fitted on a grid spanning the detector rather than on the reflections that happen to be present, so whether it is applied no longer depends on how far integration reached. * rugnux: the first-pass rotation indexing finds its spots on every worker rather than one, which is worth most on large detectors. The lattice it picks is unchanged. * rugnux: beam-stop detection is substantially faster - the pre-scan reads and accumulates its frames on several threads, the mask is built in one pass over the image rather than by repeated searches, and a worker's accumulator is allocated only if it is used. The shadow it finds is unchanged. diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index a8465297..f11dc86c 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1826,10 +1826,24 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b std::function worker = per_image_analysis ? std::function(full_worker) : std::function(azint_worker); + // How many workers the loop actually wants. Every one of them submits its own kernels to a card, + // and a card runs out of room to accept them long before it runs out of work: measured on two + // GPUs, the loop's own time falls to sixteen workers and then rises again, so forty-eight is + // slower than eight on a 16 Mpx set. The cap is per card because that is what the queue depth + // belongs to, and it only applies when -N was left at its default - an explicit -N is a + // deliberate instruction and is obeyed, which is what a previous attempt at this got wrong. + const int gpus = get_gpu_count(); + const int image_workers = (config_.nthreads_auto && gpus > 0) + ? std::min(config_.nthreads, std::max(8, 8 * gpus)) + : config_.nthreads; + if (image_workers < config_.nthreads) + logger.Info("Per-image loop: {} of {} threads ({} GPUs) - more workers than a card can take " + "queue work from make it slower, not faster; pass -N to override", + image_workers, config_.nthreads, gpus); std::vector > futures; - futures.reserve(config_.nthreads); + futures.reserve(image_workers); const auto image_loop_start = std::chrono::steady_clock::now(); - for (int i = 0; i < config_.nthreads; ++i) + for (int i = 0; i < image_workers; ++i) futures.push_back(std::async(std::launch::async, worker)); for (auto &f: futures) f.get(); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index bf7fd54c..deb9b07a 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -44,6 +44,9 @@ struct ProcessConfig { int end_image = -1; // -1 => to the end of the dataset int stride = 1; int nthreads = 1; + // True when -N was left at its default and nthreads above is just the machine's thread count. + // The per-image loop is capped in that case (see RunPipeline); an explicit -N never is. + bool nthreads_auto = false; // Output prefix for the _process.h5 (and scaled reflections). Empty => process without writing. std::string output_prefix; diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index fae3e0a6..4a0032a8 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1187,6 +1187,7 @@ static int RunRugnux(int argc, char **argv) { // behaves the same. The scale/merge engines expand 0 on their own, but the per-image processing // loop (Rugnux) spawns exactly nthreads workers, so passing 0 there would spawn none and process // nothing - hence resolving it centrally rather than relying on each consumer. + const bool nthreads_auto = nthreads <= 0; if (nthreads <= 0) { unsigned int hw = std::thread::hardware_concurrency(); nthreads = hw > 0 ? static_cast(hw) : 1; @@ -1642,6 +1643,7 @@ static int RunRugnux(int argc, char **argv) { config.end_image = end_image; config.stride = image_stride; config.nthreads = nthreads; + config.nthreads_auto = nthreads_auto; config.output_prefix = output_prefix; config.detect_beam_stop = detect_beam_stop; config.estimate_beam_center = estimate_beam_center; @@ -1686,6 +1688,7 @@ static int RunRugnux(int argc, char **argv) { config.end_image = end_image; config.stride = image_stride; config.nthreads = nthreads; + config.nthreads_auto = nthreads_auto; config.output_prefix = output_prefix; config.detect_beam_stop = detect_beam_stop; config.estimate_beam_center = estimate_beam_center; @@ -2089,6 +2092,7 @@ static int RunRugnux(int argc, char **argv) { config.end_image = end_image; config.stride = image_stride; config.nthreads = nthreads; + config.nthreads_auto = nthreads_auto; config.output_prefix = output_prefix; config.spot_finding = spot_settings; config.rotation_indexing = rotation_indexing;