From 0a28da7c2ecc252eb1babd4d00eb16ac0c90e10b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 7 Jul 2026 14:10:48 +0200 Subject: [PATCH 01/41] Acquisition errors return to Idle instead of Error state Going to the Error state after any acquisition failure forced an expensive re-initialisation (reconnect + full detector config + pedestals), even when the detector was still perfectly configured and only a transient problem occurred. Split failures into two classes: - ordinary acquisition failures (receiver/writer/analysis problems, missed packets, writer disconnect) return to Idle with an Error-severity message, so the run can be retried without re-initialising; - failures that leave the detector in an undefined state throw the new JFJochCriticalException and still go to Error, forcing re-initialisation. The detector layer is the single source of truth for "detector faulted": SLSDetectorWrapper::InternalStop and DectrisDetectorWrapper::CheckBusyOrError raise JFJochCriticalException on the ERROR state, and JFJochServices::Stop raises it when the detector does not return to idle at the end of a run. MeasurementThread dispatches purely on exception type - no detector-status field checks. Also: - JFJochServices::Start now stops the receiver it launched if the detector fails to start, so the service returns to idle cleanly and the run can be retried. - A synchronous Start rethrows the failure to the caller (HTTP layer) via a stored exception_ptr, so a failed start is no longer reported as HTTP 200. - JFJochServices::Stop no longer throws on a writer error; it is reported through receiver_output.writer_err and surfaced as an Idle + Error-severity message, which makes the previously-dead writer_err branch in MeasurementThread live. Initialisation, calibration/pedestal, SelectDetector and Deactivate keep the Error-state behaviour, as those genuinely leave the system in an undefined state. Co-Authored-By: Claude Opus 4.8 --- broker/JFJochServices.cpp | 50 +++++++++++++-------- broker/JFJochStateMachine.cpp | 40 +++++++++++++++-- broker/JFJochStateMachine.h | 3 ++ common/JFJochException.h | 10 +++++ detector_control/DectrisDetectorWrapper.cpp | 3 +- detector_control/SLSDetectorWrapper.cpp | 4 +- 6 files changed, 85 insertions(+), 25 deletions(-) diff --git a/broker/JFJochServices.cpp b/broker/JFJochServices.cpp index b5913a81..9f82919b 100644 --- a/broker/JFJochServices.cpp +++ b/broker/JFJochServices.cpp @@ -15,18 +15,35 @@ void JFJochServices::Start(const DiffractionExperiment& experiment, cannot_stop_detector = false; - if (receiver != nullptr) { - logger.Info(" ... receiver start"); - if (image_puller) - image_puller->ResumeAndClear(); - receiver->Start(experiment, pixel_mask, calibration, image_puller); - { - std::shared_lock ul(detector_mutex); - if (detector && !experiment.IsUsingInternalPacketGen()) { - logger.Info(" ... detector start"); - detector->Start(experiment); - } + if (receiver == nullptr) { + logger.Info(" Done!"); + return; + } + + logger.Info(" ... receiver start"); + if (image_puller) + image_puller->ResumeAndClear(); + receiver->Start(experiment, pixel_mask, calibration, image_puller); + + // From here the receiver is running asynchronously. If starting the detector fails, stop the + // receiver again so the service returns to idle and the run can be retried without + // re-initialising. The detector failure is then propagated (a critical detector error stays + // critical, so the broker still goes to the Error state). + try { + std::shared_lock ul(detector_mutex); + if (detector && !experiment.IsUsingInternalPacketGen()) { + logger.Info(" ... detector start"); + detector->Start(experiment); } + } catch (const std::exception &e) { + logger.Error(" ... detector failed to start ({}) - stopping receiver", e.what()); + receiver->Cancel(false); + try { + receiver->Stop(); + } catch (const std::exception &stop_error) { + logger.Warning("Receiver stop after failed start reported: {}", stop_error.what()); + } + throw; } logger.Info(" Done!"); @@ -123,12 +140,9 @@ JFJochServicesOutput JFJochServices::Stop() { } } - // A writer that broke mid-run (e.g. a lost connection) leaves a truncated file. - // Surface that as a failed acquisition instead of silently reporting success. - if (!ret.receiver_output.writer_err.empty()) - throw JFJochException(JFJochExceptionCategory::FileWriteError, - "Writer error, written data may be incomplete: " - + ret.receiver_output.writer_err); + // A writer that broke mid-run (e.g. a lost connection) leaves a truncated file. This is + // reported to the caller via receiver_output.writer_err (the state machine surfaces it as + // an error message) - it is not a detector fault, so it does not need re-initialisation. logger.Info(" ... finished with success"); } catch (const JFJochException &e) { @@ -146,7 +160,7 @@ JFJochServicesOutput JFJochServices::Stop() { throw JFJochException(*exception); if (detector_error) - throw JFJochException(JFJochExceptionCategory::Detector, "Error in detector operation"); + throw JFJochCriticalException("Error in detector operation"); return ret; } diff --git a/broker/JFJochStateMachine.cpp b/broker/JFJochStateMachine.cpp index 1343bbfb..df4cfa3e 100644 --- a/broker/JFJochStateMachine.cpp +++ b/broker/JFJochStateMachine.cpp @@ -7,6 +7,7 @@ #include "../preview/JFJochTIFF.h" #include "../common/CUDAWrapper.h" #include "../common/GitInfo.h" +#include "../common/JFJochException.h" JFJochStateMachine::JFJochStateMachine(const DiffractionExperiment& in_experiment, JFJochServices &in_services, @@ -361,10 +362,19 @@ void JFJochStateMachine::Start(const DatasetSettings &settings, bool async) { experiment.IncrementRunNumber(); + start_exception = nullptr; SetState(JFJochState::Busy, "Preparing measurement", BrokerStatus::MessageSeverity::Info); measurement = std::async(std::launch::async, &JFJochStateMachine::MeasurementThread, this); - if (!async) + if (!async) { c.wait(ul, [&]() { return state != JFJochState::Busy; }); + // A synchronous start propagates the failure to the caller. The state has already been set + // by MeasurementThread (Idle for an ordinary failure, Error for a critical detector fault). + if (start_exception) { + auto e = start_exception; + start_exception = nullptr; + std::rethrow_exception(e); + } + } } BrokerStatus JFJochStateMachine::WaitTillNotBusy(std::chrono::milliseconds timeout) { @@ -392,12 +402,26 @@ void JFJochStateMachine::MeasurementThread() { SetState(JFJochState::Measuring, "Measuring ...", BrokerStatus::MessageSeverity::Info); } c.notify_all(); - } catch (std::exception &e) { + } catch (const JFJochCriticalException &e) { + // Detector left in an undefined state - force re-initialisation via the Error state. + logger.Error("Critical error starting measurement: {}", e.what()); { std::unique_lock ul(m); SetState(JFJochState::Error, e.what(), BrokerStatus::MessageSeverity::Error); + start_exception = std::current_exception(); + } + c.notify_all(); + return; + } catch (const std::exception &e) { + // Ordinary acquisition failure - the detector is still configured/calibrated, so return to + // Idle and let the user retry without re-initialising. services.Start has already stopped the + // receiver it launched. + logger.Error("Error starting measurement: {}", e.what()); + { + std::unique_lock ul(m); + SetState(JFJochState::Idle, e.what(), BrokerStatus::MessageSeverity::Error); + start_exception = std::current_exception(); } - services.Cancel(); c.notify_all(); return; } @@ -442,9 +466,17 @@ void JFJochStateMachine::MeasurementThread() { "Data collection without problems", BrokerStatus::MessageSeverity::Success); } - } catch (const std::exception &e) { + } catch (const JFJochCriticalException &e) { + // Detector faulted during the run - it needs re-initialisation, so go to the Error state. + logger.Error("Critical error finishing measurement: {}", e.what()); std::unique_lock ul(m); SetState(JFJochState::Error, e.what(), BrokerStatus::MessageSeverity::Error); + } catch (const std::exception &e) { + // Receiver/writer problem - the data may be incomplete, but the detector is still usable, so + // return to Idle rather than forcing re-initialisation. + logger.Error("Error finishing measurement: {}", e.what()); + std::unique_lock ul(m); + SetState(JFJochState::Idle, e.what(), BrokerStatus::MessageSeverity::Error); } c.notify_all(); } diff --git a/broker/JFJochStateMachine.h b/broker/JFJochStateMachine.h index 97c681c1..40f49e5e 100644 --- a/broker/JFJochStateMachine.h +++ b/broker/JFJochStateMachine.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "../common/DiffractionExperiment.h" #include "../jungfrau/JFCalibration.h" @@ -99,6 +100,8 @@ class JFJochStateMachine { PixelMask pixel_mask; int64_t current_detector_setup; // Lock only on change std::optional scan_result; + // Set by MeasurementThread when a synchronous Start fails, so Start() can rethrow to the caller + std::exception_ptr start_exception; mutable std::mutex calibration_statistics_mutex; std::vector calibration_statistics; diff --git a/common/JFJochException.h b/common/JFJochException.h index 4c132a5a..0f574b9d 100644 --- a/common/JFJochException.h +++ b/common/JFJochException.h @@ -155,6 +155,16 @@ public: : JFJochException(JFJochExceptionCategory::WrongDAQState, description) {} }; +// Thrown when an operation leaves a subsystem (typically the detector) in an undefined state that a +// simple retry cannot recover from - the broker must re-initialise. The state machine catches this +// separately and goes to the Error state, whereas ordinary acquisition failures return to Idle. +class JFJochCriticalException : public JFJochException { +public: + explicit JFJochCriticalException(const std::string &description, + JFJochExceptionCategory in_category = JFJochExceptionCategory::Detector) + : JFJochException(in_category, description) {} +}; + class PCIeDeviceException : public JFJochException { public: explicit PCIeDeviceException(const std::string &description) diff --git a/detector_control/DectrisDetectorWrapper.cpp b/detector_control/DectrisDetectorWrapper.cpp index 136fab3c..dff0cb18 100644 --- a/detector_control/DectrisDetectorWrapper.cpp +++ b/detector_control/DectrisDetectorWrapper.cpp @@ -48,8 +48,7 @@ void DectrisDetectorWrapper::CheckBusy() { void DectrisDetectorWrapper::CheckBusyOrError() { switch(GetState()) { case DetectorState::ERROR: - throw JFJochException(JFJochExceptionCategory::Detector, - "Detector in error state"); + throw JFJochCriticalException("Detector in error state"); case DetectorState::BUSY: case DetectorState::WAITING: throw JFJochException(JFJochExceptionCategory::Detector, diff --git a/detector_control/SLSDetectorWrapper.cpp b/detector_control/SLSDetectorWrapper.cpp index c8ad5c95..3d8e0bd5 100644 --- a/detector_control/SLSDetectorWrapper.cpp +++ b/detector_control/SLSDetectorWrapper.cpp @@ -217,6 +217,8 @@ void SLSDetectorWrapper::Start(const DiffractionExperiment& experiment) { det.setNumberOfTriggers(experiment.GetNumTriggers()); det.startDetector(); + } catch (const JFJochException &) { + throw; // already a categorised JFJoch exception (incl. critical detector error) - keep its type } catch (std::exception &e) { logger.ErrorException(e); throw JFJochException(JFJochExceptionCategory::Detector, e.what()); @@ -229,7 +231,7 @@ void SLSDetectorWrapper::InternalStop() { // Assume it is executed in try-catch! auto state = GetState(); if (state == DetectorState::ERROR) - throw JFJochException(JFJochExceptionCategory::Detector, "Detector in error state"); + throw JFJochCriticalException("Detector in error state"); else if ((state == DetectorState::BUSY) || (state == DetectorState::WAITING)) { try { det.stopDetector(); -- 2.54.0 From 1caa57744ab44b9497e16f6c4311f0a6ecbc8b6b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 7 Jul 2026 22:46:58 +0200 Subject: [PATCH 02/41] Rebrand offline processing as rugnux (jfjoch_process -> rugnux) Split the naming: rugnux = data-processing subsystem, Jungfraujoch = streaming/acquisition. Executables jfjoch_process -> rugnux (source tools/rugnux_cli.cpp) and jfjoch_scale -> rugnux_scale; the processing library process/ -> rugnux/ with class/target JFJochProcess -> Rugnux (JFJochProcessObserver -> RugnuxObserver, JFJochProcessCommandLine -> RugnuxCommandLine). Doc JFJOCH_PROCESS.md -> RUGNUX.md, reconciled with the live usage message (drop dead -P/--partiality, -w/--wedge; --process-as-stills -> --force-still; add the real rot3d scaling knobs). New docs/NAMING.md explains both names, with pronunciation and a note on Romansh. rugnux now scales and merges rotation data automatically (implicit -M); stills still require an explicit -M. jfjoch_viewer and its classes keep their names (rename deferred); only their references to the renamed library are updated. The _process.h5 output suffix and ProcessConfig/Mode/Result are kept. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/build_and_test.yml | 2 +- CLAUDE.md | 7 ++- CMakeLists.txt | 6 +- docs/HDF5.md | 4 +- docs/JFJOCH_VIEWER.md | 4 +- docs/NAMING.md | 60 +++++++++++++++++++ docs/{JFJOCH_PROCESS.md => RUGNUX.md} | 49 ++++++++------- docs/TOOLS.md | 8 +-- docs/index.rst | 3 +- image_analysis/azint/ICE_RING_DETECTION.md | 2 +- .../BRAGG_INTEGRATION_ENGINE.md | 4 +- .../bragg_integration/NEXTGEN_INTEGRATOR.md | 4 +- process/CMakeLists.txt | 11 ---- rugnux/CMakeLists.txt | 11 ++++ .../JFJochProcess.cpp => rugnux/Rugnux.cpp | 8 +-- process/JFJochProcess.h => rugnux/Rugnux.h | 16 ++--- .../RugnuxCommandLine.cpp | 6 +- .../RugnuxCommandLine.h | 6 +- tests/CMakeLists.txt | 6 +- ...ocessLargeTest.cpp => RugnuxLargeTest.cpp} | 8 +-- .../{JFJochProcessTest.cpp => RugnuxTest.cpp} | 26 ++++---- tests/data/README.md | 2 +- tools/CMakeLists.txt | 18 +++--- tools/jfjoch_azint.cpp | 8 +-- tools/{jfjoch_process.cpp => rugnux_cli.cpp} | 23 ++++--- tools/{jfjoch_scale.cpp => rugnux_scale.cpp} | 10 ++-- viewer/CMakeLists.txt | 2 +- viewer/JFJochProcessController.cpp | 2 +- viewer/JFJochProcessController.h | 12 ++-- viewer/JFJochViewerWindow.cpp | 2 +- viewer/windows/JFJochProcessingJobsWindow.cpp | 4 +- viewer/windows/JFJochProcessingJobsWindow.h | 2 +- 32 files changed, 204 insertions(+), 132 deletions(-) create mode 100644 docs/NAMING.md rename docs/{JFJOCH_PROCESS.md => RUGNUX.md} (78%) delete mode 100644 process/CMakeLists.txt create mode 100644 rugnux/CMakeLists.txt rename process/JFJochProcess.cpp => rugnux/Rugnux.cpp (99%) rename process/JFJochProcess.h => rugnux/Rugnux.h (92%) rename process/JFJochProcessCommandLine.cpp => rugnux/RugnuxCommandLine.cpp (96%) rename process/JFJochProcessCommandLine.h => rugnux/RugnuxCommandLine.h (71%) rename tests/{JFJochProcessLargeTest.cpp => RugnuxLargeTest.cpp} (92%) rename tests/{JFJochProcessTest.cpp => RugnuxTest.cpp} (87%) rename tools/{jfjoch_process.cpp => rugnux_cli.cpp} (98%) rename tools/{jfjoch_scale.cpp => rugnux_scale.cpp} (98%) diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index cae9626f..2005ef02 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -135,7 +135,7 @@ jobs: run: | cd build # Build the whole viewer-only tree, not just the GUI: the "viewer" CPack component also - # contains the portable CLI tools (jfjoch_process/scale/azint/recompress/extract_hkl), + # contains the portable CLI tools (rugnux/rugnux_scale/azint/recompress/extract_hkl), # which must exist on disk before cpack installs the component. ninja -j16 cpack diff --git a/CLAUDE.md b/CLAUDE.md index 2e305818..a8754e28 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,8 +122,9 @@ generated API model and internal types. `JFJochImageAnalysis`): - `jfjoch_broker` — online, real-time (FPGA + GPU). - `jfjoch_viewer` — interactive Qt desktop (`viewer/`), results not persisted. -- `jfjoch_process` (`tools/jfjoch_process.cpp`) — offline batch over a stored HDF5; writes - `_process.h5` and `.mtz`/`.cif`/`.hkl`. `jfjoch_scale` re-scales/merges already-integrated data. +- `rugnux` (`tools/rugnux_cli.cpp`, built on the `Rugnux` library in `rugnux/`) — offline batch over + a stored HDF5; writes `_process.h5` and `.mtz`/`.cif`/`.hkl`. `rugnux_scale` re-scales/merges + already-integrated data. (rugnux = the data-processing half of the system; see `docs/NAMING.md`.) **`image_analysis/` pipeline** (subdirs): `spot_finding`, `indexing` (`ffbidx`/`fft` GPU, `fftw` CPU), `lattice_search`, `geom_refinement`, `pixel_refinement`, `bragg_prediction`, @@ -198,7 +199,7 @@ A per-image scalar (e.g. `ice_ring_score`, `bkg_estimate`, `mosaicity`) flows an (`bkg_estimate` is a clean template) at every layer: 1. **Compute** where the azint profile is finalized: `image_analysis/MXAnalysisWithoutFPGA.cpp` (CPU), - `receiver/JFJochReceiverFPGA.cpp` (FPGA), and the offline azint worker in `process/JFJochProcess.cpp`. + `receiver/JFJochReceiverFPGA.cpp` (FPGA), and the offline azint worker in `rugnux/Rugnux.cpp`. 2. **Message** (`common/JFJochMessages.h`): `std::optional` in `DataMessage`, `std::vector` in `EndMessage`. 3. **CBOR**: encode in `frame_serialize/CBORStream2Serializer.cpp` (DataMessage block *and* END block), diff --git a/CMakeLists.txt b/CMakeLists.txt index 7934715e..cd78188b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,9 +277,9 @@ IF (JFJOCH_VIEWER_ONLY) ADD_SUBDIRECTORY(image_analysis) ADD_SUBDIRECTORY(broker) ADD_SUBDIRECTORY(reader) - ADD_SUBDIRECTORY(process) + ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(viewer) - ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (process/scale/azint/extract_hkl) + ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/rugnux_scale/azint/extract_hkl) ELSE() ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) @@ -302,7 +302,7 @@ ELSE() ADD_SUBDIRECTORY(acquisition_device) ADD_SUBDIRECTORY(receiver) ADD_SUBDIRECTORY(image_analysis) - ADD_SUBDIRECTORY(process) + ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(tests) ADD_SUBDIRECTORY(tools) ENDIF() diff --git a/docs/HDF5.md b/docs/HDF5.md index 4a9e01dd..9a277b25 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -93,7 +93,7 @@ bitshuffle + Zstd; signed integer image datasets use `INTx_MIN` as the HDF5 fill ### Reprocessing output: `_process.h5` -The offline reprocessing tool [`jfjoch_process`](TOOLS.md) (`tools/jfjoch_process.cpp`) re-runs the +The offline reprocessing tool [`rugnux`](TOOLS.md) (`tools/rugnux_cli.cpp`) re-runs the full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing dataset and writes its results to a master file named **`_process.h5`**. This file uses the **integrated** format, but instead of copying the images its `/entry/data/data` is a *virtual @@ -104,7 +104,7 @@ of the raw images — without duplicating terabytes of data. This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it processed, while the `/entry/data/data` -VDS still resolves to the original images when they are available. `jfjoch_process` can also process +VDS still resolves to the original images when they are available. `rugnux` can also process an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. ## 3. NXmx-standard content diff --git a/docs/JFJOCH_VIEWER.md b/docs/JFJOCH_VIEWER.md index fe74af36..c36b4464 100644 --- a/docs/JFJOCH_VIEWER.md +++ b/docs/JFJOCH_VIEWER.md @@ -14,12 +14,12 @@ Jungfraujoch RPM/APT repositories (see [Deployment](DEPLOYMENT.md)). | --- | --- | --- | --- | | [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | | **`jfjoch_viewer`** | **Interactive, on-screen exploration** | **Qt desktop application** | **Displayed on screen (results not saved to disk)** | -| [`jfjoch_process`](JFJOCH_PROCESS.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | +| [`rugnux`](RUGNUX.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | ## Functionality - Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the - `*_process.h5` files produced by [`jfjoch_process`](JFJOCH_PROCESS.md). It also opens NXmx files + `*_process.h5` files produced by [`rugnux`](RUGNUX.md). It also opens NXmx files written by DECTRIS detectors, though that path has had only limited testing. - Runs an **embedded data-processing pipeline** — the same analysis code as the rest of Jungfraujoch — performing spot finding, indexing and integration on the displayed images. diff --git a/docs/NAMING.md b/docs/NAMING.md new file mode 100644 index 00000000..fa6a7a83 --- /dev/null +++ b/docs/NAMING.md @@ -0,0 +1,60 @@ +# Naming + +The software is Swiss, and so are its names: both halves of the system are named after +places in the Alps that are, in one way or another, about moving a *lot* of something up a +steep mountain as efficiently as possible — usually by train. Throughput, in other words. + +| Part | Name | What it does | +| --- | --- | --- | +| Streaming / acquisition | **Jungfraujoch** | Receives detector data at high data rates, runs the FPGA/GPU pipeline, and streams images out for writing. | +| Data processing | **Rugnux** | Offline crystallographic analysis of a stored dataset — indexing, integration, scaling and merging (the [`rugnux`](RUGNUX.md) tool). | + +## Jungfraujoch + +The **Jungfraujoch** is a high mountain col in the Bernese Alps, the saddle (*Joch* is German +for "yoke" or "col") between the peaks **Jungfrau** and **Mönch**, at 3,466 m. It is the site of +the [High Altitude Research Station Jungfraujoch](https://www.hfsjg.ch/), whose long-running +atmospheric measurements are **co-operated by the Paul Scherrer Institute** — the same institute +that develops this software and the JUNGFRAU detector. + +The name is also a small piece of word-play. PSI's **JUNGFRAU** detector and DECTRIS's **EIGER** +detector are both named after Bernese Alps peaks (the famous trio is *Eiger*, *Mönch*, *Jungfrau*). +The Jungfraujoch — the pass *between* Jungfrau and Mönch — is where those two detector worlds meet. + +And it fits the theme of the whole project: the Jungfraujoch is reached by the **Jungfraubahn**, +whose terminus is the **highest railway station in Europe** (3,454 m, the "Top of Europe"). It is +the closest you can get to that summit in a genuinely *high-throughput* way — by train, moving +crowds up the mountain — which is exactly what the streaming side of this software does with +detector frames. + +**Pronunciation (German):** *Jungfraujoch* ≈ **YUNG-frow-yokh**. +"Jung" as in *young*, "frau" rhymes with *cow*, and the final "joch" ends in the guttural *ch* of +Scottish *loch* or German *Bach* — not a hard *k*. + +## Rugnux + +**Piz Rugnux** is a mountain in the Rhaetian Alps of canton Graubünden, in south-eastern +Switzerland. (*Piz* is the Romansh word for "peak".) It rises above the **Albula line** of the +**Rhaetian Railway** (*Rhätische Bahn*), part of the "Rhaetian Railway in the Albula / Bernina +Landscapes" — a **UNESCO World Heritage Site** (*Welterbe*). + +That stretch of line is a masterpiece of throughput engineering: to climb a great deal of altitude +in very little horizontal distance, it corkscrews through a series of **helical (spiral) tunnels** +looping back inside the mountains. It is, again, the Swiss art of getting an enormous amount up a +steep mountain efficiently — the same idea the data-processing side of this software is built +around: pushing a large volume of diffraction data through the analysis pipeline. + +So the theme is consistent — **Swiss mountains, trains, and throughput** — while keeping the two +subsystems clearly distinct: *Jungfraujoch* streams, *Rugnux* processes. + +**Pronunciation (Romansh):** *Piz Rugnux* ≈ **peets roo-NYOOKS**. +The "gn" is a soft palatal *ñ*, as in *canyon* or Italian *gnocchi*, not two separate sounds. + +## What is Romansh? + +**Romansh** (*Rumantsch*) is the **fourth national language of Switzerland**, alongside German, +French and Italian. It is a Romance language — a direct descendant of the spoken Latin left behind +in the Alpine valleys — today spoken by only a few tens of thousands of people, almost all in the +canton of Graubünden. It survives in several regional idioms, brought together in a standard form +called *Rumantsch Grischun*. Naming the processing engine with a Romansh mountain is a small nod to +the least-spoken but no-less-Swiss corner of the country. diff --git a/docs/JFJOCH_PROCESS.md b/docs/RUGNUX.md similarity index 78% rename from docs/JFJOCH_PROCESS.md rename to docs/RUGNUX.md index 9692b9f3..7dddc771 100644 --- a/docs/JFJOCH_PROCESS.md +++ b/docs/RUGNUX.md @@ -1,6 +1,7 @@ -# jfjoch_process +# rugnux -`jfjoch_process` is the **offline** crystallographic data-analysis tool of Jungfraujoch. +`rugnux` is the **offline** crystallographic data-analysis tool of Jungfraujoch — the +data-processing half of the system (see [Naming](NAMING.md) for where the name comes from). It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the results to a `_process.h5` file, plus reflection files (`.mtz`/`.cif`/`.hkl`) when merging is @@ -9,9 +10,9 @@ requested. It runs the *same* analysis code as the online and interactive tools, just driven from the command line over a file rather than a live detector stream. -> **Note.** `jfjoch_process` is under very active development. This page describes the tool and +> **Note.** `rugnux` is under very active development. This page describes the tool and > its options at a high level; the authoritative, always-current list of options is the program's -> own usage message — run `jfjoch_process` with no arguments. +> own usage message — run `rugnux` with no arguments. ## Where it fits among the three analysis tools @@ -19,9 +20,9 @@ command line over a file rather than a live detector stream. | --- | --- | --- | --- | | [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | | [`jfjoch_viewer`](JFJOCH_VIEWER.md) | Interactive, on-screen exploration | Qt desktop application | Displayed on screen (results not saved to disk) | -| **`jfjoch_process`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | +| **`rugnux`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | -Use `jfjoch_process` to re-analyse data after acquisition, to experiment with processing +Use `rugnux` to re-analyse data after acquisition, to experiment with processing parameters, or to produce merged intensities for downstream structure solution. ## Hardware @@ -51,11 +52,11 @@ the first pass. Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. -## Re-scaling and re-merging (`jfjoch_scale`) +## Re-scaling and re-merging (`rugnux_scale`) -The companion tool `jfjoch_scale` re-scales and merges the *already-integrated* reflections stored +The companion tool `rugnux_scale` re-scales and merges the *already-integrated* reflections stored in one or more `_process.h5` files, without re-running spot finding or integration. Use it to -re-merge quickly with a different space group, partiality model, resolution limit or reference MTZ, +re-merge quickly with a different space group, resolution limit, anomalous setting or reference MTZ, or to combine several processed runs into one set of merged intensities. ## Quick start @@ -65,22 +66,23 @@ or to combine several processed runs into one set of merged intensities. Index, integrate, scale and merge a rotation sweep, fully de novo: ``` -jfjoch_process rotation_master.h5 \ +rugnux rotation_master.h5 \ -o lyso_rot -N 32 \ - -M --scaling-high-resolution 1.4 + --scaling-high-resolution 1.4 ``` Because the dataset carries a rotation goniometer axis, it is processed as **rotation data by default**: two-pass rotation indexing (index the sweep once, then process every frame against that -lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). `-M` -scales and merges; the unit cell is taken from the rotation indexer and the space group is -determined from systematic absences, and both are written into the merged `.cif`. +lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). Rotation +data is **scaled and merged automatically** (as if `-M` were given); the unit cell is taken from the +rotation indexer and the space group is determined from systematic absences, and both are written +into the merged `.cif`. Run **fully de novo** (no `-C`/`-S`) for the best result — supplying a cell or space group up front tends to *degrade* low-symmetry cases. `--scaling-high-resolution` (set it to your expected resolution) sharpens both the space-group search and the error model. To tune the first pass use `--two-pass-rotation=100` (or `-R100` — the first-pass image count); to force the sweep to be -treated as independent stills use `--process-as-stills`. +treated as independent stills use `--force-still`. ### Still / serial data @@ -89,7 +91,7 @@ stills automatically** — no flag needed. Known-cell indexing with the GPU fast then merge against a reference structure: ``` -jfjoch_process serial_master.h5 \ +rugnux serial_master.h5 \ -o lyso_serial -N 32 \ -X ffbidx -C 79,79,38,90,90,90 -S 96 \ --spot-sigma 4 \ @@ -100,7 +102,7 @@ jfjoch_process serial_master.h5 \ `ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills. For weak serial data, tightening spot finding with `--spot-sigma 4` typically raises the indexing rate substantially. If a dataset *does* carry a goniometer axis but you want per-frame stills processing -anyway, add `--process-as-stills`. +anyway, add `--force-still`. ## Command-line options @@ -123,6 +125,7 @@ Spot finding: | `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | | `--spot-high-resolution ` | High-resolution limit for spot finding, Å (default: 1.5) | | `--max-spots ` | Maximum spot count (default: 250) | +| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots and exclude ice-ring reflections from scaling/merging; overrides the dataset setting (default: use the dataset value) | Azimuthal integration (the radial profile behind the per-image ice-ring score): @@ -136,13 +139,13 @@ Azimuthal integration (the radial profile behind the per-image ice-ring score): Indexing: A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation -indexing) by default; a dataset without one is processed as independent stills. `--process-as-stills` +indexing) by default; a dataset without one is processed as independent stills. `--force-still` overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request rotation explicitly and pick the pass or lattice. | Option | Description | | --- | --- | -| `--process-as-stills` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | +| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | | `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | | `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`) | | `-S, --space-group ` | Space group number (used for indexing and scaling) | @@ -160,11 +163,12 @@ Scaling and merging: | Option | Description | | --- | --- | -| `-M, --scale-merge` | Scale and merge | -| `-P, --partiality ` | Partiality model: `fixed` \| `rot` \| `rot3d` \| `unity` (default: `rot3d` for rotation data, `fixed` for stills). `rot3d` = `rot` + 3D combine of the per-frame partials into fulls | +| `-M, --scale-merge` | Scale and merge (automatic for rotation data; only needed to force scaling on stills) | | `-A, --anomalous` | Anomalous mode (keep Friedel pairs separate) | | `-B, --refine-bfactor` | Refine a per-image B-factor | -| `-w, --wedge[=num]` | Refine the per-image rotation wedge (optional starting value) | +| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | +| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) | +| `--capture-uncertainty ` | rot3d: systematic sigma on under-captured fulls, ~num·(1−captured_fraction)·I (default: 1.0 for rotation, 0 otherwise) | | `--scaling-high-resolution ` | High-resolution limit for scaling, Å (default: no limit) | | `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02) | | `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for `rot3d`, off otherwise) | @@ -173,6 +177,7 @@ Scaling and merging: | `--scaling-iterations ` | Scaling iterations with no reference data (default: 3) | | `--scaling-output ` | Reflection output format: `cif` (mmCIF, default) \| `mtz` \| `txt` | | `-z, --reference-mtz ` | Reference MTZ (enables reference-driven scaling) | +| `--write-process-h5` | Also write the (large) `_process.h5` when merging (default: only `.mtz`/`.cif`) | Integration: diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 8e6aecfa..345ed34e 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -6,15 +6,15 @@ number of command-line tools. Each prints its own usage when run with `-h` or wi ## Data analysis -### jfjoch_process +### rugnux Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, integration, scaling/merging) on a stored HDF5 dataset, producing a `_process.h5` file and, when -merging, reflection files. See [jfjoch_process](JFJOCH_PROCESS.md). +merging, reflection files. See [rugnux](RUGNUX.md). -### jfjoch_scale +### rugnux_scale Re-scales and merges the already-integrated reflections from one or more `_process.h5` files (no re-integration). Useful to re-merge with a different space group, partiality, resolution limit -or reference MTZ, or to combine several runs. See [jfjoch_process](JFJOCH_PROCESS.md). +or reference MTZ, or to combine several runs. See [rugnux](RUGNUX.md). ### jfjoch_extract_hkl Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across diff --git a/docs/index.rst b/docs/index.rst index 62cefd66..af2cd19d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ Jungfraujoch is distributed under the GPLv3 license. VERSIONING DEPLOYMENT REPOSITORIES + NAMING CHANGELOG .. toctree:: @@ -32,7 +33,7 @@ Jungfraujoch is distributed under the GPLv3 license. JFJOCH_BROKER JFJOCH_WRITER - JFJOCH_PROCESS + RUGNUX JFJOCH_VIEWER SOFTWARE_INTEGRATION TOOLS diff --git a/image_analysis/azint/ICE_RING_DETECTION.md b/image_analysis/azint/ICE_RING_DETECTION.md index 99dcfdb3..1a512375 100644 --- a/image_analysis/azint/ICE_RING_DETECTION.md +++ b/image_analysis/azint/ICE_RING_DETECTION.md @@ -55,7 +55,7 @@ threshold needs no per-detector tuning). rings;}` with per-ring `{centre_q, half_width_q}`. - Add an `IsOnIceRing(d, const vector&)` overload in `common/Definitions.h` (per-ring centre + width), alongside the existing fixed-table one. -- `tools/jfjoch_process.cpp`: change `--detect-ice-rings` to take `auto|on|off` (default `auto`); in +- `tools/rugnux_cli.cpp`: change `--detect-ice-rings` to take `auto|on|off` (default `auto`); in `auto`, run `DetectIceRings` on the accumulated profile and set `experiment.DetectIceRings(model.present)` + feed `model.rings` to the spot-finder (`MarkIceRings`) and the scaling exclusion path. diff --git a/image_analysis/bragg_integration/BRAGG_INTEGRATION_ENGINE.md b/image_analysis/bragg_integration/BRAGG_INTEGRATION_ENGINE.md index f2c1ccf7..a2f5eeed 100644 --- a/image_analysis/bragg_integration/BRAGG_INTEGRATION_ENGINE.md +++ b/image_analysis/bragg_integration/BRAGG_INTEGRATION_ENGINE.md @@ -44,7 +44,7 @@ Pass A / the seed of the two profile modes, so it always runs. This is the same buffer `AzIntEngineGPU`/`ROIIntegrationGPU` consume. **Consequence:** the `±1` special/saturation band that `ProfileIntegrate2D` rejects on the lossy `CompressedImage` is handled upstream by the preprocessor instead — correct for the online path; sanity-check it for offline - `jfjoch_process` if that ever uses this engine on a lossy-compressed stored file. + `rugnux` if that ever uses this engine on a lossy-compressed stored file. - Config uses raw detector dims — `xpixel = GetXPixelsNum()`, `npixel = GetPixelsNum()` — matching how `MXAnalysisWithoutFPGA` sizes the `ImagePreprocessorBuffer` and the frame the predicted `predicted_x` live in. @@ -100,7 +100,7 @@ The engine is deliberately *not* called anywhere yet. Wiring it in: `MXAnalysisWithoutFPGA` chooses `AzIntEngineGPU` vs `AzIntEngineCPU`). Construct with the same `stream` the other GPU engines share. 3. **Keep the old functions** (`ProfileIntegrate2D`/`BraggIntegrate2D`) until the engine is validated - end-to-end against them on real data (`jfjoch_process --integrator ... --dump-observations`, A/B vs + end-to-end against them on real data (`rugnux --integrator ... --dump-observations`, A/B vs `XDS_ASCII.HKL`), since the `±1`-band difference above is the one behavioural change. The output `vector` is identical in shape (I, sigma, bkg, partiality, d, ...), so diff --git a/image_analysis/bragg_integration/NEXTGEN_INTEGRATOR.md b/image_analysis/bragg_integration/NEXTGEN_INTEGRATOR.md index 9d450be2..a81b7159 100644 --- a/image_analysis/bragg_integration/NEXTGEN_INTEGRATOR.md +++ b/image_analysis/bragg_integration/NEXTGEN_INTEGRATOR.md @@ -55,7 +55,7 @@ per-frame 2D integration (BraggIntegrate2D | ProfileIntegrate2D) <- -> merge + error model + stats (MergeOnTheFly: RefineErrorModel, CC1/2, R-meas, CCref, ISa) ``` -Relevant knobs (`jfjoch_process`, mirrored in the viewer's Processing-settings tabs): +Relevant knobs (`rugnux`, mirrored in the viewer's Processing-settings tabs): - `--integrator boxsum|gaussian|empirical` — default **gaussian** (`IntegratorMode::ProfileGaussian`). - `-P fixed|rot|rot3d` — partiality model. `rot3d` = `PartialityModel::Rotation` **+** the orthogonal `ScalingSettings::combine_3d` bool (see "Why rot3d is a bool" below). @@ -205,7 +205,7 @@ goniometer sphere-of-confusion). ## Tooling (reusable instruments) -- **`--dump-observations `** (`jfjoch_process`): `Combine3D` writes the unmerged fulls +- **`--dump-observations `** (`rugnux`): `Combine3D` writes the unmerged fulls (`h k l I σ d n_frames captured_fraction peak_frame`) for direct A/B vs `XDS_ASCII.HKL`. - **`lyso_test/anomalous_scoreboard.sh [label]`**: one command → SHELXC + ANODE under `qemu-x86_64-static` (the SHELX binaries hit the legacy vsyscall page; this WSL2 kernel is diff --git a/process/CMakeLists.txt b/process/CMakeLists.txt deleted file mode 100644 index 17c11c0c..00000000 --- a/process/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -# SPDX-License-Identifier: GPL-3.0-only - -ADD_LIBRARY(JFJochProcess STATIC - JFJochProcess.cpp - JFJochProcess.h - JFJochProcessCommandLine.cpp - JFJochProcessCommandLine.h -) - -TARGET_LINK_LIBRARIES(JFJochProcess JFJochReader JFJochImageAnalysis JFJochWriter) diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt new file mode 100644 index 00000000..45d8a0d1 --- /dev/null +++ b/rugnux/CMakeLists.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +# SPDX-License-Identifier: GPL-3.0-only + +ADD_LIBRARY(Rugnux STATIC + Rugnux.cpp + Rugnux.h + RugnuxCommandLine.cpp + RugnuxCommandLine.h +) + +TARGET_LINK_LIBRARIES(Rugnux JFJochReader JFJochImageAnalysis JFJochWriter) diff --git a/process/JFJochProcess.cpp b/rugnux/Rugnux.cpp similarity index 99% rename from process/JFJochProcess.cpp rename to rugnux/Rugnux.cpp index 75545757..716c7711 100644 --- a/process/JFJochProcess.cpp +++ b/rugnux/Rugnux.cpp @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -#include "JFJochProcess.h" +#include "Rugnux.h" #include #include @@ -64,13 +64,13 @@ namespace { } -JFJochProcess::JFJochProcess(JFJochHDF5Reader &reader, DiffractionExperiment experiment, +Rugnux::Rugnux(JFJochHDF5Reader &reader, DiffractionExperiment experiment, PixelMask pixel_mask, ProcessConfig config) : reader_(reader), experiment_(std::move(experiment)), pixel_mask_(std::move(pixel_mask)), config_(std::move(config)) {} -ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) { - Logger logger("JFJochProcess"); +ProcessResult Rugnux::Run(RugnuxObserver *observer) { + Logger logger("Rugnux"); ProcessResult result; const auto dataset = reader_.GetDataset(); diff --git a/process/JFJochProcess.h b/rugnux/Rugnux.h similarity index 92% rename from process/JFJochProcess.h rename to rugnux/Rugnux.h index 065ad1ec..1ba2d77e 100644 --- a/process/JFJochProcess.h +++ b/rugnux/Rugnux.h @@ -20,7 +20,7 @@ class JFJochHDF5Reader; -// Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by jfjoch_process, +// Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux, // jfjoch_azint and (later) the viewer. The full processing workflow lives here, not in the CLIs: // setup, an optional two-pass rotation-indexing pre-pass, a parallel per-image loop (std::thread), // an optional scaling/merging post-pass, and the _process.h5 output. The detector geometry and all @@ -28,7 +28,7 @@ class JFJochHDF5Reader; // carries run control. Cancellable from any thread (e.g. SIGINT or a GUI button) via Cancel(). enum class ProcessMode { AzimuthalIntegration, // preprocess + azimuthal integration only (jfjoch_azint) - FullAnalysis // spot finding + indexing + refinement + integration (jfjoch_process) + FullAnalysis // spot finding + indexing + refinement + integration (rugnux) }; struct ProcessConfig { @@ -42,7 +42,7 @@ struct ProcessConfig { // Output prefix for the _process.h5 (and scaled reflections). Empty => process without writing. std::string output_prefix; - // Write the per-image _process.h5. Defaults true; jfjoch_process turns it off when merging + // Write the per-image _process.h5. Defaults true; rugnux turns it off when merging // (the .mtz/.cif is the wanted output and the h5 is large) unless --write-process-h5 is given. bool write_process_h5 = true; @@ -95,15 +95,15 @@ struct ProcessResult { // Callbacks for progress and live results. Methods may be called from worker threads, so an // implementation must be thread-safe. The default no-ops suit the CLIs. -class JFJochProcessObserver { +class RugnuxObserver { public: - virtual ~JFJochProcessObserver() = default; + virtual ~RugnuxObserver() = default; virtual void OnPhase(const std::string &phase) {} virtual void OnProgress(uint64_t done, uint64_t total) {} virtual void OnImageProcessed(const DataMessage &msg) {} }; -class JFJochProcess { +class Rugnux { JFJochHDF5Reader &reader_; DiffractionExperiment experiment_; PixelMask pixel_mask_; @@ -111,11 +111,11 @@ class JFJochProcess { std::atomic cancelled_{false}; public: - JFJochProcess(JFJochHDF5Reader &reader, DiffractionExperiment experiment, + Rugnux(JFJochHDF5Reader &reader, DiffractionExperiment experiment, PixelMask pixel_mask, ProcessConfig config); // Runs the configured workflow to completion or until Cancel(). Throws on setup failure. - ProcessResult Run(JFJochProcessObserver *observer = nullptr); + ProcessResult Run(RugnuxObserver *observer = nullptr); // Request cancellation; safe to call from any thread (the worker loop checks between images). void Cancel() { cancelled_ = true; } diff --git a/process/JFJochProcessCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp similarity index 96% rename from process/JFJochProcessCommandLine.cpp rename to rugnux/RugnuxCommandLine.cpp index d09d035a..0c7aa6bd 100644 --- a/process/JFJochProcessCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -#include "JFJochProcessCommandLine.h" +#include "RugnuxCommandLine.h" #include "../common/DiffractionExperiment.h" #include @@ -48,12 +48,12 @@ namespace { } } -std::string JFJochProcessCommandLine(const ProcessConfig &config, +std::string RugnuxCommandLine(const ProcessConfig &config, const DiffractionExperiment &experiment, const std::string &input_file) { std::vector args; const bool azint = (config.mode == ProcessMode::AzimuthalIntegration); - args.emplace_back(azint ? "jfjoch_azint" : "jfjoch_process"); + args.emplace_back(azint ? "jfjoch_azint" : "rugnux"); auto add = [&](const std::string &flag, const std::string &val) { args.push_back(flag); diff --git a/process/JFJochProcessCommandLine.h b/rugnux/RugnuxCommandLine.h similarity index 71% rename from process/JFJochProcessCommandLine.h rename to rugnux/RugnuxCommandLine.h index da0ad3e8..197198f6 100644 --- a/process/JFJochProcessCommandLine.h +++ b/rugnux/RugnuxCommandLine.h @@ -5,14 +5,14 @@ #include -#include "JFJochProcess.h" // ProcessConfig, ProcessMode +#include "Rugnux.h" // ProcessConfig, ProcessMode class DiffractionExperiment; -// Reconstruct an equivalent jfjoch_process / jfjoch_azint command line for a configured run, so a +// Reconstruct an equivalent rugnux / jfjoch_azint command line for a configured run, so a // job set up in the GUI can be handed off to a cluster. Covers the settings that matter for the // run, not every obscure flag; geometry is taken from the input file, so geometry overrides are // not emitted. -std::string JFJochProcessCommandLine(const ProcessConfig &config, +std::string RugnuxCommandLine(const ProcessConfig &config, const DiffractionExperiment &experiment, const std::string &input_file); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 788b1add..7e2fd57d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,8 +47,8 @@ ADD_EXECUTABLE(jfjoch_test ModuleSummationTest.cpp ZMQMetadataSocketTest.cpp JFJochReaderTest.cpp - JFJochProcessTest.cpp - JFJochProcessLargeTest.cpp + RugnuxTest.cpp + RugnuxLargeTest.cpp TestData.h MovingAverageTest.cpp ImageMetadataTest.cpp @@ -81,7 +81,7 @@ ADD_EXECUTABLE(jfjoch_test ) target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochStreamWriter - JFJochProcess JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview + Rugnux JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview jfjoch_xds_plugin) target_include_directories(jfjoch_test PRIVATE .) diff --git a/tests/JFJochProcessLargeTest.cpp b/tests/RugnuxLargeTest.cpp similarity index 92% rename from tests/JFJochProcessLargeTest.cpp rename to tests/RugnuxLargeTest.cpp index 064d06ef..8f13ade2 100644 --- a/tests/JFJochProcessLargeTest.cpp +++ b/tests/RugnuxLargeTest.cpp @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -// End-to-end JFJochProcess runs over real JUNGFRAU datasets that are kept in git-LFS under +// End-to-end Rugnux runs over real JUNGFRAU datasets that are kept in git-LFS under // tests/data. They are tagged [large] and SKIP() when the data is not present (e.g. LFS not // pulled), so the default test run stays fast and CI without the data still passes. @@ -16,7 +16,7 @@ #include "../common/DiffractionExperiment.h" #include "../common/IndexingSettings.h" #include "../reader/JFJochHDF5Reader.h" -#include "../process/JFJochProcess.h" +#include "../rugnux/Rugnux.h" namespace { // Start-up hook: report once whether the large datasets are available, so it is obvious why @@ -40,7 +40,7 @@ namespace { CATCH_REGISTER_LISTENER(LargeDataListener) -TEST_CASE("JFJochProcess_LysoRotation", "[large]") { +TEST_CASE("Rugnux_LysoRotation", "[large]") { const auto master = jfjoch_test::LargeDataFile("lyso_rotation_master.h5"); if (!master) SKIP("lyso_rotation_master.h5 not available (git-lfs data not pulled)"); @@ -67,7 +67,7 @@ TEST_CASE("JFJochProcess_LysoRotation", "[large]") { config.two_pass_rotation = true; config.reuse_rotation_spots = false; // redo spot finding (raw dataset may carry no spots) - JFJochProcess process(reader, experiment, dataset->pixel_mask, config); + Rugnux process(reader, experiment, dataset->pixel_mask, config); ProcessResult result; REQUIRE_NOTHROW(result = process.Run()); diff --git a/tests/JFJochProcessTest.cpp b/tests/RugnuxTest.cpp similarity index 87% rename from tests/JFJochProcessTest.cpp rename to tests/RugnuxTest.cpp index bf25799d..f5bf5620 100644 --- a/tests/JFJochProcessTest.cpp +++ b/tests/RugnuxTest.cpp @@ -8,8 +8,8 @@ #include "../common/ScanResultGenerator.h" #include "../writer/FileWriter.h" #include "../reader/JFJochHDF5Reader.h" -#include "../process/JFJochProcess.h" -#include "../process/JFJochProcessCommandLine.h" +#include "../rugnux/Rugnux.h" +#include "../rugnux/RugnuxCommandLine.h" namespace { // Write a small VDS dataset of `n` flat images and return nothing (prefix_master.h5 + @@ -44,7 +44,7 @@ namespace { } } -TEST_CASE("JFJochProcess_AzInt", "[HDF5][Full]") { +TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") { WriteTestDataset("process_azint_in", 8); JFJochHDF5Reader reader; @@ -57,7 +57,7 @@ TEST_CASE("JFJochProcess_AzInt", "[HDF5][Full]") { config.nthreads = 2; config.output_prefix = "process_azint_out"; - JFJochProcess process(reader, dataset->experiment, dataset->pixel_mask, config); + Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config); ProcessResult result; REQUIRE_NOTHROW(result = process.Run()); @@ -83,7 +83,7 @@ TEST_CASE("JFJochProcess_AzInt", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochProcess_NoOutput", "[HDF5][Full]") { +TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") { WriteTestDataset("process_noout_in", 6); JFJochHDF5Reader reader; @@ -95,7 +95,7 @@ TEST_CASE("JFJochProcess_NoOutput", "[HDF5][Full]") { config.mode = ProcessMode::AzimuthalIntegration; config.nthreads = 3; - JFJochProcess process(reader, dataset->experiment, dataset->pixel_mask, config); + Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config); auto result = process.Run(); CHECK_FALSE(result.cancelled); @@ -108,7 +108,7 @@ TEST_CASE("JFJochProcess_NoOutput", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochProcess_Cancel", "[HDF5][Full]") { +TEST_CASE("Rugnux_Cancel", "[HDF5][Full]") { WriteTestDataset("process_cancel_in", 8); JFJochHDF5Reader reader; @@ -119,7 +119,7 @@ TEST_CASE("JFJochProcess_Cancel", "[HDF5][Full]") { config.mode = ProcessMode::AzimuthalIntegration; config.nthreads = 2; - JFJochProcess process(reader, dataset->experiment, dataset->pixel_mask, config); + Rugnux process(reader, dataset->experiment, dataset->pixel_mask, config); process.Cancel(); // cancel before running: the worker loop stops immediately auto result = process.Run(); @@ -133,7 +133,7 @@ TEST_CASE("JFJochProcess_Cancel", "[HDF5][Full]") { REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); } -TEST_CASE("JFJochProcessCommandLine_Full", "[process]") { +TEST_CASE("RugnuxCommandLine_Full", "[process]") { DiffractionExperiment x(DetJF(1)); IndexingSettings idx; idx.Algorithm(IndexingAlgorithmEnum::FFT); @@ -151,8 +151,8 @@ TEST_CASE("JFJochProcessCommandLine_Full", "[process]") { config.rotation_indexing_image_count = 30; config.spot_finding = DiffractionExperiment::DefaultDataProcessingSettings(); - const std::string cmd = JFJochProcessCommandLine(config, x, "/data/lyso_master.h5"); - CHECK(cmd.rfind("jfjoch_process", 0) == 0); + const std::string cmd = RugnuxCommandLine(config, x, "/data/lyso_master.h5"); + CHECK(cmd.rfind("rugnux", 0) == 0); CHECK(cmd.find("-N 8") != std::string::npos); CHECK(cmd.find("-e 500") != std::string::npos); CHECK(cmd.find("-o run1") != std::string::npos); @@ -162,7 +162,7 @@ TEST_CASE("JFJochProcessCommandLine_Full", "[process]") { CHECK(cmd.find("/data/lyso_master.h5") != std::string::npos); } -TEST_CASE("JFJochProcessCommandLine_AzInt", "[process]") { +TEST_CASE("RugnuxCommandLine_AzInt", "[process]") { DiffractionExperiment x(DetJF(1)); AzimuthalIntegrationSettings a; a.AzimuthalBinCount(4); @@ -173,7 +173,7 @@ TEST_CASE("JFJochProcessCommandLine_AzInt", "[process]") { config.nthreads = 2; config.output_prefix = "az"; - const std::string cmd = JFJochProcessCommandLine(config, x, "in.h5"); + const std::string cmd = RugnuxCommandLine(config, x, "in.h5"); CHECK(cmd.rfind("jfjoch_azint", 0) == 0); CHECK(cmd.find("--azimuthal-bins 4") != std::string::npos); CHECK(cmd.find("--min-q") != std::string::npos); diff --git a/tests/data/README.md b/tests/data/README.md index ccf842d5..87b06756 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -23,7 +23,7 @@ git lfs pull # or: git lfs pull --include "tests/data/*.h5" | `lyso_rotation_master.h5` | lysozyme rotation series (~1800 images) | yes (LFS) | `lyso_rotation_master.h5` (plus its `_data_NNNNNN.h5` files) is fetched by `git lfs pull` and -drives `JFJochProcess_LysoRotation`. A separate serial dataset is intentionally **not** shipped +drives `Rugnux_LysoRotation`. A separate serial dataset is intentionally **not** shipped to keep the repository small — the rotation series can be run in serial mode (full analysis without rotation indexing) to exercise that path. To add your own dataset, drop the master + its data files here as real files (not symlinks, if you intend to commit them via LFS); the master diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 5eab33f1..8984eb0c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,16 +8,16 @@ ADD_EXECUTABLE(jfjoch_extract_hkl jfjoch_extract_hkl.cpp TARGET_LINK_LIBRARIES(jfjoch_extract_hkl JFJochReader) INSTALL(TARGETS jfjoch_extract_hkl RUNTIME COMPONENT viewer) -ADD_EXECUTABLE(jfjoch_process jfjoch_process.cpp) -TARGET_LINK_LIBRARIES(jfjoch_process JFJochProcess JFJochReader JFJochImageAnalysis JFJochWriter) -INSTALL(TARGETS jfjoch_process RUNTIME COMPONENT viewer) +ADD_EXECUTABLE(rugnux rugnux_cli.cpp) +TARGET_LINK_LIBRARIES(rugnux Rugnux JFJochReader JFJochImageAnalysis JFJochWriter) +INSTALL(TARGETS rugnux RUNTIME COMPONENT viewer) -ADD_EXECUTABLE(jfjoch_scale jfjoch_scale.cpp) -TARGET_LINK_LIBRARIES(jfjoch_scale JFJochReader JFJochImageAnalysis JFJochWriter) -INSTALL(TARGETS jfjoch_scale RUNTIME COMPONENT viewer) +ADD_EXECUTABLE(rugnux_scale rugnux_scale.cpp) +TARGET_LINK_LIBRARIES(rugnux_scale JFJochReader JFJochImageAnalysis JFJochWriter) +INSTALL(TARGETS rugnux_scale RUNTIME COMPONENT viewer) ADD_EXECUTABLE(jfjoch_azint jfjoch_azint.cpp) -TARGET_LINK_LIBRARIES(jfjoch_azint JFJochProcess JFJochReader JFJochImageAnalysis JFJochWriter) +TARGET_LINK_LIBRARIES(jfjoch_azint Rugnux JFJochReader JFJochImageAnalysis JFJochWriter) INSTALL(TARGETS jfjoch_azint RUNTIME COMPONENT viewer) # In-place re-compress /entry/data/data of a _data file from bitshuffle/LZ4 to bitshuffle/zstd. @@ -28,8 +28,8 @@ INSTALL(TARGETS jfjoch_recompress RUNTIME COMPONENT viewer) # On Windows these CLIs get getopt/getopt_long from the vendored wingetopt shim (libc has none). IF (WIN32) TARGET_LINK_LIBRARIES(jfjoch_extract_hkl wingetopt) - TARGET_LINK_LIBRARIES(jfjoch_process wingetopt) - TARGET_LINK_LIBRARIES(jfjoch_scale wingetopt) + TARGET_LINK_LIBRARIES(rugnux wingetopt) + TARGET_LINK_LIBRARIES(rugnux_scale wingetopt) TARGET_LINK_LIBRARIES(jfjoch_azint wingetopt) ENDIF() diff --git a/tools/jfjoch_azint.cpp b/tools/jfjoch_azint.cpp index 87cdf564..c525a6ac 100644 --- a/tools/jfjoch_azint.cpp +++ b/tools/jfjoch_azint.cpp @@ -15,7 +15,7 @@ #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" #include "../common/print_license.h" -#include "../process/JFJochProcess.h" +#include "../rugnux/Rugnux.h" void print_usage() { std::cout << "Usage ./jfjoch_azint {} " << std::endl; @@ -105,7 +105,7 @@ bool parse_on_off(const char *arg, bool &out) { } namespace { - std::atomic g_active_process{nullptr}; + std::atomic g_active_process{nullptr}; void handle_sigint(int) { if (auto *p = g_active_process.load()) p->Cancel(); @@ -232,7 +232,7 @@ int main(int argc, char **argv) { logger.Info("Loaded dataset from {}", input_file); // 2. Build experiment: defaults from the input file, overridden by command line. Output and - // runtime invariants are set inside JFJochProcess; here we only configure the geometry and + // runtime invariants are set inside Rugnux; here we only configure the geometry and // azimuthal-integration settings. DiffractionExperiment experiment(dataset->experiment); if (beam_x.has_value()) experiment.BeamX_pxl(beam_x.value()); @@ -273,7 +273,7 @@ int main(int argc, char **argv) { config.nthreads = nthreads; config.output_prefix = output_prefix; - JFJochProcess process(reader, experiment, dataset->pixel_mask, config); + Rugnux process(reader, experiment, dataset->pixel_mask, config); g_active_process = &process; std::signal(SIGINT, handle_sigint); diff --git a/tools/jfjoch_process.cpp b/tools/rugnux_cli.cpp similarity index 98% rename from tools/jfjoch_process.cpp rename to tools/rugnux_cli.cpp index c4054e2f..bcddb708 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/rugnux_cli.cpp @@ -16,7 +16,7 @@ #include "../common/PixelMask.h" #include "../common/print_license.h" #include "../image_analysis/LoadFCalcFromMtz.h" -#include "../process/JFJochProcess.h" +#include "../rugnux/Rugnux.h" // Default rot3d per-frame scale-G smoothing range (XDS DELPHI-like), in degrees of rotation. constexpr double SMOOTH_G_DEFAULT_DEG = 5.0; @@ -26,7 +26,7 @@ constexpr double SMOOTH_G_DEFAULT_DEG = 5.0; constexpr double REJECT_OUTLIERS_DEFAULT_NSIGMA = 6.0; void print_usage() { - std::cout << "Usage jfjoch_process {} " << std::endl; + std::cout << "Usage rugnux {} " << std::endl; std::cout << "Options:" << std::endl; std::cout << " -o, --output-prefix Output file prefix (default: output)" << std::endl; std::cout << " -N, --threads Number of threads (default: 1)" << std::endl; @@ -58,7 +58,7 @@ void print_usage() { std::cout << std::endl; std::cout << " Scaling and merging" << std::endl; - std::cout << " -M, --scale-merge Scale and merge (refine mosaicity) and write scaled.hkl + image.dat" << std::endl; + std::cout << " -M, --scale-merge Scale and merge (refine mosaicity); automatic for rotation data" << std::endl; std::cout << " --scale-fulls After -P rot3d combine, refit a per-frame scale on the fulls (XDS order, Unity model); implies -M. Default ON for rot3d" << std::endl; std::cout << " --no-scale-fulls Disable the rot3d scale-fulls refit (it is on by default for rot3d)" << std::endl; std::cout << " --write-process-h5 Also write the (large) _process.h5 when merging (default: only .mtz/.cif when merging)" << std::endl; @@ -273,7 +273,7 @@ std::optional parse_lattice_arg(const char *arg) { } namespace { - std::atomic g_active_process{nullptr}; + std::atomic g_active_process{nullptr}; void handle_sigint(int) { if (auto *p = g_active_process.load()) p->Cancel(); @@ -289,9 +289,9 @@ int main(int argc, char **argv) { RegisterHDF5Filter(); - print_license("jfjoch_process"); + print_license("rugnux"); - Logger logger("jfjoch_process"); + Logger logger("rugnux"); std::string input_file; std::string output_prefix = "output"; @@ -770,6 +770,11 @@ int main(int argc, char **argv) { "indexing). Use --force-still to treat it as stills."); } + // Rotation data is scaled and merged automatically (as if -M were given); stills still require + // an explicit -M. --force-still keeps rotation_indexing false, so stills stay opt-in. + if (rotation_indexing) + run_scaling = true; + // Configure Indexing IndexingSettings indexing_settings; indexing_settings.Algorithm(indexing_algorithm); @@ -839,7 +844,7 @@ int main(int argc, char **argv) { // Azimuthal integration (default q-spacing 0.01 1/A, from AzimuthalIntegrationSettings): the profile // resolves the narrow ice rings for the ice-ring score. -q / --azim-* override. Applied before - // JFJochProcess builds the azint mapping from the experiment. + // Rugnux builds the azint mapping from the experiment. { AzimuthalIntegrationSettings azint_settings = experiment.GetAzimuthalIntegrationSettings(); if (min_q || max_q) @@ -866,7 +871,7 @@ int main(int argc, char **argv) { spot_settings.high_resolution_limit = d_min_spot_finding; // Run the shared full-analysis workflow (rotation indexing + scaling/merging live in - // JFJochProcess; the experiment above carries all algorithm settings). + // Rugnux; the experiment above carries all algorithm settings). ProcessConfig config; config.mode = ProcessMode::FullAnalysis; config.start_image = start_image; @@ -888,7 +893,7 @@ int main(int argc, char **argv) { // _process.h5 unless explicitly requested. Without merging, the _process.h5 is the only output. config.write_process_h5 = run_scaling ? write_process_h5_flag : true; - JFJochProcess process(reader, experiment, dataset->pixel_mask, config); + Rugnux process(reader, experiment, dataset->pixel_mask, config); g_active_process = &process; std::signal(SIGINT, handle_sigint); diff --git a/tools/jfjoch_scale.cpp b/tools/rugnux_scale.cpp similarity index 98% rename from tools/jfjoch_scale.cpp rename to tools/rugnux_scale.cpp index 86191bdf..67a2a4ab 100644 --- a/tools/jfjoch_scale.cpp +++ b/tools/rugnux_scale.cpp @@ -28,7 +28,7 @@ #include "../image_analysis/UpdateReflectionResolution.h" void print_usage() { - std::cout << "Usage ./jfjoch_scale {} " << std::endl; + std::cout << "Usage ./rugnux_scale {} " << std::endl; std::cout << "Options:" << std::endl; std::cout << " -o, --output-prefix Output file prefix (default: output)" << std::endl; std::cout << " -N, --threads Number of threads (default: 1)" << std::endl; @@ -88,9 +88,9 @@ static option long_options[] = { int main(int argc, char **argv) { RegisterHDF5Filter(); - print_license("jfjoch_scale"); + print_license("rugnux_scale"); - Logger logger("jfjoch_scale"); + Logger logger("rugnux_scale"); std::string input_file; std::string output_prefix = "output"; @@ -369,13 +369,13 @@ int main(int argc, char **argv) { // Print resolution-shell statistics table std::cout << merged_statistics; - // Space-group determination lives in the full jfjoch_process pipeline, where the lattice search + // Space-group determination lives in the full rugnux pipeline, where the lattice search // and a consistent integration are available; this re-scaling tool only consumes a space group // (from the file's /entry/sample/space_group_number or -S) and merges in it. const bool fixed_space_group = space_group || experiment.GetGemmiSpaceGroup().has_value(); if (!fixed_space_group) logger.Warning("No space group in the input file or on the command line - merged in P1. " - "Re-run jfjoch_process (which determines and stores the space group) or pass " + "Re-run rugnux (which determines and stores the space group) or pass " "-S to scale and merge in the correct symmetry."); const auto twin_sg_number = experiment.GetSpaceGroupNumber(); diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 7a3cc391..40358381 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -110,7 +110,7 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::Concurrent Qt6::OpenGL Qt6::OpenGLWidgets - JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis JFJochProcess + JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis Rugnux fftw3f) # Native GUI app per platform: a .app bundle on macOS, GUI subsystem (no console window) on diff --git a/viewer/JFJochProcessController.cpp b/viewer/JFJochProcessController.cpp index ff705c2a..8b7c1016 100644 --- a/viewer/JFJochProcessController.cpp +++ b/viewer/JFJochProcessController.cpp @@ -57,7 +57,7 @@ void JFJochProcessController::run_(QString file_path, DiffractionExperiment expe last_live_emit_ = {}; } - JFJochProcess process(reader, std::move(experiment), std::move(pixel_mask), std::move(config)); + Rugnux process(reader, std::move(experiment), std::move(pixel_mask), std::move(config)); active_ = &process; if (cancel_pending_) process.Cancel(); diff --git a/viewer/JFJochProcessController.h b/viewer/JFJochProcessController.h index ff64ede0..03d16eef 100644 --- a/viewer/JFJochProcessController.h +++ b/viewer/JFJochProcessController.h @@ -12,19 +12,19 @@ #include #include -#include "../process/JFJochProcess.h" +#include "../rugnux/Rugnux.h" #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" #include "../reader/JFJochReaderDataset.h" Q_DECLARE_METATYPE(ProcessResult) -// Runs one JFJochProcess job off the GUI thread and reports back via queued Qt signals. The job +// Runs one Rugnux job off the GUI thread and reports back via queued Qt signals. The job // opens its own private JFJochHDF5Reader on the file (HDF5 access is globally serialized, so this // is safe alongside the interactive reader), so the viewer becomes a processing frontend without -// blocking the UI. Cancel() is forwarded to JFJochProcess::Cancel() (atomic) and works from any +// blocking the UI. Cancel() is forwarded to Rugnux::Cancel() (atomic) and works from any // thread / at any point of the run. -class JFJochProcessController : public QObject, private JFJochProcessObserver { +class JFJochProcessController : public QObject, private RugnuxObserver { Q_OBJECT public: explicit JFJochProcessController(QObject *parent = nullptr); @@ -49,7 +49,7 @@ signals: void liveDataset(std::shared_ptr dataset); private: - // JFJochProcessObserver - called from worker threads, forwarded as queued signals. + // RugnuxObserver - called from worker threads, forwarded as queued signals. void OnPhase(const std::string &phase) override; void OnProgress(uint64_t done, uint64_t total) override; void OnImageProcessed(const DataMessage &msg) override; @@ -58,7 +58,7 @@ private: void joinWorker_(); std::thread worker_; - std::atomic active_{nullptr}; + std::atomic active_{nullptr}; std::atomic running_{false}; std::atomic cancel_pending_{false}; diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index a6547f5f..de057c2b 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -417,7 +417,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString connect(this, &JFJochViewerWindow::setAutoForeground, viewer, &JFJochDiffractionImage::setAutoForeground); - // Processing jobs: run jfjoch_process locally / copy a cluster command line, and surface + // Processing jobs: run rugnux locally / copy a cluster command line, and surface // finished runs as switchable dataset snapshots. connect(processingJobsWindow, &JFJochProcessingJobsWindow::registerSnapshot, reading_worker, &JFJochImageReadingWorker::RegisterProcessingSnapshot); diff --git a/viewer/windows/JFJochProcessingJobsWindow.cpp b/viewer/windows/JFJochProcessingJobsWindow.cpp index 648919c5..f6e56d2e 100644 --- a/viewer/windows/JFJochProcessingJobsWindow.cpp +++ b/viewer/windows/JFJochProcessingJobsWindow.cpp @@ -4,7 +4,7 @@ #include "JFJochProcessingJobsWindow.h" #include "JFJochMergeStatsWindow.h" #include "../widgets/ToolbarIcons.h" -#include "../../process/JFJochProcessCommandLine.h" +#include "../../rugnux/RugnuxCommandLine.h" #include #include @@ -230,7 +230,7 @@ void JFJochProcessingJobsWindow::newJob(bool azint) { if (action == 2) { // copy command line const QString cmd = QString::fromStdString( - JFJochProcessCommandLine(config, experiment, inputs.file.toStdString())); + RugnuxCommandLine(config, experiment, inputs.file.toStdString())); QApplication::clipboard()->setText(cmd); QMessageBox::information(this, "Command line", cmd + "\n\n(copied to clipboard)"); return; diff --git a/viewer/windows/JFJochProcessingJobsWindow.h b/viewer/windows/JFJochProcessingJobsWindow.h index 1994b916..6857fdd0 100644 --- a/viewer/windows/JFJochProcessingJobsWindow.h +++ b/viewer/windows/JFJochProcessingJobsWindow.h @@ -19,7 +19,7 @@ class QToolButton; // Makes processing a first-class GUI activity: a dockable panel with a table of processing jobs run // on the current dataset. A job can be run locally (off the GUI thread, via JFJochProcessController) -// or its jfjoch_process command line copied for a cluster. A finished local run is registered as a +// or its rugnux command line copied for a cluster. A finished local run is registered as a // reader snapshot so its results become a selectable view of the dataset. Re-processing reads a // stored HDF5 file, so the panel shows an explanatory message while connected to a live HTTP stream. class JFJochProcessingJobsWindow : public QWidget { -- 2.54.0 From 01e4f81e4430e5b34d6e3ca19daf5b88f82797c9 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 8 Jul 2026 14:09:15 +0200 Subject: [PATCH 03/41] Consolidate offline tools into rugnux; merge on by default Fold jfjoch_azint and rugnux_scale into the single rugnux CLI, and flip scaling/merging from opt-in to on-by-default. - rugnux --azint-only replaces jfjoch_azint (reuses ProcessMode:: AzimuthalIntegration; adds the correction toggles and geometry overrides the old tool carried). - rugnux --scale replaces rugnux_scale (re-scale/merge stored reflections, same simpler scale/merge as before: no space-group search or rot3d defaults). Its workflow is folded into the CLI verbatim. - Merging is now on by default for rotation and stills; --no-merge disables it, replacing -M/--scale-merge. Also fix the batch ReadReflections reader so it works on rugnux's own NXmxIntegrated _process.h5: read per-image reflections/mosaicity/lattice from the master (global index) first, falling back to the source-data locator for legacy/VDS datasets. Without this, both --scale and the former rugnux_scale read zero reflections from an integrated snapshot. RugnuxCommandLine now emits `rugnux --azint-only` and `--no-merge`; docs and tests updated to match. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 6 +- CMakeLists.txt | 2 +- docs/CHANGELOG.md | 1 + docs/RUGNUX.md | 36 ++-- docs/TOOLS.md | 49 +---- reader/HDF5MetadataSource.cpp | 19 +- rugnux/Rugnux.h | 6 +- rugnux/RugnuxCommandLine.cpp | 16 +- rugnux/RugnuxCommandLine.h | 2 +- tests/RugnuxTest.cpp | 7 +- tools/CMakeLists.txt | 12 +- tools/jfjoch_azint.cpp | 296 -------------------------- tools/rugnux_cli.cpp | 327 +++++++++++++++++++++++++--- tools/rugnux_scale.cpp | 390 ---------------------------------- 14 files changed, 359 insertions(+), 810 deletions(-) delete mode 100644 tools/jfjoch_azint.cpp delete mode 100644 tools/rugnux_scale.cpp diff --git a/CLAUDE.md b/CLAUDE.md index a8754e28..c331f080 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -123,8 +123,10 @@ generated API model and internal types. - `jfjoch_broker` — online, real-time (FPGA + GPU). - `jfjoch_viewer` — interactive Qt desktop (`viewer/`), results not persisted. - `rugnux` (`tools/rugnux_cli.cpp`, built on the `Rugnux` library in `rugnux/`) — offline batch over - a stored HDF5; writes `_process.h5` and `.mtz`/`.cif`/`.hkl`. `rugnux_scale` re-scales/merges - already-integrated data. (rugnux = the data-processing half of the system; see `docs/NAMING.md`.) + a stored HDF5; writes `_process.h5` and `.mtz`/`.cif`/`.hkl`. Merging is on by default (`--no-merge` + to disable); `--azint-only` runs only azimuthal integration and `--scale` re-scales/merges the + already-integrated reflections in a `_process.h5`. (rugnux = the data-processing half of the system; + see `docs/NAMING.md`.) **`image_analysis/` pipeline** (subdirs): `spot_finding`, `indexing` (`ffbidx`/`fft` GPU, `fftw` CPU), `lattice_search`, `geom_refinement`, `pixel_refinement`, `bragg_prediction`, diff --git a/CMakeLists.txt b/CMakeLists.txt index cd78188b..b4fc549a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -279,7 +279,7 @@ IF (JFJOCH_VIEWER_ONLY) ADD_SUBDIRECTORY(reader) ADD_SUBDIRECTORY(rugnux) ADD_SUBDIRECTORY(viewer) - ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/rugnux_scale/azint/extract_hkl) + ADD_SUBDIRECTORY(tools) # builds only the portable analysis tools (rugnux/extract_hkl) ELSE() ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b85f78d2..ecb7f121 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,6 +3,7 @@ ### 1.0.0-rc.156 This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. +* rugnux: Consolidate the offline analysis tools into one `rugnux` binary - the former `jfjoch_azint` is now `rugnux --azint-only` and `rugnux_scale` is now `rugnux --scale`. Scaling and merging are on by default (`--no-merge` disables them), replacing the previous opt-in `-M, --scale-merge`. * jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output. * jfjoch_process: Bragg integration now runs on the GPU in the offline/non-FPGA workflow (one box-sum + profile-fit engine, GPU when available, CPU otherwise); the FPGA workflow integrates on the CPU directly from the assembled image. The previous standalone integrators are removed. * jfjoch_process: Deterministic Bragg prediction - when more reflections are predicted than fit the output, they are ranked by distance to the Ewald sphere before truncation, so repeated runs produce identical reflections. diff --git a/docs/RUGNUX.md b/docs/RUGNUX.md index 7dddc771..494702fa 100644 --- a/docs/RUGNUX.md +++ b/docs/RUGNUX.md @@ -42,9 +42,10 @@ the first pass. - `_process.h5` — NXmx-compliant HDF5 with derived metadata (spots, indexing, integration, azimuthal integration, per-image statistics). See - [HDF5 / NeXus data format](HDF5.md) for the layout. -- When merging (`-M`, or whenever a `--reference-mtz` is supplied), the merged reflections are - written as `.cif` (mmCIF — the default), or `.mtz` / `.hkl` depending on + [HDF5 / NeXus data format](HDF5.md) for the layout. Written by default only when **not** merging + (i.e. under `--no-merge`); add `--write-process-h5` to also write it when merging. +- Merging is **on by default** (`--no-merge` disables it). The merged reflections are written as + `.cif` (mmCIF — the default), or `.mtz` / `.hkl` depending on `--scaling-output`. Both the mmCIF and the MTZ carry the **refined unit cell** (from rotation indexing) and the **space group determined from systematic absences** (constrained to the indexed lattice symmetry). No-reference scaling additionally emits per-iteration `_iterN_scale.dat`. @@ -52,12 +53,13 @@ the first pass. Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. -## Re-scaling and re-merging (`rugnux_scale`) +## Re-scaling and re-merging (`rugnux --scale`) -The companion tool `rugnux_scale` re-scales and merges the *already-integrated* reflections stored -in one or more `_process.h5` files, without re-running spot finding or integration. Use it to -re-merge quickly with a different space group, resolution limit, anomalous setting or reference MTZ, -or to combine several processed runs into one set of merged intensities. +The `--scale` mode re-scales and merges the *already-integrated* reflections stored in a +`_process.h5` file, without re-running spot finding or integration. Use it to re-merge quickly with a +different space group, resolution limit, anomalous setting or reference MTZ. It reuses the same +`-o/-N/-s/-e/-S/-A/-B/-z/--scaling-*` options as the full run, and (unlike the full pipeline) does +not run a space-group search, so pass `-S` for the correct symmetry. ## Quick start @@ -73,9 +75,10 @@ rugnux rotation_master.h5 \ Because the dataset carries a rotation goniometer axis, it is processed as **rotation data by default**: two-pass rotation indexing (index the sweep once, then process every frame against that -lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). Rotation -data is **scaled and merged automatically** (as if `-M` were given); the unit cell is taken from the -rotation indexer and the space group is determined from systematic absences, and both are written +lattice) with the **`rot3d`** partiality model (rotation partials combined into 3D fulls). Scaling +and merging run **by default** (for both rotation and stills; `--no-merge` turns them off); the unit +cell is taken from the rotation indexer and the space group is determined from systematic absences, +and both are written into the merged `.cif`. Run **fully de novo** (no `-C`/`-S`) for the best result — supplying a cell or space group up front @@ -95,7 +98,7 @@ rugnux serial_master.h5 \ -o lyso_serial -N 32 \ -X ffbidx -C 79,79,38,90,90,90 -S 96 \ --spot-sigma 4 \ - -M -z reference.mtz \ + -z reference.mtz \ --scaling-high-resolution 1.8 ``` @@ -117,6 +120,13 @@ General: | `-t, --stride ` | Process every *n*-th image (default: 1) | | `-v, --verbose` | Verbose output | +Modes (default: full analysis — spot finding, indexing, integration and merging): + +| Option | Description | +| --- | --- | +| `--azint-only` | Only run azimuthal integration (no spot finding/indexing); writes `_process.h5` | +| `--scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) | + Spot finding: | Option | Description | @@ -163,7 +173,7 @@ Scaling and merging: | Option | Description | | --- | --- | -| `-M, --scale-merge` | Scale and merge (automatic for rotation data; only needed to force scaling on stills) | +| `--no-merge` | Skip scaling and merging (on by default); write only the per-image `_process.h5` | | `-A, --anomalous` | Anomalous mode (keep Friedel pairs separate) | | `-B, --refine-bfactor` | Refine a per-image B-factor | | `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 345ed34e..d9f74dcf 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -9,12 +9,10 @@ number of command-line tools. Each prints its own usage when run with `-h` or wi ### rugnux Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, integration, scaling/merging) on a stored HDF5 dataset, producing a `_process.h5` file and, when -merging, reflection files. See [rugnux](RUGNUX.md). - -### rugnux_scale -Re-scales and merges the already-integrated reflections from one or more `_process.h5` files -(no re-integration). Useful to re-merge with a different space group, partiality, resolution limit -or reference MTZ, or to combine several runs. See [rugnux](RUGNUX.md). +merging, reflection files. Merging is on by default (`--no-merge` disables it). Two extra modes +narrow the work: `--azint-only` runs only azimuthal integration (no spot finding/indexing), and +`--scale` re-scales/merges the already-integrated reflections in a `_process.h5` without +re-integrating. See [rugnux](RUGNUX.md). ### jfjoch_extract_hkl Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across @@ -66,41 +64,4 @@ writing. Tests single-threaded HDF5 writer performance. ### jfjoch_simplon_test -Minimal test client for a DECTRIS SIMPLON detector API. - -### jfjoch_azint - -Runs CPU azimuthal integration on an existing Jungfraujoch HDF5 file using `N` threads and writes the -result to `_process.h5`. Example: -``` -./jfjoch_azint -o output -N 8 input.h5 -``` - -The integration settings and geometry default to the values stored in the input file; any of them can be -overridden on the command line: -``` -jfjoch_azint {} - -o, --output-prefix Output file prefix (default: output) - -N, --threads Number of threads (default: 1) - -s, --start-image Start image number (default: 0) - -e, --end-image End image number (default: all) - -t, --stride Image stride (default: 1) - -v, --verbose Verbose output - - Azimuthal integration: - --min-q Minimum Q for integration (1/A) - --max-q Maximum Q for integration (1/A) - --q-spacing Q bin spacing (1/A) - --azimuthal-bins Number of azimuthal bins (default: 1) - --polarization-correction Enable/disable polarization correction - --solid-angle-correction Enable/disable solid angle correction - - Geometry overrides: - --beam-x Beam center X (pixel) - --beam-y Beam center Y (pixel) - --detector-distance Detector distance (mm) - --wavelength Wavelength (A) - --rot1 PONI rotation 1 (rad) - --rot2 PONI rotation 2 (rad) - --polarization Polarization factor -``` \ No newline at end of file +Minimal test client for a DECTRIS SIMPLON detector API. \ No newline at end of file diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index 811ab091..f5b8af3a 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -1102,14 +1102,21 @@ std::vector HDF5MetadataSource::ReadReflections(size_t start // Generic (non-image-specific) detector geometry from experiment setup. outcome.geom = cached_geom; - // Per-image reflections and MX metadata live in the same file as the image pixels, - // at the source-local index (the locator keeps the data-file handle cached). - const auto loc = ResolveMeta(static_cast(img)); - HDF5Object *meta_file = loc.file.get(); - const size_t meta_image_id = loc.local_index; + // Per-image reflections and MX metadata are stored in this master at the global index for a + // self-contained integrated _process.h5 snapshot, or co-located with the pixels in the source + // data file at the source-local index for a legacy/VDS dataset. Prefer the master (so an + // integrated snapshot reads without its linked source data present); fall back to the source. + HDF5ReadOnlyFile *meta_file = master_file.get(); + size_t meta_image_id = img; + std::string refl_group = fmt::format("/entry/reflections/image_{:06d}", img); + if (!master_file->Exists(refl_group)) { + const auto loc = ResolveMeta(static_cast(img)); + meta_file = loc.file.get(); + meta_image_id = loc.local_index; + refl_group = fmt::format("/entry/reflections/image_{:06d}", meta_image_id); + } // ── reflections ────────────────────────────────────────────────────── - const std::string refl_group = fmt::format("/entry/reflections/image_{:06d}", meta_image_id); ReadReflectionsFromGroup(*meta_file, refl_group, outcome.reflections); // ── per-image mosaicity ─────────────────────────────────────────────── diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 1ba2d77e..69e9a059 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -20,14 +20,14 @@ class JFJochHDF5Reader; -// Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux, -// jfjoch_azint and (later) the viewer. The full processing workflow lives here, not in the CLIs: +// Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux (including its +// --azint-only mode) and (later) the viewer. The full processing workflow lives here, not in the CLIs: // setup, an optional two-pass rotation-indexing pre-pass, a parallel per-image loop (std::thread), // an optional scaling/merging post-pass, and the _process.h5 output. The detector geometry and all // algorithm settings are configured on the DiffractionExperiment by the caller; ProcessConfig only // carries run control. Cancellable from any thread (e.g. SIGINT or a GUI button) via Cancel(). enum class ProcessMode { - AzimuthalIntegration, // preprocess + azimuthal integration only (jfjoch_azint) + AzimuthalIntegration, // preprocess + azimuthal integration only (rugnux --azint-only) FullAnalysis // spot finding + indexing + refinement + integration (rugnux) }; diff --git a/rugnux/RugnuxCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp index 0c7aa6bd..315bc4cd 100644 --- a/rugnux/RugnuxCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -53,7 +53,9 @@ std::string RugnuxCommandLine(const ProcessConfig &config, const std::string &input_file) { std::vector args; const bool azint = (config.mode == ProcessMode::AzimuthalIntegration); - args.emplace_back(azint ? "jfjoch_azint" : "rugnux"); + args.emplace_back("rugnux"); + if (azint) + args.emplace_back("--azint-only"); auto add = [&](const std::string &flag, const std::string &val) { args.push_back(flag); @@ -72,10 +74,10 @@ std::string RugnuxCommandLine(const ProcessConfig &config, if (azint) { const auto a = experiment.GetAzimuthalIntegrationSettings(); - add("--min-q", num(a.GetLowQ_recipA())); - add("--max-q", num(a.GetHighQ_recipA())); - add("--q-spacing", num(a.GetQSpacing_recipA())); - add("--azimuthal-bins", std::to_string(a.GetAzimuthalBinCount())); + add("--azim-min-q", num(a.GetLowQ_recipA())); + add("--azim-max-q", num(a.GetHighQ_recipA())); + add("--azim-q-spacing", num(a.GetQSpacing_recipA())); + add("--azim-phi-bins", std::to_string(a.GetAzimuthalBinCount())); add("--polarization-correction", a.IsPolarizationCorrection() ? "on" : "off"); add("--solid-angle-correction", a.IsSolidAngleCorrection() ? "on" : "off"); } else { @@ -116,13 +118,15 @@ std::string RugnuxCommandLine(const ProcessConfig &config, args.emplace_back("--force-still"); } + // Merging is on by default; emit --no-merge only when it was turned off. if (config.run_scaling) { - args.emplace_back("-M"); const auto sc = experiment.GetScalingSettings(); if (!sc.GetMergeFriedel()) args.emplace_back("-A"); if (sc.GetRefineB()) args.emplace_back("-B"); + } else { + args.emplace_back("--no-merge"); } } diff --git a/rugnux/RugnuxCommandLine.h b/rugnux/RugnuxCommandLine.h index 197198f6..b8362f70 100644 --- a/rugnux/RugnuxCommandLine.h +++ b/rugnux/RugnuxCommandLine.h @@ -9,7 +9,7 @@ class DiffractionExperiment; -// Reconstruct an equivalent rugnux / jfjoch_azint command line for a configured run, so a +// Reconstruct an equivalent rugnux command line (including --azint-only) for a configured run, so a // job set up in the GUI can be handed off to a cluster. Covers the settings that matter for the // run, not every obscure flag; geometry is taken from the input file, so geometry overrides are // not emitted. diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index f5bf5620..553810fb 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -174,8 +174,9 @@ TEST_CASE("RugnuxCommandLine_AzInt", "[process]") { config.output_prefix = "az"; const std::string cmd = RugnuxCommandLine(config, x, "in.h5"); - CHECK(cmd.rfind("jfjoch_azint", 0) == 0); - CHECK(cmd.find("--azimuthal-bins 4") != std::string::npos); - CHECK(cmd.find("--min-q") != std::string::npos); + CHECK(cmd.rfind("rugnux", 0) == 0); + CHECK(cmd.find("--azint-only") != std::string::npos); + CHECK(cmd.find("--azim-phi-bins 4") != std::string::npos); + CHECK(cmd.find("--azim-min-q") != std::string::npos); CHECK(cmd.find("in.h5") != std::string::npos); } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 8984eb0c..4194ea3a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,18 +8,12 @@ ADD_EXECUTABLE(jfjoch_extract_hkl jfjoch_extract_hkl.cpp TARGET_LINK_LIBRARIES(jfjoch_extract_hkl JFJochReader) INSTALL(TARGETS jfjoch_extract_hkl RUNTIME COMPONENT viewer) +# rugnux is the single offline analysis CLI: full pipeline by default, plus --azint-only +# (azimuthal integration) and --scale (re-scale/merge stored reflections) modes. ADD_EXECUTABLE(rugnux rugnux_cli.cpp) TARGET_LINK_LIBRARIES(rugnux Rugnux JFJochReader JFJochImageAnalysis JFJochWriter) INSTALL(TARGETS rugnux RUNTIME COMPONENT viewer) -ADD_EXECUTABLE(rugnux_scale rugnux_scale.cpp) -TARGET_LINK_LIBRARIES(rugnux_scale JFJochReader JFJochImageAnalysis JFJochWriter) -INSTALL(TARGETS rugnux_scale RUNTIME COMPONENT viewer) - -ADD_EXECUTABLE(jfjoch_azint jfjoch_azint.cpp) -TARGET_LINK_LIBRARIES(jfjoch_azint Rugnux JFJochReader JFJochImageAnalysis JFJochWriter) -INSTALL(TARGETS jfjoch_azint RUNTIME COMPONENT viewer) - # In-place re-compress /entry/data/data of a _data file from bitshuffle/LZ4 to bitshuffle/zstd. ADD_EXECUTABLE(jfjoch_recompress jfjoch_recompress.cpp) TARGET_LINK_LIBRARIES(jfjoch_recompress JFJochHDF5Wrappers) @@ -29,8 +23,6 @@ INSTALL(TARGETS jfjoch_recompress RUNTIME COMPONENT viewer) IF (WIN32) TARGET_LINK_LIBRARIES(jfjoch_extract_hkl wingetopt) TARGET_LINK_LIBRARIES(rugnux wingetopt) - TARGET_LINK_LIBRARIES(rugnux_scale wingetopt) - TARGET_LINK_LIBRARIES(jfjoch_azint wingetopt) ENDIF() # Online / hardware tools (broker, FPGA, receiver, detector). They link the non-portable diff --git a/tools/jfjoch_azint.cpp b/tools/jfjoch_azint.cpp deleted file mode 100644 index c525a6ac..00000000 --- a/tools/jfjoch_azint.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include -#include -#include -#include -#include -#include -#include - -#include "../reader/JFJochHDF5Reader.h" -#include "../common/Logger.h" -#include "../common/Definitions.h" -#include "../common/DiffractionExperiment.h" -#include "../common/PixelMask.h" -#include "../common/print_license.h" -#include "../rugnux/Rugnux.h" - -void print_usage() { - std::cout << "Usage ./jfjoch_azint {} " << std::endl; - std::cout << "Runs CPU azimuthal integration on a Jungfraujoch HDF5 file and writes _process.h5" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << " -o, --output-prefix Output file prefix (default: output)" << std::endl; - std::cout << " -N, --threads Number of threads (default: 1)" << std::endl; - std::cout << " -s, --start-image Start image number (default: 0)" << std::endl; - std::cout << " -e, --end-image End image number (default: all)" << std::endl; - std::cout << " -t, --stride Image stride (default: 1)" << std::endl; - std::cout << " -v, --verbose Verbose output" << std::endl; - std::cout << std::endl; - - std::cout << " Azimuthal integration (defaults taken from the input file)" << std::endl; - std::cout << " --min-q Minimum Q for integration (1/A)" << std::endl; - std::cout << " --max-q Maximum Q for integration (1/A)" << std::endl; - std::cout << " --q-spacing Q bin spacing (1/A)" << std::endl; - std::cout << " --azimuthal-bins Number of azimuthal bins (default: 1)" << std::endl; - std::cout << " --polarization-correction Enable/disable polarization correction" << std::endl; - std::cout << " --solid-angle-correction Enable/disable solid angle correction" << std::endl; - std::cout << std::endl; - - std::cout << " Geometry overrides (defaults taken from the input file)" << std::endl; - std::cout << " --beam-x Beam center X (pixel)" << std::endl; - std::cout << " --beam-y Beam center Y (pixel)" << std::endl; - std::cout << " --detector-distance Detector distance (mm)" << std::endl; - std::cout << " --wavelength Wavelength (A)" << std::endl; - std::cout << " --rot1 PONI rotation 1 (rad)" << std::endl; - std::cout << " --rot2 PONI rotation 2 (rad)" << std::endl; - std::cout << " --polarization Polarization factor" << std::endl; -} - -enum { - OPT_MIN_Q = 1000, - OPT_MAX_Q, - OPT_Q_SPACING, - OPT_AZIMUTHAL_BINS, - OPT_POLARIZATION_CORRECTION, - OPT_SOLID_ANGLE_CORRECTION, - OPT_BEAM_X, - OPT_BEAM_Y, - OPT_DETECTOR_DISTANCE, - OPT_WAVELENGTH, - OPT_ROT1, - OPT_ROT2, - OPT_POLARIZATION -}; - -static option long_options[] = { - {"verbose", no_argument, nullptr, 'v'}, - {"output-prefix", required_argument, nullptr, 'o'}, - {"threads", required_argument, nullptr, 'N'}, - {"start-image", required_argument, nullptr, 's'}, - {"end-image", required_argument, nullptr, 'e'}, - {"stride", required_argument, nullptr, 't'}, - - {"min-q", required_argument, nullptr, OPT_MIN_Q}, - {"max-q", required_argument, nullptr, OPT_MAX_Q}, - {"q-spacing", required_argument, nullptr, OPT_Q_SPACING}, - {"azimuthal-bins", required_argument, nullptr, OPT_AZIMUTHAL_BINS}, - {"polarization-correction", required_argument, nullptr, OPT_POLARIZATION_CORRECTION}, - {"solid-angle-correction", required_argument, nullptr, OPT_SOLID_ANGLE_CORRECTION}, - - {"beam-x", required_argument, nullptr, OPT_BEAM_X}, - {"beam-y", required_argument, nullptr, OPT_BEAM_Y}, - {"detector-distance", required_argument, nullptr, OPT_DETECTOR_DISTANCE}, - {"wavelength", required_argument, nullptr, OPT_WAVELENGTH}, - {"rot1", required_argument, nullptr, OPT_ROT1}, - {"rot2", required_argument, nullptr, OPT_ROT2}, - {"polarization", required_argument, nullptr, OPT_POLARIZATION}, - {nullptr, 0, nullptr, 0} -}; - -bool parse_on_off(const char *arg, bool &out) { - std::string s = arg ? arg : ""; - std::transform(s.begin(), s.end(), s.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (s == "on" || s == "1" || s == "true" || s == "yes") { - out = true; - return true; - } - if (s == "off" || s == "0" || s == "false" || s == "no") { - out = false; - return true; - } - return false; -} - -namespace { - std::atomic g_active_process{nullptr}; - void handle_sigint(int) { - if (auto *p = g_active_process.load()) - p->Cancel(); - } -} - -int main(int argc, char **argv) { - for (int i = 0; i < argc; i++) - std::cout << argv[i] << " "; - std::cout << std::endl << std::endl; - - RegisterHDF5Filter(); - print_license("jfjoch_azint"); - - Logger logger("jfjoch_azint"); - - std::string output_prefix = "output"; - int nthreads = 1; - int start_image = 0; - int end_image = -1; // -1 indicates process until end - int image_stride = 1; - bool verbose = false; - - // Azimuthal integration overrides (default: keep value from input file) - std::optional min_q; - std::optional max_q; - std::optional q_spacing; - std::optional azimuthal_bins; - std::optional polarization_correction; - std::optional solid_angle_correction; - - // Geometry overrides (default: keep value from input file) - std::optional beam_x; - std::optional beam_y; - std::optional detector_distance_mm; - std::optional wavelength_A; - std::optional rot1_rad; - std::optional rot2_rad; - std::optional polarization_factor; - - if (argc == 1) { - print_usage(); - exit(EXIT_FAILURE); - } - - int opt; - int option_index = 0; - const char *short_opts = "vo:N:s:e:t:"; - - while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { - switch (opt) { - case 'o': output_prefix = optarg; break; - case 'v': verbose = true; break; - case 'N': nthreads = atoi(optarg); break; - case 's': start_image = atoi(optarg); break; - case 'e': end_image = atoi(optarg); break; - case 't': image_stride = atoi(optarg); break; - - case OPT_MIN_Q: min_q = atof(optarg); break; - case OPT_MAX_Q: max_q = atof(optarg); break; - case OPT_Q_SPACING: q_spacing = atof(optarg); break; - case OPT_AZIMUTHAL_BINS: azimuthal_bins = atoi(optarg); break; - case OPT_POLARIZATION_CORRECTION: { - bool value; - if (!parse_on_off(optarg, value)) { - logger.Error("Invalid polarization correction value (expected on|off): {}", optarg); - exit(EXIT_FAILURE); - } - polarization_correction = value; - break; - } - case OPT_SOLID_ANGLE_CORRECTION: { - bool value; - if (!parse_on_off(optarg, value)) { - logger.Error("Invalid solid angle correction value (expected on|off): {}", optarg); - exit(EXIT_FAILURE); - } - solid_angle_correction = value; - break; - } - - case OPT_BEAM_X: beam_x = atof(optarg); break; - case OPT_BEAM_Y: beam_y = atof(optarg); break; - case OPT_DETECTOR_DISTANCE: detector_distance_mm = atof(optarg); break; - case OPT_WAVELENGTH: wavelength_A = atof(optarg); break; - case OPT_ROT1: rot1_rad = atof(optarg); break; - case OPT_ROT2: rot2_rad = atof(optarg); break; - case OPT_POLARIZATION: polarization_factor = atof(optarg); break; - - default: - print_usage(); - exit(EXIT_FAILURE); - } - } - - if (optind != argc - 1) { - logger.Error("Input file not specified"); - print_usage(); - exit(EXIT_FAILURE); - } - - const std::string input_file = argv[optind]; - logger.Verbose(verbose); - - if (image_stride <= 0) { - logger.Error("Image stride must be positive"); - exit(EXIT_FAILURE); - } - - // 1. Read input file - JFJochHDF5Reader reader; - try { - reader.ReadFile(input_file); - } catch (const std::exception &e) { - logger.Error("Error reading input file: {}", e.what()); - exit(EXIT_FAILURE); - } - - const auto dataset = reader.GetDataset(); - if (!dataset) { - logger.Error("No experiment dataset found in the input file"); - exit(EXIT_FAILURE); - } - logger.Info("Loaded dataset from {}", input_file); - - // 2. Build experiment: defaults from the input file, overridden by command line. Output and - // runtime invariants are set inside Rugnux; here we only configure the geometry and - // azimuthal-integration settings. - DiffractionExperiment experiment(dataset->experiment); - if (beam_x.has_value()) experiment.BeamX_pxl(beam_x.value()); - if (beam_y.has_value()) experiment.BeamY_pxl(beam_y.value()); - if (detector_distance_mm.has_value()) experiment.DetectorDistance_mm(detector_distance_mm.value()); - if (wavelength_A.has_value()) experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / wavelength_A.value()); - if (rot1_rad.has_value()) experiment.PoniRot1_rad(rot1_rad.value()); - if (rot2_rad.has_value()) experiment.PoniRot2_rad(rot2_rad.value()); - if (polarization_factor.has_value()) experiment.PolarizationFactor(polarization_factor.value()); - - AzimuthalIntegrationSettings azint_settings = experiment.GetAzimuthalIntegrationSettings(); - if (min_q.has_value() || max_q.has_value()) - azint_settings.QRange_recipA(min_q.value_or(azint_settings.GetLowQ_recipA()), - max_q.value_or(azint_settings.GetHighQ_recipA())); - if (q_spacing.has_value()) azint_settings.QSpacing_recipA(q_spacing.value()); - if (azimuthal_bins.has_value()) azint_settings.AzimuthalBinCount(azimuthal_bins.value()); - if (polarization_correction.has_value()) azint_settings.PolarizationCorrection(polarization_correction.value()); - if (solid_angle_correction.has_value()) azint_settings.SolidAngleCorrection(solid_angle_correction.value()); - experiment.ImportAzimuthalIntegrationSettings(azint_settings); - - logger.Info("Geometry: beam ({:.2f}, {:.2f}) pxl, distance {:.2f} mm, wavelength {:.5f} A", - experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), - experiment.GetDetectorDistance_mm(), experiment.GetWavelength_A()); - logger.Info("Azimuthal integration: Q range [{:.4f}, {:.4f}] 1/A, spacing {:.4f} 1/A, {} Q bins x {} azimuthal bins", - azint_settings.GetLowQ_recipA(), azint_settings.GetHighQ_recipA(), - azint_settings.GetQSpacing_recipA(), azint_settings.GetQBinCount(), - azint_settings.GetAzimuthalBinCount()); - logger.Info("Corrections: polarization {}, solid angle {}", - azint_settings.IsPolarizationCorrection() ? "on" : "off", - azint_settings.IsSolidAngleCorrection() ? "on" : "off"); - - // 3. Run the shared azimuthal-integration workflow. - ProcessConfig config; - config.mode = ProcessMode::AzimuthalIntegration; - config.start_image = start_image; - config.end_image = end_image; - config.stride = image_stride; - config.nthreads = nthreads; - config.output_prefix = output_prefix; - - Rugnux process(reader, experiment, dataset->pixel_mask, config); - - g_active_process = &process; - std::signal(SIGINT, handle_sigint); - - ProcessResult result; - try { - result = process.Run(); - } catch (const std::exception &e) { - logger.Error("Processing failed: {}", e.what()); - exit(EXIT_FAILURE); - } - g_active_process = nullptr; - - // 4. Report statistics - std::cout << fmt::format("Processing time: {:.2f} s", result.processing_time_s) << std::endl; - std::cout << fmt::format("Frame rate: {:.2f} Hz", result.frame_rate_hz) << std::endl; - std::cout << fmt::format("Total throughput: {:.2f} MB/s", result.throughput_MBs) << std::endl; - if (result.cancelled) - logger.Warning("Processing was cancelled after {} images", result.images_processed); -} diff --git a/tools/rugnux_cli.cpp b/tools/rugnux_cli.cpp index bcddb708..2f7448f3 100644 --- a/tools/rugnux_cli.cpp +++ b/tools/rugnux_cli.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -12,10 +13,17 @@ #include "../reader/JFJochHDF5Reader.h" #include "../common/Logger.h" +#include "../common/Definitions.h" #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" #include "../common/print_license.h" #include "../image_analysis/LoadFCalcFromMtz.h" +#include "../image_analysis/UpdateReflectionResolution.h" +#include "../image_analysis/WriteReflections.h" +#include "../image_analysis/scale_merge/Merge.h" +#include "../image_analysis/scale_merge/ScaleOnTheFly.h" +#include "../image_analysis/scale_merge/RotationScaleMerge.h" +#include "../image_analysis/scale_merge/TwinningAnalysis.h" #include "../rugnux/Rugnux.h" // Default rot3d per-frame scale-G smoothing range (XDS DELPHI-like), in degrees of rotation. @@ -36,6 +44,11 @@ void print_usage() { std::cout << " -v, --verbose Verbose output" << std::endl; std::cout << std::endl; + std::cout << " Modes (default: full analysis - spot finding, indexing, integration and merging)" << std::endl; + std::cout << " --azint-only Only run azimuthal integration (no spot finding/indexing); writes _process.h5" << std::endl; + std::cout << " --scale Only re-scale/merge the already-integrated reflections in (no re-integration)" << std::endl; + std::cout << std::endl; + std::cout << " Spot finding" << std::endl; std::cout << " --spot-sigma Noise sigma level for spot finding (default: 3.0)" << std::endl; std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << std::endl; @@ -57,9 +70,9 @@ void print_usage() { std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice)" << std::endl; std::cout << std::endl; - std::cout << " Scaling and merging" << std::endl; - std::cout << " -M, --scale-merge Scale and merge (refine mosaicity); automatic for rotation data" << std::endl; - std::cout << " --scale-fulls After -P rot3d combine, refit a per-frame scale on the fulls (XDS order, Unity model); implies -M. Default ON for rot3d" << std::endl; + std::cout << " Scaling and merging (on by default)" << std::endl; + std::cout << " --no-merge Skip scaling and merging; write only the per-image _process.h5" << std::endl; + std::cout << " --scale-fulls rot3d: after the 3D combine, refit a per-frame scale on the fulls (XDS order, Unity model). Default ON for rot3d" << std::endl; std::cout << " --no-scale-fulls Disable the rot3d scale-fulls refit (it is on by default for rot3d)" << std::endl; std::cout << " --write-process-h5 Also write the (large) _process.h5 when merging (default: only .mtz/.cif when merging)" << std::endl; std::cout << " --smooth-g[=deg] rot3d: smooth per-frame scale G over a deg-degree rotation range (XDS DELPHI-like) before the combine (default: 5 for rot3d; 0 = off)" << std::endl; @@ -86,6 +99,18 @@ void print_usage() { std::cout << " --azim-min-q Azimuthal-integration minimum Q (1/A)" << std::endl; std::cout << " --azim-max-q Azimuthal-integration maximum Q (1/A)" << std::endl; std::cout << " --azim-phi-bins Number of azimuthal (phi) bins (default: 1)" << std::endl; + std::cout << " --polarization-correction Enable/disable azimuthal polarization correction" << std::endl; + std::cout << " --solid-angle-correction Enable/disable azimuthal solid angle correction" << std::endl; + std::cout << std::endl; + + std::cout << " Geometry overrides (defaults taken from the input file)" << std::endl; + std::cout << " --beam-x Beam center X (pixel)" << std::endl; + std::cout << " --beam-y Beam center Y (pixel)" << std::endl; + std::cout << " --detector-distance Detector distance (mm)" << std::endl; + std::cout << " --wavelength Wavelength (A)" << std::endl; + std::cout << " --rot1 PONI rotation 1 (rad)" << std::endl; + std::cout << " --rot2 PONI rotation 2 (rad)" << std::endl; + std::cout << " --polarization Polarization factor" << std::endl; } enum { @@ -118,7 +143,19 @@ enum { OPT_FORCE_STILL, OPT_AZIM_MIN_Q, OPT_AZIM_MAX_Q, - OPT_AZIM_PHI_BINS + OPT_AZIM_PHI_BINS, + OPT_AZINT_ONLY, + OPT_SCALE, + OPT_NO_MERGE, + OPT_POLARIZATION_CORRECTION, + OPT_SOLID_ANGLE_CORRECTION, + OPT_BEAM_X, + OPT_BEAM_Y, + OPT_DETECTOR_DISTANCE, + OPT_WAVELENGTH, + OPT_ROT1, + OPT_ROT2, + OPT_POLARIZATION }; static option long_options[] = { @@ -136,7 +173,9 @@ static option long_options[] = { {"space-group", required_argument, nullptr, 'S'}, {"anomalous", no_argument, nullptr, 'A'}, {"refine-bfactor", no_argument, nullptr, 'B'}, - {"scale-merge", no_argument, nullptr, 'M'}, + {"azint-only", no_argument, nullptr, OPT_AZINT_ONLY}, + {"scale", no_argument, nullptr, OPT_SCALE}, + {"no-merge", no_argument, nullptr, OPT_NO_MERGE}, {"scale-fulls", no_argument, nullptr, OPT_SCALE_FULLS}, {"no-scale-fulls", no_argument, nullptr, OPT_NO_SCALE_FULLS}, {"write-process-h5", no_argument, nullptr, OPT_WRITE_PROCESS_H5}, @@ -150,6 +189,15 @@ static option long_options[] = { {"azim-min-q", required_argument, nullptr, OPT_AZIM_MIN_Q}, {"azim-max-q", required_argument, nullptr, OPT_AZIM_MAX_Q}, {"azim-phi-bins", required_argument, nullptr, OPT_AZIM_PHI_BINS}, + {"polarization-correction", required_argument, nullptr, OPT_POLARIZATION_CORRECTION}, + {"solid-angle-correction", required_argument, nullptr, OPT_SOLID_ANGLE_CORRECTION}, + {"beam-x", required_argument, nullptr, OPT_BEAM_X}, + {"beam-y", required_argument, nullptr, OPT_BEAM_Y}, + {"detector-distance", required_argument, nullptr, OPT_DETECTOR_DISTANCE}, + {"wavelength", required_argument, nullptr, OPT_WAVELENGTH}, + {"rot1", required_argument, nullptr, OPT_ROT1}, + {"rot2", required_argument, nullptr, OPT_ROT2}, + {"polarization", required_argument, nullptr, OPT_POLARIZATION}, {"redo-rotation-spots", no_argument, nullptr, OPT_REDO_ROTATION_SPOTS}, {"force-rotation-lattice", required_argument, nullptr, OPT_FORCE_ROTATION_LATTICE}, @@ -192,6 +240,21 @@ bool parse_float_strict(const std::string &t, float &out) { } }; +bool parse_on_off(const char *arg, bool &out) { + std::string s = arg ? arg : ""; + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (s == "on" || s == "1" || s == "true" || s == "yes") { + out = true; + return true; + } + if (s == "off" || s == "0" || s == "false" || s == "no") { + out = false; + return true; + } + return false; +} + std::optional parse_unit_cell_arg(const char *arg) { if (!arg) return std::nullopt; @@ -301,18 +364,25 @@ int main(int argc, char **argv) { int image_stride = 1; bool verbose = false; + bool azint_only = false; // --azint-only: azimuthal integration only (no spot finding/indexing) + bool scale_only = false; // --scale: re-scale/merge stored reflections only (no re-integration) bool rotation_indexing = false; bool force_still = false; // --force-still: process a rotation dataset as stills (indexing + scaling) bool two_pass_rotation = true; bool reuse_rotation_spots = true; int rotation_indexing_image_count = 100; std::optional rotation_indexing_range; - bool run_scaling = false; + bool run_scaling = true; // merge is on by default; --no-merge turns it off std::optional scale_fulls_arg; // --scale-fulls / --no-scale-fulls; default on for rot3d bool write_process_h5_flag = false; // --write-process-h5; also write _process.h5 when merging std::optional detect_ice_rings; // --detect-ice-rings[=on|off]; unset => use the dataset (file) value std::optional min_q, max_q, q_spacing; // azimuthal integration range / -q spacing (1/A) std::optional azimuthal_bins; // --azimuthal-bins + std::optional polarization_correction; // --polarization-correction (azimuthal integration) + std::optional solid_angle_correction; // --solid-angle-correction (azimuthal integration) + + // Geometry overrides (default: keep the value stored in the input file) + std::optional beam_x, beam_y, detector_distance_mm, wavelength_A, rot1_rad, rot2_rad, polarization_factor; std::optional smooth_g_deg_arg; // --smooth-g[=deg]; default 5 deg for rot3d, 0 (off) otherwise bool anomalous_mode = false; std::optional space_group_number; @@ -352,7 +422,7 @@ int main(int argc, char **argv) { int opt; int option_index = 0; - const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABS:Mr:q:"; + const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABS:r:q:"; while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { switch (opt) { @@ -513,11 +583,16 @@ int main(int argc, char **argv) { max_spot_count_override = atoll(optarg); logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); break; - case 'M': - run_scaling = true; + case OPT_AZINT_ONLY: + azint_only = true; + break; + case OPT_SCALE: + scale_only = true; + break; + case OPT_NO_MERGE: + run_scaling = false; break; case OPT_SCALE_FULLS: - run_scaling = true; scale_fulls_arg = true; break; case OPT_NO_SCALE_FULLS: @@ -537,8 +612,6 @@ int main(int argc, char **argv) { case OPT_WRITE_PROCESS_H5: write_process_h5_flag = true; break; - run_scaling = true; - break; case OPT_SMOOTH_G: smooth_g_deg_arg = optarg ? std::stod(optarg) : SMOOTH_G_DEFAULT_DEG; break; @@ -599,6 +672,31 @@ int main(int argc, char **argv) { case OPT_AZIM_PHI_BINS: azimuthal_bins = atoi(optarg); break; + case OPT_POLARIZATION_CORRECTION: { + bool value; + if (!parse_on_off(optarg, value)) { + logger.Error("Invalid polarization correction value (expected on|off): {}", optarg); + exit(EXIT_FAILURE); + } + polarization_correction = value; + break; + } + case OPT_SOLID_ANGLE_CORRECTION: { + bool value; + if (!parse_on_off(optarg, value)) { + logger.Error("Invalid solid angle correction value (expected on|off): {}", optarg); + exit(EXIT_FAILURE); + } + solid_angle_correction = value; + break; + } + case OPT_BEAM_X: beam_x = atof(optarg); break; + case OPT_BEAM_Y: beam_y = atof(optarg); break; + case OPT_DETECTOR_DISTANCE: detector_distance_mm = atof(optarg); break; + case OPT_WAVELENGTH: wavelength_A = atof(optarg); break; + case OPT_ROT1: rot1_rad = atof(optarg); break; + case OPT_ROT2: rot2_rad = atof(optarg); break; + case OPT_POLARIZATION: polarization_factor = atof(optarg); break; case OPT_SCALING_ITERATIONS: scaling_iter = atoi(optarg); if (scaling_iter <= 0) { @@ -629,6 +727,11 @@ int main(int argc, char **argv) { input_file = argv[optind]; logger.Verbose(verbose); + if (azint_only && scale_only) { + logger.Error("--azint-only and --scale are mutually exclusive"); + exit(EXIT_FAILURE); + } + // Validate space group number early const gemmi::SpaceGroup *space_group = nullptr; if (space_group_number.has_value()) { @@ -695,6 +798,115 @@ int main(int argc, char **argv) { } } + // --scale: re-scale and merge the already-integrated reflections stored in the input file, + // without re-running spot finding or integration (folded in from the former rugnux_scale tool). + if (scale_only) { + const auto total_images = static_cast(reader.GetNumberOfImages()); + const int last_image = (end_image < 0 || end_image >= total_images) ? total_images - 1 : end_image; + auto reflections = reader.ReadReflections(start_image, last_image); + + DiffractionExperiment experiment(dataset->experiment); + experiment.BitDepthImage(32).Compression(CompressionAlgorithm::BSHUF_LZ4); + experiment.FilePrefix(output_prefix); + experiment.Mode(DetectorMode::Standard); + experiment.PixelSigned(true); + experiment.OverwriteExistingFiles(true); + experiment.PolarizationFactor(0.99); + experiment.SetFileWriterFormat(FileWriterFormat::NXmxLegacy); + // Keep the space group stored in the input file (written by the full pipeline) unless -S overrides. + if (space_group_number.has_value()) + experiment.SpaceGroupNumber(space_group_number); + experiment.NumTriggers(1); + + // A rotation (goniometer) dataset uses RotationScaleMerge unless --force-still asks for stills scaling. + IndexingSettings indexing_settings; + indexing_settings.RotationIndexing(experiment.GetGoniometer().has_value() && !force_still); + experiment.ImportIndexingSettings(indexing_settings); + + ScalingSettings scaling_settings; + if (d_min_scale_merge) + scaling_settings.HighResolutionLimit_A(d_min_scale_merge.value()); + scaling_settings.MergeFriedel(!anomalous_mode); + scaling_settings.RefineB(refine_bfactor); + scaling_settings.MinPartiality(min_partiality); + scaling_settings.MinCCForImage(min_image_cc / 100.0); // --min-image-cc is percent; the setting is a fraction + if (intensity_format) + scaling_settings.FileFormat(intensity_format.value()); + experiment.ImportScalingSettings(scaling_settings); + + if (!experiment.GetUnitCell()) { + logger.Error("Experiment unit cell not found, cannot update reflection resolution"); + exit(EXIT_FAILURE); + } + auto refl_stats = UpdateReflectionResolution(experiment.GetUnitCell().value(), reflections); + logger.Info("Read {} reflections from {} images", refl_stats.n_reflections, refl_stats.n_images); + experiment.ImagesPerTrigger(refl_stats.n_images); + + const auto scale_start = std::chrono::steady_clock::now(); + std::vector merged_reflections; + MergeStatistics merged_statistics; + double error_model_isa = 0.0; + + // Rotation (rot3d): the dedicated RotationScaleMerge does the whole self-scale -> 3D combine -> + // merge. It does not support external-reference scaling, B-factor or wedge refinement. Everything + // else (stills, reference scaling) uses ScaleOnTheFly + MergeOnTheFly. + const bool is_rotation = experiment.IsRotationIndexing(); + if (is_rotation) { + if (!reference_data.empty() || experiment.GetScalingSettings().GetRefineB() + || experiment.GetRefineRotationWedgeInScaling() + || experiment.GetScalingSettings().GetRotationWedgeForScaling().has_value()) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Rotation scaling/merging (RotationScaleMerge) does not support reference " + "scaling, B-factor refinement or wedge refinement"); + RotationScaleMerge rsm(experiment, reflections, experiment.GetUnitCell(), + scaling_iter, 0.0f, nthreads, logger); + rsm.Ingest(); + auto r = rsm.Run(false); + merged_reflections = std::move(r.merged); + merged_statistics = std::move(r.statistics); + error_model_isa = r.isa; + } else { + for (int i = 0; i < scaling_iter; i++) { + if (reference_data.empty()) + ScaleOnTheFly(experiment, MergeAll(experiment, reflections)).Scale(reflections, nthreads); + else + ScaleOnTheFly(experiment, reference_data).Scale(reflections, nthreads); + } + MergeOnTheFly merge_engine(experiment); + merge_engine.ReferenceCell(experiment.GetUnitCell()); + for (size_t i = 0; i < reflections.size(); ++i) + merge_engine.AddImage(reflections[i], static_cast(i)); + merged_reflections = merge_engine.ExportReflections(); + merged_statistics = merge_engine.MergeStats(merged_reflections, reflections, reference_data); + error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0; + } + + logger.Info("Scale + merge completed in {:.2f} s ({} unique reflections)", + std::chrono::duration(std::chrono::steady_clock::now() - scale_start).count(), + merged_reflections.size()); + + std::cout << merged_statistics; + + // Space-group determination lives in the full rugnux pipeline; --scale only consumes a space + // group (from the file or -S) and merges in it. + const bool fixed_space_group = space_group || experiment.GetGemmiSpaceGroup().has_value(); + if (!fixed_space_group) + logger.Warning("No space group in the input file or on the command line - merged in P1. " + "Re-run rugnux (which determines and stores the space group) or pass " + "-S to scale and merge in the correct symmetry."); + + const auto twin_sg_number = experiment.GetSpaceGroupNumber(); + const gemmi::SpaceGroup *twin_sg = twin_sg_number + ? gemmi::find_spacegroup_by_number(twin_sg_number.value()) : nullptr; + std::cout << std::endl << TwinningAnalysisToText(AnalyzeTwinning(merged_reflections, twin_sg)) << std::endl; + + if (!output_prefix.empty()) + WriteReflections(merged_reflections, *experiment.GetUnitCell(), experiment, merged_statistics, + error_model_isa > 0 ? fmt::format("{:.2f}", error_model_isa) : "?", + output_prefix); + return 0; + } + uint64_t total_images_in_file = reader.GetNumberOfImages(); if (end_image < 0 || end_image > total_images_in_file) end_image = total_images_in_file; @@ -722,6 +934,72 @@ int main(int argc, char **argv) { // 2. Setup Experiment & Components DiffractionExperiment experiment(dataset->experiment); + + // Geometry overrides (default: keep the value stored in the input file). Applied before the + // azimuthal-integration settings are derived, which depend on the geometry. + if (beam_x) experiment.BeamX_pxl(beam_x.value()); + if (beam_y) experiment.BeamY_pxl(beam_y.value()); + if (detector_distance_mm) experiment.DetectorDistance_mm(detector_distance_mm.value()); + if (wavelength_A) experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / wavelength_A.value()); + if (rot1_rad) experiment.PoniRot1_rad(rot1_rad.value()); + if (rot2_rad) experiment.PoniRot2_rad(rot2_rad.value()); + if (polarization_factor) experiment.PolarizationFactor(polarization_factor.value()); + + // Azimuthal integration (default q-spacing 0.01 1/A, from AzimuthalIntegrationSettings): the profile + // resolves the narrow ice rings for the ice-ring score. Shared by --azint-only and full analysis. + // -q / --azim-* / correction flags override; defaults come from the input file. + { + AzimuthalIntegrationSettings azint_settings = experiment.GetAzimuthalIntegrationSettings(); + if (min_q || max_q) + azint_settings.QRange_recipA(min_q.value_or(azint_settings.GetLowQ_recipA()), + max_q.value_or(azint_settings.GetHighQ_recipA())); + if (q_spacing) + azint_settings.QSpacing_recipA(q_spacing.value()); + if (azimuthal_bins) + azint_settings.AzimuthalBinCount(azimuthal_bins.value()); + if (polarization_correction) + azint_settings.PolarizationCorrection(polarization_correction.value()); + if (solid_angle_correction) + azint_settings.SolidAngleCorrection(solid_angle_correction.value()); + experiment.ImportAzimuthalIntegrationSettings(azint_settings); + logger.Info("Azimuthal integration: Q [{:.4f}, {:.4f}] 1/A, spacing {:.4f}, {} Q x {} azimuthal bins", + azint_settings.GetLowQ_recipA(), azint_settings.GetHighQ_recipA(), + azint_settings.GetQSpacing_recipA(), azint_settings.GetQBinCount(), + azint_settings.GetAzimuthalBinCount()); + } + + // --azint-only: azimuthal integration only (no spot finding / indexing / scaling). Rugnux reads + // the geometry and azimuthal-integration settings configured above off the experiment. + if (azint_only) { + ProcessConfig config; + config.mode = ProcessMode::AzimuthalIntegration; + config.start_image = start_image; + config.end_image = end_image; + config.stride = image_stride; + config.nthreads = nthreads; + config.output_prefix = output_prefix; + + Rugnux process(reader, experiment, dataset->pixel_mask, config); + g_active_process = &process; + std::signal(SIGINT, handle_sigint); + + ProcessResult result; + try { + result = process.Run(); + } catch (const std::exception &e) { + logger.Error("Processing failed: {}", e.what()); + exit(EXIT_FAILURE); + } + g_active_process = nullptr; + + std::cout << fmt::format("Processing time: {:.2f} s", result.processing_time_s) << std::endl; + std::cout << fmt::format("Frame rate: {:.2f} Hz", result.frame_rate_hz) << std::endl; + std::cout << fmt::format("Total throughput: {:.2f} MB/s", result.throughput_MBs) << std::endl; + if (result.cancelled) + logger.Warning("Processing was cancelled after {} images", result.images_processed); + return 0; + } + experiment.BitDepthImage(32).Compression(CompressionAlgorithm::BSHUF_LZ4); experiment.FilePrefix(output_prefix); experiment.Mode(DetectorMode::Standard); // Ensure full image analysis @@ -770,10 +1048,8 @@ int main(int argc, char **argv) { "indexing). Use --force-still to treat it as stills."); } - // Rotation data is scaled and merged automatically (as if -M were given); stills still require - // an explicit -M. --force-still keeps rotation_indexing false, so stills stay opt-in. - if (rotation_indexing) - run_scaling = true; + // Scaling and merging are on by default (run_scaling initialised true); --no-merge turns them off + // for both rotation and stills, in which case only the per-image _process.h5 is written. // Configure Indexing IndexingSettings indexing_settings; @@ -842,25 +1118,6 @@ int main(int argc, char **argv) { : "profile (empirical)"); } - // Azimuthal integration (default q-spacing 0.01 1/A, from AzimuthalIntegrationSettings): the profile - // resolves the narrow ice rings for the ice-ring score. -q / --azim-* override. Applied before - // Rugnux builds the azint mapping from the experiment. - { - AzimuthalIntegrationSettings azint_settings = experiment.GetAzimuthalIntegrationSettings(); - if (min_q || max_q) - azint_settings.QRange_recipA(min_q.value_or(azint_settings.GetLowQ_recipA()), - max_q.value_or(azint_settings.GetHighQ_recipA())); - if (q_spacing) - azint_settings.QSpacing_recipA(q_spacing.value()); - if (azimuthal_bins) - azint_settings.AzimuthalBinCount(azimuthal_bins.value()); - experiment.ImportAzimuthalIntegrationSettings(azint_settings); - logger.Info("Azimuthal integration: Q [{:.4f}, {:.4f}] 1/A, spacing {:.4f}, {} Q x {} azimuthal bins", - azint_settings.GetLowQ_recipA(), azint_settings.GetHighQ_recipA(), - azint_settings.GetQSpacing_recipA(), azint_settings.GetQBinCount(), - azint_settings.GetAzimuthalBinCount()); - } - SpotFindingSettings spot_settings; spot_settings.enable = true; spot_settings.indexing = true; diff --git a/tools/rugnux_scale.cpp b/tools/rugnux_scale.cpp deleted file mode 100644 index 67a2a4ab..00000000 --- a/tools/rugnux_scale.cpp +++ /dev/null @@ -1,390 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include -#include -#include -#include -#include -#include -#include - -#include "../reader/JFJochHDF5Reader.h" -#include "../common/Logger.h" -#include "../common/DiffractionExperiment.h" -#include "../common/time_utc.h" -#include "../common/print_license.h" -#include "../image_analysis/MXAnalysisWithoutFPGA.h" -#include "../writer/FileWriter.h" -#include "../image_analysis/IndexAndRefine.h" -#include "../common/JFJochReceiverPlots.h" -#include "../compression/JFJochCompressor.h" -#include "../image_analysis/LoadFCalcFromMtz.h" -#include "../image_analysis/scale_merge/Merge.h" -#include "../image_analysis/scale_merge/TwinningAnalysis.h" -#include "../image_analysis/scale_merge/ScaleOnTheFly.h" -#include "../image_analysis/scale_merge/RotationScaleMerge.h" -#include "../image_analysis/WriteReflections.h" -#include "../image_analysis/UpdateReflectionResolution.h" - -void print_usage() { - std::cout << "Usage ./rugnux_scale {} " << std::endl; - std::cout << "Options:" << std::endl; - std::cout << " -o, --output-prefix Output file prefix (default: output)" << std::endl; - std::cout << " -N, --threads Number of threads (default: 1)" << std::endl; - std::cout << " -s, --start-image Start image number (default: 0)" << std::endl; - std::cout << " -e, --end-image End image number (default: all)" << std::endl; - std::cout << " -v, --verbose Verbose output" << std::endl; - std::cout << "" << std::endl; - std::cout << " Scaling and merging" << std::endl; - std::cout << " -S, --space-group Space group number" << std::endl; - std::cout << " --scaling-high-resolution High resolution limit for scaling/merging (default: 0.0; no limit)" - << std::endl; - std::cout << " --force-still Scale as independent stills (ScaleOnTheFly) even for a rotation dataset " - "(default: rotation data uses RotationScaleMerge)" << std::endl; - std::cout << " -A, --anomalous Anomalous mode (don't merge Friedel pairs)" << std::endl; - std::cout << " -B, --refine-bfactor Refine per image B-factor" << std::endl; - std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << - std::endl; - std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; - std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" - << std::endl; - std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: cif)" << std::endl; - std::cout << " -z, --reference-mtz Reference MTZ file" << std::endl; - std::cout << " --reference-column