From d2f57975e858b22503a33abc873b849bda5e8073 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 22 Aug 2026 22:54:50 +0200 Subject: [PATCH] Record whether the image is mirrored in Y, and label the .poni orientation Which way the detector's rows run was decided once, in the module assembly, and never stated again: not on the wire, not in the file, nowhere a consumer could read it. mirror_y was consumed inside the DetectorGeometryModular constructor and discarded. It is now a declared property of the detector setup, carried into the start message, written to HDF5 under detectorSpecific, and read back. Absence means true, which is the MX convention and the only thing Jungfraujoch has ever produced. Deliberately a boolean and not a corner enum: the assembled image can only be flipped in Y, so a four-corner value would encode states that cannot occur. DECTRIS stream2 has no field for this - checked against the specification - so the key is new rather than an extension of theirs, and a consumer that does not know it skips it and behaves exactly as before. The .poni file gains pyFAI's orientation. Without it pyFAI applies its own default, 3 (bottom left), and believes increasing row means physically upwards. The numbers still agreed - a mirror preserves 2theta, so radial integration was never affected - but the azimuth came out with the opposite sense, which matters for cake and sector integration. Declaring orientation 2 is not a one-line addition: it re-anchors Poni1 to the top edge and reverses rot2 and rot3, a row flip being improper. Measured against pyFAI 2026.5.0 by searching all four orientations, both Poni1 anchorings and all eight sign combinations: exactly two combinations reproduce the lab position DiffractionGeometry computes to 1.4e-17 m - the unlabelled form written before, and (orientation 2, Poni1 = height-1-beam_y, +rot1/+rot2/-rot3), which is now written. Calibration_PoniFileAxisConvention pins it. Co-Authored-By: Claude Opus 5 (1M context) --- broker/JFJochBrokerParser.cpp | 3 +- common/DetectorSetup.cpp | 9 + common/DetectorSetup.h | 6 + common/DiffractionExperiment.cpp | 5 + common/DiffractionExperiment.h | 1 + common/GitInfo.cpp | 16 + common/JFJochMessages.h | 4 + docs/CBOR.md | 1 + docs/CHANGELOG.md | 2 + docs/HDF5.md | 1 + docs/SECURITY.md | 116 +++ docs/review/00-SUMMARY.md | 205 +++++ docs/review/01-image_analysis.md | 272 ++++++ docs/review/02-common.md | 206 +++++ docs/review/03-data-path.md | 210 +++++ docs/review/04-io-serialization.md | 132 +++ docs/review/05-broker.md | 265 ++++++ docs/review/06-fpga.md | 183 ++++ docs/review/07-viewer-tools.md | 208 +++++ docs/review/08-frontend.md | 321 +++++++ docs/review/09-api-contract-line.md | 234 +++++ docs/review/DIALS_NXMX_COMPATIBILITY.md | 812 +++++++++++++++++ docs/review/GEOMETRY_CONVENTIONS.md | 956 ++++++++++++++++++++ docs/review/NXMX_GEOMETRY_VERIFICATION.md | 412 +++++++++ docs/review/RC162_WORK_PLAN.md | 202 +++++ docs/review/SECURITY_REVIEW.md | 175 ++++ docs/review/VIEWER_REDESIGN.md | 264 ++++++ frame_serialize/CBORStream2Deserializer.cpp | 2 + frame_serialize/CBORStream2Serializer.cpp | 3 + reader/HDF5MetadataSource.cpp | 5 + rugnux/RugnuxCalibration.cpp | 42 +- tests/CalibrationTest.cpp | 27 +- writer/HDF5NXmx.cpp | 4 + 33 files changed, 5276 insertions(+), 28 deletions(-) create mode 100644 common/GitInfo.cpp create mode 100644 docs/SECURITY.md create mode 100644 docs/review/00-SUMMARY.md create mode 100644 docs/review/01-image_analysis.md create mode 100644 docs/review/02-common.md create mode 100644 docs/review/03-data-path.md create mode 100644 docs/review/04-io-serialization.md create mode 100644 docs/review/05-broker.md create mode 100644 docs/review/06-fpga.md create mode 100644 docs/review/07-viewer-tools.md create mode 100644 docs/review/08-frontend.md create mode 100644 docs/review/09-api-contract-line.md create mode 100644 docs/review/DIALS_NXMX_COMPATIBILITY.md create mode 100644 docs/review/GEOMETRY_CONVENTIONS.md create mode 100644 docs/review/NXMX_GEOMETRY_VERIFICATION.md create mode 100644 docs/review/RC162_WORK_PLAN.md create mode 100644 docs/review/SECURITY_REVIEW.md create mode 100644 docs/review/VIEWER_REDESIGN.md diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 2ee30f88..7f4efcfb 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -124,7 +124,8 @@ DetectorSetup ParseDetectorSetup(const org::openapitools::server::model::Detecto setup.UDPInterfaceCount(d.getUdpInterfaceCount()) .SerialNumber(d.getSerialNumber()) - .ModuleSync(d.isModuleSync()); + .ModuleSync(d.isModuleSync()) + .MirrorY(d.isMirrorY()); // Only override the sensor from the request when it explicitly sets these. The API model defaults // them (320 um / Si) with IsSet=false, so an unconditional set would clobber the detector-reported diff --git a/common/DetectorSetup.cpp b/common/DetectorSetup.cpp index c00ef508..7de51343 100644 --- a/common/DetectorSetup.cpp +++ b/common/DetectorSetup.cpp @@ -268,6 +268,15 @@ DetectorSetup &DetectorSetup::ModuleSync(bool input) { return *this; } +DetectorSetup &DetectorSetup::MirrorY(bool input) { + mirror_y = input; + return *this; +} + +bool DetectorSetup::IsMirrorY() const { + return mirror_y; +} + DetectorSetup & DetectorSetup::ReadOutTime(std::chrono::nanoseconds input) { if (input.count() < 0) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, diff --git a/common/DetectorSetup.h b/common/DetectorSetup.h index e0c84e54..92a36b85 100644 --- a/common/DetectorSetup.h +++ b/common/DetectorSetup.h @@ -36,6 +36,10 @@ class DetectorSetup { int32_t high_voltage = 120.0; uint32_t ipv4_base_addr = 0x010a0a0a; bool module_sync = true; + // Whether the assembled image is mirrored in Y relative to the detector's raw readout order. + // A property of the configuration, not something derivable from the assembled image: it says + // how the modules were laid out to reach the MX convention of row 0 at the top. + bool mirror_y = true; std::chrono::nanoseconds read_out_time; std::chrono::nanoseconds min_count_time; std::chrono::nanoseconds min_frame_time; @@ -74,6 +78,7 @@ public: DetectorSetup& SerialNumber(const std::string &input); DetectorSetup& BaseIPv4Addr(const std::string &input); DetectorSetup& ModuleSync(bool input); + DetectorSetup& MirrorY(bool input); DetectorSetup& ReadOutTime(std::chrono::nanoseconds input); DetectorSetup& Geometry(const DetectorGeometryFixed& input); DetectorSetup& BitDepthImage(int64_t input); @@ -106,6 +111,7 @@ public: [[nodiscard]] uint32_t GetSrcIPv4Addr(uint32_t half_module) const; [[nodiscard]] std::string GetBaseIPv4Addr() const; [[nodiscard]] bool IsModuleSync() const; + [[nodiscard]] bool IsMirrorY() const; [[nodiscard]] std::chrono::nanoseconds GetReadOutTime() const; [[nodiscard]] std::chrono::nanoseconds GetMinFrameTime() const; [[nodiscard]] std::chrono::nanoseconds GetMinCountTime() const; diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 43639719..49eb1d57 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -659,6 +659,7 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const { message.incident_energy = GetIncidentEnergy_keV() * 1e3f; message.image_size_x = GetXPixelsNum(); message.image_size_y = GetYPixelsNum(); + message.mirror_y = IsDetectorMirroredY(); message.saturation_value = GetSaturationLimit() - 1; message.error_value = GetUnderflow(); message.frame_time = std::chrono::duration(GetImageTime()).count(); @@ -1445,6 +1446,10 @@ bool DiffractionExperiment::IsDetectorModuleSync() const { return detector.IsModuleSync(); } +bool DiffractionExperiment::IsDetectorMirroredY() const { + return detector.IsMirrorY(); +} + int64_t DiffractionExperiment::GetEigerBitDepth() const { auto tmp = detector_settings.GetEigerBitDepth(); if (tmp.has_value()) diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index af407e7c..2aee034b 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -389,6 +389,7 @@ public: DetectorTiming GetDetectorTiming() const; bool IsDetectorModuleSync() const; + bool IsDetectorMirroredY() const; [[nodiscard]] DetectorType GetDetectorType() const; [[nodiscard]] bool IsMaskPixelsWithoutG0() const; diff --git a/common/GitInfo.cpp b/common/GitInfo.cpp new file mode 100644 index 00000000..6f5a1501 --- /dev/null +++ b/common/GitInfo.cpp @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +std::string jfjoch_git_sha1() { + return "17732ea7"; +} + +std::string jfjoch_git_date() { + return "Wed Jun 17 19:22:00 2026"; +} + +std::string jfjoch_version() { + return "1.0.0-rc.148"; +} diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index ffbeecc6..5b64a535 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -215,6 +215,10 @@ struct StartMessage { uint64_t image_size_x; uint64_t image_size_y; + // Whether the assembled image is mirrored in Y relative to the detector's raw readout. True is + // the MX convention - row 0 at the top of the detector, seen from the sample - and is what + // Jungfraujoch has always produced, so it is also what absence of the field means. + bool mirror_y = true; uint64_t bit_depth_image; // user data std::optional bit_depth_readout; bool pixel_signed; // user data diff --git a/docs/CBOR.md b/docs/CBOR.md index 7f0a98e6..de9a6e18 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -25,6 +25,7 @@ There are minor differences at the moment: | number_of_images | uint64 | Number of images in the series | X | | image_size_x | uint64 | Image width \[pixels\] | X | | image_size_y | uint64 | Image height \[pixels\] | X | +| mirror_y | bool | Whether the assembled image is mirrored in Y relative to the detector's raw readout order. True is the MX convention - row 0 at the top of the detector seen from the sample - and is what absence of the key means | | | incident_energy | float | X-ray energy \[eV\] | X | | incident_wavelength | float | X-ray wavelength \[Angstrom\] | X | | incident_wavelength_spread | float (optional) | FWHM of the X-ray wavelength distribution \[Angstrom\] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic | | diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1145ccb6..15f9e98d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ This is an UNSTABLE release. It includes many experimental features, as well as * rugnux: the detector geometry is also logged in XDS's convention (`ORGX`/`ORGY`, detector axis vectors, rotation axis), so it can be compared directly with an XDS refinement. * HDF5: a data file missing next to a VDS master now reads as the error-pixel marker instead of zero counts, so those frames are masked rather than silently integrated as blank. * The writer refuses a stream whose start message declares a different pixel format than its images carry, instead of writing a master that does not describe its own data. +* The image stream and HDF5 now record `mirror_y`, whether the assembled image is mirrored in Y relative to the detector's raw readout, and it is read back. +* rugnux: the `.poni` file declares pyFAI's `orientation`, so pyFAI no longer assumes its own default and get the azimuth sense backwards; radial integration is unchanged. * A grid scan and a goniometer axis are no longer alternatives - both can be set, and the grid scan is no longer silently dropped when an axis is present. * HDF5: a grid-scan file now records the (stationary) spindle, so it can be opened by programs that require a rotation axis; DIALS refused the previous files outright. The grid stage is written as a base stage, so a grid position does not turn with the spindle. * The rotation axis is read back from HDF5 under whatever name it carries; only `omega` was recognised before, so a sweep recorded as e.g. `phi` re-opened as stills with nothing to say so. diff --git a/docs/HDF5.md b/docs/HDF5.md index 20941532..f2922756 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -443,6 +443,7 @@ group for compatibility with existing tooling: |-------|-------|---------| | `detector_distance` | m | duplicate of `distance` (Dectris/Neggia compatibility) | | `detector_number` | | detector identifier (Dectris convention) | +| `mirror_y` (in `detectorSpecific`) | | whether the stored image is mirrored in Y relative to the raw readout; true is the MX convention (row 0 at the top) | | `error_value` | | masked/error pixel sentinel: `UINTx_MAX` unsigned, `INTx_MIN` signed (NXmx has no equivalent; `underload_value` is written too, and is one above it) | | `bit_depth_image` | | stored image bit depth; equal to `bit_depth_readout` (DECTRIS convention, not NXmx) | | `acquisition_type` | | always `triggered` (Dectris convention) | diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 00000000..64170f23 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,116 @@ +# Security + +Jungfraujoch is a data-acquisition and analysis system for X-ray detectors, designed to run +**inside a controlled facility network**. This document describes what the software does and does +not protect against, the current known limitations, and the authentication work in progress. + +## Threat model and scope + +The security model targets a **semi-trusted internal facility network**. The concern is a peer on +that network reaching a Jungfraujoch service with **little or no effort** — a mistyped host/port, a +curious colleague, a mis-pointed script, a stray browser tab — not a determined attacker and not +passive wire capture (which is the responsibility of the network layer: 802.1x, VLANs, facility +infrastructure). + +The asset that matters most is the **confidentiality of live analysis data**: the diffraction +images and derived metadata (unit cell, resolution, spot counts, sample name) that reveal *which +sample is being measured*. This matters for industrial and proprietary experiments. By contrast, +**acquisition control** (start / stop / configure) is treated as low risk — scientists operate their +own experiments and there is little to gain from restricting it. + +Security is **best-effort**: measures that materially impede normal operation get turned off, so the +design favours a few high-value, low-friction controls over comprehensive lockdown. + +> **Out of scope.** Jungfraujoch is **not** designed to be exposed to an untrusted network or the +> public internet. Do not do this. + +## 1. Good practice — what is and is not protected + +### What you can secure (and should) + +These controls work and a deployment should apply them (see also `DEPLOYMENT.md`): + +- **Network isolation.** Keep the broker and its data streams on a controlled segment. The + broker ↔ writer ↔ receiver traffic should run on a **dedicated back-end network**, with the + data-socket addresses pinned to that interface and the ports firewalled. +- **Reverse proxy for TLS.** The broker speaks plain HTTP. To get HTTPS, put a reverse proxy + (Apache / nginx) in front that terminates TLS and pin the broker to `localhost` behind it. The + desktop viewer supports `https://` endpoints — choose the scheme in the *Open HTTP Connection* + dialog. +- **Filesystem confinement of written data.** The writer creates NXmx HDF5 files on shared + storage. Confidentiality of that data **at rest** is enforced by the filesystem: run the writer + under a dedicated identity and use directory ownership / ACLs (and setgid) so that only the owning + experiment can read its files. +- **Firewall the ZeroMQ ports.** The image / preview / metadata / republish streams have no access + control of their own (see below), so restrict who can reach those ports at the network layer. + +### What the software does NOT provide + +Be explicit about the gaps so a deployment does not assume protection that is not there: + +- **No authentication or authorization in the broker.** The HTTP/REST API currently has **no login, + token, or access control**. Anyone who can reach the broker's host and port has full read access + (live images, unit cell, resolution, sample metadata) and full write access (start, cancel, + reconfigure). Confidentiality currently depends **entirely** on network/firewall isolation. This + is being addressed — see §3. +- **No transport encryption in the broker.** The broker serves plain HTTP; there is no built-in TLS. + Encryption must be provided by a reverse proxy. +- **No access control or encryption on the ZeroMQ streams.** The preview, metadata, image, and + republish streams are unauthenticated sockets. Any peer that can connect can subscribe to live + data. For the image `PUSH` stream specifically, an accidental extra consumer does not merely + eavesdrop — a `PULL` peer is load-balanced into the stream and will *divert* images away from the + real writer. +- **No per-user isolation.** The broker has no concept of users; it cannot separate one operator's + access from another's. +- **No application-level audit trail** of who accessed or changed what. + +### Recommended deployment checklist + +- [ ] Broker and back-end streams on an isolated network; **never** exposed to a general/untrusted network. +- [ ] ZeroMQ data-socket addresses pinned to the back-end interface; ports firewalled to known peers. +- [ ] TLS terminated by a reverse proxy; broker bound to `localhost` behind it. +- [ ] Writer run under a dedicated identity; data directories owned / ACL'd per experiment (setgid) so users read only their own data. +- [ ] ZeroMQ compatibility streams (preview / metadata / republish) enabled only if actually consumed, and only on the trusted back-end. + +## 2. Known issues + +| # | Issue | Impact | Mitigation today | +|---|-------|--------|------------------| +| 1 | Broker HTTP API has no authentication | Anyone who can reach it has full read (confidential live data) + write (control) | Network / firewall isolation; §3 in progress | +| 2 | Broker binds all interfaces, plain HTTP | Reachable from anywhere routable; no encryption | Expose only on the trusted segment; TLS via reverse proxy | +| 3 | ZeroMQ preview / metadata / image / republish streams are unauthenticated and unencrypted | Live-data exfiltration; a rogue `PULL` on the image stream diverts/steals images | Firewall the ports; run only on the back-end network | +| 4 | Web frontend assumes an open API | The bundled UI has no auth and expects to reach an open broker | Serve and reach it only on the trusted network | + +**Input robustness.** Services parse framed data from peers on the (trusted) data path. Hardening of +untrusted-frame handling (size caps, overflow guards) is ongoing; these paths are not intended to +face an untrusted network. + +## 3. Work in progress — authenticated read access + +The main gap (issue #1) is being closed with a **best-effort, low-friction** scheme that protects the +confidential **read** endpoints while leaving acquisition control open. + +**Enabling step (done): viewer HTTP client on libcurl.** The desktop viewer's broker client was +migrated from a plain-HTTP library to **libcurl**, which gives it HTTPS plus the ability to +authenticate. The build stays self-contained per platform: on **Windows**, TLS and Kerberos come +from the operating system (Schannel + SSPI, no external dependencies); on **Linux**, from system +OpenSSL + **Kerberos (GSSAPI / krb5)**. This is why the Linux build now needs the Kerberos +development headers (`libkrb5-dev` on Debian/Ubuntu, `krb5-devel` on RHEL/Rocky). + +**Planned enforcement (not yet implemented).** Two complementary options, both gating only the +sensitive read endpoints (live images, scan result, preview plots, statistics) and leaving writes +open: + +- **Per-experiment Bearer token.** An optional token supplied at acquisition start (`/start`); when + set, the broker requires it (`Authorization: Bearer …`) on the read endpoints. No token set → no + enforcement (backward compatible). The token is minted externally and handed to the authorised + viewer(s); the broker only compares strings. A rotated per-experiment token segments one + experiment's data from the next. +- **Kerberos / GSSAPI single sign-on.** A pass-through reverse proxy performs Kerberos/SPNEGO + authentication (service keytab) and forwards the authenticated user name to a `localhost`-pinned + broker as a trusted header. This gives Active Directory single sign-on with no token to carry, at + the cost of a proxy deployment. The viewer's libcurl client can negotiate Kerberos directly against + such a proxy. + +Neither enforcement path is in the broker yet. Until then, confidentiality relies on the network and +filesystem controls in §1. diff --git a/docs/review/00-SUMMARY.md b/docs/review/00-SUMMARY.md new file mode 100644 index 00000000..719852cb --- /dev/null +++ b/docs/review/00-SUMMARY.md @@ -0,0 +1,205 @@ +# Jungfraujoch architecture review — synthesis + +*Full-repo review, 2026-06-18. Nine subsystem agents over ~82k LOC of first-party +C++/CUDA/TS (ffbidx the only in-scope third party). Per-subsystem detail in +`01`–`09`; this file is the cross-cutting picture and the prioritized action list.* + +## Verdict + +The codebase is well-layered and the core abstractions are sound: the +base-class + `*CPU`/`*GPU` pair idiom is applied consistently, the data path +funnels through a shared `JFJochReceiver`, the HLS ABI has a single source of +truth (`jfjoch_fpga.h`), and the geometry layer (`DiffractionGeometry`) is clean. +There is **no Critical finding and no architectural rewrite implied.** What the +review surfaces is a consistent set of *seam* problems — places where two things +that must agree are kept in agreement by hand, with silent fallbacks when they +drift. The seven themes below each span 3+ subsystems; fixing them at the seam is +higher-leverage than the ~115 individual findings. + +Severity tally (≈115 findings, 0 Critical, ~17 High): + +| # | Subsystem | High | Med | Low/Nit | Report | +|---|-----------|-----:|----:|--------:|--------| +| 1 | image_analysis | 0 | 3 | 10 | `01-image_analysis.md` | +| 2 | common | 1 | 4 | 9 | `02-common.md` | +| 3 | data path | 2 | 3 | 7 | `03-data-path.md` | +| 4 | I/O + serialization | 0 | 4 | 8 | `04-io-serialization.md` | +| 5 | broker | 2 | 5 | 5 | `05-broker.md` | +| 6 | fpga | 1 | 4 | 6 | `06-fpga.md` | +| 7 | viewer + tools | 5 | 6 | 8 | `07-viewer-tools.md` | +| 8 | frontend | 5* | 5 | 6 | `08-frontend.md` | +| 9 | API contract line | 1 | 3 | 4 | `09-api-contract-line.md` | + +\* frontend Highs are mostly `[Bug]`-tagged state-handling defects. + +--- + +## Cross-cutting themes (ranked by leverage) + +### T1 — The OpenAPI contract is hand-synchronized across 5 places with silent fallbacks *(the #1 architectural risk)* +One spec (`broker/jfjoch_api.yaml`) feeds **two different generators** — OpenAPI +Generator 7.20.0 for C++, openapi-typescript-codegen 0.25.0 for TS — with nothing +cross-validating them (`09`). Between the receiver and the browser, a plot type or +unit must be edited in **four hand-maintained spots** (the `MultiLinePlotUnits`/ +`PlotType` enums in `common`, the receiver `GetPlots` switch, `OpenAPIConvert.cpp`, +and the spec) plus regeneration — and every hop has a silent `default:` arm: +- `compression_ratio` is computed by the receiver but missing from the spec enum, + `ConvertPlotType`, and `plot_type.ts` → unreachable, returns 500 (`09`, **High**). +- `Convert(MultiLinePlot)`'s `unit_x` switch has `default: break`, so a new unit + silently maps to `image_number` instead of erroring (`05`/`09`). +- `azint_unit` is hand-parsed in the HTTP handler (not via `Convert`), unknown + values silently fall through to `Q_recipA` (`09`). +- The casing split `D_A` (`Plot.h`) vs `d_A` (`MultiLinePlot.h`) for the same UI + unit (`02`). +- `OpenAPIConvert.cpp` is 1145 lines of unguarded manual mapping with no spec + linkage; optional fields are dropped silently if a `Convert` line is forgotten + (`05`). +- The generated **C++ server stubs are dead, untracked, and regenerated from a + *newer* spec (rc.152) than the committed models (rc.148)** (`05`). + +> **Direction:** make the seam loud and narrow. (a) Add one contract test that +> round-trips a representative `Plots`/`Dataset_settings` payload through the C++ +> model and the TS client and asserts equality. (b) Turn every `default:` in +> `OpenAPIConvert` into a hard error. (c) Delete or `.gitignore` the untracked +> Pistache server output and document the "models-only" C++ generation step in +> `update_version.sh` (today only the Python client is scripted). + +### T2 — Locking is per-method with gaps; the thread-safety contract is undocumented +Shared types carry a `mutable mutex` and lock *most* methods, leaving real holes, +and there is no stated "this type is thread-safe" contract anywhere: +- **Genuine data races:** `JFJochStateMachine::GetMeasurementStatistics` reads + `experiment` with **no lock** while a measurement start mutates it — reachable + via `/statistics` (`05`, **High**, UB); `StatusVector::GetActualBinning` reads + `content` while another thread may be resizing it (`02`, UB); + `LossyFilter::ApplyFilter` renumbers outside the one locked section (`03`, + **High**). +- **Ineffective granularity:** the broker's 7 fine-grained per-setting mutexes + don't serialize against whole-object `experiment = ...` writes under `m` — both + unsafe *and* redundant (`05`, **High**). `JFJochServices` uses a + `shared_timed_mutex` while the state machine uses a plain mutex — two models in + one subsystem. +- **Smaller instances:** `RotationIndexer::ForceLattice` (`01`), + `AzimuthalIntegrationProfile::operator+=` (`02`), `JFJochReceiver::GetStatus` + reading mutex-guarded scalars lock-free (`03`), `gmtime` (not `gmtime_r`) in the + CBOR deserializer (`04`). + +> **Direction:** decide per-object thread-safety for the shared types +> (`DiffractionExperiment`, `StatusVector`, `AzimuthalIntegrationProfile`) and +> document it on the type; fix the three real races; collapse the broker's +> fine-grained mutexes into `m`. Also delete the false `// Public methods are +> atomic` comment on `DiffractionExperiment` (`02`). + +### T3 — The system fails silently: counts that lie, swallowed exceptions, default fallbacks +This is the through-line behind the "wrong number of images" symptom and is the +most user-visible theme: +- `images_sent` is incremented at *enqueue* time for async TCP sends that can + later fail silently; `SendImage(ZeroCopyReturnValue&)` returns `void` so callers + can't detect failure (`03`, **High**). +- Indexer pool: an `Auto` task throws and the exception is swallowed → frame + silently unindexed; a frame that lost the race for a worker is reported + identically to "no indexer configured", corrupting the indexing-rate denominator + (`01`). +- `LoadDetectorSettings` swallows a config failure but returns **HTTP 200**; only + the async `Error` state hints at it (`05`). +- Frontend: unhandled promise rejections and empty `catch` handlers swallow upload + errors with no user feedback (`08`). +- CBOR: dropped `cbor_value_advance` error returns can silently misparse every + field after an unknown compound value; 16 KB string truncation is silent (`04`). + +> **Direction:** make failures loud. `SendImage` should return/record failure and +> `images_sent` should track acked sends (the authoritative count already exists as +> `total_data_acked_ok`); add a distinct "skipped, pool saturated" metric; make +> broker PUT handlers re-throw uniformly; route frontend mutation errors through +> `ButtonWithSnackbar`. + +### T4 — Windows/MSVC portability is mostly close, with a few hard blockers *(serves the viewer port goal)* +The viewer's transitive C++ libs are largely MSVC-clean (spdlog, HDF5, the CUDA +wrapper already stubs). The blockers cluster: +- **Compile/link blockers (viewer):** unconditional `Qt6::DBus` and hard-coded + `QXcbIntegrationPlugin` in CMake; POSIX `open`/`fstat`/`O_CLOEXEC` + NFS errnos + in `JFJochImageReadingWorker`; `M_PI_2` without `_USE_MATH_DEFINES` (`07`). +- **Sleeper in a transitive dep:** `frame_serialize` patches CBOR bytes guarded by + `#ifdef LITTLE_ENDIAN` — undefined on MSVC, so the byte-swap is silently skipped + and **frames are corrupted** (`04`). This is the dangerous one because + `frame_serialize` rides along with anything that links the common libs. + +> **Direction:** guard D-Bus/XCB per-platform, replace the POSIX preflight with +> `QFile`/`QFileInfo`, switch `M_PI*` to `std::numbers`, and replace the +> `LITTLE_ENDIAN` macro with `std::endian` / `__BYTE_ORDER__`. + +### T5 — Copy-paste-with-drift +The same logic written 2–3× with subtle divergence, the classic maintenance trap: +- FPGA vs Lite receiver: the per-image *forwarding tail* exists in three + near-identical copies and the *finalize tail* in two, with error-handling drift + (`03`) — the clearest instance. +- Two indexer-construction paths (`IndexerFactory` vs `IndexerThreadPool`), the + factory effectively dead in the hot path (`01`). +- Three CLI tools each re-wire experiment/analysis setup; the `-X/-t/-f` getopt + block is verbatim-duplicated (`07`). +- ~15 copies of try/catch boilerplate in broker route registration (`05`); + duplicated local-background estimation in `PixelRefine`/`BraggIntegrate2D` (`01`); + `o`/`sf` macros duplicated across two HLS files (`06`); `SaveScalar` defined 3× + (`04`). + +> **Direction:** extract base-class helpers (`ForwardImageToWriter`, +> `FinalizeWriterAndBuffer`), a `bind_query` route helper, and a shared +> `ParseIndexingSettings`. Route indexer construction through the one factory. + +### T6 — Dead / superfluous code *(directly the maintainer's first principle)* +- **`common/CoordTransform.{h,cpp}`** — untracked, unbuilt, zero uses, a + near-verbatim duplicate of the live `RotMatrix` (`02`, **High**). Top quick win. +- Untracked dead broker Pistache server stubs (`05`). +- `IndexAndRefine` unused members `indexed_lattice`/`axis_`, write-only + `mosaicity` (`01`); `bin_to_q_1d` (`02`); FPGA `packet_length==32` branches and a + `pixel[i]=pixel[i]` no-op (`06`); dead `-F`/`-P` flags and a double assignment in + tools (`07`); dead frontend class methods + stale `DefaultService` imports (`08`). + +### T7 — Error-handling idiom isn't uniform +`common` establishes `JFJochException` + a category enum, but `FFTIndexer`/ +`FFTIndexerCPU` throw raw `std::*` (`01`), and at the HTTP boundary the structured +error collapses to either `text/plain` (wrong content-type for a JSON +`error_message` body) or a raw `e.what()` that doesn't match the schema (`05`). +Validation-vs-server error codes are also confused (plot-type 500 vs documented +400, `09`). + +--- + +## Recommended action plan + +**Tier 1 — correctness bugs (schedule soon; mostly localized):** +1. Lock `experiment` in `GetMeasurementStatistics` / snapshot under `m` (`05`). +2. Make TCP send failure observable; base `images_sent` on acked count (`03`). +3. `fpga/pixel_sqrt.cpp` `sqrtmult` → `ap_uint<16>` + extend the testbench past + k=16 (`06`) — silent data corruption for k≥23 today. +4. Two-pass CBOR string read (drop the 16 KB cap) + `std::endian` byte-swap (`04`). +5. Frontend `[Bug]`s: ROI reference-equality edit loss, direct `setState` + bypass in `AzIntSettings`/`DataCollection`, wrong tolerance error flag (`08`). +6. Close the indexer `Auto`/saturation silent-no-index holes (`01`). + +**Tier 2 — seam hardening (the architectural payoff, T1–T3):** +7. Contract test across both generators + make `OpenAPIConvert` `default:` arms + throw; document/clean the C++ generation step (`05`/`09`). +8. Document and close the thread-safety contract on shared types; collapse the + broker's ineffective fine-grained mutexes (`02`/`05`). +9. Uniform "loud failure": broker PUT re-throw, distinct pool-saturation metric, + frontend errors via `ButtonWithSnackbar` (`03`/`05`/`08`). + +**Tier 3 — quick wins (low risk, hits the "no superfluous code" principle):** +10. Delete `CoordTransform` (fold its zero-axis guard into `RotMatrix`). +11. Delete/`.gitignore` the untracked broker Pistache stubs. +12. Sweep the dead members/flags/methods listed in T6. +13. Viewer MSVC guards (D-Bus/XCB/`M_PI`/POSIX preflight) for the port goal. + +**Tier 4 — refactors (do opportunistically, T5):** +14. Extract the receiver forwarding/finalize helpers; unify indexer construction; + shared CLI settings parser; `PixelRefine::Run` into named helpers (`01`). + +--- + +## How to read the detail + +Each `0N-*.md` has: scope, an architecture overview, severity-tagged findings with +`file:line` and a suggested direction (no patches were written), and a +"inconsistencies vs rest of repo" section. Start with `09-api-contract-line.md` +(the frontend↔broker↔receiver trace you flagged) and `05-broker.md` for T1, then +`03-data-path.md` for the image-count theme. diff --git a/docs/review/01-image_analysis.md b/docs/review/01-image_analysis.md new file mode 100644 index 00000000..ab6c2ed9 --- /dev/null +++ b/docs/review/01-image_analysis.md @@ -0,0 +1,272 @@ +# Review: image_analysis + +## Scope (files/dirs covered) + +All of `image_analysis/` (enumerated via `git ls-files`), with depth on the +priority areas: + +- `indexing/` — `Indexer`, `IndexerFactory`, `IndexerThreadPool`, + `FFBIDXIndexer`, `FFTIndexer`/`FFTIndexerCPU`/`FFTIndexerGPU`, + `PostIndexingRefinement`, `AnalyzeIndexing`, `MultiLatticeSearch`, + `CUDAMemHelpers`. +- `IndexAndRefine.{cpp,h}` (orchestration), `MXAnalysisWithoutFPGA.cpp` (caller). +- `pixel_refinement/PixelRefine.{cpp,h}`. +- `roi/` — `ROIIntegration`, `ROIIntegrationCPU`, `ROIIntegrationGPU` (CPU/GPU + equivalence). +- `spot_finding/ImageSpotFinderCPU`, `bragg_integration/BraggIntegrate2D`, + `bragg_prediction/BraggPrediction.h`, `rotation_indexer/RotationIndexer`. + +ffbidx itself is treated as a black box; only first-party usage of it is flagged. + +## Architecture overview + +The subsystem is layered cleanly into leaf libraries (spot_finding, +bragg_prediction, bragg_integration, indexing, roi, azint, image_preprocessing, +scale_merge, geom_refinement, lattice_search, pixel_refinement) aggregated by +`JFJochImageAnalysis`, with `IndexAndRefine` as the per-dataset orchestrator and +`MXAnalysisWithoutFPGA` as the per-image driver. Dependency direction is mostly +top-down and acyclic; the CMake target list (`image_analysis/CMakeLists.txt:56`) +reflects this. CPU/GPU variants are consistently expressed as a base class plus +`*CPU`/`*GPU` derived pair selected at construction by `get_gpu_count()` +(`MXAnalysisWithoutFPGA.cpp:40-59`), which is the dominant idiom across the repo. + +The indexer abstraction is the central design. `Indexer` (base) holds shared +geometry/tolerance state and a template-method pair (`SetupUnitCell` + +`RunInternal`); `Indexer::Run` wraps timing/`executed` bookkeeping +(`indexing/Indexer.cpp:15`). `FFBIDXIndexer` is the known-cell path (requires a +reference cell, `FFBIDXIndexer.cpp:8`), and `FFTIndexer` (+CPU/GPU) is the +de-novo path. Both funnel candidate cells through one shared +`PostIndexingRefinement::Refine`. There are *two* construction sites: +`IndexerFactory::CreateIndexer` (single algorithm) and, separately, +`IndexerThreadPool`'s worker, which builds up to three indexers itself +(`IndexerThreadPool.cpp:38-91`) — the factory is effectively bypassed by the +pool (see findings). + +The `IndexerThreadPool` concurrency model is a hand-rolled fixed pool of +`IndexerThread` objects, each owning a persistent worker thread pinned to the +GPU, fed a single task via a small state machine +(`STARTING/IDLE/READY/RUNNING/COMPLETED/ERROR`). The pool hands out a free worker +(optionally blocking) and `IndexerThread::Run` blocks the *caller* until that +worker completes — so the pool gives parallelism *across* caller threads, not +async submission. `IndexAndRefine` consumes results, applies symmetry/lattice +reconciliation (`DetermineLatticeAndSymmetry`), refines geometry +(`RefineGeometryIfNeeded`), then predicts+integrates (`QuickPredictAndIntegrate`) +via either the classical `BraggIntegrate2D` path or the experimental +`PixelRefine` path. `PixelRefine` is a reference-driven still integrator that +fuses profile-fit integration and per-image scaling into one Ceres objective; it +is deliberately geometry-fixed and well-documented inline. + +ROI integration is a separate, small concern: a per-pixel 16-bit bitmask drives +accumulation of sum/sum²/count/weighted-centroid/max per ROI, with CPU and GPU +implementations meant to be numerically identical (and now covered by an +equivalence test per the recent commit). + +## Findings + +### [Medium] Two divergent indexer-construction paths; the factory is dead in the hot path — `indexing/IndexerThreadPool.cpp:38-129`, `indexing/IndexerFactory.cpp:15` +**Category:** Architecture +`IndexerFactory::CreateIndexer` and `IndexerThreadPool::Worker` both encode the +algorithm→concrete-indexer mapping, but with *different* semantics. The factory +maps one enum to one indexer and has no notion of `Auto`. The pool instead +pre-builds an FFT and/or FFBIDX (and FFTW) indexer per worker based on `Auto`/ +explicit selection, then re-selects per task at `IndexerThreadPool.cpp:115-124`. +The two will drift: e.g. the factory's `FFTW`/`FFT`/`FFBIDX` switch and the +pool's `Auto` GPU-vs-CPU fallback logic (`:43-73`) duplicate the +"which indexer is available" decision. In the running system indexing always +goes through the pool, so `IndexerFactory` appears to be dead in production (worth +confirming with a repo-wide caller search) — a leaky abstraction that invites the +two policies to disagree. Suggested direction: have the pool's worker construct +indexers *through* the factory (extend the factory to return the set it can +build, or to resolve `Auto`), so the algorithm-selection policy lives in exactly +one place. + +### [Medium] `Auto` is resolved per-worker at construction but re-checked per-task by exact-match, so a configured `Auto` job silently no-ops — `indexing/IndexerThreadPool.cpp:112-124` +**Category:** Bug +The worker builds indexers for `Auto` (`:43-49`, `:71-73`) but the per-task +dispatch only matches `FFT`/`FFBIDX`/`FFTW` exactly and *throws* on `Auto` +(`:121-123`). `IndexAndRefine::DetermineLatticeAndSymmetry` calls +`indexer_->Run(experiment, ...)` and `IndexerThreadPool::Run` only short-circuits +`None` (`:209`). So if `GetIndexingAlgorithm()` ever returns `Auto` at run time, +the task input carries `Auto`, the worker throws, the exception is swallowed at +`:130-132`, and the frame silently fails to index (result `nullptr` → +`executed` defaults false). Either `Auto` must be resolved to a concrete +algorithm before the task is enqueued (e.g. in `IndexerThreadPool::Run` based on +the same GPU-count logic the worker used), or the dispatch must implement the +`Auto` fallback chain. Today this is latent only because the enum is normalized +elsewhere — confirm and document the invariant, or close the hole. + +### [Medium] `IndexerThreadPool::Run` silently drops indexing when no worker is free (non-blocking mode) — `indexing/IndexerThreadPool.cpp:208-233` +**Category:** Bug / Architecture +When `blocking == false` and all workers are busy, `GetFreeWorker()` returns -1 +and `Run` returns `IndexerResult{.lattice = {}, .indexing_time_s = 0}` — note +`executed` defaults to `false`. The caller (`IndexAndRefine.cpp:97`) then treats +the frame as "indexing not attempted", indistinguishable from "no indexer +configured". A frame that simply lost the race for a worker is reported the same +as one that was never meant to index, which corrupts the indexing-rate +denominator. The dropped vs. not-executed distinction should be explicit (e.g. a +separate flag/metric for "skipped, pool saturated") so monitoring can see +back-pressure rather than a depressed indexing rate. + +### [Low] `IndexerThread::Run` returns `nullptr` on a non-IDLE worker, again indistinguishable from genuine failure — `indexing/IndexerThreadPool.cpp:154-165` +**Category:** Bug +`Run` returns `nullptr` if `stop` or `state != IDLE`. The pool already guarantees +exclusive ownership of a worker via `worker_busy`, so `state != IDLE` should be +impossible; if it ever happens it is a logic error, not a normal result, yet it +is collapsed into the same empty-result path as a real indexing failure +(`IndexerThreadPool.cpp:230-232`). Prefer asserting/throwing on the +"can't happen" branch so a state-machine bug surfaces instead of masquerading as +"no lattice found". + +### [Low] `RotationIndexer::ForceLattice` mutates shared state without the mutex — `rotation_indexer/RotationIndexer.cpp:167-185` +**Category:** Bug (concurrency) +Every other `RotationIndexer` member takes `std::unique_lock ul(m)` +(`ProcessImage:118`, `RunIndexing:22`, `GetLattice:154`), but `ForceLattice` +writes `indexed_lattice` and `search_result_` with no lock. It is reached via +`IndexAndRefine::ForceRotationIndexerLattice` (`IndexAndRefine.cpp:604`). If that +can be called while worker threads are in `GetLattice`/`ProcessImage`, this is a +data race on `indexed_lattice`/`search_result_`. Add the lock for consistency +with the rest of the class even if the current call site happens to be quiescent. + +### [Low] `FFTIndexerCPU` serializes only plan create/destroy, not the shared `fftw_plan_mutex` rationale being honored at execute — verify thread-safety assumption — `indexing/FFTIndexerCPU.cpp:13,37-48,51-54` +**Category:** Bug (concurrency) / Inconsistency +A file-static `fftw_plan_mutex` guards `fftwf_plan_many_dft_r2c` and +`fftwf_destroy_plan` (correct: FFTW planning is not thread-safe). Each +`FFTIndexerCPU` instance is owned by one `IndexerThread`, and `ExecuteFFT` +(`:56`) executes the plan with no lock — which is fine *only* because each +instance has its own plan and its own host buffers, and a given instance is +single-threaded. This invariant (one indexer instance ⇒ one thread) is load- +bearing across `FFBIDXIndexer` (shared `indexer` member, `FFBIDXIndexer.h:31`), +`FFTIndexerGPU` (shared `result_fft`/`stream`), and `FFTIndexerCPU`, but is +nowhere stated. Document it on the `Indexer` base (these classes are correctly +non-copyable, which hints at it) so a future "share one indexer across the pool" +refactor doesn't silently introduce races. + +### [Low] `pixel_reference_` is copied wholesale and kept alongside the live `ScaleOnTheFly` reference — `IndexAndRefine.cpp:424-428`, `IndexAndRefine.h:77` +**Category:** Simplification / Architecture +`ReferenceIntensities` both builds the `ScaleOnTheFly` engine *and* keeps a full +`std::vector` copy (`pixel_reference_ = reference;`) purely so +the lazily-built `PixelRefine` can be constructed later +(`PixelRefineIntegrate:455-457`). For large reference sets this doubles a +potentially big buffer for the lifetime of the object, only one of which is used +depending on `GeomRefinementAlgorithmEnum`. Consider building the `PixelRefine` +engine eagerly here (the data is already available) and dropping the copy, or +holding a reference/`shared_ptr` rather than a value. Minor, but it is exactly +the kind of "kept for the experimental path" baggage the maintainer flags. + +### [Low] `IndexAndRefine` carries unused members `indexed_lattice` and `axis_` — `IndexAndRefine.h:29,31` +**Category:** Simplification (dead code) +`std::optional indexed_lattice;` and +`std::optional axis_;` are declared but never read or written in +`IndexAndRefine.cpp` (orientation state lives in `RotationIndexer` and in the +per-call `IndexingOutcome`). Dead members; remove. + +### [Low] `mosaicity` vector is written but never consumed — `IndexAndRefine.cpp:26,295` +**Category:** Simplification (dead code) +`mosaicity` is resized (`:26`) and assigned per frame (`mosaicity[msg.number] = +mos_deg;` `:295`) but, unlike `scale_cc`/`unit_cells`, has no getter and is read +nowhere. Either expose it (parallel to `GetImageCC`) or drop it; right now it is +write-only state guarded by no clear purpose. + +### [Nit] `wrap_deg_pm180` has a duplicated statement-terminating semicolon — `indexing/AnalyzeIndexing.cpp:32` +**Category:** Simplification +`return ...quiet_NaN();;` (double `;`) plus a stray comment "or std::nullopt +upstream". Harmless but sloppy in a file that is otherwise careful. + +### [Nit] `CudaRegisteredVector result_fft_reg` registers pinned memory that `ExecuteFFT` never uses — `indexing/FFTIndexerGPU.h:36`, `indexing/FFTIndexerGPU.cu:105,177` +**Category:** Simplification +`FFTIndexerGPU` registers `result_fft` as pinned host memory +(`result_fft_reg(result_fft)`), but the device→host copy at `:177` targets +`result_fft.data()` directly — which works whether or not it is registered. If +the registration is meant to accelerate that single small (`nDirections`-element) +async copy it is questionable value; if it is vestigial, drop it. Either way it +adds a `cudaHostRegister`/`Unregister` round-trip and a member with non-obvious +purpose. Confirm intent. + +### [Nit] `PixelRefine::Run` `bandwidth_radial_sq` lambda is computed but the `g.R_bw_sq` it feeds is only ever additive to `R0` — confirm no double-count with `BraggPredictionSettings.bandwidth_sigma` — `pixel_refinement/PixelRefine.cpp:353-359,503,620`, `IndexAndRefine.cpp:466` +**Category:** Bug (verify) +Bandwidth enters PixelRefine twice: (1) the prediction acceptance band via +`BraggPredictionSettings.bandwidth_sigma` (`PixelRefine.cpp:334`), and (2) the +radial profile width via `g.R_bw_sq` added to `R0²` (`:503`, `:620`). Both derive +from `data.bandwidth` (set once from `GetBandwidthFWHM()/2.3548`, +`IndexAndRefine.cpp:466`). That looks intentional and consistent (prediction +widens the shell; the profile widens the partiality), but the two uses apply the +sigma in different formulae (one as `|recip_z|·sigma`, one as `(bλ)²/2d⁴`). +Worth a one-line confirmation/test that they describe the same physical width and +are not silently double-broadening high-resolution partialities. + +### [Nit] `PixelRefine` `Run` is one ~380-line function spanning prediction, masking, R1 measurement, Ceres fit, extraction, and CC — `pixel_refinement/PixelRefine.cpp:319-700` +**Category:** Simplification +Given the maintainer's stated first principle (simple, readable code, no +superfluous code — *especially* here), the function is well-commented but long +and does six distinct jobs (numbered `---- 1..6` in comments). The comment +numbering is itself a sign each block is a candidate for a named helper (e.g. +`MeasureTangentialR1(groups, ...)`, `FitScaleAndB(groups, data)`, +`ExtractReflections(groups, ...)`, `ComputeCC(...)`). This would also let the +template instantiation (6 explicit instantiations, `:702-707`) shrink to a thin +typed shim over non-template helpers, cutting compile time. No behavior change +intended — purely readability/altitude. Note: this is the opposite of +over-engineering; the logic itself is lean. + +## Inconsistencies with rest of repo + +- **Mean-vs-median background:** `PixelRefine::EstimateLocalBackground` + (`PixelRefine.cpp:142-145`) and `BraggIntegrate2D::IntegrateReflection` + (`BraggIntegrate2D.cpp:110-114`) independently re-derive the same "use mean not + median, modern detectors put few zingers in the ring" decision, with near- + identical comments. Good that they agree; bad that the rationale and the + ring-sampling geometry are duplicated in two places that must be kept in sync. + Consider a shared local-background helper. +- **ROI sentinel handling is duplicated and type-divergent:** CPU `RunROI` + (`ROIIntegrationCPU.h:33-66`) is templated and uses + `numeric_limits::min/max`; the GPU kernel (`ROIIntegrationGPU.cu:55-57`) + hardcodes `INT32_MIN`/`INT32_MAX`. They are equivalent today *only* because + `ImagePreprocessorBuffer` is always `int32_t` (`ImagePreprocessorBuffer.h:12`), + making the CPU template generality dead. If the buffer type ever changes the two + paths diverge silently. Either drop the CPU templating (commit to int32) or make + the GPU path templated too — pick one convention. +- **Background-loop window equivalence (GPU vs CPU FFT):** the CPU prominence + window (`FFTIndexerCPU.cpp:113-115`, divisor `hi-lo`) and the GPU sliding window + (`FFTIndexerGPU.cu:44-52`, divisor `whi-wlo+1`) span the same + `[j-bg_half, j+bg_half]` index range and divide by the same count — verified + equivalent. Worth a comment cross-reference since the off-by-one conventions + differ between the two. +- **Time-units naming:** most timers use `_s` suffix and `duration`; one + uses `duration_cast>` (`IndexAndRefine.cpp:257`, + `refinement_time_s`) while neighbors use `duration` (`:165`, `:330`). + Cosmetic divergence in the same file. +- **Error-handling idiom:** the codebase standard is `JFJochException` with a + `JFJochExceptionCategory`. `FFTIndexer`/`FFTIndexerCPU` instead throw raw + `std::invalid_argument`/`std::runtime_error` (`FFTIndexer.cpp:35-46`, + `FFTIndexerCPU.cpp:28-47`), unlike `FFBIDXIndexer` + (`FFBIDXIndexer.cpp:8`) and the CUDA helpers which use `JFJochException`. These + are caught generically in the worker (`IndexerThreadPool.cpp:130`) so behavior is + OK, but the category/diagnostic is lost. + +## ffbidx touchpoints + +- `FFBIDXIndexer.h:19` includes `` and owns three ffbidx value + members: `config_runtime`, `config_persistent` (with `max_input_cells = 1`, + `max_spots = MAX_SPOT_COUNT`, `max_output_cells = 32`, + `num_candidate_vectors = 32`, `redundant_computations = true`), and + `refine::indexer` constructed from them. This is the only file that + binds the ffbidx type into a class member; the indexer is shared across all + `Run` calls of that instance (single-thread invariant — see findings). +- `FFBIDXIndexer.cpp:16` writes the reference cell into `indexer.iCellM()`; + `:34-40` fills `spotX/Y/Z` and calls `indexer.index(1, nspots)`; `:53` reads + `indexer.oCellM()`/`oScoreV()` and hands them to the shared + `PostIndexingRefinement::Refine`. `index(1, ...)` uses a single input cell + (consistent with `max_input_cells = 1`), so FFBIDX is strictly the *known-cell* + path — it requires a reference cell (`SetupUnitCell` throws otherwise, + `:8-10`). +- **Bounds:** `RunInternal` `assert`s `nspots <= MAX_SPOT_COUNT` and + `coord.size() <= MAX_SPOT_COUNT` (`:30-31`) but the spot-write loop (`:33-37`) + has no runtime clamp. Safe today only because `DatasetSettings` caps spot count + at 250 (`common/DatasetSettings.cpp:33`, far below the 64K + `MAX_SPOT_COUNT`); the `assert` vanishes in release builds, so this is a + latent buffer overrun if that cap ever rises. A runtime guard (clamp or throw) + would be safer than an assert for the ffbidx buffer write. +- `FFBIDXIndexer` correctly returns the reference (c-last) cell setting, which + `IndexAndRefine::DetermineLatticeAndSymmetry` relies on to treat FFBIDX as + "merge-safe" vs the de-novo FFT/FFTW reject path (`IndexAndRefine.cpp:117-135`). + This coupling is documented inline but is a non-obvious cross-module invariant + worth a test. diff --git a/docs/review/02-common.md b/docs/review/02-common.md new file mode 100644 index 00000000..a5d18e00 --- /dev/null +++ b/docs/review/02-common.md @@ -0,0 +1,206 @@ +# Review: common + +## Scope + +The shared-types backbone in `common/` (≈90 header/source pairs, ~12.7k LOC). Reviewed in depth: +the god-object candidate `DiffractionExperiment` (.h/.cpp, ~2100 LOC combined), the newly added +untracked `CoordTransform`, the geometry/coord-frame layer (`Coord`, `DiffractionGeometry`, +`RotMatrix`), and the plot/stats data types feeding broker + frontend (`Plot.h`, `MultiLinePlot`, +`Histogram.h`, `ADUHistogram`, `AzimuthalIntegrationProfile`, `StatusVector`), plus the +convention-setting headers (`JFJochException.h`, `JFJochMessages.h`, `IndexingSettings`). `ffbidx` +is the only in-scope third party; `gemmi`/`nlohmann` treated as black boxes. + +## Architecture overview + +`DiffractionExperiment` is the central aggregate: it owns ~13 sub-settings value objects +(`DatasetSettings`, `DetectorSettings`, `ImageFormatSettings`, `AzimuthalIntegrationSettings`, +`DetectorSetup`, `IndexingSettings`, `ScalingSettings`, `BraggIntegrationSettings`, +`FileWriterSettings`, `InstrumentMetadata`, `DarkMaskSettings`, `ROIMap`, …) and exposes a fluent +chained-setter API plus a large set of derived getters. Most of the 1717-line .cpp is one-line +delegation (`dataset.X()`, `detector_settings.Y()`); the real value it adds is the *derivation* +logic that fuses several sub-objects at once: timing (`GetFrameTime`/`GetImageTime`/ +`GetDetectorPeriod`), byte-depth/overflow/underflow inference, storage-cell counting, mode-gated +defaults, `FillMessage`, and geometry map generation (`CalcAzIntCorrRawCoord`, +`CalcSpotFinderResolutionMap`, `GetDiffractionGeometry`). This is a facade-over-composition design +rather than a true god-object — cohesion of the *sub-objects* is good — but the facade itself is +oversized: a single class with ~250 public methods is the one place every module couples to, and +its surface area is the main maintainability risk. The `// Public methods are atomic` comment +(line 86) is misleading: there is no synchronization and chained setters return `*this`. + +The geometry/coord layer is the cleanest abstraction here: `Coord` is a small value type with full +operator set, `DiffractionGeometry` is a focused fluent geometry calculator with clearly named +unit-suffixed methods, and `RotMatrix` provides rotation/inversion. Conventions are largely +consistent: unit suffixes (`_mm`, `_keV`, `_pxl`, `_recipA`, `_rad`, `_A`, `_deg`), fluent +setters returning `&`, `Get*` const getters, `Import*`/`Get*Settings` for sub-object swap, and a +single exception type (`JFJochException` + category enum). The plot types +(`MultiLinePlotStruct`, `Histogram`, `StatusVector`, `AzimuthalIntegrationProfile`, +`ADUHistogram`) are the broker↔frontend contract; they mix concerns (raw accumulation + plot +formatting + thread-safety) and have some self-inconsistencies noted below. + +The standout problem is the freshly added `CoordTransform` (untracked, 2026): it is a near-verbatim +duplicate of the existing `RotMatrix` class, is not in `CMakeLists.txt`, and is referenced by +nothing in the tree. It is dead code that, if adopted, would fork the rotation-matrix abstraction. + +## Findings + +### [High] `CoordTransform` is dead, unbuilt, near-verbatim duplicate of `RotMatrix` — `common/CoordTransform.h:10`, `common/CoordTransform.cpp:1` +**Category:** Simplification / Architecture +`CoordTransform` (untracked, added 2026-06-05) reimplements the same 3×3 rotation matrix already +provided by `RotMatrix` in `common/Coord.h:47` / `common/Coord.cpp:155`. The bodies of +`operator*`, `transpose`, `invert`, `operator!`, `arr` are character-for-character identical; only +the constructor shape differs (`CoordTransform::Rotation(alpha, dir)` static factory + zero-axis +throw vs `RotMatrix(alpha, dir)` ctor that silently yields a zero matrix). `CoordTransform.cpp/.h` +are **not** listed in `common/CMakeLists.txt` (which compiles `Coord.cpp` at line 44) and a repo +grep finds **zero** uses of `CoordTransform`, while `RotMatrix` is used across `DiffractionGeometry`, +`GoniometerAxis`, `CrystalLattice`, `IndexAndRefine`, `RotationIndexer`, `XtalOptimizer`, viewer, and +tests. Either fold the one genuine improvement (the non-zero-axis guard) into `RotMatrix` and delete +`CoordTransform`, or commit to replacing `RotMatrix` with `CoordTransform` everywhere — but two +copies of the same matrix class in the convention-setting module will confuse every downstream +consumer. As-is this is superfluous code (violates the maintainer's first principle). + +### [Medium] `DiffractionExperiment` public surface is ~250 methods on one facade — `common/DiffractionExperiment.h:56` +**Category:** Architecture +The class is the universal coupling point: it composes 13 settings objects and re-exposes nearly +every field through paired setter/getter delegators plus derived getters. Cohesion of the +*sub-objects* is fine, but the facade is the single hardest file to evolve and the reason a change +to any setting ripples through `common/`. Most methods are pure pass-throughs +(`BeamX_pxl`, `Compression`, `StorageCells`, `SampleName`, …). Consider exposing the sub-settings +objects directly (the `Import*`/`Get*Settings` accessors already exist for most of them) and +reserving `DiffractionExperiment` for the genuinely cross-cutting derivations (timing, byte depth, +`FillMessage`, geometry maps). This is a direction, not a quick fix; flagged because the brief asks +whether this is a god-class — it is a borderline one (facade, not logic dump). + +### [Medium] `// Public methods are atomic` is false and misleading — `common/DiffractionExperiment.h:86` +**Category:** Inconsistency / Bug +The comment claims atomicity, but the class has no mutex, mutable shared state (`series_id`, +`summation`, all sub-objects), and setters return `DiffractionExperiment&` for chaining — the +opposite of atomic, transactional access. Callers may assume thread-safe concurrent use. Either +remove the comment or document the real contract (single-threaded / externally synchronized). Note +`ImportDatasetSettings` (line 1025) *does* implement transactional rollback on validation failure, +which is the only place resembling atomicity — inconsistent with the rest. + +### [Medium] `GetStorageCellNumber` relies on an uncommented switch fall-through — `common/DiffractionExperiment.cpp:557` +**Category:** Bug (latent) / Simplification +``` +case DetectorMode::PedestalG1: + if (IsFixedGainG1()) + return storage_cells; // only returns when fixed-gain +case DetectorMode::PedestalG2: // PedestalG1 non-fixed silently falls through + ... +``` +When `mode == PedestalG1` and gain is *not* fixed, control falls into the `PedestalG2` branch. This +may be intended (non-fixed G1 behaves like G2), but there is no `[[fallthrough]]`/comment, so it +reads as a missing `return`/`break`. In a conventions-setting module this is exactly the kind of +construct that should be explicit. Add `[[fallthrough]];` with a one-line rationale or restructure. + +### [Medium] `Histogram::Add` silently drops bin 0; `total_count` is `int32_t` — `common/Histogram.h:54`, `:60` +**Category:** Bug +`Add(int32_t val)` accumulates only when `val > 0`, so all zero-valued samples are discarded and +never counted in `total_count`. `Percentile` then computes ranks over a population that excludes +zeros, which silently biases the percentile for any data where 0 is a legitimate value (e.g. ADU / +pixel counts). If excluding 0 is intentional it should be documented; the guard reads like an +off-by-one (likely meant `val >= 0`). Separately, `total_count` and `max_bin` are `int32_t` while +the per-bin `count` is `uint64_t` — for long detector runs `total_count` can overflow 2^31 and +`Percentile`'s `total_count - 1` math goes negative. Use `uint64_t` to match the bins. + +### [Low] `AzimuthalIntegrationProfile::Add(int64_t,int64_t)` updates `sum` but not `sum2`, and skips the lock — `common/AzimuthalIntegrationProfile.cpp:47` +**Category:** Bug / Inconsistency +The `Add(bin, value)` overload increments `sum`/`count` but never `sum2`, so any profile filled via +this path returns `NAN`-or-wrong `GetStd()`. The float/uint32 vector overload (line 56) *also* +conditionally skips `sum2`. Both diverge from the variance contract the class otherwise advertises +(`GetStd`, `calc_std`). Also the bounds check `bin >= sum.size()` on line 48 reads `sum.size()` +*before* taking the lock on line 50 — a benign data race today (size is fixed after construction) +but inconsistent with every other method that locks first. Decide whether `sum2` is part of the +contract and either always maintain it or document that std is unavailable for these paths. + +### [Low] `StatusVector` reads shared state outside the mutex — `common/StatusVector.cpp:64` +**Category:** Bug (thread-safety) +`GetActualBinning` accesses `content.size()` with no lock, while every other accessor +(`GetElement`, `ExportArray`, `Mean`, `GetMeanPerBin`, …) locks `m`. Since `AddElement` can resize +`content` concurrently from another thread, this is an unsynchronized read of a `std::vector` whose +storage may be reallocating — undefined behavior, not just a torn value. Lock it like the siblings. + +### [Low] `AzimuthalIntegrationProfile::operator+=` is not thread-safe but siblings are — `common/AzimuthalIntegrationProfile.cpp:174` +**Category:** Inconsistency +The class locks `m` in nearly every method, but `operator+=` (and `GetPlot`, `SetTitle`) touch +member vectors with no lock. The header comments `operator+=` as "Not thread safe", so this is +known — but it makes the class's threading contract per-method rather than per-object, which is a +trap for the broker code that shares these. Worth documenting the locking policy in one place. + +### [Low] Dead member `bin_to_q_1d` — `common/AzimuthalIntegrationProfile.h:22` +**Category:** Simplification +`bin_to_q_1d` is declared, never assigned, never read (grep finds only the declaration). `GetPlot`'s +1-D path slices `bin_to_q` instead. Remove it. + +### [Low] `Coord::operator==` uses exact float equality — `common/Coord.cpp:82` +**Category:** Bug (latent) / Inconsistency +Bitwise float `==` on three components. `Coord` is the base geometry type; downstream code that +compares transformed/derived coordinates will rarely get exact equality, and the operator gives no +tolerance hook. `AzimuthalIntegrationProfile::operator+=` already relies on exact vector `!=` +comparison of `bin_to_q` (line 175). Either document that `==` is strict-identity-only or provide a +tolerance-based comparison for geometric use. + +### [Low] Ambiguous "atomic" rollback only in one importer — `common/DiffractionExperiment.cpp:1025` +**Category:** Inconsistency +`ImportDatasetSettings` validates and rolls back (`dataset = tmp`) on any failure, but the sibling +importers (`ImportDetectorSettings`, `ImportImageFormatSettings`, `ImportScalingSettings`, …) assign +first and validate partially or not at all. The error-handling convention is therefore +per-importer. Pick one (validate-then-commit) and apply it uniformly, since these are the public +mutation entry points other modules depend on. + +### [Nit] Two enums named for plot X-axis units that overlap but differ — `common/Plot.h:21`, `common/MultiLinePlot.h:12` +**Category:** Inconsistency +`PlotAzintUnit { Q_recipA, TwoTheta_deg, D_A }` and +`MultiLinePlotUnits { ImageNumber, Angle_deg, Q_recipA, ADU, Grid_um, d_A }` both encode units, with +`Q_recipA` duplicated and `D_A` vs `d_A` differing only in case. These feed the same frontend; the +casing split (`D_A`/`d_A`) is an avoidable confusion in the broker↔UI contract. Consolidate or at +least align the spelling. + +### [Nit] `check_max/check_min/check_finite` are bare `#define` macros in the .cpp — `common/DiffractionExperiment.cpp:19` +**Category:** Simplification +These validation macros (no `do{}while(0)`, leak into the translation unit) could be `static` +inline helpers taking the param name + value. Minor, but this file sets the validation style copied +elsewhere (`CheckDataProcessingSettings`), so a clean form is worth establishing here. + +## Conventions this module establishes (and where they're shaky) + +- **Unit suffixes in names** (`_mm`, `_keV`, `_pxl`, `_recipA`, `_rad`, `_A`, `_deg`, `_um`, + `_mA`, `_K`): strong and consistent across setters/getters. Shaky spot: `D_A` (Plot.h) vs `d_A` + (MultiLinePlot.h) casing. +- **Fluent chained setters returning `&`** + **`Get*` const getters** + **`Import*` / `Get*Settings` + for whole-sub-object swap**: consistent. The `Detector()` accessor is overloaded to return both a + `const&` and a mutable `&` (DiffractionExperiment.h:300-301) — a value-vs-reference convention + worth being aware of. +- **Single exception type** `JFJochException` with a category enum + human string: consistent and + good. `DecodeCategory`'s `default: return ""` (JFJochException.h:128) means a future un-decoded + category produces an empty prefix silently. +- **Validation convention**: `check_*` macros that throw `InputParameterAboveMax/BelowMin/Invalid`. + Mostly consistent, but applied unevenly across the `Import*` methods (see Low finding above). +- **Thread-safety convention is per-method, not per-object**: the plot/stats types + (`StatusVector`, `AzimuthalIntegrationProfile`, `ADUHistogram`, `SetAverage`) carry a `mutable + std::mutex` and lock most—but not all—methods. There is no documented "this type is thread-safe" + contract, and the gaps (StatusVector::GetActualBinning, AzInt operator+=/GetPlot) are real. +- **Coordinate frame**: `DiffractionGeometry` is the single source of truth — lab frame from + detector pixels via `poni_rot * detectorCoord` (DiffractionGeometry.cpp:9), reciprocal via + `LabCoord/wavelength - S0`, beam along +z, `S0 = {0,0,1/λ}`. The `poni_rot` composition order + (`-rot3·z * -rot2·x * rot1·y`, line 184) is a convention with a literature reference only via the + Ewald-sphere methods; it is otherwise undocumented and easy to get wrong if duplicated (cf. the + `CoordTransform`/`RotMatrix` split). `PxlToRes(float)` is explicitly rotation-agnostic + (line 65) while `PxlToRes(x,y)` is not — same name, different frame assumptions. + +## ffbidx touchpoints + +`common/` only *names* the indexer; it never includes ffbidx headers. The coupling is the +`IndexingAlgorithmEnum { FFBIDX, FFT, FFTW, Auto, None }` enum (`IndexingSettings.h:8`), carried in +`IndexingSettings`, surfaced on `DiffractionExperiment` (`IndexingAlgorithm`/`GetIndexingAlgorithm`), +and serialized into `StartMessage::indexing_algorithm` (`JFJochMessages.h:299`). The live +resolution policy is `DiffractionExperiment::GetIndexingAlgorithm()` +(`DiffractionExperiment.cpp:1499`): `FFBIDX` downgrades to `None` when no unit cell is set; `Auto` +picks `FFTW` if `get_gpu_count()==0`, else `FFT` (no cell) or `FFBIDX` (cell present). This matches +the memory note that FFBIDX is the known-cell GPU path and FFT/FFTW the de-novo path. The +FFT-tuning knobs in `IndexingSettings` (`fft_*`, `viable_cell_min_spots`, `max_angle_from_ewald_deg`, +tolerances) are the parameters the indexer factory consumes downstream in `image_analysis/indexing/`. +No misuse of the ffbidx contract is visible from `common/`; the enum/settings surface is clean. The +only smell is that `unit_cell_angle_tolerance_deg` is a `static constexpr` while its sibling +`unit_cell_dist_tolerance_vs_reference` is a mutable per-instance field with a setter — an +inconsistency in how the two indexing tolerances are exposed. diff --git a/docs/review/03-data-path.md b/docs/review/03-data-path.md new file mode 100644 index 00000000..c2640d2f --- /dev/null +++ b/docs/review/03-data-path.md @@ -0,0 +1,210 @@ +# Review: data path (receiver / pushers / devices) + +## Scope + +Streaming data path of Jungfraujoch: +- `receiver/` — `JFJochReceiver` (base), `JFJochReceiverFPGA`, `JFJochReceiverLite`, `FrameTransformation`, + `LossyFilter`, `JFJochReceiverPlots`, `JFJochReceiverService`, `JFJochReceiverCurrentStatus`, `ImageMetadata`. +- `image_pusher/` — `TCPStreamPusher` (priority: zero-copy buffer-reuse fix), `ImagePusher` base. +- Supporting plumbing in `common/`: `ImageBuffer`, `ZeroCopyReturnValue`, `ThreadSafeFIFO`. +- `image_puller/`, `acquisition_device/`, `jungfrau/`, `preview/`, `detector_control/` skimmed for the data-flow and + ownership picture (not exhaustively audited). + +Vendored libs and `broker/gen/` treated as black boxes. `ffbidx` reached only transitively (via the indexer); no +direct touchpoints in this subsystem — see final section. + +## Architecture overview (threading & data flow) + +There are two receiver implementations sharing the abstract `JFJochReceiver`. The base owns all the cross-cutting +state — counters (`images_collected/sent/skipped`, compressed sizes), `MovingAverage` stat windows, the `ImageBuffer` +and `ImagePusher` references, the `ScanResultGenerator`, `LossyFilter`, indexer, and the `SendStartMessage`/ +`SendEndMessage`/`GetStatus` machinery (`JFJochReceiver.cpp:108-210`). Each subclass adds its own ingestion threads. + +**FPGA path** (`JFJochReceiverFPGA`): one `AcquireThread` per data stream drives the FPGA, plus a pool of +`FrameTransformationThread`s (count = `forward_and_sum_nthreads`). Each transformation thread pulls an image number +from a `ThreadSafeFIFO` work queue (`images_to_go`), waits on `AcquisitionDevice::Counters` for the frame, +pulls per-module `DeviceOutput` straight from the device buffer, runs metadata/azimuthal/spot-finding analysis, +geometry-transforms + compresses into an `ImageBuffer` slot, then hands the slot to the pusher +(`JFJochReceiverFPGA.cpp:272-503`). Frame ownership in the device buffer is released explicitly via +`FrameBufferRelease`. A separate `FinalizeMeasurement` future joins all worker futures, sends the END message, and +finalizes the writer (`:550-588`). CPU-summation mode fans each image out to nested `SummationThread`s. + +**Lite path** (`JFJochReceiverLite`): a `MeasurementThread` waits for the upstream START message off an `ImagePuller`, +configures the experiment from it, then a pool of `DataAnalysisThread`s each poll the puller's FIFO directly for +DATA/END/CALIBRATION messages, analyze, and forward into `ImageBuffer` slots (`JFJochReceiverLite.cpp:227-354`). +There is no explicit work-distribution queue — every analysis thread competes on `image_puller.PollImage()`, and the +END message is observed via the shared `end_message_received` flag. + +**Buffer handoff & zero-copy.** `ImageBuffer` is a fixed ring of fixed-size slots. `GetImageSlot()` returns a +`ZeroCopyReturnValue*` (pointer into a stable `std::vector`), the worker serializes/compresses the +CBOR image directly into the slot, marks `ReadyToSend()`, then calls `image_pusher.SendImage(*loc)`. For TCP the slot +is *not* copied: `TCPStreamPusher::SendImage(ZeroCopyReturnValue&)` enqueues the slot pointer into a per-connection +`ThreadSafeFIFO`; the per-connection `WriterThread` sends it and calls `z->release()` (returning the slot to the ring) +only after the bytes are handed to the kernel — and for `MSG_ZEROCOPY` sends, only after the kernel errqueue +completion notification (`TCPStreamPusher.cpp:351-388,544-568`). The synchronous `SendImage(ptr,size,number)` overload +takes a transient caller buffer (`z == nullptr`) and must never zero-copy it — this is the recently-fixed hazard. + +**Plot production** (`JFJochReceiverPlots`): both receivers call `plots.Add(message, az_int_profile)` per image from the +worker threads. Internally each metric is a `StatusVector` (documented as internally thread-safe), so the per-metric +appends need no outer lock; only the `std::map` (`roi_m`, a `shared_mutex`) and the +`az_int_profile` / XFEL maps (`m`) are mutex-guarded (`JFJochReceiverPlots.h:34-88`). The broker pulls snapshots via +`GetPlots`/`GetPlotRaw`/`GetAzIntProfile`, and `GetStatus` reads aggregate means (indexing rate, bkg estimate). So plot +data is produced incrementally on the hot path and read concurrently by broker/UI threads — characterized here; +frontend↔broker line is out of scope. + +## Findings + +### [High] `LossyFilter::ApplyFilter` mutates shared RNG/counter but the body is unsynchronized — `receiver/LossyFilter.cpp:19-29` +**Category:** Bug +`ApplyFilter` is called concurrently from every FPGA `FrameTransformationThread` and every Lite `DataAnalysisThread` +(`JFJochReceiverFPGA.cpp:438`, `JFJochReceiverLite.cpp:308`). `RollDice()` does lock `random_m`, but the rest of +`ApplyFilter` does not: `message.number = image_number++` reads-modifies-writes the `std::atomic image_number` +via `++` (atomic, OK on its own) but the *decision* to renumber and the renumber itself are not atomic together. Two +threads can interleave between `RollDice()` returning true and the `image_number++`, so the renumbered sequence is +fine numerically (atomic post-increment) but is assigned in nondeterministic order relative to original image order — +filtered serial-MX output image numbers are not guaranteed monotonic/contiguous per source order. More importantly the +`p == 1.0` fast path returns without touching `image_number`, while the `p < 1.0` path renumbers from a separate +counter, so for `0 < p < 1` the emitted `message.number` is decoupled from the original frame number across threads +with no ordering guarantee. Worth confirming this matches the intended serial-MX renumbering contract; if monotonic +per-arrival numbering is expected, the dice-roll + assignment needs to be one critical section. + +### [High] `images_sent` is incremented at enqueue time, not send time (TCP) — `receiver/JFJochReceiverFPGA.cpp:484-485`, `JFJochReceiverLite.cpp:331-332` +**Category:** Bug / Inconsistency +Both receivers do `image_pusher.SendImage(*loc); ++images_sent;` with the comment "Handle case when image not sent +properly". For the TCP pusher `SendImage(ZeroCopyReturnValue&)` is *asynchronous* — it only enqueues onto the +per-connection FIFO and returns void; the actual send can later fail (broken socket → `z->release()` with no send, +`TCPStreamPusher.cpp:552-555,1071-1094`). So `images_sent` over-counts whenever a connection breaks mid-run or the +2-second enqueue deadline expires and the slot is dropped. The authoritative "written" count comes from +`GetImagesWritten()` = `total_data_acked_ok` (ACK-based), so the receiver-side `images_sent` and the pusher-side acked +count can disagree; the "wrong number of images" symptom the zero-copy commit chased can also surface here. The +file-based pushers return a bool from `SendImage(z)` (via the base overload) but that bool is discarded for the +`ZeroCopyReturnValue` overload, which is `void` — there is no way for the caller to know the async send failed. + +### [Medium] `ZeroCopyCompletionThread` reads `c->zc_pending.empty()` without holding `zc_mutex` — `image_pusher/TCPStreamPusher.cpp:392` +**Category:** Bug (data race) +The loop condition `while (c->active || !c->zc_pending.empty())` inspects `std::deque::empty()` without `zc_mutex`, +while `EnqueueZeroCopyPending`/`ReleaseCompletedZeroCopy`/`ForceReleasePendingZeroCopy` mutate the deque under the +lock (`:362-381,351-360`). Concurrent `push_back`/`pop_front` against an unsynchronized `empty()` is a data race (UB), +even if in practice it only affects the loop's exit timing. Since `StopDataCollectionThreads` already drains via +`WaitForZeroCopyDrain` + `ForceReleasePendingZeroCopy` before joining `zc_future`, the unlocked check is also +redundant for correctness — gating purely on `c->active` (atomic) and letting the drain/force path handle the tail +would remove the race. The same loop drives `c->fd.load()` and `c->broken` (both atomic, fine). + +### [Medium] FPGA `SendStartMessage` runs before acquisition threads, but END/finalize ordering differs from Lite — `receiver/JFJochReceiverFPGA.cpp:111-137` vs `JFJochReceiverLite.cpp:94-157` +**Category:** Architecture / Inconsistency +The two receivers implement the same lifecycle (start msg → images → end msg → finalize writer → check buffer +returned) but in structurally different places and orders. FPGA sends START in the *constructor* before spawning +acquire threads and runs END/finalize in a dedicated `FinalizeMeasurement` future joined by `StopReceiver`. Lite does +START *and* END inside its single `MeasurementThread`, and crucially Lite calls `CheckIfBufferReturned` *before* +`image_pusher.Finalize()` (`:141-149`) while FPGA also does buffer-return-then-Finalize (`:575-582`) — those match, +but the surrounding error handling diverges: Lite's `MeasurementThread` `throw`s out of the future on buffer-return +timeout, FPGA's `FinalizeMeasurement` also throws, yet FPGA additionally cancels devices on END. The duplicated +"send end / check buffer / finalize / set status" tail (≈15 lines) is copy-pasted with subtle drift; a shared +protected helper on the base (e.g. `FinalizeWriterAndBuffer()`) would remove the divergence risk. This is the clearest +instance of the FPGA-vs-Lite copy-paste-with-drift the review asked to look for. + +### [Medium] `WriterThread` does not break the loop on a broken connection — `image_pusher/TCPStreamPusher.cpp:544-568` +**Category:** Smell / timeout correctness +Once `c->broken` is set, the writer keeps looping, draining the queue element-by-element and calling +`e.z->release()` per element (`:552-555`) until it finally sees `e.end`. Each `GetBlocking()` blocks until something is +enqueued, so during the 2s producer enqueue window (`SendImage` `PutTimeout` loop, `:1076-1089`) the thread spins +through release-and-continue. Functionally it drains correctly, but on a broken socket it would be cheaper and clearer +to flush the queue once and exit; as written, shutdown relies on the producer eventually pushing `{.end=true}` via +`StopDataCollectionThreads`. Confirm there is no path where `active` is cleared and the `{.end}` sentinel fails to +enqueue (the `PutTimeout` 200ms fallback at `:851-856` clears + re-Puts, which looks correct, but the interaction is +subtle). + +### [Low] `JFJochReceiverFPGA::AcquireThread` logs the same exception twice — `receiver/JFJochReceiverFPGA.cpp:192-196` +**Category:** Inconsistency +In the second catch block, `logger.ErrorException(e); Cancel(e); logger.ErrorException(e);` logs `e` twice (and +`Cancel(e)` itself logs again at `JFJochReceiver.cpp:221-222`), so a single device error produces three log entries. +The first catch block (`:179-184`) logs it once. Minor, but inconsistent and noisy. + +### [Low] `JFJochReceiver::GetStatus()` reads many non-atomic / mutex-guarded members without their locks — `receiver/JFJochReceiver.cpp:80-105` +**Category:** Bug (benign race) / Inconsistency +`GetStatus()` is `const` and is called both from worker threads (after every image) and indirectly from the broker via +`current_status`. It reads `max_image_number_sent` (guarded elsewhere by `max_image_number_sent_mutex`) and +`max_delay` (guarded by `max_delay_mutex`) without taking those mutexes, and reads plain `int64_t`/`std::optional` +members non-atomically. The atomic counters are fine; the mutex-guarded scalars are technically racy reads. In +practice these are status/telemetry values so a torn read is cosmetic, but it is inconsistent with the locking the +same fields get on the write side. Either make them atomic or take the locks. + +### [Low] `IsConnectionAlive` is racy as a pre-send gate — `image_pusher/TCPStreamPusher.cpp:169-191`, used at `:1047,1199` +**Category:** Smell +`SendImage(ptr,size,number)` checks `IsConnectionAlive(c)` then sends; the connection can break between the check and +the send, which is unavoidable and handled by `SendAll`'s error paths — so the check is an optimization, not a +guarantee. `SendCalibration` re-checks `broken/connected/active` four times before and after taking `send_mutex` +(`:1165-1203`); this is defensive but verbose (≈40 lines of repeated guards). A single re-check under the lock would be +equivalent given `broken`/`connected`/`active` are atomics. Simplification opportunity, not a bug. + +### [Low] `ImageBuffer::GetImageSlot` waits on a preview reader while holding the global mutex — `common/ImageBuffer.cpp:111-135` +**Category:** Architecture +When a recycled slot still has `readers > 0` (a preview/JPEG copy in flight), `GetImageSlot` does +`cv_preview_done.wait(ul, …)` while holding `m`. The wait releases `m`, so it is not a self-deadlock, but it stalls +*every* other producer/consumer that needs the buffer mutex (status reads, ReadyToSend, ReleaseSlot) until the preview +memcpy of one image completes. `GetImage` does drop the lock for its memcpy (`:225-230`), so the window is just the +copy duration, but on a large image at high frame rate this serializes the hot path behind a UI read. Worth noting as a +latency coupling between preview and acquisition. + +### [Nit] `nexus_mask` declared and never used — `receiver/JFJochReceiver.cpp:125` +**Category:** Simplification +`std::vector nexus_mask;` in `SendStartMessage` is dead. Remove. + +### [Nit] `SummationThread` returns `int64_t` but always returns 0 — `receiver/JFJochReceiverFPGA.cpp:241-270` +**Category:** Simplification +The return value is never read (`futures … f.get()` discards it, `:345-346`). Should be `void`. The maintainer's +"no superfluous code" principle applies. + +### [Nit] `FillNotCollectedModule` / `err_value` const-correctness and naming — `receiver/FrameTransformation.cpp:138-140` +**Category:** Nit +Minor: `precompression_buffer`/`compressed_buffer`/`err_value` are `std::vector` and repeatedly C-cast to typed +pointers; consistent `reinterpret_cast` or a typed view would read cleaner. Also `image_mode`/`pixel_depth` are +initialized in the member-init list in an order that does not match declaration order (`pixel_depth` declared after +`image_mode`’s use); harmless here but compilers with `-Wreorder` will warn. + +## FPGA vs Lite receiver: shared vs duplicated + +**Shared (good):** All telemetry/counters, `SendStartMessage`/`SendEndMessage`, `GetStatus`, `GetFinalStatistics` +skeleton, the `ImageBuffer` slot protocol, `LossyFilter`, `ScanResultGenerator`, `JFJochReceiverPlots.Add`, and the +`Cancel` base behavior all live on `JFJochReceiver`. The two subclasses correctly funnel through these. + +**Duplicated with drift (risk):** The per-image *forwarding tail* is near-identical but copy-pasted in both +subclasses: get slot → `nullptr` ⇒ `writer_queue_full=true` → else serialize into slot → `SetImageNumber/SetImageSize/ +SetIndexed` → `ReadyToSend()` (FPGA) / `release()` (Lite dark-mask) → optional preview/metadata sockets → `SendImage` ++ `++images_sent` else `release()` → `UpdateMaxImageSent`. Compare `JFJochReceiverFPGA.cpp:440-489` against +`JFJochReceiverLite.cpp:310-336`. They differ in small ways (FPGA does compression into the slot and computes +`compression_ratio` inline; Lite serializes an already-compressed `DataMessage`), but the slot lifecycle and the +preview/metadata/pusher dispatch are the same logic written twice. The MaskThread (`Lite:179-225`) duplicates a third, +slightly different copy. The finalize tail (send end → CheckIfBufferReturned → Finalize writer → set status) is also +duplicated (see Medium finding above). A shared `ForwardImageToWriter(ZeroCopyReturnValue&, DataMessage&)` helper on the +base would collapse three near-copies into one and remove the drift surface (e.g. only FPGA guards the +`image_collection_efficiency == 0` empty-image case, only Lite forwards calibration messages). + +**Efficiency / progress semantics differ legitimately:** FPGA derives efficiency from packet counts and progress from +device frame counters; Lite derives both from `images_collected` / `max_image_number_received` vs `GetFrameNum()`. +That divergence is inherent (Lite has no packet accounting) and is fine. + +## Inconsistencies with rest of repo + +- **Logging style mismatch.** `TCPStreamPusher` builds log messages with string `+` concatenation throughout + (`"... " + std::to_string(x) + ...`, e.g. `:127,565,694`) while the rest of the receiver subsystem uses the fmt-style + `logger.Info("... {}", x)` API (e.g. `JFJochReceiver.cpp:48`, `JFJochReceiverService.cpp:223`). `SendCalibration` in + the same file uses the fmt style (`:1166`), so the file is internally inconsistent too. +- **`SendImage(z)` return type asymmetry.** Base `ImagePusher::SendImage(const uint8_t*, size_t, int64_t)` returns + `bool`; the `ZeroCopyReturnValue` overload returns `void` (`ImagePusher.h:43-44`). Callers therefore cannot detect + async TCP send failures (ties into the `images_sent` over-count finding). Other pushers (HDF5/ZMQ) also override the + void overload, so the asymmetry is repo-wide, but it is the root of the silent-failure path here. +- **Mutex-guarded-vs-atomic inconsistency in the base receiver.** `max_image_number_sent`/`max_image_number_received`/ + `max_delay` use a dedicated mutex on write but are read lock-free in `GetStatus`/`GetProgress`. Elsewhere in the repo + similar telemetry is plain atomic. Pick one idiom. +- **`ThreadSafeFIFO::Get` decrements `utilization` but not via the blocking path's accounting** is consistent within the + class; no issue, just noting the FIFO is the shared primitive both paths rely on and it is sound (all ops mutex-guarded). + +## ffbidx touchpoints (if any) + +None directly in the data path. `ffbidx` is reached only transitively through `IndexAndRefine`/`IndexerThreadPool` +(`JFJochReceiver.h:23-24`, member `indexer`), which the receivers feed via `MXAnalysisAfterFPGA`/`MXAnalysisWithoutFPGA` +and finalize via `indexer.FinalizeRotationIndexing()` / `GetConsensusUnitCell()` in `SendEndMessage` +(`JFJochReceiver.cpp:165-176`). The indexer object is shared across all worker threads by reference; its internal +thread-safety is owned by the image-analysis/indexing subsystem (separate review). No ffbidx API is called from +receiver/pusher/device code in scope, so no ffbidx-specific misuse to flag here. diff --git a/docs/review/04-io-serialization.md b/docs/review/04-io-serialization.md new file mode 100644 index 00000000..b3e05c37 --- /dev/null +++ b/docs/review/04-io-serialization.md @@ -0,0 +1,132 @@ +# Review: I/O + serialization + +## Scope + +- `frame_serialize/` — CBORStream2Serializer / CBORStream2Deserializer + tinycbor (vendored) +- `writer/` — HDF5Objects, HDF5NXmx, and the plugin tree +- `reader/` — JFJochHDF5Reader (entry point only; deep reader internals not exhaustively reviewed) +- `compression/` — JFJochBitShuffleCompressor / JFJochDecompress / JFJochZstdCompressor + +Third-party vendored code (tinycbor, bitshuffle, lz4, zstd, HDF5 library, nlohmann/json, gemmi) is treated as a black box — findings concern first-party usage only. + +--- + +## Architecture overview + +The wire format is CBOR (tinycbor). The serializer (`CBORStream2Serializer`) encodes directly into a caller-supplied buffer via low-level tinycbor encoder calls collected entirely inside a single ~930-line `.cpp` file. The deserializer is a free function `CBORStream2Deserialize` backed by ~1500 lines of anonymous-namespace helpers in `CBORStream2Deserializer.cpp`. Both files follow the same structural idiom: each message type (START, IMAGE, END, CALIBRATION, METADATA) has a corresponding serialise path (a sequence of `CBOR_ENC(…)` calls) and a deserialise path (a `while (ProcessXMessageElement(…)) {}` loop). That symmetry is easy to follow and is the main architectural strength. + +The compression layer is clean: `JFJochBitShuffleCompressor` wraps bshuf+LZ4/zstd in one class; the symmetric decompressor is a set of inline template helpers in `JFJochDecompress.h`. The split (compressor is a class, decompressor is free functions) is consistent enough because only compression needs persistent state (the zstd RLE state machine). The high-performance block decompressor (`JFJochDecompressHperfPtr`) is a useful optimisation that is correctly kept separate from the reference path. + +The HDF5 layer (`HDF5Objects`) is a thin RAII wrapper over the HDF5 C API with ref-counted copy semantics on `HDF5Id`. `HDF5Object` acts as both the common base for files, groups, and datasets and as a mixin that injects convenience methods (`SaveScalar`, `SaveVector`, `Attr`, `Units`, `Transformation`). This is compact but slightly blurry: `HDF5Object` carries read methods (`ReadVector`, `GetFloat`, `GetInt`) that are only meaningful on a group/file, not on a dataset — the hierarchy is flat where a two-level hierarchy (container vs. leaf) would be cleaner. + +`HDF5NXmx` constructs the complete NXmx master file in a two-phase pattern (constructor writes static fields, `Finalize()` writes post-run fields and renames the `.tmp` file). This is a sound pattern. The destructor silently removes the `.tmp` on abnormal exit, which is the right behaviour. + +--- + +## Findings + +### [Medium] Fixed 16 384-byte string buffer in deserializer — `frame_serialize/CBORStream2Deserializer.cpp:15` +**Category:** Bug + +`GetCBORString` allocates a fixed `std::vector(16384)`, queries the actual length with `cbor_value_get_string_length`, then calls `cbor_value_copy_text_string` with `len = s.size() - 1 = 16383`. If the serialised string is longer (e.g. a large `user_data` JSON blob encoded as CBOR text, or a very long `file_prefix`), the copy silently truncates and the returned length reflects only what was copied. The serialiser itself encodes `user_data` as a JSON string via `CBOR_ENC(encoder, "user_data", message.user_data.dump())` (serializer:839) which can easily exceed 16 kB for large experiment metadata. The fix is to read the actual length first and allocate accordingly, which is the standard two-pass pattern for tinycbor string reads. + +--- + +### [Medium] `AppendImage` patches CBOR bytes manually — `frame_serialize/CBORStream2Serializer.cpp:914` +**Category:** Bug / Architecture + +`AppendImage` writes a CBOR byte-string header directly by index-patching `buffer[curr_size - 2]` then memcpy-ing a big-endian length and advancing `curr_size`. This works only when the preceding item was serialised with exactly one trailing "empty" byte that can be overwritten (the break code `0xFF` from an indefinite-length container). The invariant is fragile and undocumented; changing the order of fields in `SerializeImage` or the CBOR structure could silently corrupt frames. `GetImageAppendOffset` returns `curr_size + sizeof(size_t) - 1` without any explanation of what that offset means or why `sizeof(size_t)` is relevant at that point. A comment or a better abstraction (e.g. a reserved-slot pattern with a known placeholder) would substantially reduce risk. + +The endianness guard at line 920 uses `#ifdef LITTLE_ENDIAN` (a POSIX macro that may or may not be defined by the platform headers). The standard and portable approach is `#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__` or a `std::endian` check. On a platform where `LITTLE_ENDIAN` is not defined (e.g. some MSVC builds) the bswap is silently skipped and frames are corrupted. + +--- + +### [Medium] `cbor_value_advance` called without error check in unknown-field skip paths — `frame_serialize/CBORStream2Deserializer.cpp:179,462,521,579,652,925,958,1285,1305,1440` +**Category:** Bug / Inconsistency + +The "skip unknown field" branches (`else { cbor_value_advance(&map_value); }`) discard the `CborError` return value, while all other advance calls wrap `cborErr(…)`. For unknown simple scalars this is benign, but for compound values (maps, arrays, tagged types) a single `cbor_value_advance` does not recurse — tinycbor's advance only moves to the next peer at the same level. An unknown key whose value is a map or array leaves the iterator in the wrong position, causing all subsequent key reads to parse garbage. The two-call pattern used for the top-level message skip (lines 829–830, 876–877, 1284–1285) `cbor_value_advance` twice — once for a possible tag and once for the value — but still doesn't handle the multi-value case. A safe skip requires `cbor_value_skip_tag` followed by the appropriate recursive advance. Use `cbor_value_advance` for nested containers only if the tinycbor version deployed actually recurses (check the version; tinycbor's `cbor_value_advance` does recurse for containers, but only if `CBOR_PARSER_MAX_RECURSIONS` allows it and the error is checked). + +--- + +### [Medium] `gmtime` is not thread-safe — `frame_serialize/CBORStream2Deserializer.cpp:89` +**Category:** Bug + +`GetCBORDateTime` calls `gmtime(&t)` which returns a pointer to a single static buffer. If two threads deserialise `CborUnixTime_tTag` simultaneously the behaviour is undefined. The POSIX replacement is `gmtime_r`. This matters in any multi-threaded reader context. + +--- + +### [Low] Duplicate `SaveVectorIfMissing` call for `imageScaleFactor` — `writer/HDF5NXmx.cpp:901,903` +**Category:** Bug (benign / wasted work) + +`EndResultVectors` calls `SaveVectorIfMissing(*hdf5_file, "/entry/MX/imageScaleFactor", end.image_scale_factor)` on lines 901 and 903. The second call is a no-op because `SaveVectorIfMissing` checks `Exists()` first, but it is clearly a copy-paste error that should be removed. + +--- + +### [Low] `ProcessPixelMaskElement` uses `continue` inside a map loop — `frame_serialize/CBORStream2Deserializer.cpp:1031` +**Category:** Bug + +When `image.GetMode() != CompressedImageMode::Uint32`, the code does `continue`. Inside a `while (!cbor_value_at_end(&map_value))` loop the CBOR iterator is not advanced past the current entry — the key has already been consumed by `GetCBORString` and the multidim array by `GetCBORMultidimTypedArray`, so the position after `continue` is actually correct. However the intent is obscured. More importantly, if `GetCBORMultidimTypedArray` throws (malformed data), `cbor_value_leave_container` at line 1039 is still called and may produce an invalid state. A `try/catch` inside the loop, or an explicit `cbor_value_advance` in the skip branch, would make the contract clear. + +--- + +### [Low] `GetCBORArrayLen` fails on indefinite-length arrays — `frame_serialize/CBORStream2Deserializer.cpp:95` +**Category:** Inconsistency + +`cbor_value_get_array_length` returns `CborErrorUnknownLength` for indefinite-length arrays (confirmed in `tinycbor/cbor.h:570`). `GetCBORArrayLen` passes the error through `cborErr`, so an indefinite-length array anywhere in the wire stream throws. The serialiser only writes definite-length arrays (`cbor_encoder_create_array(&encoder, &arrayEncoder, spots.size())`), so in practice this is consistent for self-generated messages. However the error message at throw site says "Array expected" rather than "Array of known length expected", which is misleading during debugging. + +--- + +### [Low] `Reflection.image_number` decoded as float while encoded as float — `frame_serialize/CBORStream2Deserializer.cpp:509` +**Category:** Inconsistency + +`Reflection.image_number` is declared as `float` (`common/Reflection.h:16`, comment: "Can be in-between for 3D integration") and serialised as `cbor_encode_float` via `CBOR_ENC(mapEncoder, "image", r.image_number)`. This is correct. However the serialiser comment in `Serializer.cpp:271` encodes `CBOR_ENC(mapEncoder, "image", r.image_number)` as a `float` while `CBOR_ENC(mapEncoder, "image", spot.image)` at line 233 encodes `SpotToSave::image` — which is an integer — also as `"image"`. The shared key name across two different message types (spots and reflections) and the mixed integer/float encoding for the field named `"image"` creates potential confusion for downstream consumers. + +--- + +### [Low] `bshuf_read_uint32_BE` used without declaration — `compression/JFJochDecompress.h:50` +**Category:** Inconsistency + +`JFJochDecompressHperfPtr` calls `bshuf_read_uint32_BE` (line 50) but only `bshuf_read_uint64_BE` is declared `extern "C"` in the file (lines 20-22). `bshuf_read_uint32_BE` is declared in `compression/bitshuffle/bitshuffle_internals.h`, which is included earlier in the file. This is a linkage inconsistency: the 64-bit variant gets an explicit `extern "C"` declaration, the 32-bit variant relies on the transitive include from `bitshuffle_internals.h`. If that include order changes, only the 32-bit call silently picks up the wrong linkage. Declare both or neither explicitly. + +--- + +### [Nit] `CBOR_ENC_COMPRESSED` silently ignores `BSHUF_ZSTD_RLE` vs `BSHUF_ZSTD` distinction on the wire — `frame_serialize/CBORStream2Serializer.cpp:70` +**Category:** Inconsistency + +Both `BSHUF_ZSTD` and `BSHUF_ZSTD_RLE` are serialised as the string `"bszstd"`. On deserialisation (`CBORStream2Deserializer.cpp:350`) only `BSHUF_ZSTD` is restored. The deserialised data is still correct because the compressed bitstream is compatible (both use the same ZSTD frame format), but the RLE variant is lost and the decompressor will not apply the RLE-aware path. If RLE is performance-critical this is a silent accuracy-of-mode loss; if it is not used in practice the enum value is dead code. + +--- + +### [Nit] `HDF5NXmx::Finalize` checks `filesystem::exists` after closing the file — `writer/HDF5NXmx.cpp:833` +**Category:** Bug (race) + +The sequence is: close file → `exists(filename)` → `rename(tmp, filename)`. Between `exists` and `rename`, another process could create `filename`. The `rename(2)` syscall is atomic on POSIX and will overwrite the target if `overwrite == true`; the guard is therefore redundant in the overwrite case and racy in the non-overwrite case. A better pattern: attempt `rename` unconditionally with `overwrite == true`, or use `O_EXCL`-style link tricks for the non-overwrite path. + +--- + +### [Nit] `SaveScalar` defined twice with the same body in header — `writer/HDF5Objects.h:337,349,357` +**Category:** Simplification + +There are two free-function `SaveScalar` templates and one `HDF5Object::SaveScalar` method template with identical bodies. The free functions exist because some call sites pass the parent by const reference; the member function exists for chained call sites. Consolidating to one implementation called by the other would halve the maintenance surface. + +--- + +## Serializer/deserializer symmetry notes + +Overall symmetry is good: every field written by the serialiser has a corresponding branch in the deserialiser. The following asymmetries are noted: + +- **`az_int_bin_to_phi`**: serialised conditionally (`if (!message.az_int_bin_to_phi.empty())`, serializer:691) and deserialised unconditionally (the key is handled, deserializer:1274). Symmetric. +- **`end_time` rational**: the `end_time` rational in image messages is serialised (`timestamp + exptime`, serializer:772) but never decoded in `ProcessDataMessageElement`. This is intentional (consumers can derive it), but worth noting. +- **`az_int_profile_count`**: serialised as `TagUnsignedInt64BitLE` typed array (serializer:780) and decoded with `GetCBORUInt64Array` (deserializer:719). Symmetric. +- **`pixel_mask`**: serialiser always uses `BSHUF_LZ4` (serializer:421); deserialiser decompresses whatever algorithm is found in the frame (deserializer:1034). Asymmetric by design — the deserialiser is more permissive, which is appropriate. +- **`az_int_map`**: same pattern as pixel_mask. +- **Spots/reflections in metadata**: the serialiser writes spots and reflections only for non-metadata images (`!metadata_only` guard, serializer:841–845). The metadata deserialiser path (`ProcessMetadataImagesElement`) calls `ProcessDataMessageElement` which handles `spots` and `reflections` keys — they are ignored in practice because they are not present, but if a future change adds them to metadata the deserialiser already handles them silently. + +--- + +## Inconsistencies with rest of repo + +- **Error check style**: all direct `cbor_value_advance` calls in the known-field helpers use `cborErr(...)`, but the unknown-field skip branches at ~10 sites drop the return value. This is inconsistent with the repo convention that all fallible calls are checked. +- **Naming**: the serialiser uses all-uppercase macro-style names (`CBOR_ENC`, `CBOR_ENC_DATE`, etc.) for what are actually `inline` functions. The rest of the codebase uses `CamelCase` for free functions. The macros-that-are-functions are confusing because they look like they can expand tokens but cannot. +- **`HDF5NXmx` group creation in `EndResultVectors`**: creates `/entry/MX` and `/entry/instrument/detector/detectorSpecific` groups again (lines 888–889, 866–867) that were already created in the constructor. HDF5 group creation is idempotent when using `H5Gcreate` with `H5P_DEFAULT`, but it is redundant and adds confusion about which phase is responsible for group lifecycle. +- **`SaveVectorIfMissing` vs `SaveVector`**: the `SaveVectorIfMissing` helper checks `Exists()` before writing to avoid duplicate writes in `EndResultVectors`. This is a workaround for the fact that the VDS path (called earlier in `Finalize`) creates some of the same datasets. It would be cleaner to track which datasets have been written rather than relying on a file-system existence check. diff --git a/docs/review/05-broker.md b/docs/review/05-broker.md new file mode 100644 index 00000000..fb6f8a74 --- /dev/null +++ b/docs/review/05-broker.md @@ -0,0 +1,265 @@ +# Review: broker (control plane / OpenAPI) + +## Scope +`broker/` control-plane subsystem: `JFJochBrokerHttp.{cpp,h}` (HTTP transport), +`JFJochStateMachine.{cpp,h}` (domain/state machine), `JFJochServices.{cpp,h}` +(facade over receiver/detector), `OpenAPIConvert.{cpp,h}` (domain <-> generated +model mapping), `JFJochBrokerParser.cpp` and `jfjoch_broker.cpp` (config + main), +plus the relationship to `broker/gen/` (generated OpenAPI artefacts) and the spec +`broker/jfjoch_api.yaml`. Third-party libs (httplib, nlohmann, pistache) treated as +black boxes; `broker/gen/` internals not reviewed line-by-line. + +## How the OpenAPI contract is generated & integrated (findings) + +The single source of truth is **`broker/jfjoch_api.yaml`** (OpenAPI 3.0.3, +`info.version` currently `1.0.0-rc.148`, `broker/jfjoch_api.yaml:25`). + +Two consumers are generated from it with the OpenAPI Generator 7.20.0 +(`broker/gen/.openapi-generator/VERSION`): + +1. **Python client** — scripted and reproducible via `gen_python_client.sh` + (`-g python`, output to `python-client/`). This is the only documented/automated + generation path. +2. **C++ server artefacts** in `broker/gen/` — generated with the + `cpp-pistache-server` generator (`broker/gen/README.md` names Pistache; + `broker/gen/api/DefaultApi.cpp:23` takes a `Pistache::Rest::Router`). **There is + no script and no CI step that regenerates these** — `.gitlab-ci.yml` / + `gen_python_client.sh` only mention the Python client and `jfjoch_api.yaml`. + +Crucially, only a *subset* of the generated C++ is actually used. `broker/CMakeLists.txt` +compiles **only `gen/model/`** into the `JFJochAPI` static library +(`broker/CMakeLists.txt:5-7`) and includes `gen/api` headers +(`broker/CMakeLists.txt:21`), but the broker executable links **`httplib::httplib`** +(`broker/CMakeLists.txt:18`), not Pistache. The actual HTTP server, routing, request +parsing and error handling are **hand-written** in `JFJochBrokerHttp.cpp` on top of +cpp-httplib. So the generated `gen/api/ApiBase.*`, `gen/api/DefaultApi.*`, +`gen/impl/DefaultApiImpl.*`, `gen/main-api-server.cpp` and `gen/CMakeLists.txt` +(which `ExternalProject_Add`s Pistache) are **dead code**: never compiled, never +included anywhere (verified: no source references `DefaultApi`, `ApiBase`, +`DefaultApiImpl` or `main-api-server` outside `gen/` itself). + +Confirming the drift, this generated dead code is also **not even tracked in git**: +`git ls-files broker/gen` returns only the 116 `model/` files; `git status` shows +`broker/gen/api/`, `broker/gen/impl/`, `broker/gen/main-api-server.cpp`, +`broker/gen/CMakeLists.txt`, `broker/gen/README.md`, `broker/gen/.openapi-generator/` +as **untracked** — i.e. they are local regeneration debris. And the local regen +used a *newer* spec than the committed one: `gen/api/DefaultApi.cpp:5` reports +document version `1.0.0-rc.152`, whereas the committed `gen/model/Broker_status.h` +and the committed `jfjoch_api.yaml` are `1.0.0-rc.148`. The generated `model/` +classes carry `to_json`/`from_json`, `validate()`, and typed getters/setters; the +hand-written `OpenAPIConvert.cpp` maps each of those to the real domain types in +`common/`, `jungfrau/`, `image_analysis/`, etc. + +## Architecture overview + +**State machine.** `JFJochStateMachine` owns the experiment state +(`enum class JFJochState {Inactive, Idle, Measuring, Error, Busy, Calibration}`, +`common/BrokerStatus.h:9`) plus the `DiffractionExperiment`, calibration, pixel mask, +ROI, scan result and various settings. Long-running operations (initialize, pedestal, +measurement) run on a single `std::future measurement`; `SetState()` is the one +choke-point that updates both the atomic `state` and the `broker_status` snapshot +(`JFJochStateMachine.cpp:590-600`). Transitions are encoded imperatively inside each +operation (e.g. `MeasurementThread` Busy->Measuring->Idle/Error, +`JFJochStateMachine.cpp:386-450`), not in a declarative transition table, so the legal +transitions and invariants are spread across many methods. Guard checks (`IsRunning()`, +`state != Idle`) are repeated at the top of nearly every mutator. The transitions +generally follow a co-operative cancel model: a worker unlocks `m`, blocks in +`services.Stop()`, then re-locks (`JFJochStateMachine.cpp:186-190`), allowing `Cancel()` +to set `cancel_sequence` and call `services.Cancel()` while the worker runs. + +**HTTP layer.** `JFJochBrokerHttp` holds the `JFJochStateMachine`, `JFJochServices` and +a `Logger`, and registers ~60 routes in `register_routes()` +(`JFJochBrokerHttp.cpp:143-357`). Three binder helpers (`bind_noarg`, `bind_json`, +`bind_req`) wrap handlers with try/catch; routes with query parameters are inlined as +lambdas because the helpers don't model query args, leading to a lot of repeated +try/catch boilerplate (~15 copies of the same catch block). Each handler converts the +generated model to a domain type via `Convert(...)`, calls the state machine, and on the +way out converts back and serialises with nlohmann. The `main` wires everything together +and runs `httplib::Server::listen` on a worker-thread-per-connection model +(`jfjoch_broker.cpp:120-138`), which is why state-machine thread-safety matters. + +**Conversion layer.** `OpenAPIConvert.cpp` (1145 lines) is an overload set of free +`Convert()` functions, one direction per type, entirely hand-written. Enums are mapped +with explicit `switch`es; string enums (state, severity, compression, color, plot type) +are mapped with literal string comparisons. This is the single largest drift surface: it +has no compile-time link to the spec, so any field added/renamed in `jfjoch_api.yaml` +silently goes unmapped. + +## Findings + +### [High] `GetMeasurementStatistics()` reads `experiment` with no lock — data race — `broker/JFJochStateMachine.cpp:489-521` +**Category:** Bug (concurrency) +`GetMeasurementStatistics()` reads `experiment.GetFilePrefix()`, `GetRunNumber()`, +`GetExperimentGroup()`, `GetUnitCellString()`, `GetImageNum()`, etc. holding **no mutex +at all**. Concurrently, `Start()` mutates the same `experiment` +(`ImportDatasetSettings`, `StorageCellStart`, `IncrementRunNumber`, +`JFJochStateMachine.cpp:354-362`) under lock `m`, and `MeasurementThread`/`InitializeThread` +assign `experiment = local_experiment` (`JFJochStateMachine.cpp:330`) under `m`. Because +`/statistics` and `/statistics/data_collection` are served on httplib worker threads while a +measurement is starting on the async thread, this is a genuine read/write race on +`std::string`/`UnitCell` members (torn reads, UB). `GetStatus()` is called from the same +unlocked path and is fine only because it locks `broker_status_mutex` and delegates to +`services`. This getter should take `m` (or the experiment be snapshotted under lock). + +### [High] Fine-grained per-setting mutexes don't protect against the worker threads — `broker/JFJochStateMachine.cpp:84-91, 330, 354, 714-721` +**Category:** Bug (concurrency) / Architecture +Settings getters lock dedicated mutexes (e.g. `GetRadialIntegrationSettings()` locks +`experiment_azimuthal_integration_settings_mutex`, `:719-722`), and the matching setters +lock **both** `m` and the fine-grained mutex (`:708-717`). But the background workers +mutate the *same* `experiment` object holding only `m` — `InitializeThread` does +`experiment = local_experiment` (`:330`), `Start` calls `ImportDatasetSettings`/ +`IncrementRunNumber` (`:354-362`). A reader holding only +`experiment_azimuthal_integration_settings_mutex` is therefore **not** serialised against +those whole-object writes. The lock granularity gives a false sense of safety: the only +mutex that actually covers every `experiment` write is `m`, so the fine-grained mutexes +are both insufficient (don't exclude worker writes) and redundant (every setter already +holds `m`). Recommend collapsing them or making every `experiment` access go through `m`. + +### [Medium] Error responses violate the spec: JSON `error_message` sent as `text/plain`; parse errors not in schema — `broker/JFJochBrokerHttp.cpp:119-137, 144-179` +**Category:** Inconsistency +The spec declares `error_message` (object with required `msg` + enum `reason` +["WrongDAQState","Other"], `jfjoch_api.yaml:1656-1669`) as the `application/json` body for +500 responses. `handleOperationException()` correctly builds that JSON +(`JFJochBrokerHttp.cpp:122-136`) and returns `{500, j.dump()}`, but the route lambdas then +emit it via `send_plain(res, c, s)` which forces `Content-Type: text/plain` +(`:47-50, 149-151`). So a JSON error body is served with the wrong content type — clients +keying off content type (or the generated client's deserialiser) will mishandle it. +Separately, `handleParsingException()` (`:107-117`) returns the raw `e.what()` string for +400/500, never wrapped in `error_message`, so parse/validation failures don't match the +documented schema at all. Pick one error contract and apply it uniformly. + +### [Medium] Generated C++ server stubs are untracked dead code regenerated from a different spec version — `broker/gen/api/`, `broker/gen/impl/`, `broker/gen/main-api-server.cpp`, `broker/gen/CMakeLists.txt` +**Category:** Architecture / Inconsistency +`gen/api/*`, `gen/impl/*`, `gen/main-api-server.cpp`, `gen/CMakeLists.txt`, +`gen/README.md` and `gen/.openapi-generator/` are never compiled (broker uses httplib, +not the Pistache server these emit) and are **untracked in git** — only `gen/model/` is +committed. They were locally regenerated from spec `1.0.0-rc.152` +(`gen/api/DefaultApi.cpp:5`) while the committed spec and committed models are +`1.0.0-rc.148`. This is confusing and risky: a maintainer could believe the server layer +is generated when it is hand-written, and the stray newer-version files invite an +accidental commit of mismatched artefacts. Recommend either (a) deleting the unused +generator output and committing only `gen/model/`, with a documented "models only" +generation step, or (b) `.gitignore`-ing the non-model `gen/` output explicitly. Today +the C++ model-generation flow is entirely undocumented (only `gen_python_client.sh` +exists, and it generates the Python client only). + +### [Medium] `OpenAPIConvert` is a large, unguarded manual mapping with no spec linkage — `broker/OpenAPIConvert.cpp` (whole file) +**Category:** Inconsistency (drift risk) +Every field is mapped by hand in 1145 lines with no mechanism tying it back to the spec. +Optional fields rely on the `...IsSet()` guard being remembered on read and the +`has_value()` guard on write; a spec field added without a matching `Convert` line +compiles cleanly and is silently dropped. String enums (`Convert(BrokerStatus)` state, +severity; `ConvertPlotType`; `ConvertColorScale`; compression in +`Convert(Dataset_settings)`) are matched against string literals that must stay in sync +with the spec `enum:` lists by hand. See the dedicated drift section below. + +### [Medium] `LoadDetectorSettings` swallows configuration errors but still reports HTTP 200 — `broker/JFJochStateMachine.cpp:548-575` +**Category:** Bug (error propagation) +In the `Idle` branch, when no recalibration is needed, `services.ConfigureDetector()` +failure is caught and turned into `SetState(Error, ...)` but the exception is **not** +re-thrown (`:564-567`). The HTTP handler `config_detector_put` therefore returns 200 to +the client even though configuration failed; the only signal is the async `Error` state +visible on the next `/status`. Compare `SelectDetector` (`:701-705`) and +`SetIndexingSettings` (`:1037-1043`), which set Error **and** re-throw. The PUT +error-reporting behaviour is inconsistent across mutators. + +### [Medium] `wait_until_running` / `wait_till_done` default to a fixed 60 s server-side block — `broker/JFJochBrokerHttp.cpp:385-447` +**Category:** Bug / Simplification +When `timeout` is absent both handlers block the httplib worker thread for up to 1 minute +(`WaitTillNotBusy(std::chrono::minutes(1))`, `:388`, `:420`). With httplib's +thread-per-request pool this means a handful of parked long-polls can exhaust the worker +pool and stall unrelated requests (e.g. `/status`, `/cancel`). The duplicated +timeout-validation + state-to-HTTP-code switch in the two functions is also near-identical +and could be factored. + +### [Low] `Trigger()` and several getters bypass state checks and `m` — `broker/JFJochStateMachine.cpp:341-343, 966-996` +**Category:** Bug (concurrency) / Inconsistency +`Trigger()` forwards straight to `services.Trigger()` with no state guard and no lock, so +a `/trigger` issued in `Inactive`/`Error` reaches the service layer unchecked. +`GetDeviceStatus()`, `GetImagePusherStatus()`, `GetImageBufferStatus()`, +`SetPreviewSocketSettings()` etc. also delegate to `services` without `m`; that is +probably fine if `JFJochServices` is internally synchronised (`detector_mutex` is a +`shared_timed_mutex`), but the locking contract between the two classes is implicit and +undocumented. + +### [Low] `ProcessOutput` validates output and returns HTTP 500 on failure, but most handlers skip it — `broker/JFJochBrokerHttp.cpp:52-65, 499-506, 546-595` +**Category:** Inconsistency +`ProcessOutput()` runs `output.validate()` and returns 500 with the validation message if +the server-produced model is invalid. But the array/aggregate handlers +(`statistics_calibration_get` `:499-506`, `fpga_status_get` `:546-552`, `statistics_get` +`:554-595`, `xfel_*` `:918-938`) serialise with `nlohmann::json j = ...; j.dump()` +directly and never call `validate()`, so output validation is applied unevenly across +endpoints. + +### [Low] `Cancel()` only cancels in `Calibration`/`Measuring`, silently no-ops otherwise — `broker/JFJochStateMachine.cpp:452-459` +**Category:** Bug (smell) +`Cancel()` does nothing when state is `Busy` (e.g. during the configure-detector phase of +`InitializeThread` before the worker reaches `Measuring`). The comment at `:453` +("inconsistency in naming - need to solve later") flags it as known debt. A cancel issued +during the Busy window is silently ignored, with no feedback to the caller. + +### [Nit] Repeated try/catch boilerplate in route registration — `broker/JFJochBrokerHttp.cpp:198-353` +**Category:** Simplification +~15 query-param routes each inline an identical `try { ... } catch (const std::exception +&e) { auto [c,s] = handleOperationException(e); send_plain(res, c, s); }`. A +`bind_query`-style helper that adapts a parsed-args handler (mirroring `bind_noarg`) +would remove the duplication and keep the maintainer's "no superfluous code" principle. + +### [Nit] `MeasurementStatistics.images_skipped` mapped to a differently-named spec field — `broker/OpenAPIConvert.cpp:58` +**Category:** Inconsistency (naming) +The domain field `images_skipped` is written with +`setImagesDiscardedLossyCompression(...)`. The names diverge between layers; harmless +today but the kind of mismatch that makes the manual mapping hard to audit against the +spec. + +## OpenAPIConvert drift risk (specific fields at risk) + +The mapping is hand-written with no spec coupling; the following are the concrete +fault lines: + +- **String enums matched by literal** — must be kept in lockstep with spec `enum:` lists: + - `Convert(BrokerStatus)` state strings (`OpenAPIConvert.cpp:205-224`) vs + `broker_status.state` enum (`jfjoch_api.yaml:1328`). Note the deliberate remap + `JFJochState::Calibration -> "Pedestal"`; a renamed spec value would break clients + silently. + - `message_severity` strings (`:229-242`) vs `jfjoch_api.yaml:1343`. + - Compression strings `bslz4/bszstd/bszstd_rle/none` in `Convert(Dataset_settings)` + (`:603-617`) — parsed from a free-form `getCompression()` string, not a typed enum. + - `ConvertPlotType` (`:849-892`): ~38 plot-type strings hand-listed; any new `plot_type` + enum value in the spec throws `InputParameterInvalid` until added here. + - `ConvertColorScale` (`:894-915`). +- **Typed enums needing a new `case` per spec value** — `Indexing_algorithm`, + `Geom_refinement_algorithm` (`:917-1020`), `File_writer_format` (`:797-847`), + `Detector_timing` (`:95-131`), `Detector_state`/`Detector_power_state`/`Detector_type` + (`:296-361`), `Image_pusher_type` (`:1117-1146`). Several use a `default: throw`, so a + newly added spec enum value becomes a runtime error rather than a compile error. +- **Optional fields** depend on remembering `...IsSet()` / `has_value()` guards. A new + optional spec property is silently dropped if no `Convert` line is added — there is no + test asserting field coverage. High-cardinality structs most exposed: + `Convert(Dataset_settings)` (`:581-681`), `Convert(Scan_result)` (`:1022-1093`), + `Convert(MeasurementStatistics)` (`:48-93`), `Convert(Detector_settings)` (`:133-200`). +- **`Detector_list_element`** is fully hand-populated (`:363-386`); spec requires + `readout_time_ns`, `min_frame_time_ns`, `min_count_time_ns`, `udp_interface_count` + (`jfjoch_api.yaml:1092-1095`) — all currently mapped, but adding a required field to the + spec won't fail any build here. +- **ZMQ settings**: `Convert(Zeromq_*_settings)` deliberately discards the address on the + inbound path (`ret.address = ""`, `:713`, `:723`); easy to misread as a mapping bug. + +## Inconsistencies with rest of repo + +- **Generation discipline**: the Python client has a committed, parameterised generation + script (`gen_python_client.sh`); the C++ side has none, and its non-model generated + output is untracked and version-skewed (rc.152 vs committed rc.148). The repo also + ships a `redoc-static.html` (installed via `broker/CMakeLists.txt:2`) as the + human-facing spec render, so there are three derived artefacts (Python client, C++ + models, redoc) and only one is reproducible from a checked-in command. +- **Error conventions**: internally the codebase uses `JFJochException` / + `WrongDAQStateException` with categories; at the HTTP boundary these collapse to either + a schema-conformant `error_message` JSON (sent with the wrong content type) or a raw + `e.what()` string, so the structured `reason` only survives for `WrongDAQState` on the + operation path and never on the parse path. +- **Locking style**: `JFJochServices` uses a `shared_timed_mutex` (`JFJochServices.h:18`) + for reader/writer separation, whereas `JFJochStateMachine` uses a plain `std::mutex m` + plus seven fine-grained `std::mutex`es whose protection scope doesn't actually cover the + worker-thread writes — two different and partly ineffective concurrency models in the + same subsystem. diff --git a/docs/review/06-fpga.md b/docs/review/06-fpga.md new file mode 100644 index 00000000..240e800e --- /dev/null +++ b/docs/review/06-fpga.md @@ -0,0 +1,183 @@ +# Review: fpga (HLS / gateware) + +## Scope (and what was excluded as vendored) + +Reviewed all first-party files from `git ls-files fpga/ | grep -vE 'include/ap_|include/etc/'`. This covers: + +- `fpga/hls/*.cpp` and `fpga/hls/hls_jfjoch.h` — HLS kernels and the shared header +- `fpga/hls_simulation/` — software simulation layer (HLSDevice, sls_packet.h, datamover_model.h, hls_cores.h) +- `fpga/host_library/JungfraujochDevice.{cpp,h}` — host-side PCIe device wrapper +- `fpga/pcie_driver/jfjoch_fpga.h` — shared ABI header (structs, constants) +- `fpga/hls_tb/` — HLS testbenches +- `fpga/hdl/*.v` / `fpga/xdc/*.xdc` / `fpga/scripts/*.tcl` — gateware, constraints, build scripts +- `fpga/pcie_driver/*.c` — kernel driver + +Excluded as vendored: `fpga/include/ap_*.h`, `fpga/include/etc/ap_private.h`. +`fpga/hls/ip_header_checksum.h` is third-party (BSD-3, HPCN/ETH Zurich) but reviewed briefly as it is small and locally modified. + +--- + +## Architecture overview + +The gateware implements a streaming pipeline that receives UDP packets from JUNGFRAU/EIGER detectors over 100 GbE or 4×10 GbE, calibrates and processes pixel data, and DMAs results to host memory over PCIe. + +Kernel pipeline order (all free-running, `ap_ctrl_none`, connected by AXI-Stream): + +1. **Ethernet demux** (`ethernet.cpp`, `arp.cpp`, `icmp.cpp`, `ptp.cpp`, `udp.cpp`) — strips headers, dispatches ARP/ICMP/PTP/UDP. +2. **SLS packet parser** (`sls_detector.cpp`) — recognises JUNGFRAU/EIGER headers, emits raw pixel data stream and `axis_addr` metadata. +3. **`data_collection_fsm`** — gate: arms/disarms data collection, injects a 512-bit config packet at frame-start and a sentinel at end. +4. **`save_to_hbm`** — tracks which packets of each frame arrived (512-bit `packet_mask`), routes raw data into HBM via Datamover, emits `axis_completion` records. +5. **`load_from_hbm`** (`stream512to768`) — reads raw 16-bit pixels back from HBM and widens to 768-bit (24-bit per pixel × 32 pixels/beat) internal format. +6. **`eiger_reorder`** — de-interleaves EIGER's per-half-module packet order. +7. **`jf_conversion`** — fixed-point ADU → photon conversion using gain/pedestal from HBM (12 separate 256-bit HBM pseudo-ports, p0–p11). +8. **`pixel_mask`** — applies per-pixel bad-pixel mask stored as 32-bit words (1 bit per pixel). +9. **`pixel_threshold`** — zeroes/clamps pixels outside `[threshold_min, threshold_max]`. +10. **`pixel_sqrt`** — optional Anscombe/Poisson transform: `out = round(sqrt(pixel × sqrtmult²))`. +11. **`frame_summation`** / **`frame_summation_reorder_compl`** — N-frame summation with URAM accumulation. +12. Parallel branches: + - **`spot_finder_mask`** → **`spot_finder`** (SNR + count thresholding, 3-stage dataflow) → **`spot_finder_connectivity`** → **`spot_finder_merge`** + - **`integration`** — azimuthal integration into 2048 radial bins using HBM-resident bin map and weight map. + - **`roi_calc`** — per-ROI sums, sum², weighted centroid, good-pixel count, max. + - **`adu_histo`** — raw-ADU histogram (pre-conversion, 16-bit bins, 32-wide bins). + - **`pixel_calc`** — per-module statistics (min/max/sum/err/sat/masked counts). +13. **`host_writer`** — assembles results, issues Datamover commands to write pixel image + all analytics to the DMA-mapped `DeviceOutput` buffer; signals completion via 32-bit mailbox. +14. **`load_calibration`** / **`frame_generator`** — separate s_axilite kernels for calibration loading and loopback packet generation. + +The shared header (`hls_jfjoch.h`) and `jfjoch_fpga.h` together form the ABI between gateware, driver, and host software. + +--- + +## Findings + +### [High] `sqrtmult` declared `ap_uint<9>` causes silent overflow for multiplier ≥ 23 — `fpga/hls/pixel_sqrt.cpp:23` + +**Category:** Bug + +The register field `ACT_REG_SQRTMULT` carries an 8-bit value *k* (the desired output scale factor). The kernel squares it to reconstruct `k²`, then computes `sqrt(pixel * k²) = sqrt(pixel) * k`: + +```cpp +ap_uint<9> sqrtmult = ACT_REG_SQRTMULT(packet_in.data) * ACT_REG_SQRTMULT(packet_in.data); +``` + +`ap_uint<9>` holds at most 511. For *k* ≥ 23, *k*² ≥ 529 overflows: *k* = 23 → stored as 17; *k* = 255 → stored as 1. The type should be `ap_uint<16>` (maximum *k*² = 65 025). The existing testbench only exercises *k* ∈ {1, 2, 4, 8, 16}, all of which are below the overflow threshold, so this is untested. + +--- + +### [Medium] Dead `axis_addr::det_type` field shadows `detector_type` — `fpga/hls/hls_jfjoch.h:96,99` + +**Category:** Inconsistency / Bug + +`axis_addr` has two 4-bit fields: `det_type` (line 96) and `detector_type` (line 99). In `sls_detector.cpp` both are written from the same byte: + +```cpp +addr.det_type = detector_type; // line 107 +addr.detector_type = detector_type; // line 128 +``` + +Downstream, `stream512to768.cpp` branches on `addr.det_type`, while `save_to_hbm.cpp` copies `addr.detector_type` into the completion record. These are currently consistent, but they occupy different bits inside `axis_addr`, wasting space and creating a maintenance trap. One should be removed and all uses unified. + +--- + +### [Medium] `roi_calc` output bit-range constant `265+63` overlaps the next field — `fpga/hls/roi_calc.cpp:143` + +**Category:** Bug (latent / functionally harmless by coincidence) + +```cpp +packet_out(265+63, 256) = roi_good_pixels[i]; // writes bits [328:256], 73-bit slot +packet_out(256+127, 256+64) = roi_max_value[i]; // writes bits [383:320], 64-bit slot +``` + +The two ranges overlap at bits [328:320] (9 bits). The overlap is harmless only because `roi_good_pixels` is `ap_uint<32>` — HLS zero-extends it so bits [328:288] are 0 — and then `roi_max_value` overwrites [328:320] with its own data. The `ROICount` struct expects `good_pixels` at bytes 32–39 (bits [319:256]). The intended constant is `256+64-1 = 319`, not `265+63 = 328`. The code is accidentally correct but incorrect as written. + +--- + +### [Medium] `ACT_REG_NSTORAGE_CELLS` encoding differs from `ACT_REG_NMODULES` without comment — `fpga/hls/data_collection_fsm.cpp:85`, `fpga/hls/jf_conversion.cpp:174` + +**Category:** Inconsistency + +`nmodules` is stored in the action register as *count − 1* (host sets `job.nmodules = count - 1`; the consumer in `jf_conversion` adds 1 back). `nstorage_cells` is stored as *count*: the FSM adds 1 before writing (`nstorage_cells + 1`), and `jf_conversion` reads it directly as the actual count. The different conventions inside the same register packet are undocumented and are a future maintenance hazard. + +--- + +### [Medium] `packet_length == 32` branch in `save_to_hbm` is dead code — `fpga/hls/save_to_hbm.cpp:77,95` + +**Category:** Bug / Simplification + +`sls_detector.cpp` only sets `packet_length` to 128 (JUNGFRAU) or 64 (EIGER). The `== 32` branches in `save_to_hbm` are never reached. They bloat the synthesis and suggest either a planned future detector type (not documented) or a leftover from an earlier design. If they are future-proofing, a comment is warranted; otherwise they should be removed. + +--- + +### [Low] `hls_jfjoch.h` includes `jfjoch_fpga.h` twice — `fpga/hls/hls_jfjoch.h:7,34` + +**Category:** Simplification + +`#include "../pcie_driver/jfjoch_fpga.h"` appears at line 7 and again at line 34. The include guard in `jfjoch_fpga.h` prevents any harm, but the duplicate is noise. + +--- + +### [Low] `o` / `sf` macros are copy-pasted across two files — `fpga/hls/host_writer.cpp:6-7`, `fpga/hls/load_calibration.cpp:6-7` + +**Category:** Inconsistency / Simplification + +```cpp +#define o(field) offsetof(ModuleStatistics, field) +#define sf(msg, field, s) msg(o(field)*8 + s - 1, o(field)*8) +``` + +These two macros are identical in both files. They could live in `hls_jfjoch.h` (or a small helper header) to avoid drift if `ModuleStatistics` fields are reordered. + +--- + +### [Low] `pixel_sqrt.cpp:35` — no-op self-assignment — `fpga/hls/pixel_sqrt.cpp:35` + +**Category:** Simplification + +```cpp +pixel[i] = pixel[i]; // inside the INT24_MAX / INT24_MIN guard +``` + +This is a no-op that was presumably a placeholder for some future action or a mistaken edit. It confuses readers and should be replaced by `/* keep as-is */` or simply removed (the surrounding `if` branch is already a pass-through). + +--- + +### [Low] `spot_finder_snr_threshold` comment mismatch — `fpga/hls/spot_finder.cpp:234` + +**Category:** Inconsistency + +Comment at line 235 says "snr_threshold is in units of 0.25", but `snr_threshold_2` is the *square* of the SNR threshold (computed from the float value directly at line 303). The comment belongs at the `spot_finder_apply_threshold` level where the float is squared, not inside the comparator function. + +--- + +### [Nit] `GAIN_G1_MULTIPLIER` / `GAIN_G2_MULTIPLIER` are defined as `-1` but the template clamps to 0 — `fpga/pcie_driver/jfjoch_fpga.h:24-25`, `fpga/hls/load_calibration.cpp:42-46` + +**Category:** Inconsistency / Simplification + +The `convert` template multiplies the raw calibration float by `MULT = -1` and then clamps negative results to 0. This is how the sign convention of G1/G2 (inverted compared to G0) is absorbed into the unsigned fixed-point storage type. The multiplication by −1 then the non-negative clamp effectively stores `abs(gain)`. The convention is correct but non-obvious. A comment at the `#define` site explaining that gain for G1/G2 calibration files is negative would prevent future confusion. + +--- + +### [Nit] `data_collection_fsm` `RCV_LAST` sends stale `addr` — `fpga/hls/data_collection_fsm.cpp:136` + +**Category:** Bug (benign by construction) + +In the `RCV_LAST` case, `addr_out << addr` sends whatever `addr` held from the last `RCV_INIT` iteration. The `last` flag is set, so the content (module/frame fields) is unused by `save_to_hbm` (which only checks `addr.last` to exit its loop). But the code is fragile: if a consumer ever inspects the non-`last` fields of the terminating `addr`, it would see stale data. A sentinel initialisation (`addr = {};` before the loop) or an explicit `axis_addr terminal{.last=1};` would make the intent clear. + +--- + +## Host/device interface consistency with `receiver/JFJochReceiverFPGA` + +The interface is consistent. The shared header `fpga/pcie_driver/jfjoch_fpga.h` is included by both the HLS kernels (via `hls_jfjoch.h`) and by the host library / receiver code; all `DataCollectionConfig`, `DeviceOutput`, `LoadCalibrationConfig`, `SpotFinderParameters`, and related constants are defined in one place. The HBM slot assignments (p0–p11 gain/pedestal, p12–p15 integration, p16–p17 spot-finder d-map, p18–p19 ROI, p20–p21 frame-gen, p22–p26 raw) are expressed only as numeric multipliers of `hbm_size_bytes/32` scattered across the kernels — there is no single table, but `load_calibration.cpp` is the authoritative mapping and the consumer kernels use matching offsets. + +The `receiver/JFJochReceiverFPGA` path calls `acquisition_device[d].PrepareAction(experiment)` and `LoadCalibrationToFPGA`, which drives `JungfraujochDevice::SetConfig` / `LoadCalibration` — these map cleanly onto the `DataCollectionConfig` / `LoadCalibrationConfig` ABI structs. No drift was observed between the gateware ABI and the host-side wrapper at the time of review. + +--- + +## Inconsistencies with rest of repo + +1. **`ETHER_PTP = 0xF788`** (`hls_jfjoch.h:254`): the standard PTP-over-Ethernet EtherType is `0x88F7`; the bytes are in reversed (little-endian) order. This is intentional given the on-wire big-endian / HLS little-endian convention used for the network byte-swap helpers, but it differs from the host-side network code which uses standard values directly. A comment would prevent confusion. + +2. **Gain G1 coefficient naming**: `gainG1_t` is `ap_ufixed<16,4>` (max ≈ 16), while physical G1 gain factors for JUNGFRAU are typically ∼0.4–2 ADU/keV after the ×(−1) sign flip. The integer bits are wider than needed. This is a deliberate safety margin, not a bug, but is inconsistent with the tighter `gainG0_t` (`ap_ufixed<16,2>`, max ≈ 4). + +3. **`int` return type for `load_calibration`** vs `void` for all other HLS kernels: `load_calibration` and `frame_generator` use `s_axilite` control (start/done/return value), while all pipeline kernels use `ap_ctrl_none`. This is architecturally correct (loaders are invoked as discrete transactions), but the inconsistent control model is not explained by a comment. + +4. **`HLSDevice` vs `JungfraujochDevice` API surface**: the simulation device (`HLSDevice`) exposes `HW_ReadActionRegister` / `HW_WriteActionRegister` and raw packet injection helpers that do not exist on the real device. This is expected for a simulation shim, but it means full-path simulation tests exercise a code path that never executes on hardware. diff --git a/docs/review/07-viewer-tools.md b/docs/review/07-viewer-tools.md new file mode 100644 index 00000000..2763ea93 --- /dev/null +++ b/docs/review/07-viewer-tools.md @@ -0,0 +1,208 @@ +# Review: viewer + tools + +## Scope + +- `viewer/` — Qt6 desktop image viewer (Qt Widgets + Charts + D-Bus + OpenGL; ~50 source files) +- `tools/` — CLI entry points: `jfjoch_process.cpp`, `jfjoch_fpga_test.cpp`, `jfjoch_lite_perf_test.cpp`, plus `xbflash.qspi/` (Xilinx PCIe flashing utility) +- Third-party: only `ffbidx` is in scope; `xbflash.qspi/` is treated as a vendored low-level tool (robustness focus, not elegance) + +--- + +## Architecture overview + +**Viewer.** `JFJochViewerWindow` is the top-level `QMainWindow`. A `JFJochImageReadingWorker` lives on a dedicated `QThread`; all I/O (HDF5 or HTTP) and image analysis happen there. Communication is entirely via Qt signals/slots — no raw shared state is visible from the UI thread. The worker drives two readers (`JFJochHDF5Reader`, `JFJochHttpReader`) and a suite of analysis objects (`MXAnalysisWithoutFPGA`, `IndexAndRefine`, `AzimuthalIntegrationMapping`). The status-bar, toolbar, side-panel, and dock-based dataset-info chart are separate classes connected by signals. The recent "live detector status bar + dataset-follow sync mode" adds `JFJochViewerStatusBar`, a `status_timer` in the worker for broker polling, and the `HTTPSyncDataset` autoload mode. + +**tools/.** `jfjoch_process` is a standalone offline reprocessing pipeline — reads HDF5, sets up `DiffractionExperiment`, `IndexerThreadPool`, `MXAnalysisWithoutFPGA`, and `FileWriter` from scratch with `getopt_long`-based argument parsing. `jfjoch_fpga_test` exercises the live FPGA receive path. `jfjoch_lite_perf_test` exercises the receiver service with a synthetic image stream. All three re-wire experiment/analysis setup independently rather than sharing a common factory. + +--- + +## Findings + +### [High] `LoadFile` ignores its `retry` parameter — always retries — `viewer/JFJochViewerWindow.cpp:397` + +**Category:** Bug + +`JFJochViewerWindow::LoadFile(filename, image_number, summation, retry)` always emits `LoadFileRequest` with `retry=true`, discarding the `retry` argument. When the viewer opens a file at startup (line 386: `LoadFile(file, 0, 1, false)`) it silently enables retry/back-off regardless. If the caller intention was "fail immediately on startup" this is a quiet behaviour change. Either remove the `retry` parameter from `LoadFile()` and its signal (since it is always `true`), or propagate it correctly. + +--- + +### [High] POSIX syscalls (`open`, `fstat`, `O_CLOEXEC`, `ESTALE`, `ETIMEDOUT`, `ENOTCONN`) hard-block Windows port — `viewer/JFJochImageReadingWorker.cpp:4–62` + +**Category:** Portability + +The preflight function `preflight_open_ro` uses ``, ``, ``, `::open`, `::fstat`, `O_CLOEXEC` (POSIX), and the NFS-specific errnos `ESTALE`, `ETIMEDOUT`, `ENOTCONN` (not defined on Windows). Qt provides `QFile::exists()` and `QFileInfo` which are cross-platform. `O_CLOEXEC` and `ESTALE` are completely absent from the MSVC/Windows headers. Either wrap this in `#ifdef Q_OS_UNIX` / provide a no-op Windows path, or rewrite the preflight using `QFile`/`QFileInfo` (which avoids the platform issues while preserving the retry logic). + +--- + +### [High] `M_PI_2` is a POSIX extension, not standard C++ — `viewer/image_viewer/JFJochDiffractionImage.cpp:242–257` + +**Category:** Portability + +`M_PI_2` and `M_PI` are not part of ISO C++; they are POSIX/glibc extensions. MSVC does not define them unless `_USE_MATH_DEFINES` is defined before including ``. Four uses of `M_PI_2` appear in `JFJochDiffractionImage.cpp`, and `M_PI` is used in several viewer files. Define the constants locally or add `#define _USE_MATH_DEFINES` (MSVC) guarded by a feature check, or use `std::numbers::pi` (C++20) throughout. + +--- + +### [High] `qt_import_plugins(… INCLUDE Qt::QXcbIntegrationPlugin)` hard-codes X11/XCB — `viewer/CMakeLists.txt:116` + +**Category:** Portability + +This line statically imports the Linux/X11 XCB platform plugin. On Windows the plugin does not exist; linking will fail. The CMake rule needs to be platform-conditional: `Qt::QWindowsIntegrationPlugin` on Windows, `Qt::QCocoaIntegrationPlugin` on macOS, and the XCB plugin on Linux only. + +--- + +### [High] Qt D-Bus is a Linux-only Qt module; it is unconditional in the viewer — `viewer/CMakeLists.txt:8,93` + +**Category:** Portability + +`Qt6::DBus` and the D-Bus service file installation are unconditional. D-Bus does not exist on Windows or macOS. The find/link/install should be wrapped in `if(UNIX AND NOT APPLE)` (or similar), and the viewer should compile cleanly with `--dbus false` on non-Linux. The `dbus/` source files would need a compile-time guard too. + +--- + +### [Medium] `jfjoch_lite_perf_test` prints the wrong binary name in its usage string — `tools/jfjoch_lite_perf_test.cpp:17,30,32` + +**Category:** Inconsistency + +Both `print_license` and `logger` are initialised with `"jfjoch_fpga_test"` instead of `"jfjoch_lite_perf_test"`, and the usage string says `"./jfjoch_fpga_test"`. A user running `jfjoch_lite_perf_test --help` sees the wrong name, making log attribution and support confusing. + +--- + +### [Medium] `jfjoch_process` has a dead, undocumented `-F` flag — `tools/jfjoch_process.cpp:340,468–470` + +**Category:** Bug / Simplicity + +`-F` appears in `short_opts` and handles `IndexingAlgorithmEnum::FFT`, but is absent from `print_usage()` and superseded by `--indexing-algorithm fft` (`-X fft`). The silent alias is a maintenance trap (it could be mis-typed as the `-F` for file output in other tools). Remove it. + +--- + +### [Medium] `spot_settings.high_resolution_limit` is assigned twice — `tools/jfjoch_process.cpp:735,739` + +**Category:** Simplicity + +`spot_settings.high_resolution_limit = d_min_spot_finding;` is set unconditionally on line 735, then immediately set again inside `if (d_min_spot_finding > 0.0f)` on line 739. The condition is always true (default is 1.5). Remove the redundant conditional assignment. + +--- + +### [Medium] `FileWriter::Finalize()` return value silently discarded — `tools/jfjoch_process.cpp:1126` + +**Category:** Bug / Inconsistency + +`auto stats = writer->Finalize();` declares `stats` but never uses it. The compiler will likely warn (unused variable). Either log the statistics or assign to `[[maybe_unused]]`. If the statistics are useful for the process tool (compressed sizes, per-file summary) they should be printed, as `jfjoch_fpga_test` reports analogous throughput figures. + +--- + +### [Medium] Emitting signals while holding a non-recursive `QMutex` — `viewer/JFJochImageReadingWorker.cpp` (multiple locations) + +**Category:** Architecture / Bug risk + +Many `_i()` methods emit signals (`datasetLoaded`, `imageLoaded`, `brokerStatusUpdated`, etc.) while `m` is held by the caller. Because the worker is on its own thread and all connections to UI widgets are queued, the emits themselves are safe (they post to the event queue and return immediately). However, any future direct connection to a slot that acquires the same mutex would deadlock. The pattern is currently safe but fragile. Consider a "emit after unlock" pattern: collect signals in a local struct and emit them after the locker goes out of scope, or document the requirement that no direct connections to these signals may be made. + +--- + +### [Medium] `UpdateUserMask_i` lacks an "assumes m locked" comment — `viewer/JFJochImageReadingWorker.cpp:562` + +**Category:** Inconsistency + +All other `_i()` helpers have `// Assumes m locked!` at the top. `UpdateUserMask_i` does not. It also calls `LoadImage_i` (which also assumes lock held), so the invariant is real. Add the comment for consistency and safety. + +--- + +### [Low] `AddROIToUserMask` calls `azint_mapping->Resolution()` and discards the result — `viewer/JFJochImageReadingWorker.cpp:601` + +**Category:** Simplicity + +`const auto res = azint_mapping->Resolution();` is called and `res` is never used. Remove the call. + +--- + +### [Low] Signed/unsigned comparison in `FindCenter` ring loop — `viewer/JFJochImageReadingWorker.cpp:520` + +**Category:** Bug / Portability + +`for (int i = 0; i < 15 && i < ring_Q.size(); i++)` compares `int` to `size_t` — a signed/unsigned warning on all compilers. Use `for (size_t i = 0; i < 15 && i < ring_Q.size(); i++)`. + +--- + +### [Low] Splash screen shown after main window; not parented; leaks on close — `viewer/jfjoch_viewer.cpp:60–81` + +**Category:** Bug / Simplicity + +(a) `mainWindow.show()` is called first (line 62), so the main window appears before the splash — the typical Qt pattern is to show the splash before creating the main window. (b) `QSplashScreen* splash` is heap-allocated with no parent and no `deleteLater`. `QTimer::singleShot` only calls `close()`, not `delete`. The pointer is never freed (minor leak during a 5-second startup window, but still incorrect). Use `splash->setAttribute(Qt::WA_DeleteOnClose)` or call `splash->deleteLater()` from the timer callback. + +--- + +### [Low] Typo in splash screen application name — `viewer/jfjoch_viewer.cpp:74` + +**Category:** Nit + +The splash text reads `"Jungfrujoch image viewer"` (missing 'a') while the window title at `JFJochViewerWindow.cpp:52` correctly reads `"Jungfraujoch image viewer"`. Fix the splash string. Also `"Paul Scherrer Institut"` is missing the trailing `e` — the full name is "Paul Scherrer Institute". + +--- + +### [Low] `getopt` / `getopt_long` / `getuid` / `geteuid` / `libgen.h` (`basename`) in tools — `tools/jfjoch_process.cpp:14`, `tools/xbflash.qspi/main.cpp:31` + +**Category:** Portability (tools, not viewer) + +All three major tool files use POSIX `getopt`/`getopt_long`. This is already expected for Linux-only tools. Documented here for completeness; no immediate action required unless cross-platform build is planned for `jfjoch_process`. `xbflash.qspi` also uses `libgen.h::basename` and `getuid`/`geteuid` — again inherently Linux (PCIe sysfs), so acceptable. + +--- + +### [Low] `atoi`/`atof`/`atol` used throughout tools — `tools/jfjoch_process.cpp:351ff`, `tools/jfjoch_fpga_test.cpp:83ff` + +**Category:** Inconsistency + +Both tools mix `atoi`/`atof` (no error detection on bad input) with `std::stod`/`std::stof`/`std::stoi` (throw on error, used in newer options). The older options silently accept garbage. Prefer `std::stoi`/`std::stod` with a try/catch (or `parse_float_strict`) consistently. Not critical since these are developer/test tools. + +--- + +### [Nit] Misleading comment in `LoadCalibration` — `viewer/JFJochImageReadingWorker.cpp:670` + +**Category:** Nit + +The log message `"HTTP mode doesn't allow to read calibration (at the moment"` is missing the closing parenthesis. Fix the string. + +--- + +### [Nit] `JFJochViewerDatasetInfo::UpdateLabels` calls `combo_box->clear()` twice when no dataset — `viewer/JFJochViewerDatasetInfo.cpp:108–109` + +**Category:** Simplicity + +The `else` branch calls `combo_box->clear()` after already having been guaranteed to be empty (the method always clears first on line 75). Remove the redundant call. + +--- + +## Viewer MSVC/Windows portability risks + +Priority order for a Windows port: + +1. **`QXcbIntegrationPlugin`** — must be made conditional per platform (`CMakeLists.txt:116`). Blocks linking immediately on Windows. +2. **Qt D-Bus** — unconditional dependency on a Linux-only Qt module (`CMakeLists.txt:8,93`). Blocks compilation. +3. **POSIX preflight in `JFJochImageReadingWorker.cpp`** — `sys/stat.h`, `fcntl.h`, `unistd.h`, `O_CLOEXEC`, `ESTALE`, `ETIMEDOUT`, `ENOTCONN` (`cpp:4–62`). Replace with `QFile`/`QFileInfo` plus a Windows-compatible `WSAECONNRESET`/`ERROR_FILE_NOT_FOUND` mapping, or #ifdef the function. +4. **`M_PI`, `M_PI_2`** — need `_USE_MATH_DEFINES` or C++20 `std::numbers::pi`. Scattered across `JFJochDiffractionImage.cpp`, `JFJochAzIntImage.cpp`, `PowderCalibrationWidget.cpp`, `JFJochViewerMetadataWindow.cpp`. +5. **`JFJochCommon` / `JFJochImageAnalysis` transitive deps** — spdlog (MSVC-clean), HDF5 (MSVC-clean). Main risk is CUDA: `CUDAWrapper.cpp` is already MSVC-compatible (`#ifndef JFJOCH_USE_CUDA` stub). No additional POSIX headers detected in the headers included by the viewer. +6. **FFTW** — the CMake conditional (`CMakeLists.txt:118–122`) already handles absence gracefully. FFTW has a Windows build; no blocker if the lib is available. + +**Summary:** Items 1–3 are compile blockers. Items 4–5 are fixable with one-liners. The overall viewer architecture (Qt Widgets, QGraphics, Qt Charts) is cross-platform. The largest single risk for the MSVC port is the D-Bus subsystem: it is deeply wired into `JFJochViewerWindow` (construction, adaptor, service registration) and would need to be fully `#ifdef`'d out or stubbed. + +--- + +## Tools: duplicated orchestration logic + +`jfjoch_process`, `jfjoch_fpga_test`, and `jfjoch_lite_perf_test` each independently: + +- Construct a `DiffractionExperiment` with hardcoded geometry or file-loaded settings. +- Build `IndexingSettings` from CLI flags with near-identical `getopt` blocks for `-X`/`-t`/`-f` (indexing algorithm, thread count, FFT vectors). +- Instantiate `MXAnalysisWithoutFPGA` or `JFJochReceiverService` + `SpotFindingSettings` with largely duplicated defaults. + +The `-X`/`-t`/`-f` block appears verbatim in both `jfjoch_fpga_test.cpp:133–148` and `jfjoch_lite_perf_test.cpp:66–79` with identical logic. If the indexing algorithm enum changes, both must be updated. A shared helper (e.g. `ParseIndexingSettings(getopt_result) -> IndexingSettings`) would centralise this. `jfjoch_process` uses `--indexing-algorithm` long options for the same semantic — the string-to-enum mapping is duplicated in all three files. + +`jfjoch_lite_perf_test` and `jfjoch_fpga_test` both assemble `SpotFindingSettings` from `DiffractionExperiment::DefaultDataProcessingSettings()` plus field overrides — identical pattern, different defaults, no sharing. + +--- + +## Inconsistencies with rest of repo + +1. **`retry` parameter ignored** (`JFJochViewerWindow::LoadFile`): the rest of the repo's signal/slot chain uses `bool retry` correctly at the worker level but the window wrapper always overrides it to `true`. Inconsistent with the declared API. +2. **`atoi`/`atof` vs `std::stoi`/`std::stod`** in tools: newer option handling (bandwidth, outliers, radii) uses `std::stod`/`std::stof`; older options use `atoi`/`atof`. Mixed within the same switch statement. +3. **`// Assumes m locked!` comment convention** missing from `UpdateUserMask_i` while present in all other `_i()` helpers. +4. **"NUMA policy is deprecated"** warning appears in both `jfjoch_fpga_test.cpp:101` and `jfjoch_lite_perf_test.cpp:60` — the `-P` option is retained in both binaries but documented nowhere. If it is truly dead, remove the option from both tools. +5. **`print_license`/logger name copy-paste error** in `jfjoch_lite_perf_test` (uses `"jfjoch_fpga_test"` string) — inconsistent with all other tools which name themselves correctly. diff --git a/docs/review/08-frontend.md b/docs/review/08-frontend.md new file mode 100644 index 00000000..0d005152 --- /dev/null +++ b/docs/review/08-frontend.md @@ -0,0 +1,321 @@ +# Review: frontend (React/TypeScript) + +## Scope + +`frontend/src/` — all hand-written components and the App root. The generated +`frontend/src/openapi/` tree (core/, models/, services/DefaultService.ts) is +excluded from internal review but its consumption surface is assessed. + +--- + +## Architecture overview + +### Component tree + +``` +App (class, setInterval 1 s, holds jfjoch_statistics) +├── StatusBar — broker_status → toolbar buttons +├── ErrorMessage — broker_status → alert +├── DataProcessingPlots (×2) — own setInterval 1 s → DataProcessingPlot +│ └── MultiLinePlotWrapper → Plot (plotly) +├── MeasurementStatistics — measurement_statistics (pass-through) +├── PreviewImage — own setInterval 2 s, raw fetch /image_buffer/image.jpeg +├── DataCollection — local form state, POST /start +├── DetectorSettings — detector_settings (prop-driven) +├── PixelMask — pixel_mask_statistics (prop-driven) +├── DarkMaskSettings — dark_mask_settings (prop-driven) +├── FileWriterSettings — file_writer_settings (prop-driven) +├── DetectorSelection — detector_list (prop-driven) +├── DetectorStatus — detector_status (prop-driven) +├── ZeroMQPreview — zeromq_preview_settings (prop-driven) +├── IndexingSettings — indexing_settings (prop-driven) +├── ImageFormatSettings — image_format_settings (prop-driven) +├── InstrumentMetadata — instrument_metadata (prop-driven) +├── AzIntSettings — azim_int_settings (prop-driven) +├── ImagePusherStatus — image_pusher_status (prop-driven) +├── Calibration — calibration_statistics (prop-driven) +├── ROI — roi_definitions (prop-driven) +└── FpgaStatus — fpga_status (prop-driven) +``` + +### State management + +All server state is held in `App.state.s` (a `jfjoch_statistics` object) and is +fetched by a single `setInterval` at 1-second cadence via +`DefaultService.getStatistics()`. This is the polling source-of-truth. +Individual settings components receive their slice of that state as props; they +copy it into local component state on first receipt and thereafter track +server-side changes via `_.isEqual` diffing in `componentDidUpdate`. Writes are +performed by each component independently — either through `ButtonWithSnackbar` +(which issues a raw `fetch` PUT/POST) or directly via a `DefaultService` call. + +`DataProcessingPlot` and `PreviewImage` run their own independent polling +intervals (1 s and 2 s respectively) without any coordination with the root +interval. + +### Data-fetching patterns + +- **Polling only.** No WebSocket or ZeroMQ push paths exist on the frontend. + `ZeroMQPreview.tsx` is just a settings panel for a ZeroMQ socket served by + the broker; no client-side ZeroMQ connection is established. +- **Raw `fetch`** for the JPEG preview endpoint; all other reads go through the + typed `DefaultService` generated client. +- **Writes** use a mix of `ButtonWithSnackbar` (raw `fetch` PUT/POST with a JSON + body) and direct `DefaultService` calls (DataProcessingSettings sliders, a few + ImageFormatSettings shortcut buttons). + +--- + +## Findings + +### [Bug] `processRowCircleUpdate` compares by reference, not by id — `ROI.tsx:256` + +**Category:** Bug + +`let new_state = this.state.circle.map((item) => item === oldRow ? newRow : item)` +uses object identity (`===`) instead of `item.id === oldRow.id` like the box +handler at line 249 and the azim handler at line 263. After a React render the +`oldRow` reference is a fresh object, so the comparison always fails and edits to +circle ROI rows are silently dropped (the grid shows the edit but `this.state.circle` +is never updated). This is a data-loss bug during in-place editing. + +### [Bug] Upload button does not block when `azim_err` is set — `ROI.tsx:437` + +**Category:** Bug + +`disabled={(this.state.box_err !== undefined) || (this.state.circle_err !== undefined)}` +omits `|| (this.state.azim_err !== undefined)`. A user can upload an invalid +azimuthal ROI configuration (duplicate names or inverted Q bounds) while the +error message is displayed but the upload is not prevented. + +### [Bug] `IndexingSettings` sets `tolerance_error` instead of `unit_cell_dist_tolerance_error` — `IndexingSettings.tsx:180` + +**Category:** Bug + +The callback for the "UC dist. tol." `NumberTextField` (line 180) writes +`tolerance_error: err` into state, colliding with the HKL tolerance field +(line 166). `unit_cell_dist_tolerance_error` is tracked in state and checked by +`isError()` (line 125) but is never set by either callback. Consequence: an error +in the unit-cell distance tolerance field disables upload (because `tolerance_error` +is checked), but an error in the HKL tolerance field also sets `tolerance_error` +and so the actual per-field correlation is broken. The dedicated flag is dead code. + +### [Bug] `AzIntSettings` mutates state directly — `AzIntSettings.tsx:131,139` + +**Category:** Bug + +```js +this.state.s.solid_angle_corr = event.target.checked; +``` +and line 139 for `polarization_corr`. These direct mutations bypass `setState`, +so the UI will not re-render and the value does not feed back into the Upload +payload correctly on a subsequent render cycle. The checkbox `checked` prop still +reads the (now-mutated) state reference, but the surrounding `Upload` button will +not be re-evaluated and React's reconciler will not see a change. + +### [Bug] `DataCollection.getDatasetSettings` mutates state — `DataCollection.tsx:76-81` + +**Category:** Bug + +```js +getDatasetSettings = () => { + let d = this.state.s; // alias, not a copy + if (this.state.mode === 'rotation') + d.goniometer = this.state.goniometer; // mutates this.state.s + ... +``` +`d` is a reference to `this.state.s`, not a shallow copy, so assigning +`d.goniometer` or `d.grid_scan` silently mutates the state object. This is +called on every "start" button click; the mutation is harmless for the current +render but is an antipattern that can cause subtle bugs if `getDatasetSettings` +is ever called before `setState` settles. + +### [High] Unhandled promise rejections in several components + +**Category:** Bug + +- `PixelMask.handleUpload` (line 28): `DefaultService.putConfigUserMaskTiff` is + called with no `.catch()`. A network or server error silently disappears; no + snackbar or user feedback. +- `DetectorSettings.deactivate` (line 104): catches with an empty handler + `catch(error => {})`. Legitimate errors are swallowed. +- `DataProcessingSettings.putValues` (line 42): `catch(error => console.log(error))` + — errors go only to the console, no UI feedback. +- `ImageFormatSettings.raw/conv/putValues` (lines 79, 84, 92): same + console-only pattern. + +A consistent pattern for mutation errors is already established by +`ButtonWithSnackbar`; the direct `DefaultService` calls should either use that +component or replicate its error-display logic. + +### [High] `InstrumentMetadata.tsx` exports a class named `ImageFormatSettings` — `InstrumentMetadata.tsx:42,151` + +**Category:** Inconsistency / Bug + +The internal class name is `ImageFormatSettings`, identical to the class in +`ImageFormatSettings.tsx`. This is purely cosmetic (modules resolve by export +default), but it is a genuine confusion hazard and violates the one-class-per-file +naming contract that the rest of the codebase maintains. The class should be +renamed `InstrumentMetadata`. + +### [Medium] Two independent polling intervals for plots — `DataProcessingPlot.tsx:99,124-126` + +**Category:** Architecture + +`DataProcessingPlot` starts its own 1-second `setInterval` independently of the +root `App` poll. Two `DataProcessingPlots` widgets are rendered simultaneously +in `App` (lines 152 and 158), each containing a `DataProcessingPlot`, so there +are four independent polling loops for plot data plus the root stats loop — +five total — with no coordination. They fire at random offsets relative to each +other and to the root poll, creating unnecessary request bursts. Plots could be +fetched from a single shared hook or the root poll could include plot data in the +statistics bundle (though that changes the server contract). + +### [Medium] `PreviewImage` fetches on every UI interaction without debounce — `PreviewImage.tsx:256-284` + +**Category:** Architecture / Performance + +Every slider move (saturation, resolution ring, image ID) or checkbox toggle +calls `getValues()` synchronously, which fires an immediate `fetch` to +`/image_buffer/image.jpeg`. Dragging the saturation slider can therefore issue +tens of JPEG requests per second. The previous object URL is revoked only after +the response resolves (line 279), which is correct but the volume of in-flight +requests is uncontrolled. Debouncing slider handlers by ~150 ms would +substantially reduce server load. + +### [Medium] `App` uses a class component with a class-level interval — `App.tsx:53-82` + +**Category:** Architecture + +`App` is a 250-line class component managing all UI toggles plus the central +polling interval. The interval ID is stored as a class field (`this.interval`) +rather than a React ref, which is fine for class components, but the entire +component would be simpler and easier to test as a function component with a +`useEffect`/`useRef` interval. More importantly, if `getValues()` is called in +the snackbar or statistics flows while the component is unmounting, the +in-flight promise's `.then(data => this.setState(...))` will set state on an +unmounted component. The generated `CancelablePromise` can cancel the request +on unmount, but this is not done here. + +### [Medium] `deactivate` method in `DetectorSettings` is dead code — `DetectorSettings.tsx:103-106` + +**Category:** Simplification + +The `deactivate` arrow-function method calls `DefaultService.postDeactivate()` +but is never invoked (`this.deactivate` has no call sites). The Deactivate +button uses `ButtonWithSnackbar` with `path={"/deactivate"}`, which issues its +own `fetch`. The dead method should be removed. + +Similarly, `ROI.uploadButton/downloadButton` (lines 163-164), +`ImageFormatSettings.uploadButton/raw/conv` (lines 78-90) are dead class +methods. + +### [Low] `MeasurementStatistics` and `Calibration` import `DefaultService` but never use it + +**Category:** Simplification + +`MeasurementStatistics.tsx:5` imports `DefaultService` alongside +`measurement_statistics`; the service is never called. `Calibration.tsx:6` does +the same. These are stale import remnants. + +### [Low] `NumberTextField` has a fixed `id="frame_time"` on all instances — `NumberTextField.tsx:99` + +**Category:** Bug / Inconsistency + +Every rendered `NumberTextField` outputs ``. +Duplicate `id` attributes on a page are an HTML validity error and break label +association (the `htmlFor` of any `InputLabel` with `for="frame_time"` would +bind to only the first matching element). The `id` should be derived from the +`label` prop or removed. + +### [Low] `ROI.tsx:335` — column header typo `2pi_q_maz` / both d columns labeled `d_max` + +**Category:** Inconsistency + +The field name at line 335 is `'2pi_q_maz'` (typo of `'2pi_q_max'`). More +visually confusing: both the `q_min`-derived column (line 326) and the +`q_max`-derived column (line 352) render their header as `d_max [Å]`. The +first should read `d_max [Å]` and the second `d_min [Å]`. + +### [Low] `DetectorSelection` missing `key` on `TableRow` and `MenuItem` maps — `DetectorSelection.tsx:89,135` + +**Category:** Inconsistency + +The `arr.map(d => ...)` at line 89 and `this.detector_list().map(d => )` at line 135 do not pass a `key` prop, producing React reconciler +warnings and potentially causing incorrect DOM updates when the list changes. + +### [Nit] `DataProcessingPlots` mixes `String` (object type) and `string` — `DataProcessingPlots.tsx:20,34` + +**Category:** Inconsistency + +`tab: String` in `MyState` and the `SelectChangeEvent` generics use the +boxed `String` object type instead of the primitive `string`. In strict TypeScript +these are different types; `String` should be `string` throughout. + +### [Nit] `FormatNumber` in `StatusBar.tsx:17-24` checks `=== undefined` then `=== null` + +**Category:** Simplification + +These two checks are redundant in strict TypeScript where the function signature +`(x: number)` does not admit `undefined` or `null`. The function could be +`x.toFixed(1)` directly, or the guard unified as `x == null`. + +### [Nit] `
` tag in `App.tsx:242` is deprecated HTML + +**Category:** Simplification + +`
` was deprecated in HTML 4. A `
` or +MUI `` should be used instead, consistent with the +rest of the codebase. + +--- + +## TS/JS typing consistency + +Two files are plain JavaScript: + +| File | Reason cited | Impact | +|---|---|---| +| `Plot.jsx` | "plotly is not TypeScript" | The comment is outdated. `@types/react-plotly.js` is already in `devDependencies`. The file is 12 lines with no props, so migration is trivial. | +| `MultiLinePlotWrapper.jsx` | Same | All props (`xaxis`, `yaxis`, `data`, `grid_scan`, `range_x`, `range_y`) are accessed without any type guards. TypeScript would catch mismatches between `DataProcessingPlot.tsx` and this wrapper. | + +The `tsconfig.json` sets `"allowJs": true`, so these files are compiled but not +type-checked. `strict: true` applies only to `.ts`/`.tsx` files. The untyped +boundary is precisely at the point where typed Plotly data leaves +`DataProcessingPlot.tsx` and enters `MultiLinePlotWrapper.jsx` — that is the +most complex data path in the rendering pipeline and the one most worth +protecting. + +The rest of the codebase is consistently typed. All OpenAPI model types are used +as-generated (no `any` escape hatches in the hand-written code). The pattern of +`type MyProps` / `type MyState` / `Component` is uniform. The +only weak typing outside the JSX files is `PlotlyPlot` and `PlotlyData` in +`DataProcessingPlot.tsx` (lines 18-28), which are hand-written approximations of +the Plotly `Data` type rather than imports from `plotly.js-cartesian-dist-min`. + +--- + +## How it consumes the generated OpenAPI client + +- **Read path:** All polling goes through `DefaultService.getStatistics()` from + the generated client. The client is imported in `App.tsx` and in + `DataProcessingPlot.tsx` (`DefaultService.getPreviewPlot`). Returned model + objects are used directly as TypeScript types — no re-mapping or DTO + transformation layer exists. +- **Write path:** Writes are split. Some components call `DefaultService` methods + directly (`DataProcessingSettings.putValues`, `ImageFormatSettings.raw/conv/putValues`, + `PixelMask.handleUpload`). Others pass a JSON string body through + `ButtonWithSnackbar`, which issues a raw `fetch` call using the path and + `OpenAPI.BASE` from the generated config. This is a two-track write pattern: + the generated client is not used consistently for mutations, and the + `ButtonWithSnackbar`-based writes bypass the client's `CancelablePromise` + cancellation and error-normalisation machinery. +- **`OpenAPI.BASE` initialisation** happens in `App.componentDidMount` (line 79), + setting it to `''`. This must execute before any component calls the generated + service; it is safe only because the root `setInterval` fires after + `componentDidMount` completes. +- **Model consumption:** The `plots`/`plot` model is consumed correctly in + `DataProcessingPlot.tsx` — `plot[0].z` presence drives the heatmap vs. scatter + branch. The `plots.size_x/size_y` grid-scan extents are passed through to the + Plotly layout. The only risk is `plot[0].z` being `undefined` vs. an empty + array: the code checks `length > 0` (line 141), which is correct. diff --git a/docs/review/09-api-contract-line.md b/docs/review/09-api-contract-line.md new file mode 100644 index 00000000..5449d53e --- /dev/null +++ b/docs/review/09-api-contract-line.md @@ -0,0 +1,234 @@ +# Review: API contract line (frontend ↔ broker ↔ receiver/plots) + +Scope: the live plot/status/preview data flow **receiver → broker → frontend**, traced +across all three layers. All citations are to files in this repo. + +## The data flow, end to end (a precise trace with file:line at each hop) + +### 1. Producer (receiver, in-process) + +The receiver and broker run **in the same process**; there is no gRPC/Cap'n Proto/shared-memory +boundary for plot data. The broker holds a raw pointer to the receiver service and calls a method +directly: + +- `broker/JFJochServices.h:20` — `JFJochReceiverService *receiver = nullptr;` (raw pointer, set via `Receiver()`). +- `broker/JFJochServices.cpp:177-180` — `GetPlots(...)` → `return receiver->GetDataProcessingPlot(request);` +- `receiver/JFJochReceiverService.h:39` — `JFJochReceiverPlots plots;` is a plain data member. +- `receiver/JFJochReceiverService.cpp:154-156` — `GetDataProcessingPlot` → `plots.GetPlots(request);` + +Plot/stats values are accumulated per image in `JFJochReceiverPlots::Add` from a `DataMessage` +(`receiver/JFJochReceiverPlots.cpp:128-211`), into a large set of `StatusVector` members +(`receiver/JFJochReceiverPlots.h:44-108`) plus per-ROI `StatusVector`s +(`receiver/JFJochReceiverPlots.h:79-88`) and one `AzimuthalIntegrationProfile` +(`receiver/JFJochReceiverPlots.h:40`). + +A request is served by `JFJochReceiverPlots::GetPlots(const PlotRequest&)` +(`receiver/JFJochReceiverPlots.cpp:217-455`): +- The X-axis unit is decided here (`MultiLinePlotUnits`, see `common/MultiLinePlot.h:12`) based on + `request.experimental_coord`, goniometer/grid-scan presence, and `request.azint_unit` + (`receiver/JFJochReceiverPlots.cpp:219-257`). +- The plot payload is produced by `StatusVector::GetMeanPlot/GetMaxPlot/GetMeanPerBin` + (`common/StatusVector.h:33-39`) or, for azimuthal integration, by + `AzimuthalIntegrationProfile::GetPlot` (`common/AzimuthalIntegrationProfile.cpp:131-149`), where + the X axis array is chosen from precomputed `bin_to_q`/`bin_to_d`/`bin_to_2theta` + (`common/AzimuthalIntegrationProfile.cpp:119-129`). +- Result type is `MultiLinePlot` (`common/MultiLinePlot.h:21-37`) carrying `plots[]` (each with + `title`, `x`, `y`, `z`), a `units` enum, and optional `size_x`/`size_y`. + +`Plot.h` (`common/Plot.h`) defines the *request* side: `PlotType`, `PlotAzintUnit`, `PlotRequest`. +`Histogram.h` and `ADUHistogram.*` are **not** part of this REST plot flow (they are not referenced +by `GetPlots`/`OpenAPIConvert`); they feed other paths. `StatusVector.*` is the core storage type. + +### 2. Broker (C++ REST) + +- HTTP route registered at `broker/JFJochBrokerHttp.cpp:305-317` (`GET /preview/plot`), handler + `preview_plot_get` at `broker/JFJochBrokerHttp.cpp:872-906`. +- Query params parsed: `type`, `binning`, `fill`, `experimental_coord`, `azint_unit` + (`broker/JFJochBrokerHttp.cpp:307-312`). +- `type` string → `PlotType` via `ConvertPlotType` (`broker/OpenAPIConvert.cpp:849-892`). +- `azint_unit` string → `PlotAzintUnit` is parsed **inline in the handler**, NOT via OpenAPIConvert + (`broker/JFJochBrokerHttp.cpp:878-886`). +- `state_machine.GetPlots(req)` (`broker/JFJochStateMachine.cpp:602-603`) → services → receiver + (in-process, see above). +- `MultiLinePlot` → generated `Plots` model via `Convert(const MultiLinePlot&)` + (`broker/OpenAPIConvert.cpp:388-432`): copies `title/x/y/z` per plot, maps `units`→`Plot_unit_x`, + copies `size_x`/`size_y`. +- Serialized by `ProcessOutput(...)` (JSON via `to_json`) — generated model + `broker/gen/model/Plots.cpp:105-117`, `broker/gen/model/Plot_unit_x.cpp:69-97`. + +### 3. Frontend (TS) + +- Client method `DefaultService.getPreviewPlot(...)` (`frontend/src/openapi/services/DefaultService.ts:1033-1054`) + issues `GET /preview/plot`, returns `plots` (`frontend/src/openapi/models/plots.ts`). +- Polling component `DataProcessingPlot` (`frontend/src/components/DataProcessingPlot.tsx:115-126`): + calls `getPreviewPlot` once on mount and then every **1000 ms** (`setInterval`). +- Renders to Plotly through `MultiLinePlotWrapper.jsx` → `Plot.jsx` (cartesian bundle). +- Axis labels are computed **independently in the frontend** by `AxisTypeX(unit_x)` and + `AxisTypeY(plot_type)` (`frontend/src/components/DataProcessingPlot.tsx:30-96`). +- Plot selector / unit toggles: `DataProcessingPlots.tsx` (menu of plot types, `binning`, + `experimental_coord`, `azint_unit` toggle). + +### Separate preview-image path (NOT ZeroMQ in the browser) + +- `frontend/src/components/PreviewImage.tsx:256-296` fetches `GET /image_buffer/image.jpeg?...` + over plain HTTP, polling every **2000 ms** while measuring. +- Broker handler `image_buffer_image_jpeg_get` (`broker/JFJochBrokerHttp.cpp:247-253, 665-697`) + calls `state_machine.GetPreviewJPEG(...)`. Server-side rendering of overlays (spots, ROI, beam + center, resolution rings) is done in `preview/PreviewImage.cpp`. +- `frontend/src/components/ZeroMQPreview.tsx` is **only a settings panel** that configures a ZMQ + *publisher* socket on the receiver (`receiver/JFJochReceiverService.cpp:221-264`, + `ZMQPreviewSocket`). The browser never reads that ZMQ stream — it is for external downstream + consumers. See "REST plots vs ZeroMQ preview" below. + +## How the OpenAPI contract is generated for C++ and TS (and skew risk) + +There is **one** spec: `broker/jfjoch_api.yaml`. Three generators consume it (`update_version.sh:38-49`): + +| Target | Generator | Version | Invoked from | +|--------|-----------|---------|--------------| +| Python client | `openapi-generator-cli` `python` | **7.20.0** | `gen_python_client.sh:6-9`, `update_version.sh:39` | +| C++ broker server | `openapi-generator-cli` `cpp-pistache-server` | **7.20.0** | `update_version.sh:40` (also recorded `broker/gen/.openapi-generator/VERSION`) | +| TS frontend client | `openapi-typescript-codegen` | **0.25.0** (a *different* tool) | `frontend/package.json:35`, `npm run openapi` in `update_version.sh` | + +So the C++ server model and the TS client model are produced by **two different code generators** +(OpenAPI Generator 7.20.0 vs openapi-typescript-codegen 0.25.0) from the same spec. This is the +principal skew risk: the two tools interpret the spec independently (enum-string casing, +`nullable`, `required`, default handling). They happen to agree today for the plot models — verified +that C++ `Plot_unit_x::to_json` emits exactly the strings in the spec enum +(`broker/gen/model/Plot_unit_x.cpp:78-95` vs `frontend/src/openapi/models/plot_unit_x.ts:6-13`) — but +there is no automated cross-check, and a spec change that one generator handles differently would not +be caught. `update_version.sh` regenerates all three together, which mitigates *staleness* but not +*semantic divergence between generators*. + +## Findings + +### [High] `compression_ratio` plot is computed by the receiver but unreachable via the API — `broker/jfjoch_api.yaml:85-122` +**Category:** Contract drift +The receiver fully supports `PlotType::CompressionRatio` (stored in `compression_ratio`, +`receiver/JFJochReceiverPlots.cpp:160,302-303`, and exported in `GetPlotRaw`, +`receiver/JFJochReceiverPlots.cpp:573-575`). But `compression_ratio` is **not** in the spec +`plot_type` enum, **not** in `ConvertPlotType` (`broker/OpenAPIConvert.cpp:849-892` — no case), and +**not** in `plot_type.ts`. Direction: the receiver capability is dead from the REST/UI side; any +client passing `type=compression_ratio` gets a 500 ("Plot type not recognized"). Either add it to the +spec + converter or drop it from the receiver. (Note: `image_scale_factor`/`image_scale_cc`/ +`image_scale_b` ARE wired through all three layers, so only `compression_ratio` is orphaned.) + +### [Medium] `azint_unit` is parsed by hand in the HTTP handler, diverging from `ConvertPlotType`/spec — `broker/JFJochBrokerHttp.cpp:878-886` +**Category:** Contract drift | Bug +Unlike `type` (centralized in `ConvertPlotType`), the `azint_unit` query value is matched inline with +ad-hoc lowercase variants: it accepts `"Q_recipA"`/`"q_recipa"`, `"d_A"`/`"d_a"`, `"two_theta_deg"`. +The spec enum (`broker/jfjoch_api.yaml:62-66`) and TS client only ever send `"Q_recipA"`, `"d_A"`, +`"two_theta_deg"` (`frontend/src/openapi/models/azint_unit.ts`). Any **unrecognized** value silently +falls through to the default `Q_recipA` (the `if/else if` chain has no error branch), so a typo or a +future enum addition is silently mis-rendered rather than rejected. This parsing should go through a +single `Convert`/`ConvertAzintUnit` like every other enum. + +### [Medium] 2θ azimuthal-integration unit is collapsed to `angle_deg` and mislabeled as "Rotation angle" — `receiver/JFJochReceiverPlots.cpp:250-252`, `frontend/src/components/DataProcessingPlot.tsx:33-34` +**Category:** Units/semantics +When `azint_unit == two_theta_deg`, the receiver maps it to `MultiLinePlotUnits::Angle_deg` +(`receiver/JFJochReceiverPlots.cpp:250-252`), which serializes to `plot_unit_x = angle_deg`. The +frontend `AxisTypeX(angle_deg)` returns **"Rotation angle [deg]"** +(`frontend/src/components/DataProcessingPlot.tsx:33-34`). So a 2θ azimuthal plot is labeled as +goniometer rotation angle — semantically wrong (scattering angle vs sample rotation). There is no +distinct `two_theta_deg` value in `plot_unit_x`. Direction: `Plot.h`/`MultiLinePlot.h` lack a 2θ +X-unit; the spec and frontend therefore cannot disambiguate. Either add a `two_theta_deg` to +`plot_unit_x` (spec) or special-case the azint label in the frontend. + +### [Medium] Plot-type errors return HTTP 500, but the spec/client document only 400 — `broker/JFJochBrokerHttp.cpp:119-137`, `broker/OpenAPIConvert.cpp:890-891` +**Category:** Error handling | Contract drift +`ConvertPlotType` throws `JFJochException(InputParameterInvalid)` for an unknown/missing `type` +(`broker/OpenAPIConvert.cpp:850-852,890-891`), but `handleOperationException` maps every non-DAQ +`std::exception` to **500** (`broker/JFJochBrokerHttp.cpp:129-136`). The OpenAPI operation only +declares a **400** response, and `DefaultService.getPreviewPlot` lists `400: Input parsing or +validation error` (`frontend/src/openapi/services/DefaultService.ts:1050-1052`). So an +input-validation failure is reported as a server error. Practically the frontend treats any rejection +as `connection_error` (graceful), but the status-code contract is violated. Note the binning<0 path +*does* correctly return 400 (`broker/JFJochBrokerHttp.cpp:897-901`), making the inconsistency clearer. + +### [Low] No staleness/identity tagging on plot responses; 1 s polling can interleave with a new dataset — `frontend/src/components/DataProcessingPlot.tsx:123-126`, `receiver/JFJochReceiverPlots.cpp:217-455` +**Category:** Architecture +The plot endpoint returns no run number, image-count watermark, or generation counter. The frontend +polls every 1 s and unconditionally replaces state. On `Setup()` the receiver clears all vectors +(`receiver/JFJochReceiverPlots.cpp:44-126`); a poll that lands during/after a re-setup will show a +truncated or empty plot with no indication it belongs to a different run. The `plots` model +(`frontend/src/openapi/models/plots.ts`) has an optional `title` but it is never set by `Convert` +(`broker/OpenAPIConvert.cpp:388-432` calls `setPlot/setUnitX/setSizeX/setSizeY` but never `setTitle`), +so even per-plot identity is thin. Low severity because data is monotonic within a run. + +### [Low] `mosaicity`/`pixel_sum`/`packets_received`/`image_scale_*` are wired through the API but absent from the UI plot menu — `frontend/src/components/DataProcessingPlots.tsx:56-86` +**Category:** Architecture +These plot types exist in the spec enum and `ConvertPlotType`, and are renderable, but the selector in +`DataProcessingPlots.tsx` omits them (`mosaicity`, `pixel_sum`, `packets_received`, +`image_scale_factor/cc/b`, `integrated_reflections` is also absent from the menu though it has an axis +label). Not a contract bug — just capability that the UI never surfaces. Confirms the contract is +"wider" than any single consumer, which is why hand-maintained mappings drift. + +### [Low] `Plot.z` and `y` are nullable arrays in the contract, but the producer never emits nulls — `broker/jfjoch_api.yaml:1425-1436`, `frontend/src/components/DataProcessingPlot.tsx:140-165` +**Category:** Units/semantics +Spec marks `y` items and `z` items `nullable: true` (TS: `Array`, +`frontend/src/openapi/models/plot.ts:12-13`). The receiver replaces non-finite azint values with `0` +(`receiver/JFJochReceiverPlots.cpp:504-506`) and `StatusVector` fill uses `fill_value`, so nulls are +not actually produced for azint; meanwhile gap fill for binned status plots depends on `fill_value`. +The renderer's z-vs-2D detection keys only on `z.length > 0` +(`frontend/src/components/DataProcessingPlot.tsx:140-141`); a 1D plot that happens to carry a `z` +would be misrendered as a heatmap. Minor, but the nullable contract is broader than reality and the +2D detection is heuristic rather than driven by `unit_x == grid_um`. + +### [Nit] Two different code generators for one spec, with no cross-validation — `update_version.sh:39-49` +**Category:** Architecture +See the generation table above. C++ uses OpenAPI Generator 7.20.0; TS uses +openapi-typescript-codegen 0.25.0. They agree today but nothing enforces it (no contract test that +round-trips a `Plots` JSON through both). A `redocly`-validated spec is built +(`frontend/package.json:33-34`) but that validates the spec, not generator agreement. + +## Field-by-field mapping risks in OpenAPIConvert + +`Convert(const MultiLinePlot&)` (`broker/OpenAPIConvert.cpp:388-432`) is the single hand-written hop +for plot payloads. Per-field assessment: + +| Domain field (MultiLinePlot) | Mapped? | Risk | +|---|---|---| +| `plots[].title` | yes (`:391`) | OK | +| `plots[].x` | yes (`:392`) | OK | +| `plots[].y` | yes (`:393`) | OK | +| `plots[].z` | yes (`:394`) | always copied even when empty; frontend 2D detection keys on `z.length` (Low finding above) | +| `units` → `unit_x` | yes (`:400-423`) | enum mapped by switch; `default: break` leaves `Plot_unit_x` at its default `image_number` if a new `MultiLinePlotUnits` value is added — **silent fallback, not an error** | +| `size_x` / `size_y` | yes, optional (`:425-429`) | OK | +| **`MultiLinePlot` has no `title`** | n/a | the top-level `plots.title` (spec optional) is **never set** — `setTitle` is not called; UI cannot show a per-response title | + +Enum-by-string risks elsewhere on this line: +- `ConvertPlotType` (`broker/OpenAPIConvert.cpp:849-892`): hand-written string→enum with **35** cases. + Missing `compression_ratio` (High finding). Each entry must stay in lockstep with the spec enum + (`broker/jfjoch_api.yaml:85-122`) and `plot_type.ts` by hand — classic triple-maintenance. +- The `unit_x` switch and the `ConvertPlotType` chain are the two places where adding a plot + type/unit requires edits in *four* spots (Plot.h enum, receiver `GetPlots` switch, OpenAPIConvert, + spec) plus regeneration. No compiler error ties them together; the `default:` arms swallow misses. + +## REST plots vs ZeroMQ preview: the split + +There are effectively **three** transports in this feature area, and the "ZeroMQ preview" name is the +source of confusion: + +1. **REST 1D/2D plots** (`GET /preview/plot`, JSON): receiver→broker in-process, broker→browser via + `DefaultService.getPreviewPlot`, polled at 1 s. This is the line traced above. There is also a raw + binary variant `GET /preview/plot.bin` (`broker/JFJochBrokerHttp.cpp:855-870`, + `DefaultService.getPreviewPlotBin`) that returns float32 values **without** X-axis or unit metadata + — a deliberately lossy fast path, used by external tools rather than the React UI. + +2. **REST preview image** (`GET /image_buffer/image.jpeg`, JPEG): the browser's live detector image + (`frontend/src/components/PreviewImage.tsx:256-296`, polled at 2 s). Server renders overlays in + `preview/PreviewImage.cpp`. This is **not** ZeroMQ despite living near the "preview" UI. + +3. **ZeroMQ preview publisher** (`receiver/JFJochReceiverService.cpp:221-264`, `ZMQPreviewSocket`): + a push socket the receiver opens for **external** downstream consumers (writers/online analysis). + `frontend/src/components/ZeroMQPreview.tsx` only PUTs `/config/zeromq_preview` settings (enabled, + period_ms, socket_address) — the browser never subscribes to this stream. + +**Is the split justified?** Mostly yes, but the naming is accidental and misleading: +- The browser uses *polling REST* for both plots (1 s) and the preview image (2 s); neither uses push. + The "ZeroMQ preview" is an orthogonal external feed, so calling the in-browser image path a + "preview" alongside a "ZeroMQ preview settings" panel invites the wrong mental model. +- Plots are JSON (carry axis + units), the image is JPEG (server-rendered, opaque to the client), and + the ZMQ feed is raw frames for machines. These are genuinely different payloads, so three transports + is defensible. The hazard is purely cognitive/naming plus the freshness gap (no run-identity on any + of the polled REST responses — see the staleness Low finding). diff --git a/docs/review/DIALS_NXMX_COMPATIBILITY.md b/docs/review/DIALS_NXMX_COMPATIBILITY.md new file mode 100644 index 00000000..555e2507 --- /dev/null +++ b/docs/review/DIALS_NXMX_COMPATIBILITY.md @@ -0,0 +1,812 @@ +# DIALS / dxtbx NXmx compatibility review + +**Scope.** Compare the DIALS 3.27 / dxtbx NXmx reader against what the Jungfraujoch HDF5 +writer actually emits, for all three `FileWriterFormat`s. + +**Status of this document.** It is an audit. **Nothing in the repository was changed.** The +follow-ups in §6 are proposals only; none of them were implemented. + +**Provenance markers used throughout.** + +| Marker | Meaning | +| --- | --- | +| **[measured]** | I ran it; the output is quoted below or was observed directly | +| **[read]** | Verified by reading source, with `file:LINE` on both sides | +| **[inferred]** | Reasoned from code I read, but not executed | + +**Environment.** DIALS `3.27.0-g9c415ec69-release` at `/opt/dials-v3-27-0` +(`source /opt/dials-v3-27-0/dials_env.sh`), h5py with HDF5 1.14.x, `hdf5plugin` 6.0.0. +Jungfraujoch branch `2608-rc-162`, `jfjoch_hdf5_test` rebuilt from +`cmake-build-release` (reports `Version 1.0.0-rc.161 (git 538f35)`). +Generated files live in the session scratchpad, not in the repo. + +The reader path is three files, and every dxtbx citation below is one of them: + +* `dxtbx/format/FormatNXmx.py` — the format class that wins (178 lines) +* `dxtbx/nexus/__init__.py` — model construction: goniometer, beam, scan, detector, mask, raw data (595 lines) +* `nxmx/__init__.py` — the standalone `nxmx` package at + `/opt/dials-v3-27-0/conda_base/lib/python3.13/site-packages/nxmx/__init__.py` (1236 lines), + which does the actual HDF5 traversal + +--- + +## 1. Summary + +### What works today + +For the shapes the DIALS CI job exercises — a **single-data-file** `NXmxLegacy`, `NXmxVDS`, +and `NXmxIntegrated`, stills or rotation — Jungfraujoch files import into DIALS cleanly and +the geometry is right. `dials.import` picks `FormatNXmx` in every case, `dials.find_spots` runs, +the bitshuffle/LZ4 chunks decode without any extra setup, and the beam centre, pixel size, +image size, distance, wavelength, sensor material/thickness, scan oscillation and goniometer +axis all come out at the expected values **[measured]**. The `NXtransformations` chains +terminate correctly at `"."` and dxtbx walks them as intended. + +### What is at risk + +Four findings, in descending order of severity. + +1. **`NXmxLegacy` with more than one data file is silently truncated by DIALS.** dxtbx reads + only `list(nxdata.values())[0]` — the *first* `data_00000N` link — and never advances to + the next one (`dxtbx/nexus/__init__.py:583`). `dials.import` reports **only the first + data file's image count**, with no error and no warning: 40 images written as 4 files of + 10 import as **10**; 100 images written as 4 files of 25 import as **25** **[measured]**. + As a rotation sweep the count comes out right and the failure moves to first pixel + access — `dials.find_spots` dies with `IndexError: Index (25) out of range for (0-24)` + **[measured]**. dxtbx *has* the code to walk `data_\d{6}` members — its regex matches our + naming exactly — but only in `FormatNXmxEigerFilewriter`, gated behind a DECTRIS-only key + (`FormatNXmxEigerFilewriter.py:118-150`). This is the one finding that loses data on a + configuration we ship, and **CI cannot catch it** (§2.6). +2. **A missing VDS source file is silently read as zeros.** `dials.import` succeeds, reports + 100 images, and every pixel of every image reads back `0` with no warning at any log + level **[measured]**. The `NXmxLegacy` equivalent at least fails loudly (a `KeyError`, + though one that does not name the missing file). +3. **No `underload_value`, so the trusted range has no lower bound.** We write the underload + as `error_value`, flagged in our own source as "this is not NXmx" + (`writer/HDF5NXmx.cpp:369`). dxtbx therefore falls back to `-0x7FFFFFFF` + (`dxtbx/nexus/__init__.py:428-432`) and the panel comes out with + `trusted_range: (-2147483647.0, 32766.0)` **[measured]**. Gap pixels carry `-32768` and + are *inside* that range; today only the `pixel_mask` keeps them out of the analysis. +4. **Grid scans do not import at all.** A sample `depends_on` chain containing only + translations makes `get_dxtbx_goniometer` do a matmul against an empty array and raise + `ValueError: matmul: Input operand 1 has a mismatch in its core dimension 0` + (`dxtbx/nexus/__init__.py:58`) **[measured on a hand-built file reproducing + `writer/HDF5NXmx.cpp:771-800`]**. + +Plus one geometry issue that is latent because the values are almost always zero: the +exported detector tilt axes `rot1`/`rot2` carry each other's `@vector`, and the NXmx chain +pivots the module about corner pixel (0,0) while our own refinement pivots about the PONI +(§4, rows G4/G5). + +Format-class selection is safe today but has a one-key-wide margin against +`FormatNXmxEigerFilewriter` (§4, row F2). + +--- + +## 2. Empirical results + +### 2.1 Generating the files + +`jfjoch_hdf5_test` run with no arguments prints its real usage (this is authoritative; note +it does **not** advertise `-S`, which `tools/jfjoch_hdf5_test.cpp:41-43` does implement): + +``` +Usage: ./jfjoch_hdf5_test [-o ] [-n ] [-V] [-f ] [-R] +``` + +Flags, from `tools/jfjoch_hdf5_test.cpp:29-56`: `-V` → `NXmxVDS`, `-S` → `NXmxIntegrated`, +neither → `NXmxLegacy`; `-f` sets images per file; `-R` adds a goniometer +`omega` with axis `(-1,0,0)` (`tools/jfjoch_hdf5_test.cpp:79-80`) — without it the file is a +stills set with `/entry/sample/depends_on == "."` **[measured]**. + +``` +T=/home/leonarski_f/jungfraujoch/cmake-build-release/tools/jfjoch_hdf5_test +IN=/home/leonarski_f/jungfraujoch/tests/test_data/compression_benchmark.h5 + +# the three shapes the DIALS CI job builds +$T $IN -n100 -o legacy # NXmxLegacy, 1 data file +$T $IN -n100 -f25 -V -o vds # NXmxVDS, 4 data files +$T $IN -n100 -S -o single # NXmxIntegrated, self-contained + +# added here: rotation, and multi-file legacy (never built by the DIALS CI job) +$T $IN -n100 -R0.1 -o legacy_rot # NXmxLegacy rotation, 1 data file +$T $IN -n100 -f25 -o legacy4 # NXmxLegacy, 4 data files +$T $IN -n100 -f25 -R0.1 -o legacy4rot # NXmxLegacy rotation, 4 data files + +# the controlled 40-image matrix of §2.5 +$T $IN -n40 -f10 -o multi # NXmxLegacy, 4 data files +$T $IN -n40 -f10 -V -o vds # NXmxVDS, 4 data files +$T $IN -n40 -o single_legacy # NXmxLegacy, 1 data file +``` + +All nine wrote successfully. Sizes: `legacy_master.h5` 17.9 MB (the uncompressed `pixel_mask` +dominates), `vds_master.h5` 0.24 MB, `single_master.h5` 120.8 MB. + +### 2.2 `dials.import` — the three CI formats + +All three resolve to `FormatNXmx` and import without error or warning **[measured]**. +Verbatim, trimmed to the report block: + +``` +$ dials.import legacy_master.h5 + format: + template: .../legacy_master.h5 + num images: 100 + sequences: + still: 0 + sweep: 0 + num stills: 100 + +$ dials.import vds_master.h5 + format: + template: .../vds_master.h5 + num images: 100 + num stills: 100 + +$ dials.import single_master.h5 + format: + template: .../single_master.h5 + num images: 100 + num stills: 100 + +$ dials.import legacy_rot_master.h5 + format: + template: .../legacy_rot_master.h5:1:100 + num images: 100 + sequences: + still: 0 + sweep: 1 + num stills: 0 +``` + +The stills/sweep split is correct behaviour: without `-R` there is no goniometer, so +`/entry/sample/depends_on` is `"."`, `nxmx.NXsample.depends_on` returns `None` +(`nxmx:553-559`) and both `get_dxtbx_goniometer` and `get_dxtbx_scan` return `None` +(`dxtbx/nexus/__init__.py:41-42`, `:177-178`) — 100 independent stills. + +### 2.3 `dials.show` — models extracted + +`dials.show legacy_rot_imported.expt`, verbatim **[measured]**: + +``` +Detector: +Panel: + name: /entry/instrument/detector/module + type: SENSOR_PAD + pixel_size:{0.075,0.075} + image_size: {2068,2164} + trusted_range: {-2.14748e+09,32766} + thickness: 0.32 + material: Si + mu: 4.20799 + gain: 1 + pedestal: 0 + fast_axis: {1,0,0} + slow_axis: {0,-1,0} + origin: {-81.75,85.2,-75} + distance: 75 + pixel to millimeter strategy: ParallaxCorrectedPxMmStrategy + mu: 4.20799 + t0: 0.32 + +Max resolution (at corners): 1.038773 +Max resolution (inscribed): 1.325398 + +Beam: + probe: x-ray + wavelength: 1 + sample to source direction : {0,0,1} + polarization normal: {0,1,0} + polarization fraction: 0.999 + flux: 0 + transmission: 1 + +Beam centre: + mm: (81.75,85.20) + px: (1090.00,1136.00) + +Scan: + number of images: 100 + image range: {1,100} + epoch: 0 + exposure time: 0.0005 + oscillation: {0,0.0999999} + +Goniometer: + Rotation axis: {1,0,0} + Fixed rotation: {1,0,0,0,1,0,0,0,1} + Setting rotation:{1,0,0,0,1,0,0,0,1} +``` + +Everything here is correct against the values the test tool set +(`tools/jfjoch_hdf5_test.cpp:75-80`: beam 1090/1136 px, 75 mm, 1 Å) except three items worth +noting: + +* `trusted_range` lower bound `-2147483647` — finding 3 above. +* `exposure time: 0.0005` is `frame_time`, not `count_time` (0.00048). dxtbx builds the scan + exposure from `nxdetector.frame_time` and ignores `count_time` + (`dxtbx/nexus/__init__.py:227-230`) **[read]**. +* `Rotation axis: {1,0,0}` although we wrote `@vector = (-1,0,0)`. This is correct and + expected: `MCSTAS_TO_IMGCIF = np.diag([-1, 1, -1])` (`dxtbx/nexus/__init__.py:33`) is + applied at `:48`. The same conversion turns our NeXus translation + `(0.5844, 0.6091, 0.5362) × 139.88 mm` into the imgCIF origin `(-81.75, 85.2, -75)` — + which matches to 6 significant figures **[measured]**, confirming the whole detector + chain is being walked as intended. + +### 2.4 Raw data, masks, and compression + +**[measured]** via `dxtbx.model.ExperimentList`, identical for all four single-data-file sets: + +``` +trusted_range: (-2147483647.0, 32766.0) +raw dtype: int32 shape (2164, 2068) min -32768 max 32767 +most common values: 0 (2287176), 1 (1155035), 2 (462418), -32768 (239792), 3 (193195) +count(-32768): 239792 count(-1): 0 count(-2): 0 +dynamic mask (get_mask): 4129012 True / 346140 False +static mask (get_static_mask): 4129024 True / 346128 False +``` + +Notes: + +* **The bitshuffle/LZ4 chunks decode with no user setup.** `dxtbx/__init__.py:23-28` prepends + its own plugin directory to `HDF5_PLUGIN_PATH` at import, and + `dxtbx/format/nexus.py:11` does `import hdf5plugin` **[read]**. `hdf5plugin` 6.0.0 is a + hard conda dependency of DIALS (it appears in every `.conda-envs/*.txt` lock file) + **[read]**. Filter on the dataset is `32008` with `cd_values (0, 4, 2, 8192, 2)` — + bitshuffle 0.4, elem_size 2, block 8192, compression code 2 = LZ4 **[measured]**. +* **The static mask is found**, despite the mask's real storage being under + `detectorSpecific`: `writer/HDF5NXmx.cpp:422-423` creates a hard link at the standard + `/entry/instrument/detector/pixel_mask`, which is what `nxmx:881` looks up. Values present + are `0` (good, 4129024 px), `1` (239792 px), `1<<30` (24416 px), `1<<31` (81664 px), + `1<<30|1<<31` (256 px) **[measured]**. dxtbx masks anything non-zero + (`dxtbx/nexus/__init__.py:504`), so all of them are excluded — which is the intent here, + but see §4 row M2 for the bit-semantics caveat. +* The 12-pixel difference between the static and dynamic mask is the `32767` overload + marker falling outside `trusted_range`'s upper bound of 32766. +* `int16` on disk becomes `int32` in flex (`dxtbx/nexus/__init__.py:540-548`), and the + `bit_depth_readout == 16` top-two-value remap in `FormatNXmx.get_raw_data:163-177` + targets `65535`/`65534`, which cannot occur in signed 16-bit data — so it is a no-op for + us **[measured: `count(-1) == count(-2) == 0`]**. (Bit depth and signedness are a separate + audit; this is the one row of it here.) + +`dials.find_spots` runs to completion on `legacy_rot`, `vds` and `single`, finding ~5900 +strong pixels and ~850 spots per image **[measured]** — so the decode path is exercised +end to end, not just the metadata. + +### 2.5 The `NXmxLegacy` multi-file failure + +This is the headline defect. **A multi-file `NXmxLegacy` master is silently truncated to its +first data file by DIALS.** + +The controlled matrix — same image count, same data-file count, only the container format +changes **[measured, all three verified end to end]**: + +| Command | Format | Data files | `dials.import` reports | | +| --- | --- | --- | --- | --- | +| `jfjoch_hdf5_test -n40 -o single_legacy` | `NXmxLegacy` | 1 | **40** | correct | +| `jfjoch_hdf5_test -n40 -f10 -o multi` | `NXmxLegacy` | 4 | **10** | **75% of the data silently discarded** | +| `jfjoch_hdf5_test -n40 -f10 -V -o vds` | `NXmxVDS` | 4 | **40** | correct | + +Verbatim, trimmed to the report block: + +``` +$ dials.import multi_master.h5 # 40 images, 4 data files + format: + template: .../multi_master.h5 + num images: 10 + sequences: + still: 0 + sweep: 0 + num stills: 10 + +$ dials.import vds_master.h5 # 40 images, 4 data files + num images: 40 + num stills: 40 + +$ dials.import single_legacy_master.h5 # 40 images, 1 data file + num images: 40 + num stills: 40 +``` + +**No error, no warning, exit code 0.** The first case is the one that matters: a user with a +40-image legacy dataset gets a 10-image experiment list and nothing tells them. + +The same behaviour at the CI image count, and the rotation variant: + +``` +$ dials.import legacy4_master.h5 # 100 images, 4 data files, stills + format: + num images: 25 # <-- 75 images silently dropped + num stills: 25 + +$ dials.import legacy4rot_master.h5 # 100 images, 4 data files, rotation + format: + template: .../legacy4rot_master.h5:1:100 + num images: 100 # <-- count from the scan, so it looks right + sequences: + sweep: 1 +``` + +This is **not a recent regression**: the prebuilt `build/tools/jfjoch_hdf5_test` +(`Version 1.0.0-rc.153`, git `c9803c`, June 2026) produces a master that imports as +`num images: 10` from the same `-n40 -f10` invocation **[measured]**. + +Reading images out of that rotation imageset **[measured]**: + +``` +len(imageset) = 100 + image 0: OK sum=-7853392911 + image 24: OK sum=-7853255753 + image 25: IndexError: Index (25) out of range for (0-24) + image 50: IndexError: Index (50) out of range for (0-24) + image 99: IndexError: Index (99) out of range for (0-24) +``` + +and `dials.find_spots legacy4rot.expt` therefore aborts: + +``` +IndexError: Please report this error at https://github.com/dials/dials/issues or to +dials-user-group@jiscmail.ac.uk: Index (25) out of range for (0-24) +``` + +**Mechanism [read].** `writer/HDF5NXmx.cpp:104-110` creates one external link per data file, +named `/entry/data/data_000001`, `data_000002`, … dxtbx's `get_raw_data` does: + +```python +576 if nxdata.signal: +... +582 else: +583 data = list(nxdata.values())[0] +584 all_data = [] +585 sliced_outer = data[index] +``` +— `dxtbx/nexus/__init__.py:576-585`. With no `@signal` attribute on `/entry/data` +(we write none — `writer/HDF5NXmx.cpp:102`; confirmed `nxdata.signal is None` +**[measured]**), it takes the first member and indexes it with the *global* image number. +The image count for stills comes from the same first dataset +(`FormatNXmx.py:139-143`), hence `num images: 25`; for rotation it comes from the scan +length (`FormatNXmx.py:135-136`), hence a correct 100 followed by a hard failure on read. + +The same file also shows dxtbx *can* do this correctly — `FormatNXmxEigerFilewriter.py:136` +selects `[v for k, v in sorted(nxdata.items()) if DATA_FILE_RE.match(k)]` with +`DATA_FILE_RE = re.compile(r"data_\d{6}")` (`:16`) and walks the subsets at `:137-141`. Our +member names match that regex exactly. The capability is simply not reachable from +`FormatNXmx`. + +### 2.6 Why CI has never caught this + +The multi-file legacy layout **is** produced in CI — just never handed to DIALS +**[read, `.gitea/workflows/build_and_test.yml`]**: + +| Job | Line | Legacy invocation | Data files | Validator | +| --- | --- | --- | --- | --- | +| `dials-test` | `:325` | `-n100 -o legacy` | **1** | `xia2.ssx` | +| `xds-durin-test` | `:360` | `-n25 -f10` | 3 | XDS + Durin | +| `xds-test` | `:396` | `-n25 -f10` | 3 | XDS + jfjoch plugin | +| `xds-neggia-test` | `:431` | `-n25 -f10` | 3 | XDS + Neggia | + +`dials-test` (`:307-341`) is the only job that runs DIALS, and it is the only one whose +legacy step omits `-f`. The three XDS jobs all pass `-f10` and all pass — the XDS HDF5 +plugins walk the `data_00000N` links correctly. So the layout is exercised, and DIALS is +exercised, but never both at once. That is the whole of the blind spot. + +Two further weaknesses in the same job, both **[read]**, neither verified by running CI: + +* **The DIALS job asserts nothing about image count.** The steps at `:326`, `:334` and + `:342` run `xia2.ssx` and let the exit code decide. `xia2.ssx` on 10 of 100 lysozyme + stills would very likely still index and integrate successfully, so even adding `-f` to + the legacy step would not necessarily turn the job red. An explicit + `dials.import` + image-count assertion is what actually closes this. +* **The single-file step points at a filename that does not exist.** Line `:341` writes with + `-S -o single` and line `:342` then runs `xia2.ssx image=single.h5`, but `-S -o single` + produces **`single_master.h5`** — there is no `single.h5` **[measured: the only output + file is `single_master.h5`]**. I did not run `xia2.ssx` to see how it reacts to a missing + input, so I cannot say whether that step currently fails, is tolerated, or silently + no-ops — but as written it does not validate `NXmxIntegrated` against the file the writer + produces. Worth checking against a real CI log. + +### 2.7 Link resolution, missing files, and stale links + +**Relative paths resolve correctly from any working directory [measured].** Both the +`NXmxLegacy` external links and the `NXmxVDS` sources are basenames — `vds_data_000001.h5`, +etc. — because `HDF5Object::ExternalLink` strips the directory +(`writer/HDF5Objects.cpp:675-684`) and `HDF5Dcpl::SetVirtual` runs the path through +`ExtractFilename` (`writer/HDF5Objects.cpp:334-339`, `:1015-1019`) **[read]**. Opening a +master from `/` with an absolute path read image 0 correctly in both formats. Master and +data files must sit in the same directory; there is no way to express a subdirectory. + +**Missing data file, `NXmxLegacy` — fails, but late and unhelpfully [measured]:** + +``` +$ dials.import legacy_rot_master.h5 # data file deleted + num images: 100 + sequences: + sweep: 1 + Writing experiments to leg.expt # exit 0 + +>>> imageset.get_raw_data(0) +KeyError: "Unable to synchronously open object (can't open file)" +``` + +The import succeeds and the failure surfaces only on first pixel access, with an error that +does not name the file it could not open. + +**Missing source files, `NXmxVDS` — does not fail at all [measured]:** + +``` +$ dials.import vds_master.h5 # all four vds_data_*.h5 deleted + num images: 100 + num stills: 100 + Writing experiments to vdsm.expt # exit 0 + +>>> imageset.get_raw_data(0) + min 0 max 0 sum 0 unique values 1 +``` + +Every pixel reads back the HDF5 VDS fill value with no error, no warning, and no log line. +A downstream `dials.find_spots` would simply find nothing and report zero spots. This is +standard HDF5 VDS behaviour, not a dxtbx bug, but it is the failure mode our default format +has. + +### 2.8 Grid scan — hard import failure + +`jfjoch_hdf5_test` cannot emit a grid scan, so I hand-built a master reproducing the exact +shape of `writer/HDF5NXmx.cpp:771-800`: an `NXtransformations` group hard-linked as both +`goniometer` and `transformations`, containing **only** `grid_scan_x` and `grid_scan_y` +translations, with `/entry/sample/depends_on` pointing at `grid_scan_y`. **[measured]** + +``` +File ".../dxtbx/format/FormatNXmx.py", line 101, in _start + self._goniometer_model = dxtbx.nexus.get_dxtbx_goniometer(nxsample) +File ".../dxtbx/nexus/__init__.py", line 58, in get_dxtbx_goniometer + flex.vec3_double((MCSTAS_TO_IMGCIF @ axes.axes.T).T), +ValueError: matmul: Input operand 1 has a mismatch in its core dimension 0, + with gufunc signature (n?,k),(k,m?)->(n?,m?) (size 0 is different from 3) +``` + +**Mechanism [read].** `nxsample.depends_on` is non-`None`, so `get_dxtbx_goniometer` proceeds +past its guard (`dxtbx/nexus/__init__.py:41-42`). `nxmx.get_rotation_axes` skips every +transformation whose `transformation_type != "rotation"` (`nxmx:1221-1222`), so it returns +empty arrays. `len(axes.axes) == 1` is False, the `else` branch runs, `np.sum(axes.is_scan_axis)` +on an empty array is `0` so the "sequence of stills" path at `:52-53` is taken, and the +matmul at `:58` is against a `(0,)`-shaped array. + +*Caveat:* this file was constructed by me to match the writer, not produced by the writer. +The writer branch it models is unambiguous, but if grid scans matter, confirming with a real +broker-produced grid-scan master would be worth the few minutes. + +### 2.9 Zero-increment goniometer + +A configured goniometer with a zero increment (plausible from the broker) imports and behaves +sensibly: 100 single-image experiments, each a scan with `oscillation: {0,0}` and a valid +rotation axis **[measured, hand-edited master]**. `dials.import` reports `still: 1`. No +problem here. + +### 2.10 Structure, as inspected + +`h5ls -r` / `h5dump -H` confirm the datatypes **[measured]**. Two representative extracts: + +``` +$ h5dump -H -d /entry/data/data vds_master.h5 +DATASET "/entry/data/data" { + DATATYPE H5T_STD_I16LE + DATASPACE SIMPLE { ( 100, 2164, 2068 ) / ( 100, 2164, 2068 ) } + ATTRIBUTE "image_nr_high" { DATATYPE H5T_STD_I32LE DATASPACE SCALAR } + ATTRIBUTE "image_nr_low" { DATATYPE H5T_STD_I32LE DATASPACE SCALAR } +} +``` + +``` +$ h5dump -A -d /entry/instrument/detector/module/fast_pixel_direction legacy_rot_master.h5 +DATATYPE H5T_IEEE_F64LE DATASPACE SCALAR + ATTRIBUTE "depends_on" H5T_STRING STRSIZE 48 CSET H5T_CSET_ASCII + "/entry/instrument/detector/transformations/rot3" + ATTRIBUTE "offset" H5T_IEEE_F64LE (3) 0, 0, 0 + ATTRIBUTE "transformation_type" "translation" + ATTRIBUTE "units" "m" + ATTRIBUTE "vector" -1, 0, 0 +``` + +All strings are **fixed-length ASCII** (`writer/HDF5Objects.cpp:420-465` for attributes, +`:698-708` for datasets) **[read]**. That is fine: `nxmx.h5str` decodes `np.bytes_` and +`bytes` transparently (`nxmx:49-60`) **[read]**. + +Values dxtbx/nxmx extracts from a rotation master **[measured]**: + +``` +definition: NXmx instrument.name: '' +start_time -> ValueError: ISO string too short <-- see §4 row E2 +end_time -> 2026-08-22 15:24:15.875000+00:00 +sample.name -> KeyError: object 'name' doesn't exist <-- see §4 row S1 +source.name: '' nxdata.signal: None +det.description = Detector det.distance = 0.075 m +det.count_time = 0.00048 s det.frame_time = 0.0005 s +det.beam_center_x = 1090.0 pixel det.beam_center_y = 1136.0 pixel +det.underload_value = None det.saturation_value = 32766 +det.bit_depth_readout= 16 det.bit_depth_image = 16 +det.sensor_material = Si det.sensor_thickness = 0.00032 m +det.type = None det.serial_number = None +det.pixel_mask_applied = False +beam.incident_wavelength = 1.0 angstrom +beam.flux / total_flux / incident_beam_size / profile = None +n NXsample NXtransformations groups: 2 <-- the goniometer/transformations hard link +``` + +--- + +## 3. Format-class selection + +**Resolution order [read].** `dxtbx/format/Registry.py:115-132`. The registry walks the class +DAG from the root; at each level it takes the **first** child, in sorted order, whose +`understand()` returns `True`, and descends into that child only — remaining siblings are +never asked (`Registry.py:118-121`). It is *not* "most-derived wins"; the sort key is +`(0 if in priority_formats else 1, name)` (`Registry.py:112-113`), i.e. lexicographic on the +class name within a tier. `priority_formats` comes from `format_hint` plus its ancestors +(`Registry.py:100-110`), and `ExperimentList`'s `FormatChecker` feeds the previously matched +class in as the hint for the next file +(`dxtbx/model/experiment_list.py:86-100`), with inputs natural-sorted first (`:1259`). +So in a multi-file import the outcome can depend on file order. + +**Which class wins for us [measured].** `FormatNXmx`, for every file I produced. Tracing the +full set of `understand()` calls on a Jungfraujoch master gives `True` for `FormatHDF5`, +`FormatNexus`, `FormatNXmx`, and `False` for `FormatNXmxDLS`, `FormatNXmxED`, +`FormatNXmxEigerFilewriter` and all the `FormatHDF5*` siblings. + +`FormatNXmx.understand` (`FormatNXmx.py:74-84`) and `FormatNexus.understand` +(`FormatNexus.py:13-18` → `format/nexus.py:537-543`) test the same thing: an `NXentry`-classed +group containing `definition == "NXmx"` **[read]**. We satisfy it at +`writer/HDF5NXmx.cpp:45`. + +One structural detail worth recording: `FormatNXmx` beats `FormatNexusStill` only because +`'X'(0x58) < 'e'(0x65)` puts `FormatNXmx` first among `FormatNexus`'s children **[read]**. +`FormatNexusStill.understand` (`FormatNexus.py:153-168`) would return `True` for a stills +master that omitted `/entry/sample/depends_on`. We always write it +(`writer/HDF5NXmx.cpp:801`), so this does not currently matter, but the tie-break is +alphabetical luck rather than design. + +Exceptions raised *inside* an `understand()` are **not caught** by the registry — +`Registry.py:120` and `:129` call it bare **[read]** — so a single throwing sibling aborts +the whole format search for the file. Two consequences for us are in §4 rows F3 and F4. + +--- + +## 4. Pitfall inventory + +Verdicts: **OK** = works and is robust · **risk** = works today but on a narrow margin or with +a silently-wrong result in a reachable configuration · **broken** = fails or loses data in a +configuration we ship. + +### Data location and packing + +| # | Key / feature | What dxtbx does | What Jungfraujoch writes | Verdict | Consequence | +| --- | --- | --- | --- | --- | --- | +| D1 | `/entry/data` member selection | `data = list(nxdata.values())[0]`, indexed with the global image number — `dxtbx/nexus/__init__.py:583-585` | `NXmxLegacy`: one external link per data file, `data_000001`, `data_000002`, … — `writer/HDF5NXmx.cpp:104-110` | **broken** | Multi-file legacy: stills silently truncated to the first file's image count; rotation `IndexError` past image 24 **[measured, §2.5]** | +| D2 | `@signal` on `NXdata` | `nxmx:222-234` returns `attrs.get("signal")`; dxtbx falls back to the first member when absent — `:576-583` | Not written; `/entry/data` carries only `NX_class` — `writer/HDF5NXmx.cpp:102`, `:118`, `:204` | risk | Harmless for a single dataset, but it is why D1 silently picks the wrong member instead of erroring | +| D3 | Number of images | Scan length if a scan exists, else `data.shape[0]` of the first member — `FormatNXmx.py:135-143` | — | risk | Rotation hides D1 until first read; stills expose it as a wrong count | +| D4 | `image_nr_low` / `image_nr_high` | **Never read.** No reference anywhere in `FormatNXmx.py`, `dxtbx/nexus/__init__.py` or `nxmx/__init__.py` **[read]** | int32 on the data-file dataset (`writer/HDF5DataFile.cpp:80-82`) and on the VDS (`writer/HDF5NXmx.cpp:123-124`, `:255-256`) | OK | The natural fix for D1 is ignored by dxtbx; only member ordering matters | +| D5 | VDS with missing sources | Delegated to HDF5; no existence check | `NXmxVDS` — `writer/HDF5NXmx.cpp:113-194` | **risk (severe)** | Reads all-zero images, import succeeds, no warning **[measured, §2.7]** | +| D6 | External link to a missing file | Delegated to h5py | `NXmxLegacy` — `writer/HDF5Objects.cpp:675-684` | risk | Import succeeds; `KeyError` on first pixel access, error text does not name the file **[measured]** | +| D7 | Relative vs absolute link paths | h5py resolves relative to the containing file's directory | Basenames only — directory stripped at `writer/HDF5Objects.cpp:678` and `:334-339` | OK | Works from any cwd **[measured]**; the dataset just cannot be split across directories | +| D8 | Compression filter | `HDF5_PLUGIN_PATH` prepended at `dxtbx/__init__.py:23-28`; `import hdf5plugin` at `dxtbx/format/nexus.py:11` | bitshuffle id 32008 only, `cd_values {0,4,elem_size,block,2 or 3}` — `writer/HDF5Objects.cpp:963-991`, `compression/bitshuffle/bshuf_h5filter.h:43,46,47` | OK | LZ4 decodes with zero setup **[measured]**. ZSTD not exercised — see the caveat below | +| D9 | Compressed `pixel_mask` | Read through the same filter chain | bitshuffle-LZ4, except `NO_COMPRESSION` under `NXmxLegacy` — `writer/HDF5NXmx.cpp:414-421` | OK | Fine for DIALS; the uncompressed legacy copy is what makes that master 17.9 MB | +| D10 | Data-file `/entry` alone | `understand()` needs `definition == "NXmx"` | Data files write `NXentry` but no `definition` — `writer/HDF5DataFile.cpp:134` | OK | A data file resolves to the abstract `FormatHDF5` and lands in `unhandled`, logged only at DEBUG (`dxtbx/model/experiment_list.py:701-707`). `dials.import prefix_*.h5` still works **[measured]** | + +*D8 caveat.* Only bitshuffle+**LZ4** was exercised; `jfjoch_hdf5_test` gives no way to select +an algorithm. `BSHUF_ZSTD`, `BSHUF_ZSTD_RLE` and `BSHUF_ZSTD_RLE_HUFF` all write +`cd_values[4] = 3` (`writer/HDF5Objects.cpp:973-981`) and the RLE/Huffman variants emit +standards-compliant ZSTD frames (`compression/JFJochDecompress.h:77-88` decodes all three +with plain `ZSTD_decompress`) **[read]**, so they should decode with any bitshuffle build +that has ZSTD support — but **I did not verify that hdf5plugin's bitshuffle is built with +ZSTD**, and that is the single unverified item in this section. It is cheap to settle: run +`tools/jfjoch_recompress` (`tools/jfjoch_recompress.cpp:173-182`, converts a data file's +`/entry/data/data` from bitshuffle/LZ4 to bitshuffle/zstd) over a generated data file and +re-run `dials.find_spots`. + +### Format-class selection + +| # | Key / feature | What dxtbx does | What Jungfraujoch writes | Verdict | Consequence | +| --- | --- | --- | --- | --- | --- | +| F1 | `definition` | `== "NXmx"` selects `FormatNXmx` — `FormatNXmx.py:76-84` | `"NXmx"` — `writer/HDF5NXmx.cpp:45` | OK | Correct class chosen **[measured]** | +| F2 | `detectorSpecific/eiger_fw_version` | Sole predicate of `FormatNXmxEigerFilewriter` — `FormatNXmxEigerFilewriter.py:48` | We write a `detectorSpecific` group (`writer/HDF5NXmx.cpp:384-412`) but not that key | **risk** | One key away from a hostile takeover. That class forces `underload_value = 0` (`:74-75`), and its `_get_nxmx` raises `ValueError("Unknown Eiger module size")` for any module not in its table (`:84-90`) — JUNGFRAU 4M `(2164, 2068)` is not in it, so a match would be a hard load failure. Its `get_raw_data` also assumes `data_\d{6}` members and would break on a VDS master (`:136-146`) | +| F3 | `/entry/instrument` must exist, entry must be named `entry` | `FormatNXmxDLS.understand` → `get_instrument_name` hard-codes `handle["/entry/instrument"]` unguarded — `FormatNXmxDLS.py:83` → `FormatNexus.py:135`; registry does not catch — `Registry.py:120` | Always writes `/entry` and `/entry/instrument` — `writer/HDF5NXmx.cpp:45`, `:562` | OK (constraint) | Renaming the entry or dropping `NXinstrument` would abort the whole format search with a bare `KeyError` | +| F4 | `/entry/instrument/name` value | `i03`/`i04`/`i24`/`vmxi` exactly, or prefix `DLS`/`DIAMOND BEAMLINE` → `FormatNXmxDLS`; `I19-2` substring; `ebic` substring — `FormatNXmxDLS.py:84-89`, `FormatNXmxDLSI19_2.py:28`, `FormatNXmxED.py:81` | Beamline string, empty in the test tool; `X06DA` in repo fixtures — `writer/HDF5NXmx.cpp:563` | OK (constraint) | PSI names are safe. Note `@short_name` on `/entry/instrument` or on the `name` dataset takes precedence over the dataset value (`FormatNexus.py:135-141`) — we write neither | +| F5 | `/entry/source/probe` | `== "electron"` → `FormatNXmxED` — `FormatNXmxED.py:22` | Not written — `writer/HDF5NXmx.cpp:553-560` | OK | X-ray assumed by default, which is right for us | +| F6 | `FormatHDF5EigerNearlyNexus` | Requires `definition` to be **absent** — `FormatHDF5EigerNearlyNexus.py:36` | We always write it | OK | Structurally unreachable regardless of `description` | + +### Geometry + +| # | Key / feature | What dxtbx does | What Jungfraujoch writes | Verdict | Consequence | +| --- | --- | --- | --- | --- | --- | +| G1 | Detector `depends_on` chain | Walks `module.fast_pixel_direction.depends_on` (`dxtbx/nexus/__init__.py:360-394`); `nxmx.get_dependency_chain` follows to `"."` (`nxmx:1191-1204`, `:484-489`) | `fast/slow_pixel_direction` → `rot3` → `rot2` → `rot1` → `translation` → `"."` — `writer/HDF5NXmx.cpp:538`, `:543`, `:642`, `:635`, `:628`, `:613` | OK | Chain terminates correctly; origin reproduces to 6 s.f. **[measured]** | +| G2 | `NXdetector/depends_on` | **Never used.** dxtbx starts from the module, not the detector — `dxtbx/nexus/__init__.py:360` | Written as `.../transformations/rot3` — `writer/HDF5NXmx.cpp:339` | OK | Ignored; correct anyway | +| G3 | Sample `depends_on` | `nxmx:553-559` resolves the path against the `/entry/sample` group | `/entry/sample/transformations/omega`, while the group is created as `goniometer` and hard-linked as `transformations` — `writer/HDF5NXmx.cpp:718-720`, `:801` | OK | The hard link makes both names real, so it resolves **[measured]**. Side effect: `find_class` returns the same `NXtransformations` group twice (`nxmx:545`), so `nxsample.transformations` has 2 entries **[measured]**. dxtbx never uses that property; another NXmx consumer might double-count | +| G4 | `rot1` / `rot2` `@vector` | Applies rotations about the written `@vector` — `nxmx:508-534`, `dxtbx/nexus/__init__.py:369-387` | `rot1` → `(1,0,0)`, `rot2` → `(0,-1,0)` — `writer/HDF5NXmx.cpp:623-628`, `:630-635` | **risk** | Internally `rot1 = Ry(+rot1)` and `rot2 = Rx(-rot2)` (`image_analysis/geom_refinement/XtalResidual.h:53`, `:61-71`, `:83-85`). Accounting for the 180°-about-z frame difference, `rot1` should carry `(0,-1,0)` and `rot2` `(1,0,0)` — i.e. **the two are swapped**. Verified in dxtbx: `rot1=10 mrad` tilts the **slow** axis to `(0,-0.99995,0.01)`; `rot2=10 mrad` tilts the **fast** axis to `(0.99995,0,0.01)` **[measured]** — the opposite of the internal meaning. Invisible in-house because our own reader reads the values by dataset name and never consults `@vector` (`reader/HDF5MetadataSource.cpp:574-578`). Harmless while both are 0 | +| G5 | Tilt pivot | `A = T·R1·R2·R3`, so `A[:3,3] = t_translation` regardless of the rotations — origin fixed, axes rotate (`dxtbx/nexus/__init__.py:391-408`) | `translation` is outermost with `depends_on = "."` — `writer/HDF5NXmx.cpp:613` | **risk** | The module pivots about **corner pixel (0,0)**; our refinement pivots about the PONI (`XtalResidual.h:75-92` subtracts the beam centre before rotating). Measured: the origin is byte-identical at `rot1 = 0` and `rot1 = 10 mrad` **[measured]**. **[inferred]** at 10 mrad with a PONI ~85 mm from the corner the two conventions place the detector plane ~0.85 mm apart. Only matters when `poni_rot1/2 != 0` | +| G6 | `module_offset` | `nxmx:1025-1032` wraps it as an axis; `dxtbx/nexus/__init__.py:391-408` applies the chain from `fast_pixel_direction` | int32 `0` with `@vector = (0,0,0)`, no `@offset` — `writer/HDF5NXmx.cpp:547-549` | risk (cosmetic) | A zero-length translation vector is degenerate and NXmx types the field `NX_FLOAT`. dxtbx does not read it here (the module's fast/slow depend directly on `rot3`, not on `module_offset`), so it is inert today | +| G7 | `@offset` with rotation units | `nxmx:462-482`: `offset_units` falls back to `units` when absent; `nxmx:527-528` then does `offset.to("mm")` — **but only `if np.any(self.offset)`** | Goniometer axes get `@offset = (0,0,0)` with `@units = "deg"` and no `@offset_units` — `writer/HDF5NXmx.cpp:728-731` | risk (latent) | Currently safe because the offset is all zeros. A non-zero offset on a rotation axis would hit `Quantity(deg).to("mm")` → pint `DimensionalityError` | +| G8 | Goniometer with no rotation axis | `get_dxtbx_goniometer` proceeds whenever `depends_on` is set (`:41`), then matmuls an empty axis array (`:58`) | Grid scan writes translations only, with `depends_on` set — `writer/HDF5NXmx.cpp:771-800` | **broken** | `dials.import` fails with an opaque numpy `ValueError` **[measured, §2.8]** | +| G9 | Multi-axis chain (Smargon) | `len(axes.axes) > 1` → `make_multi_axis_goniometer`, `assert np.sum(axes.is_scan_axis) == 1` — `dxtbx/nexus/__init__.py:50-62` | `omega` → `chi` → `phi`, sample depends on `phi` — `writer/HDF5NXmx.cpp:702-714`, `:744` | OK **[inferred]** | Chain order (innermost first) matches DIALS's convention; `chi`/`phi` are scalars so `is_scan_axis` sums to 1. Not exercised — no Smargon fixture was available | +| G10 | `_end` | `nxmx:494-499` reads `_end` and uses it for the step: `steps = end[()] - axis[()]` — `dxtbx/nexus/__init__.py:209-210` | Written with `units` only, no transformation attrs — `writer/HDF5NXmx.cpp:733-735` | OK | Correct — `NXtransformations.__init__` only collects datasets that have a `vector` attr (`nxmx:342-345`), so `omega_end` is correctly *not* treated as an axis | +| G11 | `_increment_set` | `nxmx:501-506` reads it if present | Not written; we write `_range_average` / `_range_total` instead — `writer/HDF5NXmx.cpp:737-742` | OK | dxtbx prefers `_end`, which we do write | +| G12 | `beam_center_x/y` | Used **only** as a fallback when the derived origin is `(0,0)` — `dxtbx/nexus/__init__.py:410-419` | float32 with `units = "pixel"` — `writer/HDF5NXmx.cpp:599-600` | OK | Not on the normal path; the transformation chain wins | +| G13 | `data_size` order | `image_size = (data_size[-1], data_size[-2])` — `dxtbx/nexus/__init__.py:427` | `{image_size_y, image_size_x}` (slow, fast) — `writer/HDF5NXmx.cpp:619-621` | OK | Yields `{2068, 2164}` = (fast, slow) **[measured]** | + +### Detector values + +| # | Key / feature | What dxtbx does | What Jungfraujoch writes | Verdict | Consequence | +| --- | --- | --- | --- | --- | --- | +| V1 | `underload_value` | `float(underload_value)` if present, else `-0x7FFFFFFF` — `dxtbx/nexus/__init__.py:428-432` | **Not written.** The underload goes to the non-standard `error_value`, commented "this is not NXmx" — `writer/HDF5NXmx.cpp:367-369` | **risk** | `trusted_range = (-2147483647, 32766)` **[measured]**. Gap pixels (`error_value = -32768`, 239792 of them **[measured]**) are inside the trusted range and rely entirely on `pixel_mask`. `FormatNXmxDLS` and `FormatNXmxEigerFilewriter` both patch this to 0 for their own files (`FormatNXmxDLS.py:122-123`, `FormatNXmxEigerFilewriter.py:74-75`); the generic class does not | +| V2 | `saturation_value` | `int(...)` → trusted-range upper bound — `nxmx:924-939`, `dxtbx/nexus/__init__.py:433-437` | int64, `GetSaturationLimit()-1` = 32766 — `writer/HDF5NXmx.cpp:367` | OK | Correct; the 32767 overload marker falls outside **[measured]** | +| V3 | `sensor_material` | Looked up in `KNOWN_SENSOR_MATERIALS = {Si, Silicon, CdTe, GaAs}`; **raises `ValueError` on anything else** — `dxtbx/nexus/__init__.py:21-26`, `:441-443` | `"Si"` — `writer/HDF5NXmx.cpp:356` | OK (constraint) | A future sensor string outside that set is a hard import failure, not a warning | +| V4 | `sensor_thickness` | Mandatory: `self._handle["sensor_thickness"]` unguarded — `nxmx:907-910` | float32, `units = "m"` — `writer/HDF5NXmx.cpp:349` | OK | Feeds `ParallaxCorrectedPxMmStrategy` (`:444-447`); 0.32 mm, µ = 4.208 **[measured]** | +| V5 | `count_time` vs `frame_time` | Scan exposure comes from `frame_time`; `count_time` is never used — `dxtbx/nexus/__init__.py:227-233` | Both written — `writer/HDF5NXmx.cpp:347` | risk (minor) | DIALS records 0.0005 s (frame) rather than 0.00048 s (count) **[measured]**. Affects only dose/exposure bookkeeping | +| V6 | `pixel_mask` | `nxdetector.pixel_mask` at the standard path; masks every non-zero value — `nxmx:841-881`, `dxtbx/nexus/__init__.py:495-509` | uint32 hard-linked from `detectorSpecific` to the standard path — `writer/HDF5NXmx.cpp:421-423` | OK | 346128 pixels masked **[measured]** | +| V7 | `pixel_mask` bit semantics | dxtbx masks any non-zero, ignoring which bit | Bits 30 and 31 used for chip/module edges (106336 px **[measured]**) | risk (minor) | NXmx reserves bits 9–30 as undefined and bit 31 as "virtual pixel" (`nxmx:846-869`), and says the upper two bytes "would not be a sole reason to reject the intensity value". dxtbx's blanket rule happens to give the behaviour we want; a spec-following reader would keep those pixels | +| V8 | `bit_depth_readout` | Drives the top-two-value remap — `FormatNXmx.py:118`, `:163-177` | uint64, 16 — `writer/HDF5NXmx.cpp:366` | OK | No-op on signed 16-bit data **[measured]**. (Bit depth/signedness is a separate audit) | +| V9 | `data_scale_factor` / `data_offset` | `p.set_gain(1/data_scale_factor)` — `dxtbx/nexus/__init__.py:465-466`; `nxmx:236-279` | Not written | OK | `gain: 1`, `pedestal: 0` **[measured]** — correct for photon-counted data | +| V10 | `type`, `serial_number`, `detector_readout_time`, `threshold_energy`, `flatfield` | `nxmx:946-965` exposes `type`/`serial_number`; dxtbx reads neither | `type` absent; `serial_number`/`detector_number` only when non-empty (`writer/HDF5NXmx.cpp:360-361`) | OK | Ignored by dxtbx | + +### Entry, sample, beam + +| # | Key / feature | What dxtbx does | What Jungfraujoch writes | Verdict | Consequence | +| --- | --- | --- | --- | --- | --- | +| E1 | `NX_class` attributes | `find_classes` matches on the attribute, not the group name — `nxmx:67-91` | Fixed-length ASCII, on every NeXus group — `writer/HDF5Objects.cpp:648-651` | OK | `h5str` handles fixed-length bytes (`nxmx:49-60`); group names are free | +| E2 | `/entry/start_time` | `dateutil.parser.isoparse(h5str(...))` with **no guard** — `nxmx:173-183` | Written unconditionally from `start.arm_date`, which is **empty** in the test tool — `writer/HDF5NXmx.cpp:46` | **risk** | `nxmx.NXentry.start_time` raises `ValueError: ISO string too short` **[measured]**. `FormatNXmx` never touches it, so `dials.import` is unaffected — but `FormatNXmxDLS16MI03.understand` reads it raw (`FormatNXmxDLS16MI03.py:26-28`), and any other NXmx consumer that asks for `start_time` crashes. An empty string is worse than an absent dataset, which `nxmx:181` handles by returning `None` | +| E3 | `/entry/end_time` | Same parser — `nxmx:184-198` | Always a valid ISO-8601 UTC string — `writer/HDF5NXmx.cpp:899-905` | OK | Parses **[measured]** | +| E4 | `/entry/sample/name` | `nxmx:548-551` indexes `["name"]` unguarded | Written **only if non-empty** — `writer/HDF5NXmx.cpp:657-658` | risk (minor) | `nxsample.name` raises `KeyError` **[measured]**. dxtbx does not call it; NXmx lists it as required | +| E5 | `NXsource` location | `NXentry.__init__` finds it by `NX_class` among `/entry`'s children — `nxmx:150-155` | `/entry/source`, not `/entry/instrument/source` — `writer/HDF5NXmx.cpp:553` | OK | Found either way; dxtbx does not use it | +| E6 | `incident_wavelength` | `CachedWavelengthBeamFactory` indexes `handle["incident_wavelength"]` unguarded, converts by unit — `dxtbx/nexus/__init__.py:91-92`, `:125-134` | float32, `units = "angstrom"` — `writer/HDF5NXmx.cpp:569` | OK | 1.0 Å **[measured]**. Lower-case unit string parses fine in pint | +| E7 | `incident_wavelength_weights` / `variant` | Spectrum support — `dxtbx/nexus/__init__.py:94-119` | Not written | OK | Falls to the monochromatic branch (`:136-141`) | +| E8 | `total_flux`, `incident_beam_size` | `nxmx:1124-1143`; dxtbx does not use them for the beam model | `total_flux` only if set (`writer/HDF5NXmx.cpp:573`); no `incident_beam_size` | OK | `dials.show` reports `flux: 0` **[measured]** — dxtbx never reads flux from NXmx | +| E9 | `@default`, `NeXus_version`, `creator`, `definition@version` | Not read anywhere in the three reader files **[read]** | Not written — `writer/HDF5NXmx.cpp:43-45` | OK | Ignored | + +--- + +## 5. Data-packing notes per `FileWriterFormat` + +### `NXmxLegacy` — master + `_data_NNNNNN.h5` external links + +* Master `/entry/data` holds one external link per data file, named `data_%06d` starting at 1 + (`writer/HDF5NXmx.cpp:104-110`); each points at `/entry/data/data` in the target. +* Link file names are basenames only (`writer/HDF5Objects.cpp:675-684`); master and data must + share a directory. Resolution works from any cwd **[measured]**. +* `image_nr_low`/`image_nr_high` live on the target dataset in the data file + (`writer/HDF5DataFile.cpp:80-82`) and are visible through the link **[measured]** — but + dxtbx reads neither (row D4). +* **One data file: fully supported. More than one: silently truncated by DIALS to the first + file** (row D1, §2.5). CI does build multi-file legacy masters, but only in the three XDS + jobs, whose plugins read them correctly; the one job that runs DIALS builds a single-file + legacy master (§2.6). That split is why this has never surfaced. +* Missing data file → import succeeds, `KeyError` on first read (row D6). +* `pixel_mask` is stored **uncompressed** in this format only + (`writer/HDF5NXmx.cpp:417-418`), which is why the legacy master is 17.9 MB vs 0.24 MB for + VDS **[measured]**. + +### `NXmxVDS` — master + virtual dataset (the default) + +* A single virtual dataset `/entry/data/data`, shape `{n, h, w}`, with `image_nr_low = 1` and + `image_nr_high = n` as int32 (`writer/HDF5NXmx.cpp:119-124`). Verified: 4 source mappings + covering rows 0-24, 25-49, 50-74, 75-99, sources `vds_data_00000N.h5:/entry/data/data` + **[measured]**. +* Because there is exactly one member under `/entry/data`, `list(nxdata.values())[0]` is the + right answer and D1 does not apply. **This is the format that scales.** +* Source names are basenames (`writer/HDF5Objects.cpp:334-339`) — same directory constraint. +* **Missing sources read as zeros, silently** (row D5, §2.7). This is the format's one sharp + edge and it is severe precisely because it is the default. +* The master is written with libver bounds V110..LATEST for this format + (`writer/HDF5NXmx.cpp:39-42`), which VDS requires. +* A zero-image run creates no `/entry/data` group at all (`writer/HDF5NXmx.cpp:117`) + **[read]** — `FormatNXmx._start` would then fail at `nxmx_obj.entries[0].data[0]` + (`FormatNXmx.py:98`) with an `IndexError` **[inferred, not tested]**. + +### `NXmxIntegrated` — single self-contained file + +* `/entry/data/data` is a real chunked, compressed dataset in the master itself + (`writer/FileWriter.cpp:64-71` → `writer/HDF5DataFile.cpp:165-168`), with + `image_nr_low`/`image_nr_high` at close (`:80-82`) **[measured: a plain HardLink, + `is_virtual False`]**. +* Single member under `/entry/data`, so D1 does not apply. No external references at all, so + D5/D6/D7 do not apply either. **From a DIALS-robustness standpoint this is the safest of + the three.** +* The file additionally carries the per-image plugin groups `/entry/MX`, `/entry/detector`, + `/entry/image`, `/entry/profiling`, `/entry/reflections` at `/entry` level + **[measured]**. All are `NXcollection` and dxtbx ignores them. +* `LinkToData_ProcessingVDS` (`writer/HDF5NXmx.cpp:196-257`) is a different case — the rugnux + `_process.h5`, which builds `/entry/data/data` as a VDS onto the *original* input files. + Its source names are stripped to basenames by the same `ExtractFilename` + (`writer/HDF5Objects.cpp:336`), so a `_process.h5` written to a different directory than + its input has unresolvable mappings and would exhibit the D5 silent-zeros failure + **[read, not tested]**. + +--- + +## 6. Proposed follow-ups + +**These are proposals. None of them were implemented, and this document changed no code.** +Ordered by severity × cost. + +1. **Fix multi-file `NXmxLegacy`, or refuse to produce it.** *(row D1 — the only outright + data-loss finding.)* Three options, in the order I would consider them: + * **Preferred:** in `writer/HDF5NXmx.cpp:92-111`, additionally create a VDS named + `/entry/data/data` spanning all the data files, alongside the existing + `data_%06d` external links. dxtbx's `list(nxdata.values())[0]` picks `data` (it sorts + before `data_000001`), so DIALS gets the whole sweep, while readers that expect the + legacy links keep working. This is the smallest change that fixes DIALS without + changing the format's contract. + * Or set `@signal = "data_000001"`-style metadata — **this does not help**: dxtbx would + still index a single member (`dxtbx/nexus/__init__.py:576-581`). Recorded only so the + option is not re-proposed. + * Or upstream: teach `dxtbx/nexus/get_raw_data` the `data_\d{6}` walk that already exists + at `FormatNXmxEigerFilewriter.py:136-146`. Correct, but it fixes only future DIALS. +2. **Write `/entry/instrument/detector/underload_value`.** *(row V1.)* In + `writer/HDF5NXmx.cpp:367-369`, emit `underload_value` next to the existing `error_value` + with the same value from `DiffractionExperiment::GetUnderflow()`. One line; it gives every + NXmx reader a correct lower trusted bound instead of `-2147483647`, and makes gap pixels + rejectable without relying solely on `pixel_mask`. Keep `error_value` for back + compatibility. +3. **Do not write an empty `/entry/start_time`.** *(row E2.)* At `writer/HDF5NXmx.cpp:46`, + guard on `start.arm_date` being non-empty, exactly as `sample/name` is already guarded at + `:657-658`. An absent dataset is handled cleanly by `nxmx:181`; an empty string is a + `ValueError` in any consumer that reads it. +4. **Decide what a grid-scan master should look like to a rotation-oriented reader.** + *(row G8.)* In `writer/HDF5NXmx.cpp:771-800`, either leave `/entry/sample/depends_on` at + `"."` for a pure grid scan (dxtbx then treats it as stills, which is what a grid scan + *is*), or add a zero-valued rotation axis at the outer end of the chain. Today the file + is unreadable by DIALS. Worth confirming against a real broker-produced grid scan first. +5. **Swap the `@vector` values of `rot1` and `rot2`.** *(row G4.)* + `writer/HDF5NXmx.cpp:623-628` and `:630-635`: `rot1` should carry `(0,-1,0)` and `rot2` + `(1,0,0)` to match `Ry(+rot1)` / `Rx(-rot2)` at + `image_analysis/geom_refinement/XtalResidual.h:53`. No effect while both are zero, which + is why it has gone unnoticed — but it silently corrupts exported geometry for any dataset + with a refined tilt. **Before changing anything, confirm the sign/handedness derivation + independently**; I verified the *swap* empirically (§2.10 / row G4) but the correct signed + vectors deserve a second opinion. +6. **Consider whether the NXmx tilt pivot should be the PONI.** *(row G5.)* Reordering the + chain so `translation` is not outermost — or folding the beam-centre offset into + `module_offset` — would make the exported geometry pivot where our refinement does. This + is a design question, not a bug fix, and it interacts with item 5; both should be settled + together, and only if refined tilts are actually being exported. +7. **Give a missing VDS source a detectable signature.** *(row D5.)* HDF5 offers no + existence check, but `H5Pset_fill_value` on the virtual dataset + (`writer/HDF5NXmx.cpp:119-122`) could use the `error_value` (`-32768`) instead of the + default `0`, so an unresolved mapping reads as "bad pixel" rather than "zero counts". + Combined with item 2 that would make missing data visible to any reader instead of + silently valid. Alternatively, a writer-side or tooling-side check that the sources exist. +8. **Guard against `FormatNXmxEigerFilewriter` capture.** *(row F2.)* No code change needed + today, but if the SIMPLON/EIGER ingest path ever forwards a DECTRIS `detectorSpecific` + block into the master, `eiger_fw_version` must be filtered out at + `writer/HDF5NXmx.cpp:384-412`. Worth a comment there recording why. +9. **`module_offset` hygiene.** *(row G6.)* `writer/HDF5NXmx.cpp:547-549` writes an int32 `0` + with `@vector = (0,0,0)`; NXmx types the field `NX_FLOAT` and a zero-length translation + vector is degenerate. Inert for dxtbx; a one-line correctness tidy if the file is touched + for another reason. +10. **Close the CI gap in the `dials-test` job** (`.gitea/workflows/build_and_test.yml:307-341`). + Four separate changes, each of which stands alone (§2.6): + * **Add `-f` to the DIALS legacy step** (`:325`), so DIALS sees the same multi-file + layout the three XDS jobs already build. This is the change that would have caught + row D1. + * **Assert the image count.** The job currently only checks that `xia2.ssx` exits 0 + (`:326`, `:334`, `:342`), and `xia2.ssx` on a truncated stills set would very likely + still succeed. A `dials.import` step asserting the expected `num images` is what + actually turns row D1 red — without it, adding `-f` alone may not be enough. + * **Fix the single-file step's filename** (`:342`): it references `single.h5`, but + `-S -o single` writes `single_master.h5` **[measured]**. As written, that step does not + test `NXmxIntegrated` against the file the writer produces. + * **Add a `-R` rotation variant** of each format. Every current DIALS CI case is stills, + so the goniometer, scan and `omega_end` path is not covered at all — and it is the path + where row D1 turns from a wrong count into a hard `IndexError`. + +--- + +## 7. Things I could not determine + +* **bitshuffle+ZSTD decoding under DIALS.** Not exercised — `jfjoch_hdf5_test` offers no + algorithm switch, so every generated file used bitshuffle/LZ4. See the D8 caveat for a + cheap way to settle it with `tools/jfjoch_recompress`. +* **Smargon multi-axis goniometers** (row G9) — reasoned from code, no fixture available. +* **The grid-scan failure** (§2.8) was reproduced on a file I hand-built to match + `writer/HDF5NXmx.cpp:771-800`, not on writer output. +* **The exact correct signed `@vector` values** for `rot1`/`rot2` (item 5). The swap is + established; the corrected values are a derivation I would want checked. +* **Zero-image `NXmxVDS`** — the `IndexError` at `FormatNXmx.py:98` is inferred from reading, + not run. diff --git a/docs/review/GEOMETRY_CONVENTIONS.md b/docs/review/GEOMETRY_CONVENTIONS.md new file mode 100644 index 00000000..cc4ec30b --- /dev/null +++ b/docs/review/GEOMETRY_CONVENTIONS.md @@ -0,0 +1,956 @@ +# Detector geometry conventions: PyFAI/PONI, NXmx, imgCIF, XDS, DIALS + +Reference note, 2026-08-22. Comparative study of the four conventions Jungfraujoch has to +interoperate with. **This document describes the standards, not the jfjoch implementation** — it is +the yardstick, not the measurement. + +## How to read this + +Every statement is tagged: + +* **[QUOTED]** — verbatim from a primary source, with a URL. +* **[DERIVED]** — algebra done here from quoted definitions. +* **[VERIFIED]** — derived *and* checked numerically against a reference implementation + (pyFAI 2026.5.0, DIALS/dxtbx 3.27.0) during the preparation of this note. Residuals are given. +* **[UNVERIFIED]** — believed true, but not confirmed. Treat with suspicion. +* **[CONTRADICTION]** — the sources disagree with each other. Both sides are given. + +Numbers below were produced against pyFAI 2026.5.0 (pip) and the DIALS 3.27.0 in +`/opt/dials-v3-27-0`. + +--- + +## 0. Frames at a glance + +All five frames below are right-handed and have their origin at the sample. They differ only in +where the axes point. + +| Frame | +X | +Y | +Z | Handed | +|---|---|---|---|---| +| **NeXus / McStas** (NXmx) | left, looking downstream | up | along the beam, downstream | RH | +| **pyFAI** `(axis2, axis1, axis3)` | **right**, looking downstream | up | along the beam, downstream | LH in this order; RH as `(axis1, axis2, axis3)` | +| **imgCIF / CBF** | principal goniometer axis | completes RH set | **towards the source** | RH | +| **dxtbx / DIALS** | = imgCIF | = imgCIF | = imgCIF (beam propagates along −z) | RH | +| **XDS** (conventional MX setting) | detector fast axis = right, looking downstream | detector slow axis = down | along the beam, downstream | RH | + +Relations between them (all are *proper* rotations — no mirrors): + +``` +McStas -> imgCIF/dxtbx : diag(-1, +1, -1) 180 deg about Y (involution) +imgCIF -> XDS : diag(+1, -1, -1) 180 deg about X (involution) +McStas -> XDS : diag(-1, -1, +1) 180 deg about Z (involution) +pyFAI(1,2,3) -> McStas : [[0,-1,0],[1,0,0],[0,0,1]] = Rz(+90 deg) +``` + +The first is **[QUOTED]** from dxtbx source (`MCSTAS_TO_IMGCIF = np.diag([-1, 1, -1])`, +`dxtbx/nexus/__init__.py:33`) and corroborated by the imgCIF dictionary itself (§3). The others are +**[DERIVED]**, the last **[VERIFIED]** (§6). + +--- + +## 1. PyFAI / PONI + +Primary sources: + +* `https://pyfai.readthedocs.io/en/stable/geometry.html` + (source: `doc/source/geometry.rst` in `silx-kit/pyFAI`) +* `https://pyfai.readthedocs.io/en/stable/geometry_conversion.html` + (source: `doc/source/geometry_conversion.rst`) — **the only pyFAI page that writes the rotation + matrices down**. Authors J. Kieffer and C. Detlefs. +* `https://pyfai.readthedocs.io/en/stable/usage/tutorial/Geometry/geometry.html` +* `doc/source/img/PONI.png` — the canonical figure, reproduced in both pages above. +* `src/pyFAI/geometry/core.py` (`Geometry.rotation_matrix`, `Geometry.calc_pos_zyx`) +* `src/pyFAI/detectors/_common.py`, `src/pyFAI/detectors/orientation.py` + +### 1.1 Axes + +**[QUOTED]** `geometry_conversion.rst`, "Geometry definition of pyFAI / Coordinates": + +> `pyFAI` uses a coordinate system where the first axis (1) is vertically up (*y*), the second axis +> (2) is horizontal (*x*) towards the ring center (**starboard**), and the third axis (3) along the +> beam (*z*). Note that in this order (1, 2, 3) is a right-handed coordinate system, which makes +> *xyz* in the usual order a left-handed coordinate system! + +**[QUOTED]** same document, on the data layout: + +> We assume that looking along the beam axis into the detector, *d_H* increases towards the right +> (towards the center of the synchrotron) and *d_V* towards the top. + +**[QUOTED]** `PONI.png` (figure text, verbatim): + +> **From the sample's point of view, looking towards the detector:** +> rot₁: moves detector → to the right +> rot₂: moves detector ↓ downwards +> rot₃: moves detector ↻ clockwise +> +> Detector's origin: lower left, looking from the sample + +So, in McStas components: + +``` +axis1 (slow / d_V / poni1) = ( 0, +1, 0) up +axis2 (fast / d_H / poni2) = (-1, 0, 0) "starboard" = right when looking downstream = -X_McStas +axis3 (dist) = ( 0, 0, +1) downstream +``` + +**[DERIVED]** `axis1 x axis2 = ŷ × (−x̂) = ẑ = axis3`, so `(1,2,3)` is right-handed — consistent +with the quote. The change-of-basis from pyFAI components `(v1,v2,v3)` to McStas `(x,y,z)` is + +``` + [ 0 -1 0 ] +B = [ +1 0 0 ] = Rz(+90 deg), det B = +1 + [ 0 0 +1 ] +``` + +### 1.2 [CONTRADICTION] — the direction of axis 2 + +`geometry.rst` says the opposite: + +> **[QUOTED]** Axis 3 is built in such a way to be orthogonal to the plane (1,2). Due to constraints +> on the origin and orientation of the azimuthal angle, chi, **(1, 2, 3) is indirect orientation.** + +If `(1,2,3)` were indirect, `(x,y,z)` would be direct and pyFAI's axis 2 would equal McStas +X +(*left* looking downstream) — the mirror image of §1.1. This is not a cosmetic difference: **it +flips the sign of rot1 and rot3** in any conversion, and flips the sign of chi. + +Evidence for **axis2 = starboard = −X_McStas** (the reading adopted here): + +1. `geometry_conversion.rst` states it explicitly, twice, and is the only page that also gives the + matrices — so its statement is load-bearing rather than decorative. +2. `PONI.png` states it three independent ways (rot1 → right, rot2 → down, rot3 → clockwise, all + "from the sample's point of view"). **[VERIFIED]** numerically: `rot1 = +0.1` moves the PONI + point to `axis2 = +19.967 mm`, i.e. towards +axis2; the figure calls that "to the right". +3. The tutorial's closing note: **[QUOTED]** "While the system (x₁, x₂, x₃) is direct, the rotation + number 1 and 2 are indirect and rot3 is direct again. This is technical debt." +4. `src/pyFAI/geometry/imaged11.py` sets `poni._rot1 = -id11.tilt_z`, `poni._rot2 = id11.tilt_y`, + `poni._rot3 = id11.tilt_x`. ImageD11's frame is right-handed with *x* along the beam and *z* + vertically up, so `tilt_z` is a right-handed rotation about the vertical. `rot1 = −tilt_z` + therefore means pyFAI's rot1 is a rotation by **−rot1** about McStas +Y — which is the §1.1 + reading, not the `geometry.rst` one. + +Evidence the other way — one item: + +5. `src/pyFAI/geometry/crystfel.py` docstring: **[QUOTED]** "Luckily (xyz) matches pyFAI's + definition...", and the converter copies CrystFEL `corner_x` straight into pyFAI axis 2 with no + sign change. CrystFEL's frame is **[QUOTED]** (`doc/man/crystfel_geometry.5.md`): "+z is the beam + direction, and points along the beam (i.e. away from the source) … +y points towards the zenith + (ceiling) … +x completes the right-handed coordinate system", i.e. CrystFEL +x = McStas +X = + *left* looking downstream, corroborated by "The CrystFEL GUI shows +x horizontally (left to + right) … the GUI shows images from the 'into the beam' perspective." + +**[UNVERIFIED] Conclusion**: `geometry.rst`'s "(1,2,3) is indirect" sentence is stale (the file is +dated 01/12/2016) and pyFAI's own CrystFEL converter appears to contain a corresponding left–right +sign error. Four sources against one; the four include the only one that writes the maths down. But +this is an *inference*, not a test — see §10 for the experiment that would settle it on real data. + +Note also that `geometry.rst`'s gloss "pointing to the center of the storage ring" is +**beamline-dependent** and must never be used as the definition. "Starboard, looking downstream" is +the operative definition. + +### 1.3 The six parameters + +**[QUOTED]** `geometry.rst`: + +> In pyFAI, the beam center is not directly used as it is ill-defined with highly tilted detectors. +> Like SPD, we use the orthogonal projection of origin on the detector surface called PONI (for +> Point Of Normal Incidence). … *Poni1* and *Poni2* are distances in meter (along the *y* and *x* +> axis of the detector) … *rot1*, *rot2* and *rot3* are rotation along axis 1, 2 and 3, always +> expressed in radians. Rotations applied in the same order: *rot1* then *rot2* and finally *rot3*. + +`dist` (metres) is the sample → PONI distance along axis 3. All lengths are **metres**, all angles +**radians**. + +### 1.4 The rotation matrices, verbatim + +**[QUOTED]** `geometry_conversion.rst`, "Rotations". These are the standard **right-handed** +matrices in the `(1,2,3)` basis: + +``` + [ 1 0 0 ] [ cos(t2) 0 sin(t2) ] +R1(t1) = [ 0 cos(t1) -sin(t1) ] R2(t2)= [ 0 1 0 ] + [ 0 sin(t1) cos(t1) ] [ -sin(t2) 0 cos(t2) ] + + [ cos(t3) -sin(t3) 0 ] +R3(t3) = [ sin(t3) cos(t3) 0 ] + [ 0 0 1 ] +``` + +> The rotations 1 and 2 in `pyFAI` are left handed, i.e. the sign of θ₁ and θ₂ is inverted. +> +> The combined `pyFAI` rotation matrix is then +> +> R_pyFAI(θ₁, θ₂, θ₃) = R₃(θ₃) · R₂(−θ₂) · R₁(−θ₁) + +and the full mapping: + +> [t₁ t₂ t₃]ᵀ = R_pyFAI · ( D_pyFAI · [d_H d_V]ᵀ + [−poni₁ −poni₂ L]ᵀ ) + +with `D_pyFAI · [d_H, d_V]ᵀ = [pxsize_V·d_V, pxsize_H·d_H, 0]ᵀ` — i.e. **p1 is the slow/row +coordinate and p2 the fast/column coordinate**. + +**[QUOTED]** the same product as it appears in `pyFAI/geometry/core.py` (`Geometry.rotation_matrix`), +which is the authority for what the code actually does: + +```python +# Rotation about axis 1: Note this rotation is left-handed +rot1 = numpy.array([[1.0, 0.0, 0.0], [0.0, cos_rot1, sin_rot1], [0.0, -sin_rot1, cos_rot1]]) +# Rotation about axis 2. Note this rotation is left-handed +rot2 = numpy.array([[cos_rot2, 0.0, -sin_rot2], [0.0, 1.0, 0.0], [sin_rot2, 0.0, cos_rot2]]) +# Rotation about axis 3: Note this rotation is right-handed +rot3 = numpy.array([[cos_rot3, -sin_rot3, 0.0], [sin_rot3, cos_rot3, 0.0], [0.0, 0.0, 1.0]]) +rotation_matrix = numpy.dot(numpy.dot(rot3, rot2), rot1) # 3x3 matrix +``` + +Note `numpy.dot(numpy.dot(rot3, rot2), rot1)` = `R3·R2·R1` acting on a column vector, so **rot1 is +applied to the vector first** — matching the prose "rot1 then rot2 and finally rot3". + +**[VERIFIED]** In the McStas basis, `B · R_pyFAI · Bᵀ` is exactly + +``` +R = Rz(+rot3) · Rx(+rot2) · Ry(-rot1) (all right-handed, McStas axes) +``` + +Checked to `max|diff| = 0.00e+00` for `rot = (0.1,0,0)`, `(0,0.2,0)`, `(0,0,0.3)` and +`(0.13,−0.21,0.34)`. **Only rot1 carries a sign inversion.** + +### 1.5 Pixel positions and the half-pixel + +**[QUOTED]** `Detector.calc_cartesian_positions` docstring: "Calculate the position of each pixel +center in cartesian coordinate and in meter … **The half pixel offset is taken into account here!!!**" +The code computes `p = (index + 0.5) * pitch`. **[DERIVED]** Therefore *poni1/poni2 are measured +from the outer edge of the first row/column of pixels* — the physical detector edge, not the centre +of pixel (0,0). This agrees with dxtbx (§5) and needs no half-pixel fudge between the two. + +### 1.6 `orientation` — where poni1/poni2 count from + +`.poni` files of `poni_version: 2.1` carry an `orientation` inside `Detector_config`. +**[QUOTED]** `src/pyFAI/detectors/orientation.py`: + +> Names come from the position of the origin when looking at the sample from behind the camera. +> When looking from the sample to the detector, the right & left are swapped. +> +> `Orientation(1)` TopLeft — Camera default. Origin at the top left of the image when looking at the +> sample. +> `Orientation(2)` TopRight — Origin at the top left of the image when looking **from** the sample. +> `Orientation(3)` BottomRight — **Native orientation of pyFAI.** Origin at the bottom left when +> looking **from** the sample. +> `Orientation(4)` BottomLeft — Origin at the bottom left when looking at the sample. + +**Orientation 2 is the MX layout** (pixel (0,0) top-left as seen from the sample). +Orientation 3 (or an absent orientation, or 0) is pyFAI's native bottom-left layout. + +**[VERIFIED]** Implementation: `Detector._calc_pixel_index_from_orientation` reverses the row index +for orientations 1 and 2, and the column index for 1 and 4; `Geometry.calc_pos_zyx` then negates +`t1` for orientations 1, 2 and `t2` for orientations 1, 4 — **after** the rotation. Numerically: + +``` +10x10 px of 100 um, poni1 = 0.2 mm, poni2 = 0.3 mm; pixel of minimum 2theta: + orientation 3 (BottomRight) -> (row 2, col 3) poni counted from row 0 / col 0 + orientation 2 (TopRight) -> (row 7, col 3) poni counted from last row / col 0 + orientation 4 (BottomLeft) -> (row 2, col 6) poni counted from row 0 / last col + orientation 1 (TopLeft) -> (row 7, col 6) both from the far corners +``` + +**[DERIVED]** So `poni1`/`poni2` are *always* measured from the corner that is the origin of the +**native** (orientation-3) layout, whatever the stored array's layout is. With rot ≠ 0 that is *not* +equivalent to simply substituting `H·pitch − poni1`, because the sign flip is applied after the +rotation: the flip is a genuine **improper** transformation in the lab frame +(`det(K) = −1` for orientations 2 and 4, **[VERIFIED]**). This is 2θ-preserving and chi-reversing, +which is all pyFAI needs, but it means **the geometry pyFAI computes for an orientation-2 `.poni` is +the mirror image of the physical one**. §6.2 un-mirrors it. + +--- + +## 2. NXmx / NeXus + +Primary sources: + +* `https://manual.nexusformat.org/design.html#the-nexus-coordinate-system` +* `https://manual.nexusformat.org/classes/base_classes/NXtransformations.html` +* `https://manual.nexusformat.org/classes/base_classes/NXdetector_module.html` +* `https://manual.nexusformat.org/classes/applications/NXmx.html` + +### 2.1 The frame + +**[QUOTED]** `manual/source/design.rst`: + +> The instrument is given a global, absolute coordinate system where the *z* axis points in the +> direction of the incident beam, the *x* axis is perpendicular to the beam in the horizontal plane +> pointing left as seen from the source, and the *y* axis points upwards. … The origin of this +> coordinate system is the sample position or, if this is ambiguous, the center of the sample holder +> with all angles and translations set to zero. + +> **Note**: The NeXus definition of *+z* is opposite to that in the IUCr International Tables for +> Crystallography, volume G. + +**[DERIVED]** Right-handedness plus "y up, z downstream" *forces* `x = y × z`, which is left in the +downstream view. So "pointing left as seen from the source" is a description, not an extra +constraint — the maths already fixes it, and any reading of the prose that contradicts +right-handedness is wrong. + +**Important gap.** `NXmx.nxdl.xml` **never mentions McStas or any coordinate system** +(`grep -i "mcstas|coordinate system|right-handed" NXmx.nxdl.xml` → no hits). NXmx inherits the frame +silently from the NeXus base standard. A reader who has only read the NXmx page has not been told +what frame the vectors are in. + +### 2.2 Transformation semantics — the composition order + +**[QUOTED]** `NXtransformations.nxdl.xml`, top-level ``: + +> For a chain of three transformations, where **T₁ depends on T₂ and that in turn depends on T₃**, +> the final transformation T_f is +> +> **T_f = T₃ T₂ T₁** +> +> In explicit terms, the transformations are a subset of affine transformations expressed as 4x4 +> matrices that act on homogeneous coordinates, w=(x,y,z,1)ᵀ. +> +> For rotation and translation, +> +> T_r = [[R, o], [0₃, 1]] T_t = [[I₃, t + o], [0₃, 1]] +> +> where R is the usual 3x3 rotation matrix, *o* is an offset vector, 0₃ is a row of 3 zeros, I₃ is +> the 3x3 identity matrix and *t* is the translation vector. +> +> *o* is given by the `offset` attribute, *t* is given by the `vector` attribute multiplied by the +> field value, and R is defined as a rotation about an axis in the direction of `vector`, of angle of +> the field value. + +and: + +> `NXtransformations` follows the **active** transformation convention. This means that the +> transformation describes how an object is moved or rotated within the coordinate system. + +**[DERIVED] Reading of `T_f = T₃ T₂ T₁`.** T₁ is the *head* — the transformation you point at first, +nearest the component. T₃ is the *tail*, the one whose `depends_on` is `"."`. In the matrix product +acting on a column vector, the rightmost matrix acts first, so **the transformation nearest the +detector is applied first and the one nearest `"."` last**. Physically: the innermost stage of a +mechanical stack moves within the frame of the stage it is bolted to. + +**[QUOTED]** `design.rst` on `depends_on`: + +> The **order** is encoded through this attribute. The value is the name of the transformation upon +> which the current transformation depends on. As each transformation represents possible motion by a +> physical device, this dependency expresses the attachment order; thus, the current device is +> attached to (or mounted on) the next device referred to by the attribute. + +**[VERIFIED]** in the reference implementation (`nxmx/__init__.py`): + +```python +def get_dependency_chain(transformation): # [head, ..., tail] + transformations = [] + transform = transformation + while transform is not None: + transformations.append(transform) + transform = transform.depends_on + return DependencyChain(transformations) + +def get_cumulative_transformation(dependency_chain): + return reduce(operator.__matmul__, reversed([t.matrix for t in dependency_chain])) +``` + +`reversed([T1,T2,T3])` = `[T3,T2,T1]`, reduced left-to-right with `@` → `T3 @ T2 @ T1`. Exactly the +documented formula. + +### 2.3 Rotation sign + +**[QUOTED]** `NXtransformations.nxdl.xml`, `@vector`: + +> By default, for `rotation` axes that do not explicitly depend on a coordinate system, the direction +> should be chosen for a **right-handed rotation with increasing angle**. Note, McStas is a right +> handed coordinate system. +> +> … In a right-handed coordinate system, positive rotation about an axis is counter-clockwise when +> looking from a point on the positive axis towards its origin (from infinity towards the origin). + +`nxmx` implements this as `Rotation.from_rotvec(value * vector)` (scipy, right-handed) — +**[VERIFIED]**. + +There is no requirement that `@vector` have positive components; `(0,-1,0)` with a positive angle and +`(0,1,0)` with a negative angle are the same transformation. + +### 2.4 Module fields + +**[QUOTED]** `NXdetector_module.nxdl.xml`: + +> **data_origin** — A dimension-2 or dimension-3 field which gives the indices of the origin of the +> hyperslab of data for this module in the main area detector image in the parent NXdetector module. +> The data_origin is **0-based**. The frame number dimension (np) is omitted. … The order of indices +> (i, j or i, j, k) is **slow to fast**. +> +> **data_size** — Two or three values for the size of the module in pixels in each direction. +> Dimensionality and order of indices is the same as for data_origin. +> +> **fast_pixel_direction** — Values along the direction of fastest varying pixel direction. Each +> value in this array is **the size of a pixel** in the units specified. … The direction itself is +> given through the vector attribute. +> +> **slow_pixel_direction** — Values along the direction of slowest varying pixel direction. … +> +> **module_offset** — Offset of the module in regards to the origin of the detector in an arbitrary +> direction. + +So `fast_pixel_direction` is a **translation transformation whose value is the pixel pitch**; a +reader multiplies it by the (0-based) pixel index. The module's placement is carried either by the +shared `@offset` on the two pixel-direction fields, by a `module_offset` field, or by explicit +translation axes in the chain. + +**Gap [QUOTED-by-absence]**: NXdetector_module says nothing about whether the resulting position is +the **centre** or the **corner** of the pixel. dxtbx treats it as the corner of pixel (0,0), with +pixel centres at index + 0.5 (**[VERIFIED]**, §5). That is the de-facto convention but it is not in +the standard. + +### 2.5 beam_center_x / beam_center_y + +**[QUOTED]** `NXmx.nxdl.xml`: + +> This is the x position where the direct beam would hit the detector. This is a length and can be +> outside of the actual detector. The length can be **in physical units or pixels** as documented by +> the units attribute. **Normally, this should be derived from the axis chain**, but the direct +> specification may take precedence if it is not a derived quantity. + +and for `distance`: + +> Distance from the sample to the beam center. **Normally this value is for guidance only**, the +> proper geometry can be found following the depends_on axis chain. + +These fields are advisory. NXmx does not say which corner they are measured from, nor whether a +pixel-unit value is centre- or corner-based. dxtbx uses them only as a **fallback** when the chain +produces `origin[0] == origin[1] == 0` ("this is needed for some older dectris eiger filewriter +datasets", `dxtbx/nexus/__init__.py`). + +--- + +## 3. MX / imgCIF + +Primary source: the imgCIF dictionary `cif_img.dic` +(`https://raw.githubusercontent.com/COMCIFS/imgCIF/master/cif_img.dic`; the rendered page +`https://www.iucr.org/__data/iucr/cifdic_html/2/cif_img.dic/Caxis.html` returns HTTP 403 to +automated fetches, so the dictionary master was used — it carries the same text). + +### 3.1 The imgCIF laboratory frame + +**[QUOTED]**: + +> **THE IMGCIF STANDARD LABORATORY COORDINATE SYSTEM** +> The imgCIF standard laboratory coordinate system is a right-handed orthogonal coordinate system +> similar to that used by MOSFLM, but imgCIF puts Z along the X-ray beam, rather than putting X along +> the X-ray beam as in MOSFLM. +> +> ``` +> | Y (to complete right-handed system) +> | +> |________________X +> / principal goniometer axis +> / +> /Z (to source) +> ``` +> +> **Axis 1 (X)**: The X-axis is aligned to the mechanical axis pointing from the sample or specimen +> along the principal axis of the goniometer … +> **Axis 2 (Y)**: The Y-axis completes an orthogonal right-handed system defined by the X-axis and +> the Z-axis. +> **Axis 3 (Z)**: The Z-axis is derived from the source axis which goes **from the sample to the +> source**. + +Rotation sign **[QUOTED]**: + +> For a right-handed rotation axis, if the right hand is wrapped around the axis with the thumb +> pointed in the direction of the axis, the fingers point in the positive rotation direction, i.e. +> clockwise. +> … If the axis involved is a rotation axis, it is right-handed, i.e. as one views the object to be +> rotated **from the origin (the tail) of the unit vector**, the rotation is clockwise. + +NeXus describes the same right-hand rule from the *opposite viewpoint* ("counter-clockwise when +looking from a point on the positive axis towards its origin"). **[DERIVED]** These agree — one looks +along the axis, the other looks back down it. The apparent clockwise/counter-clockwise contradiction +is a viewpoint difference, not a convention difference. + +### 3.2 imgCIF vs NXmx — where they agree and differ + +**[QUOTED]**, from the same dictionary section, which is unusually explicit: + +> **Differences in Coordinate Frames** +> The standard coordinate frame in imgCIF/CBF aligns the X-axis to the principal goniometer axis, and +> chooses the Z-axis to point from the sample into the beam. … Let us call the NeXus coordinate axes +> X_nx, Y_nx and Z_nx … In the CBF coordinate frame, Z_nx is aligned to Beam_cbf: Z_nx = Beam_cbf. +> X_nx is defined as being horizontal at right angles to the beam, pointing to the left when seen +> from the source … X_nx = (Beam_cbf × Gravity_cbf)/||Beam_cbf × Gravity_cbf||. … Finally +> Y_nx = Beam_cbf × X_nx. +> +> **In the common case in which the beam is orthogonal to the principal goniometer axis so that +> Beam_cbf = (0, 0, −1) and the imgCIF/CBF Y-axis points upwards, the transformation inverts the X +> and Z axes.** In the other common case in which the beam is orthogonal to the principal goniometer +> axis and the imgCIF/CBF Y-axis points downwards, the transformation inverts the Y and Z axes. +> +> … Notice that X and Z have changed direction, but Y has not. + +"Inverts the X and Z axes" is exactly `diag(-1, 1, -1)` — the matrix dxtbx hard-codes. + +**Agreements**: both are right-handed, both have their origin at the sample, both use the right-hand +rule for rotations, both compose axes as dependency chains with the innermost applied first (imgCIF's +`_axis.depends_on` was the model for NeXus's — **[QUOTED]** design.rst: "The CIF scheme … is the +preferred method for expressing geometry in NeXus"). + +**Differences**: the *z* direction (beam vs anti-beam), the *x* direction, the choice of what fixes +*x* (goniometer axis vs "complete the right-handed set"), and the array index base — imgCIF's +`_array_structure_list.index` is **[QUOTED]** "the **one-based** index of the row or column", against +NXmx's explicitly 0-based `data_origin`. + +### 3.3 "Rows counted from the top down" + +**[DERIVED]** This is not stated as an axiom anywhere; it is a *consequence* of the panel vectors +that real MX software uses. In the imgCIF/dxtbx frame the standard flat MX detector is + +``` +fast = (+1, 0, 0) slow = ( 0, -1, 0) origin = (-bx, +by, -dist) +``` + +**[QUOTED]** from `dxtbx/format/FormatCBFMini.py:189-194` (the Pilatus/Eiger mini-CBF reader), which +calls `DetectorFactory.simple(..., "+x", "-y", ...)`. Converting with `diag(-1,1,-1)` to McStas: + +``` +fast_McStas = (-1, 0, 0) slow_McStas = ( 0, -1, 0) origin = (+bx, +by, +dist) +``` + +**[DERIVED]** i.e. columns increase to the *right* and rows increase *downwards* when looking +downstream from the sample. Displaying the array the normal way (col → right, row → down) therefore +reproduces the view **from the sample, looking downstream**. That, and not folklore, is what "MX +images are viewed from the crystal" means. + +Note `fast × slow = (0,0,+1)` = +beam. **[DERIVED]** pyFAI's native layout has `fast = (-1,0,0)`, +`slow = (0,+1,0)`, so `fast × slow = (0,0,-1)` = −beam. **The MX layout and pyFAI's native layout +differ by a mirror in Y, not by a rotation.** (Note this is only true under the §1.1 reading of +axis 2. Under the discarded `geometry.rst` reading they would differ by a 180° rotation about the +beam instead — see §7.) + +**[QUOTED]** the imgCIF beam-centre convention, for completeness: + +> When specifying detector axes, the axis is given to the beam centre. The location of the beam +> centre on the detector should be given in the DIFFRN_DETECTOR category in distortion-corrected +> millimetres from the (0,0) corner of the detector. + +--- + +## 4. XDS + +Primary source: `https://xds.mr.mpg.de/html_doc/coordinate_systems.html` and +`https://xds.mr.mpg.de/html_doc/xds_parameters.html`. + +**[QUOTED]**: + +> **Laboratory coordinate system** — Any convenient right-handed orthonormal system may be chosen +> with the origin at the intersection between rotation axis, direct beam and crystal. Once chosen, +> the coordinate system remains fixed throughout the diffraction experiment. +> +> `INCIDENT_BEAM_DIRECTION= 0 0 1` — The positive beam direction points along +z from the source +> towards the crystal. +> +> The x- and y- axes of the detector coordinate system are specified by the two orthonormal vectors +> ED(:,1) and ED(:,2) … These vectors — together with their cross product +> **ED(:,3) = ED(:,1) × ED(:,2)** — define a rotation matrix ED. +> +> The origin of the detector system … is specified (mm) in the laboratory coordinate system by the +> vector **ORG(:) = −ORGX·QX·ED(:,1) − ORGY·QY·ED(:,2) + F·ED(:,3)** … Note the positive sign for the +> detector distance because the detector normal ED(:,3) points away from the crystal. +> +> The pixels of a segment are enumerated by IX,IY whereby 1 ≤ x1 ≤ IX ≤ x2 and 1 ≤ y1 ≤ IY ≤ y2 … +> a subset of the index set **1 ≤ IX ≤ NX and 1 ≤ IY ≤ NY**. + +Key points: + +* **XDS does not define a lab frame.** It is whatever the user declares via + `DIRECTION_OF_DETECTOR_X-AXIS`, `DIRECTION_OF_DETECTOR_Y-AXIS`, `INCIDENT_BEAM_DIRECTION` and + `ROTATION_AXIS`. The worked SIEMENS example in the docs even has "the y-axis points vertically + **down**". Do not assume a canonical XDS frame; read the XDS.INP. +* **The conventional modern-MX setting** is `DIRECTION_OF_DETECTOR_X-AXIS = 1 0 0`, + `DIRECTION_OF_DETECTOR_Y-AXIS = 0 1 0`, `INCIDENT_BEAM_DIRECTION = 0 0 1`, which pins the lab frame + to the detector: +x = fast axis, +y = slow axis (physically *down*), +z = beam. **[DERIVED]** In + McStas terms that is `diag(-1,-1,1)` — a 180° rotation about the beam. +* **[VERIFIED]** dxtbx's exporter builds exactly this frame: `Rd = align_reference_frame(fast, + (1,0,0), slow, (0,1,0))` (`dxtbx/serialize/xds.py:188`). +* **ORGX/ORGY are 1-based, pixel-centre coordinates.** **[QUOTED]** dxtbx comment at + `dxtbx/serialize/xds.py:208`: "*Need to add 0.5 because XDS seems to do centroids in fortran + coords*", with `self.detector_origin = (x / f + 0.5, y / f + 0.5)`. So + `ORGX = beam_centre_px_corner_based + 0.5`. + +--- + +## 5. DIALS / dxtbx + +**[QUOTED]** `dxtbx/nexus/__init__.py:29-33`: + +```python +# Conversion from the McStas coordinate system as used by NeXus to the imgCIF +# coordinate system conventionally used by dxtbx: +# https://manual.nexusformat.org/design.html#design-coordinatesystem +# https://www.iucr.org/__data/iucr/cifdic_html/2/cif_img.dic/Caxis.html +MCSTAS_TO_IMGCIF = np.diag([-1, 1, -1]) +``` + +**[VERIFIED]** with `DetectorFactory.simple("PAD", 200 mm, beam_centre (100, 120) mm, "+x", "-y", +0.075 mm px, 2000x2000)`: + +``` +fast = (1, 0, 0) slow = (0, -1, 0) normal = (0, 0, -1) +origin = (-100, 120, -200) s0 = (0, 0, -1)/lambda +pixel (0,0) -> (-100.0000, 120.0000, -200) <- the CORNER of pixel (0,0) +pixel (0.5,0.5) -> ( -99.9625, 119.9625, -200) <- its centre +beam centre px = (1333.333, 1600.000) = (100/0.075, 120/0.075) +``` + +So a dxtbx panel is `origin + i_fast·pitch_f·fast + i_slow·pitch_s·slow`, with **integer pixel +coordinates at pixel corners** and centres at half-integers, and the panel `origin` at the outer +corner of pixel (0,0). The beam propagates along **−z**; `get_sample_to_source_direction()` is +`(0,0,+1)` = imgCIF +Z. This matches pyFAI's half-pixel convention exactly (§1.5). + +### 5.1 [VERIFIED] dxtbx does not rotate the module `@offset` + +Reading `dxtbx/nexus/__init__.py` (flat-detector branch, ~line 385-410): + +```python +A = nxmx.get_cumulative_transformation(dependency_chain) # chain BELOW the module +fast_axis = MCSTAS_TO_IMGCIF @ R @ module.fast_pixel_direction.vector # R = rotation part of A +origin = MCSTAS_TO_IMGCIF @ (module.fast_pixel_direction.offset + A[0, :3, 3]) +``` + +The **axes** get the chain's rotation `R`; the **offset** gets only the chain's translation. The +strict NeXus reading (§2.2) is `A · (o + j·p_f·f̂ + …)` = `R·o + t + …`. + +Confirmed with a synthetic NXmx file (`det_z` translation 200 mm ← `tilt` rotation 30° about +`(0,1,0)` ← module with `@offset = (100, 50, 0) mm`): + +``` +tilt = 0 deg : dxtbx origin = (100, 50, 200) strict R@o+t = (100, 50, 200) agree +tilt = 30 deg : dxtbx origin = (100, 50, 200) strict R@o+t = ( 86.60, 50, 150) DISAGREE + fast axis = (-0.866, 0, 0.5) <- correctly rotated +``` + +**Consequence**: if any rotation sits in the chain *below* the module and the module placement is +carried in `@offset`, DIALS puts the panel in the wrong place while getting its orientation right. +In the §7 worked example (rot ≈ 0.01–0.02 rad only) this costs **0.68 mm ≈ 9 pixels**. +**[UNVERIFIED]** whether upstream considers this a bug or an intended convention (that `@offset` is +expressed post-rotation). + +**Mitigation [VERIFIED]**: express the module placement as explicit **translation axes in the chain** +instead of as `@offset`. Translations go through `get_cumulative_transformation` and are handled +correctly, and the result then agrees with the strict NeXus reading to 1e-16. + +--- + +## 6. Conversion: PONI → McStas/NXmx + +### 6.1 Literal form (reproduces pyFAI's numbers exactly, mirror and all) + +Let `p_s` = `pixel1` (slow, m), `p_f` = `pixel2` (fast, m), array `H × W` pixels, `L` = `dist`. + +``` +s1 = -1 if orientation in {1,2} else +1 # slow-axis sign flip +s2 = -1 if orientation in {1,4} else +1 # fast-axis sign flip +a1 = H*p_s if s1 < 0 else 0 +a2 = W*p_f if s2 < 0 else 0 + +R = Rz(+rot3) · Rx(+rot2) · Ry(-rot1) # right-handed, McStas axes +N = diag(s2, s1, 1) # pyFAI's post-rotation sign flip +K = N · R # det K = s1*s2 (IMPROPER for orientations 2, 4) + +fast_vector = K · (-s2, 0, 0) +slow_vector = K · ( 0, s1, 0) +origin = K · (poni2 - a2, a1 - poni1, L) # corner of pixel (0,0) + +P(i,j) = origin + (j + 0.5)*p_f*fast_vector + (i + 0.5)*p_s*slow_vector # pixel CENTRE +``` + +**[VERIFIED]** against `Geometry.calc_pos_zyx` for all four orientations, 5 random draws each, +`H=13, W=17`, `p_s=172 µm`, `p_f=200 µm`, `dist ∈ [0.15,0.25] m`, `poni ∈ [−2,4] mm`, +`rot ∈ [−0.35,0.35] rad`: **worst residual 1.77e-10 m** (pyFAI's cython path is float32). +The McStas vector is recovered from pyFAI's `(t3,t1,t2)` return as `(x,y,z) = (−t2, t1, t3)`. + +### 6.2 Physical form (what you should actually write into an NXmx file) + +Un-mirror by the orientation label (§1.6): the label tells you where pixel (0,0) physically is, so +apply the mirror `M = diag(s2, s1, 1)` to §6.1 and simplify. `K` disappears and **R is the same in +all four cases**: + +``` +R = Rz(+rot3) · Rx(+rot2) · Ry(-rot1) +``` + +| `orientation` | `fast_pixel_direction @vector` | `slow_pixel_direction @vector` | module origin (corner of pixel 0,0) | +|---|---|---|---| +| 3 (or 0 / absent) — pyFAI native | `R·(-1, 0, 0)` | `R·(0, +1, 0)` | `R·(poni2, −poni1, L)` | +| **2 — the MX layout** | `R·(-1, 0, 0)` | `R·(0, -1, 0)` | `R·(poni2, H·p_s − poni1, L)` | +| 4 | `R·(+1, 0, 0)` | `R·(0, +1, 0)` | `R·(poni2 − W·p_f, −poni1, L)` | +| 1 | `R·(+1, 0, 0)` | `R·(0, -1, 0)` | `R·(poni2 − W·p_f, H·p_s − poni1, L)` | + +**[VERIFIED]** against pyFAI's own positions after un-mirroring, all four orientations, same random +sampling as above: **worst residual 1.77e-10 m**. + +Read it as one rule: *`poni2` is the distance from the pixel-(0,0) edge to the PONI along the fast +direction, `poni1` the distance from the pixel-(0,0) edge along the slow direction*, with the sign of +each axis set by the orientation label — and `L` along the (rotated) detector normal. + +For orientation 2 this produces `fast = R·(-1,0,0)`, `slow = R·(0,-1,0)`, which for `R = I` is +exactly the canonical MX/NXmx panel of §3.3. **pyFAI orientation 2 and the NXmx MX layout are the +same layout.** + +### 6.3 The reverse direction (NXmx → PONI) + +**[DERIVED]**, not separately verified. Given McStas `fast`, `slow`, `origin` (corner of pixel (0,0)) +for an MX-layout panel, write `n = fast × slow` (unit normal, pointing away from the sample): + +``` +dist = origin · n +PONI = dist * n # foot of the normal from the sample +poni2 = (PONI - origin) · fast # metres +poni1 = H*p_s - (PONI - origin) · slow # metres, orientation = 2 +rot2 = asin(-n_y) # from R = Rz(rot3) Rx(rot2) Ry(-rot1) +rot1 = -atan2(n_x, n_z) # careful with the branch when |n_y| -> 1 +rot3 from the residual in-plane rotation of `fast` about n +``` + +Do not implement this from the sketch above; solve `R` from the orthonormal triad +`(fast, slow, n)` = `R · (−x̂, −ŷ, ẑ)` and decompose. The decomposition is `Rz·Rx·Ry`, which is a +non-standard Euler order — most library `euler_from_matrix` helpers will not give you `rot1/rot2/rot3` +directly. + +--- + +## 7. Mirrors and the sign of rotations + +This is the part everyone gets wrong, so here it is in general form. + +**[DERIVED]** For a rotation `R(n̂,θ) = I cosθ + sinθ [n̂]× + (1−cosθ) n̂n̂ᵀ` and an orthogonal `M`: + +``` +M · R(n̂, θ) · M⁻¹ = R( M n̂ , det(M) · θ ) +``` + +because `M [n̂]× Mᵀ = det(M) [M n̂]×` (the rotation axis is a pseudovector). Two cases: + +**Mirror in Y**, `M = diag(1,-1,1)`, `det = -1`: + +| axis | image | resulting rotation | sign | +|---|---|---|---| +| x̂ | +x̂ | `R(x̂, −θ)` | **flips** | +| ŷ | −ŷ | `R(−ŷ, −θ) = R(ŷ, +θ)` | **unchanged** | +| ẑ | +ẑ | `R(ẑ, −θ)` | **flips** | + +> **A mirror in Y flips the sense of rotations about the two axes lying *in* the mirror plane (X and +> Z) and leaves rotations about the mirror normal (Y) alone.** + +**180° rotation about Z**, `M = diag(-1,-1,1)`, `det = +1`: + +| axis | resulting rotation | sign | +|---|---|---| +| x̂ | `R(−x̂, +θ) = R(x̂, −θ)` | **flips** | +| ŷ | `R(ŷ, −θ)` | **flips** | +| ẑ | `R(ẑ, +θ)` | **unchanged** | + +Applied to pyFAI's `R = Rz(rot3)·Rx(rot2)·Ry(−rot1)`: + +* Y-mirror ⇒ `rot2 → −rot2`, `rot3 → −rot3`, **`rot1` unchanged**. + This is the pyFAI-native ↔ MX-layout relation, and also exactly what pyFAI's `orientation = 2` + does internally (§1.6). **[VERIFIED]** by the §6 residuals. +* X-mirror ⇒ `rot1 → −rot1`, `rot3 → −rot3`, `rot2` unchanged. (pyFAI `orientation = 4`.) +* 180° about Z ⇒ `rot1 → −rot1`, `rot2 → −rot2`, `rot3` unchanged. (pyFAI `orientation = 1`.) +* **The `geometry.rst` reading of axis 2 (§1.2), if it were right**, would change the + pyFAI ↔ McStas relation by an extra X-mirror, i.e. it would negate `rot1` and `rot3` throughout + §6. That is the entire practical consequence of the contradiction. + +**2θ is invariant under all of these; chi is not.** That is why the ambiguity can hide for years in +an azimuthal-integration code and surface only when someone writes an NXmx file or compares an +azimuthal profile against another package. + +--- + +## 8. Worked example: a PONI triple as an NXmx chain + +Input `.poni` (written by pyFAI 2026.5.0, `poni_version: 2.1`): + +``` +poni_version: 2.1 +Detector: Detector +Detector_config: {"pixel1": 7.5e-05, "pixel2": 7.5e-05, "orientation": 2, "max_shape": [2000, 2000]} +Distance: 0.15 +Poni1: 0.0755 +Poni2: 0.0812 +Rot1: 0.01 +Rot2: -0.02 +Rot3: 0.003 +Wavelength: 1e-10 +``` + +`orientation: 2` ⇒ MX layout ⇒ row 2 of the §6.2 table. `H·p_s = W·p_f = 150 mm`, so +`H·p_s − poni1 = 150 − 75.5 = 74.5 mm`. + +### 8.1 The chain + +Chain order **from the module outwards** (`depends_on` pointing left to right), which by §2.2 is +also the order in which the transformations are applied to a vector: + +``` +fast_pixel_direction ─┐ +slow_pixel_direction ─┴→ poni_x → poni_y → det_z → rot1 → rot2 → rot3 → "." +``` + +Written out (`/entry/instrument/detector/`): + +| field | type | value | `@vector` | `@units` | `@depends_on` | +|---|---|---|---|---|---| +| `module/fast_pixel_direction` | translation | `0.075` | `(-1, 0, 0)` | mm | `transformations/poni_x` | +| `module/slow_pixel_direction` | translation | `0.075` | `(0, -1, 0)` | mm | `transformations/poni_x` | +| `transformations/poni_x` | translation | `81.2` | `(1, 0, 0)` | mm | `transformations/poni_y` | +| `transformations/poni_y` | translation | `74.5` | `(0, 1, 0)` | mm | `transformations/det_z` | +| `transformations/det_z` | translation | `150.0` | `(0, 0, 1)` | mm | `transformations/rot1` | +| `transformations/rot1` | rotation | `0.5729578` | `(0, -1, 0)` | deg | `transformations/rot2` | +| `transformations/rot2` | rotation | `-1.1459156` | `(1, 0, 0)` | deg | `transformations/rot3` | +| `transformations/rot3` | rotation | `0.1718873` | `(0, 0, 1)` | deg | `.` | + +with `/entry/instrument/detector/depends_on = ".../transformations/rot1"`, +`module/data_origin = [0, 0]`, `module/data_size = [2000, 2000]`. + +### 8.2 Derivation of each row + +* **Chain order.** pyFAI applies rot1 first (`R3·R2·R1` acting on a column vector). NeXus applies the + *head* of the chain first (`T_f = T₃T₂T₁`, T₁ = head). So rot1 must be the head-most of the three + rotations, i.e. **nearest the detector**, and rot3 nearest `"."`. Listing them "rot1, rot2, rot3" + from the module outwards therefore reproduces pyFAI's stated order — but note the matrix product is + written in the opposite order. +* **`det_z` sits *between* the PONI offsets and the rotations.** pyFAI rotates the whole vector + `(p1−poni1, p2−poni2, L)` including the `L` component, so the distance translation must be applied + *before* the rotations, i.e. closer to the module. +* **`rot1` sign.** §1.4 gives `Ry(−rot1)`. NeXus rotations are right-handed about `@vector`, so this + is either `@vector = (0,1,0)` with value `−0.5729578 deg` **or** `@vector = (0,-1,0)` with value + `+0.5729578 deg`. The two are identical to NeXus. **Recommendation: put the sign in `@vector`** — + the stored angle then matches `Rot1:` in the `.poni` character for character, which is what you + want when someone diffs the two by eye. This is a style choice; the standard is silent. +* **`rot2`, `rot3` signs.** §1.4 gives `Rx(+rot2)` and `Rz(+rot3)`: plain `(1,0,0)` and `(0,0,1)` + with the `.poni` value unchanged. +* **`poni_x` = `+poni2`, `poni_y` = `+(H·p_s − poni1)`** from §6.2, expressed as two translation + axes rather than as a module `@offset` — see §5.1 for why. `poni_x` is `+81.2` because McStas +X is + *left* looking downstream while `fast` points right, so the pixel-(0,0) corner is `poni2` to the + left of the beam. + +### 8.3 Expected result + +``` +R = Rz(0.003) · Rx(-0.02) · Ry(-0.01) = + [ 0.999944901 -0.002999396 -0.010059781 ] + [ 0.003199828 0.999795508 0.019967577 ] + [ 0.009997833 -0.019998667 0.999750017 ] + +fast (McStas) = (-0.999944901, -0.003199828, -0.009997833) +slow (McStas) = ( 0.002999396, -0.999795508, 0.019998667) +origin (McStas) = ( 79.463104, 77.739728, 149.284426 ) mm +PONI point = ( -1.508967, 2.995137, 149.962503 ) mm +beam centre = (1062.666, 1033.341) px, corner-based 0-based (fast, slow) +``` + +**[VERIFIED]** by writing this file and reading it with DIALS 3.27.0 +(`dxtbx.nexus.get_dxtbx_detector`, converting back with `diag(-1,1,-1)`): + +``` +[chain encoding] fast err 1.1e-16 slow err 3.5e-18 origin err 0.000 mm +[offset encoding] fast err 1.1e-16 slow err 3.5e-18 origin err 0.678 mm <-- §5.1 trap +``` + +The "offset encoding" row is the same geometry with `poni_x`/`poni_y` replaced by +`@offset = (81.2, 74.5, 0)` on the two pixel-direction fields. It is *strictly correct NeXus* and +DIALS still misreads it by 9 pixels. + +--- + +## 9. Traps + +1. **Chain order.** `T_f = T₃T₂T₁` with **T₁ = the head** (nearest the component). The + transformation nearest the detector is applied to the vector **first**; the one whose + `depends_on` is `"."` is applied **last**. Writing the chain "rot1 → rot2 → rot3 → ." from the + module outward gives the product `R3·R2·R1`, not `R1·R2·R3`. +2. **pyFAI's `rot1` is the only rotation with an inverted sign** in McStas + (`R = Rz(rot3)·Rx(rot2)·Ry(−rot1)`). It is tempting to negate all three or none. +3. **A mirror in Y flips rotations about X and Z, *not* about Y.** The axis perpendicular to the + mirror plane is the one that survives. (§7) +4. **pyFAI's `orientation` is not decoration.** With rotations present it is a genuine improper + transformation in pyFAI's lab frame; you cannot emulate it by substituting `H·pitch − poni1` and + leaving the rotations alone. A `.poni` with `poni_version < 2.1` has no orientation field at all — + you must know the layout out of band. +5. **DIALS does not rotate the module `@offset`** by rotations in the chain below it (§5.1, + **[VERIFIED]**, 0.68 mm error on a 1° tilt). Encode module placement as translation *axes*, not + as `@offset`, whenever the chain contains a rotation. +6. **Pixel centre vs corner.** pyFAI adds `+0.5` explicitly; dxtbx puts integer pixel coordinates at + corners. They agree. **XDS does not**: `ORGX/ORGY` are 1-based pixel-centre coordinates, so + `ORGX = corner_based_beam_x + 0.5`. NXmx does not specify centre or corner at all. +7. **Index base.** NXmx `data_origin` is 0-based; imgCIF `_array_structure_list.index` is 1-based; + XDS `IX/IY` are 1-based. +8. **`beam_center_x/y` and `distance` in NXmx are advisory** — "normally this should be derived from + the axis chain". Do not let a reader prefer them; DIALS only falls back to them when the chain + yields a zero origin. +9. **NXmx never states its own coordinate frame.** It is inherited silently from `design.rst`. +10. **XDS has no fixed lab frame.** Read the XDS.INP; do not assume `0 0 1` for the beam. +11. **imgCIF and NeXus describe the same right-hand rule from opposite viewpoints** ("clockwise from + the tail" vs "counter-clockwise from infinity towards the origin"). They agree; the wording does + not. +12. **pyFAI's "towards the center of the storage ring" is beamline-dependent** and is not a + definition. Use "starboard, looking downstream". +13. **2θ hides everything.** Every mirror and 180° rotation discussed here preserves 2θ exactly. Only + chi, the sign of the rotations, and the absolute placement of pixel (0,0) expose them. An + azimuthal-integration test will not catch a mirror; a DIALS index/refine of the same file will. + +--- + +## 10. Open questions / not verified + +* **§1.2 is the important one.** pyFAI's own documentation contradicts itself on the direction of + axis 2. Four sources (the conversion note, the canonical figure, the tutorial, the ImageD11 + converter) say "starboard / right, looking downstream"; `geometry.rst` says the opposite, and + `geometry/crystfel.py` implicitly agrees with `geometry.rst`. This note adopts "starboard". If it + is wrong, **negate `rot1` and `rot3` throughout §6 and §8**, and negate the x-component of every + vector. + *Decisive experiment*: calibrate one real image with `pyFAI-calib2` and index/refine the same image + with DIALS, then compare the two panel `fast`/`slow`/`origin` triples in a common frame. A + left–right disagreement in the x-component with the y- and z-components matching is the signature + of the wrong choice. This is worth doing before relying on §6 for anything but 2θ. +* **`geometry/crystfel.py`**: an apparent left–right sign error, inferred from two documents, not + tested end-to-end against CrystFEL. Not reported upstream. +* **dxtbx `@offset` handling (§5.1)**: the behaviour is verified; whether upstream considers it a bug + or an intentional convention (that `@offset` is expressed in the post-rotation frame) is not. + Nothing in `NXdetector_module` supports the latter reading, and dxtbx's own `nxmx_writer.py` writes + `@offset` in the *pre*-rotation frame, which suggests the round-trip is broken for a rotated + parent — but that round-trip was not tested here. +* **§6.3 (NXmx → PONI)** is derived on paper only; it was not implemented or checked. +* `https://www.iucr.org/__data/iucr/cifdic_html/2/cif_img.dic/Caxis.html` returns **HTTP 403** to + automated fetches. The COMCIFS dictionary master was used instead; the text is believed identical + but was not diffed against the rendered page. +* The CrystFEL man page was read from the GitHub mirror `taw10/crystfel`, not from the canonical + `gitlab.desy.de` (which returns an access-denied page to automated fetches). +* pyFAI's `Orientation` values 5–8 (Transpose, Rotate270, Transverse, Rotate90) are declared + `available = False` and are not handled here. +* Multi-module / hierarchical NXmx detectors (`equipment_component` grouping, several + `NXdetector_module` groups) are out of scope; dxtbx takes a different code path for them + (`Detector.hierarchy()`), and in that path it *does* apply the full affine to the origin. + +--- + +## Sources + +| What | URL | +|---|---| +| NeXus coordinate system & transformations | https://manual.nexusformat.org/design.html#the-nexus-coordinate-system | +| NXtransformations | https://manual.nexusformat.org/classes/base_classes/NXtransformations.html | +| NXdetector_module | https://manual.nexusformat.org/classes/base_classes/NXdetector_module.html | +| NXmx | https://manual.nexusformat.org/classes/applications/NXmx.html | +| NeXus definitions source | https://github.com/nexusformat/definitions | +| pyFAI geometry | https://pyfai.readthedocs.io/en/stable/geometry.html | +| pyFAI geometry conversion (the matrices) | https://pyfai.readthedocs.io/en/stable/geometry_conversion.html | +| pyFAI geometry tutorial | https://pyfai.readthedocs.io/en/stable/usage/tutorial/Geometry/geometry.html | +| pyFAI source | https://github.com/silx-kit/pyFAI | +| imgCIF dictionary (lab frame, NeXus comparison) | https://github.com/COMCIFS/imgCIF/blob/master/cif_img.dic | +| imgCIF axis category (403 to bots) | https://www.iucr.org/__data/iucr/cifdic_html/2/cif_img.dic/Caxis.html | +| XDS coordinate systems | https://xds.mr.mpg.de/html_doc/coordinate_systems.html | +| XDS input parameters | https://xds.mr.mpg.de/html_doc/xds_parameters.html | +| CrystFEL geometry man page | https://github.com/taw10/crystfel/blob/master/doc/man/crystfel_geometry.5.md | +| dxtbx NXmx reader | `/opt/dials-v3-27-0/modules/dxtbx/src/dxtbx/nexus/__init__.py` | +| dxtbx NXmx writer | `/opt/dials-v3-27-0/modules/dxtbx/src/dxtbx/format/nxmx_writer.py` | +| dxtbx XDS exporter | `/opt/dials-v3-27-0/modules/dxtbx/src/dxtbx/serialize/xds.py` | +| `nxmx` chain composition | `/opt/dials-v3-27-0/conda_base/lib/python3.13/site-packages/nxmx/__init__.py` | diff --git a/docs/review/NXMX_GEOMETRY_VERIFICATION.md b/docs/review/NXMX_GEOMETRY_VERIFICATION.md new file mode 100644 index 00000000..fc579d94 --- /dev/null +++ b/docs/review/NXMX_GEOMETRY_VERIFICATION.md @@ -0,0 +1,412 @@ +# NXmx detector-tilt encoding: empirical verification against DIALS/dxtbx + +**Question.** What are the correct `@vector` attributes for `rot1`, `rot2`, `rot3` in +`/entry/instrument/detector/transformations`, and what is the correct `depends_on` chain +order, such that a DIALS/dxtbx reading of a Jungfraujoch master file reproduces the +scattering vectors Jungfraujoch itself computes? + +**Answer, in one line.** `rot1 → (0,-1,0)`, `rot2 → (1,0,0)`, `rot3 → (0,0,-1)`, chain +`module → translation → rot1 → rot2 → rot3 → "."`. Verified in reciprocal space to +**≤ 7.3e-16 Å⁻¹** across 10 tilt settings and 11 pixels. The shipped encoding is wrong in +**three independent ways** (both `rot1` and `rot2` axes, the rotation product order, and the +tilt pivot); the previously proposed patch fixes two of them and leaves two. + +**Status.** This is a measurement report. **Nothing in the repository was changed.** §7 is a +proposal only. + +**Provenance markers.** **[measured]** = I ran it, numbers quoted below. **[read]** = +verified by reading source, `file:LINE` given on both sides. **[inferred]** = reasoned from +code I read but did not execute. + +**Environment.** DIALS `3.27.0-g9c415ec69-release` at `/opt/dials-v3-27-0`, interpreter +`/opt/dials-v3-27-0/build/bin/dials.python`. Jungfraujoch at `fccf2d91` (v1.0.0-rc.162), +working tree clean. Base file produced by +`cmake-build-release/tools/jfjoch_hdf5_test -o base -n 2 -S tests/test_data/compression_benchmark.h5` +— JF4M, 2068 × 2164 px, 0.075 mm pixel, beam centre (1090, 1136), 75 mm, 1 Å. Scratch files +in the session scratchpad, not in the repo. + +> **Caveat on the binary.** `cmake-build-release/tools/jfjoch_hdf5_test` (built 18:30) predates +> the source it sits next to (`writer/HDF5NXmx.cpp` mtime 18:33) and **emits the candidate +> encoding, not the shipped one** — its output has `rot1 @vector (0,1,0)` and chain +> `module → rot1 → rot2 → rot3 → translation`, whereas `writer/HDF5NXmx.cpp:614-647` at HEAD +> writes `(1,0,0)` and the reverse chain **[measured]**. Anything measured on a file straight +> out of that binary is measuring the candidate patch. Every encoding below was therefore +> written from scratch with h5py onto the same base file, so the comparison does not depend +> on which build produced it. + +--- + +## 1. The two frames, and why the premise about a Y mirror is wrong + +This is the crux, and getting it wrong is what produced the earlier misleading result. + +**Jungfraujoch's lab frame** (`common/DiffractionGeometry.cpp:10-15`, `:21-23`) **[read]**: + +```cpp +Coord detectorCoord = {(x - beam_x_pxl) * pixel_size_mm, + (y - beam_y_pxl) * pixel_size_mm, + det_distance_mm}; +return poni_rot * detectorCoord; // LabCoord +... +Coord GetScatteringVector() { return {0, 0, 1.0f / wavelength_A}; } +``` + +So **+x = increasing pixel column, +y = increasing pixel row (down the image), +z = beam +downstream**. There is *no* Y flip anywhere in the lab-frame code — the "MX counts rows from +the top" convention is not applied as a mirror, it is simply absorbed into the choice of +y. +The triple (right, down, downstream) is **right-handed**. + +**NeXus/McStas frame**, as encoded in the file: `fast_pixel_direction @vector = (-1,0,0)`, +`slow_pixel_direction @vector = (0,-1,0)`, translation along `+(bx·px, by·py, D)` +(`writer/HDF5NXmx.cpp:607-615`, `:646`) **[read]**. Pixel `(x,y)` therefore sits at +`(bx·px − x·px, by·py − y·py, D)` in McStas coordinates, i.e. + +``` +N : JF -> McStas = diag(-1, -1, +1) (a proper 180° rotation about z) +``` + +**dxtbx/imgCIF frame**: `MCSTAS_TO_IMGCIF = np.diag([-1, 1, -1])` +(`dxtbx/nexus/__init__.py:33`) **[read]**. Composing, + +``` +M·N : JF -> dxtbx = diag(+1, -1, -1) (a proper 180° rotation about x) +``` + +**[measured]** confirmation at zero tilt: Jungfraujoch places pixel (0,0) at +`(-81.750, -85.200, +75.000)` mm; dxtbx reports panel origin `(-81.750, +85.200, -75.000)` mm, +fast `[1,0,0]`, slow `[0,-1,0]`, `s0 = (0,0,-1)` Å⁻¹. + +**The consequence.** `diag(1,-1,-1)` is a **proper rotation**, not a mirror. Conjugating by it: + +| | JF frame | dxtbx frame | +| --- | --- | --- | +| rotation about **x** | `Rx(θ)` | `Rx(θ)` — **unchanged** | +| rotation about **y** | `Ry(θ)` | `Ry(-θ)` — **flipped** | +| rotation about **z** | `Rz(θ)` | `Rz(-θ)` — **flipped** | + +The task premise ("a Y mirror flips the sign of rotations about X and Z but not about Y") is +the opposite of the truth, because the two frames are not related by a mirror. It is **X that +is preserved and Y and Z that flip**. Applying it to +`poni_rot = Rz(-rot3)·Rx(-rot2)·Ry(+rot1)` (`common/DiffractionGeometry.cpp:183-187`, +`image_analysis/geom_refinement/XtalResidual.h:55`) **[read]**: + +``` +R_dxtbx = Rz(+rot3) · Rx(-rot2) · Ry(-rot1) +R_mcstas = Rz(-rot3) · Rx(+rot2) · Ry(-rot1) +``` + +This explains the earlier observations exactly. `Rz(+rot3)` instead of `Rz(-rot3)` looked +anomalous and was noticed; `Rx(-rot2)` is unchanged and looked right; **`Ry(-rot1)` also flips +but happens to look like the untransformed formula's `Ry(+rot1)` if you forget the frame +change** — which is why setting `rot1 @vector = (0,1,0)` appeared to "reproduce +`Rx(-rot2)·Ry(+rot1)` to 5.6e-17". It reproduces the formula written in the *wrong frame*. + +--- + +## 2. Where the rotation acts: the second, larger error + +NeXus composes a chain leaf-to-root: for `module → X1 → X2 → X3 → translation → "."`, +`nxmx.get_dependency_chain` returns `[X1, X2, X3, translation]` and +`get_cumulative_transformation` does `reduce(matmul, reversed(...))` +(`nxmx/__init__.py:1191-1211`) **[read]**, i.e. + +``` +p_lab = T_translation · R_X3 · R_X2 · R_X1 · p_module +``` + +The **outermost** (closest to `"."`) transformation is applied **last**. With `translation` at +the root the rotations act only on the in-plane part and the sample→PONI vector is *not* +rotated — dxtbx takes the panel origin from `A[0,:3,3]` of the full chain +(`dxtbx/nexus/__init__.py:390-408`) **[read]**, which for `T·R·R·R` is just `t`. + +Jungfraujoch rotates the **whole** vector including the distance component +(`poni_rot * detectorCoord`, distance in `z`). To match, the translation must be **innermost**: + +``` +p_lab = R_rot3 · R_rot2 · R_rot1 · T_translation · p_module +``` + +That is a rotation about the **sample**, which is what PONI means. This error is invisible to +any test that only compares axis directions — it moves the detector bodily and leaves the +panel orientation untouched. + +**[measured]**, `rot1 = rot2 = rot3 = 0.30/0.25/0.20 rad`, agreement of the dxtbx panel against +Jungfraujoch's own axes and PONI origin mapped through `diag(1,-1,-1)`: + +| encoding | Δfast | Δslow | Δorigin | +| --- | --- | --- | --- | +| shipped | 1.17e-01 | 9.34e-02 | **56.9 mm** | +| candidate (`rot1=(0,1,0)`, chain reversed) | 5.91e-01 | 1.14e-16 | **56.9 mm** | +| proposed | **1.24e-16** | **1.14e-16** | **7.6e-06 mm** | + +Note the candidate row: `slow` is exact to machine precision and `origin` is off by 57 mm. +An axis-only test passes it. + +--- + +## 3. Method + +**Jungfraujoch side.** A standalone program compiled against the existing +`cmake-build-release/common/libJFJochCommon.a` (not added to the repo) calls +`DiffractionGeometry::LabCoord` / `DetectorToRecip` for a list of pixels. Because +`DiffractionGeometry` is `float`, a float64 replica of `UpdatePoniRotMatrix` + `LabCoord` was +written in numpy and cross-checked against it — they agree to **≤ 9.1e-08 Å⁻¹**, which is the +float32 floor **[measured]**. All residuals below are quoted against the float64 replica. + +**dxtbx side.** `dxtbx.load(master)` → `detector[0]`, then +`panel.get_lab_coord((x·0.075, y·0.075))`. This deliberately bypasses +`get_pixel_lab_coord`, which would apply the `ParallaxCorrectedPxMmStrategy` dxtbx attaches +(`dxtbx/nexus/__init__.py:449`) and which Jungfraujoch has no counterpart for. Then +`s1 = lab/|lab|/λ`, `q = s1 − s0` with `s0 = beam.get_s0()`; dxtbx's `Beam` stores +`direction_` as the *sample-to-source* direction and returns `s0 = −direction/λ` +(`dxtbx/model/beam.h:279-282`), and the NXmx factory sets `direction=(0,0,1)` +(`dxtbx/nexus/__init__.py:138-140`) **[read]**, so `s0 = (0,0,−1)/λ` — the beam propagates +along −z in the dxtbx frame, consistent with `diag(1,-1,-1)`. + +**Comparison.** `q_dxtbx` vs `diag(1,-1,-1) · q_JF`, per pixel. + +**Pixels** (11): four corners, the beam centre, `(1090,200)` and `(1090,2000)` — both halves in +Y, `(200,1136)` and `(1900,1136)` — both halves in X, plus `(400,1800)` and `(1700,350)`. + +**Tilts** (10): zero; 10 mrad on each axis alone; 10 mrad on all three; 0.30/0.25/0.20 rad on +each alone; all three together; and a mixed-sign case `(-0.15, +0.20, -0.10)`. + +Every `(encoding, tilt)` file was written **before** any dxtbx call, since dxtbx keeps files +open. + +--- + +## 4. Results + +Worst value over the 11 pixels. "ray dev" is the angle between the JF and dxtbx diffracted-ray +directions for the same pixel; "spot shift" is that angle × 75 mm / 0.075 mm, i.e. roughly how +far a prediction lands from where Jungfraujoch puts it. **[measured]** + +| encoding | tilt (rad) | max \|Δq\| (Å⁻¹) | max \|Δlab\| (mm) | ray dev (mrad) | spot shift (px) | +| --- | --- | --- | --- | --- | --- | +| **shipped** | 0, 0, 0 | 9.3e-08 | 7.6e-06 | 0.0 | 0.0 | +| | 0.010, 0, 0 | 1.77e-02 | 2.55 | 17.7 | 18 | +| | 0, 0.010, 0 | 1.76e-02 | 2.51 | 17.5 | 18 | +| | 0, 0, 0.010 | 1.57e-02 | 1.18 | 15.7 | 16 | +| | 0.010 × 3 | 2.23e-02 | 2.30 | 22.2 | 22 | +| | 0.30, 0.25, 0.20 | 8.31e-01 | 70.3 | 857 | 857 | +| **candidate** | 0, 0, 0 | 9.3e-08 | 7.6e-06 | 0.0 | 0.0 | +| `rot1=(0,1,0)`, `rot2=(1,0,0)`, | 0.010, 0, 0 | 1.92e-02 | 2.41 | 19.2 | 19 | +| `rot3=(0,0,-1)`, chain | 0, 0.010, 0 | 1.10e-02 | 1.14 | 11.0 | 11 | +| `module→rot1→rot2→rot3→translation` | 0, 0, 0.010 | 1.57e-02 | 1.18 | 15.7 | 16 | +| | 0.010 × 3 | 2.08e-02 | 2.29 | 20.8 | 21 | +| | 0.30, 0.25, 0.20 | 5.01e-01 | 56.9 | 506 | 506 | +| **proposed** | **all 10 tilts** | **9.3e-08** | **7.6e-06** | **0.0001** | **0.00** | + +The `proposed` residual is *identical* at every tilt, including zero — it is a constant floor, +not a tilt-dependent error. Isolating it: the floor comes from the writer computing the +translation in `float` (`writer/HDF5NXmx.cpp:607-609`: `beam_center_x * start.pixel_size_x`, +both `float`), so the file stores the PONI vector as `(81.75000548, 85.20000428, 75.00000298)` +mm instead of `(81.75, 85.2, 75.0)`. Re-running the comparison with **the file's own stored +PONI vector** as the Jungfraujoch reference: + +| tilt | max \|Δq\| (Å⁻¹) | max \|Δlab\| (mm) | +| --- | --- | --- | +| 0, 0, 0 | 0.0 | 0.0 | +| 0.010, 0, 0 | 2.5e-16 | 1.4e-14 | +| 0, 0.010, 0 | 2.0e-16 | 1.4e-14 | +| 0, 0, 0.010 | 1.7e-16 | 2.1e-14 | +| 0.010 × 3 | 2.5e-16 | 2.1e-14 | +| 0.30, 0, 0 | 3.5e-16 | 2.1e-14 | +| 0, 0.25, 0 | 2.7e-16 | 1.4e-14 | +| 0, 0, 0.20 | 2.5e-16 | 2.8e-14 | +| 0.30, 0.25, 0.20 | 4.0e-16 | 2.8e-14 | +| -0.15, 0.20, -0.10 | **7.3e-16** | 5.7e-14 | + +**Exact to double precision.** The 9.3e-08 in the main table is the writer's float32 metadata +rounding and nothing else. + +`dials.import` reads the proposed file without complaint (`FormatNXmx`, 2 images) +**[measured]**. + +### 4.1 Each ingredient is individually necessary + +Single-change perturbations of `proposed`, worst \|Δq\| over all tilts **[measured]**: + +| variant | worst \|Δq\| (Å⁻¹) | which tilts fail | +| --- | --- | --- | +| `proposed` | 9.3e-08 (= floor) | none | +| `rot1 @vector = (0,+1,0)` | 5.9e-01 | every tilt with `rot1 ≠ 0` | +| `rot2 @vector = (-1,0,0)` | 4.9e-01 | every tilt with `rot2 ≠ 0` | +| `rot3 @vector = (0,0,+1)` | 3.9e-01 | every tilt with `rot3 ≠ 0` | +| rotation product order reversed (`rot3` innermost) | 1.07e-01 | only tilts with **two or more** non-zero angles | + +The last row is the reason the combined-tilt cases had to be in the test set: reversing the +rotation order is a **second-order** error (1.8e-04 Å⁻¹ at 10 mrad, 1.1e-01 at 0.2-0.3 rad) +and is invisible to any single-axis test. + +### 4.2 An equivalent alternative, for the record + +Keeping `translation` at the root and instead writing its `@vector` pre-rotated by +`R_mcstas` also reproduces Jungfraujoch exactly (**≤ 3.7e-16 Å⁻¹** over all 10 tilts, +**[measured]**) — *provided* the chain is still +`module → rot1 → rot2 → rot3 → translation` so the rotation product stays `R3·R2·R1`. With +the shipped (reversed) chain the same variant fails on combined tilts by 1.1e-01 Å⁻¹. +This alternative is **not recommended**: it makes the `translation` dataset a function of the +tilt angles, so the four numbers can no longer be edited independently, and it hides the +pivot in a precomputed vector. Recorded only to show the reorder is not the sole fix. + +--- + +## 5. The fast-axis "sign discrepancy" — explained, and it is not a discrepancy + +The writer stores `fast_pixel_direction @vector = {-1,0,0}` and dxtbx reports panel fast +`[1,0,0]`; the slow axis is `{0,-1,0}` in both. That asymmetry is entirely +`MCSTAS_TO_IMGCIF = diag(-1, 1, -1)` (`dxtbx/nexus/__init__.py:33`, applied at `:372` and +`:387`) **[read]**: + +``` +fast: diag(-1,1,-1) @ (-1, 0, 0) = ( 1, 0, 0) <- sign appears to flip +slow: diag(-1,1,-1) @ ( 0,-1, 0) = ( 0, -1, 0) <- y is preserved, so it looks unchanged +``` + +The same transformation is applied to both; it only *looks* like it hit the fast axis, because +that axis lies along x (flipped) while the slow axis lies along y (preserved). Nothing in the +writer or the reader is inconsistent here, and no change is needed. The origin gets the same +treatment: `(+81.75, +85.20, +75.00)` McStas → `(-81.75, +85.20, -75.00)` dxtbx **[measured]**. + +--- + +## 6. `mirror_y` + +**The sign convention does not differ between the mirrored and non-mirrored configurations, +and `mirror_y` does not enter the NXmx geometry at all.** + +`mirror_y` is a **detector-assembly** flag, not a lab-frame one. It is consumed only by +`DetectorGeometryModular` (`common/DetectorGeometryModular.cpp:69-70`, `:139-142`), which calls +`DetectorModuleGeometry::VerticalFlip` (`common/DetectorModuleGeometry.cpp:72-87`) to move each +module's origin to `height − y0 − 1` and swap its `Ypos`/`Yneg` fast/slow directions. That +affects **which raw module pixel lands where in the assembled image** and nothing else. Once +the image is assembled, `DiffractionGeometry` maps assembled `(x, y)` to the lab with a single +fixed convention (`common/DiffractionGeometry.cpp:10-15`), and `NXmx::Metrology` writes a fixed +`fast = {-1,0,0}` / `slow = {0,-1,0}` regardless (`writer/HDF5NXmx.cpp:646`) **[read]**. + +**[measured]** — the same JF4M built with `mirror_y = true` (`DetJF4M()`) and +`mirror_y = false` (`DetJF(8, 2, 8, 36, false)`), same beam centre / distance / tilt +`(0.30, 0.25, 0.20)`, produces byte-identical `StartMessage` metrology fields +(`image_size 2068×2164`, `pixel 7.50000036e-05 m`, `bc (1090,1136)`, `dist 0.075000003 m`, +`poni (0.3, 0.25, 0.2)`) and identical lab coordinates at every probed pixel, e.g. pixel +(1090, 2000) → `(37.717495, 74.503891, 53.391029)` mm in both. + +--- + +## 7. Proposed change — NOT made + +Two files, six edits. + +**`writer/HDF5NXmx.cpp:614-615`** — `translation` moves inside the rotations: + +```cpp + SaveScalar(transformations, "translation", vector_length)-> + Transformation("m", "/entry/instrument/detector/transformations/rot1", + "detector", "detector_arm", "translation", vector_norm); +``` + +**`writer/HDF5NXmx.cpp:625-644`** — axes corrected and the chain reversed so the product is +`R_rot3·R_rot2·R_rot1`: + +```cpp + SaveScalar(transformations, "rot1", rot1)-> + Transformation("rad", + "/entry/instrument/detector/transformations/rot2", + "detector", "detector_arm", + "rotation", + std::vector{0.0, -1.0, 0.0}); + + SaveScalar(transformations, "rot2", rot2)-> + Transformation("rad", + "/entry/instrument/detector/transformations/rot3", + "detector", "detector_arm", + "rotation", + std::vector{1.0, 0.0, 0.0}); + + SaveScalar(transformations, "rot3", rot3)-> + Transformation("rad", ".", + "detector", "detector_arm", + "rotation", + std::vector{0.0, 0.0, -1.0}); +``` + +**`writer/HDF5NXmx.cpp:646`** — the module now hangs off `translation`: + +```cpp + DetectorModule("module", origin, size, {-1,0,0}, {0,-1,0}, "translation", + start.pixel_size_x); +``` + +**`writer/HDF5NXmx.cpp:339`** — the detector's own `depends_on` must follow: + +```cpp + SaveScalar(group, "depends_on", "/entry/instrument/detector/transformations/translation"); +``` + +Notes on the change: + +* **Forward references are fine.** `rot1` names `rot2` in a `depends_on` *attribute*, which is + a plain string, not an HDF5 link, so the write order (`translation`, `rot1`, `rot2`, `rot3`) + needs no change. Confirmed empirically: the patched build already emitted such a file and + dxtbx resolved it **[measured]**. +* **`rot3 @vector` is already correct** at `(0,0,-1)` — only its position in the chain changes. +* **Jungfraujoch's own read-back is unaffected.** `reader/HDF5MetadataSource.cpp:573-579` reads + `rot1`/`rot2`/`rot3` by dataset path into `PoniRot1/2/3_rad` and never consults `@vector` or + `depends_on` **[read]**, so the stored values keep their meaning and the round-trip is + untouched. This is also why the bug has been invisible in-house. +* **`NXdetector/depends_on` is cosmetic.** dxtbx builds the panel from + `module.fast_pixel_direction.depends_on` and never reads the detector-level field + (`dxtbx/nexus/__init__.py:355-408`; `nxmx/__init__.py:742-750` exposes it but nothing in + `get_dxtbx_detector` calls it) **[read]**. Fix it anyway so the file is self-consistent for + other readers. +* **No existing correct file is invalidated.** All four numbers keep their values and meaning; + only the attributes change. Files with `rot1 = rot2 = rot3 = 0` are bit-compatible in + geometry either way — which is every file where the tilt was never set. +* **Suggested regression test**: the reciprocal-space comparison of §3 belongs in the DIALS CI + job, run at a non-zero tilt. The current CI writes zero tilts, where all seven encodings + agree, so it cannot see any of this. + +### Impact if left unfixed + +Only bites when `poni_rot1/2/3 ≠ 0`, i.e. a file carrying a refined tilt from +`rugnux` (`end.refined_poni_rot*`) or a tilt set through the broker API. At a realistic +10 mrad the diffracted-ray direction DIALS derives for a given pixel is wrong by 16–22 mrad, +≈ 18–22 px at 75 mm — enough to break indexing outright, not merely to degrade it. + +--- + +## 8. What I did not determine + +* **Whether Jungfraujoch's `rot1/rot2/rot3` match upstream PyFAI's.** Everything here verifies + that the NXmx encoding reproduces *Jungfraujoch's own* `poni_rot` + (`common/DiffractionGeometry.cpp:183-187`). Whether that in turn equals pyFAI's PONI + convention for an imported `.poni` file is a separate question and was not tested. +* **The `NXmxLegacy` / `NXmxVDS` variants.** All measurements used `NXmxIntegrated` (`-S`), + chosen because it is a single file and therefore safe to patch. `NXmx::Metrology` is + format-independent (`writer/HDF5NXmx.cpp:588-648`) so the conclusion carries **[inferred]**, + but it was not measured on the other two. +* **Hierarchical (multi-`NXdetector_module`) files.** Jungfraujoch writes exactly one module + (`writer/HDF5NXmx.cpp:646`), so dxtbx takes its flat-detector path + (`dxtbx/nexus/__init__.py:355-408`). The hierarchical path at `:264-330` uses a different + formula (`A @ (-1,0,0,1) − origin`) and was not exercised. +* **Non-dxtbx readers.** XDS/Durin/Neggia and CrystFEL take the beam centre and distance from + `NXdetector` scalars rather than walking `NXtransformations`, so they are **[inferred]** + indifferent to this — but they also then ignore the tilt entirely, which is a separate gap. +* **The stale-build question.** I did not rebuild `jfjoch_hdf5_test` from HEAD; I worked around + it by writing every encoding with h5py. If the candidate patch is still sitting in someone's + working tree, it should be reverted before the change in §7 is applied on top. + +--- + +## 9. Relation to `DIALS_NXMX_COMPATIBILITY.md` + +That audit's rows **G4** and **G5** anticipated this. G4 predicted `rot1 → (0,-1,0)` and +`rot2 → (1,0,0)` from the frame argument, flagged the derivation as needing independent +confirmation, and listed "the exact correct signed `@vector` values" as unresolved. **Both +values are confirmed exactly as G4 stated them [measured]**; the sign that later went astray +was in the working patch (`rot1 = (0,+1,0)`), not in G4. G5 flagged the pivot; this report +shows it is not merely a cosmetic mismatch but contributes the **larger** of the two errors, +and that fixing it needs the chain reversed, not only reordered. What neither row covered is +the rotation **product order**, which is a third, independent requirement (§4.1). diff --git a/docs/review/RC162_WORK_PLAN.md b/docs/review/RC162_WORK_PLAN.md new file mode 100644 index 00000000..504cd2a3 --- /dev/null +++ b/docs/review/RC162_WORK_PLAN.md @@ -0,0 +1,202 @@ +# rc.162 work plan — decisions and open items + +Working document for branch `2608-rc-162`. Nothing here is committed. It records what was changed, +what was found, and what needs a decision before anything else is written. + +Evidence is labelled: **[verified]** = reproduced or read directly in this session; +**[agent]** = reported by a review agent and not independently re-checked; +**[unverified]** = plausible, not yet confirmed. + +--- + +## 1. Already changed, on the branch, uncommitted + +| Change | Files | Status | +| --- | --- | --- | +| Volta (`sm_70`) added for CUDA 12 toolkits, gated `CUDA < 13.0`, with `-Wno-deprecated-gpu-targets` | `CMakeLists.txt` | **[verified]** all 15 `.cu` compile for `sm_70` (incl. ffbidx); reconfigure emits the flag; `cuobjdump` shows an `sm_70` cubin in the built `rugnux` | +| DECTRIS no longer defaults `bit_depth_readout` to 16; `FillMessage` emits it as optional | `common/DetectorSetup.cpp`, `common/DiffractionExperiment.cpp` | **[verified]** `DetectorSetup*`, `CBOR*`, `JFJochReceiverLite*` (647 assertions / 46 cases) and `[DiffractionExperiment]` (378 / 43) pass | +| GPU generations + minimum NVIDIA driver per released artefact | `docs/RELEASE_CONTENTS.md`, pointers in `docs/RUGNUX.md`, `docs/JFJOCH_VIEWER.md` | done | +| `1.0.0-rc.162` changelog block (3 lines) | `docs/CHANGELOG.md` | done — assumes the branch name means rc.162; renumber or drop if not | +| dxtbx/NXmx compatibility review (812 lines) | `docs/review/DIALS_NXMX_COMPATIBILITY.md` | new file, review only | + +`VERSION` untouched; `update_version.sh` not run. + +Driver floors documented: CUDA 12 artefacts → 525.60.13; CUDA 13 artefacts → 580.65.06 (Linux) / +R580 (Windows). **A V100 can only use the RHEL 8 packages or the portable Linux `.tgz`** — the +Windows installer is CUDA 13.3 and will never run on Volta. + +--- + +## 2. Decisions needed + +### D1 — `bit_depth_readout` policy *(blocks: the writer, and the value of the fix already made)* + +Plain `FormatNXmx` reads **only** `bit_depth_readout` and treats it as the *stored element width*. +`bit_depth_image` is never referenced anywhere in dxtbx/dials/cctbx **[agent, spot-checked]**. +NXmx defines only `bit_depth_readout` ("how many bits the electronics record per pixel"); +`bit_depth_image` is **not in the NXmx application definition at all** **[verified]**. DECTRIS +invented it because summation makes the image wider than the readout. + +| Option | NXmx-conformant | DIALS-correct today | +| --- | --- | --- | +| (a) true readout depth (12/16) | yes | **no** — silently remaps real counts 2ⁿ−1, 2ⁿ−2 to −1/−2 on any summed image | +| (b) stored element width | no (but dxtbx's own writer does this — `nxmx_writer.py:740`) | yes | +| (c) omit when it differs from the image depth | conformant by omission of a *recommended* field | yes, provided 32-bit is stored `int32` | + +Measured dxtbx behaviour **[verified]** — `uint32` is the trap: + +``` +uint32 bit_depth=None -> TypeError: Unsupported integer dtype uint32 +uint32 bit_depth=32 -> OK +uint8/uint16/int8/int16/int32, any -> OK +``` + +Leaning **(c)**, reverting to (a) once upstream prefers `bit_depth_image`. (a) is the defensible +purist answer. Either way the change already on the branch is a strict improvement — never worse +than the hard-coded 16 **[verified case by case]**. + +### D2 — write `underload_value`? *(recommend: yes)* + +We do not write it **[verified]**; the only `underload` hit is a JSON key at `FileWriter.cpp:193`. +Without it dxtbx sets the trusted minimum to `-0x7FFFFFFF`, so any `-1`/`-2` it manufactures is +**trusted and consumed as a real intensity**. `FormatNXmxDLS` and `FormatNXmxEigerFilewriter` both +patch `underload_value = 0`; plain `FormatNXmx` does not, so we get no such protection. +Suggested: `0` unsigned, `GetUnderflow() + 1` signed. Standards-clean, independent of D1, and worth +more than any bit-depth tweak. + +### D3 — `images_per_file` default *(currently 1000, `jfjoch_api.yaml:429-434`)* + +The value plays three roles at once: + +1. **Writer parallelism unit** — `(image_number / images_per_file) % socket.size()` + (`ZMQStream2Pusher.cpp:22,31`; `TCPStreamPusher.cpp:947,972`). Whole files round-robin. +2. **Linear writer-RAM multiplier** — every plugin reserves per file: `HDF5DataFilePluginMX.cpp:55-82` + (8+ arrays × `max_spot_count` × ipf), `HDF5DataFilePluginAzInt.cpp:27-34` + (ipf × phi_bins × q_bins × 4 B, ×3 with count+std), plus ROI/ImageStats/Performance. +3. **DIALS-legacy readability unit** — the threshold is exactly *one data file*. + +Key correction to the intuition: parallelism only collapses when `total_images ≤ ipf`. At 10⁵–10⁶ +images, ipf=5000 still gives 20–200 files across the writers, so **serial MX keeps its +parallelism**; it only loses load-balancing granularity. Rotation (1800/3600) does collapse to one +writer — which is the case that does not need several. + +Costs of a flat 5000: memory ×5 (≈40 MB/writer at default `max_spot_count` 250; ≈320 MB at the API +max of 2000, more with multi-phi azint); coarser failure granularity; later file closure (matters +for serial fast-feedback only). + +Options: **flat 5000**, or **rotation-aware** (rotation → one file; stills/serial → 1000). +Rotation is already detectable at Start — `GetGoniometer()` is `std::optional`, set only for +rotation. Leaning rotation-aware, because 5000 is a blunt setting for a value meaning three things. + +### D4 — impersonate `FormatNXmxEigerFilewriter`? *(recommend: no)* + +Writing `detectorSpecific/eiger_fw_version` would buy the `bit_depth_image` preference and a free +`underload_value = 0`. It also drags in **[verified by reading the class]**: + +- a module-size whitelist with a hard `ValueError` — we write one `NXdetector_module` whose + `data_size` is the full image (`HDF5NXmx.cpp:619-621,644`); a full-frame EIGER2 is in the list, + **every JUNGFRAU geometry and most ROI modes are not**; +- a `get_raw_data` that bypasses the virtual dataset entirely — it scans for `data_\d{6}`, so + `NXmxVDS` (our default) and `NXmxIntegrated` would crash; +- a false claim in a vendor-detection hook that already gates firmware-bug workarounds. + +Both benefits are obtainable directly (D1, D2). + +### D5 — release framing + +Keep, renumber, or drop the `1.0.0-rc.162` changelog block; decide whether `VERSION` / +`update_version.sh` run in this branch. + +--- + +## 3. Verified defects, not fixed + +Ranked by blast radius. None of these are touched on the branch. + +1. **`rugnux` mistypes every `_process.h5`.** `Rugnux.cpp:863` forces `BitDepthImage(32).PixelSigned(true)` + (correct for the reader's internal container, per the comment at `HDF5MetadataSource.cpp:692-701`), + but that experiment reaches `FillMessage` with `NXmxIntegrated` + `hdf5_source_data` and + `write_images=false`, so `HDF5NXmx.cpp:250` types the virtual `/entry/data/data` **int32 over the + original uint16/uint32 detector files**. Our own reader dodges it by resolving the VDS to the + source; only DIALS, XDS/Durin and h5py see it. Ships in every offline run. **[verified]** +2. **Multi-file `NXmxLegacy` is silently truncated by DIALS.** 40 images written, `dials.import` + reports 10; as a rotation sweep the count looks plausible and `dials.find_spots` later dies with + `IndexError`. Cause: `dxtbx/nexus/__init__.py:576-583` picks one dataset and indexes it globally. + **[verified]**, reproduced independently **[agent]**. VDS and single-file legacy are correct. +3. **`pixel_signed` never propagated.** `JFJochReceiverLite.cpp:208` copies the depth from the + stream and drops the sign, while `MXAnalysisWithoutFPGA` forwards the image byte-for-byte. + Separately, `image_format_settings.signed_output` (`OpenAPIConvert.cpp:562`) is accepted + unconditionally for DECTRIS — reachable through the REST API on stock hardware, and it + mis-types the VDS master and corrupts `saturation_value`/`error_value`. **[verified]** +4. **Missing VDS sources read as all-zeros, silently** — `dials.import` succeeds, every pixel 0, + no warning at any log level. Severe because VDS is the default. **[agent]** +5. **`image_dtype` missing or unrecognised → `{bit_depth_image=0, pixel_signed=false}`**, silently + (no `else` at `CBORStream2Deserializer.cpp:1287`, no `default` at `CBORStream2Serializer.cpp:709`). + `NXmxLegacy` then records `bit_depth_image = 0` as fact and reports success. **[agent, both]** +6. **No cross-check** anywhere between a per-image CBOR type tag and the start-message declaration. + Both agents independently propose the same single guard at `writer/HDF5DataFile.cpp:138` — + compare `msg.image.GetMode()` against `{start.bit_depth_image, start.pixel_signed}` — which + closes items 3, 5 and the VDS half of 1 at the one point where the two values meet. **[agent, both]** +7. **`saturation_value` drifts down one count per write→read cycle** — written as + `GetSaturationLimit() - 1` (`DiffractionExperiment.cpp:662`), read back without `+1` + (`HDF5MetadataSource.cpp:691`). **[verified]** +8. **Grid scans fail to import** — `ValueError: matmul: ... size 0 is different from 3` at + `dxtbx/nexus/__init__.py:58`, on a hand-built file matching `HDF5NXmx.cpp:771-800`. **[agent]** +9. **Latent geometry bug** — exported `rot1`/`rot2` appear to carry each other's `@vector`, and the + NXmx chain pivots the module about corner pixel (0,0) while refinement pivots about the PONI. + Inert while tilts are zero. **[unverified]** — needs its own check before anyone acts. +10. **Docs** — `docs/CBOR.md:72` is the only mention of `image_dtype` and lists none of the six legal + values; `bit_depth_image` does not appear in CBOR.md at all, so the only bit-depth row a reader + finds is `bit_depth_readout`. `docs/HDF5.md:429` wrongly claims `bit_depth_image`'s "NXmx + standard is `bit_depth_readout`". `pixel_signed` is undocumented in both. **[agent, both]** + +Note for item 10: `image_dtype` **is** a genuine stream2 field, but only over +`uint8`/`uint16`/`uint32` — our three `int*` forms are a jfjoch extension the doc does not +disclose. **[verified against the upstream spec]** + +--- + +## 4. CI gaps (cheap, independent of every decision above) + +- `dials-test` legacy job passes no `-f` (`build_and_test.yml:325`), so it builds a **single** data + file and never exercises multi-file legacy. Multi-file legacy *is* built in CI — by the three XDS + jobs (`-n25 -f10`), which read it correctly. Layout and DIALS are each tested, never together. +- **`single.h5` vs `single_master.h5`** — line 342 runs `xia2.ssx image=single.h5`, but `-S -o single` + writes `single_master.h5`. **[verified with the current build]** Our only DIALS test of + `NXmxIntegrated` is not testing it. +- The job asserts nothing beyond `xia2.ssx`'s exit code, so adding `-f` alone may not turn it red. + +--- + +## 5. Upstream report to DIALS + +Two independent defects, both with the fix already present in a vendor-specific subclass: + +1. **Prefer `bit_depth_image` over `bit_depth_readout`** — hoist `FormatNXmxEigerFilewriter.py:99-103` + into `FormatNXmx.get_raw_data`. Supporting points: the `8/16 ⇒ unsigned` assumption is wrong for + signed 8/16-bit data; the remap is **untested** anywhere in the tree; and dxtbx's own writer + (`nxmx_writer.py:740`) sets `bit_depth_readout = dset.dtype.itemsize * 8`, contradicting both + NXmx and DECTRIS. That inconsistency is the strongest part of the report. +2. **Multi-dataset NXdata silently truncates** — hoist the `data_\d{6}` walk out of the same + subclass, or at minimum raise instead of truncating when NXdata holds more than one image dataset. + +--- + +## 6. Measurements to run before committing D3 + +1. Single-writer sustained rate on **GPFS** at the real rotation frame rate. The 2000 Hz figure + measured here is local page cache and is not evidence for the target. +2. Writer RSS at ipf=5000 with realistic `max_spot_count` and azint bin counts. +3. **Does AutoPROC read a single-data-file legacy master?** Only DIALS has been verified, and + AutoPROC is the whole reason legacy remains the default. **This can invalidate D3 outright — + check it first.** +4. What idle writers do when a run produces fewer files than writers (empty files? dangling state?). + +--- + +## 7. Known inconsistency worth folding into D3 + +`images_per_file: 0` is documented as "write all images to a single data file" +(`jfjoch_api.yaml:434`) but **both pushers reject it** (`ZMQStream2Pusher.cpp:43-45`, +`TCPStreamPusher.cpp:838`). It is the natural spelling for what D3 wants, and it currently throws. +Either make it work or fix the description. diff --git a/docs/review/SECURITY_REVIEW.md b/docs/review/SECURITY_REVIEW.md new file mode 100644 index 00000000..36ec4837 --- /dev/null +++ b/docs/review/SECURITY_REVIEW.md @@ -0,0 +1,175 @@ +# Security Review — access control & data confidentiality + +Status: proposal / discussion. Scoped to a specific, deliberately narrow threat model. + +## 1. Threat model + +The concern is a peer **already on the semi-trusted facility network** connecting to a +Jungfraujoch service with **zero or accidental effort** — a wrong host:port, a curious +colleague, a mis-pointed script, a stray browser tab. It is *not* a sophisticated attacker +and *not* packet capture / MITM on the wire: those are owned by the network layer (802.1x, +facility infrastructure) and there are far easier targets elsewhere on the network. + +Security here is explicitly **best-effort**: any control that materially slows down normal +operation will be turned off by scientists, so the goal is the *minimal, highest-gain* set of +measures and nothing more. Per-user accounts, RBAC, mTLS everywhere, and encrypted data +streams are all **out of scope** — they fail this test. + +## 2. What we are actually protecting + +The crown jewel is **confidentiality of the live analysis data — *which sample / ligand is +being measured***. This is an industrial-fragment-screening concern: one experiment's team +should not be able to observe another's sample. The sensitive data is the live diffraction +images plus the derived metadata (unit cell, resolution, spot counts, sample naming) — the +unit cell alone can fingerprint a compound. + +By contrast, **write / control access is low risk.** Scientists starting, cancelling, or +reconfiguring collections is normal operation; there is little to gain from restricting it and +real friction if we do. The design principle is therefore: **protect reads, leave writes open.** + +## 3. The three channels the crown jewel can leak through + +The same sensitive data is reachable three independent ways. Two are handled by deployment / +infrastructure (sections 3.1–3.2) or are accepted residual risk (3.3); the remaining one +(3.4) is the open item this document is about. + +### 3.1 Data at rest on the shared filesystem — handled by FS ACLs + +Written HDF5 files (and the `file_prefix` / `sample_name` embedded in their names) sit on the +shared high-performance filesystem. Confidentiality there is enforced by **filesystem ACLs and +a setuid/setgid ownership scheme**, so that only the owning experiment can read its data. This +is a deployment control, not an application concern. + +Note for operators: the writer sets no `umask` of its own, so file/dir modes are inherited from +the launching environment — the ACL/ownership setup must not depend on the process applying a +restrictive umask itself. + +### 3.2 broker ↔ writer transport — handled by network isolation + +The internal image transport (broker → writer / receiver, over the ZeroMQ `PUSH`/`PULL` or the +raw-TCP path) is expected to run on a **dedicated back-end network** with no general user +access. The socket address is already configurable, so the data socket can be pinned to the +correct back-end interface, and access to it should be limited by **firewall** rules. This is +an infrastructure control, not an application concern. + +### 3.3 ZeroMQ compatibility streams — unsafe by design, accepted + +The broker's ZeroMQ **preview** (`preview/ZMQPreviewSocket.cpp`), **metadata** +(`preview/ZMQMetadataSocket.cpp`), and **republish** streams are integration/compatibility +interfaces for external downstream tools. They are **`PUB`/`PUSH` sockets with no +authentication and no access control** — any peer that can reach them can subscribe and +receive live images / per-image metadata. No shipped Jungfraujoch tool depends on them +(`jfjoch_viewer` and the web frontend both consume everything over HTTP; see 3.4), and they +are **not used at the main installation**. + +**Posture: these streams are documented as unauthenticated and unsafe.** If enabled, they must +be bound to a trusted back-end network and firewalled like 3.2. We deliberately do *not* add +CURVE/keying here — it carries key-management friction disproportionate to a compatibility +feature that the main installation does not run. Consumers accept the risk by enabling them. + +### 3.4 Broker HTTP/REST control plane — the open item + +This is the channel scientists actually use and the only one exposed facility-wide. + +- The broker's HTTP server binds **all interfaces** (`broker/jfjoch_broker.cpp:135`, + `listen("0.0.0.0", 5232)`), plain HTTP, and has **no authentication of any kind** — no token, + no pre-routing hook, no `securityScheme` in `broker/jfjoch_api.yaml`. Anyone who can reach + `host:5232` has full read and write access to 64 routes. +- Both scientist-facing tools reach live data **exclusively over this HTTP interface**: + - `jfjoch_viewer` — HTTP only (`reader/JFJochHttpReader.cpp`; connect via Ctrl+H, + `docs/JFJOCH_VIEWER.md`). No ZeroMQ in the viewer at all. + - the React frontend — HTTP polling only (`/image_buffer/image.jpeg`, `/preview/plot`, + `/statistics`, `/result/scan`); no WebSocket / ZeroMQ (`docs/review/08-frontend.md`). +- Because access comes from **arbitrary user workstations**, the HTTP port cannot simply be + bound to a back-end interface (that would break the viewer/frontend). The gate has to live in + the application. + +The confidentiality-sensitive **read** endpoints — the ones to gate (all handlers in +`broker/JFJochBrokerHttp.cpp`): + +| Endpoint | Carries | +|---|---| +| `GET /statistics` | aggregate live analysis state | +| `GET /image_buffer/image.{cbor,jpeg,tiff}`, `GET /image_buffer/start.cbor` | live diffraction images + start metadata | +| `GET /result/scan` | spots, indexing, **unit cell**, resolution | +| `GET /preview/plot`, `GET /preview/plot.bin` | azimuthal / ROI plots (live data) | + +Reads that were reconsidered and are **not** confidential, so they stay open: +`/preview/pedestal.tiff` and the mask routes (`/config/mask[.tiff]`, `/config/user_mask[.tiff]`) +are calibration / pixel-mask data, not sample-dependent; `/config/instrument` and +`/config/file_writer` carry only the beamline name and file-format type — the `file_prefix` and +`sample_name` are supplied **write-only** via `/start` DatasetSettings and are not read back here. +`/version`, `/status`, `*/status` are benign. All write/control routes carry no sensitive data. + +## 4. Proposal — gate the sensitive reads (§3.4) + +Two options. Both leave **writes open** and gate only the §3.4 read endpoints. **Option A** is +the simplest, self-contained step and the recommendation; **Option B** (Kerberos/GSSAPI) is +easier for users but more to build. + +### 4.1 Option A — acquisition-scoped Bearer token set at `/start` (recommended) + +The token lives with the acquisition instead of in broker config: + +- `POST /start` (`Dataset_settings`) gains an **optional token** field. + - **No token in `/start` → no enforcement** — current behaviour, fully backward-compatible. + - **Token in `/start` → that token is required** on every §3.4 route as + `Authorization: Bearer `; a missing or wrong token gets 401/403. +- The broker holds the current acquisition's token and does an **opaque equality check** — it + neither mints nor decodes it. Each `/start` sets its own token, so protection is naturally + **per experiment** and rotates when the next acquisition starts. (Because `/start` is a write + and writes are open, anyone can start a new acquisition and thereby replace the token — this is + accepted write-risk and is exactly the rotation mechanism.) +- **Token provisioning is external to `jfjoch_broker`** — e.g. a facility tool that encodes the + user name under AES-256. The same string is handed to whoever issues `/start` and to the + authorised viewer(s). The broker only compares strings. +- **`jfjoch_viewer`** accepts the token via **command-line parameter, environment variable, and + D-Bus** (`viewer/`, `reader/JFJochHttpReader.cpp`), and sends it as the Bearer header. +- **Frontend: no token handling.** When a §3.4 route returns 401/403, the frontend shows a + graceful *"permission denied"* state instead of breaking; confidential viewing goes through the + desktop viewer. +- **Transport / HTTPS: not wired into `jfjoch_broker`.** Terminate HTTPS at a front `httpd` + reverse proxy and pin the broker to localhost behind it — cheaper and standard versus building + TLS into the broker. This does require **`jfjoch_viewer` to speak HTTPS** (build its httplib + client with OpenSSL support and accept `https://` endpoints). + +Implementation touch-points: `broker/jfjoch_api.yaml` (`Dataset_settings` token field + regen +clients), `broker/OpenAPIConvert.cpp` and `common/DatasetSettings` (carry the token into broker +state), one `set_pre_routing_handler` in `broker/JFJochBrokerHttp.cpp` (enforce on the §3.4 +routes), and the viewer's token inputs + HTTPS-capable client. + +### 4.2 Option B — Kerberos / GSSAPI via a pass-through proxy (better UX, more work) + +PSI runs Active Directory, so Kerberos/GSSAPI is available. The usual PSI pattern: a +**pass-through `httpd`** terminates HTTPS, holds the service **keytab**, performs GSSAPI/SPNEGO +authentication, and forwards to a **localhost-pinned** broker with the authenticated **user name +as a plain-text header**. The broker trusts that header — safe because it only accepts +connections from the local proxy — for authorization on the §3.4 routes. + +- **Pro:** far easier for users — AD single sign-on, no token to mint, carry, or paste. +- **Con:** more development — the broker needs header-based identity/authorization (and, if + wanted, per-user rather than per-acquisition rules), plus the proxy/keytab deployment. + +Option B is the stronger long-term path if per-user AD identity is desired; Option A is the +minimal step that protects the crown jewel now, and the two are not mutually exclusive (the +Bearer-token gate can stay as the broker-native fallback behind either proxy). + +## 5. Relied-upon controls (out of application scope) + +The confidentiality posture depends on these being in place operationally: + +- **Network:** the back-end image transport (3.2) and any enabled ZeroMQ compatibility streams + (3.3) run on an isolated back-end network with firewall restrictions. +- **Filesystem:** ACL + setuid/setgid ownership (3.1) confines who can read written data and + does not rely on the writer applying its own umask. +- **Front proxy:** HTTPS termination (and, under Option B, the keytab + GSSAPI) runs in a reverse + `httpd`; `jfjoch_broker` is pinned to localhost behind it. +- **Infrastructure:** 802.1x / no untrusted parties passively on the wire. + +## 6. What is explicitly *not* recommended + +TLS wired **into** `jfjoch_broker` (terminate it at the front proxy instead, §4.1), mTLS, and +encrypted / CURVE-keyed ZeroMQ data streams (§3.3) — all disproportionate to this threat model +and likely to be disabled in practice. Per-user RBAC is deliberately absent from Option A; it is +effectively what Option B (§4.2) would provide via AD if we decide the extra complexity is +warranted — a considered choice, not an oversight. diff --git a/docs/review/VIEWER_REDESIGN.md b/docs/review/VIEWER_REDESIGN.md new file mode 100644 index 00000000..60ace386 --- /dev/null +++ b/docs/review/VIEWER_REDESIGN.md @@ -0,0 +1,264 @@ +# jfjoch_viewer — redesign proposal (v2, post-review) + +*Working document. Goal: rethink the viewer so it centers on what makes Jungfraujoch unique +(whole-dataset triage + reprocessing), not on what every diffraction viewer already does. Keep +the salmon identity, add a navy second colour, replace text toolbars with icons, surface a +surgical subset of processing settings, and let the user track several quantities at once.* + +v2 incorporates the design-review feedback: multiple simultaneous plots beside the image, +thumbnail hit feed, adjustable/saveable layouts (presets **and** custom), inline surgical +settings with an MX⇄AzInt toggle, consolidation of the per-image plot into the reusable chart, +and the coral + **navy `#1F3A5F`** palette. + +--- + +## 1. What the viewer is today (audit) + +Single `QMainWindow`: menu (File/Mask/Window/Charts/Help) · two **text** toolbars (unicode +arrows `|⇐ ⇐ ⇒ ⇒|`) · big diffraction image · a right panel of **seven solid 50-px `#FA7268` +banners** that stack into a "venetian blind" and still scroll · a ~340-px bottom dock holding +the dataset plot and the processing jobs · status bar. Plus ~10 **satellite windows** (image +list, spot list, reflections, metadata, **Processing settings** [tabbed], calibration, +reciprocal space, azint-2D, magnifier, extra plots) reachable only from menus. + +**Keep (genuinely good):** the dataset-info **chart engine** +(`charts/JFJochDatasetInfoChartView.cpp`, `JFJochViewerDatasetInfo.cpp`) — ~20 metrics, +binning, goniometer x-axis, **run overlays**, **grid-scan 2D**, hover/click-to-load, FFT; the +**processing jobs** panel with switchable snapshot overlays +(`windows/JFJochProcessingJobsWindow.cpp`); the **tabbed settings** +(`windows/JFJochSettingsWindow.cpp`); the **salmon palette**. + +**Problems:** (1) Albula DNA — screen budget favours the single image; the two differentiating +features (dataset plot, reprocessing) are the smallest things on screen. (2) Banner clutter — +equal weight for always-used and rarely-used controls; panel scrolls. (3) Fragmentation — key +tables and **settings** live in hidden windows. (4) Text toolbars, no icon set. (5) One rigid +layout for every mode (file/live, stills/rotation, grid/sweep), nothing adapts and nothing is +resizable away on a small screen. + +--- + +## 2. Inspiration + +Closest professional analogue: **cryoSPARC Live / Thermo Smart EPU** — monitor *many* exposures +live, drill into *one*, reprocess, one app. Reference image the reviewer singled out: +`cryosparc.com/images/home/live-apo-2.png` — **a large micrograph with several diagnostic plots +arranged beside it**, each tracking a different quantity. That "image + a stack of live plots" +is the target feel. Transferable patterns: stage-oriented workspaces instead of scattered +windows; an **Overview** of per-exposure attributes used to *triage*; an **exposure feed** +(thumbnail strip with progress/reject marks); reprocessing with "apply to all/future". + +Crystallography cues: SSX beamline tools (SLS **DA+** / *adp-tracker*, DESY **P11** CrystFEL) +foreground live **hit/index rate** as the primary readout. `dials.image_viewer` / ALBULA / adxv +define the *image pane* baseline (HDR red-dot, histogram contrast, cross-sections) — match that +for the image, don't let it define the app. + +Sources: guide.cryosparc.com/live/ui-overview · thermofisher.com embedded-cryosparc-live · +journals.iucr.org (DA+, P11 SSX) · dials.github.io · dectris ALBULA · scripps.edu adxv. + +--- + +## 3. North-star reframe + +> From **"an image viewer with extras"** → **"a diffraction-experiment monitor that drills down +> to images,"** with a layout the user can shape to their screen and their job. + +### 3.1 Everything is a dock; layout is saved and switchable + +Rebuild the window as a set of **dockable panels** around a central area, using +`QMainWindow::saveState/restoreState` + `saveGeometry/restoreGeometry` persisted in `QSettings`. +This answers the reviewer's question directly: **Qt fully supports save/resume of dock +arrangements** (each dock needs a stable `objectName`, including the dynamically-created plot +docks). + +Two delivery mechanisms, not either/or: + +- **Perspective presets** (baked-in `QByteArray` states, restored on a click): + - **Image mode** — big central image, inspector docked right, plots/processing hidden. + For small screens and pure viewing. + - **Processing/monitor mode** — image shares the screen with a **grid of plots**, processing + + settings + hit feed visible. The cryoSPARC-style triage layout. + - **AzInt/powder mode** — image + **radial profile always on screen**, AzInt settings. Powder + rings only when a unit cell exists (Scanning-SAXS data has none); powder's key role here is + **geometry refinement**, which must remain available. +- **Custom** — drag any dock anywhere, float it, hide it; the arrangement is remembered per + user and restored on launch. A "Reset layout" returns to the active preset. + +A slim **perspective switcher** (segmented control in the top toolbar, or a thin left rail of +icons) flips presets; the menu `Window` becomes show/hide toggles for each dock (the wiring for +this already exists via `toggleViewAction()` in `JFJochViewerMenu::AddDockEntry`). + +### 3.2 Multiple plots, side by side (key new capability) + +Today only **one** metric is plotted at a time (a single combo box). The reviewer wants to +track several at once. Make the plot a **tileable dock type**: each plot dock is an independent +`JFJochDatasetInfoChartView` carrying *its own* metric selector. Open as many as you like; they +tile/stack beside the image. The infrastructure is already multi-instance — `NewDatasetInfo()` +in `JFJochViewerWindow.cpp` already creates additional dataset-info docks and wires them to the +same signals; this generalises that from "extra bottom plots" to "first-class plot grid". + +Default **Processing mode** ships with, e.g., a 2×2 of Spot count · Indexed count · Resolution +estimate · Scale CC, all sharing the run-overlay colours, all clickable to jump to a frame. + +### 3.3 One chart component for dataset *and* per-image plots + +The side panel's separate "Image statistics plot" (`widgets/JFJochViewerSidePanelChart.cpp`, +a per-image radial/profile plot) is folded into the **same** chart component with a +per-image vs per-dataset selector — so a plot dock can show either. This **removes a banner +from the venetian blind** and means one plotting widget to maintain. + +### 3.4 Thumbnail hit feed (nice-to-have) + +A horizontal **strip of recent/strong images** (thumbnail + index, marker for indexed/hit), +dockable, click to open in the image pane. Mirrors the cryoSPARC exposure feed; valuable for +serial data. Lower priority than plots + settings. + +### Recommended default — Processing/monitor mode + +``` +┌───┬───────────────────────────────────────────────────────────────────┐ +│▣ │ ab_c2_new · PILATUS4 2M · 1800 img · Δφ 0.2° ● Live 12 Hz │ context header +│Img│───────────────────────────────────────────────┬───────────────────│ +│▦ │ │ Spot count ╱╲ │ plot grid +│Pro│ DIFFRACTION IMAGE │ ····╱╲····╱╲···· │ (each tile = its +│⚙ │ spots · predictions · rings ├───────────────────┤ own metric, all +│Set│ │ Resolution ▔▔╲ │ clickable, run +│ │ │ ········▁▁▁▁▁▁▁ │ overlays shared) +│ │───────────────────────────────────────────────┼───────────────────┤ +│ │ hit feed ▸ [▤][▤][▤][▤][▤][▤]… click=open │ Scale CC ▁▁▔▔ │ +│ │ Runs ●Original ●Full1 ●Live Processing[New…] │ Indexed ╱▔▔╲ │ +└───┴───────────────────────────────────────────────┴───────────────────┘ +``` + +### Recommended default — Image mode (small screens / pure viewing) + +``` +┌───┬───────────────────────────────────────────────────────────────────┐ +│▣ │ img 742/1800 ⤓ ◂ [742] ▸ ⤒ foreground ──○── Auto HDR 🎨▾ │ image toolbar (icons) +│Img│──────────────────────────────────────────────────┬────────────────│ +│▦ │ │ ▾ Statistics │ accordion inspector +│Pro│ DIFFRACTION IMAGE (large) │ ▾ Overlays │ (rarely-used groups +│⚙ │ spots · predictions · rings │ ▸ Resolution │ collapsed; plots & +│Set│ │ ▸ ROI │ settings hidden) +│ │ │ ▸ Mini-plot │ +└───┴──────────────────────────────────────────────────┴────────────────┘ +``` + +--- + +## 4. Surgical inline settings — MX ⇄ AzInt toggle + +Settings must be **visible**, not buried in the `Window` menu — but only a curated subset; the +full tabbed window stays for the long tail. A compact **Settings dock** with a segmented toggle +at the top — **[ MX ‖ AzInt ]** — which also tells the two user communities apart at a glance. + +**MX page** (feeds `DiffractionExperiment` geometry, `SpotFindingSettings`, `IndexingSettings`, +`ScalingSettings`): +- **Geometry:** beam center x/y · detector distance · photon energy (⇄ wavelength) · rot1/rot2 + detector tilt. +- **Unit cell (a,b,c,α,β,γ) + space group** — **NEW; the viewer has no such input today.** This + is what enables known-cell `ffbidx` indexing (`-C`) and `-S` symmetry. Important gap to close. +- **Spot finding:** SNR (σ) threshold · photon-count threshold · min pixels per spot. +- **Indexing algorithm:** FFBIDX / FFT / FFTW / Auto. +- **Refinement:** none / orientation / beam_and_lattice / pixelrefine. +- **Reference MTZ** (file picker) for reference-driven scaling/merging. + +**AzInt page** (feeds `AzimuthalIntegrationSettings` + a calibrant model): +- q min · q max · q spacing · azimuthal bins · total radial bins. +- **Unit cell** for powder-ring overlay, with a **calibrant preset** dropdown — + **[ Custom · LaB₆ · Ag-behenate · CeO₂ · Si ]** that auto-fills the cell / ring d-spacings + (this extends the existing `widgets/PowderCalibrationWidget.cpp`). Rings need a cell, so they + are optional (Scanning-SAXS has none); the powder calibrant primarily drives **geometry + refinement** (`findBeamCenter`), which stays available regardless of rings. + +The MX/AzInt pages target different groups, so they need not co-exist on one screen — the toggle +is the right model. Changing any field re-runs analysis through the existing +`UpdateSpotFindingSettings` / `UpdateAzintSettings` / `UpdateScalingSettings` worker slots. + +--- + +## 5. Visual system — salmon + navy + +Reviewer-selected second colour: **deep navy/ink `#1F3A5F`** (instrument-grade, high contrast on +blush). Coral stays the identity. + +| Role | Colour | Today | +| --- | --- | --- | +| Canvas | blush `#FFEBE6` | `QPalette::Window` ✓ | +| Brand / grouping | coral `#FA7268` | `TitleLabel` banners ✓ | +| **Action / accent (2nd colour)** | **navy `#1F3A5F`** | replaces teal `#2A9D8F` for buttons/selection/active tab | +| Ink / text | `#2B2B2B` | default | +| State pos/neg | green `#5CB85C` / red `#D9534F` | status bar ✓ | + +- **Coral = identity & section markers** — stop filling 50-px bars; use a slim header (coral + small-caps + a 2-px coral left-rule, or a 24-px tinted strip). Salmon stays the signature; the + venetian blind goes. +- **Navy = anything you click + chrome** — primary buttons, active perspective, selected run, + the "current image" plot marker, the live-rate badge. (Drop teal so two hues don't compete; + keep green/red strictly for state.) +- Entry fields stay white for legibility. +- **Icon toolbars:** one monochrome SVG set (ink default, navy on hover/active) + tooltips; + retire the unicode glyphs and text-only buttons. Min set: first/prev/next/last · play · live · + reanalyze · auto · HDR · colourmap · grid · reset-zoom · new-job · perspective switch. + `QStyle` standard icons can stand in for v1. + +--- + +## 6. Phasing + +**Phase 0 — quick wins (no structural change):** +1. Slim the `TitleLabel` banners. 2. Accordion side panel (collapse Powder/ROI/Data-analysis by +default). 3. Icon toolbars + tooltips. 4. Adopt navy accent on buttons + live badge. 5. Taller +plot dock + "maximise ⤢". + +**Phase 1 — dockable shell + visible settings:** +6. Convert panels to docks with stable `objectName`s; add `saveState/restoreState` to +`QSettings` and two preset perspectives (Image / Processing) + a switcher. 7. Build the inline +**MX⇄AzInt settings dock** (incl. the new unit-cell + space-group input and calibrant presets). +8. Fold the per-image side-panel plot into the reusable chart component. + +**Phase 2 — the monitor:** +9. Multi-plot grid (several `JFJochDatasetInfoChartView` docks, each with its own metric). 10. +Thumbnail hit feed. 11. Mode-aware chrome (hide rotation-/grid-/live-only controls when +irrelevant). 12. Drag-to-threshold triage on plots. + +No working logic is discarded: the chart engine, process controller, settings objects, powder +widget and side-panel signal wiring are all reused — the change is *where things live, how much +room they get, how many you can have at once*, plus the visual layer. + +--- + +## 7. Resolved (design review) +- "Min spot per pixel" = **min pixels per spot** ✓. +- Calibrant presets: **LaB₆ · Ag-behenate · CeO₂ · Si** ✓. +- **Three** perspectives: Image · Processing · **AzInt/powder** (radial profile always; rings + only when a cell exists; powder = geometry refinement) ✓. + +## 8. Future ideas (parked — concept only, solve later) +- **(a) Filtered image navigation.** Today browsing is strictly first→last. Add modes to step + through *indexed images only*, or images with *≥ N spots*, or *next hit* — i.e. navigation + driven by a per-image predicate, not just the frame index. +- **(b) Navigating huge sparse serial datasets.** With ~200k images and rare hits (crystal only + occasionally in the jet), the per-dataset plot is too dense to click the right frame. Need a + smart aid: mouse-wheel **zoom along X** on the plot (Y varies little, so X-only zoom is the + useful axis), and/or a **magnifier**, and/or **snap-to-outlier / next-hit**. Ties into (a). + +## 9. Two hero actions + branding (review note) + +The two capabilities that make Jungfraujoch special must be the eye-catching, unmistakable +controls — bespoke icons, a distinct accent treatment (navy fill / coral highlight, not the +default flat button), and a prominent fixed home (top toolbar or the context header), not buried +among the other buttons: + +- **Reanalyze image** — re-run the embedded pipeline on the *current* image (today: the side + panel "Full analysis" button → `analyze`, and the toolbar "Reanalyze" toggle). +- **Reanalyze dataset** — re-process the *whole* dataset (today: Processing panel "New job…" → + `jfjoch_process`). Rename toward "Reanalyze dataset" wording. + +Design intent: give them purpose-drawn icons (a single-frame glyph vs a stack/grid-of-frames +glyph), one shared accent colour so they read as a pair, and place them together so the +image→dataset relationship is obvious. + +**PSI logo.** Add the PSI logo somewhere unobtrusive but present (user has an SVG). Candidate +homes: a corner of the context header, the bottom-right of the status bar, or the toolbar's +right edge. Drop the SVG at `viewer/resources/psi_logo.svg` and add it to +`viewer/resources/resources.qrc`; render it tinted/monochrome so it sits on the blush canvas. diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 692e9b6e..cd7a7249 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1269,6 +1269,8 @@ namespace { message.jungfrau_conversion_enabled = GetCBORBool(value); else if (key == "geometry_transformation_enabled") message.geometry_transformation_enabled = GetCBORBool(value); + else if (key == "mirror_y") + message.mirror_y = GetCBORBool(value); else if (key == "jungfrau_conversion_factor") message.jungfrau_conversion_factor = GetCBORFloat(value); else if (key == "arm_date") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index ddab842c..936c199c 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -697,6 +697,9 @@ void CBORStream2Serializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC(mapEncoder, "jungfrau_conversion_enabled", message.jungfrau_conversion_enabled); CBOR_ENC(mapEncoder, "jungfrau_conversion_factor", message.jungfrau_conversion_factor); CBOR_ENC(mapEncoder, "geometry_transformation_enabled", message.geometry_transformation_enabled); + // Not a DECTRIS field - stream2 has nothing for the row direction, so a consumer that does not + // know this key simply skips it and gets today's behaviour, which is what absence means. + CBOR_ENC(mapEncoder, "mirror_y", message.mirror_y); CBOR_ENC_PIXEL_MASK(mapEncoder, message); CBOR_ENC_AZINT_MAP(mapEncoder, message); diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index 6f5e234a..65a2cc27 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -701,6 +701,11 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen DetectorSetup detector = DetDECTRIS(image_size_x, image_size_y, detector_name, {}); detector.PixelSize_um(master_file->GetFloat("/entry/instrument/detector/x_pixel_size") * 1e6); + // Whether the stored image is mirrored in Y. A file written before this was recorded is + // mirrored - that is the only thing Jungfraujoch has ever produced - so absence means true. + detector.MirrorY(master_file + ->GetOptBool("/entry/instrument/detector/detectorSpecific/mirror_y") + .value_or(true)); // Sensor thickness/material drive the parallax/absorption model, so take them from the file // rather than the DetectorSetup default (NXmx stores thickness in metres). if (master_file->Exists("/entry/instrument/detector/sensor_thickness")) diff --git a/rugnux/RugnuxCalibration.cpp b/rugnux/RugnuxCalibration.cpp index 78300b67..04a27c4d 100644 --- a/rugnux/RugnuxCalibration.cpp +++ b/rugnux/RugnuxCalibration.cpp @@ -97,29 +97,39 @@ void WritePoniFile(const std::string &path, const DiffractionExperiment &experim // pyFAI integrates sits half a pixel off ours. const double half_pixel_m = 0.5 * pixel_m; f << fmt::format("# Calibration done by Jungfraujoch rugnux {}\n", jfjoch_version()); - f << "poni_version: 2\n"; + // poni_version 2.1 is what pyFAI introduced "orientation" with (pyFAI 2024.01). + f << "poni_version: 2.1\n"; f << "Detector: Detector\n"; - f << fmt::format("Detector_config: {{\"pixel1\": {:g}, \"pixel2\": {:g}, \"max_shape\": [{}, {}]}}\n", + // orientation 2 is pyFAI's "origin at the top left of the image when looking FROM the sample", + // which is the MX convention Jungfraujoch assembles to. Without it pyFAI assumes its own default, + // orientation 3 (bottom left), and quietly believes increasing row means physically upwards. The + // radial integration is identical either way - a mirror preserves 2theta - but the azimuth comes + // out with the opposite sense, which matters for anything that uses chi (cake or sector + // integration, texture). + f << fmt::format("Detector_config: {{\"pixel1\": {:g}, \"pixel2\": {:g}, \"max_shape\": [{}, {}], " + "\"orientation\": 2}}\n", pixel_m, pixel_m, experiment.GetYPixelsNumConv(), experiment.GetXPixelsNumConv()); f << fmt::format("Distance: {:.9g}\n", geom.GetDetectorDistance_mm() * 1e-3); - f << fmt::format("Poni1: {:.9g}\n", geom.GetBeamY_pxl() * pixel_m + half_pixel_m); + // Poni1 is measured from pyFAI's own origin, so declaring orientation 2 re-anchors it to the top + // edge: the same physical point is now (height - 1 - beam_y) rows down from there. + f << fmt::format("Poni1: {:.9g}\n", + (experiment.GetYPixelsNumConv() - 1 - geom.GetBeamY_pxl()) * pixel_m + half_pixel_m); f << fmt::format("Poni2: {:.9g}\n", geom.GetBeamX_pxl() * pixel_m + half_pixel_m); - // Only rot2 changes sign on the way out; rot1 and rot3 do not. The mapping - // (Rot1, Rot2, Rot3) = (+rot1, -rot2, +rot3) - // is pinned empirically against pyFAI itself, not derived from its documentation, which - // contradicts itself on the direction of its axis 2: feeding these values to pyFAI reproduces the - // lab position DiffractionGeometry computes, over the whole detector and each angle separately, to - // 1.4e-17 m. Negating rot3 instead puts a pixel 25 mm out. - // Poni1/Poni2 need no change: they are distances from the corner of the sensor along each axis, - // which the direction the axis runs in does not affect. - // A LaB6 check confirmed the rot2 flip early on (rings sharpen from a peak of 42 to 132, mean - // ring-position error 0.0045 -> 0.0005 1/A) but could not have tested rot3, which is a rotation - // about the beam and so leaves q and 2theta untouched - it moves only the azimuth. + // With orientation declared, rot2 and rot3 change sign and rot1 does not: + // (Rot1, Rot2, Rot3) = (+rot1, +rot2, -rot3) + // A row flip is an improper transformation, so it reverses the sense of rotations about x and + // about the beam while leaving the one about the vertical alone. Pinned empirically against + // pyFAI 2026.5.0: over the whole detector, this reproduces the lab position DiffractionGeometry + // computes to 1.4e-17 m, and an exhaustive search over all four orientations, both Poni1 + // anchorings and all eight sign combinations found exactly two exact solutions - this one, and + // the unlabelled orientation-3 form that was written before. + // A LaB6 check confirmed the earlier rot2 flip but could not have tested rot3, which is a + // rotation about the beam and so moves only the azimuth. f << fmt::format("Rot1: {:.9g}\n", geom.GetPoniRot1_rad()); // negate() rather than a bare minus so an unrefined angle prints as 0 and not -0. const auto negate = [](float v) { return v == 0.0f ? 0.0f : -v; }; - f << fmt::format("Rot2: {:.9g}\n", negate(geom.GetPoniRot2_rad())); - f << fmt::format("Rot3: {:.9g}\n", geom.GetPoniRot3_rad()); + f << fmt::format("Rot2: {:.9g}\n", geom.GetPoniRot2_rad()); + f << fmt::format("Rot3: {:.9g}\n", negate(geom.GetPoniRot3_rad())); f << fmt::format("Wavelength: {:.9g}\n", geom.GetWavelength_A() * 1e-10); f.flush(); if (!f) diff --git a/tests/CalibrationTest.cpp b/tests/CalibrationTest.cpp index a12578c2..82d16b20 100644 --- a/tests/CalibrationTest.cpp +++ b/tests/CalibrationTest.cpp @@ -98,23 +98,28 @@ TEST_CASE("Calibration_PoniFileAxisConvention", "[DetGeomCalib]") { std::remove(path.c_str()); const double pixel_m = geom.GetPixelSize_mm() * 1e-3; - CHECK(keys["poni_version"] == "2"); + CHECK(keys["poni_version"] == "2.1"); + // orientation 2 = "top left seen from the sample", the MX convention we assemble to. Without it + // pyFAI applies its own default (3, bottom left) and gets the azimuth sense backwards. + CHECK(keys["Detector_config"].find("\"orientation\": 2") != std::string::npos); // The half pixel is the origin convention (docs/DETECTOR_GEOMETRY.md): our beam centre is // pixel-centred, pyFAI measures from the edge of the sensor and puts the centre of pixel i at // (i + 0.5) * pixel size. - CHECK(std::stod(keys["Poni1"]) == Catch::Approx(1275.5 * pixel_m)); // slow axis = y + // Declaring orientation 2 anchors Poni1 at the top edge, so the same physical point is + // (height - 1 - beam_y) rows down from it. + CHECK(std::stod(keys["Poni1"]) + == Catch::Approx((x.GetYPixelsNumConv() - 1 - 1275 + 0.5) * pixel_m)); // slow axis = y CHECK(std::stod(keys["Poni2"]) == Catch::Approx(1000.5 * pixel_m)); // fast axis = x CHECK(std::stod(keys["Distance"]) == Catch::Approx(0.150)); - // Only rot2 is NEGATED into pyFAI's frame; rot1 and rot3 are not. Pinned by feeding these exact - // values to pyFAI and checking it reproduces the lab position DiffractionGeometry computes: the - // mapping below agrees to 1.4e-17 m over the whole detector, with each angle tested separately, - // while negating rot3 puts a pixel 25 mm out. Do not "fix" these signs without repeating that - // check against pyFAI itself - its own documentation contradicts itself on the direction of its - // axis 2, and a powder-ring check cannot test rot3, which is a rotation about the beam and moves - // only the azimuth. + // With orientation declared, (Rot1, Rot2, Rot3) = (+rot1, +rot2, -rot3): a row flip is improper, + // so it reverses rotations about x and about the beam and leaves the one about the vertical. + // Pinned against pyFAI 2026.5.0 - an exhaustive search over all four orientations, both Poni1 + // anchorings and all eight sign combinations found exactly two exact solutions, this one and the + // unlabelled orientation-3 form written before. Do not "fix" these signs without repeating that + // search: a powder-ring check cannot test rot3, which moves only the azimuth. CHECK(std::stod(keys["Rot1"]) == Catch::Approx(0.01)); - CHECK(std::stod(keys["Rot2"]) == Catch::Approx(0.02)); - CHECK(std::stod(keys["Rot3"]) == Catch::Approx(0.03)); + CHECK(std::stod(keys["Rot2"]) == Catch::Approx(-0.02)); + CHECK(std::stod(keys["Rot3"]) == Catch::Approx(-0.03)); CHECK(std::stod(keys["Wavelength"]) == Catch::Approx(geom.GetWavelength_A() * 1e-10)); // max_shape is [rows, cols] - the same slow-then-fast order as Poni1/Poni2. const std::string shape = "[" + std::to_string(x.GetYPixelsNumConv()) + ", " diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 525e30c8..0f729e81 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -418,6 +418,10 @@ void NXmx::Detector(const StartMessage &start) { SaveScalar(group, "number_of_cycles", start.summation); HDF5Group det_specific(group, "detectorSpecific"); + // Not NXmx: NXmx states the row direction only through the module axis vectors, which say what + // the geometry is but not how it was arrived at. This records the assembly setting itself, so a + // re-opened file knows whether the stored image was mirrored rather than having to infer it. + SaveScalar(det_specific, "mirror_y", start.mirror_y); det_specific.NXClass("NXcollection"); if (!start.jfjoch_release.empty())