From d55f3257ede4717bcc4975c7f2a0485d5916e84f Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 2 Aug 2026 20:21:50 +0200 Subject: [PATCH] ci: regenerate the published API artefacts, and build Linux with LTO update_version.sh had not been run for the adaptive spot-finding schema change. Running it leaves the C++ server model and the TypeScript client byte-identical to what the generators produced directly, but it also regenerates two artefacts the direct calls do not touch and which are tracked: the Python client's published documentation and the Redoc bundle. Both now carry adaptive_threshold and false_pixels_per_frame. LTO joins -march in the CI flags, which is why MARCH_CMAKE_FLAGS is now LINUX_CMAKE_FLAGS - it no longer describes only the architecture. Measured on rugnux against an otherwise identical build: 7-10% fewer retired instructions and a 9% smaller binary, but only ~1.5% off the wall clock, because the pipeline is GPU- and I/O-bound. It costs about 3x on an incremental rebuild (9.8 s -> 30.1 s for one file plus link), so it stays out of CMakeLists and out of a developer's edit cycle: CI builds from scratch and ships the result, paying the link once. It links against CUDA with no special handling, and both CI images already put gcc-toolset-13 on PATH, which -flto=auto requires. MSVC is left alone: its LTO is a different flag (/GL + /LTCG) and nothing here measured it. Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/build_and_test.yml | 26 +++++++++++++------ CLAUDE.md | 13 ++++++---- broker/redoc-static.html | 19 ++++++++++---- .../python_client/docs/SpotFindingSettings.md | 2 ++ 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index 07d27f5e..6973cca9 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -1,7 +1,17 @@ name: Build Packages env: - MARCH_CMAKE_FLAGS: '-DCMAKE_CXX_FLAGS="-march=x86-64-v3" -DCMAKE_C_FLAGS="-march=x86-64-v3"' + # Architecture and link-time optimisation for every Linux configure. Both are deliberately absent + # from CMakeLists.txt so a site can pick its own (x86-64-v4 on an AVX-512 cluster, -march=native, + # or the plain baseline); CI pins what it ships. + # + # -flto=auto needs GCC >= 10 and both CI images put gcc-toolset-13 on PATH. Measured on rugnux + # against an otherwise identical build: 7-10% fewer retired instructions and a 9% smaller binary, + # but only ~1.5% off the wall clock, because the offline pipeline is GPU- and I/O-bound rather + # than CPU-instruction-bound. It costs about 3x on an INCREMENTAL rebuild (one file plus link, + # 9.8 s -> 30.1 s), which is why it belongs here and not in CMakeLists: CI always builds from + # scratch and ships the result, so it pays the link once and a developer never pays it at all. + LINUX_CMAKE_FLAGS: '-DCMAKE_CXX_FLAGS="-march=x86-64-v3 -flto=auto" -DCMAKE_C_FLAGS="-march=x86-64-v3 -flto=auto"' # MSVC has no spelling for the x86-64-v2 level; /arch:AVX is the nearest and implies SSE4.1/4.2, # which is what matters here - without it Eigen has no vectorised round and falls back to a libm # call per element. AVX is Sandy Bridge (2011) and up, a safe floor for a desktop viewer. @@ -50,7 +60,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.MARCH_CMAKE_FLAGS }} .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. ninja -j48 jfjoch_test jfjoch_hdf5_test jfjoch_hdf5_enospc_test enospc_shim - name: Run unit tests shell: bash @@ -137,7 +147,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DJFJOCH_VIEWER_ONLY=ON -DJFJOCH_USE_CUDA=${{ matrix.use_cuda }} ${{ env.MARCH_CMAKE_FLAGS }} -DCMAKE_BUILD_TYPE=Release .. + cmake -G Ninja -DJFJOCH_VIEWER_ONLY=ON -DJFJOCH_USE_CUDA=${{ matrix.use_cuda }} ${{ env.LINUX_CMAKE_FLAGS }} -DCMAKE_BUILD_TYPE=Release .. - name: Build viewer tgz shell: bash run: | @@ -236,7 +246,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DJFJOCH_INSTALL_DRIVER_SOURCE=ON -DJFJOCH_VIEWER_BUILD=ON ${{ env.MARCH_CMAKE_FLAGS }} -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flags }} .. + cmake -G Ninja -DJFJOCH_INSTALL_DRIVER_SOURCE=ON -DJFJOCH_VIEWER_BUILD=ON ${{ env.LINUX_CMAKE_FLAGS }} -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flags }} .. - name: Build packages shell: bash run: | @@ -294,7 +304,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.MARCH_CMAKE_FLAGS }} .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. ninja -j16 jfjoch_hdf5_test - name: Run DIALS processing on legacy format shell: bash @@ -330,7 +340,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.MARCH_CMAKE_FLAGS }} .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. ninja -j16 jfjoch_hdf5_test - name: Run XDS with Durin and legacy HDF5 format shell: bash @@ -366,7 +376,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.MARCH_CMAKE_FLAGS }} .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. ninja -j16 jfjoch_hdf5_test ninja -j16 jfjoch_xds_plugin - name: Run XDS with legacy HDF5 format @@ -402,7 +412,7 @@ jobs: run: | mkdir -p build cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.MARCH_CMAKE_FLAGS }} .. + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. ninja -j16 jfjoch_hdf5_test - name: Run XDS with Neggia and legacy HDF5 format shell: bash diff --git a/CLAUDE.md b/CLAUDE.md index c043d8ec..5720a3a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,13 +43,16 @@ Key CMake options: - `SLS9` (default OFF) — build against slsDetectorPackage 9.2.0 instead of 8.0.2. - `JFJOCH_INSTALL_DRIVER_SOURCE` (default OFF) — install the PCIe driver source for DKMS/RPM. -### `-march` is deliberately NOT set in CMakeLists +### `-march` and LTO are deliberately NOT set in CMakeLists -The build system sets no architecture flags, so a site can pick its own — `x86-64-v4` on an +The build system sets no architecture or LTO flags, so a site can pick its own — `x86-64-v4` on an AVX-512 cluster, `-march=native`, or the plain baseline. **CI passes them explicitly** -(`.gitea/workflows/build_and_test.yml`): every Linux configure gets -`-march=x86-64-v3`, and the MSVC viewer job gets `/arch:AVX` (MSVC has no x86-64-v2 spelling; -`/arch:AVX` is the nearest and implies SSE4.1/4.2). +(`.gitea/workflows/build_and_test.yml`): every Linux configure gets `LINUX_CMAKE_FLAGS` +(`-march=x86-64-v3 -flto=auto`), and the MSVC viewer job gets `/arch:AVX` (MSVC has no x86-64-v2 +spelling; `/arch:AVX` is the nearest and implies SSE4.1/4.2). LTO is CI-only on purpose: measured on +rugnux it removes 7–10% of retired instructions and 9% of the binary but only ~1.5% of the wall +clock (the pipeline is GPU- and I/O-bound), while costing ~3× on an incremental rebuild — a bad +trade for a developer, a fine one for a build that happens once and is shipped. **This matters when profiling.** A plain `cmake -DCMAKE_BUILD_TYPE=Release ..` produces a *baseline* binary that is not what CI or production runs, and the difference is not uniform: diff --git a/broker/redoc-static.html b/broker/redoc-static.html index 973da47c..533c4e79 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -716,16 +716,25 @@ as far as the detector reaches, i.e. the detection is not clipped in resolution. If enabled it will likely reduce performance of Jungfraujoch for datasets with a very high indexing rate. (experimental feature)

ice_ring_width_q_recipA
required
number <float> [ 0 .. 1 ]
Default: 0.02

Width of ice ring in q-space in reciprocal space

-
high_res_gap_Q_recipA
number <float> [ 0.1 .. 5 ]
Default: 1.5

This parameter is used to remove spurious spots at a very high resolution, that sometimes appear due to very low background close to the edge of the detector. +

high_res_gap_Q_recipA
number <float> [ 0.1 .. 5 ]
Default: 1.5

This parameter is used to remove spurious spots at a very high resolution, that sometimes appear due to very low background close to the edge of the detector. If there is a gap in (1/d)-space between spots of at least this size, spots on the side of the gap with high resolution will be discarded. This is optional parameter. This option should be turned OFF for small molecule datasets or for crystals with very low mosaicity, when it is expected to see only few spots in any case.

+
adaptive_threshold
boolean
Default: false

Self-calibrating spot detection: replace the fixed photon_count_threshold by a per-resolution-ring +threshold derived from each image's own noise, so the same setting works across datasets without +per-dataset tuning. photon_count_threshold is then ignored and false_pixels_per_frame sets the +operating point instead. +Only available on detectors whose images are analysed in software (the DECTRIS/SIMPLON workflow). +The JUNGFRAU and EIGER workflows find spots on the FPGA, which applies its own fixed threshold, +so enabling this there is rejected rather than silently ignored.

+
false_pixels_per_frame
number <float> [ 1 .. 100000 ]
Default: 100

Operating point of the adaptive threshold: the number of noise pixels tolerated per frame. +About 100 suits a multi-megapixel detector. Ignored unless adaptive_threshold is set.

Responses

Request samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5
}

Get data processing configuration

Can be done anytime

+
http://localhost:5232/config/spot_finding

Request samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5,
  • "adaptive_threshold": false,
  • "false_pixels_per_frame": 100
}

Get data processing configuration

Can be done anytime

Responses

Response samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5
}

Configure azimuthal integration

Can be done when detector is Inactive or Idle

+
http://localhost:5232/config/spot_finding

Response samples

Content type
application/json
{
  • "enable": true,
  • "indexing": true,
  • "signal_to_noise_threshold": 0.1,
  • "photon_count_threshold": 0,
  • "min_pix_per_spot": 1,
  • "max_pix_per_spot": 1,
  • "high_resolution_limit": 0.1,
  • "low_resolution_limit": 0.1,
  • "high_resolution_limit_for_spot_count_low_res": 2,
  • "quick_integration": false,
  • "ice_ring_width_q_recipA": 0.02,
  • "high_res_gap_Q_recipA": 1.5,
  • "adaptive_threshold": false,
  • "false_pixels_per_frame": 100
}

Configure azimuthal integration

Can be done when detector is Inactive or Idle

Request Body schema: application/json
polarization_corr
required
boolean
Default: true

Apply polarization correction for azimuthal integration (polarization factor must be configured in dataset settings)

solid_angle_corr
required
boolean
Default: true

Apply solid angle correction for azimuthal integration

high_q_recipA
number <float> [ 0.00002 .. 10 ]

Upper q limit of the azimuthal integration [1/Angstrom]. Optional: if omitted, the integration @@ -844,7 +853,7 @@ This can only be done when detector is Idle, Error or

Request samples

Content type
application/json
{
  • "box": {
    },
  • "circle": {
    },
  • "azim": {
    }
}

Response samples

Content type
application/json
{
  • "msg": "Detector in wrong state",
  • "reason": "WrongDAQState"
}

Get general statistics

Responses

Response samples

Content type
application/json
{
  • "detector": {
    },
  • "detector_list": {
    },
  • "detector_settings": {
    },
  • "image_format_settings": {
    },
  • "instrument_metadata": {
    },
  • "file_writer_settings": {
    },
  • "data_processing_settings": {
    },
  • "measurement": {
    },
  • "broker": {
    },
  • "fpga": [
    ],
  • "calibration": [
    ],
  • "zeromq_preview": {
    },
  • "zeromq_metadata": {
    },
  • "dark_mask": {
    },
  • "pixel_mask": {
    },
  • "roi": {
    },
  • "az_int": {
    },
  • "buffer": {
    },
  • "indexing": {
    },
  • "bragg_integration": {
    },
  • "image_pusher": {
    }
}

Get data collection statistics

Results of the last data collection

+
http://localhost:5232/statistics

Response samples

Content type
application/json
{
  • "detector": {
    },
  • "detector_list": {
    },
  • "detector_settings": {
    },
  • "image_format_settings": {
    },
  • "instrument_metadata": {
    },
  • "file_writer_settings": {
    },
  • "data_processing_settings": {
    },
  • "measurement": {
    },
  • "broker": {
    },
  • "fpga": [
    ],
  • "calibration": [
    ],
  • "zeromq_preview": {
    },
  • "zeromq_metadata": {
    },
  • "dark_mask": {
    },
  • "pixel_mask": {
    },
  • "roi": {
    },
  • "az_int": {
    },
  • "buffer": {
    },
  • "indexing": {
    },
  • "bragg_integration": {
    },
  • "image_pusher": {
    }
}

Get data collection statistics

Results of the last data collection

Responses