diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index 0982ec085..0b5a95c23 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -49,7 +49,7 @@ jobs: # already been tested by the dispatch run it came from. if: github.ref_type != 'tag' container: - image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2511 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b options: --gpus all timeout-minutes: 90 env: @@ -72,7 +72,7 @@ jobs: mkdir -p build cd build cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. - ninja -j48 jfjoch_test jfjoch_hdf5_test jfjoch_hdf5_enospc_test enospc_shim + ninja -j16 jfjoch_test jfjoch_hdf5_test jfjoch_hdf5_enospc_test enospc_shim - name: Run unit tests shell: bash run: | @@ -138,6 +138,8 @@ jobs: build-viewer-tgz: name: build:viewer-tgz:${{ matrix.variant }} runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b timeout-minutes: 120 strategy: fail-fast: false @@ -191,6 +193,8 @@ jobs: build-rugnux-tgz: name: build:rugnux-tgz (x86_64) runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b timeout-minutes: 120 steps: - uses: actions/checkout@v4 @@ -296,7 +300,7 @@ jobs: # rebuilding and re-pushing the image, and a fix to it would appear to have no effect. runs-on: jfjoch_ubuntu2404 container: - image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2404:2508 + image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2404:2608b steps: - uses: actions/checkout@v4 - name: Clean previous build @@ -368,56 +372,68 @@ jobs: build-rpm: name: build:rpm (${{ matrix.distro }}) runs-on: ${{ matrix.runner }} + container: + image: ${{ matrix.image }} strategy: fail-fast: false matrix: include: - runner: jfjoch_rocky8 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b distro: rocky8 cmake_flags: -DJFJOCH_USE_CUDA=ON pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12/upload - runner: jfjoch_rocky8 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b distro: rocky8_sls9 cmake_flags: -DJFJOCH_USE_CUDA=ON -DSLS9=ON pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12/upload - runner: jfjoch_rocky8 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b distro: rocky8_nocuda cmake_flags: -DJFJOCH_USE_CUDA=OFF pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda/upload - runner: jfjoch_rocky9 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky9:2607b distro: rocky9 cmake_flags: -DJFJOCH_USE_CUDA=ON pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13/upload - runner: jfjoch_rocky9 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky9:2607b distro: rocky9_sls9 cmake_flags: -DJFJOCH_USE_CUDA=ON -DSLS9=ON pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13/upload - runner: jfjoch_rocky9 + image: gitea.psi.ch/leonarski_f/jfjoch_rocky9:2607b distro: rocky9_nocuda cmake_flags: -DJFJOCH_USE_CUDA=OFF pkg_glob: "*.rpm" upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda/upload - runner: jfjoch_ubuntu2204 + image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2204:2608b distro: ubuntu2204 cmake_flags: -DJFJOCH_USE_CUDA=ON pkg_glob: "*.deb" upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/jammy/cuda13/upload - runner: jfjoch_ubuntu2204 + image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2204:2608b distro: ubuntu2204_nocuda cmake_flags: -DJFJOCH_USE_CUDA=OFF pkg_glob: "*.deb" upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/jammy/nocuda/upload - runner: jfjoch_ubuntu2404 + image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2404:2608b distro: ubuntu2404 cmake_flags: -DJFJOCH_USE_CUDA=ON pkg_glob: "*.deb" upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/noble/cuda13/upload - runner: jfjoch_ubuntu2404 + image: gitea.psi.ch/leonarski_f/jfjoch_ubuntu2404:2608b distro: ubuntu2404_nocuda cmake_flags: -DJFJOCH_USE_CUDA=OFF pkg_glob: "*.deb" @@ -481,9 +497,11 @@ jobs: python3 gitea_upload_file.py "$file" done fi - dials-test: - name: DIALS test + hdf5-consumer-tests: + name: HDF5 consumer tests (DIALS, XDS) runs-on: jfjoch_rocky9 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky9:2607b steps: - uses: actions/checkout@v4 - name: Build processing test @@ -492,157 +510,99 @@ jobs: mkdir -p build cd build cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. - ninja -j16 jfjoch_hdf5_test - # dials.import before xia2.ssx, checking the image COUNT: dxtbx reports a short scan rather - # than failing when it cannot reach every image, and xia2.ssx still succeeds on the images it - # did get, so an exit code alone does not notice. Multi-file NXmxLegacy is exercised by the - # XDS jobs below but deliberately not here: dxtbx reads only the first data file of a legacy - # master (it takes one dataset out of NXdata and indexes it globally), so that combination - # cannot pass until the limitation is fixed upstream. - - name: Run DIALS processing on legacy format + ninja -j16 jfjoch_hdf5_test jfjoch_xds_plugin + # One job for what used to be four, each of which configured and built the same + # jfjoch_hdf5_test on the same image. Every case still runs when an earlier one fails -- each + # is a subshell under `set -e`, so a case stops at its own first bad command without taking + # the rest down -- and the summary at the end decides the job's result. + - name: Run every consumer against every file layout shell: bash run: | - source /opt/dials-v3-27-0/dials_env.sh - mkdir -p test01 - cd test01 - ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -o legacy - dials.import legacy_master.h5 | tee import.log - grep -q "num images: 100" import.log || { echo "DIALS did not read all 100 images"; exit 1; } - xia2.ssx image=legacy_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 - - name: Run DIALS processing on VDS (master + 4 linked image files) - shell: bash - run: | - source /opt/dials-v3-27-0/dials_env.sh - mkdir -p test02 - cd test02 - ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -f25 -V -o vds - dials.import vds_master.h5 | tee import.log - grep -q "num images: 100" import.log || { echo "DIALS did not read all 100 images"; exit 1; } - xia2.ssx image=vds_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 - - name: Run DIALS processing on single file format - shell: bash - run: | - source /opt/dials-v3-27-0/dials_env.sh - mkdir -p test03 - cd test03 + cd "$GITHUB_WORKSPACE" + HDF5_TEST=$PWD/build/tools/jfjoch_hdf5_test + DATA=$PWD/tests/test_data/compression_benchmark.h5 + # Unquoted on use: these are two arguments, not one. + CELL="space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000" + + # -- DIALS ------------------------------------------------------------------------- + # dials.import before xia2.ssx, checking the image COUNT: dxtbx reports a short scan + # rather than failing when it cannot reach every image, and xia2.ssx still succeeds on + # the images it did get, so an exit code alone does not notice. Multi-file NXmxLegacy is + # exercised by the XDS cases below but deliberately not here: dxtbx reads only the first + # data file of a legacy master (it takes one dataset out of NXdata and indexes it + # globally), so that combination cannot pass until the limitation is fixed upstream. + dials_case() { # + local dir=$1 prefix=$2; shift 2 + source /opt/dials-v3-27-0/dials_env.sh + rm -rf "$dir"; mkdir -p "$dir"; cd "$dir" + "$HDF5_TEST" "$DATA" "$@" -o "$prefix" + dials.import "${prefix}_master.h5" | tee import.log + grep -q "num images: 100" import.log || { echo "DIALS did not read all 100 images"; return 1; } + xia2.ssx image="${prefix}_master.h5" $CELL + } + dials_legacy() { dials_case test01 legacy -n100; } + dials_vds() { dials_case test02 vds -n100 -f25 -V; } # -S writes _master.h5, not .h5 - naming this wrong meant xia2.ssx was # pointed at a file that never existed, so the integrated format went untested. - ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -S -o single - dials.import single_master.h5 | tee import.log - grep -q "num images: 100" import.log || { echo "DIALS did not read all 100 images"; exit 1; } - xia2.ssx image=single_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000 - # Everything above writes an UNTILTED detector, where every plausible encoding of the tilt - # agrees exactly - so none of it constrains the NXmx transformation chain. This does. - - name: Check tilted detector geometry against DIALS - shell: bash - run: | - source /opt/dials-v3-27-0/dials_env.sh - mkdir -p test04 - cd test04 - ../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n1 -S -o geom - dials.python ../tests/nxmx_geometry_dials_test.py geom_master.h5 - xds-durin-test: - name: XDS test (durin plugin) - runs-on: jfjoch_rocky9 - steps: - - uses: actions/checkout@v4 - - name: Build processing test - shell: bash - run: | - mkdir -p build - cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. - ninja -j16 jfjoch_hdf5_test - - name: Run XDS with Durin and legacy HDF5 format - shell: bash - run: | - cd tests/xds_durin - rm -rf *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 - /opt/xds/xds_par - test -f IDXREF.LP - - name: Run XDS with Durin and VDS HDF5 format - shell: bash - run: | - cd tests/xds_durin - rm -rf *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 -V - /opt/xds/xds_par - test -f IDXREF.LP - - name: Run XDS with Durin and integrated HDF5 format - shell: bash - run: | - cd tests/xds_durin - rm -rf *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S - /opt/xds/xds_par - test -f IDXREF.LP - xds-test: - name: XDS test (JFJoch plugin) - runs-on: jfjoch_rocky9 - steps: - - uses: actions/checkout@v4 - - name: Build processing test - shell: bash - run: | - mkdir -p build - cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. - ninja -j16 jfjoch_hdf5_test - ninja -j16 jfjoch_xds_plugin - - name: Run XDS with legacy HDF5 format - shell: bash - run: | - cd tests/xds - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 - /opt/xds/xds_par - test -f IDXREF.LP - - name: Run XDS with VDS HDF5 format - shell: bash - run: | - cd tests/xds - rm -f *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 -V - /opt/xds/xds_par - test -f IDXREF.LP - - name: Run XDS with single HDF5 format - shell: bash - run: | - cd tests/xds - rm -f *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S - /opt/xds/xds_par - test -f IDXREF.LP - xds-neggia-test: - name: XDS test (neggia plugin) - runs-on: jfjoch_rocky9 - steps: - - uses: actions/checkout@v4 - - name: Build processing test - shell: bash - run: | - mkdir -p build - cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. - ninja -j16 jfjoch_hdf5_test - - name: Run XDS with Neggia and legacy HDF5 format - shell: bash - run: | - cd tests/xds_neggia - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 - /opt/xds/xds_par - test -f IDXREF.LP - - name: Run XDS with Neggia and single HDF5 format - shell: bash - run: | - cd tests/xds_neggia - rm -f *.h5 *.LP *.HKL - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S - /opt/xds/xds_par - test -f IDXREF.LP + dials_single() { dials_case test03 single -n100 -S; } + # Everything above writes an UNTILTED detector, where every plausible encoding of the tilt + # agrees exactly - so none of it constrains the NXmx transformation chain. This does. + dials_tilted() { + source /opt/dials-v3-27-0/dials_env.sh + rm -rf test04; mkdir -p test04; cd test04 + "$HDF5_TEST" "$DATA" -n1 -S -o geom + dials.python "$GITHUB_WORKSPACE/tests/nxmx_geometry_dials_test.py" geom_master.h5 + } + + # -- XDS, once per read plugin ----------------------------------------------------- + # Each case clears its directory first. The runner reuses the workspace volume, so a + # leftover .h5/.LP from an earlier run could let XDS "pass" on files this case never + # wrote -- which is what the first case of two of these jobs used to risk. + xds_case() { # + local dir=$1; shift + cd "tests/$dir" + rm -f ./*.h5 ./*.LP ./*.HKL + "$HDF5_TEST" "$DATA" "$@" + /opt/xds/xds_par + test -f IDXREF.LP + } + durin_legacy() { xds_case xds_durin -n25 -f10; } + durin_vds() { xds_case xds_durin -n25 -f10 -V; } + durin_single() { xds_case xds_durin -n25 -S; } + jfjoch_legacy() { xds_case xds -n25 -f10; } + jfjoch_vds() { xds_case xds -n25 -f10 -V; } + jfjoch_single() { xds_case xds -n25 -S; } + # Neggia cannot read a VDS master, so it gets the two layouts it supports. + neggia_legacy() { xds_case xds_neggia -n25 -f10; } + neggia_single() { xds_case xds_neggia -n25 -S; } + + fail=0 + summary="" + for t in dials_legacy dials_vds dials_single dials_tilted \ + durin_legacy durin_vds durin_single \ + jfjoch_legacy jfjoch_vds jfjoch_single \ + neggia_legacy neggia_single; do + echo "::group::$t" + # Run the subshell as a command of its own and test $? afterwards. A subshell used + # directly as an `if` condition has its `set -e` ignored, so the case would carry on + # past its first failure and report only the last command's status. + ( set -e; cd "$GITHUB_WORKSPACE"; "$t" ) + rc=$? + if [ "$rc" -eq 0 ]; then + summary="$summary PASS $t"$'\n' + else + summary="$summary FAIL $t (exit $rc)"$'\n'; fail=1 + fi + echo "::endgroup::" + done + echo "===== HDF5 consumer tests =====" + printf '%s' "$summary" + exit "$fail" python-client: name: Generate python client runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b steps: - uses: actions/checkout@v4 - name: Build python client @@ -656,6 +616,8 @@ jobs: documentation: name: Build documentation runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b steps: - uses: actions/checkout@v4 - name: Build documentation @@ -682,6 +644,8 @@ jobs: create-release: name: Create release runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2607b if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) steps: - uses: actions/checkout@v4