leonarski_f
4a852b4d6b
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 12m57s
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 13m4s
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 11m18s
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 13m12s
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 13m51s
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 13m59s
Build Packages / build:rpm (ubuntu2204) (push) Successful in 10m45s
Build Packages / build:rpm (rocky8) (push) Successful in 12m29s
Build Packages / build:rpm (ubuntu2404) (push) Successful in 12m2s
Build Packages / Generate python client (push) Successful in 24s
Build Packages / XDS test (durin plugin) (push) Successful in 9m50s
Build Packages / Create release (push) Has been skipped
Build Packages / build:rpm (rocky9) (push) Successful in 14m15s
Build Packages / Build documentation (push) Successful in 1m6s
Build Packages / DIALS test (push) Successful in 13m10s
Build Packages / XDS test (JFJoch plugin) (push) Successful in 6m45s
Build Packages / XDS test (neggia plugin) (push) Successful in 5m58s
Build Packages / Unit tests (push) Successful in 1h20m42s
This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. * jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long * jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata * XDS plugin: Add generation of Jungfraujoch plugin for XDS * CI: Add tests with XDS and DIALS (`xia2.ssx`) Reviewed-on: #43
352 lines
13 KiB
YAML
352 lines
13 KiB
YAML
name: Build Packages
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- '**'
|
|
tags:
|
|
- '**'
|
|
pull_request:
|
|
workflow_dispatch:
|
|
inputs:
|
|
create_release:
|
|
type: boolean
|
|
description: 'Create release'
|
|
required: false
|
|
default: false
|
|
|
|
jobs:
|
|
build-rpm:
|
|
name: build:rpm (${{ matrix.distro }})
|
|
if: github.ref_type != 'workflow_dispatch'
|
|
runs-on: ${{ matrix.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- runner: jfjoch_rocky8
|
|
distro: rocky8
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-cuda12/upload
|
|
- runner: jfjoch_rocky8
|
|
distro: rocky8_sls9
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON -DSLS9=ON
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet9-cuda12/upload
|
|
- runner: jfjoch_rocky8
|
|
distro: rocky8_nocuda
|
|
cmake_flags: -DJFJOCH_USE_CUDA=OFF
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el8/slsdet8-nocuda/upload
|
|
- runner: jfjoch_rocky9
|
|
distro: rocky9
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-cuda13/upload
|
|
- runner: jfjoch_rocky9
|
|
distro: rocky9_sls9
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON -DSLS9=ON
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet9-cuda13/upload
|
|
- runner: jfjoch_rocky9
|
|
distro: rocky9_nocuda
|
|
cmake_flags: -DJFJOCH_USE_CUDA=OFF
|
|
pkg_glob: "*.rpm"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/rpm/centos/el9/slsdet8-nocuda/upload
|
|
- runner: jfjoch_ubuntu2204
|
|
distro: ubuntu2204
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON
|
|
pkg_glob: "*.deb"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/jammy/cuda13/upload
|
|
- runner: jfjoch_ubuntu2204
|
|
distro: ubuntu2204_nocuda
|
|
cmake_flags: -DJFJOCH_USE_CUDA=OFF
|
|
pkg_glob: "*.deb"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/jammy/nocuda/upload
|
|
- runner: jfjoch_ubuntu2404
|
|
distro: ubuntu2404
|
|
cmake_flags: -DJFJOCH_USE_CUDA=ON
|
|
pkg_glob: "*.deb"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/noble/cuda13/upload
|
|
- runner: jfjoch_ubuntu2404
|
|
distro: ubuntu2404_nocuda
|
|
cmake_flags: -DJFJOCH_USE_CUDA=OFF
|
|
pkg_glob: "*.deb"
|
|
upload_url: https://gitea.psi.ch/api/packages/mx/debian/pool/noble/nocuda/upload
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup build (cmake)
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DJFJOCH_INSTALL_DRIVER_SOURCE=ON -DJFJOCH_VIEWER_BUILD=ON -DCMAKE_BUILD_TYPE=Release ${{ matrix.cmake_flags }} ..
|
|
- name: Build packages
|
|
shell: bash
|
|
run: |
|
|
cd build
|
|
ninja frontend
|
|
ninja -j16 package
|
|
- name: Upload packages
|
|
if: github.ref_type == 'tag'
|
|
shell: bash
|
|
env:
|
|
TOKEN: ${{ secrets.PIP_REPOSITORY_API_TOKEN }}
|
|
run: |
|
|
set -euo pipefail
|
|
cd build
|
|
shopt -s nullglob
|
|
files=(${{ matrix.pkg_glob }})
|
|
if [ ${#files[@]} -eq 0 ]; then
|
|
echo "No package files found for pattern: ${{ matrix.pkg_glob }}"
|
|
exit 1
|
|
fi
|
|
for file in "${files[@]}"; do
|
|
echo "Uploading $file -> ${{ matrix.upload_url }}"
|
|
curl --fail --user __token__:"$TOKEN" --upload-file "$file" "${{ matrix.upload_url }}"
|
|
done
|
|
|
|
cd ..
|
|
|
|
if [ "${{ matrix.distro }}" = "rocky8_nocuda" ]; then
|
|
for file in build/jfjoch-viewer*.rpm build/jfjoch-writer*rpm build/xds-plugin/libjfjoch_xds_plugin.so.*; do
|
|
python3 gitea_upload_file.py "$file"
|
|
done
|
|
elif [ "${{ matrix.distro }}" = "rocky9_nocuda" ]; then
|
|
for file in build/jfjoch-viewer*.rpm build/jfjoch-writer*rpm; do
|
|
python3 gitea_upload_file.py "$file"
|
|
done
|
|
elif [ "${{ matrix.distro }}" = "ubuntu2204_nocuda" ]; then
|
|
for file in build/jfjoch*viewer*.deb build/jfjoch*writer*.deb; do
|
|
python3 gitea_upload_file.py "$file"
|
|
done
|
|
fi
|
|
dials-test:
|
|
name: DIALS test
|
|
runs-on: jfjoch_rocky9
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build processing test
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
|
|
ninja -j16 jfjoch_hdf5_test
|
|
- name: Run DIALS processing on legacy format
|
|
shell: bash
|
|
run: |
|
|
source /opt/dials-v3-27-0/dials_env.sh
|
|
mkdir -p test01
|
|
cd test01
|
|
../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -o legacy
|
|
xia2.ssx image=legacy_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000
|
|
- name: Run DIALS processing on VDS (master + 4 linked image files)
|
|
shell: bash
|
|
run: |
|
|
source /opt/dials-v3-27-0/dials_env.sh
|
|
mkdir -p test02
|
|
cd test02
|
|
../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -f25 -V -o vds
|
|
xia2.ssx image=vds_master.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000
|
|
- name: Run DIALS processing on single file format
|
|
shell: bash
|
|
run: |
|
|
source /opt/dials-v3-27-0/dials_env.sh
|
|
mkdir -p test03
|
|
cd test03
|
|
../build/tools/jfjoch_hdf5_test ../tests/test_data/compression_benchmark.h5 -n100 -S -o single
|
|
xia2.ssx image=single.h5 space_group=P43212 unit_cell=78.551,78.551,36.914,90.000,90.000,90.000
|
|
xds-durin-test:
|
|
name: XDS test (durin plugin)
|
|
runs-on: jfjoch_rocky9
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build processing test
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
|
|
ninja -j16 jfjoch_hdf5_test
|
|
- name: Run XDS with Durin and legacy HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds_durin
|
|
rm -rf *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
- name: Run XDS with Durin and VDS HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds_durin
|
|
rm -rf *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 -V
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
- name: Run XDS with Durin and integrated HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds_durin
|
|
rm -rf *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
xds-test:
|
|
name: XDS test (JFJoch plugin)
|
|
runs-on: jfjoch_rocky9
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build processing test
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
|
|
ninja -j16 jfjoch_hdf5_test
|
|
ninja -j16 jfjoch_xds_plugin
|
|
- name: Run XDS with legacy HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
- name: Run XDS with VDS HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds
|
|
rm -f *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 -V
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
- name: Run XDS with single HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds
|
|
rm -f *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
xds-neggia-test:
|
|
name: XDS test (neggia plugin)
|
|
runs-on: jfjoch_rocky9
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build processing test
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
|
|
ninja -j16 jfjoch_hdf5_test
|
|
- name: Run XDS with Neggia and legacy HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds_neggia
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
- name: Run XDS with Neggia and single HDF5 format
|
|
shell: bash
|
|
run: |
|
|
cd tests/xds_neggia
|
|
rm -f *.h5 *.LP *.HKL
|
|
../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S
|
|
/opt/xds/xds_par
|
|
test -f IDXREF.LP
|
|
python-client:
|
|
name: Generate python client
|
|
runs-on: jfjoch_rocky8
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build python client
|
|
shell: bash
|
|
run: bash gen_python_client.sh
|
|
- name: Upload Package to Gitea PyPI
|
|
if: github.ref_type == 'tag'
|
|
run: |
|
|
twine upload -u __token__ -p ${{ secrets.CI_PYPI_TOKEN }} --skip-existing dist/*
|
|
twine upload -u __token__ -p ${{ secrets.PIP_REPOSITORY_API_TOKEN }} --repository-url https://gitea.psi.ch/api/packages/mx/pypi dist/*
|
|
documentation:
|
|
name: Build documentation
|
|
runs-on: jfjoch_rocky8
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build documentation
|
|
shell: bash
|
|
run: bash make_doc.sh
|
|
- name: Upload documentation
|
|
if: github.ref_type == 'tag'
|
|
shell: bash
|
|
run: |
|
|
git config --global user.email "filip.leonarski@psi.ch"
|
|
git config --global user.name "Filip Leonarski (Gitea)"
|
|
git checkout --orphan gitea-pages
|
|
git reset --hard
|
|
# Remove everything except .git and public
|
|
git clean -fdx -e public -e .git
|
|
|
|
# Move public contents to root and remove the now-empty public dir
|
|
shopt -s dotglob nullglob
|
|
mv public/* .
|
|
rm -rf public
|
|
git add .
|
|
git commit -m "Deploy site"
|
|
git push -f https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git gitea-pages
|
|
unit-tests:
|
|
name: Unit tests
|
|
runs-on: jfjoch_rocky8
|
|
if: github.ref_type != 'tag' && github.ref_type != 'workflow_dispatch'
|
|
container:
|
|
image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2511
|
|
options: --gpus all
|
|
timeout-minutes: 90
|
|
env:
|
|
CTEST_OUTPUT_ON_FAILURE: '1'
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Build tests
|
|
shell: bash
|
|
run: |
|
|
mkdir -p build
|
|
cd build
|
|
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ..
|
|
ninja -j48 jfjoch_test jfjoch_hdf5_test
|
|
- name: Run unit tests
|
|
shell: bash
|
|
run: |
|
|
cd build/tests
|
|
./jfjoch_test
|
|
- name: Run hdf5 test
|
|
shell: bash
|
|
run: |
|
|
cd build/tools
|
|
./jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5
|
|
|
|
create-release:
|
|
name: Create release
|
|
runs-on: jfjoch_rocky8
|
|
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: 'true' # Optional; should be the default
|
|
- name: Configure auth and fetch LFS
|
|
shell: bash
|
|
env:
|
|
GITEA_TOKEN: ${{ secrets.PIP_REPOSITORY_API_TOKEN }}
|
|
run: |
|
|
git lfs install --local
|
|
AUTH=$(git config --local http.${{ github.server_url }}/.extraheader)
|
|
git config --local --unset http.${{ github.server_url }}/.extraheader
|
|
git config --local http.${{ github.server_url }}/${{ github.repository }}.git/info/lfs/objects/batch.extraheader "$AUTH"
|
|
git lfs pull
|
|
- name: Create release
|
|
env:
|
|
TOKEN: ${{ secrets.PIP_REPOSITORY_API_TOKEN }}
|
|
shell: bash
|
|
run: |
|
|
python3 gitea_create_release.py |