FIX: retrograde git to 2.42.0 for GH

FIX: move spack-psi bye one folder

See merge request lsm-hpce/spack-psi!8

.gitea/workflows/ippl_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: IPPL GH Merlin7 Scheduler
+run-name: IPPL GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/ippl_gh.sbatch
+        sbatch .gitea/workflows/sbatch/ippl_gh.sbatch

.gitea/workflows/ippl_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: IPPL GPU Merlin7 Scheduler
+run-name: IPPL GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/ippl_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/ippl_gpu.sbatch

.gitea/workflows/opal-x_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: OPAL-X GH Merlin7 Scheduler
+run-name: OPAL-X GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/opal-x_gh.sbatch
+        sbatch .gitea/workflows/sbatch/opal-x_gh.sbatch

.gitea/workflows/opal-x_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: OPAL-X-GPU Merlin7 Scheduler
+run-name: OPAL-X-GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/opal-x_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/opal-x_gpu.sbatch

.gitea/workflows/q-e_cpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-CPU Merlin7 Scheduler
+run-name: QE-CPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch CPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_cpu.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_cpu.sbatch

.gitea/workflows/q-e_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-GH Merlin7 Scheduler
+run-name: QE-GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_gh.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_gh.sbatch

.gitea/workflows/q-e_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-GPU Merlin7 Scheduler
+run-name: QE-GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_gpu.sbatch

.gitea/workflows/sbatch/ippl_gh.sbatch

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --job-name="ippl-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module purge
+module use Spack unstable
+module load gcc/13.2.0 openmpi/5.0.3-3lmi-GH200-gpu  
+module load boost/1.82.0-3ns6 fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.7.1 h5hut/2.0.0rc7 openblas/0.3.26 cmake/3.31.4-u2nm
+
+git clone https://github.com/IPPL-framework/ippl.git
+cd ippl
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_HOPPER90=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=ON -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/ippl_gpu.sbatch

@@ -0,0 +1,30 @@
+#!/bin/bash
+#SBATCH --job-name="ippl-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/13.2.0 openmpi/4.1.6-57rc-A100-gpu                 
+module load boost/1.82.0-e7gp fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.8 h5hut/2.0.0rc7 openblas/0.3.26-omp cmake/3.31.6-oe7u
+
+git clone https://github.com/IPPL-framework/ippl.git
+cd ippl
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=ON -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/opal-x_gh.sbatch

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name="opal-x-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module purge
+module use Spack unstable
+module load gcc/13.2.0 openmpi/5.0.3-3lmi-GH200-gpu  
+module load boost/1.82.0-3ns6 fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.7.1 h5hut/2.0.0rc7 openblas/0.3.26 cmake/3.31.4-u2nm
+
+git clone https://gitlab.psi.ch/OPAL/opal-x/src.git opal-x
+cd opal-x
+./gen_OPALrevision
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_HOPPER90=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=OFF -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/opal-x_gpu.sbatch

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --job-name="opal-x-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/13.2.0 openmpi/4.1.6-57rc-A100-gpu                 
+module load boost/1.82.0-e7gp fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.8 h5hut/2.0.0rc7 openblas/0.3.26-omp cmake/3.31.6-oe7u
+
+git clone https://gitlab.psi.ch/OPAL/opal-x/src.git opal-x
+cd opal-x
+./gen_OPALrevision
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=OFF -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/q-e_cpu.sbatch

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-CPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tmerlin7
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/12.3 openmpi/main-syah fftw/3.3.10.6-omp hdf5/1.14.5-t46c openblas/0.3.29-omp cmake/3.31.6-oe7u
+
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitea/workflows/sbatch/q-e_gh.sbatch

@@ -0,0 +1,27 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load nvhpc/25.3 openmpi/5.0.7-e3bf-GH200-gpu fftw/3.3.10-sfpw-omp hdf5/develop-2.0-ztvo nvpl-blas/0.4.0.1-3zpg nvpl-lapack/0.3.0-ymy5  netlib-scalapack/2.2.2-qrhq cmake/3.31.6-5dl7
+
+module load git/2.42.0-y4e4
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DQE_ENABLE_SCALAPACK:BOOL=ON -DQE_ENABLE_CUDA:BOOL=ON -DQE_ENABLE_MPI_GPU_AWARE:BOOL=ON -DQE_ENABLE_OPENACC:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitea/workflows/sbatch/q-e_gpu.sbatch

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load nvhpc/25.3 openmpi/main-6bnq-A100-gpu fftw/3.3.10.6-qbxu-omp hdf5/develop-2.0-rjgu netlib-scalapack/2.2.2-3hgw cmake/3.31.6-oe7u
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DQE_ENABLE_SCALAPACK:BOOL=ON -DQE_ENABLE_CUDA:BOOL=ON -DQE_ENABLE_MPI_GPU_AWARE:BOOL=ON -DQE_ENABLE_OPENACC:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitlab-ci.yml

@@ -1,32 +0,0 @@
-stages: [build_test]
-
-variables:
-  GIT_STRATEGY: clone
-  MAX_JOB_NB: 4
-  SPACK_BRANCH: develop
-  SPACK_PCKG: ffbidx
-  SPACK_PCKG_VERSION: main
-  SPACK_REPO: https://github.com/spack/spack.git
-
-build_test:
-  tags:
-    - merlin
-  stage: build_test
-  script:
-    - git clone ${SPACK_REPO} -b ${SPACK_BRANCH}
-    - . "./spack/share/spack/setup-env.sh"
-    - source /opt/psi/config/profile.bash
-    - module load git-lfs
-    - spack clean -a # Make sure spack doesn't have any caches left
-    - rm -rf /scratch/${USER}/spack-dev/${SPACK_PCKG} # Remove previous packages
-    - spack uninstall -Ray
-    - SPACK_PCKG_LOWERCASE=$(echo "${SPACK_PCKG,,}")
-    - if [[ "$SPACK_PCKG_LOWERCASE" == "hiqsimulator" ]]; then
-    - SPACK_PCKG_LOWERCASE="py-$SPACK_PCKG_LOWERCASE"
-    - fi
-    - spack env activate --without-view env/$(spack arch -o)/${SPACK_PCKG_LOWERCASE}
-    - spack develop --path /scratch/\$user/spack-dev/${SPACK_PCKG} --force ${SPACK_PCKG_LOWERCASE}@${SPACK_PCKG_VERSION}
-    - spack install -j ${MAX_JOB_NB}
-    - spack load
-#    - JOB_ID=$(sbatch --parsable ../test/run.slurm)
-#    - cat slurm-${JOB_ID}.out

README.md

@@ -2,7 +2,7 @@
 
 [Official Spack documentation](https://spack.readthedocs.io/en/latest/)
 
-## How to use on Merlin systems (Merlin6/Merlin7)
+## How to use on Merlin systems (Merlin6/Merlin7/Ra)
 
 Load the spack module
 

env/rhel7/x86_64/admin/stable/spack.yaml

@@ -14,13 +14,17 @@ spack:
       - opal@master/l4dkgyf
   include:
   - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
         tcl: $spack/opt/spack/stable/modulefiles
+      lmod:
+        core_compilers:
+        - gcc@=4.8.5
   specs:
   - matrix:
     - [$compilers]

env/rhel7/x86_64/admin/testing/spack.yaml

@@ -1,4 +1,3 @@
----
 spack:
   concretizer:
     unify: false
@@ -9,12 +8,14 @@ spack:
       padded_length: 256
   include:
   - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/compilers.yaml
+  #include_concrete:
+  #- /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/testing
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
         tcl: $spack/opt/spack/testing/modulefiles
-  specs: []
+  specs:
+  - opencv
   view: false

env/rhel7/x86_64/admin/unstable/spack.yaml

@@ -12,6 +12,7 @@ spack:
       - ffbidx@main/vyjua6z
       - sratoolkit@3.1.1/2szh2jy
       - star@2.7.11a/icumpki
+      - opencv@4.10.0/7fjszca
     - gcc_12.3.0_openmpi:
       - cp2k@2024.1/2bayb45 # ~cuda
       - cp2k@2024.1/exkrpix # +cuda
@@ -22,13 +23,14 @@ spack:
       - gromacs@2024.1/pi5t2pp # ~cuda
   include:
     - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/unstable/modulefiles
+        lmod: $spack/opt/spack/unstable/modulefiles
   specs:
   - matrix:
     - [$gcc_12.3.0]

env/rhel7/x86_64/sysconfigs/packages.yaml

@@ -69,8 +69,6 @@ packages:
     require: '%gcc'
   freetype:
     require: '%gcc@4.8.5'
-  gams:
-    require: '%gcc@4.8.5'
   gawk:
     require: '%gcc@4.8.5'
   gcc:
@@ -147,7 +145,7 @@ packages:
   libxml2:
     require: '%gcc@4.8.5'
   libxpm:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   libxt:
     require: '%gcc'
   m4:
@@ -171,7 +169,7 @@ packages:
       prefix: /usr
   openmpi:
     require:
-      - spec: schedulers=slurm fabrics=auto pmi-interface=pmi2 +cxx +cuda +legacylaunchers
+      - spec: schedulers=slurm fabrics=auto +internal-pmix +cxx +cuda +legacylaunchers
       - spec: '+cxx_exceptions'
         when: '%gcc'
   openssh:

env/rhel7/x86_64/users/stable/spack.yaml

@@ -3,12 +3,14 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
   specs: []
   upstreams:
     stable:
       install_tree: >-
         $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/rhel7/x86_64/users/unstable/spack.yaml

@@ -3,14 +3,17 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
   specs: []
   upstreams:
     stable:
       install_tree: $spack/opt/spack/stable
     unstable:
       install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/rhel8/x86_64/admin/stable/spack.yaml

@@ -13,8 +13,9 @@ spack:
     - gcc_12.3.0_openmpi:
       - opal@master/l4dkgyf
   include:
-  - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:

env/rhel8/x86_64/admin/testing/spack.yaml

@@ -8,8 +8,9 @@ spack:
       # Strongly adviced to avoid problems with buildcaches
       padded_length: 256
   include:
-    - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/testing
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:

env/rhel8/x86_64/admin/unstable/spack.yaml

@@ -22,7 +22,8 @@ spack:
       - gromacs@2024.1/pi5t2pp # ~cuda
   include:
     - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:

env/rhel8/x86_64/sysconfigs/packages.yaml

@@ -166,7 +166,7 @@ packages:
     require: '%gcc'
   openmpi:
     require:
-      - spec: schedulers=slurm fabrics=auto pmi-interface=pmix +cxx +cuda +legacylaunchers
+      - spec: schedulers=slurm fabrics=auto pmi-interface=pmi2 +cxx +cuda +legacylaunchers
       - spec: '+cxx_exceptions'
         when: '%gcc'
   openssh:

env/rhel8/x86_64/users/stable/spack.yaml

@@ -3,12 +3,14 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
   specs: []
   upstreams:
     stable:
       install_tree: >-
         $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/rhel8/x86_64/users/unstable/spack.yaml

@@ -3,14 +3,17 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
     - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
   specs: []
   upstreams:
     stable:
       install_tree: $spack/opt/spack/stable
     unstable:
       install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/sles15/aarch64/admin/stable/spack.yaml

@@ -6,20 +6,40 @@ spack:
     install_tree:
       root: $spack/opt/spack/stable
   definitions:
-  - gcc_12.3_mpich:
+  - compilers:
-    - relion@4.0.1/ve5sish     #+cuda
+    - gcc@13.2.0
+    - gcc@7.5.0
+    - gcc@=12.3
+    - nvhpc@24.3
+    - cce@17.0.0
   include:
     - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/stable/modulefiles
+        lmod: $spack/opt/spack/stable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+        core_compilers:
+        - gcc@=7.5.0
   specs:
   - matrix:
-    - [$gcc_12.3_mpich]
+    - [$compilers]
-    - ['%gcc@12.3']
+    - ['%gcc@7.5.0']
-    - [^cray-mpich]
   view: false

env/sles15/aarch64/admin/testing/spack.yaml

@@ -1,6 +1,12 @@
 spack:
   concretizer:
-    unify: false
+    unify: when_possible
+    targets:
+      granularity: microarchitectures
+    reuse:
+      from:
+      - type: local
+      - type: external
   config:
     install_tree:
       root: $spack/opt/spack/testing
@@ -8,12 +14,49 @@ spack:
       padded_length: 256
   include:
   - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/testing/modulefiles
+        lmod: $spack/opt/spack/testing/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+  packages:
+    # overwrite any existing compiler
+    'cce:':
+      buildable: true
+    'gcc:':
+      buildable: true
+    'nvhpc:':
+      buildable: true
   specs: []
   view: false
+  compilers:
+  - compiler:
+      spec: gcc@=7.5.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []

env/sles15/aarch64/admin/unstable/spack.yaml

@@ -5,41 +5,81 @@ spack:
     install_tree:
       root: $spack/opt/spack/unstable
   definitions:
+  - compilers_gcc_7.5.0:
+    - gcc@13.2.0
+    - gcc@7.5.0
+    - gcc@=12.3
+    - gcc@14.2.0/tln2ck4
+    - nvhpc@24.3
+    - cce@17.0.0
+  - compilers_gcc_13.2.0:
+    - nvhpc@24.11/6wi4jhx
   - gcc_7.5.0:
-    - visit@3.3.3/fo37wwb
+    - ffbidx@main/3xig2i
-  - gcc_12.3:
+  - gcc_13.2.0:
-    - fastqc@0.12.1/eyuqqvf
+    - dssp@2.3.0/xfc2pth
-    - ffbidx@main/lfacuhw
+  - gcc_14.2.0:
-    - star@2.7.11a/gdgr2ea
+    - osu-micro-benchmarks@7.5/mit3iil
-    - sratoolkit@3.1.1/7oqp6a3
+    - osu-micro-benchmarks@7.5/6ycu464
-  - gcc_12.3_mpich:
+    - aws-ofi-nccl@1.13.0/nz3b3yk
-    - cp2k@2024.1/2jesise     #~cuda
-    - cp2k@2024.1/zqm5fme     #+cuda
-    - quantum-espresso@7.3.1/n6mulig     #~cuda
-    - opal@2024.1.0/mhg3mdx
-    - py-alphafold@2.3.2/xkw7tfm     #+cuda
   - gcc_12.3_openmpi:
-    - gromacs@2024.1/3n7qspb     #+cuda
+    - gromacs@2024.1/x33hfjx
-  - nvhpc_23.9_openmpi:
+    - cp2k@2024.3/m3eow6e #+cuda
-    - quantum-espresso@7.3.1/le35al7 #+cuda
+    - relion@4.0.1/rudfcxt
+    - relion@5.0.0/oyf4n7i
+  - gcc_12.3_cray_mpich:
+    - relion@5.0.0/wk4d62r
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/ttg7vdz
+  - gcc_14.2_openmpi:
+    - py-torchvision@0.20.1/ysfcgu5
+    - py-torchaudio@2.5.1/trrglyu
+  - nvhpc_24.11_openmpi:
+    - quantum-espresso@7.4/oed4w6w  #+cuda +mpigpu hdf5=parallel
+    - quantum-espresso@7.3.1/d5wpqzw
   include:
   - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/unstable/modulefiles
+        lmod: $spack/opt/spack/unstable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+        core_compilers:
+        - gcc@=7.5.0
   specs:
+  - matrix:
+    - [$compilers_gcc_7.5.0]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$compilers_gcc_13.2.0]
+    - ['%gcc@13.2.0']
   - matrix:
     - [$gcc_7.5.0]
     - ['%gcc@7.5.0']
   - matrix:
-    - [$gcc_12.3]
+    - [$gcc_13.2.0]
-    - ['%gcc@12.3']
+    - ['%gcc@13.2.0']
   - matrix:
-    - [$gcc_12.3_mpich]
+    - [$gcc_14.2.0]
+    - ['%gcc@14.2.0']
+  - matrix:
+    - [$gcc_12.3_cray_mpich]
     - ['%gcc@12.3']
     - [^cray-mpich]
   - matrix:
@@ -47,7 +87,55 @@ spack:
     - ['%gcc@12.3']
     - [^openmpi]
   - matrix:
-    - [$nvhpc_23.9_openmpi]
+    - [$gcc_13.2_openmpi]
-    - ['%nvhpc@23.9']
+    - ['%gcc@13.2']
+    - [^openmpi]
+  - matrix:
+    - [$gcc_14.2_openmpi]
+    - ['%gcc@14.2']
+    - [^openmpi]
+  - matrix:
+    - [$nvhpc_24.11_openmpi]
+    - ['%nvhpc@24.11']
     - [^openmpi]
   view: false
+  compilers:
+  - compiler:
+      spec: gcc@=14.2.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gcc
+        cxx: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/g++
+        f77: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: gcc@=7.5.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: gcc@=12.3.0
+      paths:
+        cc: /usr/bin/gcc-12
+        cxx: /usr/bin/g++-12
+        f77: /usr/bin/gfortran-12
+        fc: /usr/bin/gfortran-12
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []

env/sles15/aarch64/sysconfigs/compilers.yaml

@@ -1,64 +1,31 @@
 compilers:
 - compiler:
-    spec: cce@=17.0.0
+    spec: gcc@=13.2.0
     paths:
-      cc: cc
+      cc: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gcc
-      cxx: CC
+      cxx: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/g++
-      f77: ftn
+      f77: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gfortran
-      fc: ftn
+      fc: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gfortran
     flags: {}
     operating_system: sles15
     target: aarch64
     modules:
-      - libfabric/1.15.2.0
+    - libfabric/1.22.0
-      - PrgEnv-cray/8.5.0
-      - cce/17.0.0
     environment: {}
     extra_rpaths: []
 - compiler:
-    spec: gcc@=12.3
+    spec: nvhpc@=24.11
     paths:
-      cc: /usr/bin/gcc-12
+      cc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvc
-      cxx: /usr/bin/g++-12
+      cxx: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvc++
-      f77: /usr/bin/gfortran-12
+      f77: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvfortran
-      fc: /usr/bin/gfortran-12
+      fc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvfortran
     flags: {}
     operating_system: sles15
     target: aarch64
     modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
-      - PrgEnv-gnu/8.5.0
+      - /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/modulefiles/nvhpc-nompi/24.11
-      - gcc-native/12.3
     environment: {}
-    extra_rpaths: []
+    extra_rpaths:
-- compiler:
+      - /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/math_libs/lib64/stubs
-    spec: gcc@=7.5.0
-    paths:
-      cc: /usr/bin/gcc
-      cxx: /usr/bin/g++
-      f77: /usr/bin/gfortran
-      fc: /usr/bin/gfortran
-    flags: {}
-    operating_system: sles15
-    target: aarch64
-    modules:
-    - libfabric/1.15.2.0
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: nvhpc@=24.3
-    paths:
-      cc: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc
-      cxx: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc++
-      f77: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvfortran
-      fc: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvfortran
-    flags: {}
-    operating_system: sles15
-    target: aarch64
-    modules:
-      - libfabric/1.15.2.0
-      - nvhpc-nompi/24.3
-    environment:
-      prepend_path:
-        LD_LIBRARY_PATH: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/math_libs/lib64/stubs
-    extra_rpaths: []

env/sles15/aarch64/sysconfigs/packages.yaml

@@ -1,17 +1,24 @@
 packages:
   all:
-    compiler: [gcc@12.3, cce, nvhpc]
+    compiler: [gcc, nvhpc, cce]
+    prefer:
+    - spec: '^openmpi'
+      when: '^mpi'
+    - spec: '^openblas'
+      when: '%gcc ^blas'
+    - spec: '^netlib-scalapack'
+      when: '+scalapack'
     providers:
-      blas: [cray-libsci, openblas, nvhpc]
+      blas: [openblas, nvhpc, nvpl-blas]
-      lapack: [cray-libsci, openblas, nvhpc]
+      lapack: [openblas, nvhpc, nvpl-lapack]
-      mpi: [cray-mpich, openmpi]
+      mpi: [openmpi, cray-mpich]
       pkgconfig: [pkg-config]
-      scalapack: [cray-libsci]
+      scalapack: [netlib-scalapack]
     require:
     - spec: "os=sles15"
     - spec: "target=aarch64"
       when: "%gcc"
-      - spec: "target=zen3"
+    - spec: "target=neoverse_v2"
       when: "%nvhpc"
     # Grace Hopper arch
     variants:
@@ -19,11 +26,25 @@ packages:
   berkeley-db:
     require: '%gcc@7.5.0'
   boost:
-    require: '%gcc@7.5.0'
+    require: '%gcc'
+  cassini-headers:
+    require: '@main'
   ca-certificates-mozilla:
     require: '%gcc@7.5.0'
-  cairo:
+  cce:
-    require: '%gcc@7.5.0'
+    externals:
+      - modules:
+        - libfabric/1.22.0
+        - PrgEnv-cray/8.5.0
+        - cce/17.0.0
+        spec: cce@=17.0.0
+        extra_attributes:
+          compilers:
+            c: cc
+            cxx: CC
+            fortran: ftn
+          operating_system: sles15
+          target: aarch64
   cmake:
     require: '%gcc@7.5.0'
   coreutils:
@@ -43,10 +64,12 @@ packages:
     buildable: false
     externals:
       - modules:
+          - PrgEnv-cce
           - cray-mpich
         prefix: /opt/cray/pe/mpich/8.1.28/ofi/cray/17.0
         spec: cray-mpich@8.1.28%cce@17.0.0
       - modules:
+          - PrgEnv-gnu
           - cray-mpich
         prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3
         spec: cray-mpich@8.1.28%gcc@12.3
@@ -59,15 +82,26 @@ packages:
         spec: cray-pmi@6.1.13%gcc
   cuda:
     prefer:
-    - spec: '@12.2.0'
+    - spec: '@12.8.0'
     require: '@12.2.0:'
     externals:
       - modules:
         - nvhpc-nompi/24.3
         prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/cuda
         spec: cuda@12.2.0%nvhpc@24.3
+        extra_attributes:
+          environment:
+            prepend_path:
+              LD_LIBRARY_PATH: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/math_libs/lib64/stubs
   curl:
     require: '%gcc@7.5.0'
+  cxi-driver:
+    require: '@main'
+  egl:
+    buildable: False
+    externals:
+      - spec: egl@1.5.0
+        prefix: /usr/
   expat:
     require: '%gcc@7.5.0'
   fftw:
@@ -87,53 +121,61 @@ packages:
   gawk:
     require: '%gcc@7.5.0'
   gcc:
-    require: '%gcc@7.5.0'
+    externals:
+      - prefix: /usr
+        spec: gcc@7.5.0
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++
+            fortran: /usr/bin/gfortran
+          operating_system: sles15
+          target: aarch64
+        modules:
+          - libfabric/1.22.0
+      - prefix: /usr
+        spec: gcc@=12.3
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+            fortran: /usr/bin/gfortran-12
+          operating_system: sles15
+          target: aarch64
+        modules:
+          - libfabric/1.22.0
+          - PrgEnv-gnu/8.5.0
+          - gcc-native/12.3
   gdbm:
     require: '%gcc@7.5.0'
   gettext:
-    externals:
+    require: '%gcc@7.5.0'
-      - prefix: /usr
-        spec: 'gettext@0.20.2'
   git:
     require: '%gcc@7.5.0'
   gmake:
     require: '%gcc@7.5.0'
   groff:
     require: '%gcc@7.5.0'
-  gtkplus:
-    require: 'default_library=shared,static'
-  hdf5:
-    prefer:
-      - spec: '~mpi'
-    externals:
-      - modules:
-        - cray-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%cce@17.0.0
-      - modules:
-        - cray-parallel-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%cce@17.0.0
-      - modules:
-        - cray-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%gcc@12.3
-      - modules:
-        - cray-parallel-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%gcc@12.3
   libbsd:
     require: '%gcc@7.5.0'
+  libcxi:
+    require: '@main'
   libevent:
     require: '%gcc@7.5.0'
   # Crayism
   libfabric:
-    buildable: false
+    buildable: true
+    require: '@main+cuda+uring fabrics=cxi,sockets,tcp,udp,xpmem'
     externals:
       - modules:
-          - libfabric/1.15.2.0
+          - libfabric/1.22.0 
-        prefix: /opt/cray/libfabric/1.15.2.0
+        prefix: /opt/cray/libfabric/1.22.0
-        spec: libfabric@1.15.2.0
+        spec: libfabric@1.22.0
+  libfuse:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: libfuse@2.9.7
   libiconv:
     require: '%gcc@7.5.0'
   libidn2:
@@ -144,6 +186,11 @@ packages:
     require: '%gcc@7.5.0'
   libsigsegv:
     require: '%gcc@7.5.0'
+  librsvg:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: librsvg@2.48.0
   libtool:
     require: '%gcc@7.5.0'
   libxml2:
@@ -193,9 +240,17 @@ packages:
     require: '%gcc@7.5.0'
   nvhpc:
     externals:
-      - modules:
+      - prefix: /opt/nvidia/hpc_sdk
-        - nvhpc-nompi/24.3
         spec: nvhpc@24.3
+        extra_attributes:
+          compilers:
+            c: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc
+            cxx: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc++
+            fortran: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvfortran
+          operating_system: sles15
+          modules: nvhpc-nompi/24.3
+          extra_rpaths:
+            - /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/math_libs/lib64/stubs
   opal:
     require:
       - spec: '%gcc'
@@ -211,10 +266,10 @@ packages:
         spec: openldap@2.4.46
   openmpi:
     require:
-      - spec: 'schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +cxx_exceptions +cuda +legacylaunchers'
+      - spec: "schedulers=slurm fabrics=ofi,xpmem +cuda +internal-pmix"
-        when: '%gcc'
+        when: "@5.0.0:"
-      - spec: 'schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +cuda +legacylaunchers'
+      - spec:  "schedulers=slurm fabrics=ofi,xpmem +cxx +cuda +legacylaunchers"
-        when: '%nvhpc'
+        when: "@:5.0.0"
   openssh:
     require: '%gcc@7.5.0'
   openssl:
@@ -238,13 +293,19 @@ packages:
     require: '%gcc@7.5.0'
   pkgconf:
     require: '%gcc@7.5.0'
+  pmix:
+    require: '+munge'
   py-fypp:
     require: '%gcc@7.5.0'
   python:
     require: '%gcc@7.5.0'
-  # QE is unable to find out cray-libsci
   quantum-espresso:
-    require: ~scalapack ^openblas
+    require:
+      - spec: "^nvpl-blas ^nvpl-lapack"
+        when: "%nvhpc"
+    conflict:
+    - spec: '^acfl'
+      when: '%nvhpc'
   rdma-core:
     require: '%gcc@7.5.0'
   readline:
@@ -269,9 +330,10 @@ packages:
   xpmem:
     buildable: false
     externals:
-      - modules:
+      - spec: xpmem@2.9.6-1.1
+        prefix: /usr
+        modules:
         - xpmem/2.9.6-1.1_20240511212657__g087dc11fc19d
-        spec: xpmem@2.9.6-1.1
   xxhash:
     require: '%gcc@7.5.0'
   xz:

env/sles15/aarch64/users/stable/spack.yaml

@@ -3,11 +3,12 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
   specs: []
   upstreams:
     stable:
       install_tree: $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/sles15/aarch64/users/unstable/spack.yaml

@@ -3,7 +3,6 @@ spack:
     unify: false
   include:
   - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
@@ -13,4 +12,6 @@ spack:
       install_tree: $spack/opt/spack/stable
     unstable:
       install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/sles15/x86_64/admin/stable/spack.yaml

@@ -6,20 +6,46 @@ spack:
     install_tree:
       root: $spack/opt/spack/stable
   definitions:
+  - compilers:
+    - gcc@7.5.0
+    - gcc@=12.3.0
+    - intel-oneapi-compilers@=2024.1.0
+    - intel-oneapi-compilers@=2022.1.0
+    - gcc@=13.2.0
+    - nvhpc@23.9
+    - cce@17.0.0
   - gcc_7.5.0:
     - fpart@1.6.0/di4akxx
   - gcc_12.3_mpich:
     - relion@4.0.1/ve5sish     #+cuda
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/q4ulqv3 #+cuda cuda_arch=80
   include:
     - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
     - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/stable/modulefiles
+        lmod: $spack/opt/spack/stable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64'
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
   specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@7.5.0']
   - matrix:
     - [$gcc_7.5.0]
     - ['%gcc@7.5.0']
@@ -27,4 +53,8 @@ spack:
     - [$gcc_12.3_mpich]
     - ['%gcc@12.3']
     - [^cray-mpich]
+  - matrix:
+    - [$gcc_13.2_openmpi]
+    - ['%gcc@13.2']
+    - [^openmpi]
   view: false

env/sles15/x86_64/admin/testing/spack.yaml

@@ -8,12 +8,25 @@ spack:
       padded_length: 256
   include:
   - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+  - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/testing
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/testing/modulefiles
+        lmod: $spack/opt/spack/testing/modulefiles
-  specs: []
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              LD_LIBRARY_PATH: /opt/cray/libfabric/1.22.0/lib64
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+  specs:
+  - 
   view: false

env/sles15/x86_64/admin/unstable/spack.yaml

@@ -5,45 +5,91 @@ spack:
     install_tree:
       root: $spack/opt/spack/unstable
   definitions:
+  - compilers:
+    - gcc@7.5.0
+    - gcc@9.5.0/ow3yhui
+    - gcc@=12.3
+    - intel-oneapi-compilers@=2024.1.0
+    - intel-oneapi-compilers@=2022.1.0
+    - gcc@=13.2.0 arch=linux-sles15-x86_64
+    - nvhpc@23.9
+    - cce@17.0.0
   - gcc_7.5.0:
+    - fpart@1.6.0/di4akxx
     - visit@3.3.3/fo37wwb
   - gcc_12.3:
     - fastqc@0.12.1/eyuqqvf
     - ffbidx@main/lfacuhw
     - star@2.7.11a/gdgr2ea
     - sratoolkit@3.1.1/7oqp6a3
+  - gcc_13.2:
+    - dmtcp@3.0.0/e76t6c5
+    - gams@49.1/vztjqs4
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/q4ulqv3 #+cuda cuda_arch=80
   - gcc_12.3_mpich:
     - cp2k@2024.1/2jesise     #~cuda
     - cp2k@2024.1/zqm5fme     #+cuda
+    - relion@4.0.1/ve5sish
     - quantum-espresso@7.3.1/n6mulig     #~cuda
     - opal@2024.1.0/mhg3mdx
     - py-alphafold@2.3.2/xkw7tfm     #+cuda
+    - sundials@7.0.0/xxysszi #sundials@7.0.0+ARKODE+CVODE+CVODES+IDA+IDAS+KINSOL+cuda+examples+examples-install~f2003~fcmix~ginkgo~hypre~int64~ipo~klu~kokkos~kokkos-kernels~lapack~magma~monitoring+mpi+openmp~petsc~profiling~pthread~raja~rocm+shared+static~superlu-dist~superlu-mt~sycl~trilinos build_system=cmake build_type=Release cstd=99 cuda_arch=80 cxxstd=14 generator=make logging-level=2 precision=double
+  - gcc_9.5.0_openmpi:
+    - q-e-sirius@develop-ristretto/bb4v7wz
   - gcc_12.3_openmpi:
     - gromacs@2024.1/3n7qspb     #+cuda
+    - hdf5@1.14.3/wqcdnm7 #+mpi ^openmpi@5.0.3%gcc@12.3
   - intel_2021.6.0:
-    - openmpi@4.1.6/eqhrgv5
+    - hdf5@1.14.3/yrxkczx
   - nvhpc_23.9_openmpi:
     - quantum-espresso@7.3.1/le35al7 #+cuda
   include:
   - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+  - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
   mirrors:
     local_psi: file:///afs/psi.ch/software/spack/testing
   modules:
     default:
       roots:
-        tcl: $spack/opt/spack/unstable/modulefiles
+        lmod: $spack/opt/spack/unstable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              LD_LIBRARY_PATH: /opt/cray/libfabric/1.22.0/lib64
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
   specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@7.5.0']
   - matrix:
     - [$gcc_7.5.0]
     - ['%gcc@7.5.0']
   - matrix:
     - [$gcc_12.3]
     - ['%gcc@12.3']
+  - matrix:
+    - [$gcc_13.2]
+    - ['%gcc@13.2']
+  - matrix:
+    - [$gcc_13.2_openmpi]
+    - ['%gcc@13.2']
+    - [^openmpi]
   - matrix:
     - [$gcc_12.3_mpich]
     - ['%gcc@12.3']
     - [^cray-mpich]
+  - matrix:
+    - [$gcc_9.5.0_openmpi]
+    - ['%gcc@9.5.0']
+    - [^openmpi]
   - matrix:
     - [$gcc_12.3_openmpi]
     - ['%gcc@12.3']

env/sles15/x86_64/sysconfigs/compilers.yaml

@@ -1,92 +1,57 @@
 compilers:
-- compiler:
-    spec: cce@=17.0.0
-    paths:
-      cc: cc
-      cxx: CC
-      f77: ftn
-      fc: ftn
-    flags: {}
-    operating_system: sles15
-    target: x86_64
-    modules:
-      - libfabric/1.15.2.0
-      - PrgEnv-cray/8.5.0
-      - cce/17.0.0
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: gcc@=12.3
-    paths:
-      cc: /usr/bin/gcc-12
-      cxx: /usr/bin/g++-12
-      f77: /usr/bin/gfortran-12
-      fc: /usr/bin/gfortran-12
-    flags: {}
-    operating_system: sles15
-    target: x86_64
-    modules:
-      - libfabric/1.15.2.0
-      - PrgEnv-gnu/8.5.0
-      - gcc-native/12.3
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: gcc@=7.5.0
-    paths:
-      cc: /usr/bin/gcc
-      cxx: /usr/bin/g++
-      f77: /usr/bin/gfortran
-      fc: /usr/bin/gfortran
-    flags: {}
-    operating_system: sles15
-    target: x86_64
-    modules:
-    - libfabric/1.15.2.0
-    environment: {}
-    extra_rpaths: []
-- compiler:
-    spec: nvhpc@=23.9
-    paths:
-      cc: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc
-      cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc++
-      f77: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran
-      fc: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran
-    flags: {}
-    operating_system: sles15
-    target: x86_64
-    modules:
-      - nvhpc-nompi/23.9
-      - libfabric/1.15.2.0
-    environment:
-      prepend_path:
-        LD_LIBRARY_PATH: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/math_libs/lib64/stubs
-    extra_rpaths: []
 - compiler:
       spec: intel@=2021.6.0
       paths:
-        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/intel64/icc
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/icc
-        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/intel64/icpc
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/icpc
-        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/intel64/ifort
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/ifort
-        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/intel64/ifort
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/ifort
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
       environment: {}
       extra_rpaths: []
 - compiler:
       spec: oneapi@=2022.1.0
       paths:
-        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/icx
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/icx
-        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-jjfubnqeg5udrhlho6owzlpi6odqsey5/compiler/2022.1.0/linux/bin/icpx
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/icpx
         f77: null
         fc: null
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
+      environment: {}
+      extra_rpaths: []
+- compiler:
+      spec: gcc@=13.2.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gcc
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/g++
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: x86_64
+      modules:
+      - libfabric/1.22.0
+      environment: {}
+      extra_rpaths: []
+- compiler:
+      spec: gcc@=9.5.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gcc
+        cxx: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/g++
+        f77: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: x86_64
+      modules:
+      - libfabric/1.22.0
       environment: {}
       extra_rpaths: []

env/sles15/x86_64/sysconfigs/packages.yaml

@@ -1,10 +1,13 @@
 packages:
   all:
-    compiler: [gcc@12.3, cce, nvhpc]
+    compiler: [gcc, nvhpc, cce]
+    prefer:
+    - spec: '^openmpi'
+      when: '^mpi'
     providers:
-      blas: [cray-libsci, openblas, nvhpc]
+      blas: [openblas, nvhpc, cray-libsci]
-      lapack: [cray-libsci, openblas, nvhpc]
+      lapack: [openblas, nvhpc, cray-libsci]
-      mpi: [cray-mpich, openmpi]
+      mpi: [openmpi, cray-mpich]
       pkgconfig: [pkg-config]
       scalapack: [cray-libsci]
     require:
@@ -13,7 +16,6 @@ packages:
         when: "%gcc"
       - spec: "target=zen3"
         when: "%nvhpc"
-    # Grace Hopper arch
     variants:
       - cuda_arch=80
   autoconf:
@@ -26,10 +28,26 @@ packages:
     require: '%gcc@7.5.0'
   ca-certificates-mozilla:
     require: '%gcc@7.5.0'
+  cce:
+    externals:
+      - modules:
+        - libfabric/1.22.0
+        - PrgEnv-cray/8.5.0
+        - cce/17.0.0
+        spec: cce@=17.0.0
+        extra_attributes:
+          compilers:
+            c: cc
+            cxx: CC
+            fortran: ftn
+          operating_system: sles15
+          target: x86_64
   cairo:
     require: '%gcc@7.5.0'
+  cassini-headers:
+    require: '@main'
   cmake:
-    require: '%gcc@7.5.0'
+    require: '%gcc'
   coreutils:
     require: '%gcc@7.5.0'
   cray-libsci:
@@ -63,7 +81,7 @@ packages:
         spec: cray-pmi@6.1.13%gcc
   cuda:
     prefer:
-    - spec: '@12.2.0'
+    - spec: '@12.8.0'
     require: '@12.2.0:'
     externals:
       - modules:
@@ -72,6 +90,8 @@ packages:
         spec: cuda@12.2.0%nvhpc@23.9
   curl:
     require: '%gcc@7.5.0'
+  cxi-driver:
+    require: '@main'
   expat:
     require: '%gcc@7.5.0'
   fftw:
@@ -91,7 +111,31 @@ packages:
   gawk:
     require: '%gcc@7.5.0'
   gcc:
-    require: '%gcc@7.5.0'
+    externals:
+      - prefix: /usr
+        spec: gcc@7.5.0
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++
+            fortran: /usr/bin/gfortran
+          operating_system: sles15
+          target: x86_64
+        modules:
+          - libfabric/1.22.0
+      - prefix: /usr
+        spec: gcc@=12.3
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+            fortran: /usr/bin/gfortran-12
+          operating_system: sles15
+          target: x86_64
+        modules:
+          - libfabric/1.22.0
+          - PrgEnv-gnu/8.5.0
+          - gcc-native/12.3
   gdbm:
     require: '%gcc@7.5.0'
   gettext:
@@ -106,38 +150,21 @@ packages:
     require: '%gcc@7.5.0'
   gtkplus:
     require: 'default_library=shared,static'
-  hdf5:
-    prefer:
-      - spec: '~mpi'
-    externals:
-      - modules:
-        - cray-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%cce@17.0.0
-      - modules:
-        - cray-parallel-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%cce@17.0.0
-      - modules:
-        - cray-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%gcc@12.3
-      - modules:
-        - cray-parallel-hdf5/1.12.2.9
-        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%gcc@12.3
   libbsd:
     require: '%gcc@7.5.0'
+  libcxi:
+    require: '@main'
   libevent:
     require: '%gcc@7.5.0'
   # Crayism
   libfabric:
-    buildable: false
+    buildable: true
+    require: '@main+cuda+uring fabrics=cxi,sockets,tcp,udp,xpmem'
     externals:
       - modules:
-          - libfabric/1.15.2.0
+          - libfabric/1.22.0
-        prefix: /opt/cray/libfabric/1.15.2.0
+        prefix: /opt/cray/libfabric/1.22.0
-        spec: libfabric@1.15.2.0
+        spec: libfabric@1.22.0
   libiconv:
     require: '%gcc@7.5.0'
   libidn2:
@@ -197,9 +224,18 @@ packages:
     require: '%gcc@7.5.0'
   nvhpc:
     externals:
-      - modules:
+      - spec: nvhpc@23.9
-        - nvhpc-nompi/23.9
+        prefix: /opt/nvidia/hpc_sdk
-        spec: nvhpc@23.9
+        extra_attributes:
+          compilers:
+            c: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc
+            cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc++
+            fortran: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran
+          operating_system: sles15
+          modules: nvhpc-nompi/23.9
+          environment:
+            prepend_path:
+              LD_LIBRARY_PATH: '/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/math_libs/lib64/stubs'
   opal:
     require:
       - spec: '%gcc'
@@ -215,11 +251,10 @@ packages:
         spec: openldap@2.4.46
   openmpi:
     require:
-      - spec: 'schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +cxx_exceptions +cuda +legacylaunchers'
+      - spec: "schedulers=slurm fabrics=ofi,xpmem +cuda +internal-pmix"
-        when: '%gcc'
+        when: "@5.0.0:"
-      - spec: 'schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +cuda +legacylaunchers'
+      - spec:  "schedulers=slurm fabrics=ofi,xpmem +pmi +cxx +cxx_exceptions +cuda +legacylaunchers"
-        when: 'nvhpc'
+        when: "@:5.0.0"
-      - spec: 'schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +legacylaunchers'
   openssh:
     require: '%gcc@7.5.0'
   openssl:
@@ -243,6 +278,8 @@ packages:
     require: '%gcc@7.5.0'
   pkgconf:
     require: '%gcc@7.5.0'
+  pmix:
+    require: '+munge'
   py-fypp:
     require: '%gcc@7.5.0'
   python:
@@ -276,6 +313,7 @@ packages:
     externals:
       - modules:
           - xpmem/2.8.2-1.0_3.9__g84a27a5.shasta
+        prefix: /usr
         spec: xpmem@2.8.2-1
   xxhash:
     require: '%gcc@7.5.0'

env/sles15/x86_64/users/stable/spack.yaml

@@ -3,11 +3,13 @@ spack:
     unify: false
   include:
     - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/packages.yaml
-    - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
   specs: []
   upstreams:
     stable:
       install_tree: $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/sles15/x86_64/users/unstable/spack.yaml

@@ -3,14 +3,17 @@ spack:
     unify: false
   include:
   - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/packages.yaml
-  - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/compilers.yaml
   include_concrete:
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/stable
     - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
   specs: []
   upstreams:
     stable:
       install_tree: $spack/opt/spack/stable
     unstable:
       install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

packages/gams/package.py

@@ -11,3 +11,4 @@ from spack.pkg.builtin.gams import Gams as SpackGams
 
 class Gams(SpackGams):
     version("48.1", md5="5a59a973edfc4e3170b9535760593d62", expand=False)
+    version("49.1", md5="bc9b82d015c9672f8c153003c7a5dd2a", expand=False)

packages/mfem/pachage.py

@@ -1,13 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-
-from spack.package import *
-from spack.pkg.builtin.mfem import Mfem as SpackMfem
-
-
-class Mfem(SpackMfem):
-
-    patch(mfem_conduit_spack.patch)

packages/quantum-espresso/package.py

@@ -1,14 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import os
-
-from spack.package import *
-from spack.pkg.builtin.quantum_espresso import QuantumEspresso as SpackQuantumEspresso
-
-class QuantumEspresso(SpackQuantumEspresso):
-    
-    with when("%nvhpc +cuda"):
-        depends_on("cuda", type=("build","link","run"))

packages/relion/package.py

@@ -0,0 +1,24 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+from spack.pkg.builtin.relion import Relion as SpackRelion
+
+
+class Relion(SpackRelion):
+    version("5.0.0", sha256="800ad0c0aa778cbf584fcf8986976645f2b25d677a80f168e5397975b9db6e47")
+
+    variant("python-exe-path", default="/data/project/bio/hpce/relion-5.0/miniconda3/envs/relion-5.0/bin/python", description="Path to conda env python binary")
+    variant("torch-home-path", default="/data/project/bio/hpce/relion-5.0/torch", description="Path to pretrained model weights")
+
+    def cmake_args(self):
+        args = super().cmake_args()
+
+        if "@5.0.0" in self.spec:
+            args += ["-DPYTHON_EXE_PATH={0}".format(self.spec.variants["python-exe-path"].value)]
+            args += ["-DTORCH_HOME_PATH={0}".format(self.spec.variants["torch-home-path"].value)]
+
+        return args

templates/user_hierachical_paths.lua

@@ -0,0 +1,20 @@
+{% extends "modules/modulefile.lua" %}
+{% block footer %}
+ 
+local user_module_root= pathJoin("/afs/psi.ch/sys/spack/user", os.getenv("USER"), "spack-install/modulefiles")
+
+{% if 'mpi' in provides %}
+local user_projection= "{{ spec.architecture }}/{{ spec.name }}/{{ version_part }}/{{ spec.compiler.name }}/{{ spec.compiler.version }}"
+{% elif 'compiler' in provides %}
+local user_projection= "{{ spec.architecture }}/{{ spec.name }}/{{ spec.version }}"
+{% endif %}
+
+
+if user_projection then
+local user_path= pathJoin(user_module_root, user_projection)
+  if isDir(user_path) then
+prepend_path("MODULEPATH", user_path, ":")
+  end
+end
+
+{% endblock %}