FIX: retrograde git to 2.42.0 for GH

FIX: move spack-psi bye one folder

See merge request lsm-hpce/spack-psi!8

.gitea/workflows/ippl_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: IPPL GH Merlin7 Scheduler
+run-name: IPPL GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/ippl_gh.sbatch
+        sbatch .gitea/workflows/sbatch/ippl_gh.sbatch

.gitea/workflows/ippl_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: IPPL GPU Merlin7 Scheduler
+run-name: IPPL GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/ippl_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/ippl_gpu.sbatch

.gitea/workflows/opal-x_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: OPAL-X GH Merlin7 Scheduler
+run-name: OPAL-X GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/opal-x_gh.sbatch
+        sbatch .gitea/workflows/sbatch/opal-x_gh.sbatch

.gitea/workflows/opal-x_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: OPAL-X-GPU Merlin7 Scheduler
+run-name: OPAL-X-GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/opal-x_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/opal-x_gpu.sbatch

.gitea/workflows/q-e_cpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-CPU Merlin7 Scheduler
+run-name: QE-CPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch CPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_cpu.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_cpu.sbatch

.gitea/workflows/q-e_gh_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-GH Merlin7 Scheduler
+run-name: QE-GH Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch GH GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_gh.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_gh.sbatch

.gitea/workflows/q-e_gpu_merlin7.yml

@@ -0,0 +1,17 @@
+name: QE-GPU Merlin7 Scheduler
+run-name: QE-GPU Merlin7 Scheduler
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  checkout:
+    runs-on: merlin7
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Sbatch A100 GPU script
+      run: |
+        chmod +x .gitea/workflows/sbatch/q-e_gpu.sbatch
+        sbatch .gitea/workflows/sbatch/q-e_gpu.sbatch

.gitea/workflows/sbatch/ippl_gh.sbatch

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --job-name="ippl-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module purge
+module use Spack unstable
+module load gcc/13.2.0 openmpi/5.0.3-3lmi-GH200-gpu  
+module load boost/1.82.0-3ns6 fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.7.1 h5hut/2.0.0rc7 openblas/0.3.26 cmake/3.31.4-u2nm
+
+git clone https://github.com/IPPL-framework/ippl.git
+cd ippl
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_HOPPER90=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=ON -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/ippl_gpu.sbatch

@@ -0,0 +1,30 @@
+#!/bin/bash
+#SBATCH --job-name="ippl-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/13.2.0 openmpi/4.1.6-57rc-A100-gpu                 
+module load boost/1.82.0-e7gp fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.8 h5hut/2.0.0rc7 openblas/0.3.26-omp cmake/3.31.6-oe7u
+
+git clone https://github.com/IPPL-framework/ippl.git
+cd ippl
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=ON -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/opal-x_gh.sbatch

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name="opal-x-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module purge
+module use Spack unstable
+module load gcc/13.2.0 openmpi/5.0.3-3lmi-GH200-gpu  
+module load boost/1.82.0-3ns6 fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.7.1 h5hut/2.0.0rc7 openblas/0.3.26 cmake/3.31.4-u2nm
+
+git clone https://gitlab.psi.ch/OPAL/opal-x/src.git opal-x
+cd opal-x
+./gen_OPALrevision
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_HOPPER90=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=OFF -DUSE_ALTERNATIVE_VARIANT=ON -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/opal-x_gpu.sbatch

@@ -0,0 +1,31 @@
+#!/bin/bash
+#SBATCH --job-name="opal-x-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/13.2.0 openmpi/4.1.6-57rc-A100-gpu                 
+module load boost/1.82.0-e7gp fftw/3.3.10 gnutls/3.8.3 googletest/1.14.0 gsl/2.8 h5hut/2.0.0rc7 openblas/0.3.26-omp cmake/3.31.6-oe7u
+
+git clone https://gitlab.psi.ch/OPAL/opal-x/src.git opal-x
+cd opal-x
+./gen_OPALrevision
+
+mkdir build
+cd build
+
+cmake -DCMAKE_BUILD_TYPE=Release -DKokkos_ARCH_AMPERE80=ON -DCMAKE_CXX_STANDARD=20 -DIPPL_ENABLE_FFT=ON -DIPPL_ENABLE_TESTS=OFF -DIPPL_ENABLE_SOLVERS=ON -DIPPL_ENABLE_ALPINE=True -DIPPL_PLATFORMS=cuda ..
+make -j 16
+

.gitea/workflows/sbatch/q-e_cpu.sbatch

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-CPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tmerlin7
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load gcc/12.3 openmpi/main-syah fftw/3.3.10.6-omp hdf5/1.14.5-t46c openblas/0.3.29-omp cmake/3.31.6-oe7u
+
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitea/workflows/sbatch/q-e_gh.sbatch

@@ -0,0 +1,27 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-GH-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=gh-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load nvhpc/25.3 openmpi/5.0.7-e3bf-GH200-gpu fftw/3.3.10-sfpw-omp hdf5/develop-2.0-ztvo nvpl-blas/0.4.0.1-3zpg nvpl-lapack/0.3.0-ymy5  netlib-scalapack/2.2.2-qrhq cmake/3.31.6-5dl7
+
+module load git/2.42.0-y4e4
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DQE_ENABLE_SCALAPACK:BOOL=ON -DQE_ENABLE_CUDA:BOOL=ON -DQE_ENABLE_MPI_GPU_AWARE:BOOL=ON -DQE_ENABLE_OPENACC:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitea/workflows/sbatch/q-e_gpu.sbatch

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --job-name="qe-scheduled-A100-GPU"
+#SBATCH --output=_scheduler-stdout.txt
+#SBATCH --error=_scheduler-stderr.txt
+#SBATCH --partition=a100-daily
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=16
+#SBATCH --time=01:00:00
+#SBATCH --cluster=tgmerlin7
+#SBATCH --gpus=1
+#SBATCH --wait
+
+export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
+export REQUESTS_CA_BUNDLE=/etc/ssl/ca-bundle.pem
+PMODULES_VERSION='2.0.0.2'
+source /opt/psi/config/profile.bash
+
+module use Spack unstable
+module load nvhpc/25.3 openmpi/main-6bnq-A100-gpu fftw/3.3.10.6-qbxu-omp hdf5/develop-2.0-rjgu netlib-scalapack/2.2.2-3hgw cmake/3.31.6-oe7u
+git clone https://gitlab.com/QEF/q-e.git
+cd q-e
+mkdir build
+cd build
+cmake -DQE_ENABLE_MPI:BOOL=ON -DQE_ENABLE_OPENMP:BOOL=ON -DQE_ENABLE_SCALAPACK:BOOL=ON -DQE_ENABLE_CUDA:BOOL=ON -DQE_ENABLE_MPI_GPU_AWARE:BOOL=ON -DQE_ENABLE_OPENACC:BOOL=ON -DCMAKE_C_COMPILER:STRING=mpicc -DCMAKE_Fortran_COMPILER:STRING=mpif90 -DQE_ENABLE_HDF5:BOOL=ON ..
+make -j 16

.gitlab-ci.yml

@@ -1,32 +0,0 @@
-stages: [build_test]
-
-variables:
-  GIT_STRATEGY: clone
-  MAX_JOB_NB: 4
-  SPACK_BRANCH: develop
-  SPACK_PCKG: ffbidx
-  SPACK_PCKG_VERSION: main
-  SPACK_REPO: https://github.com/spack/spack.git
-
-build_test:
-  tags:
-    - merlin
-  stage: build_test
-  script:
-    - git clone ${SPACK_REPO} -b ${SPACK_BRANCH}
-    - . "./spack/share/spack/setup-env.sh"
-    - source /opt/psi/config/profile.bash
-    - module load git-lfs
-    - spack clean -a # Make sure spack doesn't have any caches left
-    - rm -rf /scratch/${USER}/spack-dev/${SPACK_PCKG} # Remove previous packages
-    - spack uninstall -Ray
-    - SPACK_PCKG_LOWERCASE=$(echo "${SPACK_PCKG,,}")
-    - if [[ "$SPACK_PCKG_LOWERCASE" == "hiqsimulator" ]]; then
-    - SPACK_PCKG_LOWERCASE="py-$SPACK_PCKG_LOWERCASE"
-    - fi
-    - spack env activate --without-view env/$(spack arch -o)/${SPACK_PCKG_LOWERCASE}
-    - spack develop --path /scratch/\$user/spack-dev/${SPACK_PCKG} --force ${SPACK_PCKG_LOWERCASE}@${SPACK_PCKG_VERSION}
-    - spack install -j ${MAX_JOB_NB}
-    - spack load
-#    - JOB_ID=$(sbatch --parsable ../test/run.slurm)
-#    - cat slurm-${JOB_ID}.out

README.md

@@ -2,68 +2,260 @@
 
 [Official Spack documentation](https://spack.readthedocs.io/en/latest/)
 
-## How to use on Merlin7
+## How to use on Merlin systems (Merlin6/Merlin7/Ra)
 
-### Install your own software
+Load the spack module
-```bash
-# Check if your package is already in Spack
-user@login001:~> spack list $pkg_name # e.g opal
 
-# Check the package variants you want to set e.g +cuda +openmp ^openmpi ...
+```shell
-user@login001:~> spack info $pkg_name # e.g opal
+[stable] user@login001:~> module load spack
-
-# Check the whole dependency tree of your spec and 
-# if you don't need to set more variants
-user@login001:~> spack spec $pkg_name@version +variant1 ^dep1@version ...
-
-# e.g spack spec opal@2022.1%gcc@7.5.0 +mithra ~python ^openmpi@4.1.6 ^openblas
-
-# Install your package
-user@login001:~> spack install $pkg_name@version +variant1 ^dep1@version1
-
-# Load your package
-user@login001:~> spack load $pkg_name@version +variant1 ^dep1@version1
-
-# Submit your script
-user@login001:~> sbatch batch.script
 ```
 
-### Develop your own software
+Voilà you're done you can use Spack! :)
-```bash
-# Check if your package is already in Spack
-user@login001:~> spack list $pkg_name # e.g opal
 
-# Check the package variants you want to set e.g +cuda +openmp ^openmpi ...
++ Side note: the default software stack loaded for you is stable, which might not contain all the packages you're looking for, </br>
-user@login001:~> spack info $pkg_name # e.g opal
+so if you want to use the unstable software stack switch to it using:
 
-# Create your own env and load it
+```shell
-user@login001:~> spack env activate --create -p myenv
+spack env activate -p unstable
+```
 
-# Add the spec you want to build to your environement
+## Find out what's already installed
-user@login001:~> spack add $pkg_name@version +variant1 ^dep1@version ...
 
-# e.g spack add opal@master%gcc@7.5.0 +mithra ~python ^openmpi@4.1.6 ^openblas
+```shell
+[unstable] user@login001:~>  spack find
+==> In environment unstable
+==> 1 root specs
+ - []
 
-# Tell Spack you want to develop locally under
+==> Included specs
-# /afs/psi.ch/sys/spack.x86_64_cos3.0/user/$USER/spack-environment/$pkg_name
+-- no arch / gcc ------------------------------------------------
-user@login001:~> spack develop $pkg_name@version
+gromacs%gcc
 
-# Install the package locally
+-- no arch / gcc@12.3.0 -----------------------------------------
-user@login001:~> spack install # first time, then you can just use make directly
+cp2k@2024.1%gcc@12.3.0  opal@master%gcc@12.3.0         py-alphafold@2.3.2%gcc@12.3.0
+cp2k@2024.1%gcc@12.3.0  py-alphafold@2.3.2%gcc@12.3.0  quantum-espresso@7.3.1%gcc@12.3.0
 
-# Load your package
+-- no arch / oneapi@2024.1.0 ------------------------------------
-user@login001:~> spack load $pkg_name@version +variant1 ^dep1@version1
+gromacs@2024.1%oneapi@2024.1.0
 
-# Submit your script
+==> Installed packages
-user@login001:~> sbatch batch.script
+-- linux-rhel7-x86_64 / gcc@4.8.5 -------------------------------
+autoconf@2.72                curl@8.7.1         gdbm@1.23             libffi@3.4.6       mpfr@4.2.1      pkg-config@0.29.2     readline@8.2
+autoconf@2.72                diffutils@3.10     gettext@0.19.8.1      libidn2@2.3.7      mpfr@4.2.1      pkg-config@0.29.2     sqlite@3.43.2
+autoconf-archive@2023.02.20  diffutils@3.10     gettext@0.19.8.1      libmd@1.0.4        ncurses@6.5     py-fypp@3.1           texinfo@7.0.3
+autoconf-archive@2023.02.20  expat@2.6.2        git@2.42.0            libpciaccess@0.17  ncurses@6.5     py-fypp@3.1           util-linux-uuid@2.36.2
+automake@1.16.5              findutils@4.9.0    glibc@2.17            libsigsegv@2.14    nghttp2@1.57.0  py-pip@23.1.2         util-macros@1.19.3
+automake@1.16.5              findutils@4.9.0    glibc@2.17            libsigsegv@2.14    ninja@1.11.1    py-setuptools@59.4.0  xz@5.4.6
+berkeley-db@18.1.40          flex@2.6.3         gmake@4.4.1           libtool@2.4.7      numactl@2.0.14  py-wheel@0.41.2       zlib-ng@2.1.6
+berkeley-db@18.1.40          gawk@5.3.0         gmake@4.4.1           libunistring@1.2   nvhpc@24.3      python@3.8.18         zlib-ng@2.1.6
+binutils@2.42                gawk@5.3.0         gmp@6.2.1             libxcrypt@4.4.35   openssh@9.7p1   python@3.9.18         zstd@1.5.6
+binutils@2.42                gcc@12.3.0         gmp@6.2.1             libxml2@2.10.3     openssl@3.3.0   python@3.11.7         zstd@1.5.6
+binutils@2.42                gcc@12.3.0         krb5@1.20.1           m4@1.4.19          openssl@3.3.0   python-venv@1.0
+bison@3.8.2                  gcc-runtime@4.8.5  libbsd@0.12.1         m4@1.4.19          pcre2@10.43     python-venv@1.0
+bzip2@1.0.8                  gcc-runtime@4.8.5  libedit@3.1-20230828  mpc@1.3.1          perl@5.38.0     re2c@2.2
+bzip2@1.0.8                  gdbm@1.23          libevent@2.1.12       mpc@1.3.1          perl@5.38.0     readline@8.2
+
+-- linux-rhel7-x86_64 / gcc@12.3.0 ------------------------------
+abseil-cpp@20240116.2  libogg@1.3.5            py-beniget@0.3.0               py-ml-collections@0.1.0     py-tabulate@0.8.9
+amrex@18.07            libpng@1.6.39           py-biopython@1.79              py-networkx@2.7.1           py-tensorboard@2.11.2
+aria2@1.37.0           libssh2@1.11.0          py-bottleneck@1.3.2            py-numexpr@2.8.4            py-tensorboard-data-server@0.6.1
+boost@1.85.0           libtheora@1.1.1         py-cachetools@5.2.0            py-numpy@1.17.5             py-tensorboard-plugin-wit@1.8.1
+boost@1.85.0           libxc@6.2.2             py-certifi@2023.7.22           py-numpy@1.21.6             py-tensorflow@2.11.0
+c-ares@1.27.0          libxsmm@1.17            py-charset-normalizer@3.3.0    py-oauthlib@3.2.2           py-termcolor@1.1.0
+cmake@3.27.9           libyaml@0.2.5           py-chex@0.0.7                  py-opt-einsum@3.3.0         py-tomli@2.0.1
+cp2k@2024.1            llvm@16.0.6             py-contextlib2@21.6.0          py-packaging@23.1           py-toolz@0.12.0
+cp2k@2024.1            lua@5.3.6               py-cython@0.29.36              py-pandas@1.3.4             py-typing-extensions@4.8.0
+cuda@12.4.0            lz4@1.9.4               py-decorator@5.1.1             py-pdbfixer@1.7             py-urllib3@1.26.12
+fftw@3.3.10            mesa-glu@9.0.2          py-dm-haiku@0.0.9              py-pip@23.0                 py-websocket-client@1.6.3
+fftw@3.3.10            mesa18@18.3.6           py-dm-tree@0.1.6               py-pip@23.1.2               py-werkzeug@2.0.2
+freetype@2.10.2        metis@5.1.0             py-docker@5.0.3                py-ply@3.11                 py-wheel@0.37.1
+gcc-runtime@12.3.0     mithra@2.0              py-etils@0.9.0                 py-poetry-core@1.8.1        py-wheel@0.41.2
+gcc-runtime@12.3.0     netlib-scalapack@2.2.0  py-flatbuffers@23.5.26         py-protobuf@3.19.4          py-wrapt@1.15.0
+gettext@0.19.8.1       opal@master             py-flit-core@3.9.0             py-pyasn1@0.4.8             py-zipp@3.17.0
+gl2ps@1.4.2            openblas@0.3.25         py-gast@0.4.0                  py-pyasn1-modules@0.2.8     python@3.8.18
+googletest@1.12.1      openblas@0.3.26         py-google-auth@2.27.0          py-pybind11@2.12.0          python@3.11.7
+gsl@2.7.1              openblas@0.3.26         py-google-auth-oauthlib@0.4.6  py-pytest-runner@6.0.0      python-venv@1.0
+h5hut@2.0.0rc6         openjdk@11.0.20.1_1     py-google-pasta@0.2.0          py-python-dateutil@2.8.2    python-venv@1.0
+hdf5@1.12.3            openmm@7.5.1            py-grpcio@1.60.1               py-pythran@0.9.11           quantum-espresso@7.3.1
+hdf5@1.14.3            openmpi@4.1.6           py-h5py@3.11.0                 py-pytz@2023.3              re2@2023-09-01
+hdf5@1.14.3            openmpi@4.1.6           py-idna@3.4                    py-pyyaml@6.0               rust-bootstrap@1.78.0
+hh-suite@3.3.0         osmesa@11.2.0           py-immutabledict@2.2.1         py-requests@2.31.0          slurm@23-11-0-1
+hmmer@3.4              parmetis@4.0.3          py-importlib-resources@5.12.0  py-requests-oauthlib@1.3.1  swig@4.1.1
+hwloc@2.9.1            pcre@8.45               py-jax@0.3.25                  py-rsa@4.9                  trilinos@13.4.0
+jsoncpp@1.9.5          perl-data-dumper@2.173  py-jaxlib@0.3.25               py-scipy@1.7.0              unzip@6.0
+kalign@3.4.0           protobuf@3.19.4         py-jmp@0.0.2                   py-setuptools@57.4.0        zip@3.0
+libgcrypt@1.10.3       py-absl-py@1.0.0        py-libclang@16.0.0             py-setuptools@69.2.0
+libgpg-error@1.49      py-alphafold@2.3.2      py-mako@1.2.4                  py-setuptools@69.2.0
+libint@2.6.0           py-alphafold@2.3.2      py-markdown@3.3.4              py-setuptools-scm@8.0.4
+libjpeg@9f             py-astunparse@1.6.3     py-markupsafe@2.1.3            py-six@1.16.0
+
+-- linux-rhel7-x86_64 / oneapi@2024.1.0 -------------------------
+gromacs@2024.1  hwloc@2.9.1  intel-oneapi-mkl@2024.0.0  intel-oneapi-runtime@2024.1.0  intel-tbb@2021.9.0  openmpi@4.1.6  slurm@23-11-0-1
+==> 257 installed packages
+```
+
+The included specs and installed packages sections shows you all the packages that are pre-installed for you. </br>
+In case of a package having multiple installations, the following command helps you to find out which software you're interested about.  </br>
+
+For example let's say you are interested in py-alphafold but only in the gpu version; do:
 
 ```
+[unstable] user@login001:~> spack find -vl py-alphafold
+==> In environment unstable
+==> 1 root specs
+ -  dpjaszc visit
+
+==> Included specs
+-- no arch / gcc ------------------------------------------------
+------- gromacs%gcc
+
+-- no arch / gcc@12.3.0 -----------------------------------------
+------- cp2k@2024.1%gcc@12.3.0  ------- opal@master%gcc@12.3.0         ------- py-alphafold@2.3.2%gcc@12.3.0
+------- cp2k@2024.1%gcc@12.3.0  ------- py-alphafold@2.3.2%gcc@12.3.0  ------- quantum-espresso@7.3.1%gcc@12.3.0
+
+-- no arch / oneapi@2024.1.0 ------------------------------------
+------- gromacs@2024.1%oneapi@2024.1.0
+
+==> Installed packages
+-- linux-rhel7-x86_64 / gcc@12.3.0 ------------------------------
+rrfdppp py-alphafold@2.3.2~cuda build_system=python_pip  tqziprv py-alphafold@2.3.2+cuda build_system=python_pip cuda_arch=60
+==> 2 installed packages
+```
+
+As you can see there are two installation of py-alphafold, one with +cuda and one without. </br>
+In this particular example, there is only one variant that interests you, so you can do:
+
+``` shell
+spack load py-alphafold +cuda
+```
+
+However if multiple variants are interesting to you and you don't want to have to cite all of them you can do:
+
+``` shell
+spack load /tqziprv # refer to the installation with its hash directly.
+```
+
+## Install your own software
+
+1. Check if your package is already implemented in Spack. <br/>
+ You can also use the following website: https://packages.spack.io/
+
+```shell
+spack list $pkg_name # e.g opal
+```
+
+2. Check the package variants you want to set e.g +cuda +openmp ^openmpi ...
+
+```shell
+spack info $pkg_name # e.g opal
+```
+
+3. Add your package to your personal environment
+
+```shell
+spack add $pkg_name@version +variant1 ^dep1@version1
+```
+
++ If you want to install from local source clone your source under <br/>
+/afs/psi.ch/sys/spack/user/$USER/spack-environment/$pkg_name <br/>
+and tell Spack you want to use it using:
+
+```shell
+spack develop $pkg_name@version # Skip this if you don't want to develop from local source
+```
+4. Check the whole dependency tree of your spec and <br/>
+if you don't need to set more variants
+
+```shell
+spack concretize
+```
+
+5. Install the packages in your environment
+
+```shell
+spack install # -v for verbose
+```
+
+6. Load your package
+
+```shell
+spack load $pkg_name@version +variant1 ^dep1@version1
+```
+
+7. Submit your script
+
+```shell
+sbatch batch.script
+```
+## Switching between environments
+
+There are two environment available to users; stable and unstable. <br/>
+The first one is loaded by default when loading the module. <br/>
+If you want to switch between the two and use different software stack you can use the following command:
+
+```shell
+spack env activate -p unstable # or stable
+```
+
+## Managing your environments
+
+When adding the following command you actually add a package to your own software stack;
+
+```shell
+spack add $pkg_name@version +variant1 ^dep1@version1
+```
+
+You can check which package will be concretized and are in your environment using;
+
+```shell
+[unstable] user@login001:~> spack find
+==> In environment unstable
+==> 1 root specs
+ -  visit ~adios2+vtkm+gui
+
+==> Included specs
+-- no arch / gcc ------------------------------------------------
+gromacs%gcc
+
+-- no arch / gcc@12.3.0 -----------------------------------------
+cp2k@2024.1%gcc@12.3.0  opal@master%gcc@12.3.0         py-alphafold@2.3.2%gcc@12.3.0
+cp2k@2024.1%gcc@12.3.0  py-alphafold@2.3.2%gcc@12.3.0  quantum-espresso@7.3.1%gcc@12.3.0
+
+-- no arch / oneapi@2024.1.0 ------------------------------------
+gromacs@2024.1%oneapi@2024.1.0
+
+==> Installed packages
+...
+==> 257 installed packages
+```
+
+The root spec is the specs that are personal to you and that will be concretized and installed. <br/>
+The installed packages are either packages that were previously installed by admins or packages that you already installed but that are not root. <br/>
+
+If you want to remove this package from your personal env, you can use the following command: <br/>
+
+```shell
+spack rm $pkg_name@version +variant1 ^dep1@version1
+```
+
+or
+
+```shell
+[stable] user@login001:~> spack config edit
+spack:
+  ...
+  specs:
+--  - visit~adios2+gui+vtkm ^harfbuzz%gcc@12.3.0
+++ []
+  ...
+```
+and remove it manually from the spec list.
 
 ## How to use on other systems
 
 ### Install Spack
 
-```bash
+```shell
 user@supersystem:~> cd /scratch/$USER
 
 user@supersystem:~> git clone -c feature.manyFiles=true https://github.com/spack/spack.git

env/rhel7/x86_64/admin/stable/spack.yaml

@@ -1,20 +1,39 @@
----
 spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/stable
-  include_concrete:
+  definitions:
-    - $spack/spack-psi/env/$os/$target_family/admin/unstable
+    - compilers:
+      - gcc@=12.3.0
+      - intel-oneapi-compilers@=2024.1.0
+    - gcc_4.8.5:
+      - fpart@1.6.0/lcygtky
+    - gcc_12.3.0_openmpi:
+      - opal@master/l4dkgyf
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
   mirrors:
-    local_stable: file:///afs/psi.ch/software/spack/stable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+  modules:
-  specs: []
+    default:
+      roots:
+        tcl: $spack/opt/spack/stable/modulefiles
+      lmod:
+        core_compilers:
+        - gcc@=4.8.5
+  specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@4.8.5']
+  - matrix:
+    - [$gcc_4.8.5]
+    - ['%gcc@4.8.5']
+  - matrix:
+    - [$gcc_12.3.0_openmpi]
+    - [^openmpi%gcc]
+    - ['%gcc@12.3.0']
   view: false

env/rhel7/x86_64/admin/testing/spack.yaml

@@ -1,24 +1,21 @@
----
 spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/testing
-  upstreams:
+      # Strongly adviced to avoid problems with buildcaches
-    spack-instance-1:
+      padded_length: 256
-      install_tree: $spack/opt/spack/stable
-    spack-instance-2:
-      install_tree: $spack/opt/spack/unstable
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  #include_concrete:
+  #- /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/testing
   mirrors:
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_testing: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        tcl: $spack/opt/spack/testing/modulefiles
   specs:
-    - gcc@12.3.0
+  - opencv
   view: false

env/rhel7/x86_64/admin/unstable/spack.yaml

@@ -3,23 +3,44 @@ spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/unstable
-  upstreams:
+  definitions:
-    spack-instance-1:
+    - gcc_12.3.0:
-      install_tree: $spack/opt/spack/stable
+      - cellranger@8.0.1/6h63gx
-  include_concrete:
+      - fastqc@0.12.1/dkkcfu5
-    - $spack/spack-psi/env/$os/$target_family/admin/testing
+      - ffbidx@main/vyjua6z
+      - sratoolkit@3.1.1/2szh2jy
+      - star@2.7.11a/icumpki
+      - opencv@4.10.0/7fjszca
+    - gcc_12.3.0_openmpi:
+      - cp2k@2024.1/2bayb45 # ~cuda
+      - cp2k@2024.1/exkrpix # +cuda
+      - quantum-espresso@7.3.1/yuxkoz2 # hdf5=parallel
+      - py-alphafold@2.3.2/rrfdppp # ~cuda
+      - py-alphafold@2.3.2/tqziprv # +cuda
+    - oneapi_2024.1.0_openmpi:
+      - gromacs@2024.1/pi5t2pp # ~cuda
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/rhel7/$target_family/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
   mirrors:
-    local_stable: file:///afs/psi.ch/software/spack/stable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+  modules:
-    local_testing: file:///afs/psi.ch/software/spack/testing
+    default:
+      roots:
+        lmod: $spack/opt/spack/unstable/modulefiles
   specs:
-    - gcc@12.3.0
+  - matrix:
+    - [$gcc_12.3.0]
+    - ['%gcc@12.3.0']
+  - matrix:
+    - [$gcc_12.3.0_openmpi]
+    - [^openmpi%gcc]
+    - ['%gcc@12.3.0']
+  - matrix:
+    - [$oneapi_2024.1.0_openmpi]
+    - [^openmpi%oneapi]
+    - ['%oneapi@2024.1.0']
   view: false

env/rhel7/x86_64/sysconfigs/compilers.yaml

@@ -12,3 +12,29 @@ compilers:
         fc: /usr/bin/gfortran
       spec: gcc@4.8.5
       target: x86_64
+  - compiler:
+      spec: gcc@=12.3.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/gcc-12.3.0-zpn75bqd4uuxamtoonk5tkiproitfqpt/bin/gcc
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/gcc-12.3.0-zpn75bqd4uuxamtoonk5tkiproitfqpt/bin/g++
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/gcc-12.3.0-zpn75bqd4uuxamtoonk5tkiproitfqpt/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/gcc-12.3.0-zpn75bqd4uuxamtoonk5tkiproitfqpt/bin/gfortran
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: oneapi@=2024.1.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/intel-oneapi-compilers-2024.1.0-4xhusak77uwpnggnimldlea4qydtzxy7/compiler/2024.1/bin/icx
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/intel-oneapi-compilers-2024.1.0-4xhusak77uwpnggnimldlea4qydtzxy7/compiler/2024.1/bin/icpx
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/intel-oneapi-compilers-2024.1.0-4xhusak77uwpnggnimldlea4qydtzxy7/compiler/2024.1/bin/ifx
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-rhel7-x86_64/gcc-4.8.5/intel-oneapi-compilers-2024.1.0-4xhusak77uwpnggnimldlea4qydtzxy7/compiler/2024.1/bin/ifx
+      flags: {}
+      operating_system: rhel7
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []

env/rhel7/x86_64/sysconfigs/packages.yaml

@@ -1,12 +1,19 @@
 ---
 packages:
   all:
-    compiler: [gcc@4.8.5, nvhpc]
+    compiler: [gcc@=12.3.0, gcc@4.8.5, oneapi, nvhpc]
+    prefer:
+    - spec: '^openmpi'
     providers:
-      blas: [openblas, nvhpc]
+      blas: [openblas, intel-oneapi-mkl, nvhpc]
-      lapack: [openblas, nvhpc]
+      lapack: [openblas, intel-oneapi-mkl, nvhpc]
-      mpi: [openmpi]
+      mpi: [openmpi, mpich, intel-oneapi-mpi]
       pkgconfig: [pkg-config]
+    require:
+    - spec: 'target=sandybridge'
+      when: "%nvhpc"
+      message: "nvhpc compilers cannot target generic x86_64 microarchitecture."
+    - spec: "os=rhel7"
     target: [x86_64]
     # Default GPU arch
     variants:
@@ -31,10 +38,15 @@ packages:
     require: '%gcc'
   coreutils:
     require: '%gcc@4.8.5'
+  cp2k:
+    require: 'build_system=makefile'
   cpio:
     require: '%gcc'
   ctffind:
     require: '%gcc'
+  # Makes sure the binary works on any sytem (incl. Gwendolen)
+  cuda:
+    require: '@12.2:'
   curl:
     require: '%gcc@4.8.5'
   cvs:
@@ -42,7 +54,7 @@ packages:
   diffutils:
     require: '%gcc@4.8.5'
   doxygen:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   expat:
     require: '%gcc@4.8.5'
   file:
@@ -52,13 +64,15 @@ packages:
   flex:
     require: '%gcc@4.8.5'
   fltk:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   fontconfig:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
-  gams:
+  freetype:
     require: '%gcc@4.8.5'
   gawk:
     require: '%gcc@4.8.5'
+  gcc:
+    require: '%gcc@4.8.5'
   gdbm:
     require: '%gcc@4.8.5'
   gettext:
@@ -71,15 +85,19 @@ packages:
   git:
     require: '%gcc@4.8.5'
   glew:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   gmake:
     require: '%gcc@4.8.5'
   groff:
     require: '%gcc@4.8.5'
   harfbuzz:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   hcoll:
     require: '%gcc@4.8.5'
+  icu4c:
+    require: '@67.1%gcc@12.3.0'
+  intel-oneapi-compilers:
+    require: '%gcc@4.8.5'
   libbsd:
     require: '%gcc@4.8.5'
   libcroco:
@@ -109,25 +127,27 @@ packages:
   libreadline:
     require: '%gcc@4.8.5'
   librsvg:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   libsigsegv:
     require: '%gcc@4.8.5'
   libtiff:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   libtool:
     require: '%gcc@4.8.5'
   libx11:
     require: '%gcc@4.8.5'
+  libxcb:
+    require: '%gcc@4.8.5'
   libxext:
     require: '%gcc@4.8.5'
   libxft:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   libxml2:
     require: '%gcc@4.8.5'
   libxpm:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   libxt:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   m4:
     require: '%gcc@4.8.5'
   ncurses:
@@ -138,11 +158,20 @@ packages:
     require: '%gcc@4.8.5'
   numactl:
     require: '%gcc@4.8.5'
+  nvhpc:
+    require: '%gcc@4.8.5 install_type=network'
   openblas:
     require: '%gcc'
+  opengl:
+    buildable: False
+    externals:
+    - spec: opengl@1.7.0
+      prefix: /usr
   openmpi:
     require:
-      - schedulers=slurm fabrics=auto +pmi +cxx +cxx_exceptions +cuda +legacylaunchers
+      - spec: schedulers=slurm fabrics=auto +internal-pmix +cxx +cuda +legacylaunchers
+      - spec: '+cxx_exceptions'
+        when: '%gcc'
   openssh:
     require: '%gcc@4.8.5'
   openssl:
@@ -153,10 +182,10 @@ packages:
     require: '%gcc@4.8.5'
   pkg-config:
     require: '%gcc@4.8.5'
-  py-fypp:
+  py-numpy:
-    require: '%gcc@4.8.5'
+    require: '%gcc@12.3.0'
   python:
-    require: '%gcc@4.8.5'
+    require: '%gcc'
   readline:
     require: '%gcc@4.8.5'
   rsync:

env/rhel7/x86_64/users/stable/spack.yaml

@@ -1,17 +1,16 @@
 spack:
   concretizer:
     unify: false
-  config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
-    install_tree:
-      root: $spack/user/$user/spack-install
-    upstreams:
-      spack-instance-1:
-        install_tree: $spack/opt/spack/stable
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
-    specs:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: >-
+        $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/rhel7/x86_64/users/unstable/spack.yaml

@@ -1,19 +1,19 @@
 spack:
   concretizer:
     unify: false
-  config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
-    install_tree:
-      root: $spack/user/$user/spack-install
-    upstreams:
-      spack-instance-1:
-        install_tree: $spack/opt/spack/stable
-      spack-instance-2:
-        install_tree: $spack/opt/spack/unstable
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/rhel7/x86_64/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
-    specs:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+    unstable:
+      install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/rhel8/x86_64/admin/stable/spack.yaml

@@ -0,0 +1,36 @@
+spack:
+  concretizer:
+    unify: false
+  config:
+    install_tree:
+      root: $spack/opt/spack/stable
+  definitions:
+    - compilers:
+      - gcc@=12.3.0
+      - intel-oneapi-compilers@=2024.1.0
+    - gcc_4.8.5:
+      - fpart@1.6.0/lcygtky
+    - gcc_12.3.0_openmpi:
+      - opal@master/l4dkgyf
+  include:
+  - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        tcl: $spack/opt/spack/stable/modulefiles
+  specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@4.8.5']
+  - matrix:
+    - [$gcc_4.8.5]
+    - ['%gcc@4.8.5']
+  - matrix:
+    - [$gcc_12.3.0_openmpi]
+    - [^openmpi%gcc]
+    - ['%gcc@12.3.0']
+  view: false

env/rhel8/x86_64/admin/testing/spack.yaml

@@ -0,0 +1,21 @@
+---
+spack:
+  concretizer:
+    unify: false
+  config:
+    install_tree:
+      root: $spack/opt/spack/testing
+      # Strongly adviced to avoid problems with buildcaches
+      padded_length: 256
+  include:
+    - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/testing
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        tcl: $spack/opt/spack/testing/modulefiles
+  specs: []
+  view: false

env/rhel8/x86_64/admin/unstable/spack.yaml

@@ -0,0 +1,45 @@
+---
+spack:
+  concretizer:
+    unify: false
+  config:
+    install_tree:
+      root: $spack/opt/spack/unstable
+  definitions:
+    - gcc_12.3.0:
+      - cellranger@8.0.1/6h63gx
+      - fastqc@0.12.1/dkkcfu5
+      - ffbidx@main/vyjua6z
+      - sratoolkit@3.1.1/2szh2jy
+      - star@2.7.11a/icumpki
+    - gcc_12.3.0_openmpi:
+      - cp2k@2024.1/2bayb45 # ~cuda
+      - cp2k@2024.1/exkrpix # +cuda
+      - quantum-espresso@7.3.1/yuxkoz2 # hdf5=parallel
+      - py-alphafold@2.3.2/rrfdppp # ~cuda
+      - py-alphafold@2.3.2/tqziprv # +cuda
+    - oneapi_2024.1.0_openmpi:
+      - gromacs@2024.1/pi5t2pp # ~cuda
+  include:
+    - $spack/../spack-psi/env/rhel8/$target_family/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        tcl: $spack/opt/spack/unstable/modulefiles
+  specs:
+  - matrix:
+    - [$gcc_12.3.0]
+    - ['%gcc@12.3.0']
+  - matrix:
+    - [$gcc_12.3.0_openmpi]
+    - [^openmpi%gcc]
+    - ['%gcc@12.3.0']
+  - matrix:
+    - [$oneapi_2024.1.0_openmpi]
+    - [^openmpi%oneapi]
+    - ['%oneapi@2024.1.0']
+  view: false

env/rhel8/x86_64/sysconfigs/compilers.yaml

@@ -0,0 +1,14 @@
+compilers:
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags: {}
+      modules: []
+      operating_system: rhel8
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      spec: gcc@8.5.0
+      target: x86_64

env/rhel8/x86_64/sysconfigs/packages.yaml

@@ -0,0 +1,224 @@
+---
+packages:
+  all:
+    compiler: [gcc@8.5.0, oneapi, nvhpc]
+    prefer:
+    - spec: '^openmpi'
+    providers:
+      blas: [openblas, intel-oneapi-mkl, nvhpc]
+      lapack: [openblas, intel-oneapi-mkl, nvhpc]
+      mpi: [openmpi, mpich, intel-oneapi-mpi]
+      pkgconfig: [pkg-config]
+    require:
+    - spec: 'target=skylake_avx512'
+      when: "%nvhpc"
+      message: "nvhpc compilers cannot target generic x86_64 microarchitecture."
+    - spec: "os=rhel8"
+    target: [x86_64]
+    # Default GPU arch
+    variants:
+      - cuda_arch=60
+  autoconf:
+    require: '%gcc@8.5.0'
+  automake:
+    require: '%gcc@8.5.0'
+  berkeley-db:
+    require: '%gcc@8.5.0'
+  binutils:
+    require: '%gcc@8.5.0'
+  boost:
+    require: '%gcc'
+  bzip2:
+    require: '%gcc@8.5.0'
+  ca-certificates-mozilla:
+    require: '%gcc@8.5.0'
+  cairo:
+    require: '%gcc@8.5.0'
+  cmake:
+    require: '%gcc'
+  coreutils:
+    require: '%gcc@8.5.0'
+  cp2k:
+    require: 'build_system=makefile'
+  cpio:
+    require: '%gcc'
+  ctffind:
+    require: '%gcc'
+  # Makes sure the binary works on any sytem (incl. Gwendolen)
+  cuda:
+    require: '@12.2:'
+  curl:
+    require: '%gcc@8.5.0'
+  cvs:
+    require: '%gcc@8.5.0'
+  diffutils:
+    require: '%gcc@8.5.0'
+  doxygen:
+    require: '%gcc'
+  expat:
+    require: '%gcc@8.5.0'
+  file:
+    require: '%gcc@8.5.0'
+  findutils:
+    require: '%gcc@8.5.0'
+  flex:
+    require: '%gcc@8.5.0'
+  fltk:
+    require: '%gcc'
+  fontconfig:
+    require: '%gcc'
+  freetype:
+    require: '%gcc@8.5.0'
+  gams:
+    require: '%gcc@8.5.0'
+  gawk:
+    require: '%gcc@8.5.0'
+  gcc:
+    require: '%gcc@8.5.0'
+  gdbm:
+    require: '%gcc@8.5.0'
+  gettext:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: gettext@0.19.8.1
+  ghostscript:
+    require: '%gcc@8.5.0'
+  git:
+    require: '%gcc@8.5.0'
+  glew:
+    require: '%gcc'
+  gmake:
+    require: '%gcc@8.5.0'
+  groff:
+    require: '%gcc@8.5.0'
+  harfbuzz:
+    require: '%gcc'
+  hcoll:
+    require: '%gcc@8.5.0'
+  icu4c:
+    require: '@67.1%gcc@12.3.0'
+  intel-oneapi-compilers:
+    require: '%gcc@8.5.0'
+  libbsd:
+    require: '%gcc@8.5.0'
+  libcroco:
+    require: '%gcc@8.5.0'
+  libepoxy:
+    require: '%gcc@8.5.0'
+  libevent:
+    require: '%gcc@8.5.0'
+  libfuse:
+    require: '%gcc@8.5.0'
+  libice:
+    require: '%gcc@8.5.0'
+  libiconv:
+    require: '%gcc@8.5.0'
+  libidn2:
+    require: '%gcc@8.5.0'
+  libjpeg-turbo:
+    require: '%gcc@8.5.0'
+  libmd:
+    require: '%gcc@8.5.0'
+  libpciaccess:
+    require: '%gcc@8.5.0'
+  libpthread:
+    require: '%gcc@8.5.0'
+  libpthread-stubs:
+    require: '%gcc@8.5.0'
+  libreadline:
+    require: '%gcc@8.5.0'
+  librsvg:
+    require: '%gcc'
+  libsigsegv:
+    require: '%gcc@8.5.0'
+  libtiff:
+    require: '%gcc'
+  libtool:
+    require: '%gcc@8.5.0'
+  libx11:
+    require: '%gcc@8.5.0'
+  libxcb:
+    require: '%gcc@8.5.0'
+  libxext:
+    require: '%gcc@8.5.0'
+  libxft:
+    require: '%gcc'
+  libxml2:
+    require: '%gcc@8.5.0'
+  libxpm:
+    require: '%gcc@8.5.0'
+  libxt:
+    require: '%gcc'
+  m4:
+    require: '%gcc@8.5.0'
+  ncurses:
+    require: '%gcc@8.5.0'
+  ninja:
+    require: '%gcc@8.5.0'
+  nlohmann-json:
+    require: '%gcc@8.5.0'
+  numactl:
+    require: '%gcc@8.5.0'
+  nvhpc:
+    require: '%gcc@8.5.0 install_type=network'
+  openblas:
+    require: '%gcc'
+  openmpi:
+    require:
+      - spec: schedulers=slurm fabrics=auto pmi-interface=pmi2 +cxx +cuda +legacylaunchers
+      - spec: '+cxx_exceptions'
+        when: '%gcc'
+  openssh:
+    require: '%gcc@8.5.0'
+  openssl:
+    require: '%gcc@8.5.0'
+  pcre2:
+    require: '%gcc@8.5.0'
+  perl:
+    require: '%gcc@8.5.0'
+  pkg-config:
+    require: '%gcc@8.5.0'
+  py-numpy:
+    require: '%gcc@12.3.0'
+  python:
+    require: '%gcc'
+  readline:
+    require: '%gcc@8.5.0'
+  rsync:
+    require: '%gcc@8.5.0'
+  ruby:
+    require: '%gcc@8.5.0'
+  sed:
+    require: '%gcc@8.5.0'
+  slurm:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: slurm@24-05-2-1 +pmix +hwloc +hdf5 +mariadb +pam
+  sqlite:
+    require: '%gcc@8.5.0'
+  subversion:
+    require: '%gcc@8.5.0'
+  tar:
+    require: '%gcc@8.5.0'
+  texinfo:
+    require: '%gcc@8.5.0'
+  unuran:
+    require: '%gcc@8.5.0'
+  util-linux-uuid:
+    require: '%gcc@8.5.0'
+  util-macros:
+    require: '%gcc@8.5.0'
+  valgrind:
+    require: '%gcc@8.5.0'
+  xerces-c:
+    require: '%gcc@8.5.0'
+  xxhash:
+    require: '%gcc@8.5.0'
+  xz:
+    require: '%gcc@8.5.0'
+  zlib:
+    require: '%gcc@8.5.0'
+  zstd:
+    require: '%gcc@8.5.0'

env/rhel8/x86_64/users/stable/spack.yaml

@@ -0,0 +1,16 @@
+spack:
+  concretizer:
+    unify: false
+  include:
+    - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: >-
+        $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
+  view: false

env/rhel8/x86_64/users/unstable/spack.yaml

@@ -0,0 +1,19 @@
+spack:
+  concretizer:
+    unify: false
+  include:
+    - $spack/../spack-psi/env/rhel8/x86_64/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel8/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/rhel7/x86_64/admin/unstable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+    unstable:
+      install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
+  view: false

env/sles15/aarch64/admin/stable/spack.yaml

@@ -0,0 +1,45 @@
+---
+spack:
+  concretizer:
+    unify: false
+  config:
+    install_tree:
+      root: $spack/opt/spack/stable
+  definitions:
+  - compilers:
+    - gcc@13.2.0
+    - gcc@7.5.0
+    - gcc@=12.3
+    - nvhpc@24.3
+    - cce@17.0.0
+  include:
+    - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        lmod: $spack/opt/spack/stable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+        core_compilers:
+        - gcc@=7.5.0
+  specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@7.5.0']
+  view: false

env/sles15/aarch64/admin/testing/spack.yaml

@@ -0,0 +1,62 @@
+spack:
+  concretizer:
+    unify: when_possible
+    targets:
+      granularity: microarchitectures
+    reuse:
+      from:
+      - type: local
+      - type: external
+  config:
+    install_tree:
+      root: $spack/opt/spack/testing
+      # Strongly adviced to avoid problems with buildcaches
+      padded_length: 256
+  include:
+  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        lmod: $spack/opt/spack/testing/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+  packages:
+    # overwrite any existing compiler
+    'cce:':
+      buildable: true
+    'gcc:':
+      buildable: true
+    'nvhpc:':
+      buildable: true
+  specs: []
+  view: false
+  compilers:
+  - compiler:
+      spec: gcc@=7.5.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []

env/sles15/aarch64/admin/unstable/spack.yaml

@@ -0,0 +1,141 @@
+spack:
+  concretizer:
+    unify: false
+  config:
+    install_tree:
+      root: $spack/opt/spack/unstable
+  definitions:
+  - compilers_gcc_7.5.0:
+    - gcc@13.2.0
+    - gcc@7.5.0
+    - gcc@=12.3
+    - gcc@14.2.0/tln2ck4
+    - nvhpc@24.3
+    - cce@17.0.0
+  - compilers_gcc_13.2.0:
+    - nvhpc@24.11/6wi4jhx
+  - gcc_7.5.0:
+    - ffbidx@main/3xig2i
+  - gcc_13.2.0:
+    - dssp@2.3.0/xfc2pth
+  - gcc_14.2.0:
+    - osu-micro-benchmarks@7.5/mit3iil
+    - osu-micro-benchmarks@7.5/6ycu464
+    - aws-ofi-nccl@1.13.0/nz3b3yk
+  - gcc_12.3_openmpi:
+    - gromacs@2024.1/x33hfjx
+    - cp2k@2024.3/m3eow6e #+cuda
+    - relion@4.0.1/rudfcxt
+    - relion@5.0.0/oyf4n7i
+  - gcc_12.3_cray_mpich:
+    - relion@5.0.0/wk4d62r
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/ttg7vdz
+  - gcc_14.2_openmpi:
+    - py-torchvision@0.20.1/ysfcgu5
+    - py-torchaudio@2.5.1/trrglyu
+  - nvhpc_24.11_openmpi:
+    - quantum-espresso@7.4/oed4w6w  #+cuda +mpigpu hdf5=parallel
+    - quantum-espresso@7.3.1/d5wpqzw
+  include:
+  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  mirrors:
+    local_psi: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        lmod: $spack/opt/spack/unstable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+        openmpi@main:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              FI_HMEM_CUDA_USE_DMABUF: '1'
+              OMPI_MCA_coll: '^han'
+        core_compilers:
+        - gcc@=7.5.0
+  specs:
+  - matrix:
+    - [$compilers_gcc_7.5.0]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$compilers_gcc_13.2.0]
+    - ['%gcc@13.2.0']
+  - matrix:
+    - [$gcc_7.5.0]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$gcc_13.2.0]
+    - ['%gcc@13.2.0']
+  - matrix:
+    - [$gcc_14.2.0]
+    - ['%gcc@14.2.0']
+  - matrix:
+    - [$gcc_12.3_cray_mpich]
+    - ['%gcc@12.3']
+    - [^cray-mpich]
+  - matrix:
+    - [$gcc_12.3_openmpi]
+    - ['%gcc@12.3']
+    - [^openmpi]
+  - matrix:
+    - [$gcc_13.2_openmpi]
+    - ['%gcc@13.2']
+    - [^openmpi]
+  - matrix:
+    - [$gcc_14.2_openmpi]
+    - ['%gcc@14.2']
+    - [^openmpi]
+  - matrix:
+    - [$nvhpc_24.11_openmpi]
+    - ['%nvhpc@24.11']
+    - [^openmpi]
+  view: false
+  compilers:
+  - compiler:
+      spec: gcc@=14.2.0
+      paths:
+        cc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gcc
+        cxx: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/g++
+        f77: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-7.5.0/gcc-14.2.0-tln2ck4lolcipi2fj2klu5dei3oac4sv/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: gcc@=7.5.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+  - compiler:
+      spec: gcc@=12.3.0
+      paths:
+        cc: /usr/bin/gcc-12
+        cxx: /usr/bin/g++-12
+        f77: /usr/bin/gfortran-12
+        fc: /usr/bin/gfortran-12
+      flags: {}
+      operating_system: sles15
+      target: aarch64
+      modules: []
+      environment: {}
+      extra_rpaths: []

env/sles15/aarch64/sysconfigs/compilers.yaml

@@ -0,0 +1,31 @@
+compilers:
+- compiler:
+    spec: gcc@=13.2.0
+    paths:
+      cc: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gcc
+      cxx: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/g++
+      f77: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gfortran
+      fc: /afs/psi.ch/sys/spack/system/opt/spack/stable/linux-sles15-aarch64/gcc-7.5.0/gcc-13.2.0-a3pki4ocu2ipqupkaf6ghg6hry4pkxg3/bin/gfortran
+    flags: {}
+    operating_system: sles15
+    target: aarch64
+    modules:
+    - libfabric/1.22.0
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: nvhpc@=24.11
+    paths:
+      cc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvc
+      cxx: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvc++
+      f77: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvfortran
+      fc: /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/compilers/bin/nvfortran
+    flags: {}
+    operating_system: sles15
+    target: aarch64
+    modules:
+      - libfabric/1.22.0
+      - /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/modulefiles/nvhpc-nompi/24.11
+    environment: {}
+    extra_rpaths:
+      - /afs/psi.ch/sys/spack/develop/opt/spack/unstable/linux-sles15-aarch64/gcc-13.2.0/nvhpc-24.11-6wi4jhxcgbkzgrw6fgrutvten74l5ovl/Linux_aarch64/24.11/math_libs/lib64/stubs

env/sles15/aarch64/sysconfigs/packages.yaml

@@ -0,0 +1,346 @@
+packages:
+  all:
+    compiler: [gcc, nvhpc, cce]
+    prefer:
+    - spec: '^openmpi'
+      when: '^mpi'
+    - spec: '^openblas'
+      when: '%gcc ^blas'
+    - spec: '^netlib-scalapack'
+      when: '+scalapack'
+    providers:
+      blas: [openblas, nvhpc, nvpl-blas]
+      lapack: [openblas, nvhpc, nvpl-lapack]
+      mpi: [openmpi, cray-mpich]
+      pkgconfig: [pkg-config]
+      scalapack: [netlib-scalapack]
+    require:
+    - spec: "os=sles15"
+    - spec: "target=aarch64"
+      when: "%gcc"
+    - spec: "target=neoverse_v2"
+      when: "%nvhpc"
+    # Grace Hopper arch
+    variants:
+      - cuda_arch=90
+  berkeley-db:
+    require: '%gcc@7.5.0'
+  boost:
+    require: '%gcc'
+  cassini-headers:
+    require: '@main'
+  ca-certificates-mozilla:
+    require: '%gcc@7.5.0'
+  cce:
+    externals:
+      - modules:
+        - libfabric/1.22.0
+        - PrgEnv-cray/8.5.0
+        - cce/17.0.0
+        spec: cce@=17.0.0
+        extra_attributes:
+          compilers:
+            c: cc
+            cxx: CC
+            fortran: ftn
+          operating_system: sles15
+          target: aarch64
+  cmake:
+    require: '%gcc@7.5.0'
+  coreutils:
+    require: '%gcc@7.5.0'
+  cray-libsci:
+    buildable: false
+    externals:
+      - modules:
+        - cray-libsci
+        prefix: /opt/cray/pe/libsci/23.12.5/cray/17.0/aarch64
+        spec: cray-libsci@23.12.5%cce@17.0.0 +mpi +openmp
+      - modules:
+        - cray-libsci
+        prefix: /opt/cray/pe/libsci/23.12.5/GNU/12.3/aarch64/
+        spec: cray-libsci@23.12.5%gcc@12.3 +mpi +openmp
+  cray-mpich:
+    buildable: false
+    externals:
+      - modules:
+          - PrgEnv-cce
+          - cray-mpich
+        prefix: /opt/cray/pe/mpich/8.1.28/ofi/cray/17.0
+        spec: cray-mpich@8.1.28%cce@17.0.0
+      - modules:
+          - PrgEnv-gnu
+          - cray-mpich
+        prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3
+        spec: cray-mpich@8.1.28%gcc@12.3
+  cray-pmi:
+    buildable: false
+    externals:
+      - modules:
+          - cray-pmi
+        prefix: /opt/cray/pe/pmi/6.1.13
+        spec: cray-pmi@6.1.13%gcc
+  cuda:
+    prefer:
+    - spec: '@12.8.0'
+    require: '@12.2.0:'
+    externals:
+      - modules:
+        - nvhpc-nompi/24.3
+        prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/cuda
+        spec: cuda@12.2.0%nvhpc@24.3
+        extra_attributes:
+          environment:
+            prepend_path:
+              LD_LIBRARY_PATH: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/math_libs/lib64/stubs
+  curl:
+    require: '%gcc@7.5.0'
+  cxi-driver:
+    require: '@main'
+  egl:
+    buildable: False
+    externals:
+      - spec: egl@1.5.0
+        prefix: /usr/
+  expat:
+    require: '%gcc@7.5.0'
+  fftw:
+    externals:
+      - modules:
+        - cray-fftw/3.3.10.6
+        prefix: /opt/cray/pe/fftw/3.3.10.6/aarch64
+        spec: fftw@3.3.10.6+mpi+openmp~pfft_patches precision=double,float%gcc@12.3
+  findutils:
+    require: '%gcc@7.5.0'
+  flex:
+    require: '%gcc@7.5.0'
+  fltk:
+    require: '%gcc@7.5.0'
+  fontconfig:
+    require: '%gcc@7.5.0'
+  gawk:
+    require: '%gcc@7.5.0'
+  gcc:
+    externals:
+      - prefix: /usr
+        spec: gcc@7.5.0
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++
+            fortran: /usr/bin/gfortran
+          operating_system: sles15
+          target: aarch64
+        modules:
+          - libfabric/1.22.0
+      - prefix: /usr
+        spec: gcc@=12.3
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+            fortran: /usr/bin/gfortran-12
+          operating_system: sles15
+          target: aarch64
+        modules:
+          - libfabric/1.22.0
+          - PrgEnv-gnu/8.5.0
+          - gcc-native/12.3
+  gdbm:
+    require: '%gcc@7.5.0'
+  gettext:
+    require: '%gcc@7.5.0'
+  git:
+    require: '%gcc@7.5.0'
+  gmake:
+    require: '%gcc@7.5.0'
+  groff:
+    require: '%gcc@7.5.0'
+  libbsd:
+    require: '%gcc@7.5.0'
+  libcxi:
+    require: '@main'
+  libevent:
+    require: '%gcc@7.5.0'
+  # Crayism
+  libfabric:
+    buildable: true
+    require: '@main+cuda+uring fabrics=cxi,sockets,tcp,udp,xpmem'
+    externals:
+      - modules:
+          - libfabric/1.22.0 
+        prefix: /opt/cray/libfabric/1.22.0
+        spec: libfabric@1.22.0
+  libfuse:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: libfuse@2.9.7
+  libiconv:
+    require: '%gcc@7.5.0'
+  libidn2:
+    require: '%gcc@7.5.0'
+  libmd:
+    require: '%gcc@7.5.0'
+  libpciaccess:
+    require: '%gcc@7.5.0'
+  libsigsegv:
+    require: '%gcc@7.5.0'
+  librsvg:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: librsvg@2.48.0
+  libtool:
+    require: '%gcc@7.5.0'
+  libxml2:
+    require: '%gcc@7.5.0'
+  llvm:
+    require: '%gcc@7.5.0'
+  m4:
+    require: '%gcc@7.5.0'
+  munge:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: munge@0.5.15
+  ncurses:
+    require: '%gcc@7.5.0'
+  netcdf-c:
+    externals:
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
+        spec: netcdf-c@4.9.0.9~dap~hdf4~jna~mpi~parallel-netcdf+shared%cce@17.0.0
+      - modules:
+        - cray-netcdf-hdf5parallel/4.9.0.9
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/crayclang/17.0
+        spec: netcdf-c@4.9.0.9~dap~hdf4~jna+mpi~parallel-netcdf+shared%cce@17.0.0
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
+        spec: netcdf-c@4.9.0.9~dap~hdf4~jna~mpi~parallel-netcdf+shared%gcc@12.3
+      - modules:
+        - cray-netcdf-hdf5parallel/4.9.0.9
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
+        spec: netcdf-c@4.9.0.9~dap~hdf4~jna+mpi~parallel-netcdf+shared%gcc@12.3
+  netcdf-fortran:
+    externals:
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
+        spec: netcdf-fortran@4.9.0.9~doc+shared%cce@17.0.0
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
+        spec: netcdf-fortran@4.9.0.9~doc+shared%gcc@12.3
+  ninja:
+    require: '%gcc@7.5.0'
+  numactl:
+    require: '%gcc@7.5.0'
+  nvhpc:
+    externals:
+      - prefix: /opt/nvidia/hpc_sdk
+        spec: nvhpc@24.3
+        extra_attributes:
+          compilers:
+            c: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc
+            cxx: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvc++
+            fortran: /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/compilers/bin/nvfortran
+          operating_system: sles15
+          modules: nvhpc-nompi/24.3
+          extra_rpaths:
+            - /opt/nvidia/hpc_sdk/Linux_aarch64/24.3/math_libs/lib64/stubs
+  opal:
+    require:
+      - spec: '%gcc'
+      - message: Opal 2022.1.0 only builds with gcc@10.4.0
+        spec: '%gcc@10.4.0 ^openmpi'
+        when: '@2022.1.0'
+  opengl:
+    require: '%gcc@7.5.0'
+  openldap:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: openldap@2.4.46
+  openmpi:
+    require:
+      - spec: "schedulers=slurm fabrics=ofi,xpmem +cuda +internal-pmix"
+        when: "@5.0.0:"
+      - spec:  "schedulers=slurm fabrics=ofi,xpmem +cxx +cuda +legacylaunchers"
+        when: "@:5.0.0"
+  openssh:
+    require: '%gcc@7.5.0'
+  openssl:
+    require: '%gcc@7.5.0'
+  parallel-netcdf:
+    externals:
+      - modules:
+        - cray-parallel-netcdf/1.12.3.9
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/crayclang/17.0
+        spec: parallel-netcdf@1.12.3.9+cxx+fortran%cce@17.0.0
+      - modules:
+        - cray-parallel-netcdf/1.12.3.9
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
+        spec: parallel-netcdf@1.12.3.9+cxx+fortran%gcc@12.3
+  perl:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: perl@5.26.1
+  pkg-config:
+    require: '%gcc@7.5.0'
+  pkgconf:
+    require: '%gcc@7.5.0'
+  pmix:
+    require: '+munge'
+  py-fypp:
+    require: '%gcc@7.5.0'
+  python:
+    require: '%gcc@7.5.0'
+  quantum-espresso:
+    require:
+      - spec: "^nvpl-blas ^nvpl-lapack"
+        when: "%nvhpc"
+    conflict:
+    - spec: '^acfl'
+      when: '%nvhpc'
+  rdma-core:
+    require: '%gcc@7.5.0'
+  readline:
+    require: '%gcc@7.5.0'
+  slurm:
+    buildable: false
+    externals:
+      - prefix: /usr
+        spec: slurm@24.05.3%gcc@7.5.0 +cgroup+gtk+hwloc+mariadb+nvml+pam+pmix+restd
+  sqlite:
+    require: '%gcc@7.5.0'
+  tar:
+    require: '%gcc@7.5.0'
+  texinfo:
+    require: '%gcc@7.5.0'
+  unuran:
+    require: '%gcc@7.5.0'
+  util-linux-uuid:
+    require: '%gcc@7.5.0'
+  util-macros:
+    require: '%gcc@7.5.0'
+  xpmem:
+    buildable: false
+    externals:
+      - spec: xpmem@2.9.6-1.1
+        prefix: /usr
+        modules:
+        - xpmem/2.9.6-1.1_20240511212657__g087dc11fc19d
+  xxhash:
+    require: '%gcc@7.5.0'
+  xz:
+    externals:
+      - prefix: /usr
+        spec: xz@5.2.3
+  zlib:
+    require: '%gcc@7.5.0'
+  zstd:
+    require: '%gcc@7.5.0'

env/sles15/aarch64/users/stable/spack.yaml

@@ -0,0 +1,14 @@
+spack:
+  concretizer:
+    unify: false
+  include:
+    - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
+  view: false

env/sles15/aarch64/users/unstable/spack.yaml

@@ -0,0 +1,17 @@
+spack:
+  concretizer:
+    unify: false
+  include:
+  - $spack/../spack-psi/env/sles15/aarch64/sysconfigs/packages.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+    unstable:
+      install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
+  view: false

env/sles15/x86_64/admin/stable/spack.yaml

@@ -3,18 +3,58 @@ spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/stable
-  include_concrete:
+  definitions:
-    - $spack/spack-psi/env/$os/$target_family/admin/unstable
+  - compilers:
+    - gcc@7.5.0
+    - gcc@=12.3.0
+    - intel-oneapi-compilers@=2024.1.0
+    - intel-oneapi-compilers@=2022.1.0
+    - gcc@=13.2.0
+    - nvhpc@23.9
+    - cce@17.0.0
+  - gcc_7.5.0:
+    - fpart@1.6.0/di4akxx
+  - gcc_12.3_mpich:
+    - relion@4.0.1/ve5sish     #+cuda
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/q4ulqv3 #+cuda cuda_arch=80
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+    - $spack/../spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
   mirrors:
-    local_stable: file:///afs/psi.ch/software/spack/stable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+  modules:
-  specs: []
+    default:
+      roots:
+        lmod: $spack/opt/spack/stable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              LD_LIBRARY_PATH: '/opt/cray/libfabric/1.22.0/lib64'
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+  specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$gcc_7.5.0]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$gcc_12.3_mpich]
+    - ['%gcc@12.3']
+    - [^cray-mpich]
+  - matrix:
+    - [$gcc_13.2_openmpi]
+    - ['%gcc@13.2']
+    - [^openmpi]
   view: false

env/sles15/x86_64/admin/testing/spack.yaml

@@ -2,24 +2,31 @@ spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/testing
-  upstreams:
+      # Strongly adviced to avoid problems with buildcaches
-    spack-instance-1:
+      padded_length: 256
-      install_tree: $spack/opt/spack/stable
-    spack-instance-2:
-      install_tree: $spack/opt/spack/unstable
   include:
-  - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+  - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/testing
   mirrors:
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_testing: file:///afs/psi.ch/software/spack/testing
+  modules:
+    default:
+      roots:
+        lmod: $spack/opt/spack/testing/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              LD_LIBRARY_PATH: /opt/cray/libfabric/1.22.0/lib64
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: pmix
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
   specs:
-    - cp2k@2024.1%gcc~cuda
+  - 
-    - opal@master%gcc
-    - opal@master%gcc ^openmpi
   view: false

env/sles15/x86_64/admin/unstable/spack.yaml

@@ -2,22 +2,103 @@ spack:
   concretizer:
     unify: false
   config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
     install_tree:
       root: $spack/opt/spack/unstable
-  upstreams:
+  definitions:
-    spack-instance-1:
+  - compilers:
-      install_tree: $spack/opt/spack/stable
+    - gcc@7.5.0
-  include_concrete:
+    - gcc@9.5.0/ow3yhui
-  - $spack/spack-psi/env/$os/$target_family/admin/testing
+    - gcc@=12.3
+    - intel-oneapi-compilers@=2024.1.0
+    - intel-oneapi-compilers@=2022.1.0
+    - gcc@=13.2.0 arch=linux-sles15-x86_64
+    - nvhpc@23.9
+    - cce@17.0.0
+  - gcc_7.5.0:
+    - fpart@1.6.0/di4akxx
+    - visit@3.3.3/fo37wwb
+  - gcc_12.3:
+    - fastqc@0.12.1/eyuqqvf
+    - ffbidx@main/lfacuhw
+    - star@2.7.11a/gdgr2ea
+    - sratoolkit@3.1.1/7oqp6a3
+  - gcc_13.2:
+    - dmtcp@3.0.0/e76t6c5
+    - gams@49.1/vztjqs4
+  - gcc_13.2_openmpi:
+    - opal-x@fixSolverUnits/q4ulqv3 #+cuda cuda_arch=80
+  - gcc_12.3_mpich:
+    - cp2k@2024.1/2jesise     #~cuda
+    - cp2k@2024.1/zqm5fme     #+cuda
+    - relion@4.0.1/ve5sish
+    - quantum-espresso@7.3.1/n6mulig     #~cuda
+    - opal@2024.1.0/mhg3mdx
+    - py-alphafold@2.3.2/xkw7tfm     #+cuda
+    - sundials@7.0.0/xxysszi #sundials@7.0.0+ARKODE+CVODE+CVODES+IDA+IDAS+KINSOL+cuda+examples+examples-install~f2003~fcmix~ginkgo~hypre~int64~ipo~klu~kokkos~kokkos-kernels~lapack~magma~monitoring+mpi+openmp~petsc~profiling~pthread~raja~rocm+shared+static~superlu-dist~superlu-mt~sycl~trilinos build_system=cmake build_type=Release cstd=99 cuda_arch=80 cxxstd=14 generator=make logging-level=2 precision=double
+  - gcc_9.5.0_openmpi:
+    - q-e-sirius@develop-ristretto/bb4v7wz
+  - gcc_12.3_openmpi:
+    - gromacs@2024.1/3n7qspb     #+cuda
+    - hdf5@1.14.3/wqcdnm7 #+mpi ^openmpi@5.0.3%gcc@12.3
+  - intel_2021.6.0:
+    - hdf5@1.14.3/yrxkczx
+  - nvhpc_23.9_openmpi:
+    - quantum-espresso@7.3.1/le35al7 #+cuda
   include:
-  - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
-  - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
+  - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
   mirrors:
-    local_stable: file:///afs/psi.ch/software/spack/stable
+    local_psi: file:///afs/psi.ch/software/spack/testing
-    local_unstable: file:///afs/psi.ch/software/spack/unstable
+  modules:
-    local_testing: file:///afs/psi.ch/software/spack/testing
+    default:
-  specs: []
+      roots:
+        lmod: $spack/opt/spack/unstable/modulefiles
+      lmod:
+        cray-mpich:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              LD_LIBRARY_PATH: /opt/cray/libfabric/1.22.0/lib64
+        openmpi:
+          environment:
+            set:
+              SLURM_MPI_TYPE: 'pmix'
+              OMPI_MCA_mtl_ofi_disable_hmem: 'true'
+  specs:
+  - matrix:
+    - [$compilers]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$gcc_7.5.0]
+    - ['%gcc@7.5.0']
+  - matrix:
+    - [$gcc_12.3]
+    - ['%gcc@12.3']
+  - matrix:
+    - [$gcc_13.2]
+    - ['%gcc@13.2']
+  - matrix:
+    - [$gcc_13.2_openmpi]
+    - ['%gcc@13.2']
+    - [^openmpi]
+  - matrix:
+    - [$gcc_12.3_mpich]
+    - ['%gcc@12.3']
+    - [^cray-mpich]
+  - matrix:
+    - [$gcc_9.5.0_openmpi]
+    - ['%gcc@9.5.0']
+    - [^openmpi]
+  - matrix:
+    - [$gcc_12.3_openmpi]
+    - ['%gcc@12.3']
+    - [^openmpi]
+  - matrix:
+    - [$intel_2021.6.0]
+    - ['%intel@2021.6.0']
+  - matrix:
+    - [$nvhpc_23.9_openmpi]
+    - ['%nvhpc@23.9']
+    - [^openmpi]
   view: false

env/sles15/x86_64/sysconfigs/compilers.yaml

@@ -1,62 +1,57 @@
 compilers:
 - compiler:
-    spec: cce@=17.0.0
+      spec: intel@=2021.6.0
       paths:
-      cc: cc
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/icc
-      cxx: CC
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/icpc
-      f77: ftn
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/ifort
-      fc: ftn
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/intel64/ifort
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
-      - PrgEnv-cray/8.5.0
-      - cce/17.0.0
       environment: {}
       extra_rpaths: []
 - compiler:
-    spec: gcc@=12.3
+      spec: oneapi@=2022.1.0
       paths:
-      cc: /usr/bin/gcc-12
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/icx
-      cxx: /usr/bin/g++-12
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/intel-oneapi-compilers-2022.1.0-bf6lcbnmvgeb57jaihsfu22xlbgmmqe7/compiler/2022.1.0/linux/bin/icpx
-      f77: /usr/bin/gfortran-12
+        f77: null
-      fc: /usr/bin/gfortran-12
+        fc: null
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
-      - PrgEnv-gnu/8.5.0
-      - gcc-native/12.3
       environment: {}
       extra_rpaths: []
 - compiler:
-    spec: gcc@=7.5.0
+      spec: gcc@=13.2.0
       paths:
-      cc: /usr/bin/gcc
+        cc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gcc
-      cxx: /usr/bin/g++
+        cxx: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/g++
-      f77: /usr/bin/gfortran
+        f77: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gfortran
-      fc: /usr/bin/gfortran
+        fc: /afs/psi.ch/sys/spack/0.22.0/opt/spack/stable/linux-sles15-x86_64/gcc-7.5.0/gcc-13.2.0-2k3w3v2u5u6bfnoxl5rgnnna4on3lku7/bin/gfortran
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-    - libfabric/1.15.2.0
+      - libfabric/1.22.0
       environment: {}
       extra_rpaths: []
 - compiler:
-    spec: nvhpc@=22.3
+      spec: gcc@=9.5.0
       paths:
-      cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/nvc
+        cc: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gcc
-      cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/nvc++
+        cxx: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/g++
-      f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/nvfortran
+        f77: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gfortran
-      fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/nvfortran
+        fc: /afs/psi.ch/sys/spack/0.23.0/opt/spack/unstable/linux-sles15-x86_64/gcc-7.5.0/gcc-9.5.0-ow3yhuibzp7u753hyh5xngfgegxgijxp/bin/gfortran
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
-      - libfabric/1.15.2.0
+      - libfabric/1.22.0
-      - nvhpc-nompi/22.3
       environment: {}
       extra_rpaths: []

env/sles15/x86_64/sysconfigs/packages.yaml

@@ -1,17 +1,26 @@
 packages:
   all:
-    compiler: [gcc@12.3, cce, nvhpc]
+    compiler: [gcc, nvhpc, cce]
+    prefer:
+    - spec: '^openmpi'
+      when: '^mpi'
     providers:
-      blas: [cray-libsci, openblas, nvhpc]
+      blas: [openblas, nvhpc, cray-libsci]
-      lapack: [cray-libsci, openblas, nvhpc]
+      lapack: [openblas, nvhpc, cray-libsci]
-      mpi: [cray-mpich, openmpi]
+      mpi: [openmpi, cray-mpich]
       pkgconfig: [pkg-config]
       scalapack: [cray-libsci]
-    target: [x86_64]
+    require:
-    # Grace Hopper arch
+      - spec: "os=sles15"
+      - spec: "target=x86_64"
+        when: "%gcc"
+      - spec: "target=zen3"
+        when: "%nvhpc"
     variants:
-      - cuda_arch=90
+      - cuda_arch=80
-  amrex:
+  autoconf:
+    require: '%gcc@7.5.0'
+  automake:
     require: '%gcc@7.5.0'
   berkeley-db:
     require: '%gcc@7.5.0'
@@ -19,45 +28,77 @@ packages:
     require: '%gcc@7.5.0'
   ca-certificates-mozilla:
     require: '%gcc@7.5.0'
+  cce:
+    externals:
+      - modules:
+        - libfabric/1.22.0
+        - PrgEnv-cray/8.5.0
+        - cce/17.0.0
+        spec: cce@=17.0.0
+        extra_attributes:
+          compilers:
+            c: cc
+            cxx: CC
+            fortran: ftn
+          operating_system: sles15
+          target: x86_64
   cairo:
     require: '%gcc@7.5.0'
+  cassini-headers:
+    require: '@main'
   cmake:
-    require: '%gcc@7.5.0'
+    require: '%gcc'
   coreutils:
     require: '%gcc@7.5.0'
   cray-libsci:
     buildable: false
     externals:
-      - prefix: /opt/cray/pe/libsci/23.12.5/GNU/12.3/x86_64/
+      - modules:
+        - cray-libsci
+        prefix: /opt/cray/pe/libsci/23.12.5/cray/17.0/x86_64
         spec: cray-libsci@23.12.5%cce@17.0.0 +mpi +openmp
-      - prefix: /opt/cray/pe/libsci/23.12.5/GNU/12.3/x86_64/
+      - modules:
+        - cray-libsci
+        prefix: /opt/cray/pe/libsci/23.12.5/GNU/12.3/x86_64/
         spec: cray-libsci@23.12.5%gcc@12.3 +mpi +openmp
   cray-mpich:
     buildable: false
     externals:
       - modules:
-          - PrgEnv-cce/17.0.0
           - cray-mpich
+        prefix: /opt/cray/pe/mpich/8.1.28/ofi/cray/17.0
         spec: cray-mpich@8.1.28%cce@17.0.0
       - modules:
-          - PrgEnv-gnu/8.5.0
           - cray-mpich
+        prefix: /opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3
         spec: cray-mpich@8.1.28%gcc@12.3
   cray-pmi:
     buildable: false
     externals:
       - modules:
           - cray-pmi
+        prefix: /opt/cray/pe/pmi/6.1.13
         spec: cray-pmi@6.1.13%gcc
-  ctffind:
+  cuda:
-    require: '%gcc@7.5.0'
+    prefer:
+    - spec: '@12.8.0'
+    require: '@12.2.0:'
+    externals:
+      - modules:
+          - nvhpc-nompi/23.9
+        prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda
+        spec: cuda@12.2.0%nvhpc@23.9
   curl:
     require: '%gcc@7.5.0'
+  cxi-driver:
+    require: '@main'
   expat:
     require: '%gcc@7.5.0'
   fftw:
     externals:
-      - prefix: /opt/cray/pe/fftw/3.3.10.6/x86_64
+      - modules:
+        - cray-fftw/3.3.10.6
+        prefix: /opt/cray/pe/fftw/3.3.10.6/x86_64
         spec: fftw@3.3.10.6+mpi+openmp~pfft_patches precision=double,float%gcc@12.3
   findutils:
     require: '%gcc@7.5.0'
@@ -70,38 +111,60 @@ packages:
   gawk:
     require: '%gcc@7.5.0'
   gcc:
-    require: '%gcc@7.5.0'
+    externals:
+      - prefix: /usr
+        spec: gcc@7.5.0
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++
+            fortran: /usr/bin/gfortran
+          operating_system: sles15
+          target: x86_64
+        modules:
+          - libfabric/1.22.0
+      - prefix: /usr
+        spec: gcc@=12.3
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc-12
+            cxx: /usr/bin/g++-12
+            fortran: /usr/bin/gfortran-12
+          operating_system: sles15
+          target: x86_64
+        modules:
+          - libfabric/1.22.0
+          - PrgEnv-gnu/8.5.0
+          - gcc-native/12.3
   gdbm:
     require: '%gcc@7.5.0'
   gettext:
-    require: '%gcc@7.5.0'
+    externals:
+      - prefix: /usr
+        spec: 'gettext@0.20.2'
   git:
     require: '%gcc@7.5.0'
   gmake:
     require: '%gcc@7.5.0'
   groff:
     require: '%gcc@7.5.0'
-  hdf5:
+  gtkplus:
-    externals:
+    require: 'default_library=shared,static'
-      - prefix: /opt/cray/pe/hdf5/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%cce@17.0.0
-      - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/crayclang/17.0
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%cce@17.0.0
-      - prefix: /opt/cray/pe/hdf5/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl~mpi+shared~szip~threadsafe%gcc@12.3
-      - prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
-        spec: hdf5@1.12.2+cxx+fortran+hl+mpi+shared~szip~threadsafe%gcc@12.3
   libbsd:
     require: '%gcc@7.5.0'
+  libcxi:
+    require: '@main'
   libevent:
     require: '%gcc@7.5.0'
   # Crayism
   libfabric:
-    buildable: false
+    buildable: true
+    require: '@main+cuda+uring fabrics=cxi,sockets,tcp,udp,xpmem'
     externals:
       - modules:
-          - libfabric/1.15.2.0
+          - libfabric/1.22.0
-        spec: libfabric@1.15.2.0
+        prefix: /opt/cray/libfabric/1.22.0
+        spec: libfabric@1.22.0
   libiconv:
     require: '%gcc@7.5.0'
   libidn2:
@@ -129,32 +192,56 @@ packages:
     require: '%gcc@7.5.0'
   netcdf-c:
     externals:
-      - prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
         spec: netcdf-c@4.9.0.9~dap~hdf4~jna~mpi~parallel-netcdf+shared%cce@17.0.0
-      - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/crayclang/17.0
+      - modules:
+        - cray-netcdf-hdf5parallel/4.9.0.9
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/crayclang/17.0
         spec: netcdf-c@4.9.0.9~dap~hdf4~jna+mpi~parallel-netcdf+shared%cce@17.0.0
-      - prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
         spec: netcdf-c@4.9.0.9~dap~hdf4~jna~mpi~parallel-netcdf+shared%gcc@12.3
-      - prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
+      - modules:
+        - cray-netcdf-hdf5parallel/4.9.0.9
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
         spec: netcdf-c@4.9.0.9~dap~hdf4~jna+mpi~parallel-netcdf+shared%gcc@12.3
   netcdf-fortran:
     externals:
-      - prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/crayclang/17.0
         spec: netcdf-fortran@4.9.0.9~doc+shared%cce@17.0.0
-      - prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
+      - modules:
+        - cray-netcdf/4.9.0.9
+        prefix: /opt/cray/pe/netcdf/4.9.0.9/gnu/12.3
         spec: netcdf-fortran@4.9.0.9~doc+shared%gcc@12.3
   ninja:
     require: '%gcc@7.5.0'
   numactl:
     require: '%gcc@7.5.0'
+  nvhpc:
+    externals:
+      - spec: nvhpc@23.9
+        prefix: /opt/nvidia/hpc_sdk
+        extra_attributes:
+          compilers:
+            c: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc
+            cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvc++
+            fortran: /opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/bin/nvfortran
+          operating_system: sles15
+          modules: nvhpc-nompi/23.9
+          environment:
+            prepend_path:
+              LD_LIBRARY_PATH: '/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/23.9/math_libs/lib64/stubs'
   opal:
     require:
       - spec: '%gcc'
       - message: Opal 2022.1.0 only builds with gcc@10.4.0
         spec: '%gcc@10.4.0 ^openmpi'
         when: '@2022.1.0'
-  openblas:
-    require: '%gcc@7.5.0'
   opengl:
     require: '%gcc@7.5.0'
   openldap:
@@ -164,16 +251,23 @@ packages:
         spec: openldap@2.4.46
   openmpi:
     require:
-      - schedulers=slurm fabrics=ofi +pmi pmi-interface=cray +cxx +cxx_exceptions +cuda +legacylaunchers
+      - spec: "schedulers=slurm fabrics=ofi,xpmem +cuda +internal-pmix"
+        when: "@5.0.0:"
+      - spec:  "schedulers=slurm fabrics=ofi,xpmem +pmi +cxx +cxx_exceptions +cuda +legacylaunchers"
+        when: "@:5.0.0"
   openssh:
     require: '%gcc@7.5.0'
   openssl:
     require: '%gcc@7.5.0'
   parallel-netcdf:
     externals:
-      - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/crayclang/17.0
+      - modules:
+        - cray-parallel-netcdf/1.12.3.9
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/crayclang/17.0
         spec: parallel-netcdf@1.12.3.9+cxx+fortran%cce@17.0.0
-      - prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
+      - modules:
+        - cray-parallel-netcdf/1.12.3.9
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
         spec: parallel-netcdf@1.12.3.9+cxx+fortran%gcc@12.3
   perl:
     buildable: false
@@ -184,6 +278,8 @@ packages:
     require: '%gcc@7.5.0'
   pkgconf:
     require: '%gcc@7.5.0'
+  pmix:
+    require: '+munge'
   py-fypp:
     require: '%gcc@7.5.0'
   python:
@@ -199,7 +295,7 @@ packages:
     buildable: false
     externals:
       - prefix: /usr
-        spec: slurm@23-11-0-1 +pmix +hwloc +hdf5 +mariadb +pam
+        spec: slurm@23.11.8%gcc@7.5.0 +cgroup+gtk+hwloc+mariadb+nvml+pam+pmix+restd
   sqlite:
     require: '%gcc@7.5.0'
   tar:
@@ -217,6 +313,7 @@ packages:
     externals:
       - modules:
           - xpmem/2.8.2-1.0_3.9__g84a27a5.shasta
+        prefix: /usr
         spec: xpmem@2.8.2-1
   xxhash:
     require: '%gcc@7.5.0'

env/sles15/x86_64/users/stable/spack.yaml

@@ -1,17 +1,15 @@
 spack:
   concretizer:
     unify: false
-  config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
-    install_tree:
-      root: $spack/user/$user/spack-install
-    upstreams:
-      spack-instance-1:
-        install_tree: $spack/opt/spack/stable
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+    - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
-    specs:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

env/sles15/x86_64/users/unstable/spack.yaml

@@ -1,19 +1,19 @@
 spack:
   concretizer:
     unify: false
-  config:
-    allow_sgid: false
-    build_stage:
-      - $spack/user/$user/spack-stage
-    install_tree:
-      root: $spack/user/$user/spack-install
-    upstreams:
-      spack-instance-1:
-        install_tree: $spack/opt/spack/stable
-      spack-instance-2:
-        install_tree: $spack/opt/spack/unstable
   include:
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/packages.yaml
+  - $spack/../spack-psi/env/sles15/x86_64/sysconfigs/packages.yaml
-    - $spack/spack-psi/env/$os/$target_family/sysconfigs/compilers.yaml
+  include_concrete:
-    specs:
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/x86_64/admin/unstable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/stable
+    - /afs/psi.ch/sys/spack/spack-psi/env/sles15/aarch64/admin/unstable
+  specs: []
+  upstreams:
+    stable:
+      install_tree: $spack/opt/spack/stable
+    unstable:
+      install_tree: $spack/opt/spack/unstable
+  repos:
+  - $spack/../user/$user/spack-repo
   view: false

packages/cuda/package.py

@@ -0,0 +1,18 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+from spack.pkg.builtin.cuda import Cuda as SpackCuda
+
+
+class Cuda(SpackCuda):
+
+    def setup_run_environment(self, env):
+        super().setup_run_environment(env)
+
+        if '%nvhpc' in self.spec:
+          env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib64)
+          env.prepend_path("LD_LIBRARY_PATH", join_path(self.prefix, '../math_libs/lib64'))

packages/gams/package.py

@@ -0,0 +1,14 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+from spack.pkg.builtin.gams import Gams as SpackGams
+
+
+class Gams(SpackGams):
+    version("48.1", md5="5a59a973edfc4e3170b9535760593d62", expand=False)
+    version("49.1", md5="bc9b82d015c9672f8c153003c7a5dd2a", expand=False)

packages/h5hut/package.py

@@ -5,9 +5,71 @@
 
 
 from spack.package import *
-from spack.pkg.builtin.h5hut import H5hut as SpackH5hut
 
 
-class H5hut(SpackH5hut):
+class H5hut(AutotoolsPackage):
+    """H5hut (HDF5 Utility Toolkit).
+    High-Performance I/O Library for Particle-based Simulations."""
 
+    homepage = "https://amas.psi.ch/H5hut/"
+    url = "https://amas.web.psi.ch/Downloads/H5hut/H5hut-2.0.0rc3.tar.gz"
+    git = "https://gitlab.psi.ch/H5hut/src.git"
+
+    version("2.0.0rc7", sha256="bc058c4817c356b7b7acfe386c586923103b90bdfa83575db3a91754767e6fab")
     version("2.0.0rc6", sha256="980a5c634877265bd3a862fbd439d973d44be9c1d2f840c3635e8c6375d62d7e")
+    version("2.0.0rc3", sha256="1ca9a9478a99e1811ecbca3c02cc49258050d339ffb1a170006eab4ab2a01790")
+
+    version("master", branch="master")
+
+    variant("fortran", default=True, description="Enable Fortran support")
+    variant("mpi", default=True, description="Enable MPI support")
+
+    depends_on("autoconf", type="build", when="build_system=autotools")
+    depends_on("automake", type="build", when="build_system=autotools")
+    depends_on("libtool", type="build", when="build_system=autotools")
+
+    depends_on("mpi", when="+mpi")
+    # h5hut +mpi uses the obsolete function H5Pset_fapl_mpiposix:
+    depends_on("hdf5@1.8:+mpi", when="+mpi")
+    depends_on("hdf5@1.8:", when="~mpi")
+
+    # If built in parallel, the following error message occurs:
+    # install: .libs/libH5hut.a: No such file or directory
+    parallel = False
+
+    @run_before("configure")
+    def validate(self):
+        """Checks if Fortran compiler is available."""
+
+        if "+fortran" in self.spec and not self.compiler.fc:
+            raise RuntimeError("Cannot build Fortran variant without a Fortran compiler.")
+
+    def flag_handler(self, name, flags):
+        build_system_flags = []
+        if name == "cflags" and self.spec.version < Version("2.0.0rc7"):
+            build_system_flags = ["-DH5_USE_110_API"]
+        return flags, None, build_system_flags
+
+    def autoreconf(self, spec, prefix):
+        which("bash")("autogen.sh")
+
+    def configure_args(self):
+        spec = self.spec
+        config_args = ["--enable-shared"]
+
+        if "+fortran" in spec:
+            config_args.append("--enable-fortran")
+
+        if "+mpi" in spec:
+            config_args.extend(
+                [
+                    "--enable-parallel",
+                    "CC={0}".format(spec["mpi"].mpicc),
+                    "CXX={0}".format(spec["mpi"].mpicxx),
+                ]
+            )
+
+            if "+fortran" in spec:
+                config_args.append("FC={0}".format(spec["mpi"].mpifc))
+
+        return config_args

packages/ippl/package.py

@@ -0,0 +1,99 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+# ----------------------------------------------------------------------------
+# If you submit this package back to Spack as a pull request,
+# please first remove this boilerplate and all FIXME comments.
+#
+# This is a template package file for Spack.  We've put "FIXME"
+# next to all the things you'll want to change. Once you've handled
+# them, you can save this file and test your package like this:
+#
+#     spack install ippl
+#
+# You can edit this file again by typing:
+#
+#     spack edit ippl
+#
+# See the Spack documentation for more information on packaging.
+# ----------------------------------------------------------------------------
+
+from spack.package import *
+
+
+class Ippl(CMakePackage, CudaPackage):
+    """Independent Parallel Particle Layer (IPPL) is a performance portable C++ library for Particle-Mesh methods. IPPL makes use of Kokkos (https://github.com/kokkos/kokkos), HeFFTe (https://github.com/icl-utk-edu/heffte), and MPI (Message Passing Interface) to deliver a portable, massively parallel toolkit for particle-mesh methods. IPPL supports simulations in one to six dimensions, mixed precision, and asynchronous execution in different execution spaces (e.g. CPUs and GPUs)."""
+
+    homepage = "https://github.com/IPPL-framework/ippl"
+    url = "https://github.com/IPPL-framework/ippl/archive/refs/tags/IPPL-3.2.0.tar.gz"
+    
+    git="https://github.com/IPPL-framework/ippl.git"
+    
+    maintainers("germanne", "gsell")
+
+    license("GPLv3")
+
+    version("3.2.0", sha256="041a4efbddaba5b477dae01ab166354da7dee01cf625706e19f5b60c0c3f6b88")
+
+    version('opalx-fielddump', branch="opalx-fielddump")
+
+    variant(
+        "build_type",
+        default="Release",
+        description="CMake build type",
+        values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"),
+    )
+    variant("mpi", default=True, description="Enable MPI support")
+    variant(
+            "platforms", 
+            default=True, 
+            description="Set IPPL platforms",
+            values=("serial", "openmp", "cuda", "openmp/cuda")
+    )
+    variant("solvers", default=True, description="Enable solvers")
+    variant("fft", default=True, description="Enable full-wave solver")
+    variant("tests", default=False, description="Enable tests")
+    variant("unit_tests", default=False, description="Enable unit tests")
+    variant("alternative_variant", default=False, description="Use alternative variant")
+    variant("alpine", default=False, description="Enable Alpine")
+
+    depends_on("cmake@3.25.2:", type="build")
+    depends_on("fftw@3.3.10", when="+fft")
+    depends_on("mpi", when="+mpi")
+
+    conflicts("~mpi", msg="IPPL can not run without mpi!")
+    conflicts("cuda_arch=none", when="+cuda",
+       msg="CUDA architecture is required")
+    conflicts("~cuda", when="platform=cuda",
+       msg="CUDA is required for platform cuda.")
+
+    def cmake_args(self):
+        args = [
+            "-DCMAKE_CXX_STANDARD=20",
+            self.define_from_variant("ENABLE_FFT", "fft"),
+            self.define_from_variant("ENABLE_SOLVERS", "solvers"),
+            self.define_from_variant("ENABLE_TESTS", "tests"),
+            self.define_from_variant("ENABLE_UNIT_TESTS", "unit_tests"),
+            self.define_from_variant("USE_ALTERNATIVE_VARIANT", "alternative_variant"),
+            self.define_from_variant("ENABLE_ALPINE", "alpine"),
+        ]
+
+        if 'platforms=cuda' in self.spec:
+            args.append("-DIPPL_PLATFORMS=CUDA")
+        elif 'platforms=openmp' in self.spec:
+            args.append("-DIPPL_PLATFORMS=OPENMP")
+        elif 'platforms=openmp/cuda' in self.spec:
+            args.append("-DIPPL_PLATFORMS=OPENMP;CUDA")
+        else:
+            args.append("-DIPPL_PLATFORMS=SERIAL")
+
+        if 'cuda_arch=90' in self.spec:
+          args.append("-DKokkos_ARCH_HOPPER90=ON")
+        elif 'cuda_arch=80' in self.spec:
+          args.append("-DKokkos_ARCH_AMPERE80=ON")
+        elif 'cuda_arch=60' in self.spec:
+          args.append("-DKokkos_ARCH_PASCAL61=ON")
+
+        return args

packages/mfem/mfem_conduit_spack.patch

@@ -0,0 +1,37 @@
+diff --git a/config/defaults.mk b/config/defaults.mk
+index d6b8d59a1..d1ba6fb9a 100644
+--- a/config/defaults.mk
++++ b/config/defaults.mk
+@@ -452,7 +452,7 @@ FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+ 
+ # Conduit and required libraries configuration
+ CONDUIT_DIR = @MFEM_DIR@/../conduit
+-CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
++CONDUIT_OPT = -I$(CONDUIT_DIR)/include
+ CONDUIT_LIB = \
+    $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+    -lconduit -lconduit_relay -lconduit_blueprint  -ldl
+@@ -469,8 +469,8 @@ endif
+ # Sidre and required libraries configuration
+ # Be sure to check the HDF5_DIR (set above) is correct
+ SIDRE_DIR = @MFEM_DIR@/../axom
+-SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\
+- -I$(HDF5_DIR)/include
++SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit \
++            -I$(CONDUIT_DIR)/include -I$(HDF5_DIR)/include
+ SIDRE_LIB = \
+    $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
+    $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+diff --git a/fem/conduitdatacollection.hpp b/fem/conduitdatacollection.hpp
+index 6ff1804e0..ad0a272a4 100644
+--- a/fem/conduitdatacollection.hpp
++++ b/fem/conduitdatacollection.hpp
+@@ -17,7 +17,7 @@
+ #ifdef MFEM_USE_CONDUIT
+ 
+ #include "datacollection.hpp"
+-#include <conduit.hpp>
++#include <conduit/conduit.hpp>
+ 
+ namespace mfem
+ {

packages/mithra/mithra-fPIC.patch

@@ -6,7 +6,7 @@ diff -u mithra-2.0.orig/makefile mithra-2.0/makefile
  SHELL = /bin/sh
  COMP = mpic++
  
-+CFLAGS+=-fPIC
++CFLAGS+=-fPIC -fPIE
  CFLAGS+=-std=c++11
  CFLAGS+=-O3
  

packages/mithra/package.py

@@ -18,6 +18,8 @@ class Mithra(MakefilePackage):
 
     version("2.0", sha256="66f27ba6ca2c1ab05abb76338d00e661a6fe96b95283b85409c37625b4758deb")
 
+    patch("mithra-fPIC.patch")
+
     variant("mpi", default=True, description="Enable MPI support")
 
     depends_on("mpi", when="+mpi")

packages/opal-x/package.py

@@ -0,0 +1,112 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+# ----------------------------------------------------------------------------
+# If you submit this package back to Spack as a pull request,
+# please first remove this boilerplate and all FIXME comments.
+#
+# This is a template package file for Spack.  We've put "FIXME"
+# next to all the things you'll want to change. Once you've handled
+# them, you can save this file and test your package like this:
+#
+#     spack install opal-x
+#
+# You can edit this file again by typing:
+#
+#     spack edit opal-x
+#
+# See the Spack documentation for more information on packaging.
+# ----------------------------------------------------------------------------
+
+from spack.package import *
+
+
+class OpalX(CMakePackage, CudaPackage):
+    """OPAL (Object Oriented Parallel Accelerator Library) is a parallel open source tool for charged-particle optics in linear accelerators and rings,
+    including 3D space charge. Using the MAD language with extensions, OPAL can run on a laptop as well as on the largest high performance computing systems.
+    OPAL is built from the ground up as a parallel application exemplifying the fact that high performance computing is the third leg of science,
+    complementing theory and experiment."""
+
+    homepage = "https://amas.web.psi.ch/opal/Documentation/master/OPAL_Manual.html"
+    git = "https://gitlab.psi.ch/OPAL/opal-x/src.git"
+
+    maintainers("germanne", "gsell")
+
+    license("GPLv3")
+
+    version('master', branch="master")
+    version('fixSolverUnits', branch="fixSolverUnits")
+
+    variant(
+        "build_type",
+        default="Release",
+        description="CMake build type",
+        values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"),
+    )
+    variant("mpi", default=True, description="Enable MPI support")
+    variant("solvers", default=True, description="Enable solvers")
+    variant("fft", default=True, description="Enable full-wave solver")
+    variant("tests", default=False, description="Enable tests")
+    variant("alternative_variant", default=False, description="Use alternative variant")
+    variant("alpine", default=False, description="Enable Alpine")
+    variant("unit_tests", default=False, description="Build unit tests")
+    variant("ippl_git_tag", default="master", description="IPPL git tag")
+
+    depends_on("blas")
+    depends_on("boost@1.82.0+mpi+chrono+filesystem+iostreams+regex+serialization+system+timer+python+shared")
+    depends_on("cmake@3.25.2:", type="build")
+    depends_on("fftw@3.3.10 +mpi")
+    depends_on("gsl@2.7+shared")
+    depends_on("gnutls@3.5.19:")
+    depends_on("cuda@12.4.0:")
+
+    depends_on("h5hut+mpi", when="+mpi")
+
+    depends_on("googletest@1.13.0: +shared", when="+tests")
+    # @master
+    depends_on("h5hut@2.0.0rc6:", when="@master")
+
+    # @2022.1%gcc@10.4.0
+    depends_on("h5hut@2.0.0rc6", when="@2022.1")
+
+    conflicts("~mpi", msg="OPAL can not run without mpi!")
+    conflicts("cuda_arch=none", when="+cuda",
+          msg="CUDA architecture is required")
+    conflicts("%gcc@:8.5.0", msg="gcc bigger than 8.5.0 is required")
+
+    @run_before("cmake")
+    def cmake_configure(self):
+      spec = self.spec
+      cmake_configure = Executable("./gen_OPALrevision")
+      cmake_configure()
+
+    def cmake_args(self):
+        args = [
+            "-DCMAKE_CXX_STANDARD=20",
+            self.define_from_variant("ENABLE_FFT", "fft"),
+            self.define_from_variant("ENABLE_SOLVERS", "solvers"),
+            self.define_from_variant("ENABLE_TESTS", "tests"),
+            self.define_from_variant("USE_ALTERNATIVE_VARIANT", "alternative_variant"),
+            self.define_from_variant("ENABLE_ALPINE", "alpine"),
+            self.define_from_variant("ENABLE_UNIT_TESTS", "unit_tests"),
+            "-DIPPL_GIT_TAG=opalx-fielddump",
+            "-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF",
+        ]
+
+        if '+cuda' in self.spec:
+            args.append("-DIPPL_PLATFORMS=CUDA")
+        else:
+            args.append("-DIPPL_PLATFORMS=openmp")
+
+        if 'cuda_arch=90' in self.spec:
+          args.append("-DKokkos_ARCH_HOPPER90=ON")
+        elif 'cuda_arch=80' in self.spec:
+          args.append("-DKokkos_ARCH_AMPERE80=ON")
+        elif 'cuda_arch=60' in self.spec:
+          args.append("-DKokkos_ARCH_PASCAL61=ON")
+
+
+        return args
+

packages/opal/package.py

@@ -22,6 +22,7 @@ class Opal(CMakePackage):
 
     version('master', branch="master")
     version("2022.1.0", sha256="31f6811032e9d5400169d28a8ae6c7dee2e4a803ee82e90584d31ff7a0a18d80")
+    version("2024.1.0", sha256="65c4c263d6394b4b234323a74fc55eb963fb75cb80ea904d6c82054cfc64033c")
 
     variant(
         "build_type",
@@ -45,12 +46,12 @@ class Opal(CMakePackage):
     depends_on("blas")
     depends_on("boost+chrono+filesystem+iostreams+regex+serialization+system+timer")
     depends_on("boost+mpi", when="+mpi")
-    depends_on("boost+python", when="+python")
+    depends_on("boost+python+shared", when="+python")
-    depends_on("python@3.0", when="+python")
+    depends_on("python@3.0:", when="+python")
     depends_on(
         "boost+mpi+chrono+filesystem+iostreams+regex+serialization+system+timer", when="+mpi"
     )
-    depends_on("gsl~shared")
+    depends_on("gsl+shared")
 
     depends_on("h5hut+mpi", when="+mpi")
     depends_on("h5hut~mpi", when="~mpi")
@@ -59,14 +60,14 @@ class Opal(CMakePackage):
         "amrex@18.07 precision=double dimensions=3 +mpi~openmp+particles+pic~eb",
         when="+amr+amr_mg_solver+mpi",
     )
-    depends_on("googletest@1.10.0:", when="+tests")
+    depends_on("googletest@1.10.0: +shared", when="+tests")
     depends_on("mithra@2.0", when="+opal_fel")
     depends_on(
-        "trilinos@12.0.1: cxxstd=17 +mpi+muelu+nox+zoltan+zoltan2+isorropia gotype=long",
+        "trilinos@12.0.1: cxxstd=17 +mpi+muelu+nox+zoltan+zoltan2+isorropia+shared gotype=long",
         when="+saamg_solver",
     )
     depends_on(
-        "trilinos@12.0.1: cxxstd=17 +mpi+muelu+nox+zoltan+zoltan2+isorropia gotype=long",
+        "trilinos@12.0.1: cxxstd=17 +mpi+muelu+nox+zoltan+zoltan2+isorropia+shared gotype=long",
         when="+amr_mg_solver",
     )
     depends_on("parmetis@4.0.3:", when="+saamg_solver")
@@ -74,8 +75,8 @@ class Opal(CMakePackage):
 
     # @master
     depends_on("h5hut@2.0.0rc6", when="@master")
-    depends_on("trilinos@13.4.0:", when="@master+saamg_solver")
+    depends_on("trilinos@13.4.0: +shared", when="@master+saamg_solver")
-    depends_on("trilinos@13.4.0:", when="@master+amr_mg_solver")
+    depends_on("trilinos@13.4.0: +shared", when="@master+amr_mg_solver")
 
     # @2022.1%gcc@10.4.0
     depends_on("h5hut@2.0.0rc6", when="@2022.1")
@@ -83,6 +84,7 @@ class Opal(CMakePackage):
     depends_on("trilinos@13.4.0", when="@2022.1+amr_mg_solver")
 
     conflicts("~amr", "+amr_mg_solver")
+    conflicts("~mpi", msg="OPAL can not run without mpi!")
 
     # AMRex version is too old for any newer Trilinos version
     requires(
@@ -105,11 +107,12 @@ class Opal(CMakePackage):
             self.define_from_variant("ENABLE_AMR", "amr"),
             self.define_from_variant("ENABLE_AMR_MG_SOLVER", "amr_mg_solver"),
             self.define_from_variant("ENABLE_BANDRF", "bandrf"),
-            self.define_from_variant("USE_STATIC_LIBRARIES", "static"),
             self.define_from_variant("ENABLE_OPAL_FEL", "opal_fel"),
             self.define_from_variant("ENABLE_SAAMG_SOLVER", "saamg_solver"),
             self.define_from_variant("BUILD_OPAL_PYTHON", "python"),
+            self.define_from_variant("WILL_BUILD_SHARED_LIBRARY", "python"),
             self.define_from_variant("ENABLE_IPPLTESTS", "ippltests"),
             self.define_from_variant("ENABLE_MSLANG", "mslang"),
         ]
+
         return args

packages/relion/package.py

@@ -0,0 +1,24 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+from spack.package import *
+from spack.pkg.builtin.relion import Relion as SpackRelion
+
+
+class Relion(SpackRelion):
+    version("5.0.0", sha256="800ad0c0aa778cbf584fcf8986976645f2b25d677a80f168e5397975b9db6e47")
+
+    variant("python-exe-path", default="/data/project/bio/hpce/relion-5.0/miniconda3/envs/relion-5.0/bin/python", description="Path to conda env python binary")
+    variant("torch-home-path", default="/data/project/bio/hpce/relion-5.0/torch", description="Path to pretrained model weights")
+
+    def cmake_args(self):
+        args = super().cmake_args()
+
+        if "@5.0.0" in self.spec:
+            args += ["-DPYTHON_EXE_PATH={0}".format(self.spec.variants["python-exe-path"].value)]
+            args += ["-DTORCH_HOME_PATH={0}".format(self.spec.variants["torch-home-path"].value)]
+
+        return args

packages/slurm/package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+from spack.pkg.builtin.slurm import Slurm as SpackSlurm
+
+
+class Slurm(SpackSlurm):
+
+    version("24-05-1-1", sha256="8859d90b29e6c81fdc80a16c9063c1ca58fda4245c3835016124cc044a44daec")
+    version("23-11-8-1", sha256="0e5aba8eb2b6a5b567a3918190a27d8ae3b60ce03acec225ec7ed2584fa450a1")
+
+    variant("cray_shasta", default=False, description="Enable Cray Shasta support")
+
+    def configure_args(self):
+        spec = self.spec
+
+        args=super().configure_args()
+
+        if "+cray_shasta" in spec:
+            args.append("--with-cray_shasta")
+
+        return args

packages/sratoolkit/package.py

@@ -0,0 +1,14 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+from spack.package import *
+from spack.pkg.builtin.sratoolkit import Sratoolkit as SpackSratoolkit
+
+
+class Sratoolkit(SpackSratoolkit):
+
+    version("3.1.1", sha256="600ff38589c15033c6d0fb4d1d055519e56175ea031b949a2d3f5a74151c6c12")

templates/user_hierachical_paths.lua

@@ -0,0 +1,20 @@
+{% extends "modules/modulefile.lua" %}
+{% block footer %}
+ 
+local user_module_root= pathJoin("/afs/psi.ch/sys/spack/user", os.getenv("USER"), "spack-install/modulefiles")
+
+{% if 'mpi' in provides %}
+local user_projection= "{{ spec.architecture }}/{{ spec.name }}/{{ version_part }}/{{ spec.compiler.name }}/{{ spec.compiler.version }}"
+{% elif 'compiler' in provides %}
+local user_projection= "{{ spec.architecture }}/{{ spec.name }}/{{ spec.version }}"
+{% endif %}
+
+
+if user_projection then
+local user_path= pathJoin(user_module_root, user_projection)
+  if isDir(user_path) then
+prepend_path("MODULEPATH", user_path, ":")
+  end
+end
+
+{% endblock %}