From 437fdc899f850d3655165bb9a4be919c647b435a Mon Sep 17 00:00:00 2001 From: Germann Elsa Sylvia Date: Tue, 11 Oct 2022 15:13:16 +0200 Subject: [PATCH] FIX: gpu node build --- env/merlin/HyQuas/packages/hyquas/package.py | 16 +++++++++++----- env/merlin/HyQuas/spack.yaml | 8 ++++++-- env/merlin/HyQuas/sysconfigs/compilers.yaml | 10 +++++----- env/merlin/HyQuas/sysconfigs/packages.yaml | 20 +++++--------------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/env/merlin/HyQuas/packages/hyquas/package.py b/env/merlin/HyQuas/packages/hyquas/package.py index f7d83e7..0a3117f 100644 --- a/env/merlin/HyQuas/packages/hyquas/package.py +++ b/env/merlin/HyQuas/packages/hyquas/package.py @@ -52,20 +52,26 @@ class Hyquas(CMakePackage, CudaPackage): variant("mpi", default=False, description="Use mpi") variant("overlap_mat", default=True, description="Overlap initMatirx") variant("log_eval", default=False, description="Show logging of evaluator") - variant("mat_size", default='6', description="Mat size", values=('4','5','6','7')) + variant("mat_size", default='7', description="Mat size", values=('4','5','6','7')) conflicts("~cuda", msg="HyQuas requires CUDA") def setup_build_environment(self, env): - cuda_arch = self.spec.variants['cuda_arch'].value[0] - env.set('CUDA_NVCC_FLAGS', '-Xcompiler -fopenmp -std=c++14 -O2 -g -arch=compute_' + cuda_arch + '-code=sm_' + cuda_arch + '--ptxas-options=-v -lineinfo -keep') - env.set('GENCODE_FLAGS', '-gencode arch=compute_' + cuda_arch +',code=sm_' + cuda_arch) env.set("NCCL_ROOT", self.spec['nccl'].prefix) - env.set("CUDA_HOME", self.spec['cuda'].prefix) + + def setup_run_environment(self, env): + env.set("HYQUAS_ROOT", self.stage.source_path) @run_before('cmake') def build_cutt(self): + cuda_arch = self.spec.variants['cuda_arch'].value[0] + + CMakefile = FileFilter('CMakeLists.txt') + CMakefile.filter(r'^\s*CUDA_NVCC_FLAGS\s*=.*', 'CUDA_NVCC_FLAGS := -Xcompiler -fopenmp -std=c++14 -O2 -g -arch=compute_' + cuda_arch + '-code=sm_' + cuda_arch + '--ptxas-options=-v -lineinfo -keep' ) with working_dir('./third-party/cutt'): + makefile = FileFilter('Makefile') + + makefile.filter(r'^\s*GENCODE_FLAGS\s*=.*', 'GENCODE_FLAGS := -gencode arch=compute_' + cuda_arch +',code=sm_' + cuda_arch ) make() def cmake_args(self): diff --git a/env/merlin/HyQuas/spack.yaml b/env/merlin/HyQuas/spack.yaml index 0c31795..f491d69 100644 --- a/env/merlin/HyQuas/spack.yaml +++ b/env/merlin/HyQuas/spack.yaml @@ -1,6 +1,6 @@ spack: concretizer: - unify: true + unify: when_possible include: - sysconfigs/config.yaml - sysconfigs/compilers.yaml @@ -8,7 +8,11 @@ spack: repos: - . specs: - - hyquas@main%gcc+cuda cuda_arch=61 ~mpi+micro_bench+eval_pp backend=mix ^openmpi + - openmpi@4.0.5%gcc+cuda fabrics=ucx,xpmem,knem,ofi,cma schedulers=slurm ^ucx@1.13.0 + ^hwloc@2.7.1 ^knem@1.1.4 ^xpmem@2.6.3 ^pmix@4.1.2 + - cuda@11.0.2 + - hyquas@main%gcc+cuda cuda_arch=80 ~mpi+schedule+use_double+micro_bench+eval_pp~disable_assert + backend=mix mat_size=7 ^cuda@11.0.2 develop: hyquas: path: /data/user/germann_e/HyQuas diff --git a/env/merlin/HyQuas/sysconfigs/compilers.yaml b/env/merlin/HyQuas/sysconfigs/compilers.yaml index f0662b8..00c322e 100644 --- a/env/merlin/HyQuas/sysconfigs/compilers.yaml +++ b/env/merlin/HyQuas/sysconfigs/compilers.yaml @@ -46,11 +46,11 @@ compilers: modules: [] operating_system: rhel7 paths: - cc: /opt/psi/Programming/gcc/8.3.0/bin/gcc - cxx: /opt/psi/Programming/gcc/8.3.0/bin/g++ - f77: /opt/psi/Programming/gcc/8.3.0/bin/gfortran - fc: /opt/psi/Programming/gcc/8.3.0/bin/gfortran - spec: gcc@8.3.0 + cc: /opt/psi/Programming/gcc/9.5.0/bin/gcc + cxx: /opt/psi/Programming/gcc/9.5.0/bin/g++ + f77: /opt/psi/Programming/gcc/9.5.0/bin/gfortran + fc: /opt/psi/Programming/gcc/9.5.0/bin/gfortran + spec: gcc@9.5.0 target: x86_64 - compiler: environment: {} diff --git a/env/merlin/HyQuas/sysconfigs/packages.yaml b/env/merlin/HyQuas/sysconfigs/packages.yaml index dc1f53e..06db280 100644 --- a/env/merlin/HyQuas/sysconfigs/packages.yaml +++ b/env/merlin/HyQuas/sysconfigs/packages.yaml @@ -64,11 +64,6 @@ packages: externals: - prefix: /usr spec: findutils@4.5.11 - flex: - buildable: false - externals: - - prefix: /usr - spec: flex@2.5.37+lex # E.G: only available from already downloaded source, so I put it here gams: externals: @@ -129,16 +124,6 @@ packages: externals: - prefix: /usr spec: libcroco@0.6.13 - libevent: - buildable: false - externals: - - prefix: /usr - spec: libevent@2.0 - libfabric: - buildable: false - externals: - - prefix: /usr - spec: libfabric@1.10.2 libfuse: buildable: false externals: @@ -260,6 +245,11 @@ packages: externals: - prefix: /usr spec: sed@4.2.2 + slurm: + buildable: false + externals: + - prefix: /usr + spec: slurm@21-08-8-2 sqlite: buildable: false externals: