From 5148aaf5961f7c1b9211c8d5d92c6116578436e5 Mon Sep 17 00:00:00 2001 From: Derek Feichtinger Date: Tue, 3 Sep 2019 12:06:13 +0200 Subject: [PATCH 1/3] removed dask from datascience_py36 due to errors --- .../{datascience_36 => datascience_py36}/datascience_36.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename Programming/anaconda/2019.07/conda-env-defs/{datascience_36 => datascience_py36}/datascience_36.yml (66%) diff --git a/Programming/anaconda/2019.07/conda-env-defs/datascience_36/datascience_36.yml b/Programming/anaconda/2019.07/conda-env-defs/datascience_py36/datascience_36.yml similarity index 66% rename from Programming/anaconda/2019.07/conda-env-defs/datascience_36/datascience_36.yml rename to Programming/anaconda/2019.07/conda-env-defs/datascience_py36/datascience_36.yml index fcb0fa7..8f9f851 100644 --- a/Programming/anaconda/2019.07/conda-env-defs/datascience_36/datascience_36.yml +++ b/Programming/anaconda/2019.07/conda-env-defs/datascience_py36/datascience_36.yml @@ -14,7 +14,9 @@ dependencies: - pytables - ipython - keras=2.1.6 - - dask - deap - nb_conda_kernels - ipywidgets +# Note that the dask 2.1.0 with python 3.6 led to a non-functioning configuration +# (serialization errors in dask). Therefore we exclude it here. +# - dask From fee046050155df5134ef8bead7b96cc1fd90b717 Mon Sep 17 00:00:00 2001 From: Derek Feichtinger Date: Tue, 3 Sep 2019 12:07:08 +0200 Subject: [PATCH 2/3] added chaospy on request by A. Adelmann --- .../anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml b/Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml index dfe2a39..fffdff1 100644 --- a/Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml +++ b/Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml @@ -17,3 +17,4 @@ dependencies: - deap - nb_conda_kernels - ipywidgets + - chaospy From ba7f38d99de6cde1b7d06552920bd1667a265f7f Mon Sep 17 00:00:00 2001 From: Derek Feichtinger Date: Wed, 4 Sep 2019 16:25:21 +0200 Subject: [PATCH 3/3] bigger update of python environments for anaconda/2019.07 --- .../conda-env-defs/datascience_py36/README.md | 3 +++ .../datascience_py37/datascience_py37.yml | 22 +++++++++++++++++++ .../conda-env-defs/hpce-tools/hpce-tools.yml | 9 ++++++++ .../{talos_36 => talos_py36}/README.md | 0 .../talos_py36.yml} | 0 Programming/anaconda/README.md | 17 ++++++++------ 6 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 Programming/anaconda/2019.07/conda-env-defs/datascience_py36/README.md create mode 100644 Programming/anaconda/2019.07/conda-env-defs/datascience_py37/datascience_py37.yml create mode 100644 Programming/anaconda/2019.07/conda-env-defs/hpce-tools/hpce-tools.yml rename Programming/anaconda/2019.07/conda-env-defs/{talos_36 => talos_py36}/README.md (100%) rename Programming/anaconda/2019.07/conda-env-defs/{talos_36/talos_36.yml => talos_py36/talos_py36.yml} (100%) diff --git a/Programming/anaconda/2019.07/conda-env-defs/datascience_py36/README.md b/Programming/anaconda/2019.07/conda-env-defs/datascience_py36/README.md new file mode 100644 index 0000000..ed00ac9 --- /dev/null +++ b/Programming/anaconda/2019.07/conda-env-defs/datascience_py36/README.md @@ -0,0 +1,3 @@ +Note that the dask 2.1.0 with python 3.6 led to a non-functioning configuration (serialization errors in dask). Therefore we exclude it here. + +It would be better if users used the datascience_py37 environment. diff --git a/Programming/anaconda/2019.07/conda-env-defs/datascience_py37/datascience_py37.yml b/Programming/anaconda/2019.07/conda-env-defs/datascience_py37/datascience_py37.yml new file mode 100644 index 0000000..7eeab07 --- /dev/null +++ b/Programming/anaconda/2019.07/conda-env-defs/datascience_py37/datascience_py37.yml @@ -0,0 +1,22 @@ +# Clean environment based on pure conda-forge packages +name: datascience_py37 +channels: + - conda-forge +dependencies: + - python=3.7 + - pandas + - numpy + - scipy + - scikit-learn + - matplotlib + - seaborn + - tensorflow=1.13.1 + - pytables + - ipython + # keras 2.1.6 not available from conda-forge for py37 + # - keras=2.1.6 + - deap + - nb_conda_kernels + - ipywidgets + - dask + - dask-jobqueue diff --git a/Programming/anaconda/2019.07/conda-env-defs/hpce-tools/hpce-tools.yml b/Programming/anaconda/2019.07/conda-env-defs/hpce-tools/hpce-tools.yml new file mode 100644 index 0000000..2aca8ee --- /dev/null +++ b/Programming/anaconda/2019.07/conda-env-defs/hpce-tools/hpce-tools.yml @@ -0,0 +1,9 @@ +# Clean environment based on pure conda-forge packages +name: hpce-tools +channels: + - conda-forge + - http://conda-pkg.intranet.psi.ch +dependencies: + - python=3.6 + - ldapuserdir + diff --git a/Programming/anaconda/2019.07/conda-env-defs/talos_36/README.md b/Programming/anaconda/2019.07/conda-env-defs/talos_py36/README.md similarity index 100% rename from Programming/anaconda/2019.07/conda-env-defs/talos_36/README.md rename to Programming/anaconda/2019.07/conda-env-defs/talos_py36/README.md diff --git a/Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml b/Programming/anaconda/2019.07/conda-env-defs/talos_py36/talos_py36.yml similarity index 100% rename from Programming/anaconda/2019.07/conda-env-defs/talos_36/talos_36.yml rename to Programming/anaconda/2019.07/conda-env-defs/talos_py36/talos_py36.yml diff --git a/Programming/anaconda/README.md b/Programming/anaconda/README.md index 9d086a9..61ed522 100644 --- a/Programming/anaconda/README.md +++ b/Programming/anaconda/README.md @@ -5,11 +5,12 @@ * The anaconda module just provides the **conda** package management tool together with its directory infrastructure which contains *conda environments* and a cache of downloaded packages * Python and user software is provided in **conda environments**. These environments are located within the directory tree belonging to the anaconda module, e.g. `/afs/psi.ch/sys/psi.merlin/Programming/anaconda/2019.03/conda/envs/` * The software in these environments can be accessed by users through - * loading the anaconda module and then using `conda activate somemodule_py36` - * a seperate pmodule that transparently switches to that installed environment by just setting the correct PATH - * jupyter installations running from one of the environments which discover the other environments if they contain the correct packages (**nb_conda_kernels**) + 1) loading the anaconda module and then using `conda activate somemodule_py36` + 1) a seperate pmodule that transparently switches to that installed environment by just setting the correct PATH to the python binary. + 1) jupyter installations running from one of the environments which discover the other environments if they contain the correct packages (**nb_conda_kernels**) * The `conda` tool has frequent updates, and our experience shows that they should be installed. However, it would be a waste to every time produce a new module, because with the new module would also be associated a new area for environments. So, we prefer to update conda in place, and only make a new anaconda module if their are special incentives - * Environments are self sufficient and do not depend on the conda tool at all. All depending libraries are installed by conda. Conda makes consisten use of **rpath** definitions for executables and libraries, i.e. there is no reason to set `LD_LIBRARY_PATH` at all. + * Most environments are self sufficient and do not depend on the conda tool at all after the instalation: Conda took care of installing all depending libraries, and the builds that conda provides make consistent use of **rpath** definitions for executables and libraries, i.e. there is no reason to set `LD_LIBRARY_PATH` at all. + * There is one important exception: If your environment needs additional setups (activation hooks), then it will rely on the `conda activate` call, since these hooks are only run inside of this call. ## Building a central conda environment @@ -47,12 +48,14 @@ In most cases you will want to go ahead with `pip` installs. However, after runn Proceed as above by defining a YAML file and use conda to first install all the conda based packages. -Even though the YAML file also allows for the specification of pip packages, I advise to do this step separately. The pip steps can fail for various reasons, and it is better to do them interactively. Describe what you have to do in a README.md inside of the `conda-env-defs/${myenv}` folder. +Even though the YAML file also allows for the specification of pip packages, I advise to do this step separately. The pip steps can fail for various reasons, and it is better to do them interactively. Describe what you have to do in a README.md inside of the `conda-env-defs/${myenv}** folder. + +**Note** that if pip triggers compilations, the package may pick up shared libraries from outside the environment. This can lead to problems if the build is done on pmod6.psi.ch which runs SL6, while most of the production environments are now on REHL7! ### installation of a conda environment and adding source compiled packages -**DRAFT!!!** +**This is still a DRAFT!!!** This works if the python package has a correct setup.py build @@ -62,7 +65,7 @@ This works if the python package has a correct setup.py build * document it in `conda-env-defs/${myenv}/README.md` * downlad and store the sources in the install area under `/opt/psi/Programming/anaconda/2019.07/xxxx/mypackage` - * Use pip to install them into the environment (requires the setup.py) + * Use pip to install them into the environment (requires that the package comes with a correct `setup.py`) ``` cd /opt/psi/Programming/anaconda/2019.07/xxxx/mypackage pip install .