pearl-public/pmsco-public
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: pearl-public:rev-distro-2.0.0
Branches Tags
pearl-public:distro
pearl-public:rev-distro-4.2.0 pearl-public:rev-distro-3.0.0 pearl-public:rev-distro-2.2.0 pearl-public:rev-distro-2.1.0 pearl-public:rev-distro-2.0.0 pearl-public:rev-distro-1.0.0
..
compare: pearl-public:rev-distro-4.2.0
Branches Tags
pearl-public:distro
pearl-public:rev-distro-4.2.0 pearl-public:rev-distro-3.0.0 pearl-public:rev-distro-2.2.0 pearl-public:rev-distro-2.1.0 pearl-public:rev-distro-2.0.0 pearl-public:rev-distro-1.0.0

5 Commits
rev-distro ... rev-distro

Author SHA1 Message Date
matthias muntwiler
b64beb694c public release 4.2.0 - see README.md and CHANGES.md for details 2026-01-08 19:10:45 +01:00
matthias muntwiler
ef781e2db4 public release 3.0.0 - see README and CHANGES for details 2021-02-09 12:46:20 +01:00
matthias muntwiler
2b3dbd8bac update README 2020-09-04 16:31:45 +02:00
matthias muntwiler
7c61eb1b41 public release 2.2.0 - see README.md and CHANGES.md for details 2020-09-04 16:22:42 +02:00
matthias muntwiler
fbd2d4fa8c public distro 2.1.0 2019-07-19 12:54:54 +02:00
207 changed files with 46224 additions and 5892 deletions
Expand all files Collapse all files

51
.gitea/workflows/build-test.yaml Normal file
View File

@@ -0,0 +1,51 @@
name: build and test the package
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: check runner environment
run: |
uname -a
lsb_release -a
echo "Runner home: $HOME"
- name: check out
uses: actions/checkout@v5
- name: set up compilers
run: |
sudo apt-get update
sudo apt-get -y install binutils build-essential g++ gcc gfortran libblas-dev liblapack-dev openmpi-bin openmpi-common sqlite3
- name: set up python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.9.18"
enable-cache: true
- name: lint with ruff
# configuration is in pyproject.toml
run: |
uvx ruff check --extend-exclude=.venv,build pmsco
- name: install dependencies
run: uv sync --locked --all-extras --dev
- name: tests
run: |
uv run nosetests

45
.gitea/workflows/deploy-pages.yaml Normal file
View File

@@ -0,0 +1,45 @@
name: build and deploy documentation
on:
push:
branches:
- master
jobs:
build-and-deploy:
runs-on: ubuntu-latest
container:
image: gitea.psi.ch/pearl/docs
credentials:
username: ${{ gitea.actor }}
password: ${{ secrets.package_token }}
steps:
- name: checkout
working-directory: /app
run: |
git clone --branch master --single-branch https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git
- name: build
working-directory: /app/pmsco/docs
run: |
export REVISION=$(shell git describe --always --tags --dirty --long || echo "unknown, "`date +"%F %T %z"`)
export OUTDIR=/app/build
doxygen config.dox
- name: configure git
working-directory: /app/pmsco
run: |
git config --global user.name "Gitea Actions"
git config --global user.email "actions@gitea.local"
- name: push to gitea-pages
working-directory: /app/pmsco
run: |
git checkout --orphan gitea-pages
git reset --hard
cp -r /app/build/html/* .
git add .
git commit -m "Deploy documentation to gitea"
git push -f https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git gitea-pages

12
.githooks/install-hooks.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
echo "Setting up Git hooks..."
cd "$(dirname "$0")"
cd ..
# Create symlinks
ln -sf ../../.githooks/pre-commit .git/hooks/pre-commit
ln -sf ../../.githooks/pre-push .git/hooks/pre-push
chmod +x .git/hooks/*
echo "Git hooks installed successfully!"

34
.githooks/pre-commit Executable file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
# .git/hooks/pre-commit
# requires uv
# Track overall status
PASS=true
# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${YELLOW}Running pre-commit checks...${NC}"
PY_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.py$')
# Python checks
if [ -n "$PY_FILES" ]; then
echo -e "${YELLOW}Checking Python files...${NC}"
if ! uvx ruff check --extend-exclude=.*,build*; then
PASS=false
fi
fi
# Final status
if [ "$PASS" = true ]; then
echo -e "${GREEN}All checks passed!${NC}"
exit 0
else
echo -e "${RED}Some checks failed. Please fix issues before committing.${NC}"
exit 1
fi

43
.githooks/pre-push Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
# .git/hooks/pre-push
# Track overall status
PASS=true
# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${YELLOW}Running pre-push checks...${NC}"
PY_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.py$')
# Python checks
echo -e "${YELLOW}Checking Python files...${NC}"
# Lint
if ! uvx ruff check --extend-exclude=.*,build*; then
PASS=false
fi
# Compile
uv sync
# Run different test suites based on changed files
echo -e "${YELLOW}Running Python tests...${NC}"
if ! uv run nosetests; then
echo -e "Tests failed. Push aborted."
PASS=false
fi
# Final status
if [ "$PASS" = true ]; then
echo -e "${GREEN}All checks passed!${NC}"
exit 0
else
echo -e "${RED}Some checks failed. Please fix issues before committing.${NC}"
exit 1
fi

3
.gitignore vendored
View File

@@ -2,6 +2,8 @@ work/*
debug/*
lib/*
dev/*
build/*
__pycache__/*
*.pyc
*.o
*.so
@@ -15,3 +17,4 @@ dev/*
.ropeproject/*
.fuse*
.trash
.wraplock

72
CHANGES.md Normal file
View File

@@ -0,0 +1,72 @@
Release 4.2.0 (2026-01-01)
==========================
- Switch to Astral-UV package manager
- CI lint, build, test workflow in gitea
- Automated documentation workflow in gitea
- Recommended Python version 3.12 (compatibility 3.10-3.13)
- Multipole expansion
- Table optimization mode
- Integrate phagen scattering amplitude calculator
- Select modulation and R-factor functions in runfile
- Parametric holo scan generator
- Namespace package installation, support for editable installation
- Simplified command line
- Meson build system
- Differential cross section in periodic table
- Configurable reports
- Path resolution
- Database interface for reports
- Runfile based job scheduling
Release 3.0.0 (2021-02-01)
==========================
- Compatibility with recent conda and singularity versions
- Installation: include plantuml.jar
- Documentation: replace doxypy by doxypypy
- Redefine output_file property
- Documentation of run file interface
- Introduce runfile interface
- Set legacy Fortran for compatibility with recent compiler
- Graphics: fixed color range for modulation functions
- Cluster: build_element accepts symbol or number
- Graphics: swarm plot
- Graphics: genetic chart
- Periodic table: use common binding energies in condensed matter XPS
- Periodic table: reformat bindingenergy.json, add more import/export functions
- Spectrum: add plot cross section function
Release 2.2.0 (2020-09-04)
==========================
- Demo project for arbitrary molecule (cluster file)
- Bugfix: DATA CORRUPTION in phagen translator (emitter mix-up)
- Bugfix: load native cluster file: wrong column order
- Bugfix: initial-state command line option not respected
- Photoionization cross sections and spectrum simulator
- Database: use local lock objects
- Database: create view on results and models
- REFACTORING: Domain -> ModelSpace, Params -> CalculatorParams
- REFACTORING: symmetry -> domain
- Possible conda/mpi4py conflict - changed installation procedure
- Cluster: new calc_scattering_angles function
- Include a periodic table of binding energies of the elements
- Clean up files in the main loop and in the end
- Bugfix: database ingestion overwrites results from previous jobs
- Time out at least 10 minutes before the hard time limit given on the command line
- Cluster: new get_center function
- Bugfix: type error in grid optimizer
- Bugfix: file extension in phagen/makefile
- Dispatch: new algo to distribute processing slots to task levels
- Bugfix: load single-line data files correctly!
- Cluster generator for zincblende crystal
- Phagen translator: fix phase convention (MAJOR), fix single-energy
- Dispatch: give more priority to mid-level tasks in single mode
- Improve performance of cluster add_bulk, add_layer and rotate
- Unit test for Cluster.translate, bugfix in translate and relax
- Fix compatibility with numpy >= 1.14, require numpy >= 1.13
- Database: introduce job-tags
- qpmsco: delete code after execution

201
LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2015-2020 Paul Scherrer Institut
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

4
NOTICE.md
View File

@@ -5,10 +5,10 @@ List of Contributors
Original Author
---------------
Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
Contributors
------------
- Frederik Schirdewahn, <mailto:frederik.schirdewahn@psi.ch>

111
README.md
View File

@@ -1,70 +1,129 @@
Introduction
============
PMSCO stands for PEARL multiple-scattering cluster calculations and structural optimization.
It is a collection of computer programs to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data.
PMSCO (PSI multiple-scattering cluster calculations and structural optimization)
is a Python-based workflow engine to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data using machine learning techniques.
PMSCO was developed at the [Paul Scherrer Institut (PSI)](https://www.psi.ch/)
by the team of the [PEARL beamline](https://www.psi.ch/en/sls/pearl).
The actual scattering calculation is done by code developed by other parties.
PMSCO wraps around that program and facilitates parameter handling, cluster building, structural optimization and parallel processing.
PMSCO wraps around those programs and facilitates parameter handling, cluster building, structural optimization and parallel processing.
In the current version, the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code
developed by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley (1999) is used for scattering calculations.
Other code can be integrated as well.
Instead of EDAC built-in routines, alternatively,
the PHAGEN program from [MsSpec-1.0](https://msspec.cnrs.fr/index.html) can be used to calculate atomic scattering factors.
Highlights
----------
- angle or energy scanned XPD.
- various scanning modes including energy, polar angle, azimuthal angle, analyser angle.
- averaging over multiple symmetries (domains or emitters).
- angle and energy scanned XPD.
- various scanning modes including energy, manipulator angle (polar/azimuthal), emission angle.
- averaging over multiple domains and emitters.
- global optimization of multiple scans.
- structural optimization algorithms: particle swarm optimization, grid search, gradient search.
- structural optimization algorithms: particle swarm optimization, genetic algorithm, grid scan, table scan.
- detailed reports and graphs of result files.
- calculation of the modulation function.
- calculation of the weighted R-factor.
- automatic parallel processing using OpenMPI.
- compatible with Slurm resource manager on Linux cluster machines.
Installation
============
PMSCO is written in Python 3.6 and compatible with Python 2.7.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a cluster with 20-50 available processor cores is recommended.
PMSCO is written in Python. The recommended Python version is 3.12.
Further requirements are the GNU compiler collection, BLAS/LAPACK libraries, OpenMPI and a package manager such as uv, pip or conda.
For optimization jobs, a cluster machine with 20-50 available processor cores is recommended.
Smaller jobs run on any recent Linux workstation.
The code requires about 2 GB of RAM per process.
Detailed installation instructions and dependencies can be found in the documentation
(docs/src/installation.dox).
A [Doxygen](http://www.stack.nl/~dimitri/doxygen/index.html) compiler with Doxypy is required to generate the documentation in HTML or LaTeX format.
The public distribution of PMSCO does not contain the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code.
Please obtain the EDAC source code from the original author, copy it to the pmsco/edac directory, and apply the edac_all.patch patch.
License
=======
The source code of PMSCO is licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
Please read and respect the license agreement.
This _does not include_ the calculation packages contained in the subprojects folder which are licensed separately.
Please share your extensions of the code with the original author.
The gitlab facility can be used to create forks and to submit pull requests.
Attribution notices for your contributions shall be added to the NOTICE.md file.
- Please read and respect the respective license agreements.
- Please acknowledge the use of the code.
- Please consider sharing your developments with the original author.
Due to different copyright terms, the third-party calculation programs are not contained in the public software repository.
These programs may not be used without an explicit agreement by the respective original authors.
Author
------
Authors
-------
Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Frederik Schirdewahn, <mailto:frederik.schirdewahn@psi.ch>
Copyright
---------
Copyright 2015-2018 by [Paul Scherrer Institut](http://www.psi.ch)
Copyright 2015-2025 by [Paul Scherrer Institut](http://www.psi.ch)
Release Notes
=============
For a detailed list of changes, see the CHANGES.md file.
4.2.0 (2026-01-01)
------------------
- Recommended Python version 3.12 (compatibility 3.10-3.13)
- Build system and package environment
- Switch to Astral-UV package manager
- Meson build system for Fortran, C and C++ extension modules
- Namespace package installation, support for editable installation
- CI lint, build, test workflow in gitea
- Automated documentation workflow in gitea
- User interface
- Simplified command line, all configuration via runfile and/or project class
- Select modulation and R-factor functions in runfile
- Parametric holo scan generator
- Configurable reports
- Path resolution in runfile
- Database interface for reports
- Runfile based job scheduling
- Calculation features
- Multipole expansion
- Table optimization mode
- Integrate phagen scattering amplitude calculator
- Differential cross section in periodic table
3.0.0 (2021-02-08)
------------------
- Run file interface replaces command line arguments:
- Specify all run-time parameters in a JSON-formatted text file.
- Override any public attribute of the project class.
- Only the name of the run file is needed on the command line.
- The command line interface is still available, some default values and the handling of directory paths have changed.
Check your code for compatibility.
- Integrated job scheduling with the Slurm resource manager:
- Declare all job arguments in the run file and have PMSCO submit the job.
- Graphics scripts for genetic chart and swarm population (experimental feature).
- Update for compatibility with recent Ubuntu (20.04), Anaconda (4.8) and Singularity (3.7).
- Drop compatibility with Python 2.7, minimum requirement is Python 3.6.
2.2.0 (2020-09-04)
------------------
This release breaks existing project code unless the listed refactorings are applied.
- Major refactoring: The 'symmetry' calculation level is renamed to 'domain'.
The previous Domain class is renamed to ModelSpace, Params to CalculatorParams.
The refactorings must be applied to project code as well.
- Included periodic table of elements with electron binding energies and scattering cross-sections.
- Various bug fixes in cluster routines, data file handling, and in the PHAGEN interface.
- Experimental sqlite3 database interface for optimization results.

157
bin/pmsco.ra.template
View File

@@ -1,157 +0,0 @@
#!/bin/bash
#
# Slurm script template for PMSCO calculations on the Ra cluster
# based on run_mpi_HPL_nodes-2.sl by V. Markushin 2016-03-01
#
# Use:
# - enter the appropriate parameters and save as a new file.
# - call the sbatch command to pass the job script.
# request a specific number of nodes and tasks.
# example:
# sbatch --nodes=2 --ntasks-per-node=24 --time=02:00:00 run_pmsco.sl
#
# PMSCO arguments
# copy this template to a new file, and set the arguments
#
# PMSCO_WORK_DIR
# path to be used as working directory.
# contains the script derived from this template.
# receives output and temporary files.
#
# PMSCO_PROJECT_FILE
# python module that declares the project and starts the calculation.
# must include the file path relative to $PMSCO_WORK_DIR.
#
# PMSCO_SOURCE_DIR
# path to the pmsco source directory
# (the directory which contains the bin, lib, pmsco sub-directories)
#
# PMSCO_SCAN_FILES
# list of scan files.
#
# PMSCO_OUT
# name of output file. should not include a path.
#
# all paths are relative to $PMSCO_WORK_DIR or (better) absolute.
#
#
# Further arguments
#
# PMSCO_JOBNAME (required)
# the job name is the base name for output files.
#
# PMSCO_WALLTIME_HR (integer, required)
# wall time limit in hours. must be integer, minimum 1.
# this value is passed to PMSCO.
# it should specify the same amount of wall time as requested from the scheduler.
#
# PMSCO_MODE (optional)
# calculation mode: single, swarm, grid, gradient
#
# PMSCO_CODE (optional)
# calculation code: edac, msc, test
#
# PMSCO_LOGLEVEL (optional)
# request log level: DEBUG, INFO, WARNING, ERROR
# create a log file based on the job name.
#
# PMSCO_PROJECT_ARGS (optional)
# extra arguments that are parsed by the project module.
#
#SBATCH --job-name="_PMSCO_JOBNAME"
#SBATCH --output="_PMSCO_JOBNAME.o.%j"
#SBATCH --error="_PMSCO_JOBNAME.e.%j"
PMSCO_WORK_DIR="_PMSCO_WORK_DIR"
PMSCO_JOBNAME="_PMSCO_JOBNAME"
PMSCO_WALLTIME_HR=_PMSCO_WALLTIME_HR
PMSCO_PROJECT_FILE="_PMSCO_PROJECT_FILE"
PMSCO_MODE="_PMSCO_MODE"
PMSCO_CODE="_PMSCO_CODE"
PMSCO_SOURCE_DIR="_PMSCO_SOURCE_DIR"
PMSCO_SCAN_FILES="_PMSCO_SCAN_FILES"
PMSCO_OUT="_PMSCO_JOBNAME"
PMSCO_LOGLEVEL="_PMSCO_LOGLEVEL"
PMSCO_PROJECT_ARGS="_PMSCO_PROJECT_ARGS"
module load psi-python27/2.4.1
module load gcc/4.8.5
module load openmpi/1.10.2
source activate pmsco
echo '================================================================================'
echo "=== Running $0 at the following time and place:"
date
/bin/hostname
cd $PMSCO_WORK_DIR
pwd
ls -lA
#the intel compiler is currently not compatible with mpi4py. -mm 170131
#echo
#echo '================================================================================'
#echo "=== Setting the environment to use Intel Cluster Studio XE 2016 Update 2 intel/16.2:"
#cmd="source /opt/psi/Programming/intel/16.2/bin/compilervars.sh intel64"
#echo $cmd
#$cmd
echo
echo '================================================================================'
echo "=== The environment is set as following:"
env
echo
echo '================================================================================'
echo "BEGIN test"
echo "=== Intel native mpirun will get the number of nodes and the machinefile from Slurm"
which mpirun
cmd="mpirun /bin/hostname"
echo $cmd
$cmd
echo "END test"
echo
echo '================================================================================'
echo "BEGIN mpirun pmsco"
echo "Intel native mpirun will get the number of nodes and the machinefile from Slurm"
echo
echo "code revision"
cd "$PMSCO_SOURCE_DIR"
git log --pretty=tformat:'%h %ai %d' -1
python -m compileall pmsco
python -m compileall projects
cd "$PMSCO_WORK_DIR"
echo
PMSCO_CMD="python $PMSCO_SOURCE_DIR/pmsco $PMSCO_PROJECT_FILE"
PMSCO_ARGS="$PMSCO_PROJECT_ARGS"
if [ -n "$PMSCO_SCAN_FILES" ]; then
PMSCO_ARGS="-s $PMSCO_SCAN_FILES $PMSCO_ARGS"
fi
if [ -n "$PMSCO_CODE" ]; then
PMSCO_ARGS="-c $PMSCO_CODE $PMSCO_ARGS"
fi
if [ -n "$PMSCO_MODE" ]; then
PMSCO_ARGS="-m $PMSCO_MODE $PMSCO_ARGS"
fi
if [ -n "$PMSCO_OUT" ]; then
PMSCO_ARGS="-o $PMSCO_OUT $PMSCO_ARGS"
fi
if [ "$PMSCO_WALLTIME_HR" -ge 1 ]; then
PMSCO_ARGS="-t $PMSCO_WALLTIME_HR $PMSCO_ARGS"
fi
if [ -n "$PMSCO_LOGLEVEL" ]; then
PMSCO_ARGS="--log-level $PMSCO_LOGLEVEL --log-file $PMSCO_JOBNAME.log $PMSCO_ARGS"
fi
which mpirun
ls -l "$PMSCO_SOURCE_DIR"
ls -l "$PMSCO_PROJECT_FILE"
# Do no use the OpenMPI specific options, like "-x LD_LIBRARY_PATH", with the Intel mpirun.
cmd="mpirun $PMSCO_CMD $PMSCO_ARGS"
echo $cmd
$cmd
echo "END mpirun pmsco"
echo '================================================================================'
date
ls -lAtr
echo '================================================================================'
exit 0

178
bin/pmsco.sge.template
View File

@@ -1,178 +0,0 @@
#!/bin/bash
#
# SGE script template for MSC calculations
#
# This script uses the tight integration of openmpi-1.4.5-gcc-4.6.3 in SGE
# using the parallel environment (PE) "orte".
# This script must be used only with qsub command - do NOT run it as a stand-alone
# shell script because it will start all processes on the local node.
#
# PhD arguments
# copy this template to a new file, and set the arguments
#
# PHD_WORK_DIR
# path to be used as working directory.
# contains the SGE script derived from this template.
# receives output and temporary files.
#
# PHD_PROJECT_FILE
# python module that declares the project and starts the calculation.
# must include the file path relative to $PHD_WORK_DIR.
#
# PHD_SOURCE_DIR
# path to the pmsco source directory
# (the directory which contains the bin, lib, pmsco sub-directories)
#
# PHD_SCAN_FILES
# list of scan files.
#
# PHD_OUT
# name of output file. should not include a path.
#
# all paths are relative to $PHD_WORK_DIR or (better) absolute.
#
#
# Further arguments
#
# PHD_JOBNAME (required)
# the job name is the base name for output files.
#
# PHD_NODES (required)
# number of computing nodes (processes) to allocate for the job.
#
# PHD_WALLTIME_HR (required)
# wall time limit (hours)
#
# PHD_WALLTIME_MIN (required)
# wall time limit (minutes)
#
# PHD_MODE (optional)
# calculation mode: single, swarm, grid, gradient
#
# PHD_CODE (optional)
# calculation code: edac, msc, test
#
# PHD_LOGLEVEL (optional)
# request log level: DEBUG, INFO, WARNING, ERROR
# create a log file based on the job name.
#
# PHD_PROJECT_ARGS (optional)
# extra arguments that are parsed by the project module.
#
PHD_WORK_DIR="_PHD_WORK_DIR"
PHD_JOBNAME="_PHD_JOBNAME"
PHD_NODES=_PHD_NODES
PHD_WALLTIME_HR=_PHD_WALLTIME_HR
PHD_WALLTIME_MIN=_PHD_WALLTIME_MIN
PHD_PROJECT_FILE="_PHD_PROJECT_FILE"
PHD_MODE="_PHD_MODE"
PHD_CODE="_PHD_CODE"
PHD_SOURCE_DIR="_PHD_SOURCE_DIR"
PHD_SCAN_FILES="_PHD_SCAN_FILES"
PHD_OUT="_PHD_JOBNAME"
PHD_LOGLEVEL="_PHD_LOGLEVEL"
PHD_PROJECT_ARGS="_PHD_PROJECT_ARGS"
# Define your job name, parallel environment with the number of slots, and run time:
#$ -cwd
#$ -N _PHD_JOBNAME.job
#$ -pe orte _PHD_NODES
#$ -l ram=2G
#$ -l s_rt=_PHD_WALLTIME_HR:_PHD_WALLTIME_MIN:00
#$ -l h_rt=_PHD_WALLTIME_HR:_PHD_WALLTIME_MIN:30
#$ -V
###################################################
# Fix the SGE environment-handling bug (bash):
source /usr/share/Modules/init/sh
export -n -f module
# Load the environment modules for this job (the order may be important):
module load python/python-2.7.5
module load gcc/gcc-4.6.3
module load mpi/openmpi-1.4.5-gcc-4.6.3
module load blas/blas-20110419-gcc-4.6.3
module load lapack/lapack-3.4.2-gcc-4.6.3
export LD_LIBRARY_PATH=$PHD_SOURCE_DIR/lib/:$LD_LIBRARY_PATH
###################################################
# Set the environment variables:
MPIEXEC=$OPENMPI/bin/mpiexec
# OPENMPI is set by the mpi/openmpi-* module.
export OMP_NUM_THREADS=1
export OMPI_MCA_btl='openib,sm,self'
# export OMPI_MCA_orte_process_binding=core
##############
# BEGIN DEBUG
# Print the SGE environment on master host:
echo "================================================================"
echo "=== SGE job JOB_NAME=$JOB_NAME JOB_ID=$JOB_ID"
echo "================================================================"
echo DATE=`date`
echo HOSTNAME=`hostname`
echo PWD=`pwd`
echo "NSLOTS=$NSLOTS"
echo "PE_HOSTFILE=$PE_HOSTFILE"
cat $PE_HOSTFILE
echo "================================================================"
echo "Running environment:"
env
echo "================================================================"
echo "Loaded environment modules:"
module list 2>&1
echo
# END DEBUG
##############
##############
# Setup
cd "$PHD_SOURCE_DIR"
python -m compileall .
cd "$PHD_WORK_DIR"
ulimit -c 0
###################################################
# The command to run with mpiexec:
CMD="python $PHD_PROJECT_FILE"
ARGS="$PHD_PROJECT_ARGS"
if [ -n "$PHD_SCAN_FILES" ]; then
ARGS="-s $PHD_SCAN_FILES -- $ARGS"
fi
if [ -n "$PHD_CODE" ]; then
ARGS="-c $PHD_CODE $ARGS"
fi
if [ -n "$PHD_MODE" ]; then
ARGS="-m $PHD_MODE $ARGS"
fi
if [ -n "$PHD_OUT" ]; then
ARGS="-o $PHD_OUT $ARGS"
fi
if [ "$PHD_WALLTIME_HR" -ge 1 ]
then
ARGS="-t $PHD_WALLTIME_HR $ARGS"
else
ARGS="-t 0.5 $ARGS"
fi
if [ -n "$PHD_LOGLEVEL" ]; then
ARGS="--log-level $PHD_LOGLEVEL --log-file $PHD_JOBNAME.log $ARGS"
fi
# The MPI command to run:
MPICMD="$MPIEXEC --prefix $OPENMPI -x PATH -x LD_LIBRARY_PATH -x OMP_NUM_THREADS -x OMPI_MCA_btl -np $NSLOTS $CMD $ARGS"
echo "Command to run:"
echo "$MPICMD"
echo
exec $MPICMD
exit 0

145
bin/qpmsco.ra.sh
View File

@@ -1,145 +0,0 @@
#!/bin/sh
#
# submission script for PMSCO calculations on the Ra cluster
if [ $# -lt 1 ]; then
echo "Usage: $0 [NOSUB] DESTDIR JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]"
echo ""
echo " NOSUB (optional): do not submit the script to the queue. default: submit."
echo " DESTDIR: destination directory. must exist. a sub-dir \$JOBNAME is created."
echo " JOBNAME (text): name of job. use only alphanumeric characters, no spaces."
echo " NODES (integer): number of computing nodes. (1 node = 24 or 32 processors)."
echo " do not specify more than 2."
echo " TASKS_PER_NODE (integer): 1...24, or 32."
echo " 24 or 32 for full-node allocation."
echo " 1...23 for shared node allocation."
echo " WALLTIME:HOURS (integer): requested wall time."
echo " 1...24 for day partition"
echo " 24...192 for week partition"
echo " 1...192 for shared partition"
echo " PROJECT: python module (file path) that declares the project and starts the calculation."
echo " MODE: PMSCO calculation mode (single|swarm|gradient|grid)."
echo " ARGS (optional): any number of further PMSCO or project arguments (except mode and time)."
echo ""
echo "the job script is written to \$DESTDIR/\$JOBNAME which is also the destination of calculation output."
exit 1
fi
# location of the pmsco package is derived from the path of this script
SCRIPTDIR="$(dirname $(readlink -f $0))"
SOURCEDIR="$SCRIPTDIR/.."
PMSCO_SOURCE_DIR="$SOURCEDIR"
# read arguments
if [ "$1" == "NOSUB" ]; then
NOSUB="true"
shift
else
NOSUB="false"
fi
DEST_DIR="$1"
shift
PMSCO_JOBNAME=$1
shift
PMSCO_NODES=$1
PMSCO_TASKS_PER_NODE=$2
PMSCO_TASKS=$(expr $PMSCO_NODES \* $PMSCO_TASKS_PER_NODE)
shift 2
PMSCO_WALLTIME_HR=$1
PMSCO_WALLTIME_MIN=$(expr $PMSCO_WALLTIME_HR \* 60)
shift
# select partition
if [ $PMSCO_WALLTIME_HR -ge 25 ]; then
PMSCO_PARTITION="week"
else
PMSCO_PARTITION="day"
fi
if [ $PMSCO_TASKS_PER_NODE -lt 24 ]; then
PMSCO_PARTITION="shared"
fi
PMSCO_PROJECT_FILE="$(readlink -f $1)"
shift
PMSCO_MODE="$1"
shift
PMSCO_PROJECT_ARGS="$*"
# use defaults, override explicitly in PMSCO_PROJECT_ARGS if necessary
PMSCO_SCAN_FILES=""
PMSCO_LOGLEVEL=""
PMSCO_CODE=""
# set up working directory
cd "$DEST_DIR"
if [ ! -d "$PMSCO_JOBNAME" ]; then
mkdir "$PMSCO_JOBNAME"
fi
cd "$PMSCO_JOBNAME"
WORKDIR="$(pwd)"
PMSCO_WORK_DIR="$WORKDIR"
# provide revision information, requires git repository
cd "$SOURCEDIR"
PMSCO_REV=$(git log --pretty=format:"Data revision %h, %ai" -1)
if [ $? -ne 0 ]; then
PMSCO_REV="Data revision unknown, "$(date +"%F %T %z")
fi
cd "$WORKDIR"
echo "$PMSCO_REV" > revision.txt
# generate job script from template
sed -e "s:_PMSCO_WORK_DIR:$PMSCO_WORK_DIR:g" \
-e "s:_PMSCO_JOBNAME:$PMSCO_JOBNAME:g" \
-e "s:_PMSCO_NODES:$PMSCO_NODES:g" \
-e "s:_PMSCO_WALLTIME_HR:$PMSCO_WALLTIME_HR:g" \
-e "s:_PMSCO_PROJECT_FILE:$PMSCO_PROJECT_FILE:g" \
-e "s:_PMSCO_PROJECT_ARGS:$PMSCO_PROJECT_ARGS:g" \
-e "s:_PMSCO_CODE:$PMSCO_CODE:g" \
-e "s:_PMSCO_MODE:$PMSCO_MODE:g" \
-e "s:_PMSCO_SOURCE_DIR:$PMSCO_SOURCE_DIR:g" \
-e "s:_PMSCO_SCAN_FILES:$PMSCO_SCAN_FILES:g" \
-e "s:_PMSCO_LOGLEVEL:$PMSCO_LOGLEVEL:g" \
"$SCRIPTDIR/pmsco.ra.template" > $PMSCO_JOBNAME.job
chmod u+x "$PMSCO_JOBNAME.job"
# request nodes and tasks
#
# The option --ntasks-per-node is meant to be used with the --nodes option.
# (For the --ntasks option, the default is one task per node, use the --cpus-per-task option to change this default.)
#
# sbatch options
# --cores-per-socket=16
# 32 cores per node
# --partition=[shared|day|week]
# --time=8-00:00:00
# override default time limit (2 days in long queue)
# time formats: "minutes", "minutes:seconds", "hours:minutes:seconds", "days-hours", "days-hours:minutes", "days-hours:minutes:seconds"
# --mail-type=ALL
# --test-only
# check script but do not submit
#
SLURM_ARGS="--nodes=$PMSCO_NODES --ntasks-per-node=$PMSCO_TASKS_PER_NODE"
if [ $PMSCO_TASKS_PER_NODE -gt 24 ]; then
SLURM_ARGS="--cores-per-socket=16 $SLURM_ARGS"
fi
SLURM_ARGS="--partition=$PMSCO_PARTITION $SLURM_ARGS"
SLURM_ARGS="--time=$PMSCO_WALLTIME_HR:00:00 $SLURM_ARGS"
CMD="sbatch $SLURM_ARGS $PMSCO_JOBNAME.job"
echo $CMD
if [ "$NOSUB" != "true" ]; then
$CMD
fi
exit 0

128
bin/qpmsco.sge
View File

@@ -1,128 +0,0 @@
#!/bin/sh
#
# submission script for PMSCO calculations on Merlin cluster
#
if [ $# -lt 1 ]; then
echo "Usage: $0 [NOSUB] JOBNAME NODES WALLTIME:HOURS PROJECT MODE [LOG_LEVEL]"
echo ""
echo " NOSUB (optional): do not submit the script to the queue. default: submit."
echo " WALLTIME:HOURS (integer): sets the wall time limits."
echo " soft limit = HOURS:00:00"
echo " hard limit = HOURS:00:30"
echo " for short.q: HOURS = 0 (-> MINUTES=30)"
echo " for all.q: HOURS <= 24"
echo " for long.q: HOURS <= 96"
echo " PROJECT: python module (file path) that declares the project and starts the calculation."
echo " MODE: PMSCO calculation mode (single|swarm|gradient|grid)."
echo " LOG_LEVEL (optional): one of DEBUG, INFO, WARNING, ERROR if log files should be produced."
echo ""
echo "the job script complete with the program code and input/output data is generated in ~/jobs/\$JOBNAME"
exit 1
fi
# location of the pmsco package is derived from the path of this script
SCRIPTDIR="$(dirname $(readlink -f $0))"
SOURCEDIR="$SCRIPTDIR/.."
PHD_SOURCE_DIR="$SOURCEDIR"
PHD_CODE="edac"
# read arguments
if [ "$1" == "NOSUB" ]; then
NOSUB="true"
shift
else
NOSUB="false"
fi
PHD_JOBNAME=$1
shift
PHD_NODES=$1
shift
PHD_WALLTIME_HR=$1
PHD_WALLTIME_MIN=0
shift
PHD_PROJECT_FILE="$(readlink -f $1)"
PHD_PROJECT_ARGS=""
shift
PHD_MODE="$1"
shift
PHD_LOGLEVEL=""
if [ "$1" == "DEBUG" ] || [ "$1" == "INFO" ] || [ "$1" == "WARNING" ] || [ "$1" == "ERROR" ]; then
PHD_LOGLEVEL="$1"
shift
fi
# ignore remaining arguments
PHD_SCAN_FILES=""
# select allowed queues
QUEUE=short.q,all.q,long.q
# for short queue (limit 30 minutes)
if [ "$PHD_WALLTIME_HR" -lt 1 ]; then
PHD_WALLTIME_HR=0
PHD_WALLTIME_MIN=30
fi
# set up working directory
cd ~
if [ ! -d "jobs" ]; then
mkdir jobs
fi
cd jobs
if [ ! -d "$PHD_JOBNAME" ]; then
mkdir "$PHD_JOBNAME"
fi
cd "$PHD_JOBNAME"
WORKDIR="$(pwd)"
PHD_WORK_DIR="$WORKDIR"
# provide revision information, requires git repository
cd "$SOURCEDIR"
PHD_REV=$(git log --pretty=format:"Data revision %h, %ad" --date=iso -1)
if [ $? -ne 0 ]; then
PHD_REV="Data revision unknown, "$(date +"%F %T %z")
fi
cd "$WORKDIR"
echo "$PHD_REV" > revision.txt
# generate job script from template
sed -e "s:_PHD_WORK_DIR:$PHD_WORK_DIR:g" \
-e "s:_PHD_JOBNAME:$PHD_JOBNAME:g" \
-e "s:_PHD_NODES:$PHD_NODES:g" \
-e "s:_PHD_WALLTIME_HR:$PHD_WALLTIME_HR:g" \
-e "s:_PHD_WALLTIME_MIN:$PHD_WALLTIME_MIN:g" \
-e "s:_PHD_PROJECT_FILE:$PHD_PROJECT_FILE:g" \
-e "s:_PHD_PROJECT_ARGS:$PHD_PROJECT_ARGS:g" \
-e "s:_PHD_CODE:$PHD_CODE:g" \
-e "s:_PHD_MODE:$PHD_MODE:g" \
-e "s:_PHD_SOURCE_DIR:$PHD_SOURCE_DIR:g" \
-e "s:_PHD_SCAN_FILES:$PHD_SCAN_FILES:g" \
-e "s:_PHD_LOGLEVEL:$PHD_LOGLEVEL:g" \
"$SCRIPTDIR/pmsco.sge.template" > $PHD_JOBNAME.job
chmod u+x "$PHD_JOBNAME.job"
if [ "$NOSUB" != "true" ]; then
# suppress bash error [stackoverflow.com/questions/10496758]
unset module
# submit the job script
# EMAIL must be defined in the environment
if [ -n "$EMAIL" ]; then
qsub -q $QUEUE -m ae -M $EMAIL $PHD_JOBNAME.job
else
qsub -q $QUEUE $PHD_JOBNAME.job
fi
fi
exit 0

633
docs/config.dox
View File

File diff suppressed because it is too large Load Diff

13
docs/makefile
View File

@@ -10,26 +10,25 @@ SHELL=/bin/sh
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .html
.PHONY: all docs clean
.PHONY: all docs html clean
DOX=doxygen
DOXOPTS=
LATEX_DIR=latex
REVISION=$(shell git describe --always --tags --dirty --long || echo "unknown, "`date +"%F %T %z"`)
export REVISION
OUTDIR=
export OUTDIR
all: docs
all: html
docs: doxygen pdf
docs: html
doxygen:
$(DOX) $(DOXOPTS) config.dox
pdf: doxygen
-$(MAKE) -C $(LATEX_DIR)
html: doxygen
clean:
-rm -r latex/*
-rm -r html/*

2
docs/py_filter.sh Executable file
View File

@@ -0,0 +1,2 @@
#!/bin/bash
python -m doxypypy.doxypypy -a -c $1

28
docs/readme.md Normal file
View File

@@ -0,0 +1,28 @@
To compile the source code documentation in HTML format on Ubuntu, follow the instructions below.
~~~~~~{.sh}
apt-get update
apt-get install -y --no-install-recommends \
default-jre \
doxygen \
gawk \
git \
graphviz \
pandoc \
wget
pip install --no-cache-dir \
doxypypy \
meson \
meson-python \
ninja \
pynose
wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
export PLANTUML_JAR_PATH=/app/plantuml.jar
cd pmsco/docs
doxygen config.dox
~~~~~~
Open `pmsco/docs/html/index.html` in your browser.

7
docs/readme.txt
View File

@@ -1,7 +0,0 @@
to compile the source code documentation, you need the following packages (naming according to Debian):
doxygen
doxygen-gui (optional)
doxypy
graphviz
latex (optional)

146
docs/src/commandline.dox
View File

@@ -1,128 +1,48 @@
/*! @page pag_command Command Line
\section sec_command Command Line
This section describes the command line arguments for a direct call of PMSCO from the shell.
For batch job submission to Slurm see @ref sec_slurm.
Assuming that PMSCO has been installed in the active Python environment (@ref pag_install),
the basic command line of PMSCO is as follows:
Since PMSCO is started indirectly by a call of the specific project module,
the syntax of the command line arguments is defined by the project module.
However, to reduce the amount of custom code and documentation and to avoid confusion
it is recommended to adhere to the standard syntax described below.
~~~~~~{.sh}
[mpiexec -np NPROCESSES] python -m pmsco [options]
~~~~~~
The basic command line is as follows:
@code{.sh}
[mpiexec -np NPROCESSES] python path/to/pmsco path/to/project.py [common args] [project args]
@endcode
The first portion between square brackets is necessary for parallel execution using MPI.
Replace `NPROCESSES` by the number of processes.
Include the first portion between square brackets if you want to run parallel processes.
Specify the number of processes as the @c -np option.
@c path/to/pmsco is the directory where <code>__main.py__</code> is located.
Do not include the extension <code>.py</code> or a trailing slash.
@c path/to/project.py should be the path and name to your project module.
Common args and project args are described below.
Note: In contrast to earlier versions, the project module is not executed directly any more.
Rather, it is loaded by the main pmsco module as a 'plug-in'.
The PMSCO main program has a limited number of `common arguments` that are described below.
Usually, all parameters should be declared in a @ref pag_runfile so that they can be archived with the results.
However, in some cases it may be necessary to override some common parameters, e.g. the job name, on the command line.
\subsection sec_common_args Common Arguments
\subsection sec_command_common Common Arguments
All common arguments are optional and default to more or less reasonable values if omitted.
They can be added to the command line in arbitrary order.
All common arguments can also be set in the project code or the run-file (recommended).
In that case, only the run-file is specified on the command line.
However, there are a number of options that override settings from the run-file.
The arguments can appear in arbitrary order.
The following table is ordered by importance.
| Option | Values | Description |
| --- | --- | --- |
| -h , --help | | Display a command line summary and exit. |
| -m , --mode | single (default), grid, swarm, genetic | Operation mode. |
| -d, --data-dir | file system path | Directory path for experimental data files (if required by project). Default: current working directory. |
| -o, --output-file | file system path | Base path and/or name for intermediate and output files. Default: pmsco_data |
| -t, --time-limit | decimal number | Wall time limit in hours. The optimizers try to finish before the limit. Default: 24.0. |
| -k, --keep-files | list of file categories | Output file categories to keep after the calculation. Multiple values can be specified and must be separated by spaces. By default, cluster and model (simulated data) of a limited number of best models are kept. See @ref sec_file_categories below. |
| --log-level | DEBUG, INFO, WARNING (default), ERROR, CRITICAL | Minimum level of messages that should be added to the log. |
| --log-file | file system path | Name of the main log file. Under MPI, the rank of the process is inserted before the extension. Default: output-file + log, or pmsco.log. |
| --log-disable | | Disable logging. By default, logging is on. |
| --pop-size | integer | Population size (number of particles) in swarm optimization mode. The default value is the greater of 4 or two times the number of calculation processes. |
| --seed-file | file system path | Name of the population seed file. Population data of previous optimizations can be used to seed a new optimization. The file must have the same structure as the .pop or .dat files. See @ref pmsco.project.Project.seed_file. |
| --table-file | file system path | Name of the model table file in table scan mode. |
| -c, --code | edac (default) | Scattering code. At the moment, only edac is supported. |
| Option | Values | Description | Run File |
| --- | --- | --- | --- |
| -r, --run-file | file path | JSON-formatted configuration file that defines run-time parameters. The format and content of a run file is described in a section @ref pag_runfile. | no |
| -o, --output-dir | file path | Base path and/or name for intermediate and output files. | see note below |
| -j , --job-name | string | Job name | job-name |
| -m, --module | file path | Project module | __module__ |
| -c, --project-class | string | Project class | __class__ |
| -h, --help | | Display a command line summary and exit. | no |
The job name is used as a prefix of output file names.
It is also registered in the `jobs` table of the results database (if used),
and it is used to identify the job with a job scheduling system.
\subsubsection sec_file_categories File Categories
\note It is important that the job name be unique within a project.
Specifically, you need to *provide a new job name each time you start pmsco*, otherwise the job may fail.
It may be more natural to specify the job name on the command line using the `-j` argument
than to change the run file every time.
Unfortunately, PMSCO cannot auto-generate, auto-increment or verify the job name.
The following category names can be used with the @c --keep-files option.
Multiple names can be specified and must be separated by spaces.
| Category | Description | Default Action |
| --- | --- | --- |
| all | shortcut to include all categories | |
| input | raw input files for calculator, including cluster and phase files in custom format | delete |
| output | raw output files from calculator | delete |
| phase | phase files in portable format for report | delete |
| cluster | cluster files in portable XYZ format for report | keep |
| debug | debug files | delete |
| model | output files in ETPAI format: complete simulation (a_-1_-1_-1_-1) | keep |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | keep |
| symmetry | output files in ETPAI format: symmetry (a_b_c_-1_-1) | delete |
| emitter | output files in ETPAI format: emitter (a_b_c_d_-1) | delete |
| region | output files in ETPAI format: region (a_b_c_d_e) | delete |
| report| final report of results | keep |
| population | final state of particle population | keep |
| rfac | files related to models which give bad r-factors | delete |
\subsection sec_project_args Project Arguments
The following table lists a few recommended options that are handled by the project code.
Project options that are not listed here should use the long form to avoid conflicts in future versions.
| Option | Values | Description |
| --- | --- | --- |
| -s, --scans | project-dependent | Nick names of scans to use in calculation. The nick name selects the experimental data file and the initial state of the photoelectron. Multiple values can be specified and must be separated by spaces. |
\subsection sec_scanfile Experimental Scan Files
The recommended way of specifying experimental scan files is using nick names (dictionary keys) and the @c --scans option.
A dictionary in the module code defines the corresponding file name, chemical species of the emitter and initial state of the photoelectron.
The location of the files is selected using the common @c --data-dir option.
This way, the file names and photoelectron parameters are versioned with the code,
whereas command line arguments may easily get forgotten in the records.
\subsection sec_project_example Argument Handling
To handle command line arguments in a project module,
the module must define a <code>parse_project_args</code> and a <code>set_project_args</code> function.
An example can be found in the twoatom.py demo project.
\section sec_slurm Slurm Job Submission
The command line of the Slurm job submission script for the Ra cluster at PSI is as follows.
This script is specific to the configuration of the Ra cluster but may be adapted to other Slurm-based queues.
@code{.sh}
qpmsco.sh [NOSUB] DESTDIR JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]
@endcode
Here, the first few arguments are positional and their order must be strictly adhered to.
After the positional arguments, optional arguments of the PMSCO project command line can be added in arbitrary order.
If you execute the script without arguments, it displays a short summary.
The job script is written to @c $DESTDIR/$JOBNAME which is also the destination of calculation output.
| Argument | Values | Description |
| --- | --- | --- |
| NOSUB (optional) | NOSUB or omitted | If NOSUB is present as the first argument, create the job script but do not submit it to the queue. Otherwise, submit the job script. |
| DESTDIR | file system path | destination directory. must exist. a sub-dir $JOBNAME is created. |
| JOBNAME | text | Name of job. Use only alphanumeric characters, no spaces. |
| NODES | integer | Number of computing nodes. (1 node = 24 or 32 processors). Do not specify more than 2. |
| TASKS_PER_NODE | 1...24, or 32 | Number of processes per node. 24 or 32 for full-node allocation. 1...23 for shared node allocation. |
| WALLTIME:HOURS | integer | Requested wall time. 1...24 for day partition, 24...192 for week partition, 1...192 for shared partition. This value is also passed on to PMSCO as the @c --time-limit argument. |
| PROJECT | file system path | Python module (file path) that declares the project and starts the calculation. |
| MODE | single, swarm, grid, genetic | PMSCO operation mode. This value is passed on to PMSCO as the @c --mode argument. |
| ARGS (optional) | | Any further arguments are passed on verbatim to PMSCO. You don't need to specify the mode and time limit here. |
*/
*/

114
docs/src/concepts-atomscat.dox Normal file
View File

@@ -0,0 +1,114 @@
/*! @page pag_concepts_atomscat Atomic scattering
\section sec_atomscat Atomic scattering
\subsection sec_atomscat_intro Introduction
The process of calculating atomic scattering factors (phase shifts) can be customized in several ways.
1. Internal processing.
Some multiple scattering programs, like EDAC, contain a built-in facility to calculate phase shifts.
This is the most simple and default behaviour.
2. Automatic calculation in a separate program.
PMSCO has an interface to run the PHAGEN program from
the [MsSpec-1.0 package](https://ipr.univ-rennes1.fr/msspec) to calculate scattering factors.
Note that the PHAGEN code is not included in the public distribution of PMSCO.
3. Manual calculation.
Scattering files created manually using an external program can be used by providing the file names.
The files must have the format required by the multiple scattering code,
and they must be linked to the corresponding atoms of the cluster.
In the case of automatic calculation, the project code can optionally hook into the process
and modify clusters before and after scattering factors are calculated.
For instance, it may provide an extended cluster in order to reduce boundary effects,
or it may modify the assignment of scattering files to cluster atoms
so that the scattering factors of selected atom classes are used
(cf. section \ref sec_atomscat_atomclass).
\subsection sec_atomscat_usage Usage
\subsubsection sec_atomscat_internal Internal processing
This is the default behaviour selected in the inherited pmsco.project.Project class.
Make sure not to override the `atomic_scattering_factory` attribute.
Its default value is pmsco.calculators.calculator.InternalAtomicCalculator.
\subsubsection sec_atomscat_external Automatic calculation in a separate program
To select the atomic scattering calculator,
assign its interface class to the project's `atomic_scattering_factory` attribute.
For example, to use PHAGEN, add the following code to your project's `__init__` constructor:
@code{.py}
from pmsco.calculators.phagen import PhagenCalculator
self.atomic_scattering_factory = PhagenCalculator
@endcode
\subsubsection sec_atomscat_manual Manual calculation
If you want to keep the scattering factors constant during an optimization,
you should run PMSCO in _single_ mode and provide the model parameters and cluster
that will return the desired scattering files.
In the `create_params` method of your project,
you should then set the `phase_files` attribute,
which is a dictionary that maps atom classes to the names of the scattering files.
Unless you set specific values in the cluster object, the atom class defaults to the element number.
The file names should include a path relative to the working directory.
\subsection sec_atomscat_implement Implementation
\subsubsection sec_atomscat_atomclass Atom classes
Atomic scattering programs classify atoms based on chemical element, charge state and symmetry of the local environment.
This means that two atoms of the same chemical element may have different scattering factors.
For example, if you have EDAC output the cluster after calculation of the muffin tin potential,
you will find that the chemical element number has been replaced by an arbitrary integer.
By default, PMSCO will do the linking of atom classes and scattering files transparently.
However, if you want to reduce the number of atom classes,
or if you have the scattering factors calculated on a reference cluster,
you will have to provide project code to do the assignment.
This is described further below.
\subsubsection sec_atomscat_calculator Atomic scattering calculator
The project selects the atomic scattering calculation mode by specifying its `atomic_scattering_factory` attributed.
This is the name of a class that inherits from @ref pmsco.calculators.calculator.AtomicCalculator.
The following calculators are currently implemented:
| Class | Description |
| --- | --- |
| pmsco.calculators.calculator.InternalAtomicCalculator | Calculate the atomic scattering factors in the multiple-scattering program. |
| pmsco.calculators.phagen.PhagenCalculator | Calculate the atomic scattering factors in the PHAGEN program. |
An atomic calculator class essentially defines a `run` method that operates on a cluster and scattering parameters object.
It generates the necessary scattering files, updates the cluster with the new atom classes
and updates the parameters with the file names of the scattering files.
Note that the scattering files have to be in the correct format for the multiple scattering calculator.
\subsubsection sec_atomscat_hooks Project hooks
Before and after calculation of the scattering factors,
the project's `before_atomic_scattering` and `after_atomic_scattering` methods are called
with the cluster and input parameters.
The _before_ method provides the cluster to be used for atomic scattering calculations.
It may,
1. just return the original cluster,
2. modify the provided cluster to include additional atoms or modify the charge state of the emitter,
3. create a completely different cluster,
4. return None to suppress the atomic scattering calculation.
The method is called once at the beginning of the PMSCO job with model -1,
where it may return the global reference cluster.
Later on it is called once for each calculation task with the specific task index.
Similarly, the _after_ method collects the results and updates the `phase_files` dictionary of the input parameters.
It is free to consolidate atom classes and remove unwanted atoms.
However, it must make sure that for each atom class in the cluster,
there is a corresponding link to a scattering file.
*/

4
docs/src/concepts-emitter.dox
View File

@@ -105,12 +105,12 @@ is assigned to the project's cluster_generator attribute.
1. Implement a count_emitters method in your project class
if the project uses more than one emitter configurations.
It must have same method contract as pmsco.cluster.ClusterGenerator.count_emitters.
Specifically, it must return the number of emitter configurations of a given model, scan and symmetry.
Specifically, it must return the number of emitter configurations of a given model, scan and domain.
If there is only one configuration, the method does not need to be implemented.
2. Implement a create_cluster method in your project class.
It must have same method contract as pmsco.cluster.ClusterGenerator.create_cluster.
Specifically, it must return a cluster.Cluster object for the given model, scan, symmetry and emitter configuration.
Specifically, it must return a cluster.Cluster object for the given model, scan, domain and emitter configuration.
The emitter atoms must be marked according to the emitter configuration specified by the index argument.
Note that, depending on the index.emit argument, all emitter atoms must be marked
or only the ones of the corresponding emitter configuration.

34
docs/src/concepts-symmetry.dox
View File

@@ -1,32 +1,32 @@
/*! @page pag_concepts_symmetry Symmetry
/*! @page pag_concepts_domain Domain
\section sec_symmetry Symmetry and Domain Averaging
\section sec_domain Domain Averaging
A _symmetry_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
A _domain_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is derived from the same set of model parameters
and that contributes incoherently to the measured diffraction pattern.
A symmetry may be represented by a special symmetry parameter which is not subject to optimization.
A domain may be represented by special domain parameters that are not subject to optimization.
For instance, a real sample may have additional rotational domains that are not present in the cluster,
increasing the symmetry from three-fold to six-fold.
For instance, a real sample may have rotational domains that are not present in the cluster,
changing the symmetry from three-fold to six-fold.
Or, an adsorbate may be present in a number of different lateral configurations on the substrate.
In the first case, it may be sufficient to fold calculated data in the proper way to generate the same symmetry as in the measurement.
In the latter case, it may be necessary to execute a scattering calculation for each possible orientation or a representative number of possible orientations.
PMSCO provides the basic framework to spawn multiple calculations according to the number of symmetries (cf. \ref sec_tasks).
The actual data reduction from multiple symmetries to one measurement needs to be implemented on the project level.
PMSCO provides the basic framework to spawn multiple calculations according to the number of domains (cf. \ref sec_tasks).
The actual data reduction from multiple domain to one measurement needs to be implemented on the project level.
This section explains the necessary steps.
1. Your project needs to populate the pmsco.project.Project.symmetries list.
For each symmetry, add a dictionary of symmetry parameters, e.g. <code>{'angle_azi': 15.0}</code>.
There must be at least one symmetry in a project, otherwise no calculation is executed.
1. Your project needs to populate the pmsco.project.Project.domains list.
For each domain, add a dictionary of domain parameters, e.g. <code>{'angle_azi': 15.0}</code>.
At least one domain must be declared in a project, otherwise no calculation is executed.
2. The project may apply the symmetry of a task to the cluster and parameter file if necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular symmetry in addition to the model parameters.
3. The project combines the results of the calculations for the various symmetries into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with equal weight.
If you need more control, you need to override the pmsco.project.Project.combine_symmetries method and implement your own algorithm.
2. The project may use the domain index of a task to build the cluster and parameter file as necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular domain in addition to the model parameters.
3. The project combines the results of the calculations for the various domains into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with customizable weight.
It uses the special model parameters `wdom1`, `wdom2`, ... (if defined, default 1) to weight each domain.
If you need more control, override the pmsco.project.Project.combine_domains method and implement your own algorithm.
*/

60
docs/src/concepts-tasks.dox
View File

@@ -12,7 +12,7 @@ mandated by the project but also efficient calculations in a multi-process envir
A concrete set of parameters is called @ref sec_task_model.
2. The sample was measured multiple times or under different conditions (initial states, photon energy, emission angle).
Each contiguous measured dataset is called a @ref sec_task_scan.
3. The measurement averages over multiple inequivalent domains, cf. @ref sec_task_symmetry.
3. The measurement averages over multiple inequivalent domains, cf. @ref sec_task_domain.
4. The measurement includes multiple geometrically inequivalent emitters, cf. @ref sec_task_emitter.
5. The calculation should be distributed over multiple processes that run in parallel to reduce the wall time, cf. @ref sec_task_region.
@@ -24,7 +24,7 @@ as shown schematically in the following diagram.
class CalculationTask {
model
scan
symmetry
domain
emitter
region
..
@@ -55,7 +55,7 @@ class Scan {
alphas
}
class Symmetry {
class Domain {
index
..
rotation
@@ -75,13 +75,13 @@ class Region {
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Symmetry
CalculationTask *-- Domain
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
symmetries
domains
model_handler
cluster_generator
}
@@ -98,7 +98,7 @@ class ModelHandler {
Model ..> ModelHandler
Scan ..> Project
Symmetry ..> Project
Domain ..> Project
Emitter ..> ClusterGenerator
Region ..> Project
@@ -141,29 +141,29 @@ PMSCO runs a separate calculation for each scan file and compares the combined r
This is sometimes called a _global fit_.
\subsection sec_task_symmetry Symmetry
\subsection sec_task_domain Domain
A _symmetry_ is a discrete variant of a set of calculation parameters (including the atomic cluster)
A _domain_ is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is independent of the _model_ and contributes incoherently to the measured diffraction pattern.
For instance, for a system that includes two inequivalent structural domains,
two separate clusters have to be generated and calculated for each model.
The symmetry parameter is not subject to optimization.
The domain parameter is not subject to optimization.
However, if the branching ratio is unknown a priori, a model parameter can be introduced
to control the relative contribution of a particular symmetry to the diffraction pattern.
In that case, the @ref pmsco.project.Project.combine_symmetries method must be overridden.
to control the relative contribution of a particular domain to the diffraction pattern.
The basic @ref pmsco.project.Project.combine_domains method reads the special model parameters `wdom1`, `wdom2`, etc. to weight the individual domains.
A symmetry is identified by its index which is an index into the project's symmetries table (pmsco.project.Project.symmetries).
It is up to the user project to give a physical description of the symmetry, e.g. a rotation angle,
by assigning a meaningful value (e.g. a dictionary with key-value pairs) to the symmetries table.
A domain is identified by its index which is an index into the project's domains table (pmsco.project.Project.domains).
It is up to the user project to give a physical description of the domain, e.g. a rotation angle,
by assigning a meaningful value (e.g. a dictionary with key-value pairs) to the domains table.
The cluster generator can then read the value from the table rather than from constants in the code.
The figure shows two examples of symmetry parameters.
The corresponding symmetry table could be set up like this:
The figure shows two examples of domain parameters.
The corresponding domains table could be set up like this:
@code{.py}
project.add_symmetry = {'rotation': 0.0, 'registry': 0.0}
project.add_symmetry = {'rotation': 30.0, 'registry': 0.0}
project.add_domain({'rotation': 0.0, 'registry': 0.0})
project.add_domain({'rotation': 30.0, 'registry': 0.0})
@endcode
@@ -173,9 +173,9 @@ The _emitter_ component of the calculation task selects a specific emitter confi
This is merely an index whose interpretation is up to the cluster generator.
The default emitter handler enumerates the emitter index from 1 to the emitter count reported by the cluster generator.
The emitter count and list of emitters may depend on model, scan and symmetry.
The emitter count and list of emitters may depend on model, scan and domain.
The cluster generator can tailor a cluster to the given model, scan, symmetry and emitter index.
The cluster generator can tailor a cluster to the given model, scan, domain and emitter index.
For example, in a large unit cell with many inequivalent emitters,
the generator might return a small sub-cluster around the actual emitter for better calculation performance
since the distant atoms of the unit cell do not contribute to the diffraction pattern.
@@ -237,20 +237,20 @@ scan
object ScanHandler
object "Sym: CalculationTask" as Sym {
object "Domain: CalculationTask" as Domain {
index = (i,j,k,-1,-1)
model
scan
symmetry
domain
}
object "SymmetryHandler" as SymHandler
object "DomainHandler" as DomainHandler
object "Emitter: CalculationTask" as Emitter {
index = (i,j,k,l,-1)
model
scan
symmetry
domain
emitter
}
@@ -260,7 +260,7 @@ object "Region: CalculationTask" as Region {
index = (i,j,k,l,m)
model
scan
symmetry
domain
emitter
region
}
@@ -270,14 +270,14 @@ object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Sym
Sym "1" o.. "1..*" Emitter
Scan "1" o.. "1..*" Domain
Domain "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Sym) .. SymHandler
(Sym, Emitter) .. EmitterHandler
(Scan, Domain) .. DomainHandler
(Domain, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml
@@ -293,7 +293,7 @@ and the tasks are passed back through the task handler stack.
In this phase, each level joins the datasets from the sub-tasks to the data requested by the parent task.
For example, at the lowest level, one result file is present for each region.
The region handler gathers all files that correspond to the same parent task
(i.e. have the same emitter, symmetry, scan and model attributes),
(i.e. have the same emitter, domain, scan and model attributes),
joins them to one file which includes all regions,
links the file to the parent task and passes the result to the next higher level.

20
docs/src/concepts.dox
View File

@@ -8,28 +8,30 @@ The code for a PMSCO job consists of the following components.
skinparam componentStyle uml2
component "project" as project
component "PMSCO" as pmsco
component "project" as project
component "scattering code\n(calculator)" as calculator
interface "command line" as cli
interface "input files" as input
interface "output files" as output
interface "experimental data" as data
interface "results" as results
interface "output files" as output
cli --> pmsco
data -> project
project ..> pmsco
pmsco ..> project
pmsco ..> calculator
cli --> project
input -> calculator
calculator -> output
pmsco -> results
@enduml
The main entry point is the _PMSCO_ module.
It implements a task loop to carry out the structural optimization
and provides an interface between calculation programs and project-specific code.
It also provides common utility classes and functions for the handling project data.
The _project_ consists of program code, system and experimental parameters
The _project_ consists of program code and parameters
that are specific to a particular experiment and calculation job.
The project code reads experimental data, defines the parameter dictionary of the model,
and contains code to generate the cluster, parameter and phase files for the scattering code.
@@ -40,10 +42,6 @@ which accepts detailed input files
(parameters, atomic coordinates, emitter specification, scattering phases)
and outputs an intensity distribution of photoelectrons versus energy and/or angle.
The _PMSCO core_ interfaces between the project and the calculator.
It carries out the structural optimization and manages the calculation tasks.
It generates and sends input files to the calculator and reads back the output.
\section sec_control_flow Control flow

22
docs/src/dataflow.dot
View File

@@ -10,7 +10,7 @@ digraph G {
create_params;
calc_modf;
calc_rfac;
comb_syms;
comb_doms;
comb_scans;
}
*/
@@ -24,11 +24,11 @@ digraph G {
model_handler -> model_creator [constraint=false, label="optimize"];
}
subgraph cluster_symmetry {
label = "symmetry handler";
subgraph cluster_domain {
label = "domain handler";
rank = same;
sym_creator [label="expand models", group=creators];
sym_handler [label="combine symmetries", group=handlers];
dom_creator [label="expand models", group=creators];
dom_handler [label="combine domains", group=handlers];
}
subgraph cluster_scan {
@@ -47,15 +47,15 @@ digraph G {
calculator [label="calculator (EDAC)", shape=box];
model_creator -> sym_creator [label="model", style=bold];
sym_creator -> scan_creator [label="models", style=bold];
model_creator -> dom_creator [label="model", style=bold];
dom_creator -> scan_creator [label="models", style=bold];
scan_creator -> calc_creator [label="models", style=bold];
calc_creator -> calculator [label="clusters,\rparameters", style=bold];
calculator -> calc_handler [label="output files", style=bold];
calc_handler -> scan_handler [label="raw data files", style=bold];
scan_handler -> sym_handler [label="combined scans", style=bold];
sym_handler -> model_handler [label="combined symmetries", style=bold];
scan_handler -> dom_handler [label="combined scans", style=bold];
dom_handler -> model_handler [label="combined domains", style=bold];
mode [shape=parallelogram];
mode -> model_creator [lhead="cluster_model"];
@@ -76,8 +76,8 @@ digraph G {
calc_rfac [shape=cds, label="R-factor function"];
calc_rfac -> model_handler [style=dashed];
comb_syms [shape=cds, label="symmetry combination rule"];
comb_syms -> sym_handler [style=dashed];
comb_doms [shape=cds, label="domain combination rule"];
comb_doms -> dom_handler [style=dashed];
comb_scans [shape=cds, label="scan combination rule"];
comb_scans -> scan_handler [style=dashed];

142
docs/src/execution.dox
View File

@@ -2,52 +2,44 @@
\section sec_run Running PMSCO
To run PMSCO you need the PMSCO code and its dependencies (cf. @ref pag_install),
a code module that contains the project-specific code,
and one or several files containing the scan parameters and experimental data.
a customized code module that contains the project-specific code,
one or several files containing the scan parameters and experimental data,
and a run-file specifying the calculation parameters.
Please check the <code>projects</code> folder for examples of project modules.
For a detailed description of the command line, see @ref pag_command.
\subsection sec_run_single Single Process
Run PMSCO from the command prompt:
The following instructions assume that PMSCO was installed as a Python site-package according to @ref pag_install.
To run PMSCO from the command prompt:
@code{.sh}
cd work-dir
python pmsco-dir project-dir/project.py [pmsco-arguments] [project-arguments]
python -m pmsco -j job-name -r run-file
@endcode
where <code>work-dir</code> is the destination directory for output files,
<code>pmsco-dir</code> is the directory containing the <code>__main__.py</code> file,
<code>project.py</code> is the specific project module,
and <code>project-dir</code> is the directory where the project file is located.
PMSCO is run in one process which handles all calculations sequentially.
where:
The command line arguments are divided into common arguments interpreted by the main pmsco code (pmsco.py),
and project-specific arguments interpreted by the project module.
| `work-dir` | Destination directory for output files |
| `run-file` | JSON-formatted configuration file that defines run-time parameters. The format and content of a run file is described in a section @ref pag_runfile. |
| `job-name` | (optional) The job name appears mainly as the prefix of all output files but is also used in the database and other places. The job name can also be declared in the run file. |
In this form, PMSCO is run in one process which handles all calculations sequentially.
Example command line for a single EDAC calculation of the two-atom project:
@code{.sh}
cd work/twoatom
python ../../pmsco ../../projects/twoatom/twoatom.py -s ea -o twoatom-demo -m single
python -m pmsco -j job0001 -r twoatom-hemi.json
@endcode
This command line executes the main pmsco module <code>pmsco.py</code>.
The main module loads the project file <code>twoatom.py</code> as a plug-in
and starts processing the common arguments.
The <code>twoatom.py</code> module contains only project-specific code
with several defined entry-points called from the main module.
The information which project to load is contained in the <code>twoatom-hemi.json</code> file,
along with all common and specific project arguments.
In the command line above, the <code>-o twoatom-demo</code> and <code>-m single</code> arguments
are interpreted by the pmsco module.
<code>-o</code> sets the base name of output files,
and <code>-m</code> selects the operation mode to a single calculation.
The scan argument is interpreted by the project module.
It refers to a dictionary entry that declares the scan file, the emitting atomic species, and the initial state.
In this example, the project looks for the <code>twoatom_energy_alpha.etpai</code> scan file in the project directory,
and calculates the modulation function for a N 1s initial state.
The kinetic energy and emission angles are contained in the scan file.
This example can be run for testing.
All necessary parameters and data files are included in the code repository.
\subsection sec_run_parallel Parallel Processes
@@ -61,30 +53,104 @@ The slave processes will run the scattering calculations, while the master coord
and optimizes the model parameters (depending on the operation mode).
For optimum performance, the number of processes should not exceed the number of available processors.
To start a two-hour optimization job with multiple processes on an quad-core workstation with hyperthreading:
To start an optimization job with multiple processes on a quad-core workstation with hyperthreading:
@code{.sh}
cd work/my_project
mpiexec -np 8 pmsco-dir/pmsco project-dir/project.py -o my_job_0001 -t 2 -m swarm
mpiexec -np 8 --use-hwthread-cpus python -m pmsco -j my_job002 -r my_project.json
@endcode
The `--use-hwthread` option is necessary on certain hyperthreading architectures.
\subsection sec_run_hpc High-Performance Cluster
The script @c bin/qpmsco.ra.sh takes care of submitting a PMSCO job to the slurm queue of the Ra cluster at PSI.
The script can be adapted to other machines running the slurm resource manager.
The script generates a job script based on @c pmsco.ra.template,
substituting the necessary environment and parameters,
and submits it to the queue.
PMSCO is ready to run with resource managers on cluster machines.
Code for submitting jobs to Slurm queues is included and can be customized for many machines.
For example, code for the slurm queue of the Ra cluster at PSI is included in the pmsco.schedule module.
Other machines can be supported by sub-classing pmsco.schedule.JobSchedule or pmsco.schedule.SlurmSchedule.
Execute @c bin/qpmsco.ra.sh without arguments to see a summary of the arguments.
To have PMSCO submit a job, the arguments for the queue are entered in the schedule section of the run file,
cf. @ref pag_runfile.
Then, the same command as for starting a calculation directly will instead submit a job to the queue:
To submit a job to the PSI clusters (see also the PEARL-Wiki page MscCalcRa),
the analog command to the previous section would be:
@code{.sh}
bin/qpmsco.ra.sh my_job_0001 1 8 2 projects/my_project/project.py swarm
python -m pmsco -j job-name -r run-file.json
@endcode
The command creates a separate work directory with copies of the project source, the run-file and the job script.
This job directory will also receive the calculation results.
The full path of the job directory is _output-dir/job-name_.
The directory must not exist when you run the above command to prevent overwriting of previous data.
The job name can be declared in the run file or on the command line.
The command above also loads the project module and scan files.
Many parameter errors are caught this way and can be fixed before the job is submitted to the queue.
The run file offers an option to prepare a script file and not to submit the job immediately
so that you can inspect the job files and submit the job manually.
Be sure to consider the resource allocation policy of the cluster
before you decide on the number of processes.
Requesting less resources will prolong the run time but might increase the scheduling priority.
\subsection sec_run_dirs Directories
Code and data files are typically located in different, possibly machine-specific locations.
This can make it difficult to port a project to another machine and to repeat calculations.
Ideally, a calculation job should be repeatable on different machines
with a minimum of changes to code, input data and parameter files.
Project code (which is under version control)
should never need modifications for porting to another machine.
Run-files (which are considered part of the data) can follow a project-specific or machine-specific directory structure.
PMSCO provides directory resolution at run-time to facilitate writing of portable code.
This is done by a number of directory aliases that can be included as shell-like placeholders, e.g. `${project}`, in file paths.
Some aliases are preset to system-based defaults,
further aliases can be added by the project code or declared in the run file.
Directory aliases can be used in Project.directories
as well as in other Project attributes that hold a file name.
The table below shows the aliases defined and/or required by PMSCO.
The paths are stored in Project.directories.
The aliases are resolved before the actual calculations start (in the Project.validate() method).
The resolved paths are printed to the log at warning level.
| Key | Description | Source | Use |
| --- | --- | --- | --- |
| work | Working directory at program start | PMSCO | |
| home | User's home directory | PMSCO | |
| project | Location of the project module. | PMSCO | Can be used to find auxiliary files that are part of the repository. |
| output | Intermediate and output files. | Must be set by the project or run file | The `output_file` property which serves as the basis of all output files is a concatenation of the `output` directory and `job_name`. |
| report | Directory for graphical output (reports) | Default: `${output}/report` | |
| data (optional) | Location of data (scan) files. | Project or run file | Usage is up to the project. |
| temp | Temporary files | | Reserved. Currently not supported |
| (job tag) | Any job_tags key that maps to a legal directory name can be included in a path | run file | project or run file |
| mode, job_name, project_name | These project attributes can be included in a path if they contain a valid directory name | | |
\subsection sec_run_stop Stopping a PMSCO job
A PMSCO optimization job stops on any one of the following events.
- The model handler is done.
Depending on the run mode, this happens when the optimization has converged or
the planned number of iterations or calculations has been reached.
- The number of calculation tasks exceeds the limit configured in `dispatch.MscoMaster.max_calculations`.
This is meant to prevent excessive and runaway jobs.
The default value is 1000000. It can be adjusted by the project code if necessary.
- The master process receives a SIGTERM, SIGUSR1 or SIGUSR2 from the operating system.
The signal can be sent, e.g., by the `kill` command on Linux.
This doesn´t work on all platforms.
- The time limit configured in `Project.timedelta_limit` is reached.
This is a soft limit and should be set shorter than the job reservation with the resource manager.
- A file named `finish_pmsco` is present in the output directory.
This is an easy way for a user to stop a running optimization.
The file doesn´t need any content.
It can be created by the `touch` command.
All these stop conditions cause graceful stops.
Running calculation tasks are waited for, but some results on the model level may not be complete.
Final reports of complete models are produced and the output folder is cleaned up.
Stops caused by resource managers such as Slurm are typically not graceful.
The results are in an undefined state, reports are not generated, and temporary files may be left over.
*/

372
docs/src/installation.dox
View File

@@ -3,66 +3,66 @@
\subsection sec_general General Remarks
The PMSCO code is maintained under [Git](https://git-scm.com/).
The central repository for PSI-internal projects is at https://git.psi.ch/pearl/pmsco,
the public repository at https://gitlab.psi.ch/pearl/pmsco.
The central repository for development and PSI-internal projects is at https://gitea.psi.ch/pearl/pmsco,
the public repository at https://gitea.psi.ch/pearl-public/pmsco.
For their own developments, users should clone the repository.
Changes to common code should be submitted via pull requests.
Scientific projects should be maintained in a separate directory tree, cf. @ref sec_project.
The program code of PMSCO and its external programs is written in Python, C++ and Fortran.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a workstation with at least 4 processor cores
For optimization jobs with parallel execution, a workstation with at least 4 processor cores
or cluster with 20-50 available processor cores is recommended.
The program requires about 2 GB of RAM per process.
The recommended IDE is [PyCharm (community edition)](https://www.jetbrains.com/pycharm).
The documentation in [Doxygen](http://www.stack.nl/~dimitri/doxygen/index.html) format is part of the source code.
The Doxygen compiler can generate separate documentation in HTML or LaTeX.
[Spyder](https://docs.spyder-ide.org/index.html) is a good alternative with a focus on scientific data.
The documentation in [Doxygen](https://www.doxygen.nl/index.html) format is part of the source code.
The Doxygen compiler can generate documentation in HTML.
@attention Due to rapidly evolving computing environments
some of the installation instructions on this page may be outdated or incompatible with certain environments.
\subsection sec_requirements Requirements
Please note that in some environments (particularly shared high-performance machines)
it may be important to choose specific compiler and library versions.
In order to maintain backward compatibility with some of these older machines,
it may be important to choose specific compiler and library versions that are tailored to the hardware platform.
In order to maintain backward compatibility with older installations,
code that requires new versions of compilers and libraries should be introduced carefully.
The code depends on the following libraries:
The following basic tools and libraries are required:
- GCC >= 4.8
- OpenMPI >= 1.10
- F2PY
- F2C
- SWIG
- GCC (C, C++, Fortran) >= 4.8
- BLAS
- LAPACK
- Python 2.7 or 3.6
- Numpy >= 1.11
- Python packages from PyPI listed in the requirements.txt file
- OpenMPI >= 1.10
- Git
Most of these requirements are available from the Linux distribution.
For an easily maintainable Python environment, [Miniconda](https://conda.io/miniconda.html) is recommended.
The Python environment distributed with the OS often contains outdated packages,
and it's difficult to switch between different Python versions.
For the Python environment,
the [uv](https://docs.astral.sh/uv/) package and environment manager is recommended.
It can be installed by non-privileged users.
Other package managers like pip and conda may work as well but are not described here.
On the PSI cluster machines, the environment must be set using the module system and conda (on Ra).
Details are explained in the PEARL Wiki.
PMSCO runs under Python 2.7 or Python 3.6 or higher.
Since Python 2 is being deprecated, the code has been ported to Python 3.6.
Compatibility with Python 2.7 is maintained by using
the [future package](http://python-future.org/compatible_idioms.html).
New code should be written according to their guidelines.
The following tools are required to compile the documentation.
They are not needed in calculations.
- doxygen
- doxypypy
- graphviz
- Java runtime environment (JRE)
- [plantUML](https://plantuml.com)
\subsection sec_install_instructions Instructions
Installation instructions are given for Ubuntu 24.04.
On managed HPC clusters use the compilers and libraries recommended by the administrator
(often provided by a module system).
\subsubsection sec_install_ubuntu Installation on Ubuntu
The following instructions install the necessary dependencies on Ubuntu, Debian or related distributions.
The Python environment is provided by [Miniconda](https://conda.io/miniconda.html).
@code{.sh}
sudo apt update
@@ -71,8 +71,6 @@ sudo apt install \
binutils \
build-essential \
doxygen \
doxypy \
f2c \
g++ \
gcc \
gfortran \
@@ -81,98 +79,17 @@ graphviz \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
python3 \
python3-venv \
sqlite3 \
swig \
wget
@endcode
On systems where the link to libblas is missing (see @ref sec_compile below),
the following lines are necessary.
@code{.sh}
cd /usr/lib
sudo ln -s /usr/lib/libblas/libblas.so.3 libblas.so
@endcode
Install Miniconda according to their [instructions](https://conda.io/docs/user-guide/install/index.html),
then configure the Python environment:
@code{.sh}
conda create -q --yes -n pmsco python=2.7
source activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
future \
statsmodels
pip install periodictable attrdict fasteners
@endcode
\subsubsection sec_install_singularity Installation in Singularity container
A [Singularity](https://www.sylabs.io/guides/2.5/user-guide/index.html) container
contains all OS and Python dependencies for running PMSCO.
Besides the Singularity executable, nothing else needs to be installed in the host system.
This may be the fastest way to get PMSCO running.
For installation of Singularity,
see their [user guide](https://www.sylabs.io/guides/2.5/user-guide/installation.html).
On newer Linux systems (e.g. Ubuntu 18.04), Singularity is available from the package manager.
Installation in a virtual machine on Windows or Mac are straightforward
thanks to the [Vagrant system](https://www.vagrantup.com/).
After installing Singularity,
check out PMSCO as explained in the @ref sec_compile section:
@code{.sh}
cd ~
mkdir containers
git clone git@git.psi.ch:pearl/pmsco.git pmsco
cd pmsco
git checkout master
git checkout -b my_branch
@endcode
Then, either copy a pre-built container into `~/containers`,
or build one from a script provided by the PMSCO repository:
@code{.sh}
cd ~/containers
sudo singularity build pmsco.simg ~/containers/pmsco/extras/singularity/singularity_python2
@endcode
To work with PMSCO, start an interactive shell in the container and switch to the pmsco environment.
Note that the PMSCO code is outside the container and can be edited with the usual tools.
@code{.sh}
cd ~/containers
singularity shell pmsco.simg
source activate pmsco
cd ~/containers/pmsco
make all
nosetests -w tests/
@endcode
Or call PMSCO from outside:
@code{.sh}
cd ~/containers
mkdir output
cd output
singularity run ../pmsco.simg python ~/containers/pmsco/pmsco path/to/your-project.py arg1 arg2 ...
@endcode
For parallel processing, prepend `mpirun -np X` to the singularity command as needed.
In addition, download and install [uv](https://docs.astral.sh/uv/).
PSI users should configure uv to use PSI's PyPI package cache (cf. documentation on the intranet).
\subsubsection sec_install_extra Additional Applications
@@ -188,73 +105,212 @@ gitg \
meld
@endcode
To produce documentation in PDF format (not recommended on virtual machine), install LaTeX:
To compile the documentation install the following tools.
The basic documentation is in HTML format and can be opened in any internet browser.
@code{.sh}
sudo apt-get install texlive-latex-recommended
sudo apt install \
doxygen \
graphviz \
default-jre
wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
sudo mkdir /opt/plantuml/
sudo mv plantuml.jar /opt/plantuml/
echo "export PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar" | sudo tee /etc/profile.d/pmsco-env.sh
@endcode
\subsection sec_compile Compilation
\subsection sec_distro Download PMSCO Source Code
Make sure you have access to the PMSCO Git repository and set up your Git environment.
Depending on your setup, location and permissions, one of the following addresses may work.
Private key authentication is usually recommended except on shared computers.
Clone or download the code from one of these repository addresses:
| Repository | Access |
| --- | --- |
| `git@git.psi.ch:pearl/pmsco.git` | PSI intranet, SSH private key authentication |
| `https://git.psi.ch/pearl/pmsco.git` | PSI intranet, password prompt |
| `git@gitlab.psi.ch:pearl/pmsco.git` | Public repository, SSH private key authentication |
| `https://gitlab.psi.ch/pearl/pmsco.git` | Public repository, password prompt |
Clone the code repository using one of these repositiory addresses and switch to the desired branch:
| https://gitea.psi.ch/pearl/pmsco | PSI internal |
| https://gitea.psi.ch/pearl-public/pmsco-public | Public |
@code{.sh}
git clone git@git.psi.ch:pearl/pmsco.git pmsco
cd ~
git clone {repo-address see above} pmsco
cd pmsco
git checkout master
@endcode
These instructions download the base package of PMSCO.
The public repository does not contain external programs (EDAC, PHAGEN, LOESS).
You need to obtain the source code for these programs from their respective owners,
copy them to the respective subprojects directories and
apply the patches included in the PMSCO distribution.
Please respect the respective license terms and acknowledge the use of the codes.
\subsection sec_install_environment Set up the Python Environment
The following instructions are for the [uv](https://docs.astral.sh/uv/) package manager.
For other package managers, the pyproject.toml and requirements.txt files list the necessary dependencies.
\subsubsection sec_install_uv Virtual Environment with uv
By default, uv creates the virtual environment automatically in a `.venv` folder inside the source directory tree.
In this case, no explicit setup is necessary, and pmsco can be called by:
~~~~~~{.sh}
uv run pmsco -h
~~~~~~
On some platforms, however, it may be necessary to separate the environment from the code,
e.g. because of limited storage space or quota in the home directory.
In this case, create the environment as follows:
~~~~~~{.sh}
cd ~
mkdir envs
cd envs
uv venv --clear my_pmsco_env
~~~~~~
The `--clear` option resets an existing environment to empty.
To activate this environment, call this command once in every terminal:
~~~~~~{.sh}
source ~/envs/my_pmsco_env/bin/activate
~~~~~~
\subsubsection sec_normal_install Installing PMSCO
to install PMSCO and all dependencies into the active environment,
run the following commands in the top-level PMSCO directory (where `pyproject.toml` is located).
The commands compile the Fortran and C++ code of the calculation programs using the
[Meson build system](https://mesonbuild.com/meson-python/index.html)
and install the binaries and Python code in the site-packages folder of the active Python environment.
~~~~~~{.sh}
uv sync --active
~~~~~~
To use the default `.venv` environment, omit the `--active` option (also in the uv commands shown further below).
Now, run the unit tests to check the installation:
~~~~~~{.sh}
uv run --active nosetests
~~~~~~
And check the help page:
~~~~~~{.sh}
uv run --active pmsco -h
~~~~~~
In the explicit environment, these commands can alternatively be called directly:
~~~~~~{.sh}
nosetests
pmsco -h
~~~~~~
The PMSCO packages are now accessible in Python import statements.
Verify it by opening a Python shell and entering:
~~~~~~{.py}
import pmsco.project
dir(pmsco.project)
~~~~~~
Note: By default, uv installs the Python code in editable mode.
Changes in the PMSCO source tree are visible as soon as you start a new Python interpreter.
This does not apply to the subpackages, however.
After modifying the subpackages, you need to clear and re-sync the environment.
\subsection sec_test Test project
Run the twoatom project to check that everything is installed correctly:
~~~~~~{.py}
cd ~
mkdir -p work/twoatom
cd work/twoatom
nice python -m pmsco -r {path-to-pmsco}/projects/twoatom/twoatom-hemi.json
~~~~~~
You should get a number of result files whose names start with `twoatom0001` in `~/work/twoatom/`,
including a hologram plot of the modulation function.
To learn more about running PMSCO, see @ref pag_run.
\subsection sec_install_projects Installing Namespace Packages
Instructions on how to set up your own projects as namespace packages are given in section \ref sec_project.
To install them into the pmsco namespace, call uv with the `--inexact` option.
Without `--inexact`, uv would remove the previously installed packages (including PMSCO).
~~~~~~{.sh}
uv sync --active --inexact
~~~~~~
\subsection sec_install_singularity Installation in a Singularity container
Singularity containers are currently unmaintained.
The PMSCO source includes an install script for the [Singularity](https://sylabs.io/singularity/) container system
under `extras/singularity`.
To get started with Singularity,
download it from [sylabs.io](https://www.sylabs.io/singularity/) and install it according to their instructions.
On Windows, Singularity can be installed in a virtual machine using the [Vagrant](https://www.vagrantup.com/)
script included under `extras/vagrant`.
After installing Singularity,
check out PMSCO as explained in the @ref sec_compile section:
@code{.sh}
cd ~
mkdir containers
cd containers
git clone git@gitea.psi.ch:pearl-public/pmsco-public.git pmsco
cd pmsco
git checkout master
git checkout -b my_branch
@endcode
Compile the code and run the unit tests to check that it worked.
Then, either copy a pre-built container into `~/containers`,
or build one from the definition file included under extras/singularity.
You may need to customize the definition file to match the host OS
or to install compatible OpenMPI libraries,
cf. [Singularity user guide](https://sylabs.io/guides/3.7/user-guide/mpi.html).
@code{.sh}
make all
nosetests -w tests/
cd ~/containers
sudo singularity build pmsco.sif ~/containers/pmsco/extras/singularity/singularity_python3
@endcode
If the compilation of _loess.so failes due to a missing BLAS library,
try to set a link to the BLAS library as follows (the actual file names may vary due to the actual distribution or version):
@code{.sh}
cd /usr/lib
sudo ln -s /usr/lib/libblas/libblas.so.3 libblas.so
@endcode
\subsection sec_test Tests
Run the unit tests.
They should pass successfully.
Re-check from time to time.
To work with PMSCO, start an interactive shell in the container and switch to the pmsco environment.
Note that the PMSCO code is outside the container and can be edited with the usual tools.
@code{.sh}
cd ~/pmsco
nosetests -w tests/
cd ~/containers
singularity shell pmsco.sif
. /opt/miniconda/etc/profile.d/conda.sh
conda activate pmsco
cd ~/containers/pmsco
meson setup build
meson compile -C build
meson install -C build
meson test -C build
@endcode
Run the twoatom project to check the compilation of the calculation programs.
Or call PMSCO from outside:
@code{.sh}
cd ~/pmsco
mkdir work
cd work
mkdir twoatom
cd twoatom/
nice python ~/pmsco/pmsco ~/pmsco/projects/twoatom/twoatom.py -s ea -o twoatom_energy_alpha -m single
cd ~/containers
mkdir output
cd output
singularity run -e ../pmsco.sif python -m pmsco -r path/to/your-runfile
@endcode
Runtime warnings may appear because the twoatom project does not contain experimental data.
For parallel processing, prepend `mpirun -np X` to the singularity command as needed.
Note that this requires compatible OpenMPI versions on the host and container to avoid runtime errors.
To learn more about running PMSCO, see @ref pag_run.
*/

58
docs/src/introduction.dox
View File

@@ -1,49 +1,53 @@
/*! @mainpage Introduction
\section sec_intro Introduction
PMSCO stands for PEARL multiple-scattering cluster calculations and structural optimization.
It is a collection of computer programs to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data.
PMSCO (PSI multiple-scattering cluster calculations and structural optimization)
is a Python-based workflow engine to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data using machine learning techniques.
PMSCO was developed at the [Paul Scherrer Institut (PSI)](https://www.psi.ch/)
by the team of the [PEARL beamline](https://www.psi.ch/en/sls/pearl).
The actual scattering calculation is done by code developed by other parties.
While the scattering program typically calculates a diffraction pattern based on a set of static parameters and a specific coordinate file in a single process,
PMSCO wraps around that program to facilitate parameter handling, cluster building, structural optimization and parallel processing.
In the current version, the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code
developed by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley (1999) is used for scattering calculations.
Other code can be integrated as well.
Initially, support for the MSC program by Kaduwela, Friedman, and Fadley was planned but is currently not maintained.
PMSCO is written in Python 2.7.
EDAC is written in C++, MSC in Fortran.
PMSCO interacts with the calculation programs through Python wrappers for C++ or Fortran.
In the current version, PMSCO can make use of the following programs.
Other programs may be integrated as well.
- [EDAC](https://garciadeabajos-group.icfo.es/widgets/edac/)
by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley,
[Phys. Rev. B 63 (2001) 075404](http://dx.doi.org/10.1103/PhysRevB.63.075404)
- PHAGEN from the [MsSpec package](https://ipr.univ-rennes1.fr/msspec)
by C. R. Natoli and D. Sébilleau,
[Comp. Phys. Comm. 182 (2011) 2567](http://dx.doi.org/10.1016/j.cpc.2011.07.012)
\section sec_intro_highlights Highlights
- angle or energy scanned XPD.
- various scanning modes including energy, polar angle, azimuthal angle, analyser angle.
- averaging over multiple symmetries (domains or emitters).
- averaging over multiple domains and emitters.
- global optimization of multiple scans.
- structural optimization algorithms: particle swarm optimization, grid search, gradient search.
- structural optimization algorithms: genetic, particle swarm, grid search.
- calculation of the modulation function.
- calculation of the weighted R-factor.
- automatic parallel processing using OpenMPI.
- integrated and extensible reporting, database storage of results.
- automatic parallel processing using OpenMPI and job submission to scheduling systems.
\section sec_project Optimization Projects
\section sec_intro_project Optimization Projects
To set up a new optimization project, you need to:
- create a new directory under projects.
- create a new Python module in this directory, e.g., my_project.py.
- implement a sub-class of project.Project in my_project.py.
- override the create_cluster, create_params, and create_domain methods.
- optionally, override the combine_symmetries and combine_scans methods.
- add a global function create_project to my_project.py.
- provide experimental data files (intensity or modulation function).
- implement a sub-class of pmsco.project.Project in my_project.py.
- override the create_cluster, create_params, and create_model_space methods.
- optionally, override the combine_domains and combine_scans methods.
- add a global function create_project to my_project.py or create a @ref pag_runfile.
- prepare experimental data files (intensity or modulation function).
For details, see the documentation of the Project class,
and the example projects.
For details, see @ref pag_project, the documentation of the pmsco.project.Project class and the example projects.
\section sec_intro_start Getting Started
@@ -52,22 +56,24 @@ and the example projects.
- @ref pag_concepts_tasks
- @ref pag_concepts_emitter
- @ref pag_install
- @ref pag_project
- @ref pag_run
- @ref pag_command
- @ref pag_opt
\section sec_license License Information
An open distribution of PMSCO is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) at <https://gitlab.psi.ch/pearl-public/pmsco>.
The source code of PMSCO is licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
This _does not include_ the calculation packages contained in the subprojects folder which are licensed separately.
- Please read and respect the respective license agreements.
- Please acknowledge the use of the code.
- Please share your development of the code with the original author.
- Please consider sharing your developments with the original author.
Due to different copyright, the MSC and EDAC programs are not contained in the public software repository.
Due to different copyright terms, the third-party calculation programs are not contained in the public software repository.
These programs may not be used without an explicit agreement by the respective original authors.
\author Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
\version This documentation is compiled from version $(REVISION).
\copyright 2015-2018 by [Paul Scherrer Institut](http://www.psi.ch)
\copyright 2015-2025 by [Paul Scherrer Institut](http://www.psi.ch)
\copyright Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
*/

93
docs/src/optimizers.dox
View File

@@ -3,33 +3,89 @@
\subsection sec_opt_swarm Particle swarm
\subsection sec_opt_swarm Particle swarm optimization (PSO)
The particle swarm algorithm is adapted from
The particle swarm optimization (PSO) algorithm seeks to find a global optimum in a multi-dimensional model space
by employing the _swarm intelligence_ of a number of particles traversing space,
each at its own velocity and direction,
but adjusting its trajectory based on its own experience and the results of its peers.
The PSO algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012).
It is implemented in the @ref pmsco.optimizers.swarm module.
The general parameters of the genetic algorithm are specified in the @ref Project.optimizer_params dictionary.
The general parameters of the algorithm are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | |
| pop_size | --pop-size | &ge; 1 | Recommended 20..50 |
| position_constrain_mode | | default bounce | Resolution of domain limit violations. |
| seed_file | --seed-file | a file path, default none | |
| seed_limit | --seed-limit | 0..pop_size | |
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The domain parameters have the following meanings:
The model space attributes have the following meaning:
| Parameter | Description |
| --- | --- |
| start | Seed model. The start values are copied into particle 0 of the initial population. |
| start | Start value of particle 0 in first iteration. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Not used. |
\subsubsection sec_opt_seed Seeding a population
By default, one particle is initialized with the start value declared with the model space,
and the other ones are initialized at random positions in the model space.
You may initialize more particles of the population with specific values by providing a seed file.
The seed file must have a similar format as the result `.dat` files
with a header line specifying the column names and data rows containing the values for each particle.
A good practice is to use a previous `.dat` file and remove unwanted rows.
The `.dat` file from a previous optimization job can be used as is to continue the optimization,
also in a different optimization mode.
The seeding procedure can be tweaked by several optimizer parameters (see above).
PMSCO normally loads the first rows up to population size - 1 or up to the `seed_limit` parameter,
whichever is lower.
If an `_rfac` column is present, the file is first sorted by R-factor and only the best models are loaded.
Models that resulted in an R-factor above the `rfac_limit` parameter are ignored in any case.
In the first iteration of the optimization run, the models loaded from the seed file are re-calculated by default.
This may waste CPU time if the calculation is run under the same conditions
and would result in exactly the same R-factor,
as is the case if the seed is used to continue a previous optimization, for example.
In these situations, the `recalc_seed` parameter can be set to False,
and PMSCO will use the R-factor value from the seed file rather than calculating the model again.
\subsubsection sec_opt_patch Patching a running optimization
While an optimization job is running, the user can manually patch the population with arbitrary values,
for instance, to kick the population out of a local optimum or to drive it to a less sampled parameter region.
To patch a running population, prepare a population file named `pmsco_patch.pop` and copy it to the work directory.
The patch file must have the same format as the result `.dat` files
with a header line specifying the column names and data rows containing the values.
It should contain as many rows as particles to be patched but not more than the size of the population.
The columns must include a `_particle` column and the model parameters to be changed.
The `_particle` column specifies the index of the particle that is patched (ranging from 0 to population size - 1).
Parameters that should remain unaffected can be left out,
extra columns including `_gen`, `_rfac` etc. are ignored.
PMSCO checks the file for syntax errors and ignores it if errors are present.
Individual parameter values that lie outside the domain boundary are silently ignored.
Successful or failed patching is logged at warning level.
PMSCO keeps track of the time stamp of the file and re-applies the patch whenever the time stamp has changed.
\attention Since each change of time stamp may trigger patching,
do not edit the patch file in the working directory
to prevent it from being read in an unfinished state or multiple times!
\subsection sec_opt_genetic Genetic optimization
The genetic algorithm evolves a population of individuals
@@ -54,7 +110,7 @@ Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | |
| pop_size | --pop-size | &ge; 1 | Recommended 10..40 |
| mating_factor | | 1..pop_size, default 4 | |
| strong_mutation_probability | | 0..1, default 0.01 | Probability that a parameter undergoes a strong mutation. |
| weak_mutation_probability | | 0..1, default 1 | Probability that a parameter undergoes a weak mutation. This parameters should be left at 1. Lower values tend to produce discrete parameter values. Weak mutations can be tuned by the step domain parameters. |
@@ -64,7 +120,7 @@ Some of them can be changed on the command line.
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The domain parameters have the following meanings:
The model space attributes have the following meaning:
| Parameter | Description |
| --- | --- |
@@ -73,11 +129,18 @@ The domain parameters have the following meanings:
| max | Upper limit of the parameter range. |
| step | Standard deviation of the Gaussian distribution of weak mutations. The step should not be much lower than the the parameter range divided by the population size and not greater than one third of the parameter range. |
The population of the genetic optimizer can be seeded and patched in the same way as the particle swarm,
cf. sections @ref sec_opt_seed and @ref sec_opt_swarm.
\subsection sec_opt_grid Grid search
The grid search algorithm samples the parameter space at equidistant steps.
The order of calculations is randomized so that early results represent various parts of the parameter space.
It is implemented in the @ref pmsco.optimizers.grid module.
The model space attributes have the following meaning.
The order of calculations is random so that results from different parts of the model space become available early.
| Parameter | Description |
| --- | --- |
@@ -97,15 +160,19 @@ The table scan calculates models from an explicit table of model parameters.
It can be used to recalculate models from a previous optimization run on other experimental data,
as an interface to external optimizers,
or as a simple input of manually edited model parameters.
It is implemented in the @ref pmsco.optimizers.table module.
The table can be stored in an external file that is specified on the command line,
or supplied in one of several forms by the custom project class.
The table can be left unchanged during the calculations,
or new models can be added on the go.
Duplicate models are ignored.
@attention Because it is not easily possible to know when and which models have been read from the table file, if you do modify the table file during processing, pay attention to the following hints:
1. The file on disk must not be locked for more than a second. Do not keep the file open unnecessarily.
2. _Append_ new models to the end of the table rather than overwriting previous ones. Otherwise, some models may be lost before they have been calculated.
@attention Because it is not easily possible to know when the table file is read,
if you do modify the table file while calculations are running,
1. Do not keep the file locked for longer than a second.
2. Append new models to the end of the table rather than overwriting previous ones.
3. Delete lines only if you're sure that they are not needed any more.
The general parameters of the table scan are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line or in the project class (depending on how the project class is implemented).
@@ -115,7 +182,7 @@ Some of them can be changed on the command line or in the project class (dependi
| pop_size | --pop-size | &ge; 1 | Number of models in a generation (calculated in parallel). In table mode, this parameter is not so important and can be left at the default. It has nothing to do with table size. |
| table_file | --table-file | a file path, default none | |
The domain parameters have the following meanings.
The model space attributes have the following meaning.
Models that violate the parameter range are not calculated.
| Parameter | Description |

514
docs/src/project.dox Normal file
View File

@@ -0,0 +1,514 @@
/*! @page pag_project Setting up a new project
\section sec_project Setting up a new project
This topic guides you through the setup of a new project.
Be sure to check out the examples in the projects folder
and the code documentation as well.
The basic steps are:
1. Create a new package folder under `pmsco/projects`.
To keep your code and PMSCO separate, you are suggested to start your own pmsco/projects tree
in a convenient location separate from the PMSCO source code.
2. Add the parent directory of your pmsco/projects tree to the Python path.
3. In the new folder, create a Python module for the project (subsequently called _the project module_).
4. In the project module, define a cluster generator class which inherits from @ref pmsco.cluster.ClusterGenerator.
5. In the project module, define a project class which inherits from @ref pmsco.project.Project.
6. Create one or more run files.
The basic steps listed above are recommended and explained in the following.
In previous versions, other mechanisms of project invocation were available.
They are now obsolete.
\subsection sec_packages Namespace packages
[Python namespace packages](https://realpython.com/python-namespace-package/) provide an easy way
to inject project modules into the PMSCO namespace
while their source files are kept separate from the core PMSCO packages.
This way, PMSCO and the project modules can be under separate version control.
Namespace packages work by extending the Python module search path.
The module loader looks for packages in every entry of the search path
and does not stop at the first match as it would do for a regular package.
The recommended folder structure is:
~~~~~~
pmsco-projects/
+-- pyproject.toml
+-- pmsco/
+-- projects/
+-- project1/
+-- project1.py
+-- run1.json
+-- ...
+-- project2/
+-- ...
~~~~~~
In place of `pmsco-projects`, `project1`, `project2`, `run1`, you should use distinct names.
The two levels `pmsco` and `projects` should be left as is.
If you now include `pmsco-projects` in the Pyton path,
all of your projects become available within the `pmsco` namespace, i.e.,
you can `import pmsco.projects.project1.project1` in Python.
Furthermore, you can call the module in a run-file without specifying a file path.
You may install multiple project packages if needed.
The recommended way to add `pmsco-projects` to the Python path is by an editable installation.
This will allow you to keep editing your project sources in place.
1. Place your project files in a directory tree similar to `pmsco-projects/pmsco/projects/project1/`.
The `pmsco/projects` level is mandatory as a part of the path.
Replace `pmsco-projects` and `project1` by your own choice.
2. Be sure not to create any `__init__.py` files in this directory tree.
3. Copy the `pyproject.toml` file from the PMSCO source into your `pmsco-projects` and adjust its contents.
At least give the package a distinct name.
4. Select another build backend if necessary.
The default [uv_build](https://docs.astral.sh/uv/concepts/build-backend/) is recommended for pure Python projects.
5. 'Install' the project locally.
With uv, call `uv sync --active --inexact`
while you are in the directory that contains the `pyproject.toml` file.
In plain pip the corresponding command would be
`pip install --editable .`.
6. Check that you can `import pmsco.projects.project1.project1` (or whatever your project is called) in a Python shell.
If you encounter problems importing the pmsco modules, check the Python path in a Python shell.
It must contain the `site-packages` directory of your Python environment.
Make sure it does not contain any pmsco or project source directory explicitly.
Also make sure that you don't have any `__init__.py` files in your project tree,
and do not use explicit paths to pmsco or your project anywhere in your source code or shell configuration files.
Be careful not to install packages multiple times in different locations.
In case of trouble, set up a fresh environment.
\subsection sec_project_module Project module
A skeleton of the project module file (with some common imports) may look like this:
~~~~~~{.py}
import logging
import math
import numpy as np
import periodictable as pt
from pathlib import Path
import pmsco.cluster
import pmsco.data
import pmsco.dispatch
import pmsco.elements.bindingenergy
import pmsco.project
logger = logging.getLogger(__name__)
class MyClusterGenerator(pmsco.cluster.ClusterGenerator):
def create_cluster(self, model, index):
clu = pmsco.cluster.Cluster()
# ...
return clu
def count_emitters(self, model, index):
# ...
return 1
class MyProject(pmsco.project.Project):
def __init__(self):
super().__init__()
# ...
self.cluster_generator = MyClusterGenerator(self)
def create_model_space():
spa = pmsco.project.ModelSpace()
# ...
return spa
def create_params(self, model, index):
par = pmsco.project.CalculatorParams()
# ...
return par
~~~~~~
The main purpose of the `MyProject` class is to bundle the project-specific calculation parameters and code.
The purpose of the `MyClusterGenerator` class is to produce atomic clusters as a function of a number of model parameters.
For the project to be useful, some of the methods in the skeleton above need to be implemented.
The individual methods are discussed in the following.
Further descriptions can be found in the documentation of the code.
\subsection sec_project_cluster Cluster generator
The cluster generator is a project-specific Python object that produces a cluster, i.e., a list of atomic coordinates,
based on a small number of model parameters whenever PMSCO requires it.
The most important method of a cluster generator is `create_cluster`.
At least this method must be implemented for a functional cluster generator.
A generic `count_emitters` method is implemented in the base class.
It needs to be overridden if inequivalent emitters should be calculated in parallel.
\subsubsection sec_project_cluster_create Cluster definition
The `create_cluster` method takes the model parameters (a dictionary)
and the task index (a pmsco.dispatch.CalcID, cf. @ref pag_concepts_tasks) as arguments.
Given these arguments, it creates and fills a @ref pmsco.cluster.Cluster object.
See @ref pmsco.cluster.ClusterGenerator.create_cluster for details on the method contract.
As an example, have a look at the following simplified excerpt from the `twoatom` demo project.
~~~~~~{.py}
class TwoatomCluster(ClusterGenerator):
# ...
def create_cluster(self, model, index):
# access model parameters
# dAB - distance between atoms in Angstroms
# th - polar angle in degrees
# ph - azimuthal angle in degrees
r = model['dAB']
th = math.radians(model['th'])
ph = math.radians(model['ph'])
# prepare a cluster object
clu = pmsco.cluster.Cluster()
# the comment line is optional but can be useful
clu.comment = "{0} {1}".format(self.__class__, index)
# set the maximum radius of the cluster (outliers will be ignored)
clu.set_rmax(r * 2.0)
# calculate atomic vectors
dx = r * math.sin(th) * math.cos(ph)
dy = r * math.sin(th) * math.sin(ph)
dz = r * math.cos(th)
a_top = np.array((0.0, 0.0, 0.0))
a_bot = np.array((-dx, -dy, -dz))
# add an oxygen atom at a_top position and mark it as emitter
clu.add_atom('O', a_top, 1)
# add a copper atom at a_bot position
clu.add_atom('Cu', a_bot, 0)
# pass the created cluster to the calculator
return clu
~~~~~~
In this example, two atoms are added to the cluster.
The @ref pmsco.cluster.Cluster class provides several methods to simplify the task,
such as adding layers or bulk regions, rotation, translation, trim, emitter selection, etc.
Please refer to the documentation of its code for details.
It may also be instructive to have a look at the demo projects.
The main purposes of the cluster object are to store an array of atoms and to read/write cluster files in a variety of formats.
For each atom, the following properties are stored:
- sequential atom index (1-based, maintained by cluster code)
- atom type (chemical element number)
- chemical element symbol from periodic table
- x coordinate of the atom position
- y coordinate of the atom position
- z coordinate of the atom position
- emitter flag (0 = scatterer, 1 = emitter, default 0)
- charge/ionicity (units of elementary charge, default 0)
- scatterer class (default 0)
All of these properties except the scatterer class can be set by the `add_xxxx` methods of the cluster.
The scatterer class is used internally by the atomic scattering factor calculators.
Whether the charge/ionicity is used, depends on the particular calculators, EDAC does not use it, for instance.
\note You do not need to take care how many emitters a calculator allows,
or whether the emitter needs to be at the origin or the first place of the array.
These technical aspects are handled by PMSCO code transparently.
\subsubsection sec_project_cluster_domains Domains
Domains refer to regions of inequivalent structure in the probing region.
This may include regions of different orientation, different lattice constant, or even different structure.
The cluster methods read the requested domain from the `index.domain` argument.
This is an index into the @ref pmsco.project.Project.domains list where each item is a dictionary
that holds additional, invariable structural parameters.
A common case are rotational domains.
In this case, the list of domains may look like `[{"zrot": 0.0}, {"zrot": 60.0}]`, for example,
and the `create_cluster` method would include additional code to rotate the cluster:
~~~~~~{.py}
def create_cluster(self, model, index):
# filling atoms here
# ...
dom = self.domains[index.domain]
try:
z_rot = dom['zrot']
except KeyError:
z_rot = 0.0
if z_rot:
clu.rotate_z(z_rot)
# selecting emitters
# ...
return clu
~~~~~~
Depending on the complexity of the system, it is advisable to split the code into a separate method for each domain.
The @ref pmsco.project.Project class includes generic code to add intensities of domains incoherently
(cf. @ref pmsco.project.Project.combine_domains).
In this case, the model space should contain parameters 'wdom0', 'wdom1', etc.,
that define the weights of domain 0, 1, etc.
To avoid correlations between parameters, one domain must have a fixed weight:
Typically, 'wdom0' is left undefined and defaults to 1.
\subsubsection sec_project_cluster_emitters Emitter configurations
If a project uses a large cluster and/or many emitters,
it may be more efficient to generate emitter-specific cluster configurations,
for instance to leverage process parallelization,
or to produce small, local clusters around the emitter site.
This concept is called _emitter configurations_ and explained in detail in @ref pag_concepts_emitter.
To implement emitter configurations, override the `count_emitters` method to return the number of emitter configurations.
In the simplest case, this is the number of inequivalent emitters:
~~~~~~{.py}
def count_emitters(self, model, index):
index = index._replace(emit=-1)
clu = self.create_cluster(model, index)
return clu.get_emitter_count()
~~~~~~
Next, modify the `create_cluster` method to check the emitter index (`index.emit`).
If it is -1, the method must return the full cluster with all inequivalent emitters marked.
If it is positive, only the corresponding emitter configuration must be marked.
For example, if each emitting atom represents a separate emitter configuration:
~~~~~~{.py}
def create_cluster(self, model, index):
# filling atoms here
# ...
# select all possible emitters (atoms of a specific element) in a cylindrical volume
# idx_emit is an array of atom numbers (0-based atom index)
idx_emit = clu.find_index_cylinder(origin, r_xy, r_z, self.project.scans[index.scan].emitter)
# if PMSCO asks for a specific emitter, restrict the array index:
if index.emit >= 0:
idx_emit = idx_emit[index.emit]
# mark the selected emitters
clu.data['e'][idx_emit] = 1
return clu
~~~~~~
Now, the individual emitter configurations are calculated in separate tasks
which can run in parallel in a multi-process environment.
Note that the processing time of EDAC scales linearly with the number of emitters.
\subsection sec_project_project Project class
Most commonly, a project class overrides the `__init__`, `create_model_space` and `create_params` methods.
Most other inherited methods can be overridden optionally,
for instance `validate`, `setup`, `calc_modulation`, `rfactor`,
as well as the combine methods `combine_rfactors`, `combine_domains`, `combine_emitters`, etc.
This introduction shall focus on the three most important methods.
\subsubsection sec_project_project_init Initialization and defaults
The `__init__` method defines and initializes project properties with default values.
It may also redefine properties of the base class.
The following code is just an example to give some ideas.
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def __init__(self):
# call the inherited method first
super().__init__()
# re-define an inherited property
self.directories["data"] = Path("/home/pmsco/data")
# define a scan dictionary
self.scan_dict = {}
# fill the scan dictionary
self.build_scan_dict()
# create the cluster generator
self.cluster_generator = MyClusterGenerator(self)
# declare the list of domains (at least one is required)
self.domains = [{"zrot": 0.}]
def build_scan_dict(self):
self.scan_dict["empty"] = {"filename": "${pmsco}/projects/common/empty-hemiscan.etpi",
"emitter": "Si", "initial_state": "2p3/2"}
self.scan_dict["Si2p"] = {"filename": "${data}/xpd-Si2p.etpis",
"emitter": "Si", "initial_state": "2p3/2"}
~~~~~~
A scan dictionary is one way to specify locations and metadata of experimental files centrally in the project code.
The scan can then be selected by the dictionary key rather than copying file locations.
Note that all public attributes can be assigned from a run file.
This happens after the `__init__` method.
The values set by `__init__` serve as default values.
\subsubsection sec_project_project_space Model space
The model space defines the keys and value ranges of the model parameters.
There are three ways to declare the model space in order of priority:
1. Declare the model space in the run-file.
2. Assign a ModelSpace to the self.model_space property directly in the `__init__` method.
3. Implement the `create_model_space` method.
The third way may look like this:
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def create_model_space(self):
# create an empty model space
spa = pmsco.project.ModelSpace()
# add parameters
spa.add_param('dAB', 2.05, width=0.25, step=0.05)
spa.add_param('th', 15.00, 0.00, 30.00, 1.00)
spa.add_param('ph', 90.00)
spa.add_param('V0', 21.96, width=10.0, step=1.0)
spa.add_param('Zsurf', 1.50)
spa.add_param('wdom1', 0.5, 0.10, 10.00, 0.10)
# return the model space
return spa
~~~~~~
This code declares six model parameters: `dAB`, `th`, `ph`, `V0`, `Zsurf` and `wdom1`.
Three of them are structural parameters (used by the cluster generator above),
two are used by the `create_params` method (see below),
and `wdom1` is used in @ref pmsco.project.Project.combine_domains
while summing up contributions from different domains.
The values in the arguments list correspond to the start value (initial guess),
the lower and upper boundaries of the value range,
and the step size for optimizers that require it.
If just one value is given the parameter is held constant during the optimization.
The range can, alternatively, be specified by the `width` argument.
A similar declaration in a run-file could look like as follows (some parameters omitted for brevity).
Parameter values can be numeric constants,
or simple Python math expressions in double quotes.
~~~~~~{.py}
{
"project": {
// ...
"model_space": {
"dAB": {
"start": "2.0 / math.cos(math.radians(15.0))",
"width": 0.25,
"step": 0.05
},
"th": {
"start": 15.0,
"min": 0.0,
"max": 30.0,
"step": 1.0
},
"Zsurf": {
"start": 1.50
}
// ...
}
}
}
~~~~~~
\subsubsection sec_project_project_params Calculation parameters
Non-structural parameters that are needed for the input files of the calculators are passed
in a @ref pmsco.project.CalculatorParams object.
This object is created and filled in the `create_params` method of the project class.
The following example is from the `twoatoms` demo project:
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def create_params(self, model, index):
params = pmsco.project.CalculatorParams()
# meta data
params.title = "two-atom demo"
params.comment = "{0} {1}".format(self.__class__, index)
# initial state and binding energy
initial_state = self.scans[index.scan].initial_state
params.initial_state = initial_state
emitter = self.scans[index.scan].emitter
params.binding_energy = pt.elements.symbol(emitter).binding_energy[initial_state]
# experimental setup
params.polarization = "H"
params.polar_incidence_angle = 60.0
params.azimuthal_incidence_angle = 0.0
params.experiment_temperature = 300.0
# material parameters
params.z_surface = model['Zsurf']
params.work_function = 4.5
params.inner_potential = model['V0']
params.debye_temperature = 356.0
# multiple-scattering parameters (EDAC)
params.emitters = []
params.lmax = 15
params.dmax = 5.0
params.orders = [25]
return params
~~~~~~
Most of the code is generic and can be copied to other projects.
Only the experimental and material parameters need to be adjusted.
Other properties can be changed as needed, see @ref pmsco.project.CalculatorParams.
\subsection sec_project_args Passing run-time parameters
The recommended way of passing calculation parameters is via @ref pag_runfile.
Run-files allow for a complete separation of code and data in a generic and flexible way.
Program code can be managed by a version control system,
and run-files can be stored along with the results.
This simplifies the reproduction of previous calculations and documentation of the workflow.
For testing and simple projects, it is possible to hard-code all parameters in the project class.
\subsubsection sec_project_args_runfile Setting up a run-file
The usage and format of run-files is described in detail under @ref pag_runfile.
\subsubsection sec_project_args_code Hard-coded arguments
Though it's normally recommended to declare all parameters in the run-file,
parameter values can also be hard-coded in the constructor and/or the validate method of the project class.
Which method to use depends on the processing stage.
The constructor can set default values for rarely changing parameters.
The declarations in the run-file override the defaults from the constructor.
If some parameters need adjusting _after_ the run-file has been loaded,
this can be done in the validate` method.
The call sequence of the methods is as follows.
1. `Project.__init__`:
The constructor is usually overridden by the project.
The constructor must call the superclass before applying its values.
2. `Project.set_properties`:
Sets the parameters from the run-file and resolves class names.
This method can be overridden if additional classes need resolving after loading the run-file.
It must call the superclass.
3. `Project.validate`: Parameters are validated, i.e., checked and made consistent.
Handler classes are resolved.
The `validate` method or its sub-methods can be overridden by the project.
The inherited method should be called.
*/

135
docs/src/reports.dox Normal file
View File

@@ -0,0 +1,135 @@
/*! @page pag_reports Reports
\section sec_reports Reports
The main output of PMSCO is the model parameters to R-factor mapping.
By default, it is produced in the form of a text file (.dat) as well as an sqlite3 database file (.db).
Graphical representations of the result data, called _reports_ in PMSCO, can be produced automatically at run-time or
manually after the calculation has ended.
PMSCO provides a number of pre-defined reports as well as an interface for custom reports.
Essentially, a report is defined by a Python class which derives from `pmsco.reports.base.ProjectReport`.
Instances of reports are added to the project's `reports` list during initialization of the calculation job.
They are called by the calculation handlers whenever a new model-level result is available in the database.
While reports typically produce graphics files for diagnostics,
report classes could basically produce any derived data including data files in different formats.
By default, no report is produced during a project run.
There are several ways to generate reports:
- Add instances of reports to the `reports` list of the project object.
This can be done in the project code or in the @ref pag_runfile.
One or multiple reports (of different classes) can be added and configured.
- Some report modules have their own command line interface.
This allows you to produce a report at any time during or after the project run.
- Lastly, all reports are Python classes and can be instantiated and executed in a Python shell.
The remainder of this page describes some of the pre-defined reports and their configuration parameters (attributes).
@note Reporting is still under development.
The configuration parameters and behaviour is subject to change, and the documentation may be partially outdated.
Be sure to check the in-line documentation as well as the source code for the latest information.
\subsection sec_reports_common Common Parameters
The reports share some common parameters which may, however, be used differently or ignored by some reports.
| Key | Values | Description |
| --- | --- | --- |
| `filename_format` | template string using `${key}`-type placeholders | Template string for file names of reports. Possible placeholders are listed below. |
| `title_format` | template string using `${key}`-type placeholders | Template string for graph titles. Possible placeholders are listed below. |
| `canvas` | string. default: `matplotlib.backends.backend_agg.FigureCanvasAgg` (PNG) | A matplotlib figure canvas such as FigureCanvasAgg, FigureCanvasPdf or FigureCanvasSVG. |
The `filename_format` and `title_format` attributes are template strings which can contain `${key}` type placeholders.
placeholders are replaced according to the following table.
Some of these values may not be available if you call the reports outside of an optimization run
(e.g., from the command line of a report module).
| Key | Description |
| --- | --- |
| `base` | Base file name. Default: job name |
| `mode` | optimization mode |
| `job_name` | job name |
| `project_name` | project name |
| any directories key | corresponding directories value |
| any job_tags key | corresponding job_tags value |
\subsection sec_reports_convergence Convergence Plot
The convergence plot is a violin plot where each violin represents the R-factor distribution of one generation.
The minimum, maximum and mean values are marked, and the distribution is indicated by the body.
Convergence plots are suitable for genetic or swarm optimizations.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.ConvergencePlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
\subsection sec_reports_genetic Genetic Chart
A genetic chart is a pseudo-colour representation of the coordinates of each individual in the model space.
The chart shows the amount of diversity in the population
and - by comparing charts of different generations - the changes due to mutation.
The axes are the model parameters (x) and particle number (y).
The colour is mapped from the relative parameter value within the parameter range.
Genetic charts are suitable for genetic or swarm optimizations.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.GeneticPlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
| cmap | string: 'viridis', 'plasma' (default), 'inferno', 'magma', 'cividis' | Name of colour map supported by matplotlib. |
| params | list of model parameter names | |
In addition to the common template substitutions,
the genetic chart report replaces the following placeholders
of the `filename_format` and `title_format` template strings.
| Key | Description |
| --- | --- |
| `gen` | Generation index (population reports only) |
\subsection sec_reports_swarm Particle Swarm Plot
The particle swarm plot shows the current positions and velocities of particles projected onto two dimensions.
The plot contains three elements:
- a pseudo-color scatter plot of all R-factors in the background,
- a scatter plot of particle positions.
- a quiver plot indicating the velocities of the particles.
Particle swarm plots are suitable in particle swarm optimization mode only.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.SwarmPlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
| cmap | string: 'viridis', 'plasma' (default), 'inferno', 'magma', 'cividis' | Name of colour map supported by matplotlib. |
| params | nested list of pairs of model parameter names | |
In addition to the common template substitutions,
the particle swarm plot report replaces the following placeholders
of the `filename_format` and `title_format` template strings.
| Key | Description |
| --- | --- |
| `gen` | Generation index (population reports only) |
| `param0` | Parameter name 0 (population reports only) |
| `param1` | Parameter name 1 (population reports only) |
\subsection sec_reports_misc Miscellaneous
To make a video from swarm or genetic plots, you may use ffmpeg on Linux:
~~~~~~{.sh}
ffmpeg -framerate 5 -i basename-%00d.geneticplot.png -c:v libx264 -profile:v high -crf 20 -pix_fmt yuv420p basename.geneticplot.mp4
~~~~~~
*/

506
docs/src/runfile.dox Normal file
View File

@@ -0,0 +1,506 @@
/*! @page pag_runfile Run File
\section sec_runfile Run File
This section describes the format of a run-file.
Run-files are a flexible way of passing arguments to a PMSCO process.
The benefits are:
- contain all essential parameters to repeat a calculation - no need to remember or record the command line
- avoid cluttering up the command line or frequent changes of source code
- can be versioned or stored separately from the code, maintain a single file or multiple files - up to the user
- any property and sub-property of the project object can be assigned in a generic way - even custom properties that are unknown to PMSCO
- no necessity for the project code to parse the command line
- schema validation can help to find syntax errors while editing
\subsection sec_runfile_how How It Works
Run-files are text files in machine and human readable [JSON](https://en.wikipedia.org/wiki/JSON) format.
In PMSCO, run-files contain dictionaries of parameters to be passed to the project object.
For the calculations, internally, the project object is the main container of calculation parameters, model objects and input data.
Upon launching PMSCO, a generic parser reads the run-file,
constructs the project object from the specified custom project class
and assigns the attributes defined in the run-file.
Run-files are a sort of script that assign data to the project.
The parser does not expect specific data types or classes.
It merely copies data items to the project attributes of the same name.
The validation and interpretation of the data is up to the project object.
The parser handles the following situations:
- Strings, numbers as well as dictionaries and lists of simple objects are assigned directly to project attributes.
If the project class declares a setter method for the attribute, the setter is called.
Else, the existing attribute is overwritten.
Setters can execute custom code to validate the data value.
- If specified in the run-file, the parser creates objects from classes in the namespace of the project module
and recursively assigns their properties.
\note There are no implicit checks of correctness of the assigned data objects!
The author of the run-file must make sure that the run-file is compatible with the project class,
else the calculation process might fail.
There are three ways to check assigned attributes before the calculations are started.
All have to be implemented explicitly by the project maintainer:
1. The run-file can be validated against a JSON schema before launching PMSCO (see below).
Schema validation may catch some obvious mistakes
but is not complete in the sense that it cannot guarantee error-free execution of the project code.
2. The classes used with run-files define property setters.
The setters can raise an exception or post an error in the log.
(The latter won't stop the calculation process.)
3. The project class implements a validation method to check and fix important or error-prone attributes.
It can write warnings and errors to the log, or raise an exception if the process should be aborted.
\subsection sec_runfile_general General File Format
Run-files must adhere to the [JSON](https://en.wikipedia.org/wiki/JSON) format.
Specifically, a JSON file can declare dictionaries, lists and simple objects
such as strings, numbers and `null`.
The syntax of these basic elements is similar to Python source code (there are some differences, though).
At the top level, a PMSCO run-file contains a dictionary with up to two items:
1. The _project_ item is the most important, it is described in the following under @ref sec_runfile_project.
2. The _schedule_ item is an optional section for passing the parameters to a job queue of a computing cluster.
See @ref sec_runfile_schedule .
\subsection sec_runfile_schema Schema
The structure of a JSON file can be described in a _schema_ file that can be used to check the syntax and structure programmatically.
The `schema/runfile.schema.json` file of the PMSCO distribution describes the structure of a run-file as well as common properties of the project.
The schema is, however, rather basic and does not cover all parameters, conditional cases or custom project properties.
A run-file can be easily validated against the schema while editing in the PyCharm IDE.
Alternatively, the jsonschema validator from the Python distribution can be used on the command line.
\subsection sec_runfile_project Project Specification
The following minimum run-file from the twoatom project demonstrates how to specify the project:
~~~~~~{.py}
{
"project": {
"__module__": "twoatom",
"__class__": "TwoatomProject",
"mode": "single",
"job_name": "twoatom0001"
}
}
~~~~~~
Here, the `project` keyword denotes the dictionary that is used to construct the project object.
Within the project dictionary, the `__module__` key selects the Python module file that contains the project code,
and `__class__` refers to the name of the actual project class.
Further dictionary items correspond to attributes of the project class.
The module name is the same as would be used in a Python import statement.
It must be findable on the Python path.
Alternatively, a file path may be specified.
PMSCO ensures that the directory containing the `pmsco` and `projects` sub-directories is on the Python path.
The class name must be in the namespace of the loaded module.
As PMSCO starts, it imports the specified module,
constructs an object of the specified project class,
and assigns any further items to project attributes.
In the example above, it creates an object of type `TwoatomProject` from the `twoatom` module
and assigns `single` to the `mode` property and `twoatom0001` to the `job_name` property.
Any attributes not specified in the run-file remain at their default values
that were set by the `__init__` constructor of the project class.
Note that parameter names must start with an alphabetic character, else they are ignored
(useful for comments as JSON does not have a syntax for comments).
Also note that PMSCO does not spell-check parameter names.
The parameter values are just written to the corresponding object attribute.
If a name is misspelled, the value will be written to the wrong attribute.
PMSCO carries out only some most important checks on the given parameter values.
Incorrect values may lead to improper operation or exceptions later in the calculations.
The project class can explicitly check and fix important or error-prone attributes, or report errors.
The following sub-sections describe the most common properties of the project class.
\subsubsection sec_runfile_common Common Arguments
The following table lists some important parameters controlling the calculations.
They are declared in the pmsco.projects.Project class.
| Key | Values | Description |
| --- | --- | --- |
| mode | `single` (default), `grid`, `swarm`, `genetic`, `table`, `test`, `validate` | Operation mode. `validate` can be used to check the syntax of the run-file, the process exits before starting calculations. |
| directories | dictionary | This dictionary lists common file paths used in the project. It contains keys such as `home`, `project`, `output` (see documentation of Project class in pmsco.project). |
| output_dir | path | Shortcut for directories["output"] |
| data_dir | path | Shortcut for directories["data"] |
| job_name | string, must be a valid and unique file name (see note below) | Base name for all produced output files. It is recommended to set a unique name for each calculation run. Do not include a path. The path can be set in _output_dir_. |
| cluster_generator | dictionary | Class name and attributes of the cluster generator. See below. |
| atomic_scattering_factory | string<br>Default: InternalAtomicCalculator from pmsco.calculators.calculator | Class name of the atomic scattering calculator. This name must be in the namespace of the project module. |
| multiple_scattering_factory | string<br>Default: EdacCalculator from pmsco.calculators.edac | Class name of the multiple scattering calculator. This name must be in the namespace of the project module. |
| model_space | dictionary | See @ref sec_runfile_space below. |
| domains | list of dictionaries | See @ref sec_runfile_domains below. |
| scans | list of dictionaries | See @ref sec_runfile_scans below. |
| optimizer_params | dictionary | See @ref sec_runfile_optimizer below. |
\note The *job name* parameter appears most visibly as the prefix of output file names.
It is also registered in the `jobs` table of the results database (if used),
and it is used to identify the job with a job scheduling system.
For these reasons, it is important that the job name be unique within the respective subsystem.
Specifically, you need to *provide a new job name each time you start pmsco*, otherwise the job may fail.
It may be more natural to specify the job name on the command line using the `-o` argument
rather than changing the run file every time.
Unfortunately, PMSCO cannot auto-generate, auto-increment or verify the job name.
File names specified in a runfile can include an explicit path or a placeholder.
Placeholders have the format `${key}` where `key` must be one of the keys of the `directories` dictionary.
The placeholder will then be replaced by the corresponding value before the calculation starts
(as a part of the pmsco.project.Project.validate method).
The `directories` dictionary can be filled by the project class or in the runfile.
In addition, a number of keys are defined by PMSCO and can be used as placeholders in other directories and file paths.
| Key | Type | Description |
| --- | --- | --- |
| data | absolute | Directory with experimental data. Must be set by user if needed. |
| home | absolute | Home directory of the current user |
| pmsco | absolute | Directory that contains the loaded pmsco.py module. Note: This may be in a site packages directory. |
| output | absolute | Output directory. Must be set by the user. |
| project | absolute | Directory where the project module is located. |
| project_name | relative | Name of the project. By default, the name of the project class. |
| job_name | relative | Name of the calculation job. |
| mode | relative | Calculation mode |
| report | absolute | Report directory. Defaults to `${output}/report`. |
| run | absolute | Directory where the runfile is located (if used). |
| temp | absolute | Directory for temporary files. Currently not used. |
| work | absolute | Current working directory |
Placeholders of absolute paths must be used at the beginning of a path.
Relative paths can be used at any position in a file path.
Some of the keys may have empty values if PMSCO was loaded in a non-standard way.
For verification of the path resolution, all directories are printed to the log file at WARNING level (default).
The following table lists some common control parameters and metadata
that affect the behaviour of the program but do not affect the calculation results.
The job metadata is used to identify and describe a job in the results database if requested.
| Key | Values | Description |
| --- | --- | --- |
| db_file | new or existing file path or `:memory:` | SQLite3 database file to receive the optimization results. If the database exists, results are inserted under the given job name. If it doesn't exist, a new file is created. If the attribute is `:memory:`, an in-memory database is used internally and flushed at the end of processing. |
| job_tags | dictionary of strings | User-specified job tags in key-value format (metadata). |
| description | string | Description of the calculation job (metadata) |
| time_limit | decimal number<br>Default: 24. | Wall time limit in hours. The optimizers try to finish before the limit. This cannot be guaranteed, however. |
| keep_files | list of file categories | Output file categories to keep after the calculation. Multiple values can be specified and must be separated by spaces. By default, cluster and model (simulated data) of a limited number of best models are kept. See @ref sec_runfile_files below. |
| keep_best | integer number<br>Default: 10 | number of best models for which result files should be kept. |
| keep_levels | integer number<br>Default: 1 | numeric task level down to which files are kept. 1 = scan, 2 = domain, 3 = emitter. |
| log_level | DEBUG, INFO, WARNING, ERROR, CRITICAL | Minimum level of messages that should be added to the log. Empty string turns off logging. |
| log_file | file system path<br>Default: job_name + ".log". | Name of the main log file. Under MPI, the rank of the process is inserted before the extension. The log name is created in the working directory. |
\subsubsection sec_runfile_space Model Space
The `model_space` parameter is a dictionary of model parameters.
The key is the name of the parameter as used by the cluster and input-formatting code,
the value is a dictionary holding the `start`, `min`, `max`, `step` values to be used by the optimizer.
Instead of `min` and `max` you may declare the `width`, which will center the space on the start value.
All parameter values can be declared as numbers or as simple Python expressions in double quotes.
Expressions are evaluated by the Python `eval` function.
All functions in the namespace of the project module.
Note that you have to import the `math` or `numpy` modules in your project module
if you want to use their functions.
~~~~~~{.py}
{
"project": {
// ...
"model_space": {
"dAB": {
"start": 2.109,
"min": "2.109 - 0.1",
"max": "2.109 + 0.1",
"step": 0.05
},
"pAB": {
"start": "4 * 3.56 / math.sqrt(3.0)",
"width": 4.0,
"step": 0.5
},
// ...
}
}
}
~~~~~~
Alternatively, the `model_space` can be declared as a `ModelSpace` object.
However, this shall not be described in detail here.
\subsubsection sec_runfile_domains Domains
Domains is a list of dictionaries.
Each dictionary holds keys describing the domain to the cluster and input-formatting code.
The meaning of these keys is up to the project.
An example:
~~~~~~{.py}
{
"project": {
// ...
"domains": [
{"surface": "Te", "doping": null, "zrot": 0.0},
{"surface": "Te", "doping": null, "zrot": 60.0}
],
}
}
~~~~~~
\subsection sec_runfile_scans Experimental Scan Files
The pmsco.scan.Scan objects used in the calculation cannot be instantiated from the run-file directly.
Instead, the scans object of the run-file is a list of scan creators/loaders which specify how to create a Scan object.
The pmsco.scan module defines four scan creators: `ScanLoader`, `ScanCreator`, `HoloScanCreator` and `ScanKey`.
The following code block shows examples:
~~~~~~{.py}
{
"project": {
// ...
"scans": [
{
"__class__": "ScanCreator",
"filename": "twoatom_energy_alpha.etpai",
"emitter": "N",
"initial_state": "1s",
"positions": {
"e": "np.arange(10, 400, 5)",
"t": "0",
"p": "0",
"a": "np.linspace(-30, 30, 31)"
}
},
{
"__class__": "ScanLoader",
"filename": "${project}/twoatom_hemi_250e.etpi",
"emitter": "N",
"initial_state": "1s",
"is_modf": false
},
{
"__class__": "HoloScanCreator",
"filename": "${project}/twoatom_scan3.etpi",
"emitter": "N",
"initial_state": "1s",
"generator": "pmsco.data.holo_grid",
"generator_args": {
"theta_start": 90,
"theta_step": 1,
"theta_range": 90,
"phi_start": 0,
"phi_range": 360,
"phi_refinement": 1
},
"other_positions": {"e": 250, "a": 0}
},
{
"__class__": "HoloScanCreator",
"filename": "${project}/twoatom_scan4.etpi",
"emitter": "N",
"initial_state": "1s",
"other_positions": {"e": 250, "a": 0}
}
]
}
}
~~~~~~
The class name must be specified as it would be called in the custom project module.
For the example shown above, the following import statements are necessary in the `pmsco.projects.twoatom.py` module.
(Other forms of the import statement can be used accordingly.)
~~~~~~{.py}
import numpy as np
import pmsco.data
from pmsco.scan import ScanKey, ScanLoader, ScanCreator, HoloScanCreator
~~~~~~
The *ScanCreator* object creates a scan using Numpy array constructors in `positions`.
In the example above, a two-dimensional rectangular energy-alpha scan grid is created.
The values of the positions axes are passed to Python's `eval` function
and must return a one-dimensional Numpy `ndarray`.
The `emitter` and `initial_state` keys define the probed core level.
The *HoloScanCreator* object creates a /holo scan/, i.e., an angle scan of the theta and phi axes.
The distribution of the grid points is defined by a separate generator function.
Usually, the default pmsco.data.holo_grid function is used
which generates the well-known Osterwalder holo scan
with constant point density in solid angle and equidistant polar steps.
The `generator` and `generator_args` properties have default values.
The two example holo scans above are equivalent,
as the first one above just uses default values explicitly.
If you want to specify a generator function explicitly,
you must import it into the namespace of your project.
E.g. for `pmsco.data.holo_grid` you have to `import pmsco.data`.
The *ScanLoader* object loads a data file, specified under `filename`.
The filename can include a placeholder which is replaced by the corresponding item from Project.directories.
Note that some of the directories (including `project`) are pre-set by PMSCO.
It is recommended to add a `data` key under `directories` in the run-file
if the data files are outside of the PMSCO directory tree.
The `is_modf` key indicates whether the file contains a modulation function (`true`) or intensity (`false`).
In the latter case, the modulation function is calculated after loading.
The *ScanKey* is the shortest scan specification in the run-file.
It should not be used in new projects as it uses hard-coded data links in program code.
ScanKey is a shortcut to a complete scan dictionary in the project object.
The `scan_dict` must be set up in the `__init__` method of the project class.
The `key` item specifies which key of `scan_dict` should be used to create the Scan object.
Each item of `scan_dict` holds a dictionary
that holds the attributes for either a `ScanCreator`, `HoloScanCreator` or a `ScanLoader`.
If it contains a `positions` (`other_positions`) key, it represents a `ScanCreator` (`HoloScanCreator`), else a `ScanLoader`.
\subsection sec_runfile_optimizer Optimizer Parameters
The `optimizer_params` is a dictionary holding one or more of the following items.
| Key | Values | Description |
| --- | --- | --- |
| pop_size | integer<br>The default value is the greater of 4 or the number of parallel calculation processes. | Population size (number of particles) in swarm and genetic optimization mode. |
| seed_file | file system path | Name of the population seed file. Population data of previous optimizations can be used to seed a new optimization. The file must have the same structure as the .pop or .dat files. See @ref pmsco.optimizers.population.Population.seed_from_file. |
| seed_limit | integer | Number of seed models to import. |
| recalc_seed | true or false | If true, the seed models are calculated. Otherwise, the R-factor from the seed file is used as result. Use true if the seed file contains no or outdated R-factors. |
| table_source | file system path | Name of the model table file in table scan mode. |
\subsubsection sec_runfile_files File Categories
The following category names can be used with the `keep_files` option.
Multiple names can be specified as a list.
| Category | Description | Default Action |
| --- | --- | --- |
| all | shortcut to include all categories | |
| input | raw input files for calculator, including cluster and phase files in custom format | delete |
| output | raw output files from calculator | delete |
| atomic | atomic scattering and emission files in portable format | delete |
| cluster | cluster files in portable XYZ format for report | keep |
| debug | debug files | delete |
| model | output files in ETPAI format: complete simulation (a_-1_-1_-1_-1) | keep |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | keep |
| domain | output files in ETPAI format: domain (a_b_c_-1_-1) | delete |
| emitter | output files in ETPAI format: emitter (a_b_c_d_-1) | delete |
| region | output files in ETPAI format: region (a_b_c_d_e) | delete |
| report| final report of results | keep always |
| population | final state of particle population | keep |
| rfac | files related to models which give bad r-factors, see warning below | delete |
\note
The `report` category is always kept and cannot be turned off.
The `model` category is always kept in single calculation mode.
\warning
If you want to specify `rfac` with the `keep_files` option,
you have to add the file categories that you want to keep, e.g.,
`"keep_files": ["rfac", "cluster", "model", "scan", "population"]`
(to return the default categories for all calculated models).
Do not specify `rfac` alone as this will effectively not return any file.
\subsection sec_runfile_reports Reports
Run-time graphical reports are configured in the `reports` section.
The section is organized as a list of dictionaries.
Each dictionary sets up a specific report.
For example:
~~~~~~{.py}
{
"project": {
// ...
"reports": [
{
"__class__": "ConvergencePlot",
"filename_format": "${base}.convergence",
"title_format": "my_calc"
},
{
"__class__": "SwarmPlot",
"filename_format": "${base}-${param0}-${param1}-${gen}.swarmplot",
"title_format": "my_calc ${param0}-${param1} gen ${gen}",
"params": [["A", "B"], ["C", "D"]]
}
]
}
}
~~~~~~
The class name must be specified as it would be called in the custom project module.
For the example above, the import section of the project must include:
~~~~~~{.py}
from pmsco.reports.convergence import ConvergencePlot
from pmsco.reports.swarm import SwarmPlot
~~~~~~
For details on reports and their configuration, see @ref sec_reports.
\subsection sec_runfile_schedule Job Scheduling
To submit a job to a resource manager such as Slurm, add a `schedule` section to the run file
(section ordering is not important):
~~~~~~{.py}
{
"schedule": {
"__module__": "pmsco.schedule",
"__class__": "PsiRaSchedule",
"nodes": 1,
"tasks_per_node": 24,
"walltime": "2:00",
"manual_run": true,
"enabled": true
},
"project": {
"__module__": "projects.twoatom.twoatom",
"__class__": "TwoatomProject",
"mode": "single",
"output_file": "${home}/pmsco/twoatom0001",
...
}
}
~~~~~~
In the same way as for the project, the `__module__` and `__class__` keys select the class that handles the job submission.
In this example, it is pmsco.schedule.PsiRaSchedule which is tied to the Ra cluster at PSI.
For other machines, you can sub-class one of the classes in the pmsco.schedule module and include it in your project package.
The derived job submission class must prepare the code, run file and job script, and submit the job to the queue.
It should copy the code to the calculation directory to avoid version conflicts if the user continues to edit the code.
Compilation of the code can be done before submission or as a part of the job script.
@note It is difficult to check the run file and code against errors that may abort job execution.
New code and run files should be tested with a modified, fast-running calculation.
The parameters of pmsco.schedule.PsiRaSchedule are as follows.
Information about the computing nodes and partitions can be printed by the `sinfo -Nel` and `sinfo --long` commands.
| Key | Values | Description |
| --- | --- | --- |
| nodes | integer: 1..2 | Number of compute nodes (main boards on Ra). The maximum number available for PEARL is 2. |
| tasks_per_node | integer: 1..24, 32 | Number of tasks (CPU cores on Ra) per node. Jobs with less than 24 tasks are assigned to the shared partition. |
| wall_time | string: [days-]hours[:minutes[:seconds]] <br> dict: with any combination of days, hours, minutes, seconds | Maximum run time (wall time) of the job. |
| manual | bool | Manual submission (true) or automatic submission (false). Manual submission allows you to inspect the job files before submission. |
| enabled | bool | Enable scheduling (true). Otherwise, the calculation is started directly (false). |
@note The calculation job may run in a different working directory than the current one.
It is important to specify absolute data and output directories in the run file (project/directories section).
Placeholders like `${home}` can be used to make run files portable, cf. @ref sec_run_dirs.
*/

22
docs/src/tasks.dot
View File

@@ -38,15 +38,15 @@ custom_scan [label="scan\nconfiguration", shape=note];
{rank=same; custom_scan; create_scan; combine_scan;}
custom_scan -> create_scan [lhead=cluster_scan];
subgraph cluster_symmetry {
label="symmetry handler";
subgraph cluster_domain {
label="domain handler";
rank=same;
create_symmetry [label="define\nsymmetry\ntasks"];
combine_symmetry [label="gather\nsymmetry\nresults"];
create_model_space [label="define\ndomain\ntasks"];
combine_domain [label="gather\ndomain\nresults"];
}
custom_symmetry [label="symmetry\ndefinition", shape=cds];
{rank=same; create_symmetry; combine_symmetry; custom_symmetry;}
custom_symmetry -> combine_symmetry [lhead=cluster_symmetry];
custom_domain [label="domain\ndefinition", shape=cds];
{rank=same; create_model_space; combine_domain; custom_domain;}
custom_domain -> combine_domain [lhead=cluster_domain];
subgraph cluster_emitter {
label="emitter handler";
@@ -80,11 +80,11 @@ create_cluster -> edac;
create_model -> create_scan [label="level 1 tasks"];
evaluate_model -> combine_scan [label="level 1 results", dir=back];
create_scan -> create_symmetry [label="level 2 tasks"];
combine_scan -> combine_symmetry [label="level 2 results", dir=back];
create_scan -> create_model_space [label="level 2 tasks"];
combine_scan -> combine_domain [label="level 2 results", dir=back];
create_symmetry -> create_emitter [label="level 3 tasks"];
combine_symmetry -> combine_emitter [label="level 3 results", dir=back];
create_model_space -> create_emitter [label="level 3 tasks"];
combine_domain -> combine_emitter [label="level 3 results", dir=back];
create_emitter -> create_region [label="level 4 tasks"];
combine_emitter -> combine_region [label="level 4 results", dir=back];

2
docs/src/uml/CalculationTask-class.puml
View File

@@ -28,7 +28,7 @@ remove_task_file()
class CalcID {
model
scan
sym
domain
emit
region
}

24
docs/src/uml/CalculationTask-objects.puml
View File

@@ -43,15 +43,15 @@ parent = 2, -1, -1, -1, -1
model = {'d': 7}
}
Scan11 o.. Sym111
Scan11 o.. Dom111
object Sym111 {
object Dom111 {
id = 1, 1, 1, -1, -1
parent = 1, 1, -1, -1, -1
model = {'d': 5}
}
Sym111 o.. Emitter1111
Dom111 o.. Emitter1111
object Emitter1111 {
id = 1, 1, 1, 1, -1
@@ -90,18 +90,18 @@ scan
object ScanHandler
object "Sym: CalculationTask" as Sym {
object "Domain: CalculationTask" as Domain {
model
scan
symmetry
domain
}
object "SymmetryHandler" as SymHandler
object "DomainHandler" as DomainHandler
object "Emitter: CalculationTask" as Emitter {
model
scan
symmetry
domain
emitter
}
@@ -110,7 +110,7 @@ object EmitterHandler
object "Region: CalculationTask" as Region {
model
scan
symmetry
domain
emitter
region
}
@@ -120,14 +120,14 @@ object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Sym
Sym "1" o.. "1..*" Emitter
Scan "1" o.. "1..*" Domain
Domain "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Sym) .. SymHandler
(Sym, Emitter) .. EmitterHandler
(Scan, Domain) .. DomainHandler
(Domain, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml

10
docs/src/uml/calculation-task.puml
View File

@@ -4,7 +4,7 @@
class CalculationTask {
model
scan
symmetry
domain
emitter
region
..
@@ -35,7 +35,7 @@ class Scan {
alphas
}
class Symmetry {
class Domain {
index
..
rotation
@@ -55,13 +55,13 @@ class Region {
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Symmetry
CalculationTask *-- Domain
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
symmetries
domains
model_handler
cluster_generator
}
@@ -78,7 +78,7 @@ class ModelHandler {
Model ..> ModelHandler
Scan ..> Project
Symmetry ..> Project
Domain ..> Project
Emitter ..> ClusterGenerator
Region ..> Project

32
docs/src/uml/database.puml
View File

@@ -9,14 +9,6 @@ name
code
}
class Scan << (T,orchid) >> {
id
..
job_id
..
name
}
class Job << (T,orchid) >> {
id
..
@@ -30,6 +22,22 @@ datetime
description
}
class Tag << (T,orchid) >> {
id
..
..
key
}
class JobTag << (T,orchid) >> {
id
..
tag_id
job_id
..
value
}
class Model << (T,orchid) >> {
id
..
@@ -46,10 +54,12 @@ id
model_id
..
scan
sym
domain
emit
region
rfac
timestamp
secs
}
class Param << (T,orchid) >> {
@@ -66,11 +76,13 @@ param_id
model_id
..
value
delta
}
Project "1" *-- "*" Job
Job "1" *-- "*" JobTag
Tag "1" *-- "*" JobTag
Job "1" *-- "*" Model
Job "1" *-- "*" Scan
Param "1" *-- "*" ParamValue
Model "1" *-- "*" ParamValue
Model "1" *-- "*" Result

4
docs/src/uml/handler-activity.puml
View File

@@ -20,7 +20,7 @@ repeat
partition "generate tasks" {
:define model tasks;
:define scan tasks;
:define symmetry tasks;
:define domain tasks;
:define emitter tasks;
:define region tasks;
}
@@ -34,7 +34,7 @@ end fork
partition "collect results" {
:gather region results;
:gather emitter results;
:gather symmetry results;
:gather domain results;
:gather scan results;
:gather model results;
}

6
docs/src/uml/minimum-project-classes.puml
View File

@@ -5,10 +5,10 @@ package pmsco {
mode
code
scans
symmetries
domains
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_domain()
{abstract} create_model_space()
}
}
@@ -18,7 +18,7 @@ package projects {
__init__()
create_cluster()
create_params()
create_domain()
create_model_space()
}
}

12
docs/src/uml/project-classes.puml
View File

@@ -4,13 +4,13 @@ abstract class Project {
mode : str = "single"
code : str = "edac"
scans : Scan [1..*]
symmetries : dict [1..*]
domains : dict [1..*]
cluster_generator : ClusterGenerator
handler_classes
files : FileTracker
{abstract} create_cluster() : Cluster
{abstract} create_params() : Params
{abstract} create_domain() : Domain
{abstract} create_params() : CalculatorParams
{abstract} create_model_space() : ModelSpace
}
class Scan {
@@ -25,10 +25,10 @@ class Scan {
thetas
phis
alphas
set_scan()
import_scan_file()
}
class Domain {
class ModelSpace {
start : dict
min : dict
max : dict
@@ -37,7 +37,7 @@ class Domain {
get_param(name)
}
class Params {
class CalculatorParams {
title
comment
cluster_file

86
docs/src/uml/scan-classes.puml Normal file
View File

@@ -0,0 +1,86 @@
@startuml
'https://plantuml.com/class-diagram
class ConfigurableObject
class Scan {
filename: str
raw_data: numpy.ndarray
dtype: numpy.dtype
modulation: numpy.ndarray
modulation_func: Callable
modulation_args: Dict
rfactor_func: Callable
rfactor_args: Dict
mode: List
emitter: str
initial_state: str
positions: Dict
__init__()
copy()
load()
define_scan()
import_scan_file()
analyse_raw_data()
generate_holo_scan()
}
class ScanSpec {
filename: str
emitter: str
initial_state: str
modulation_func: Callable
modulation_args: Dict
rfactor_func: Callable
rfactor_args: Dict
__init__()
load()
}
class ScanKey {
project: pmsco.project.Project
key: str
__init__()
load()
}
class ScanLoader {
is_modf: bool
patch: Dict
__init__()
load()
}
class ScanCreator {
positions: Dict
__init__()
load()
}
class HoloScanCreator {
generator: Callable
generator_args: Dict
__init__()
load()
set_property()
}
ConfigurableObject <|-- ScanSpec
ConfigurableObject <|-- ScanKey
ScanSpec <|-- ScanCreator
ScanSpec <|-- ScanLoader
ScanSpec <|-- HoloScanCreator
ScanKey --> ScanCreator: creates
ScanKey --> HoloScanCreator: creates
ScanKey --> ScanLoader: creates
ScanLoader --> Scan: creates
ScanCreator --> Scan: creates
HoloScanCreator --> Scan: creates
@enduml

2
docs/src/uml/top-activity-partitions.puml
View File

@@ -25,7 +25,7 @@ stop
|pmsco|
start
:define task (model, scan, symmetry, emitter, region);
:define task (model, scan, domain, emitter, region);
|project|
:create cluster;
:create parameters;

10
docs/src/uml/top-components.puml
View File

@@ -2,21 +2,19 @@
skinparam componentStyle uml2
component "project" as project
component "PMSCO" as pmsco
component "project" as project
component "scattering code\n(calculator)" as calculator
interface "command line" as cli
interface "input files" as input
interface "output files" as output
interface "experimental data" as data
interface "results" as results
interface "output files" as output
cli --> pmsco
data -> project
project ..> pmsco
pmsco ..> project
pmsco ..> calculator
cli --> project
input -> calculator
calculator -> output
pmsco -> results

10
docs/src/uml/user-project-classes.puml
View File

@@ -5,16 +5,16 @@ package pmsco {
mode
code
scans
symmetries
domains
cluster_generator
handler_classes
__
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_domain()
{abstract} create_model_space()
..
combine_scans()
combine_symmetries()
combine_domains()
combine_emitters()
calc_modulation()
calc_rfactor()
@@ -34,9 +34,9 @@ package projects {
setup()
..
create_params()
create_domain()
create_model_space()
..
combine_symmetries()
combine_domains()
}
class UserClusterGenerator {

30
extras/docker-docs/Dockerfile Normal file
View File

@@ -0,0 +1,30 @@
FROM python:3.12
# docker container to build PMSCO documentation
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
default-jre \
doxygen \
gawk \
git \
graphviz \
pandoc \
wget \
&& rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir \
doxypypy \
meson \
meson-python \
ninja \
pynose
RUN wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
ENV PLANTUML_JAR_PATH=/app/plantuml.jar
COPY . .
CMD ["sh"]

118
extras/singularity/singularity_python2
View File

@@ -1,118 +0,0 @@
BootStrap: debootstrap
OSVersion: bionic
MirrorURL: http://ch.archive.ubuntu.com/ubuntu/
%help
a singularity container for PMSCO.
git clone requires an ssh key for git.psi.ch.
try agent forwarding (-A option to ssh).
#%setup
# executed on the host system outside of the container before %post
#
# this will be inside the container
# touch ${SINGULARITY_ROOTFS}/tacos.txt
# this will be on the host
# touch avocados.txt
#%files
# files are copied before %post
#
# this copies to root
# avocados.txt
# this copies to /opt
# avocados.txt /opt
#
# this does not work
# ~/.ssh/known_hosts /etc/ssh/ssh_known_hosts
# ~/.ssh/id_rsa /etc/ssh/id_rsa
%labels
Maintainer Matthias Muntwiler
Maintainer_Email matthias.muntwiler@psi.ch
Python_Version 2.7
%environment
export PATH="/usr/local/miniconda3/bin:$PATH"
export PYTHON_VERSION=2.7
export SINGULAR_BRANCH="singular"
export LC_ALL=C
%post
export PYTHON_VERSION=2.7
export LC_ALL=C
sed -i 's/$/ universe/' /etc/apt/sources.list
apt-get update
apt-get -y install \
binutils \
build-essential \
doxygen \
doxypy \
f2c \
g++ \
gcc \
gfortran \
git \
graphviz \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
sqlite3 \
swig \
wget
apt-get clean
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
bash ~/miniconda.sh -b -p /usr/local/miniconda3
export PATH="/usr/local/miniconda3/bin:$PATH"
conda create -q --yes -n pmsco python=${PYTHON_VERSION}
. /usr/local/miniconda3/bin/activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
future \
statsmodels
conda clean --all -y
/usr/local/miniconda3/envs/pmsco/bin/pip install periodictable attrdict fasteners
#%test
# test the image after build
%runscript
# executes command from command line
. /usr/local/miniconda3/bin/activate pmsco
exec echo "$@"
%apprun install
. /usr/local/miniconda3/bin/activate pmsco
cd ~
git clone https://git.psi.ch/pearl/pmsco.git pmsco
cd pmsco
git checkout develop
git checkout -b ${SINGULAR_BRANCH}
make all
nosetests
%apprun python
. /usr/local/miniconda3/bin/activate pmsco
exec python "${@}"
%apprun conda
. /usr/local/miniconda3/bin/activate pmsco
exec conda "${@}"

104
extras/singularity/singularity_python3
View File

@@ -1,12 +1,13 @@
BootStrap: debootstrap
OSVersion: bionic
OSVersion: focal
MirrorURL: http://ch.archive.ubuntu.com/ubuntu/
%help
a singularity container for PMSCO.
A singularity container for PMSCO.
git clone requires an ssh key for git.psi.ch.
try agent forwarding (-A option to ssh).
singularity run -e pmsco.sif path/to/pmsco -r path/to/your-runfile
path/to/pmsco must point to the directory that contains the __main__.py file.
#%setup
# executed on the host system outside of the container before %post
@@ -31,25 +32,28 @@ try agent forwarding (-A option to ssh).
%labels
Maintainer Matthias Muntwiler
Maintainer_Email matthias.muntwiler@psi.ch
Python_Version 3
Python_Version 3.8
%environment
export PATH="/usr/local/miniconda3/bin:$PATH"
export PYTHON_VERSION=3
export SINGULAR_BRANCH="singular"
export LC_ALL=C
export PYTHON_VERSION=3
export CONDA_ROOT=/opt/miniconda
export PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar
export SINGULAR_BRANCH="singular"
%post
export PYTHON_VERSION=3
export LC_ALL=C
export PYTHON_VERSION=3.8
export CONDA_ROOT=/opt/miniconda
export PLANTUML_ROOT=/opt/plantuml
sed -i 's/$/ universe/' /etc/apt/sources.list
apt-get update
apt-get -y install \
binutils \
build-essential \
default-jre \
doxygen \
doxypy \
f2c \
g++ \
gcc \
@@ -59,59 +63,75 @@ try agent forwarding (-A option to ssh).
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
sqlite3 \
swig \
wget
apt-get clean
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
bash ~/miniconda.sh -b -p /usr/local/miniconda3
export PATH="/usr/local/miniconda3/bin:$PATH"
wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -O ~/miniforge3.sh
bash ~/miniforge3.sh -b -p ${CONDA_ROOT}
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate base
conda create -q --yes -n pmsco python=${PYTHON_VERSION}
. /usr/local/miniconda3/bin/activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
conda install -q --yes -n pmsco -c conda-forge \
commentjson \
fasteners \
future \
statsmodels
gitpython
ipython \
ipykernel \
jsonschema \
h5py \
matplotlib \
meson \
mock \
pynose \
"numpy>=1.13" \
pandas \
periodictable \
pip \
scikit-learn \
scipy \
seaborn \
sqlalchemy \
statsmodels \
swig
conda clean --all -y
/usr/local/miniconda3/envs/pmsco/bin/pip install periodictable attrdict fasteners
${CONDA_ROOT}/envs/pmsco/bin/pip install meson-python mpi4py netgraph networkx doxypypy
mkdir ${PLANTUML_ROOT}
wget -O ${PLANTUML_ROOT}/plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
#%test
# test the image after build
%runscript
# executes command from command line
source /usr/local/miniconda3/bin/activate pmsco
exec echo "$@"
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
exec python "$@"
%apprun install
source /usr/local/miniconda3/bin/activate pmsco
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
cd ~
git clone https://git.psi.ch/pearl/pmsco.git pmsco
cd pmsco
git checkout develop
git checkout master
git checkout -b ${SINGULAR_BRANCH}
make all
nosetests
%apprun python
source /usr/local/miniconda3/bin/activate pmsco
exec python "${@}"
%apprun conda
source /usr/local/miniconda3/bin/activate pmsco
exec conda "${@}"
meson setup build
cd build
meson compile
meson install
meson test
%apprun compile
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
cd build
meson compile
meson install
meson test

4
extras/vagrant/Vagrantfile vendored
View File

@@ -12,8 +12,8 @@ Vagrant.configure("2") do |config|
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "singularityware/singularity-2.4"
config.vm.box_version = "2.4"
config.vm.box = "sylabs/singularity-3.7-ubuntu-bionic64"
config.vm.box_version = "3.7"
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs

55
makefile
View File

@@ -1,55 +0,0 @@
SHELL=/bin/sh
# makefile for all programs, modules and documentation
#
# required libraries for LOESS module: libblas, liblapack, libf2c
# (you may have to set soft links so that linker finds them)
#
# on shared computing systems (high-performance clusters)
# you may have to switch the environment before running this script.
#
# note: the public distribution does not include third-party code
# (EDAC in particular) because of incompatible license terms.
# please obtain such code from the original authors
# and copy it to the proper directory before compilation.
#
# the MSC and MUFPOT programs are currently not used.
# they are not built by the top-level targets all and bin.
#
# the make system uses the compiler executables of the current environment.
# to override the executables, you may set the following variables.
# to switch between python versions, however, the developers recommend miniconda.
#
# PYTHON = python executable (default: python)
# PYTHONOPTS = python options (default: none)
# CC = C and Fortran compiler executable (default: gcc)
# CCOPTS = C compiler options (default: none)
# CXX = C++ compiler executable (default: g++)
# CXXOPTS = C++ compiler options (default: none)
#
# make all PYTHON=/usr/bin/python2.7
#
# or:
#
# export PYTHON=/usr/bin/python2.7
# make all
#
.PHONY: all bin docs clean edac loess msc mufpot
PMSCO_DIR = pmsco
DOCS_DIR = docs
all: edac loess docs
bin: edac loess
edac loess msc mufpot:
$(MAKE) -C $(PMSCO_DIR)
docs:
$(MAKE) -C $(DOCS_DIR)
clean:
$(MAKE) -C $(PMSCO_DIR) clean
$(MAKE) -C $(DOCS_DIR) clean

0
pmsco/__init__.py
View File

13
pmsco/__main__.py
View File

@@ -8,16 +8,13 @@ python pmsco [pmsco-arguments]
@endverbatim
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from pathlib import Path
import sys
import os.path
file_dir = os.path.dirname(__file__) or '.'
root_dir = os.path.join(file_dir, '..')
root_dir = os.path.abspath(root_dir)
sys.path[0] = root_dir
pmsco_root = Path(__file__).resolve().parent.parent
if str(pmsco_root) not in sys.path:
sys.path.insert(0, str(pmsco_root))
if __name__ == '__main__':
import pmsco.pmsco

0
pmsco/calculators/__init__.py
View File

20
pmsco/calculators/calculator.py
View File

@@ -11,7 +11,7 @@ TestCalcInterface is provided for testing the PMSCO code quickly without calling
@author Matthias Muntwiler
@copyright (c) 2015-18 by Paul Scherrer Institut @n
@copyright (c) 2015-19 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
@@ -42,7 +42,7 @@ class Calculator(object):
or <code>output_file + '.etpai'</code> depending on scan mode.
all other intermediate files are deleted unless keep_temp_files is True.
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@@ -56,11 +56,25 @@ class Calculator(object):
@arg the first element is the name of the main ETPI or ETPAI result file to be further processed.
@arg the second element is a dictionary that lists the names of all created data files with their category.
the dictionary key is the file name,
the value is the file category (cluster, phase, etc.).
the value is the file category (cluster, atomic, etc.).
"""
return None, None
class AtomicCalculator(Calculator):
"""
abstract interface class to the atomic scattering calculation program.
"""
pass
class InternalAtomicCalculator(AtomicCalculator):
"""
dummy atomic scattering class if scattering factors are calculated internally by the multiple scattering calculator.
"""
pass
class TestCalculator(Calculator):
"""
interface class producing random data for testing the MSCO code without calling an external program.

79
pmsco/calculators/edac.py
View File

@@ -11,23 +11,23 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import numpy as np
import os
import pmsco.calculators.calculator as calculator
from pmsco.compat import open
import pmsco.data as md
import pmsco.cluster as mc
import pmsco.edac.edac as edac
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
import edac
except (ImportError, ModuleNotFoundError) as e:
edac = None
logger.critical("Error importing the edac package.", exc_info=e)
class EdacCalculator(calculator.Calculator):
def write_input_file(self, params, scan, filepath):
@@ -49,14 +49,17 @@ class EdacCalculator(calculator.Calculator):
if alpha is defined, theta is implicitly set to normal emission! (to be generalized)
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param filepath: (str) name and path of the file to be created.
@return dictionary of created files {filename: category}
"""
with open(filepath, "w") as f:
files = {}
with open(filepath, "wt", encoding="latin1") as f:
f.write("verbose off\n")
f.write("cluster input {0}\n".format(params.cluster_file))
f.write("emitters {0:d} l(A)\n".format(len(params.emitters)))
@@ -146,9 +149,16 @@ class EdacCalculator(calculator.Calculator):
scatterers = ["scatterer {at} {fi}\n".format(at=at, fi=fi)
for (at, fi) in params.phase_files.items()
if os.path.isfile(fi)]
if scatterers:
rme = ["rmat {fi}\n".format(fi=fi)
for (at, fi) in params.rme_files.items()
if at == params.emitters[0][3] and os.path.isfile(fi)] or \
["rmat inline 1 regular1 {l0} {pv} {pd} {mv} {md}\n".format(l0=params.l_init,
pv=params.rme_plus_value, pd=params.rme_plus_shift,
mv=params.rme_minus_value, md=params.rme_minus_shift)]
if scatterers and rme:
for scat in scatterers:
f.write(scat)
f.write(rme[0])
else:
f.write("muffin-tin\n")
@@ -162,21 +172,32 @@ class EdacCalculator(calculator.Calculator):
f.write("orders {0:d} ".format(len(params.orders)))
f.write(" ".join(format(order, "d") for order in params.orders) + "\n")
f.write("emission angle window {0:F}\n".format(params.angular_resolution / 2.0))
# f.write("cluster output l(A) out.clu")
# problems:
# - muffin-tin relabels atoms
# - there can be multiple atom types for the same chemical element
# - we have to compare coordinates to find the mapping between input and output cluster
# f.write("scan scatterer i phase-shifts i.pha")
# f.write("scan scatterer i potential i.pot")
# scattering factor output (see project.CalculatorParams.phase_output_classes)
if params.phase_output_classes is not None:
fn = "{0}.clu".format(params.output_file)
f.write("cluster output l(A) {fn}\n".format(fn=fn))
files[fn] = "output"
try:
cls = (cl for cl in params.phase_output_classes)
except TypeError:
cls = range(params.phase_output_classes)
for cl in cls:
fn = "{of}.{cl}.scat".format(cl=cl, of=params.output_file)
f.write("scan scatterer {cl} phase-shifts {fn}\n".format(cl=cl, fn=fn))
files[fn] = "output"
f.write("scan pd {0}\n".format(params.output_file))
files[params.output_file] = "output"
f.write("end\n")
return files
def run(self, params, cluster, scan, output_file):
"""
run EDAC with the given parameters and cluster.
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@@ -198,20 +219,22 @@ class EdacCalculator(calculator.Calculator):
dat_filename = out_filename
if params.fixed_cluster:
etpi_filename = base_filename + ".etpai"
dtype = md.DTYPE_ETPAI
else:
etpi_filename = base_filename + ".etpi"
dtype = md.DTYPE_ETPI
# fix EDAC particularities
params.cluster_file = clu_filename
params.output_file = out_filename
params.data_file = dat_filename
params.emitters = cluster.get_emitters()
params.emitters = cluster.get_emitters(['x', 'y', 'z', 'c'])
# save parameter files
logger.debug("writing cluster file %s", clu_filename)
cluster.save_to_file(clu_filename, fmt=mc.FMT_EDAC)
logger.debug("writing input file %s", par_filename)
self.write_input_file(params, scan, par_filename)
files = self.write_input_file(params, scan, par_filename)
# run EDAC
logger.info("calling EDAC with input file %s", par_filename)
@@ -225,13 +248,10 @@ class EdacCalculator(calculator.Calculator):
result_etpi['e'] -= params.work_function
if 't' in scan.mode and 'p' in scan.mode:
hemi_tpi = scan.raw_data.copy()
hemi_tpi['i'] = 0.0
try:
hemi_tpi['s'] = 0.0
except ValueError:
pass
result_etpi = md.interpolate_hemi_scan(result_etpi, hemi_tpi)
dest_tpi = np.zeros(scan.raw_data.shape, dtype)
dest_tpi['t'] = scan.thetas
dest_tpi['p'] = scan.phis
result_etpi = md.interpolate_hemi_scan(result_etpi, dest_tpi)
if params.fixed_cluster:
expected_shape = max(scan.energies.shape[0], 1) * max(scan.alphas.shape[0], 1)
@@ -244,6 +264,9 @@ class EdacCalculator(calculator.Calculator):
logger.debug("save result to file %s", etpi_filename)
md.save_data(etpi_filename, result_etpi)
files = {clu_filename: 'input', par_filename: 'input', dat_filename: 'output',
etpi_filename: 'region'}
files[clu_filename] = 'input'
files[par_filename] = 'input'
files[dat_filename] = 'output'
files[etpi_filename] = 'region'
return etpi_filename, files

56
pmsco/calculators/msc.py
View File

@@ -18,7 +18,7 @@ from __future__ import division
from __future__ import print_function
import pmsco.calculators.calculator as calculator
import pmsco.data as md
import pmsco.msc.msc as msc
import subprojects.msc.msc as msc
import logging
logger = logging.getLogger(__name__)
@@ -27,42 +27,42 @@ logger = logging.getLogger(__name__)
class MscCalculator(calculator.Calculator):
def write_input_file(self, params, filepath):
with open(filepath, "w") as f:
f.write(" %s\n" % (params.title) )
f.write(" %s\n" % (params.comment) )
f.write(" %s\n" % (params.title))
f.write(" %s\n" % (params.comment))
l_init = "spdf".index(params.initial_state[1])
f.write(" %4u\n" % (l_init) )
f.write(" %4u\n" % (params.spherical_order) )
f.write(" %s\n" % (params.polarization) )
f.write(" %4u\n" % (params.scattering_level) )
f.write(" %7.2f%7.2f\n" % (params.fcut, params.cut) )
f.write(" %12.6f\n" % (params.angular_resolution) )
f.write(" %12.6f\n" % (params.lattice_constant) )
f.write(" %12.6f\n" % (params.z_surface) )
f.write(" %4u\n" % (params.atom_types) )
f.write(" %4u\n" % (l_init))
f.write(" %4u\n" % (params.spherical_order))
f.write(" %s\n" % (params.polarization))
f.write(" %4u\n" % (params.scattering_level))
f.write(" %7.2f%7.2f\n" % (params.fcut, params.cut))
f.write(" %12.6f\n" % (params.angular_resolution))
f.write(" %12.6f\n" % (params.lattice_constant))
f.write(" %12.6f\n" % (params.z_surface))
f.write(" %4u\n" % (params.atom_types))
for iat in range(params.atom_types):
f.write(" %4u %s\n" % (params.atomic_number[iat], "..."))
f.write(" %4u %s\n" % (params.atomic_number[iat], params.phase_file[iat]))
f.write(" %12.6f\n" % (params.msq_displacement[iat]) )
f.write(" %12.6f\n" % (params.planewave_attenuation) )
f.write(" %12.6f\n" % (params.inner_potential) )
f.write(" %12.6f\n" % (params.symmetry_range) )
f.write(" %12.6f\n" % (params.polar_incidence_angle) )
f.write(" %12.6f\n" % (params.azimuthal_incidence_angle) )
f.write(" %s\n" % (params.vibration_model) )
f.write(" %12.6f\n" % (params.substrate_atomic_mass) )
f.write(" %12.6f\n" % (params.experiment_temperature) )
f.write(" %12.6f\n" % (params.debye_temperature) )
f.write(" %12.6f\n" % (params.debye_wavevector) )
f.write(" %12.6f%7.3f\n" % (params.rme_minus_value, params.rme_minus_shift) )
f.write(" %12.6f%7.3f\n" % (params.rme_plus_value, params.rme_plus_shift) )
f.write(" %4u\n" % (1) )
f.write(" %4u %12.6f\n" % (1, 1.0) )
f.write(" %12.6f\n" % (params.msq_displacement[iat]))
f.write(" %12.6f\n" % (params.planewave_attenuation))
f.write(" %12.6f\n" % (params.inner_potential))
f.write(" %12.6f\n" % (params.symmetry_range))
f.write(" %12.6f\n" % (params.polar_incidence_angle))
f.write(" %12.6f\n" % (params.azimuthal_incidence_angle))
f.write(" %s\n" % (params.vibration_model))
f.write(" %12.6f\n" % (params.substrate_atomic_mass))
f.write(" %12.6f\n" % (params.experiment_temperature))
f.write(" %12.6f\n" % (params.debye_temperature))
f.write(" %12.6f\n" % (params.debye_wavevector))
f.write(" %12.6f%7.3f\n" % (params.rme_minus_value, params.rme_minus_shift))
f.write(" %12.6f%7.3f\n" % (params.rme_plus_value, params.rme_plus_shift))
f.write(" %4u\n" % (1))
f.write(" %4u %12.6f\n" % (1, 1.0))
def run(self, params, cluster, scan, output_file):
"""
run the MSC program with the given parameters and cluster.
@param params: a project.Params() object with all necessary values except cluster and output files set.
@param params: a project.CalculatorParams() object with all necessary values except cluster and output files set.
@param cluster: a cluster.Cluster(format=FMT_MSC) object with all atom positions set.

173
pmsco/calculators/phagen/runner.py Normal file
View File

@@ -0,0 +1,173 @@
"""
@package pmsco.calculators.phagen.runner
Natoli/Sebilleau PHAGEN interface
This module runs the PHAGEN program to calculate scattering factors and radial matrix elements.
Requires PHAGEN version 2.2 from https://git.ipr.univ-rennes.fr/epsi/msspec_python3.git (contained in subprojects).
@author Matthias Muntwiler
@copyright (c) 2015-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import os
import shutil
import tempfile
from pathlib import Path
import sys
from pmsco.calculators.calculator import AtomicCalculator
from pmsco.calculators.phagen.translator import Translator
import pmsco.cluster
from pmsco.helpers import stdout_redirected
import pmsco.project
logger = logging.getLogger(__name__)
try:
import phagen
except (ImportError, ModuleNotFoundError) as e:
phagen = None
logger.critical("Error importing the phagen package.", exc_info=e)
class PhagenCalculator(AtomicCalculator):
"""
use the PHAGEN program to calculate scattering factors and radial matrix element.
this produces scatterer, radial matrix element and cluster files for EDAC.
"""
def run(self,
params: pmsco.project.CalculatorParams,
cluster: pmsco.cluster.Cluster,
scan: pmsco.project.Scan,
output_file: str):
"""
create the input file, run PHAGEN, and translate the output to EDAC format.
the following files are created in the job work directory:
- scattering factor files in EDAC format.
their names are `output_file + "_{atomclass}.scat"`.
- radial matrix element file in EDAC format.
its name is `output_file + ".rme"`.
- cluster file in PMSCO format.
its name is `output_file + ".clu"`.
the cluster and params objects are updated and linked to the scattering files
so that they can be passed to EDAC without further modification.
the radial matrix element is currently not used.
note that the scattering files are numbered according to the atomic environment and not chemical element.
this means that the updated cluster (cluster object or ".clu" file)
must be used in the scattering calculation.
atomic index is not preserved - atoms in the input and output clusters can only be related by coordinate!
because PHAGEN generates a lot of files with hard-coded names,
the function creates a temporary directory for PHAGEN and deletes it before returning.
@param params: pmsco.project.CalculatorParams object.
the phase_files attribute is updated with the paths of the scattering files.
@param cluster: pmsco.cluster.Cluster object.
the cluster is updated with the one returned from PHAGEN.
the atom classes are linked to the scattering files.
@param scan: pmsco.project.Scan object.
the scan object is used to determine the kinetic energy range.
@param output_file: base path and name of the output files.
@return (None, dict) where dict is a list of output files with their category.
the category is "atomic" for all output files.
"""
assert cluster.get_emitter_count() == 1, "PHAGEN cannot handle more than one emitter at a time"
transl = Translator()
transl.params.set_params(params)
transl.params.set_cluster(cluster)
transl.params.set_scan(scan)
phagen_cluster = pmsco.cluster.Cluster()
files = {}
prev_wd = Path.cwd()
try:
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
in_path = temp_path / "input"
in_path.mkdir(exist_ok=True)
out_path = temp_path / "output"
out_path.mkdir(exist_ok=True)
infile = in_path / "input.ms"
try:
transl.write_input(infile)
report_infile = os.path.join(prev_wd, output_file + ".phagen.in")
shutil.copy(infile, report_infile)
files[report_infile] = "input"
except IOError:
logger.warning("error writing phagen input file {fi}.".format(fi=infile))
report_listfile = os.path.join(prev_wd, output_file + ".phagen.list")
files[report_listfile] = "log"
# call phagen, redirect stdout (unit 6)
os.chdir(out_path)
with open(report_listfile, "wb") as f:
with stdout_redirected(f):
phagen.phagen()
phafile = out_path / "div" / "phases.dat"
radfile = out_path / "fort.55"
# tlfile = out_path / "fort.35"
clufile = out_path / "clus" / "clus.out"
# collect results
try:
transl.parse_phagen_phase(phafile)
report_phafile = os.path.join(prev_wd, output_file + ".phagen.pha")
shutil.copy(phafile, report_phafile)
files[report_phafile] = "output"
except IOError:
logger.error("error loading phagen phase file {fi}".format(fi=phafile))
try:
transl.parse_radial_file(radfile)
report_radfile = os.path.join(prev_wd, output_file + ".phagen.rad")
shutil.copy(radfile, report_radfile)
files[report_radfile] = "output"
except IOError:
logger.error("error loading phagen radial file {fi}".format(fi=radfile))
try:
phagen_cluster.load_from_file(clufile, pmsco.cluster.FMT_PHAGEN_OUT)
except IOError:
logger.error("error loading phagen cluster file {fi}".format(fi=clufile))
finally:
os.chdir(prev_wd)
# write edac files
scatfile = output_file + "_{}.scat"
scatfiles = transl.write_edac_scattering(scatfile)
params.phase_files = scatfiles.copy()
files.update({f: "atomic" for f in params.phase_files.values()})
rmefile = output_file + "_{}.rme"
rmefiles = transl.write_edac_emission(rmefile)
params.rme_files = rmefiles.copy()
files.update({f: "atomic" for f in params.rme_files.values()})
cluster.update_atoms(phagen_cluster, {'c'})
clufile = output_file + ".pmsco.clu"
cluster.save_to_file(clufile, pmsco.cluster.FMT_PMSCO)
files[clufile] = "cluster"
return None, files

503
pmsco/calculators/phagen/translator.py Normal file
View File

@@ -0,0 +1,503 @@
"""
@package pmsco.calculators.phagen.translator
Natoli/Sebilleau PHAGEN interface
this module provides conversion between input/output files of PHAGEN and EDAC.
@author Matthias Muntwiler
@copyright (c) 2015-19 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import numpy as np
from pmsco.cluster import Cluster
logger = logging.getLogger(__name__)
## rydberg energy in electron volts
ERYDBERG = 13.6056923
def state_to_edge(state):
"""
translate spectroscopic notation to edge notation.
@param state: spectroscopic notation: "1s", "2s", "2p1/2", etc.
@return: edge notation: "k", "l1", "l2", etc.
note: if the j-value is not given, the lower j edge is returned.
"""
jshells = ['s', 'p1/2', 'p3/2', 'd3/2', 'd5/2', 'f5/2', 'f7/2']
lshells = [s[0] for s in jshells]
shell = int(state[0])
try:
subshell = jshells.index(state[1:]) + 1
except ValueError:
subshell = lshells.index(state[1]) + 1
except IndexError:
subshell = 1
edge = "klmnop"[shell-1]
if shell > 1:
edge += str(subshell)
return edge
class TranslationParams(object):
"""
project parameters needed for translation.
energy unit is eV.
"""
def __init__(self):
self.initial_state = "1s"
self.binding_energy = 0.
self.cluster = None
self.kinetic_energies = np.empty(0, dtype=float)
@property
def l_init(self):
return "spdf".index(self.initial_state[1])
@property
def edge(self):
return state_to_edge(self.initial_state)
def set_params(self, params):
"""
set the translation parameters.
@param params: a pmsco.project.CalculatorParams object or
a dictionary containing some or all public fields of this class.
@return: None
"""
try:
self.initial_state = params.initial_state
self.binding_energy = params.binding_energy
except AttributeError:
for key in params:
self.__setattr__(key, params[key])
def set_scan(self, scan):
"""
set the scan parameters.
@param scan: a pmsco.project.Scan object
@return: None
"""
try:
energies = scan.energies
except AttributeError:
try:
energies = scan['e']
except KeyError:
energies = scan
if not isinstance(energies, np.ndarray):
energies = np.array(energies)
self.kinetic_energies = np.resize(self.kinetic_energies, energies.shape)
self.kinetic_energies = energies
def set_cluster(self, cluster):
"""
set the initial cluster.
@param cluster: a pmsco.cluster.Cluster object
@return: None
"""
self.cluster = cluster
class Translator(object):
"""
data conversion to/from phagen input/output files.
usage:
1. set the translation parameters self.params.
2. call write_input_file to create the phagen input files.
3. call phagen on the input file.
4. call parse_phagen_phase.
5. call parse_radial_file.
6. call write_edac_scattering to produce the EDAC scattering matrix files.
7. call write_edac_emission to produce the EDAC emission matrix file.
"""
## @var params
#
# project parameters needed for translation.
#
# fill the attributes of this object before using any translator methods.
## @var scattering
#
# t-matrix storage
#
# the t-matrix is stored in a flat, one-dimensional numpy structured array consisting of the following fields:
# @arg e (float) energy (eV)
# @arg a (int) atom index (1-based)
# @arg l (int) angular momentum quantum number l
# @arg t (complex) scattering matrix element, t = exp(-i * delta) * sin delta
#
# @note PHAGEN uses the convention t = exp(-i * delta) * sin delta,
# whereas EDAC uses t = exp(i * delta) * sin delta (complex conjugate).
# this object stores the t-matrix according to the PHAGEN convention.
# the conversion to the EDAC convention occurs in write_edac_scattering_file().
## @var emission
#
# radial matrix element storage
#
# the radial matrix elemnts are stored in a flat, one-dimensional numpy structured array
# consisting of the following fields:
# @arg e (float) energy (eV)
# @arg dw (complex) matrix element for the transition to l-1
# @arg up (complex) matrix element for the transition to l+1
## @var cluster
#
# cluster object for PHAGEN
#
# this object is created by translate_cluster().
def __init__(self):
"""
initialize the object instance.
"""
self.params = TranslationParams()
dt = [('e', 'f4'), ('a', 'i4'), ('l', 'i4'), ('t', 'c16')]
self.scattering = np.empty(0, dtype=dt)
dt = [('e', 'f4'), ('dw', 'c16'), ('up', 'c16')]
self.emission = np.empty(0, dtype=dt)
self.cluster = None
def translate_cluster(self):
"""
translate the cluster into a form suitable for PHAGEN.
specifically, move the (first and hopefully only) emitter to the first atom position.
the method copies the cluster from self.params into a new object
and stores it under self.cluster.
@return: None
"""
self.cluster = Cluster()
self.cluster.copy_from(self.params.cluster)
ems = self.cluster.get_emitters(['i'])
self.cluster.move_to_first(idx=ems[0][0]-1)
def write_cluster(self, f):
"""
write the cluster section of the PHAGEN input file.
@param f: file or output stream (an object with a write method)
@return: None
"""
for atom in self.cluster.data:
d = {k: atom[k] for k in atom.dtype.names}
f.write("{s} {t} {x} {y} {z}\n".format(**d))
f.write("-1 -1 0. 0. 0.\n")
def write_ionicity(self, f):
"""
write the ionicity section of the PHAGEN input file.
ionicity is read from the 'q' column of the cluster.
all atoms of a chemical element must have the same charge state
because ionicity has to be specified per element.
this function writes the average of all charge states of an element.
@param f: file or output stream (an object with a write method)
@return: None
"""
data = self.cluster.data
elements = np.unique(data['t'])
for element in elements:
idx = np.where(data['t'] == element)
charge = np.mean(data['q'][idx])
f.write("{t} {q}\n".format(t=element, q=charge))
f.write("-1\n")
def write_input(self, f):
"""
write the PHAGEN input file.
@param f: file path or output stream (an object with a write method).
@return: None
"""
phagen_params = {}
self.translate_cluster()
phagen_params['absorber'] = 1
phagen_params['emin'] = self.params.kinetic_energies.min() / ERYDBERG
phagen_params['emax'] = self.params.kinetic_energies.max() / ERYDBERG
if self.params.kinetic_energies.shape[0] > 1:
phagen_params['delta'] = (phagen_params['emax'] - phagen_params['emin']) / \
(self.params.kinetic_energies.shape[0] - 1)
else:
phagen_params['delta'] = 0.1
phagen_params['edge'] = state_to_edge(self.params.initial_state)
phagen_params['edge1'] = 'm4' # auger not supported
phagen_params['edge2'] = 'm4' # auger not supported
phagen_params['cip'] = self.params.binding_energy / ERYDBERG
if phagen_params['cip'] < 0.001:
raise ValueError("binding energy parameter is zero.")
if np.sum(np.abs(self.cluster.data['q'])) > 0.:
phagen_params['ionzst'] = 'ionic'
else:
phagen_params['ionzst'] = 'neutral'
if hasattr(f, "write") and callable(f.write):
f.write("&job\n")
f.write("calctype='xpd',\n")
f.write("coor='angs',\n")
f.write("cip={cip},\n".format(**phagen_params))
f.write("absorber={absorber},\n".format(**phagen_params))
f.write("edge='{edge}',\n".format(**phagen_params))
f.write("edge1='{edge1}',\n".format(**phagen_params))
f.write("edge2='{edge1}',\n".format(**phagen_params))
f.write("gamma=0.03,\n")
f.write("lmax_mode=2,\n")
f.write("lmaxt=50,\n")
f.write("emin={emin},\n".format(**phagen_params))
f.write("emax={emax},\n".format(**phagen_params))
f.write("delta={delta},\n".format(**phagen_params))
f.write("potgen='in',\n")
f.write("potype='hedin',\n")
f.write("norman='stdcrm',\n")
f.write("ovlpfac=0.0,\n")
f.write("ionzst='{ionzst}',\n".format(**phagen_params))
f.write("charelx='ex',\n")
f.write("l2h=4\n")
f.write("&end\n")
f.write("comment 1\n")
f.write("comment 2\n")
f.write("\n")
self.write_cluster(f)
self.write_ionicity(f)
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_input(fi)
def parse_phagen_phase(self, f):
"""
parse the phase output file from PHAGEN.
the phase file is written to div/phases.dat.
it contains the following columns:
@arg e energy (Ry)
@arg x1 unknown 1
@arg x2 unknown 2
@arg na atom index (1-based)
@arg nl angular momentum quantum number l
@arg tr real part of the scattering matrix element
@arg ti imaginary part of the scattering matrix element
@arg ph phase shift
the data is translated into the self.scattering array.
@arg e energy (eV)
@arg a atom index (1-based)
@arg l angular momentum quantum number l
@arg t complex scattering matrix element
@note PHAGEN uses the convention t = exp(-i * delta) * sin delta,
whereas EDAC uses t = exp(i * delta) * sin delta (complex conjugate).
this class stores the t-matrix according to the PHAGEN convention.
the conversion to the EDAC convention occurs in write_edac_scattering_file().
@param f: file or path (any file-like or path-like object that can be passed to numpy.genfromtxt).
@return: None
"""
dt = [('e', 'f4'), ('x1', 'f4'), ('x2', 'f4'), ('na', 'i4'), ('nl', 'i4'),
('tr', 'f8'), ('ti', 'f8'), ('ph', 'f4')]
data = np.atleast_1d(np.genfromtxt(f, dtype=dt))
self.scattering = np.resize(self.scattering, data.shape)
scat = self.scattering
scat['e'] = data['e'] * ERYDBERG
scat['a'] = data['na']
scat['l'] = data['nl']
scat['t'] = data['tr'] + 1j * data['ti']
def write_edac_scattering(self, filename_format, phases=False):
"""
write scatterer files for EDAC.
produces one file for each atom class in self.scattering.
@param filename_format: file name including a placeholder {} for the atom class.
@param phases: write phase files instead of t-matrix files.
@return: dictionary that maps atom classes to file names
"""
if phases:
write = self.write_edac_phase_file
else:
write = self.write_edac_scattering_file
scat = self.scattering
atoms = np.unique(scat['a'])
files = {}
for atom in atoms:
f = filename_format.format(atom)
sel = scat['a'] == atom
idx = np.where(sel)
atom_scat = scat[idx]
write(f, atom_scat)
files[atom] = f
return files
def write_edac_scattering_file(self, f, scat):
"""
write a scatterer file for EDAC.
@param f: file path or output stream (an object with a write method).
@param scat: a slice of the self.scattering array belonging to the same atom class.
@return: None
"""
if hasattr(f, "write") and callable(f.write):
energies = np.unique(scat['e'])
ne = energies.shape[0]
lmax = scat['l'].max()
if ne == 1:
f.write("1 {lmax} regular tl\n".format(lmax=lmax))
else:
f.write("{nk} E(eV) {lmax} regular tl\n".format(nk=ne, lmax=lmax))
for energy in energies:
sel = scat['e'] == energy
idx = np.where(sel)
energy_scat = scat[idx]
if ne > 1:
f.write("{0:.3f} ".format(energy))
for item in energy_scat:
f.write(" {0:.6f} {1:.6f}".format(item['t'].real, -item['t'].imag))
for i in range(len(energy_scat), lmax + 1):
f.write(" 0 0")
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_edac_scattering_file(fi, scat)
def write_edac_phase_file(self, f, scat):
"""
write a phase file for EDAC.
@param f: file path or output stream (an object with a write method).
@param scat: a slice of the self.scattering array belonging to the same atom class.
@return: None
"""
if hasattr(f, "write") and callable(f.write):
energies = np.unique(scat['e'])
ne = energies.shape[0]
lmax = scat['l'].max()
if ne == 1:
f.write("1 {lmax} regular real\n".format(lmax=lmax))
else:
f.write("{nk} E(eV) {lmax} regular real\n".format(nk=ne, lmax=lmax))
for energy in energies:
sel = scat['e'] == energy
idx = np.where(sel)
energy_scat = scat[idx]
if ne > 1:
f.write("{0:.3f} ".format(energy))
for item in energy_scat:
pha = np.sign(item['t'].real) * np.arcsin(np.sqrt(np.abs(item['t'].imag)))
f.write(" {0:.6f}".format(pha))
for i in range(len(energy_scat), lmax + 1):
f.write(" 0")
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_edac_phase_file(fi, scat)
def parse_radial_file(self, f):
"""
parse the radial matrix element output file from phagen version 2.2.
the file contains 7 header lines and one data line per requested energy.
the data line contains real and imaginary parts of the matrix elements.
the first four columns contain the electric dipole transitions Rd(li --> li - 1) and Rd(li --> li + 1),
followed by higher orders that we do not use here.
@param f: file or path (any file-like or path-like object that can be passed to numpy.genfromtxt).
@return: None
@raise ValueError if the file is in a wrong format.
"""
data = np.atleast_2d(np.genfromtxt(f, skip_header=7))
if data.shape[0] != self.params.kinetic_energies.shape[0] or data.shape[1] < 4:
raise ValueError(f"Unexpected array size of Phagen radial matrix elements output: "
f"expected ({self.params.kinetic_energies.shape[0]}, >= 4), received {data.shape}")
self.emission = np.resize(self.emission, data.shape[0:1])
emission = self.emission
emission['e'] = self.params.kinetic_energies
emission['dw'] = data[:, 0] + 1j * data[:, 1]
emission['up'] = data[:, 2] + 1j * data[:, 3]
def write_edac_emission_file(self, f):
"""
write the radial photoemission matrix element in EDAC format.
requires self.emission, self.params.kinetic_energies and self.params.initial_state.
@param f: file path or output stream (an object with a write method).
@return: None
"""
if hasattr(f, "write") and callable(f.write):
l0 = self.params.l_init
energies = self.params.kinetic_energies
emission = self.emission
emission['e'] = energies
ne = energies.shape[0]
if ne == 1:
f.write("1 regular2 {l0}\n".format(l0=l0))
else:
f.write("{nk} E(eV) regular2 {l0}\n".format(nk=ne, l0=l0))
for item in emission:
if ne > 1:
f.write("{0:.3f} ".format(item['e']))
f.write(" {0:.6f} {1:.6f}".format(item['up'].real, item['up'].imag))
f.write(" {0:.6f} {1:.6f}".format(item['dw'].real, item['dw'].imag))
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as of:
self.write_edac_emission_file(of)
def write_edac_emission(self, filename_format):
"""
write the radial photoemission matrix element in EDAC format.
requires self.scattering, self.emission, self.params.kinetic_energies and self.params.initial_state.
@param filename_format: file name including, optionally, a placeholder {} for the atom class.
since phagen calculates only one emitter, the placeholder is not necessary.
@return: dictionary that maps atom classes to file names.
since phagen calculates only one emitter, this dictionary will contain just one entry.
"""
scat = self.scattering
atom = scat['a'][0]
f = filename_format.format(atom)
self.write_edac_emission_file(f)
files = {atom: f}
return files

595
pmsco/cluster.py Normal file → Executable file
View File

@@ -1,12 +1,15 @@
#!/usr/bin/env python
"""
@package pmsco.cluster
cluster tools for MSC and EDAC
cluster building and handling
the Cluster class is provided to facilitate the construction and import/export of clusters.
a cluster can be built by adding single atoms, layers, or a half-space bulk lattice.
the class can import from/export to EDAC, MSC, and XYZ cluster files.
the class can import from/export to various file formats.
XYZ allows for export to 3D visualizers, e.g. Avogadro.
the module has a command line interface to convert cluster files.
@pre requires the periodictable package (https://pypi.python.org/pypi/periodictable)
@code{.sh}
pip install --user periodictable
@@ -14,18 +17,20 @@ pip install --user periodictable
@author Matthias Muntwiler
@copyright (c) 2015-18 by Paul Scherrer Institut
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import math
import numpy as np
import periodictable as pt
import sys
import pmsco.config as config
## default file format identifier
FMT_DEFAULT = 0
## MSC file format identifier
@@ -34,6 +39,12 @@ FMT_MSC = 1
FMT_EDAC = 2
## XYZ file format identifier
FMT_XYZ = 3
## PHAGEN output file format identifier
FMT_PHAGEN_OUT = 4
## PHAGEN input file format identifier
FMT_PHAGEN_IN = 5
## native file format identifier
FMT_PMSCO = 6
# python version dependent type of chemical symbol
if sys.version_info[0] >= 3:
@@ -41,13 +52,16 @@ if sys.version_info[0] >= 3:
else:
_SYMBOL_TYPE = 'S2'
## numpy.array datatype of Cluster.data array
DTYPE_CLUSTER_INTERNAL = [('i', 'i4'), ('t', 'i4'), ('s', _SYMBOL_TYPE), ('x', 'f4'), ('y', 'f4'), ('z', 'f4'),
('e', 'u1')]
## file format of internal Cluster.data array
FMT_CLUSTER_INTERNAL = ["%5u", "%2u", "%s", "%7.3f", "%7.3f", "%7.3f", "%1u"]
## field (column) names of internal Cluster.data array
FIELDS_CLUSTER_INTERNAL = ['i', 't', 's', 'x', 'y', 'z', 'e']
## numpy.array datatype of internal Cluster.data array
DTYPE_CLUSTER_INTERNAL = [('i', 'i4'), ('t', 'i4'), ('s', _SYMBOL_TYPE), ('c', 'i4'),
('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('e', 'u1'), ('q', 'f4')]
## string formatting of native file format
FMT_CLUSTER_INTERNAL = ["%5u", "%2u", "%s", "%5u", "%7.3f", "%7.3f", "%7.3f", "%1u", "%7.3f"]
## field (column) names of native file format
FIELDS_CLUSTER_INTERNAL = ['i', 't', 's', 'c', 'x', 'y', 'z', 'e', 'q']
## column names of native file format
NAMES_CLUSTER_INTERNAL = {'i': 'index', 't': 'element', 's': 'symbol', 'c': 'class', 'x': 'x', 'y': 'y', 'z': 'z',
'e': 'emitter', 'q': 'charge'}
## numpy.array datatype of cluster for MSC cluster file input/output
DTYPE_CLUSTER_MSC = [('i', 'i4'), ('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('t', 'i4')]
@@ -57,19 +71,57 @@ FMT_CLUSTER_MSC = ["%5u", "%7.3f", "%7.3f", "%7.3f", "%2u"]
FIELDS_CLUSTER_MSC = ['i', 'x', 'y', 'z', 't']
## numpy.array datatype of cluster for EDAC cluster file input/output
DTYPE_CLUSTER_EDAC= [('i', 'i4'), ('t', 'i4'), ('x', 'f4'), ('y', 'f4'), ('z', 'f4')]
DTYPE_CLUSTER_EDAC = [('i', 'i4'), ('c', 'i4'), ('x', 'f4'), ('y', 'f4'), ('z', 'f4')]
## file format of EDAC cluster file
FMT_CLUSTER_EDAC = ["%5u", "%2u", "%7.3f", "%7.3f", "%7.3f"]
## field (column) names of EDAC cluster file
FIELDS_CLUSTER_EDAC = ['i', 't', 'x', 'y', 'z']
FIELDS_CLUSTER_EDAC = ['i', 'c', 'x', 'y', 'z']
## numpy.array datatype of cluster for XYZ file input/output
DTYPE_CLUSTER_XYZ= [('s', _SYMBOL_TYPE), ('x', 'f4'), ('y', 'f4'), ('z', 'f4')]
DTYPE_CLUSTER_XYZ = [('s', _SYMBOL_TYPE), ('x', 'f4'), ('y', 'f4'), ('z', 'f4')]
## file format of XYZ cluster file
FMT_CLUSTER_XYZ = ["%s", "%10.5f", "%10.5f", "%10.5f"]
## field (column) names of XYZ cluster file
FIELDS_CLUSTER_XYZ = ['s', 'x', 'y', 'z']
## numpy.array datatype of cluster for PHAGEN output file input/output
DTYPE_CLUSTER_PHAGEN_OUT = [('i', 'i4'), ('s', _SYMBOL_TYPE), ('t', 'i4'), ('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('c', 'i4')]
## file format of PHAGEN cluster output file
FMT_CLUSTER_PHAGEN_OUT = ["%5u", "%s", "%2u", "%7.3f", "%7.3f", "%7.3f", "%5u"]
## field (column) names of PHAGEN cluster output file
FIELDS_CLUSTER_PHAGEN_OUT = ['i', 's', 't', 'x', 'y', 'z', 'c']
## numpy.array datatype of cluster for PHAGEN input file input/output
DTYPE_CLUSTER_PHAGEN_IN = [('s', _SYMBOL_TYPE), ('t', 'i4'), ('x', 'f4'), ('y', 'f4'), ('z', 'f4'), ('q', 'f4')]
## file format of PHAGEN input file, cluster section
FMT_CLUSTER_PHAGEN_IN = ["%s", "%2u", "%7.3f", "%7.3f", "%7.3f", "%7.3f"]
## field (column) names of PHAGEN input file, cluster section
FIELDS_CLUSTER_PHAGEN_IN = ['s', 't', 'x', 'y', 'z', 'q']
## dictionary of supported cluster data types
CLUSTER_DTYPES = {FMT_DEFAULT: DTYPE_CLUSTER_INTERNAL,
FMT_MSC: DTYPE_CLUSTER_MSC,
FMT_EDAC: DTYPE_CLUSTER_EDAC,
FMT_XYZ: DTYPE_CLUSTER_XYZ,
FMT_PHAGEN_OUT: DTYPE_CLUSTER_PHAGEN_OUT,
FMT_PHAGEN_IN: DTYPE_CLUSTER_PHAGEN_IN}
## dictionary of supported cluster file formats
CLUSTER_FMTS = {FMT_DEFAULT: FMT_CLUSTER_INTERNAL,
FMT_MSC: FMT_CLUSTER_MSC,
FMT_EDAC: FMT_CLUSTER_EDAC,
FMT_XYZ: FMT_CLUSTER_XYZ,
FMT_PHAGEN_OUT: FMT_CLUSTER_PHAGEN_OUT,
FMT_PHAGEN_IN: FMT_CLUSTER_PHAGEN_IN}
## dictionary of supported cluster field names
CLUSTER_FIELDS = {FMT_DEFAULT: FIELDS_CLUSTER_INTERNAL,
FMT_MSC: FIELDS_CLUSTER_MSC,
FMT_EDAC: FIELDS_CLUSTER_EDAC,
FMT_XYZ: FIELDS_CLUSTER_XYZ,
FMT_PHAGEN_OUT: FIELDS_CLUSTER_PHAGEN_OUT,
FMT_PHAGEN_IN: FIELDS_CLUSTER_PHAGEN_IN}
class Cluster(object):
"""
@@ -84,6 +136,8 @@ class Cluster(object):
- t coordinate of the atom position
- z coordinate of the atom position
- emitter flag
- charge/ionicity
- scatterer class
the class also defines methods that add or manipulate atoms of the cluster.
see most importantly the set_rmax, add_atom, add_layer and add_bulk functions.
@@ -122,10 +176,12 @@ class Cluster(object):
# @arg @c 'i' (int) atom index (1-based)
# @arg @c 't' (int) atom type (chemical element number)
# @arg @c 's' (string) chemical element symbol
# @arg @c 'c' (int) scatterer class
# @arg @c 'x' (float32) x coordinate of the atom position
# @arg @c 'y' (float32) t coordinate of the atom position
# @arg @c 'z' (float32) z coordinate of the atom position
# @arg @c 'e' (uint8) 1 = emitter, 0 = regular atom
# @arg @c 'q' (float32) charge/ionicity
## @var comment (str)
# one-line comment that can be included in some cluster files
@@ -152,6 +208,9 @@ class Cluster(object):
@param cluster: (Cluster) other Cluster object.
"""
self.data = cluster.data.copy()
self.rmax = cluster.rmax
self.dtype = cluster.dtype
self.comment = cluster.comment
def set_rmax(self, r):
"""
@@ -166,36 +225,50 @@ class Cluster(object):
"""
self.rmax = r
def build_element(self, index, element_number, x, y, z, emitter):
def build_element(self, index, element, x, y, z, emitter, charge=0., scatterer_class=0):
"""
build a tuple in the format of the internal data array.
@param index: (int) index
@param element_number: (int) chemical element number
@param element: chemical element number (int) or symbol (str)
@param x, y, z: (float) atom coordinates in the cluster
@param emitter: (int or bool) True = emitter, False = scatterer
@param charge: (float) ionicity. default = 0
@param scatterer_class: (int) scatterer class. default = 0.
"""
symbol = pt.elements[element_number].symbol
element = (index, element_number, symbol, x, y, z, int(emitter))
try:
element_number = int(element)
symbol = pt.elements[element_number].symbol
except ValueError:
symbol = element
element_number = pt.elements.symbol(symbol.strip()).number
element = (index, element_number, symbol, scatterer_class, x, y, z, int(emitter), charge)
return element
def add_atom(self, atomtype, v_pos, is_emitter):
def add_atom(self, atomtype, v_pos, is_emitter=False, charge=0.):
"""
add a single atom to the cluster.
@param atomtype: (int) chemical element number
@param atomtype: chemical element number (int) or symbol (str)
@param v_pos: (numpy.ndarray, shape = (3)) position vector
@param is_emitter: (int or bool) True = emitter, False = scatterer
@param charge: (float) ionicity. default = 0
@return array index of added atom
"""
n0 = self.data.shape[0] + 1
element = self.build_element(n0, atomtype, v_pos[0], v_pos[1], v_pos[2], int(is_emitter))
self.data = np.append(self.data, np.array(element,
dtype=self.data.dtype))
element = self.build_element(n0, atomtype, v_pos[0], v_pos[1], v_pos[2], is_emitter, charge)
self.data = np.append(self.data, np.array(element, dtype=self.data.dtype))
return n0 - 1
def add_layer(self, atomtype, v_pos, v_lat1, v_lat2):
"""
@@ -205,7 +278,7 @@ class Cluster(object):
self.rmax (maximum distance from the origin).
all atoms are non-emitters.
@param atomtype: (int) chemical element number
@param atomtype: chemical element number (int) or symbol (str)
@param v_pos: (numpy.ndarray, shape = (3))
position vector of the first atom (basis vector)
@@ -215,29 +288,30 @@ class Cluster(object):
"""
r_great = max(self.rmax, np.linalg.norm(v_pos))
n0 = self.data.shape[0] + 1
n1 = max(int(r_great / np.linalg.norm(v_lat1)) + 1, 3) * 2
n2 = max(int(r_great / np.linalg.norm(v_lat2)) + 1, 3) * 2
nn = 0
buf = np.empty((2 * n1 + 1) * (2 * n2 + 1), dtype=self.dtype)
for i1 in range(-n1, n1 + 1):
for i2 in range(-n2, n2 + 1):
v = v_pos + v_lat1 * i1 + v_lat2 * i2
if np.linalg.norm(v) <= self.rmax:
buf[nn] = self.build_element(nn + n0, atomtype, v[0], v[1], v[2], 0)
nn += 1
buf = np.resize(buf, nn)
n1 = max(int(r_great / np.linalg.norm(v_lat1)) + 1, 4) * 3
n2 = max(int(r_great / np.linalg.norm(v_lat2)) + 1, 4) * 3
idx = np.mgrid[-n1:n1+1, -n2:n2+1]
idx = idx.reshape(idx.shape[0], -1)
lat = np.array([v_lat1, v_lat2])
v = v_pos + np.matmul(idx.T, lat)
rsq = np.sum(np.square(v), axis=-1)
b1 = rsq <= self.rmax**2
sel = b1.nonzero()[0]
buf = np.empty((len(sel)), dtype=self.dtype)
for nn, ii in enumerate(sel):
buf[nn] = self.build_element(nn + n0, atomtype, v[ii, 0], v[ii, 1], v[ii, 2], 0)
self.data = np.append(self.data, buf)
def add_bulk(self, atomtype, v_pos, v_lat1, v_lat2, v_lat3, z_surf=0.0):
"""
add bulk atoms to the cluster.
the lattice is expanded up to the limits given by
the lattice is expanded up to the limits given by
self.rmax (maximum distance from the origin)
and z_surf (position of the surface).
all atoms are non-emitters.
@param atomtype: (int) chemical element number
@param atomtype: chemical element number (int) or symbol (str)
@param v_pos: (numpy.ndarray, shape = (3))
position vector of the first atom (basis vector)
@@ -253,16 +327,18 @@ class Cluster(object):
n1 = max(int(r_great / np.linalg.norm(v_lat1)) + 1, 4) * 3
n2 = max(int(r_great / np.linalg.norm(v_lat2)) + 1, 4) * 3
n3 = max(int(r_great / np.linalg.norm(v_lat3)) + 1, 4) * 3
nn = 0
buf = np.empty((2 * n1 + 1) * (2 * n2 + 1) * (n3 + 1), dtype=self.dtype)
for i1 in range(-n1, n1 + 1):
for i2 in range(-n2, n2 + 1):
for i3 in range(-n3, n3 + 1):
v = v_pos + v_lat1 * i1 + v_lat2 * i2 + v_lat3 * i3
if np.linalg.norm(v) <= self.rmax and v[2] <= z_surf:
buf[nn] = self.build_element(nn + n0, atomtype, v[0], v[1], v[2], 0)
nn += 1
buf = np.resize(buf, nn)
idx = np.mgrid[-n1:n1+1, -n2:n2+1, -n3:n3+1]
idx = idx.reshape(idx.shape[0], -1)
lat = np.array([v_lat1, v_lat2, v_lat3])
v = v_pos + np.matmul(idx.T, lat)
rsq = np.sum(np.square(v), axis=-1)
b1 = rsq <= self.rmax**2
b2 = v[:, 2] <= z_surf
ba = np.all([b1, b2], axis=0)
sel = ba.nonzero()[0]
buf = np.empty((len(sel)), dtype=self.dtype)
for nn, ii in enumerate(sel):
buf[nn] = self.build_element(nn + n0, atomtype, v[ii, 0], v[ii, 1], v[ii, 2], 0)
self.data = np.append(self.data, buf)
def add_cluster(self, cluster, check_rmax=False, check_unique=False, tol=0.001):
@@ -283,26 +359,28 @@ class Cluster(object):
@param tol: tolerance for checking uniqueness.
positions of two atoms are considered equal if all coordinates lie within the tolerance interval.
@return: None
@return: None
"""
assert isinstance(cluster, Cluster)
data = self.data.copy()
source = cluster.data.copy()
if check_rmax and source.shape[0] > 0:
source_xyz = source[['x', 'y', 'z']].copy()
source_xyz = source_xyz.view((source_xyz.dtype[0], len(source_xyz.dtype.names)))
source_xyz = cluster.get_positions()
b_rmax = np.linalg.norm(source_xyz, axis=1) <= self.rmax
idx = np.where(b_rmax)
source = source[idx]
data = np.append(data, source)
if check_unique and data.shape[0] > 0:
data_xyz = data[['x', 'y', 'z']].copy()
data_xyz = data_xyz.view((data_xyz.dtype[0], len(data_xyz.dtype.names)))
tol_xyz = np.round(data_xyz / tol)
uni_xyz = tol_xyz.view(tol_xyz.dtype.descr * 3)
_, idx = np.unique(uni_xyz, return_index=True)
data_xyz = np.empty((data.shape[0], 3))
data_xyz[:, 0] = data['x']
data_xyz[:, 1] = data['y']
data_xyz[:, 2] = data['z']
tol *= 2
uni_xyz = np.round(data_xyz / tol)
# this requires numpy 1.13 or later
_, idx = np.unique(uni_xyz, return_index=True, axis=0)
data = data[np.sort(idx)]
self.data = data
@@ -322,8 +400,10 @@ class Cluster(object):
the returned coordinates may not be identical to any atom coordinate of a layer
but deviate up to the given tolerance.
"""
self_z = self.data['z'].view(np.float32).reshape(self.data.shape)
z2 = np.round(self_z.copy() / tol)
tol *= 2
self_z = np.empty(self.data.shape, np.float32)
self_z[:] = self.data['z']
z2 = np.round(self_z / tol)
layers = np.unique(z2) * tol
return layers
@@ -338,7 +418,8 @@ class Cluster(object):
by default (element = 0), all atoms are moved.
@return: (numpy.ndarray) indices of the atoms that have been shifted.
"""
self_z = self.data['z'].view(np.float32).reshape(self.data.shape)
self_z = np.empty(self.data.shape, np.float32)
self_z[:] = self.data['z']
b_z = self_z <= z_cut
b_all = b_z
@@ -352,15 +433,47 @@ class Cluster(object):
idx = np.where(b_all)
self.data['z'][idx] += z_shift
return idx
return idx[0]
def translate(self, vector, element=0):
def get_center(self, element=None):
"""
get the geometric center of the cluster or a class of atoms.
@param element: chemical element number (int) or symbol (str)
if atoms of a specific element should be considered only.
by default (element == None or 0 or ""),
all atoms are included in the calculation.
@return: (numpy.ndarray) 3-dimensional vector.
"""
if element:
try:
sel = self.data['t'] == int(element)
except ValueError:
sel = self.data['s'] == element
else:
sel = np.ones_like(self.data['t'])
idx = np.where(sel)
center = np.zeros(3)
center[0] = np.mean(self.data['x'][idx])
center[1] = np.mean(self.data['y'][idx])
center[2] = np.mean(self.data['z'][idx])
return center
def translate(self, vector, element=None):
"""
translate the cluster or all atoms of a specified element.
translation shifts each selected atom by the given vector.
@param vector: (numpy.ndarray) 3-dimensional displacement vector.
@param element: (int) chemical element number if atoms of a specific element should be affected.
by default (element = 0), all atoms are moved.
@param element: chemical element number (int) or symbol (str)
if atoms of a specific element should be affected only.
by default (element == None or 0 or ""),
all atoms are translated.
@return: (numpy.ndarray) indices of the atoms that have been shifted.
"""
if element:
@@ -375,7 +488,7 @@ class Cluster(object):
self.data['y'][idx] += vector[1]
self.data['z'][idx] += vector[2]
return idx
return idx[0]
def matrix_transform(self, matrix):
"""
@@ -385,61 +498,69 @@ class Cluster(object):
@param matrix: transformation matrix
@return: None
@return: None
"""
for atom in self.data:
v = np.matrix([atom['x'], atom['y'], atom['z']])
w = matrix * v.transpose()
atom['x'] = float(w[0])
atom['y'] = float(w[1])
atom['z'] = float(w[2])
pos = np.empty((3, self.data.shape[0]), np.float32)
pos[0, :] = self.data['x']
pos[1, :] = self.data['y']
pos[2, :] = self.data['z']
pos = np.matmul(matrix, pos)
self.data['x'] = pos[0, :]
self.data['y'] = pos[1, :]
self.data['z'] = pos[2, :]
def rotate_x(self, angle):
"""
rotate cluster about the surface normal axis
rotate cluster about the x-axis
@param angle (float) in degrees
"""
angle = math.radians(angle)
s = math.sin(angle)
c = math.cos(angle)
matrix = np.matrix([[1, 0, 0], [0, c, -s], [0, s, c]])
matrix = np.array([[1, 0, 0], [0, c, -s], [0, s, c]])
self.matrix_transform(matrix)
def rotate_y(self, angle):
"""
rotate cluster about the surface normal axis
rotate cluster about the y-axis
@param angle (float) in degrees
"""
angle = math.radians(angle)
s = math.sin(angle)
c = math.cos(angle)
matrix = np.matrix([[c, 0, s], [0, 1, 0], [-s, 0, c]])
matrix = np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]])
self.matrix_transform(matrix)
def rotate_z(self, angle):
"""
rotate cluster about the surface normal axis
rotate cluster about the z-axis (surface normal)
@param angle (float) in degrees
"""
angle = math.radians(angle)
s = math.sin(angle)
c = math.cos(angle)
matrix = np.matrix([[c, -s, 0], [s, c, 0], [0, 0, 1]])
matrix = np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]])
self.matrix_transform(matrix)
def find_positions(self, pos, tol=0.001):
"""
find all atoms which occupy a given position.
@param pos: (numpy.array, shape = (3)) position vector.
@param pos: position vector.
this can be a numpy.ndarray with shape (3)
or any type where pos[0] represents the x-coordinate, pos[1] y, and pos[2] z.
@param tol: (float) matching tolerance per coordinate.
@return numpy.array of indices which match v_pos.
"""
if isinstance(pos, np.ndarray):
assert pos.shape == (3,)
else:
pos = np.array((pos[0], pos[1], pos[2]))
b2 = np.abs(pos - self.get_positions()) < tol
b1 = np.all(b2, axis=1)
idx = np.where(b1)
@@ -463,8 +584,9 @@ class Cluster(object):
@return numpy.array of indices which match v_pos.
"""
pos_xy = pos[0:2]
self_xy = self.data[['x', 'y']].copy()
self_xy = self_xy.view((self_xy.dtype[0], len(self_xy.dtype.names)))
self_xy = np.empty((self.data.shape[0], 2), np.float32)
self_xy[:, 0] = self.data['x']
self_xy[:, 1] = self.data['y']
b_xy = np.linalg.norm(self_xy - pos_xy, axis=1) <= r_xy
pos_z = pos[2]
@@ -497,8 +619,9 @@ class Cluster(object):
@return: None
"""
self_xy = self.data[['x', 'y']].copy()
self_xy = self_xy.view((self_xy.dtype[0], len(self_xy.dtype.names)))
self_xy = np.empty((self.data.shape[0], 2), np.float32)
self_xy[:, 0] = self.data['x']
self_xy[:, 1] = self.data['y']
b_xy = np.linalg.norm(self_xy, axis=1) <= r_xy
self_z = self.data['z']
@@ -545,8 +668,7 @@ class Cluster(object):
@return: None
"""
self_xyz = self.data[['x', 'y', 'z']].copy()
self_xyz = self_xyz.view((self_xyz.dtype[0], len(self_xyz.dtype.names)))
self_xyz = self.get_positions()
b_xyz = np.linalg.norm(self_xyz, axis=1) <= radius
idx = np.where(b_xyz)
self.data = self.data[idx]
@@ -562,7 +684,8 @@ class Cluster(object):
@return: None
"""
coord = self.data[axis].view(np.float32).reshape(self.data.shape)
coord = np.empty(self.data.shape, np.float32)
coord[:] = self.data[axis]
sel = np.abs(coord - center) <= depth / 2
idx = np.where(sel)
self.data = self.data[idx]
@@ -617,15 +740,17 @@ class Cluster(object):
def get_positions(self):
"""
get an array of the atom coordinates.
get the atom coordinates in a two-dimensional array.
the returned array is an independent copy of the original data.
changes will not affect the original cluster.
@return numpy.ndarray, shape = (N,3)
"""
pos = self.data[['x', 'y', 'z']].copy()
pos = pos.view((pos.dtype[0], len(pos.dtype.names)))
pos = np.empty((self.data.shape[0], 3), np.float32)
pos[:, 0] = self.data['x']
pos[:, 1] = self.data['y']
pos[:, 2] = self.data['z']
return pos
def set_positions(self, positions):
@@ -689,14 +814,16 @@ class Cluster(object):
rec = self.data[index]
return rec['s']
def get_emitters(self):
def get_emitters(self, fields):
"""
get a list of all emitters.
@return list of tuples (x, y, z, atomtype)
@param fields: list of field (column) names to return
@return list of tuples. each tuple contains the values of the requested fields.
"""
idx = self.data['e'] != 0
ems = self.data[['x', 'y', 'z', 't']][idx]
ems = self.data[fields][idx]
return [tuple(em) for em in ems]
def get_emitter_count(self):
@@ -708,13 +835,72 @@ class Cluster(object):
idx = self.data['e'] != 0
return np.sum(idx)
def calc_scattering_angles(self, index_emitter, radius):
"""
calculate forward-scattering angles of the cluster atoms
for each atom within a given radius of the emitter,
the connecting vector between emitter and scatterer is calculated
and returned in cartesian and polar coordinates.
@param index_emitter: atom index of the emitter.
all angles are calculated with respect to this atom.
@param radius: include only atoms within this radius of the emitter.
@note back-scattering angles can be obtained by inverting the angle on the unit sphere:
th' = 180 - th, ph' = -ph.
@return dictionary with results.
each item is a numpy.ndarray of shape (N, M)
where N is the number of scatterers
and M = 3 for dict['xyz'] and M = 1 otherwise.
@arg dict['index']: atom index into the cluster array.
@arg dict['xyz']: connecting vector between the emitter and the atom in cartesian coordinates.
@arg dict['dist']: distance between the emitter and the atom.
@arg dict['polar']: polar angle with respect to the z-axis.
@arg dict['azimuth']: azimuthal angle with respect to the x-axis.
"""
# position of emitter atom
em = self.data[index_emitter]
em = np.asarray((em['x'], em['y'], em['z']))
# relative positions of scattering atoms
xyz = self.get_positions()
xyz -= em
dist = np.linalg.norm(xyz, axis=1)
sel1 = dist <= radius
sel2 = dist > 0.
idx = np.where(np.all([sel1, sel2], axis=0))
xyz = xyz[idx]
dist = dist[idx]
# angles
v1 = np.asarray([0, 0, 1])
v2 = np.transpose(xyz / dist.reshape((dist.shape[0], 1)))
th = np.degrees(np.arccos(np.clip(np.dot(v1, v2), -1., 1.)))
ph = np.degrees(np.arctan2(v2[1], v2[0]))
return {'index': idx[0], 'xyz': xyz, 'dist': dist, 'polar': th, 'azimuth': ph}
def load_from_file(self, f, fmt=FMT_DEFAULT):
"""
load a cluster from a file created by the scattering program.
the file formats differ in the columns that they contain.
only the 'x', 'y', 'z' coordinates are common to all formats.
at least one of the 's' and 't' columns must be present.
missing columns are initialized as follows.
@arg 'i': reset to a 1-based sequential index (@ref update_index).
@arg 's': derived from the 't' column (@ref update_symbols).
@arg 't': derived from the 's' column (@ref update_atomtypes).
@arg 'e': set to 0.
@arg 'c': set equal to the 't' column (@ref init_atomclasses).
@arg 'q': set to 0.
@param f (string/handle): path name or open file handle of the cluster file.
@param f: path name or open file handle of the cluster file.
@param fmt (int): file format.
@param fmt: file format.
must be one of the FMT_ constants.
if FMT_DEFAULT, self.file_format is used.
@@ -735,12 +921,25 @@ class Cluster(object):
dtype = DTYPE_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
sh = 2
elif fmt == FMT_PHAGEN_OUT:
dtype = DTYPE_CLUSTER_PHAGEN_OUT
fields = FIELDS_CLUSTER_PHAGEN_OUT
sh = 1
elif fmt == FMT_PHAGEN_IN:
dtype = DTYPE_CLUSTER_PHAGEN_IN
fields = FIELDS_CLUSTER_PHAGEN_IN
sh = 0
elif fmt == FMT_PMSCO:
dtype = DTYPE_CLUSTER_INTERNAL
fields = FIELDS_CLUSTER_INTERNAL
sh = 1
else:
dtype = DTYPE_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
sh = 2
raise ValueError("unknown file format {}".format(fmt))
data = np.atleast_1d(np.genfromtxt(f, dtype=dtype, skip_header=sh))
if fmt == FMT_PHAGEN_IN and data['t'][-1] < 1:
data = data[:-1]
data = np.genfromtxt(f, dtype=dtype, skip_header=sh)
self.data = np.empty(data.shape, dtype=self.dtype)
self.data['x'] = data['x']
self.data['y'] = data['y']
@@ -753,14 +952,23 @@ class Cluster(object):
self.data['t'] = data['t']
if 's' in fields:
self.data['s'] = data['s']
else:
elif 't' in fields:
self.update_symbols()
if 't' not in fields:
self.update_atomtypes()
if 's' in fields:
self.update_atomtypes()
if 'e' in fields:
self.data['e'] = data['e']
else:
self.data['e'] = 0
if 'c' in fields:
self.data['c'] = data['c']
else:
self.data['c'] = 0
if 'q' in fields:
self.data['q'] = data['q']
else:
self.data['q'] = 0.
pos = self.get_positions()
# note: np.linalg.norm does not accept axis argument in version 1.7
@@ -788,17 +996,80 @@ class Cluster(object):
for atom in self.data:
atom['t'] = pt.elements.symbol(atom['s'].strip()).number
def init_atomclasses(self, field_or_value='t', default_only=False):
"""
initialize atom classes from atom types.
atom classes identify the atomic scattering potential or scattering factors
to be used in the multiple scattering program.
if the scattering factors are calculated in the PMSCO process (by EDAC or PHAGEN),
the atom classes must be set equal to the element type
or left at the default value 0 in which case PMSCO sets the correct values.
if the scattering factors are loaded from existing files,
the atom class corresponds to the key of the pmsco.project.CalculatorParams.phase_files dictionary.
in this case the meaning of the class value is up to the project,
and the class must be set either by the cluster generator
or the project's after_atomic_scattering hook.
@param field_or_value: name of a cluster data field, e.g. 't', or an integer constant.
@param default_only: initialize classes only if they are at their default value (0).
@return None
"""
if not default_only or np.sum(np.abs(self.data['c'])) == 0:
if isinstance(field_or_value, str):
self.data['c'] = self.data[field_or_value]
else:
self.data['c'] = field_or_value
def update_index(self):
"""
update the index column.
if you have modified the order or number of elements in the self.data array directly,
you may need to re-index the atoms if your code uses functions that rely on the index.
you may need to re-index the atoms if your code uses functions that rely on the index.
@return: None
@return None
"""
self.data['i'] = np.arange(1, self.data.shape[0] + 1)
def update_atoms(self, clu, fields):
"""
update atom properties from another cluster.
this method copies selected fields from another cluster.
the other cluster must contain the same atoms (same coordinates) in a possibly random order.
the atoms of this and the other cluster are matched up by sorting them by coordinate.
atomic scattering calculators often change the order of atoms in a cluster based on domain,
and return atom classes versus atomic coordinates.
this method allows to import the atom classes into the original cluster.
the method checks that the other cluster contains the same number of atoms.
it does not check that the clusters contain the same atomic positions.
linear translations are acceptable.
@param clu: cluster.Cluster object
@param fields: subset of field names out of FIELDS_CLUSTER_INTERNAL.
'i', 'x', 'y', 'z' are ignored.
the set can be specified in any type that converts into a set of strings.
@return: None
@raise AssertError if the clusters do not contain the same number of atoms
"""
assert self.data.shape == clu.data.shape
fields = set(fields) - {'i', 'x', 'y', 'z'}
common_order = ('z', 'y', 'x')
index_self = np.argsort(self.data, order=common_order)
index_other = np.argsort(clu.data, order=common_order)
for field in fields:
self.data[field][index_self] = clu.data[field][index_other]
def save_to_file(self, f, fmt=FMT_DEFAULT, comment="", emitters_only=False):
"""
save the cluster to a file which can be read by the scattering program.
@@ -846,16 +1117,27 @@ class Cluster(object):
file_format = FMT_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
header = "{nat}\n{com}".format(nat=data.shape[0], com=comment)
elif fmt == FMT_PHAGEN_IN:
file_format = FMT_CLUSTER_PHAGEN_IN
fields = FIELDS_CLUSTER_PHAGEN_IN
header = None
elif fmt == FMT_PHAGEN_OUT:
file_format = FMT_CLUSTER_PHAGEN_OUT
fields = FIELDS_CLUSTER_PHAGEN_OUT
header = ""
elif fmt == FMT_PMSCO:
file_format = FMT_CLUSTER_INTERNAL
fields = FIELDS_CLUSTER_INTERNAL
names = NAMES_CLUSTER_INTERNAL
header = "# " + " ".join([names[field] for field in fields])
else:
file_format = FMT_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
header = "{nat}\n{com}".format(nat=data.shape[0], com=comment)
raise ValueError("unknown file format {}".format(fmt))
data = data[fields]
np.savetxt(f, data, fmt=file_format, header=header, comments="")
class ClusterGenerator(object):
class ClusterGenerator(config.ConfigurableObject):
"""
cluster generator class.
@@ -873,13 +1155,14 @@ class ClusterGenerator(object):
@param project: reference to the project object.
cluster generators may need to look up project parameters.
"""
super().__init__()
self.project = project
def count_emitters(self, model, index):
"""
return the number of emitter configurations for a particular model, scan and symmetry.
return the number of emitter configurations for a particular model, scan and domain.
the number of emitter configurations may depend on the model parameters, scan index and symmetry index.
the number of emitter configurations may depend on the model parameters, scan index and domain index.
by default, the method returns 1, which means that there is only one emitter configuration.
emitter configurations are mainly a way to distribute the calculations to multiple processes
@@ -906,9 +1189,9 @@ class ClusterGenerator(object):
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index must be -1.
@arg scan scan index (index into Project.scans)
@arg domain domain index (index into Project.domains)
@arg emit emitter index must be -1.
@return number of emitter configurations.
this implementation returns the default value of 1.
@@ -920,23 +1203,23 @@ class ClusterGenerator(object):
create a Cluster object given the model parameters and calculation index.
the generated cluster will typically depend on the model parameters.
depending on the project, it may also depend on the scan index, symmetry index and emitter index.
depending on the project, it may also depend on the scan index, domain index and emitter index.
the scan index can be used to generate a different cluster for different scan geometry,
e.g., if some atoms can be excluded due to a longer mean free path.
if this is not the case for the specific project, the scan index can be ignored.
the symmetry index may select a particular domain that has a different atomic arrangement.
in this case, depending on the value of index.sym, the function must generate a cluster corresponding
to the particular domain/symmetry.
the method can ignore the symmetry index if the project defines only one symmetry,
or if the symmetry does not correspond to a different atomic structure.
the domain index may select a particular domain that has a different atomic arrangement.
in this case, depending on the value of index.domain, the function must generate a cluster corresponding
to the particular domain.
the method can ignore the domain index if the project defines only one domain,
or if the domain does not correspond to a different atomic structure.
the emitter index selects a particular emitter configuration.
depending on the value of the emitter index, the method must react differently:
1. if the value is -1, return the full cluster and mark all inequivalent emitter atoms.
emitters which are reproduced by a symmetry expansion in combine_emitters() should not be marked.
emitters which are reproduced by a domain expansion in combine_emitters() should not be marked.
the full diffraction scan will be calculated in one calculation.
2. if the value is greater or equal to zero, generate the cluster with the emitter configuration
@@ -958,9 +1241,9 @@ class ClusterGenerator(object):
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index.
@arg scan scan index (index into Project.scans)
@arg domain domain index (index into Project.domains)
@arg emit emitter index.
if -1, generate the full cluster and mark all emitters.
if greater or equal to zero, the value is a zero-based index of the emitter configuration.
@@ -980,7 +1263,7 @@ class LegacyClusterGenerator(ClusterGenerator):
"""
def __init__(self, project):
super(LegacyClusterGenerator, self).__init__(project)
super().__init__(project)
def count_emitters(self, model, index):
"""
@@ -996,3 +1279,67 @@ class LegacyClusterGenerator(ClusterGenerator):
redirect the call to the corresponding project method.
"""
return self.project.create_cluster(model, index)
def parse_cli():
"""
parse the command line
@return: Namespace object created by the argument parser.
"""
import argparse
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="""
cluster conversion
""")
format_choices = ["PMSCO", "MSC", "EDAC", "XYZ", "PHAGEN_OUT", "PHAGEN_IN"]
parser.add_argument('input_format',
choices=format_choices,
help="format of input file")
parser.add_argument('input_file',
help="path and name of input file")
parser.add_argument('output_format',
choices=format_choices,
help="format of output file")
parser.add_argument('output_file',
help="path and name of output file")
args = parser.parse_args()
return args
def convert_cli(args):
"""
convert cluster files from one format into another
this function is part of the command line interface
@param args: command line arguments
@return: None
"""
clu = Cluster()
clu.file_format = FMT_PMSCO
input_format = globals()["FMT_" + args.input_format.upper()]
output_format = globals()["FMT_" + args.output_format.upper()]
clu.load_from_file(args.input_file, input_format)
clu.save_to_file(args.output_file, output_format)
def main_cli():
"""
command line interface to convert cluster files
see @ref convert_cli.
@return: None
"""
args = parse_cli()
convert_cli(args)
if __name__ == '__main__':
main_cli()
sys.exit(0)

40
pmsco/compat.py
View File

@@ -1,40 +0,0 @@
"""
@package pmsco.compat
compatibility code
code bits to provide compatibility for different python versions.
currently supported 2.7 and 3.6.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from io import open as io_open
def open(fname, mode='r', encoding='latin1'):
"""
open a data file for read/write/append using the default str type
this is a drop-in for io.open
where data is exchanged via the built-in str type of python,
whether this is a byte string (python 2) or unicode string (python 3).
the file is assumed to be a latin-1 encoded binary file.
@param fname: file name and path
@param mode: 'r', 'w' or 'a'
@param encoding: 'latin1' (default), 'ascii' or 'utf-8'
@return file handle
"""
if isinstance(b'b', str):
# python 2
mode += 'b'
kwargs = {}
else:
# python 3
mode += 't'
kwargs = {'encoding': encoding}
return io_open(fname, mode, **kwargs)

163
pmsco/config.py Normal file
View File

@@ -0,0 +1,163 @@
"""
@package pmsco.config
infrastructure for configurable objects
@author Matthias Muntwiler
@copyright (c) 2021-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import collections.abc
import inspect
import logging
import os
from pathlib import Path
from string import Template
from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union
logger = logging.getLogger(__name__)
PathLike = Union[str, os.PathLike]
DataDict = Mapping[str, Union[str, int, float, Iterable, Mapping]]
def resolve_path(path: PathLike, dirs: Mapping[str, Any]):
"""
Resolve a file path by replacing placeholders.
Placeholders are enclosed in curly braces.
Values for all possible placeholders are provided in a dictionary.
@param path: str, Path or other path-like.
Example: '${work}/test/testfile.dat'.
@param dirs: Dictionary mapping placeholders to project paths.
The paths can be str, Path or other path-like
Example: {'work': '/home/user/work'}
@return: pathlib.Path object
"""
return Path(*(Template(p).substitute(dirs) for p in Path(path).parts))
class ConfigurableObject(object):
"""
Parent class for objects that can be configured from a runfile
The runfile is a JSON file that contains object data in a nested dictionary structure.
In the dictionary structure the keys are property or attribute names of the object to be initialized.
Keys starting with a non-alphabetic character (except for some special keys like __class__) are ignored.
These can be used as comments, or they protect private attributes.
The values can be numeric values, strings, lists or dictionaries.
Simple values are simply assigned using setattr.
This may call a property setter if defined.
Lists are iterated. Each item is appended to the attribute.
The attribute must implement an append method in this case.
If an item is a dictionary and contains the special key '__class__',
an object of that class is instantiated and recursively initialized with the dictionary elements.
This requires that the class can be found in the module scope passed to the parser methods,
and that the class inherits from this class.
Cases that can't be covered easily using this mechanism
should be implemented in a property setter.
Value-checking should also be done in a property setter (or the append method in sequence-like objects).
Attributes
----------
project_symbols: Dictionary of symbols that should be used to resolve class and function names.
This is usually the globals() dictionary of the project module.
"""
def __init__(self):
super().__init__()
self.project_symbols: Optional[Mapping[str, Any]] = None
def set_properties(self, symbols: Optional[Mapping[str, Any]],
data_dict: DataDict,
project: 'ConfigurableObject') -> None:
"""
Set properties from dictionary.
@param symbols: Dictionary of symbols that should be used to resolve class names.
This is usually the globals() dictionary of the project module.
Classes are resolved using the eval function.
@param data_dict: Dictionary of properties to set.
See the class description for details.
@param project: Reference to the project object.
@return: None
"""
self.project_symbols = symbols
for key in data_dict:
if key[0].isalpha():
self.set_property(symbols, key, data_dict[key], project)
def set_property(self, symbols: Optional[Mapping[str, Any]],
key: str,
value: DataDict,
project: 'ConfigurableObject') -> None:
"""
Set one property.
@param symbols: Dictionary of symbols that should be used to resolve class names.
This is usually the globals() dictionary of the project module.
Classes are resolved using the eval function.
@param key: Attribute name to set.
@param value: New value of the attribute.
@param project: Reference to the project object.
@return: None
"""
obj = self.parse_object(symbols, value, project)
if hasattr(self, key):
if obj is not None:
if isinstance(obj, collections.abc.MutableSequence):
attr = getattr(self, key)
for item in obj:
attr.append(item)
elif isinstance(obj, collections.abc.Mapping):
d = getattr(self, key)
if d is not None and isinstance(d, collections.abc.MutableMapping):
d.update(obj)
else:
setattr(self, key, obj)
else:
setattr(self, key, obj)
else:
setattr(self, key, obj)
else:
logger.warning(f"class {self.__class__.__name__} does not have attribute {key}.")
def parse_object(self, symbols: Optional[Mapping[str, Any]],
value: DataDict,
project: 'ConfigurableObject') -> object:
if isinstance(value, collections.abc.MutableMapping) and "__class__" in value:
cn = value["__class__"]
try:
c = eval(cn, symbols)
except (AttributeError, KeyError, NameError, ValueError):
logger.critical(f"can't resolve class name {cn}")
raise
s = inspect.signature(c)
if 'project' in s.parameters:
o = c(project=project)
else:
o = c()
o.set_properties(symbols, value, project)
elif isinstance(value, collections.abc.MutableSequence):
o = [self.parse_object(symbols, i, project) for i in value]
else:
o = value
return o

252
pmsco/data.py
View File

@@ -1,32 +1,36 @@
"""
@package pmsco.data
import, export, evaluation of msc data.
Import, export, evaluation of msc data.
this module provides common functions for loading/saving and manipulating PED scan data sets.
This module provides common functions for loading/saving and manipulating PED scan data sets.
@author Matthias Muntwiler
@copyright (c) 2015-17 by Paul Scherrer Institut @n
@copyright (c) 2015-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
import numpy.typing as npt
import os
import scipy.special
import scipy.optimize as so
from pmsco.compat import open
import pmsco.loess.loess as loess
from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union
import h5py
logger = logging.getLogger(__name__)
try:
import loess
except (ModuleNotFoundError, ImportError) as e:
loess = None
logger.critical("Error importing the loess package.", exc_info=e)
## energy, intensity
DTYPE_EI = [('e', 'f4'), ('i', 'f4')]
## energy, theta, phi, intensity
@@ -43,9 +47,11 @@ DTYPE_TP = [('t', 'f4'), ('p', 'f4')]
DTYPE_TPI = [('t', 'f4'), ('p', 'f4'), ('i', 'f4')]
## theta, phi, intensity, sigma (standard deviation)
DTYPE_TPIS = [('t', 'f4'), ('p', 'f4'), ('i', 'f4'), ('s', 'f4')]
## intensity, theta, phi
DTYPE_ITP = [('i', 'f4'), ('t', 'f4'), ('p', 'f4')]
DTYPES = {'EI': DTYPE_EI, 'ETPI': DTYPE_ETPI, 'ETPIS': DTYPE_ETPIS, 'ETPAI': DTYPE_ETPAI, 'ETPAIS': DTYPE_ETPAIS,
'TP': DTYPE_TP, 'TPI': DTYPE_TPI, 'TPIS': DTYPE_TPIS, }
'TP': DTYPE_TP, 'TPI': DTYPE_TPI, 'TPIS': DTYPE_TPIS, 'ITP': DTYPE_ITP, }
DATATYPES = DTYPES.keys
## supportd scan types
@@ -55,8 +61,11 @@ DATATYPES = DTYPES.keys
# @arg @c 'TP' theta - phi (holo scan)
SCANTYPES = ['E', 'EA', 'ET', 'TP']
GenTextFileLike = Union[str, os.PathLike, Iterable[str], int]
OSFileLike = Union[str, os.PathLike, int]
def create_etpi(shape, sigma_column=True):
def create_etpi(shape: Tuple[int], sigma_column: bool = True) -> np.ndarray:
"""
create an ETPI array of a given size.
@@ -64,6 +73,7 @@ def create_etpi(shape, sigma_column=True):
the array is initialized with zeroes.
@param shape (tuple) shape of the array
@param sigma_column: whether the array should include a sigma field (ETPIS type instead of ETPI)
"""
if sigma_column:
data = np.zeros(shape, dtype=DTYPE_ETPIS)
@@ -72,7 +82,7 @@ def create_etpi(shape, sigma_column=True):
return data
def create_data(shape, datatype='', dtype=None):
def create_data(shape: Tuple[int], datatype: str = '', dtype: Optional[npt.DTypeLike] = None) -> np.ndarray:
"""
create a data array of a given size and type.
@@ -90,7 +100,108 @@ def create_data(shape, datatype='', dtype=None):
return data
def load_plt(filename, int_column=-1):
def holo_grid(theta_start: float = 90., theta_step: float = 1., theta_range: float = 90.,
phi_start: float = 0., phi_range: float = 360., phi_refinement: float = 1.):
"""
Generator of a holo grid with constant point density in solid angle.
The generator yields the polar coordinates of a hologram scan in the traditional Osterwalder fashion,
where the grid points are distributed evenly on the hemisphere by varying the azimuthal step size,
while the polar step size is constant.
The yield are tuples (theta, phi) in degrees.
Theta is the polar, phi the azimuthal coordinate.
@param theta_start Maximum polar angle in degrees, 0..90. Defaults to 90 (grazing emission).
@param theta_step Polar angle step in degrees, 1..90. Defaults to 1.
@param theta_range Polar angle range in degrees, 1..th_start. Defaults to 90.
@param phi_start Azimuthal start angle in degrees. Defaults to 0.
This azimuth is included at every polar step.
@param phi_range Azimuthal range in degrees. Defaults to 360.
@param phi_refinement Azimuthal refinement/oversampling (scalar). Defaults to 1.
A refinement of 2 yields a factor 2 more grid points in the azimuthal sub-scans.
@return yield tuples (theta, phi) in degrees
"""
deg2rad = 0.01745329
def calc_phi_step(th):
if th < 0.5 or int(phi_range * math.sin(th * deg2rad) * phi_refinement / theta_step) == 0:
phi_st = 0.0
else:
phi_st = phi_range / int(th / theta_start * phi_range / theta_step)
if abs(phi_st) < 0.001:
phi_st = 360.
return phi_st
for theta in np.arange(theta_range, -theta_step, -theta_step):
phi_step = calc_phi_step(theta)
for phi in np.arange(phi_start, phi_range, phi_step):
yield theta, phi
def holo_array(generator: Callable[..., Iterable[Tuple[float, float]]],
generator_args: Dict,
datatype: str = 'TP',
dtype: Optional[npt.DTypeLike] = None) -> np.ndarray:
"""
Create an hologram scan grid in a numpy array.
A holo data array is a numpy structured array containing at least
a column for theta (polar angle) and phi (azimuthal angle).
The theta and phi columns are filled with angles from the holo_grid (or custom generator) function.
The array can contain further columns for energy, intensity, etc. according to the data type specified.
These columns are initialized with zeroes.
@param generator Generator that yields tuples (theta, phi) for each grid point,
given the keyword arguments kwargs.
Defaults to holo_grid, the traditional Osterwalder holo scan.
@param generator_args Keyword arguments to be passed to the generator.
For arguments of the traditional holo scan, see the documentation of holo_grid.
@param datatype See DATATYPES. Must contain 'T' and 'P' dimensions. Defaults to 'TP'.
@param dtype See DTYPES. Must contain a 't' and 'p' column. Takes precedence over datatype.
Defaults to None (not specified).
"""
if not dtype:
dtype = DTYPES[datatype]
tp = np.fromiter(generator(**generator_args), dtype=DTYPES['TP'])
result = np.zeros(tp.shape, dtype=dtype)
result['t'] = tp['t']
result['p'] = tp['p']
return result
def analyse_holoscan_steps(holoscan: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Find the polar and azimuthal steps in a holoscan.
@param holoscan:
@return: thetas: unique theta angles. sorted.
dtheta: theta steps for each theta
dphi: phi step for each theta
"""
thetas, indices, counts = np.unique(holoscan['t'], return_index=True, return_counts=True)
dtheta = np.diff(thetas)
dtheta = np.append(dtheta, dtheta[-1])
adjusted_phis = np.append(holoscan['p'], holoscan['p'][-1])
phis0 = adjusted_phis[indices]
phis1 = adjusted_phis[indices+1]
dphi = phis1 - phis0
phi_range = counts[-1] * dphi[-1]
dphi[counts <= 1] = phi_range
return thetas, dtheta, dphi
def load_plt(filename: GenTextFileLike, int_column: int = -1) -> np.ndarray:
"""
loads ETPI data from an MSC output (plt) file
@@ -117,12 +228,13 @@ def load_plt(filename, int_column=-1):
data[i]['p'] = phi
data[i]['i'] = selected intensity column
"""
data = np.genfromtxt(filename, usecols=(0, 2, 3, int_column), dtype=DTYPE_ETPI)
data = np.atleast_1d(np.genfromtxt(filename, usecols=(0, 2, 3, int_column), dtype=DTYPE_ETPI))
sort_data(data)
return data
def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_cluster=False):
def load_edac_pd(filename: OSFileLike, int_column: int = -1,
energy: float = 0.0, theta: float = 0.0, phi: float = 0.0, fixed_cluster: bool = False) -> np.ndarray:
"""
load ETPI or ETPAI data from an EDAC PD output file.
@@ -157,7 +269,8 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
data[i]['i'] = selected intensity column
@endverbatim
"""
with open(filename, "r") as f:
with open(filename, "rt", encoding="latin1") as f:
header1 = f.readline().strip()
header2 = f.readline().strip()
if not header1 == '--- scan PD':
@@ -189,7 +302,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
logger.warning("unexpected EDAC output file column name")
break
cols = tuple(cols)
raw = np.genfromtxt(filename, usecols=cols, dtype=dtype, skip_header=2)
raw = np.atleast_1d(np.genfromtxt(filename, usecols=cols, dtype=dtype, skip_header=2))
if fixed_cluster:
etpi = np.empty(raw.shape, dtype=DTYPE_ETPAI)
@@ -218,7 +331,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
return etpi
def load_etpi(filename):
def load_etpi(filename: GenTextFileLike) -> np.ndarray:
"""
loads ETPI or ETPIS data from a text file
@@ -253,7 +366,7 @@ def load_etpi(filename):
return data
def load_data(filename, dtype=None):
def load_data(filename: GenTextFileLike, dtype: Optional[npt.DTypeLike] = None):
"""
load column data (ETPI, and the like) from a text file.
@@ -288,7 +401,7 @@ def load_data(filename, dtype=None):
return data
def format_extension(data):
def format_extension(data: np.ndarray) -> str:
"""
format the file extension based on the contents of an array.
@@ -299,7 +412,7 @@ def format_extension(data):
return "." + "".join(data.dtype.names)
def save_data(filename, data):
def save_data(filename: OSFileLike, data: npt.ArrayLike) -> None:
"""
save column data (ETPI, and the like) to a text file.
@@ -315,7 +428,7 @@ def save_data(filename, data):
np.savetxt(filename, data, fmt='%g')
def sort_data(data):
def sort_data(data: np.ndarray) -> None:
"""
sort scan data (ETPI and the like) in a consistent order.
@@ -338,7 +451,8 @@ def sort_data(data):
data.sort(kind='mergesort', order=sort_key)
def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
def restructure_data(data: np.ndarray, dtype: Optional[npt.DTypeLike] = None,
defaults: Optional[Mapping] = None) -> np.ndarray:
"""
restructure the type of a data array by adding or removing columns.
@@ -361,6 +475,8 @@ def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
@return: re-structured numpy array or
@c data if the new and original data types are the same.
"""
if dtype is None:
dtype = DTYPE_ETPAIS
if data.dtype == dtype:
return data
else:
@@ -378,7 +494,7 @@ def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
return new_data
def common_dtype(scans):
def common_dtype(scans: Iterable[Union[npt.ArrayLike, npt.DTypeLike]]) -> npt.DTypeLike:
"""
determine the common data type for a number of scans.
@@ -409,7 +525,7 @@ def common_dtype(scans):
return dtype
def detect_scan_mode(data):
def detect_scan_mode(data: np.ndarray) -> Tuple[List[str], Dict[str, np.ndarray]]:
"""
detect the scan mode and unique scan positions in a data array.
@@ -495,7 +611,7 @@ def detect_scan_mode(data):
return scan_mode, scan_positions
def filter_tp(data, filter):
def filter_tp(data: np.ndarray, _filter: np.ndarray) -> np.ndarray:
"""
select data points from an ETPI array that match theta and phi coordinates of another ETPI array.
@@ -503,7 +619,7 @@ def filter_tp(data, filter):
@param data ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
@param filter ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
@param _filter ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
only 't' and 'p' columns are used.
@return filtered data (numpy.ndarray)
@@ -512,18 +628,19 @@ def filter_tp(data, filter):
"""
# copy theta,phi into separate structured arrays
data_tp = np.zeros_like(data, dtype=[('t', '<i4'), ('p', '<i4')])
filter_tp = np.zeros_like(filter, dtype=[('t', '<i4'), ('p', '<i4')])
filt_tp = np.zeros_like(_filter, dtype=[('t', '<i4'), ('p', '<i4')])
# multiply by 10, round to integer
data_tp['t'] = np.around(data['t'] * 10.0)
data_tp['p'] = np.around(data['p'] * 10.0)
filter_tp['t'] = np.around(filter['t'] * 10.0)
filter_tp['p'] = np.around(filter['p'] * 10.0)
filt_tp['t'] = np.around(_filter['t'] * 10.0)
filt_tp['p'] = np.around(_filter['p'] * 10.0)
# calculate intersection
idx = np.in1d(data_tp, filter_tp)
idx = np.in1d(data_tp, filt_tp)
result = data[idx]
return result
def interpolate_hemi_scan(rect_tpi, hemi_tpi):
def interpolate_hemi_scan(rect_tpi: np.ndarray, hemi_tpi: np.ndarray) -> np.ndarray:
"""
interpolate a hemispherical scan from a rectangular angle scan.
@@ -555,7 +672,9 @@ def interpolate_hemi_scan(rect_tpi, hemi_tpi):
hemi_tpi['i'][sel_theta] = result
return hemi_tpi
def reshape_2d(flat_data, axis_columns, return_column='i'):
def reshape_2d(flat_data: np.ndarray, axis_columns: Sequence[str], return_column: str = 'i') -> \
Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
reshape an ETPI-like array into a two-dimensional array according to the scan axes.
@@ -564,7 +683,9 @@ def reshape_2d(flat_data, axis_columns, return_column='i'):
the array must be sorted in the order of axis_labels.
@param axis_columns list of column names that designate the axes
@param return_column: name of field to return in two dimensions
@return the tuple (result_data, axis0, axis1), where
@arg result_data (ndarray) new two-dimensional ndarray of the scan
@arg axis0 (ndarray) scan positions along the first dimension
@@ -579,7 +700,7 @@ def reshape_2d(flat_data, axis_columns, return_column='i'):
return data.copy(), axis0, axis1
def calc_modfunc_mean(data):
def calc_modfunc_mean(data: np.ndarray) -> np.ndarray:
"""
calculates the modulation function using the mean value of data.
this is a simplified calculation method
@@ -615,7 +736,7 @@ def calc_modfunc_mean(data):
return modf
def calc_modfunc_loess(data, smth=0.4):
def calc_modfunc_loess(data: np.ndarray, smth: float = 0.4) -> np.ndarray:
"""
calculate the modulation function using LOESS (locally weighted regression) smoothing.
@@ -669,20 +790,27 @@ def calc_modfunc_loess(data, smth=0.4):
return modf
def rfactor(experiment, theory):
def square_diff_rfactor(experiment: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a calculated modulation function.
Calculate the R-factor from the normalized sum of squared differences.
if the sigma column is present in experiment and non-zero,
If the sigma column is present in experiment and non-zero,
the R-factor terms are weighted by 1/sigma**2.
the input arrays must have the same shape and the coordinate columns must be identical (they are ignored).
the array elements are compared element-by-element.
terms having NaN intensity are ignored.
The input arrays must have the same shape and the coordinate columns must be identical.
The array elements are compared element-by-element.
The values of the coordinate arrays do not influence the result.
Terms having NaN intensity are ignored.
@param experiment: ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
This function can be specified in the Scan.rfactor_func parameter of the project.
@param theory: ETPI or ETPAI array containing the calculated modulation functions.
@param experiment: (numpy structured array)
ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
If an `s` field is present and non-zero,
the R-factor terms are weighted by 1/sigma**2.
@param theory: (numpy structured array)
ETPI or ETPAI array containing the theoretical function.
@return scalar R-factor in the range from 0.0 to 2.0.
@@ -702,7 +830,7 @@ def rfactor(experiment, theory):
return sum1 / sum2
def scaled_rfactor(scale, experiment, weights, theory):
def scaled_rfactor_func(scale: float, experiment: np.ndarray, weights: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a modulation function against the measurement with scaled amplitude.
@@ -732,6 +860,7 @@ def scaled_rfactor(scale, experiment, weights, theory):
@raise ValueError if all experiments and theory values or all weights are zero.
"""
difs = weights * (scale * experiment - theory) ** 2
sums = weights * (scale ** 2 * experiment ** 2 + theory ** 2)
sum1 = difs.sum(dtype=np.float64)
@@ -739,7 +868,7 @@ def scaled_rfactor(scale, experiment, weights, theory):
return sum1 / sum2
def optimize_rfactor(experiment, theory):
def optimize_rfactor(experiment: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a calculated modulation function against the measurement, adjusting their amplitude.
@@ -750,13 +879,15 @@ def optimize_rfactor(experiment, theory):
this is useful if the amplitudes of the two functions do not match due to systematic effects
of the calculation or the measurement.
the optimization is done in a scipy.optimize.least_squares optimization of the scaled_rfactor() function.
the optimization is done in a scipy.optimize.least_squares optimization of the scaled_rfactor_func() function.
the initial guess of the scaling factor is 0.7, the constraining boundaries are 1/10 and 10.
the input arrays must have the same shape and the coordinate columns must be identical (they are ignored).
the array elements are compared element-by-element.
terms having NaN intensity are ignored.
This function can be specified in the Scan.rfactor_func parameter of the project.
@param experiment: ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
@param theory: ETPI or ETPAI array containing the calculated modulation functions.
@@ -773,13 +904,13 @@ def optimize_rfactor(experiment, theory):
else:
wgts = np.ones_like(experiment['i'])
result = so.least_squares(scaled_rfactor, 0.7, bounds=(0.1, 10.0), args=(experiment['i'], wgts, theory['i']))
result_r = scaled_rfactor(result.x, experiment['i'], wgts, theory['i'])
result = so.least_squares(scaled_rfactor_func, 0.7, bounds=(0.1, 10.0), args=(experiment['i'], wgts, theory['i']))
result_r = scaled_rfactor_func(result.x, experiment['i'], wgts, theory['i'])
return result_r
def alpha_average(data):
def alpha_average(data: np.ndarray) -> np.ndarray:
"""
average I(alpha, theta, phi) over alpha.
@@ -809,7 +940,7 @@ def alpha_average(data):
return result
def phi_average(data):
def phi_average(data: np.ndarray) -> np.ndarray:
"""
average I(theta, phi) over phi.
@@ -827,9 +958,9 @@ def phi_average(data):
names = list(data.dtype.names)
names.remove('p')
dtype = [(name, data.dtype[name].str) for name in names]
result = create_data((nt), dtype=dtype)
result = create_data((nt,), dtype=dtype)
for i,t in enumerate(t_axis):
for i, t in enumerate(t_axis):
sel = np.abs(scan_positions['t'] - t) < 0.01
for name in names:
result[name][i] = np.mean(data[name][sel], dtype=np.float64)
@@ -839,7 +970,7 @@ def phi_average(data):
return result
def alpha_mirror_average(data):
def alpha_mirror_average(data: np.ndarray) -> np.ndarray:
"""
calculate the average of I(alpha, theta, phi) and I(-alpha, theta, phi).
@@ -871,3 +1002,14 @@ def alpha_mirror_average(data):
logger.warning('asymmetric alpha scan. skipping alpha mirror average.')
return result1
if loess is not None:
default_modfunc = calc_modfunc_loess
logger.info("pmsco.data.default_modfunc = pmsco.data.calc_modfunc_loess")
else:
default_modfunc = calc_modfunc_mean
logger.warning("pmsco.data.default_modfunc = pmsco.data.calc_modfunc_mean")
default_rfactor = square_diff_rfactor
logger.info("pmsco.data.default_rfactor = pmsco.data.square_diff_rfactor")

1312
pmsco/database.py
View File

File diff suppressed because it is too large Load Diff

169
pmsco/database/access.py Normal file
View File

@@ -0,0 +1,169 @@
"""
@package pmsco.database.access
wrapper classes for access to a pmsco database
the most import class to be used is DatabaseAccess.
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access here
# ...
# commit transaction
session.commit()
# continue in new transaction
# at the end of the context
# the session is closed and orm objects are detached from the database.
~~~~~~
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import fasteners
import logging
from pathlib import Path
import pmsco.database.orm as orm
logger = logging.getLogger(__name__)
class _DummyLock(object):
"""
dummy lock used for in memory database.
"""
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class LockedSession(object):
"""
database session context manager
this context manager (to be used in a with statement)
acquires a lock on the database lock file
and provides a database session (orm.Session()).
the session is closed (and pending transactions committed) on exit.
if an exception occurs, pending transactions are rolled back before the session is closed.
@note the term _session_ refers to a session in sqlalchemy.
"""
def __init__(self, lock_file=None):
self.lock_file = lock_file
self._session = None
self._lock = None
def __enter__(self):
self._lock = self.lock()
self._lock.__enter__()
self._session = orm.Session()
return self._session
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self._session.close()
else:
self._session.rollback()
self._session.close()
self._lock.__exit__(exc_type, exc_val, exc_tb)
self._lock = None
def lock(self):
"""
create a file-lock context manager for the database.
this is either a fasteners.InterProcessLock object on self._lock_filename
or a _DummyLock object if the database is in memory.
InterprocessLock allows to serialize access to the database by means of a lock file.
this is necessary if multiple pmsco instances require access to the same database.
_DummyLock is used with an in-memory database which does not require locking.
the lock object can be used as context-manager in a with statement.
"""
if self.lock_file:
return fasteners.InterProcessLock(self.lock_file)
else:
return _DummyLock()
class DatabaseAccess(object):
"""
basic database connection
this class maintains a database connection and builds session objects.
a _session_ corresponds to an sqlalchemy session, which defines the lifecycle of mapped objects.
a session can open one or multiple (subsequent) transactions.
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access
session.commit()
~~~~~~
the session object is a context handler.
it commits the transaction and closes the session at the end of the context.
if an exception occurs, it rolls back the transaction and closes the session before passing the exception.
"""
def __init__(self):
self.db_file = ""
self.lock_file = ""
def connect(self, db_file, lock_file=""):
"""
connect to a new or existing database file.
if the file does not exist, or if it is empty, a new database schema is created.
@param db_file: name of a file or ":memory:" for an in-memory database.
@param lock_file: name of a file that is used to lock the database.
by default, the db_filename with a suffix of ".lock" is used.
for most uses, the default should be fine.
the argument is provided mainly for testing the locking functionality.
this must be a file that is not used for anything else.
the file does not need to exist.
it's best if the file is in the same directory as the database file.
all clients of a database must use the same lock file.
@return: None
"""
self.db_file = db_file
if lock_file:
self.lock_file = lock_file
elif db_file == ":memory:":
self.lock_file = ""
else:
self.lock_file = Path(str(db_file) + ".lock")
orm.connect(orm.sqlite_link(self.db_file))
def session(self):
"""
open a database session.
this function returns a pmsco.database.util.LockedSession object
which is a context handler that provides an sqlalchemy session
that is locked against concurrent access from other DatabaseAccess instances.
see the class description for an example usage pattern.
@return: pmsco.database.util.LockedSession() object.
"""
return LockedSession(self.lock_file)

329
pmsco/database/common.py Normal file
View File

@@ -0,0 +1,329 @@
"""
@package pmsco.database.common
common database operations
this module gathers a number of common database operations.
all functions require an open session object from pmsco.database.access.DatabaseAccess.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import sqlalchemy
import pmsco.database.orm as orm
logger = logging.getLogger(__name__)
def filter_project(query, project_or_name_or_id):
"""
filter a query by project
@param query: sqlalchemy query object
@param project_or_name_or_id: orm.Project object or project name or project id.
@return: modified query
"""
if isinstance(project_or_name_or_id, orm.Project):
query = query.filter(orm.Project == project_or_name_or_id)
elif isinstance(project_or_name_or_id, int):
query = query.filter(orm.Project.id == project_or_name_or_id)
else:
query = query.filter(orm.Project.name == project_or_name_or_id)
return query
def filter_job(query, job_or_name_or_id):
"""
filter a query by job
@param query: sqlalchemy query object
@param job_or_name_or_id: orm.Job object or job name or job id.
@return: modified query
"""
if isinstance(job_or_name_or_id, orm.Job):
query = query.filter(orm.Job == job_or_name_or_id)
elif isinstance(job_or_name_or_id, int):
query = query.filter(orm.Job.id == job_or_name_or_id)
else:
query = query.filter(orm.Job.name == job_or_name_or_id)
return query
def query_params(session, project=None, job=None):
"""
query parameter names and their associated objects from the database
the result is a dictionary of orm.Param objects mapped to their respective keys.
the parameters can be filtered by project and/or job.
if no arguments are given, parameters from all projects are returned.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: dictionary of parameters
"""
query = session.query(orm.Param).join(orm.ParamValue).join(orm.Model).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
params = query.all()
params = {param.key: param for param in params}
return params
def query_tags(session, project=None, job=None):
"""
query tag names and their associated objects from the database
the result is a dictionary of orm.Tag objects mapped to their respective keys.
the tags can be filtered by project and/or job.
if no arguments are given, tags from all projects are returned.
@note the orm.Job.tags mapping is an alternative way to access job tags.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: dictionary of tags
"""
query = session.query(orm.Tag).join(orm.JobTag).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
tags = query.all()
tags = {tag.key: tag for tag in tags}
return tags
def query_job_tags(session, project=None, job=None):
"""
query tags (keys and values) from the database
the result is a dictionary of tag values (str) mapped to their respective keys (str).
the tags can be filtered by project and/or job.
if no arguments are given, tags from all projects are returned.
@note for one specific job, this is equivalent to the orm.Job.tags mapping.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: tags dictionary {key: value}
"""
query = session.query(orm.JobTag).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
job_tags = query.all()
job_tags = {jt.tag.key: jt.value for jt in job_tags}
return job_tags
def register_project(session, name, code, allow_existing=False):
"""
register (insert or query) a project with the database.
a new project record with the given parameters is inserted into the database.
if a project of the same name already exists, the existing record is returned.
@attention the orm.Project.id field is undefined until the session is committed!
it's better to identify a project by name or orm.Project object.
@note make sure previous changes have been committed. else the query may not find an existing project.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if a new project entry has been added.
@param name: project name. must be unique within the database.
@param code: name of the project module.
@param allow_existing: selects the behaviour if a project record exists in the database:
return the corresponding orm.Project (True) or raise an exception (False, default).
the exception is ValueError.
@return: orm.Project object.
the object can be used and modified as long as the session is active.
note that the id attribute is invalid until the session is committed!
@raise ValueError if the job exists and allow_existing is False.
"""
query = session.query(orm.Project)
query = query.filter(orm.Project.name == name)
project = query.one_or_none()
if project is None:
project = orm.Project(name=name, code=code)
session.add(project)
session.commit()
elif not allow_existing:
raise ValueError(f"project {project.name} exists")
return project
def get_project(session, project_or_name_or_id):
"""
resolve a project by name or id.
this function resolves a project specification to an orm.Project object.
if `project_or_name_or_id` is an orm.Project object, it just returns that object without any checks.
else, the project is looked up in the database.
@attention if `project_or_name_or_id` is an orm.Project object the function returns it without checks!
that means if the object is detached, you cannot use it to query results from the database.
if you need an object that is valid and in sync with the database,
resolve it by name or id!
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project_or_name_or_id: orm.Project object or project name or project id.
@return: orm.Project object
"""
if isinstance(project_or_name_or_id, orm.Project):
project = project_or_name_or_id
elif isinstance(project_or_name_or_id, int):
project = session.query(orm.Project).get(project_or_name_or_id)
else:
query = session.query(orm.Project)
query = query.filter(orm.Project.name == project_or_name_or_id)
project = query.one()
return project
def register_job(session, project, job_name, allow_existing=False, **job_attr):
"""
register (insert or query) a new job with the database.
a new job record with the given parameters is inserted into the database.
if a job of the same name exists within the given project, the existing record is returned
(without modifications!).
@note make sure previous changes have been committed. else the query may not find an existing project.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if a new job entry has been added.
@param project: orm.Project object or project name or project id.
@param job_name: name of job. unique in the project
@param job_attr: optional attributes of the job.
the keywords correspond to attribute names of the pmsco.database.Job object.
@param allow_existing: selects the behaviour if a job record exists in the database:
return the corresponding orm.Job (True) or raise an exception (False, default).
the exception is ValueError.
@return: orm.Job object.
the object can be used and modified as long as the session is active.
note that the id attribute is invalid until the session is committed!
@raise ValueError if the job exists and allow_existing is False.
"""
project = get_project(session, project)
query = session.query(orm.Job).join(orm.Project)
query = query.filter(orm.Project.name == project.name)
query = query.filter(orm.Job.name == job_name)
job = query.one_or_none()
if job is None:
job = orm.Job()
job.name = job_name
job.project = project
optional_args = {'mode', 'machine', 'git_hash', 'datetime', 'processes', 'hours', 'description'}
for name, value in job_attr.items():
if name in optional_args:
setattr(job, name, value)
session.add(job)
session.commit()
elif not allow_existing:
raise ValueError(f"a job {job_name} exists in project {project.name}")
return job
def get_job(session, project_or_name_or_id, job_or_name_or_id):
"""
resolve a job by name or id.
this function resolves any combination of project and job specification to an orm.Job object.
if `job_or_name_or_id` is an orm.Job object, it just returns that object without any checks.
else, the job is looked up in the database.
@attention if `job_or_name_or_id` is an orm.Job object the function returns it without checks!
that means if the object is detached, you cannot query results from the database.
if you need an object that is valid and in sync with the database,
query the job by name or id!
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project_or_name_or_id: orm.Project object or project name or project id.
@param job_or_name_or_id: orm.Job object or job name or job id.
@return: orm.Job object
"""
if isinstance(job_or_name_or_id, orm.Job):
job = job_or_name_or_id
elif isinstance(job_or_name_or_id, int):
job = session.query(orm.Job).get(job_or_name_or_id)
else:
project = get_project(session, project_or_name_or_id)
query = session.query(orm.Job).join(orm.Project)
query = query.filter(orm.Project.name == project.name)
query = query.filter(sqlalchemy.or_(orm.Job.id == job_or_name_or_id,
orm.Job.name == job_or_name_or_id))
job = query.one()
return job
def register_job_tags(session, job, tags):
"""
insert or update key-value tags of a job
this is one of many options to populate the Tag and JobTag tables.
it is not required to use this function.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param job: orm.Job object
@param tags: dictionary of tags
@return: None
"""
for k, v in tags.items():
job.tags[k] = v
if tags:
session.commit()
def register_params(session, params):
"""
register (insert missing) parameter names
add new parameter names to the global list of parameter names.
this is one of many options to populate the Param table.
it is not required to use this function.
this function implies a session flush.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if new parameters have been added
@param params: sequence of parameter names
param names with leading underscore are ignored.
@return: None
"""
existing_params = query_params(session).keys()
params = [param for param in params if param[0] != '_']
new_params = set(params) - set(existing_params)
for k in new_params:
session.add(orm.Param(key=k))
if new_params:
session.commit()

57
pmsco/database/git.py Normal file
View File

@@ -0,0 +1,57 @@
"""
@package pmsco.database.git
git metadata
this module retrieves the git hash of the running code for job metadata.
this requires that the code is run from a git repository
and that the gitpython package is installed.
gitpython is loaded on demand.
common errors (missing gitpython or invalid repository) are handled.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import importlib
def git():
"""
import the git module from GitPython
@return: git module or None if an error occurred
"""
try:
return importlib.import_module('git')
except ImportError:
return None
def get_git_hash(repo_path=None):
"""
get the git commit (hash) of the running code (HEAD)
the method looks for a git repository in the source tree of this module.
if successful, it returns the hash string of the HEAD commit.
@return: hexadecimal hash string.
empty string if the file is not in a git repository.
"""
if repo_path is None:
repo_path = __file__
_git = git()
if _git is not None:
try:
repo = _git.Repo(repo_path, search_parent_directories=True)
except _git.exc.InvalidGitRepositoryError:
return ""
else:
return repo.head.commit.hexsha
else:
return ""

406
pmsco/database/ingest.py Normal file
View File

@@ -0,0 +1,406 @@
"""
@package pmsco.database.ingest
ingest existing data such as flat results files (.dat or .tasks.dat) into a database.
the results file is a space-delimited, general text file
such as produced by pmsco.optimizers.population.Population.save_array().
each line contains one result dataset, the columns correspond to the regular and special parameters.
the first row contains the parameter names.
the main function is ingest_job_results().
the other functions require an open database session from pmsco.database.access.DatabaseAccess.session(),
and ingest the metadata and the actual results, respectively.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
import logging
import numpy as np
from pathlib import Path
from pmsco.database.access import DatabaseAccess
import pmsco.database.common as common
import pmsco.database.orm as orm
import pmsco.database.util as util
logger = logging.getLogger(__name__)
def insert_result(session, job, index, result, delta=None):
"""
add or update a calculation result including index and model to the database.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the model may not be found.
this function does not commit the transaction.
@param job: (orm.Job) job object.
use pmsco.database.common.get_object to retrieve by id or name.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
@return: (orm.Model, orm.Result) model and result objects
"""
model_obj = store_model(session, job, index, result)
result_obj = store_result_data(session, model_obj, index, result)
store_param_values(session, model_obj, result, delta)
return model_obj, result_obj
def store_model(session, job, index, result):
"""
add or update the model entry for a calculation result in the database.
the method updates the Models table.
the model is identified by job and index.model.
the result is identified by job and index.
if the model exists in the database, it is updated.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the model may not be found.
this function does not commit the transaction.
@param job: (orm.Job) job object.
use pmsco.database.common.get_object to retrieve by id or name.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen' and '_particle'.
'_gen' and '_particle' default to None if not present.
@return: (orm.Model) updated model object
"""
assert isinstance(job, orm.Job)
model_dict = {'gen': None, 'particle': None}
model_dict.update(util.special_params(result))
try:
model_dict['model'] = index.model
except AttributeError:
model_dict['model'] = index['_model']
q = session.query(orm.Model)
q = q.filter(orm.Model.job == job)
q = q.filter(orm.Model.model == model_dict['model'])
model_obj = q.one_or_none()
if model_obj is None:
model_obj = orm.Model()
model_obj.job = job
model_obj.model = model_dict['model']
session.add(model_obj)
model_obj.gen = model_dict['gen']
model_obj.particle = model_dict['particle']
return model_obj
def store_result_data(session, model_obj, index, result):
"""
add or update a result in the database.
the method updates the Results table.
the model is identified by model_id.
the result is identified by model_id and index.
if the result exists in the database, it is updated.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the result entry may not be found.
this function does not commit the transaction.
@param model_obj: (orm.Model) model object that is already part of the session.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@return: (orm.Result) updated Results object.
"""
assert isinstance(model_obj, orm.Model)
result_dict = util.special_params(result)
result_dict.update(util.special_params(index))
q = session.query(orm.Result)
q = q.filter(orm.Result.model == model_obj)
q = q.filter(orm.Result.scan == result_dict['scan'])
q = q.filter(orm.Result.domain == result_dict['domain'])
q = q.filter(orm.Result.emit == result_dict['emit'])
q = q.filter(orm.Result.region == result_dict['region'])
result_obj = q.one_or_none()
if result_obj is None:
result_obj = orm.Result()
result_obj.model = model_obj
result_obj.scan = result_dict['scan']
result_obj.domain = result_dict['domain']
result_obj.emit = result_dict['emit']
result_obj.region = result_dict['region']
session.add(result_obj)
result_obj.rfac = result_dict['rfac']
try:
result_obj.timestamp = result_dict['timestamp'].timestamp()
except KeyError:
result_obj.timestamp = datetime.datetime.now().timestamp()
except AttributeError:
result_obj.timestamp = result_dict['timestamp']
try:
result_obj.secs = result_dict['secs']
except KeyError:
pass
return result_obj
def store_param_values(session, model_obj, result, delta=None):
"""
add or update parameter values of a model in the database.
the method updates the ParamValues table.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the result entry may not be found.
this function flushes the session at the end.
it does not commit the transaction.
@param model_obj: (orm.Model) model object that is already part of the session.
@param result: (dict) dictionary containing the parameter values.
the parameter names must exist in the Params table and in the self._model_params dictionary.
special values (with a leading underscore) are ignored.
extra parameters may raise a KeyError.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
@return: None
@raise: KeyError if a parameter key is not registered.
"""
assert isinstance(model_obj, orm.Model)
for key in util.regular_params(result).keys():
pv = orm.ParamValue()
pv.model = model_obj
pv.param_key = key
pv.value = result[key]
try:
pv.delta = delta[key]
except (TypeError, KeyError):
pass
session.add(pv)
session.flush()
def ingest_results_file(session, project, job, filename):
"""
import a results file into the database.
this is a sub-method used by ingest().
a job entry with the given id must exist,
but there must be no model entries referencing the job.
it is not possible to update existing models, results or parameter values using this method.
instead, you have to delete the job (which also deletes all dependent entries)
and re-import the results.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is flushed but not committed at the end of this function.
@param project: orm.Project object or project name or project id.
@param job: orm.Job object or job name or job id.
@param filename: path and name of the results file.
@return: None.
@raise ValueError if the job already has model entries.
"""
job = common.get_job(session, project, job)
assert isinstance(job, orm.Job)
data = np.atleast_1d(np.genfromtxt(filename, names=True))
try:
unique_models, unique_index = np.unique(data['_model'], True)
except ValueError:
unique_models = np.array([0])
unique_index = np.array([0])
unique_data = data[unique_index]
special_params = util.special_params(data.dtype.names)
model_objs = {}
# iterate on models
for _data in unique_data:
try:
_model = _data['_model']
except ValueError:
_model = unique_models[0]
model = orm.Model(job=job, model=_model)
if 'gen' in special_params:
model.gen = _data['_gen']
if 'particle' in special_params:
model.particle = _data['_particle']
session.add(model)
model_objs[_model] = model
for key, value in util.regular_params(_data).items():
model.values[key] = value
session.flush()
# iterate on results
for _data in data:
try:
_model = _data['_model']
except ValueError:
_model = unique_models[0]
result_entry = {'model': None,
'scan': -1,
'domain': -1,
'emit': -1,
'region': -1,
'rfac': None}
result_entry.update(util.special_params(_data))
result_entry['model'] = model_objs[_model]
result = orm.Result()
for key, value in result_entry.items():
setattr(result, key, value)
session.add(result)
session.flush()
def ingest_job_metadata(session, **kwargs):
"""
ingest job metadata
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is flushed but not committed at the end of this function.
@param kwargs: dictionary of function arguments.
the dictionary contains the following values.
all arguments are required unless noted.
@arg 'resultsfile' (required) name of the .tasks.dat results file.
@arg 'project' (required) unique name of the project.
@arg 'code' (optional) name of the project code.
@arg 'job' (required) name of the calculation job. job name must not exist for the project yet.
@arg 'mode' (required) pmsco optimization mode.
@arg 'machine' (optional) name of the machine where the job ran.
@arg 'processes' (optional) number of processes.
@arg 'hours' (optional) run time in hours (wall time).
@arg 'git_hash' (optional) git hash of the code revision.
@arg 'datetime' (datetime.datetime) time stamp (optional).
if not specified, the argument defaults to the time stamp of the results file.
hint: the constructor of a datetime object is
`datetime.datetime(year, month, day, hour, minute, second)`.
@arg 'description' (optional) meaningful description of the calculation job, up to the user.
@arg 'jobtags' (dict, optional) key=value tags to be associated with the job
@return (orm.Project, orm.Job) orm objects of the inserted records.
@raise sqlalchemy.exc.IntegrityError if the job already exists in the database.
"""
if 'datetime' not in kwargs:
rf = Path(kwargs['resultsfile'])
kwargs['datetime'] = datetime.datetime.fromtimestamp(rf.stat().st_mtime)
project = common.register_project(session, kwargs['project'], kwargs['code'])
job = common.register_job(session, project, kwargs['job'], **kwargs)
try:
common.register_job_tags(session, job, kwargs['jobtags'])
except KeyError:
pass
session.flush()
return project, job
def ingest_job_results(**kwargs):
"""
import results from a calculation job.
this function contains all steps necessary to import the results (tasks.dat)
from a calculation job into a database.
it registers the project and job, and imports the results data.
the project may exist in the database, the job must not exist (raises an exception).
arguments can be specified as dict (**d) or in keyword=value form.
@param kwargs: dictionary of function arguments.
the dictionary contains the following values.
all arguments are required unless noted.
@arg 'workdir' (optional) path to the working directory.
the working directory of the operating system is changed.
this is the root for relative paths of the database and results files.
if not specified, the working directory is unchanged.
@arg 'dbfile' (required) name of the database file.
@arg 'project' (required) unique name of the project.
@arg 'code' (optional) name of the project code.
@arg 'job' (required) name of the calculation job. job name must not exist for the project yet.
@arg 'mode' (required) pmsco optimization mode.
@arg 'machine' (optional) name of the machine where the job ran.
@arg 'processes' (optional) number of processes.
@arg 'hours' (optional) run time in hours (wall time).
@arg 'git_hash' (optional) git hash of the code revision.
@arg 'datetime' (datetime.datetime) time stamp (optional).
if not specified, the argument defaults to the time stamp of the results file.
hint: the constructor of a datetime object is
`datetime.datetime(year, month, day, hour, minute, second)`.
@arg 'description' (optional) meaningful description of the calculation job, up to the user.
@arg 'jobtags' (dict, optional) key=value tags to be associated with the job
@arg 'resultsfile' (required) name of the .tasks.dat results file.
@return dict with 'project_id' and 'job_id'.
these are the database ids of the project and job records.
@raise sqlalchemy.exc.IntegrityError if the job already exists in the database.
"""
try:
wd = Path(kwargs['workdir'])
except KeyError:
pass
else:
wd.cwd()
dba = DatabaseAccess()
dba.connect(kwargs['dbfile'])
with dba.session() as session:
project, job = ingest_job_metadata(session, **kwargs)
ingest_results_file(session, project, job, kwargs['resultsfile'])
session.commit()
ref = {'project_id': project.id, 'job_id': job.id}
return ref

746
pmsco/database/orm.py Normal file
View File

@@ -0,0 +1,746 @@
"""
@package pmsco.database.orm
pmsco results database object-relational mapper
this module declares the database schema and object mapping.
the object-relational mapping uses
the [sqlalchemy framework](https://docs.sqlalchemy.org/en/13/orm/tutorial.html).
the database backend is sqlite3.
for examples how to use the database, see the ingest module and the unit tests.
@author Matthias Muntwiler
@copyright (c) 2021 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
from sqlalchemy import create_engine
from sqlalchemy import event
from sqlalchemy import Column, Sequence, ForeignKey
from sqlalchemy import Boolean, Integer, Float, String, DateTime
from sqlalchemy.engine import Engine
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import object_session
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import validates
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm.exc import NoResultFound
import numpy as np
import sqlite3
from pmsco.dispatch import CalcID
import pmsco.database.util as db_util
# make sure sqlite understands numpy data types
sqlite3.register_adapter(np.float64, float)
sqlite3.register_adapter(np.float32, float)
sqlite3.register_adapter(np.int64, int)
sqlite3.register_adapter(np.int32, int)
Base = declarative_base()
engine = None
Session = sessionmaker()
class Project(Base):
"""
database object representing a project
@note there is an implicit constructor with keyword arguments that correspond to the attributes.
"""
## @var id
# (int, primary key) database id of the project
## @var name
# project name, should be short, must be unique within a project
## @var jobs
# collection of related jobs
#
# defines the relationship between Project and Job objects.
# the instance attribute maps job names (str) to Job objects.
__tablename__ = "Projects"
id = Column(Integer, Sequence('project_id_seq'), primary_key=True)
name = Column(String(50, collation='NOCASE'), nullable=False, unique=True)
code = Column(String(50, collation='NOCASE'))
jobs = relationship('Job', backref='project',
collection_class=attribute_mapped_collection('name'),
cascade="all, delete, delete-orphan", lazy='joined')
def __repr__(self):
return f'Project({repr(self.name), repr(self.code)})'
class Job(Base):
"""
database object representing a calculation job
a job object holds several descriptive values of a calculation job.
it also refers to a project.
tags are key-value pairs that describe the job in standardized terms.
they can provide a consistent classification scheme across jobs and projects.
for example, they can store special project arguments that may be important
to distinguish calculations in different stages or contexts.
the class also defines mapping and proxy objects that simplify the use of tags and models.
explicit creation of Tag and JobTag objects is then not necessary.
@attention after modifying the mapped collections job_tags, tags or models
make sure to call flush() or commit() on the session
before accessing those mappings in other objects
else integrity errors may occur!
"""
## @var id
# (int, primary key) database id of the job
## @var project_id
# (int, foreign key) database id of the related project
## @var name
# job name, should be short, must be unique within a project
## @var mode
# pmsco calculation mode
## @var machine
# name of the computing facility
## @var git_hash
# git hash of the used code if under version control
## @var datetime
# start date and time of the job, ISO format (yyyy-mm-dd hh:mm:ss)
## @var processes
# number of processes
## @var hours
# job run time (wall time) in hours
## @var description
# up to the user
## @var job_tags
# collection of related job tags
#
# defines the relationship between Job and JobTag objects.
# the instance attribute maps tag keys (str) to JobTag objects.
## @var tags
# collection of tags
#
# maps tag keys (str) to tag values (str).
# this is an association proxy of job_tags.
## @var models
# collection of related models
#
# defines the relationship between Job and Model objects.
# the instance attribute maps model numbers to Model objects
__tablename__ = "Jobs"
id = Column(Integer, Sequence('job_id_seq'), primary_key=True)
project_id = Column(Integer, ForeignKey('Projects.id'), index=True)
name = Column(String(50, collation='NOCASE'), nullable=False)
mode = Column(String(20, collation='NOCASE'))
machine = Column(String(50, collation='NOCASE'))
git_hash = Column(String(50, collation='NOCASE'))
datetime = Column(String(50))
processes = Column(Integer)
hours = Column(Float)
description = Column(String(200, collation='NOCASE'))
job_tags = relationship('JobTag', back_populates='job',
collection_class=attribute_mapped_collection('tag_key'),
cascade="all, delete, delete-orphan")
# mapping tag_key -> tag_value
tags = association_proxy('job_tags', 'value', creator=lambda k, v: JobTag(key=k, value=v))
models = relationship('Model', back_populates='job',
collection_class=attribute_mapped_collection('model'),
cascade="all, delete, delete-orphan")
def __repr__(self):
try:
project_name = repr(self.project.name)
except AttributeError:
project_name = None
try:
job_name = repr(self.name)
except AttributeError:
job_name = None
return f'Job({project_name}, {job_name}, {repr(self.mode)})'
class Tag(Base):
"""
database object representing a tag name
"""
## @var id
# (int, primary key) database id of the tag name
## @var key
# tag name/key, should be short, must be unique
## @var tag_jobs
# collection of related JobTag objects
#
# defines the relationship between Tag and JobTag objects.
__tablename__ = "Tags"
id = Column(Integer, Sequence('tag_id_seq'), primary_key=True)
key = Column(String(20, collation='NOCASE'), nullable=False, unique=True)
tag_jobs = relationship('JobTag', back_populates='tag', cascade="all, delete, delete-orphan")
def __init__(self, key):
self.key = key
def __repr__(self):
return f'Tag({repr(self.key)})'
class JobTag(Base):
"""
association object class for job tags
Job - Tag is a many-to-many relationship built using this association class.
by using the dictionary-like Job.tags proxy, explicit creation of association objects can be avoided.
the class applies the
[UniqueObjectValidateOnPending pattern](https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObjectValidatedOnPending)
to look up existing tags in the database when a Tag object is needed and only the key is given.
"""
## @var id
# (int, primary key) database id of the job tag
## @var tag_id
# (int, foreign key) database id of the related tag name
## @var job_id
# (int, foreign key) database id of the related job
## @var value
# value (str) of the job tag
## @var tag
# associated Tag object
#
# defines the relationship between JobTag and Tag objects
## @var job
# associated Job object
#
# defines the relationship between JobTag and Job objects
## @var tag_key
# key (name) of the asscoiated Tag object
#
# this is an association proxy that provides direct access to tag.key
# or links to or creates a Tag object behind the scenes.
__tablename__ = "JobTags"
id = Column(Integer, Sequence('jobtag_id_seq'), primary_key=True)
tag_id = Column(Integer, ForeignKey('Tags.id'), index=True)
job_id = Column(Integer, ForeignKey('Jobs.id'), index=True)
value = Column(String(200, collation='NOCASE'))
tag = relationship("Tag", back_populates="tag_jobs")
job = relationship("Job", back_populates="job_tags")
tag_key = association_proxy("tag", "key")
def __init__(self, key=None, value=None):
if key is not None:
self.tag_key = key
self.value = value
@validates("tag")
def _validate_tag(self, key, value):
"""
receive the event that occurs when `jobtag.tag` is set.
if the object is present in a Session, then make sure it's the Tag
object that we looked up from the database.
otherwise, do nothing and we'll fix it later when the object is
put into a Session.
@param key: attribute name, i.e., 'tag'
@param value: a JobTag object
"""
sess = object_session(self)
if sess is not None:
return _setup_tag(sess, value)
else:
return value
@event.listens_for(Session, "transient_to_pending")
def _validate_tag(session, object_):
"""
receive a JobTag object when it gets attached to a Session to correct its unique Tag relationship.
"""
if isinstance(object_, JobTag):
if object_.tag is not None and object_.tag.id is None:
old_tag = object_.tag
new_tag = _setup_tag(session, object_.tag)
if new_tag is not old_tag:
if old_tag in session:
session.expunge(old_tag)
object_.tag = new_tag
def _setup_tag(session, tag_object):
"""
given a Session and a Tag object, return the correct Tag object from the database.
"""
with session.no_autoflush:
try:
return session.query(Tag).filter_by(key=tag_object.key).one()
except NoResultFound:
return tag_object
class Model(Base):
"""
database object representing a model
the object holds the model number (which is unique within the context of a single job only),
the diagnostic generation and particle values, and refers to the job where the model is used.
the class also defines relationship properties that simplify access to referenced objects.
for instance, parameter values can be accessed via the values['param_key'] mapping proxy.
examples:
~~~~~~{.py}
model = Model(model=10, gen=5, particle=2)
model.job = job1_object
model.values['dA'] = 25.6
model.deltas['dA'] = 0.1
pv = ParamValue(value=39.0, delta=-0.3)
model.param_values['dB'] = pv
result = Result(calc_id=calc_id, rfac=0.77)
model.results.append(result)
~~~~~~
@attention after modifying the mapped collections param_values, values or deltas,
make sure to call flush() or commit() on the session
before accessing those mappings in another model
else integrity errors may occur!
"""
## @var id
# (int, primary key) database id of the model
## @var job_id
# (int, foreign key) database id of the related job
## @var model
# (int) model number as used in the task index of pmsco
#
# @note the model number is not unique in the database as multiple jobs can produce same task indices.
# the unique number, self.id is not used in pmsco code.
## @var gen
# (int) generation number assigned by some optimizers. defaults to None.
## @var particle
# (int) particle number assigned by some optimizers. defaults to None.
## @var job
# associated Job
#
# defines the relationship between Model and Job objects.
## @var results
# collection of Result objects
#
# defines the relationship between Model and Result objects.
## @var param_values
# collection of ParamValue objects
#
# defines the relationship between Model and ParamValue objects.
# the instance attribute maps parameter keys to ParamValue objects.
## @var values
# collection of parameter values
#
# this is an association proxy that maps parameter keys to parameter values (ParamValue.value).
# ParamValue objects are accessed and created behind the scene.
## @var deltas
# collection of delta values
#
# this is an association proxy that maps parameter keys to parameter deltas (ParamValue.delta.
# ParamValue objects are accessed and created behind the scene.
__tablename__ = "Models"
id = Column(Integer, Sequence('model_id_seq'), primary_key=True)
job_id = Column(Integer, ForeignKey('Jobs.id'), index=True)
model = Column(Integer, index=True)
gen = Column(Integer)
particle = Column(Integer)
job = relationship("Job", back_populates="models")
results = relationship('Result', back_populates='model', cascade="all, delete, delete-orphan")
# mapping param_key -> ParamValue object
param_values = relationship('ParamValue', back_populates='model',
collection_class=attribute_mapped_collection('param_key'),
cascade="all, delete, delete-orphan")
# mapping param_key -> param_value
values = association_proxy('param_values', 'value', creator=lambda k, v: ParamValue(key=k, value=v))
deltas = association_proxy('param_values', 'delta', creator=lambda k, v: ParamValue(key=k, delta=v))
def __repr__(self):
return f'Model(id={repr(self.id)}, job_id={repr(self.job_id)}, model={repr(self.model)})'
def as_dict(self):
"""
object properties in a dictionary.
the dictionary keys correspond to the column names of numpy arrays.
the mapping db_field -> column name is declared in pmsco.database.util.DB_SPECIAL_PARAMS
@return: (dict)
"""
d = {'_db_model_id': self.id}
for attr, key in db_util.DB_SPECIAL_PARAMS.items():
try:
d[key] = getattr(self, attr)
except AttributeError:
pass
return d
class Result(Base):
"""
database object representing a calculation result
the result object holds the calculated R-factor per job and calculation index.
the calculation index (CalcID) is not unique in the database because it may contain results from multiple jobs.
thus, the object links to a Model object which is unique.
the calc_id property can be used to reconstruct a CalcID.
"""
## @var id
# (int, primary key) database id of the result
## @var model_id
# (int, foreign key) database id of the related model
## @var model
# associated Model object
#
# defines the relationship between Result and Model objects.
#
# @attention do not confuse the Result.model and Model.model attributes of same name!
# to obtain the model number to which a result belongs, use Result.model.model.
## @var scan
# (int) scan index as used in the calculations
## @var domain
# (int) domain index as used in the calculations
## @var emit
# (int) emitter index as used in the calculations
## @var region
# (int) region index as used in the calculations
## @var rfac
# (float) calculated R-factor
## @var timestamp
# (float) end date and time of this calculation task
#
# the float value represents seconds since jan 1, 1970 (datetime.datetime.timestamp).
# the datetime proxy converts to and from python datetime.datetime.
## @var datetime
# (datetime.datetime) end date and time of this calculation task
#
# this is a conversion proxy for timestamp.
## @var secs
# (float) total duration of the calculation task in seconds
#
# total cpu time necessary to get this result (including child tasks) in seconds.
## @var calc_id
# (CalcID) calculation task index
#
# conversion proxy for the task index components.
#
# on assignment, the scan, domain, emit and region attributes are updated.
# it does not update the model index as it is not stored by this object!
# the model index must be set separately in the linked Model object.
__tablename__ = "Results"
id = Column(Integer, Sequence('result_id_seq'), primary_key=True)
model_id = Column(Integer, ForeignKey('Models.id'), index=True)
scan = Column(Integer, index=True)
domain = Column(Integer, index=True)
emit = Column(Integer, index=True)
region = Column(Integer, index=True)
rfac = Column(Float)
timestamp = Column(Float)
secs = Column(Float)
model = relationship("Model", back_populates="results")
def __init__(self, calc_id=None, scan=None, domain=None, emit=None, region=None,
rfac=None, timestamp=None, secs=None):
if calc_id is not None:
self.calc_id = calc_id
else:
self.scan = scan
self.domain = domain
self.emit = emit
self.region = region
self.rfac = rfac
self.timestamp = timestamp
self.secs = secs
def __repr__(self):
return f'Result(model_id={repr(self.model_id)}, calc_id={repr(self.calc_id)}, rfac={repr(self.rfac)})'
@property
def calc_id(self):
return CalcID(self.model.model, self.scan, self.domain, self.emit, self.region)
@calc_id.setter
def calc_id(self, calc_id):
self.scan = calc_id.scan
self.domain = calc_id.domain
self.emit = calc_id.emit
self.region = calc_id.region
@property
def datetime(self):
return datetime.datetime.fromtimestamp(self.timestamp)
@datetime.setter
def datetime(self, value):
self.timestamp = value.timestamp()
def as_dict(self):
"""
object properties in a dictionary.
the dictionary keys correspond to the column names of numpy arrays.
the mapping db_field -> column name is declared in pmsco.database.util.D.B_SPECIAL_PARAMS
@return: (dict)
"""
d = {'_db_result_id': self.id}
for attr, key in db_util.DB_SPECIAL_PARAMS.items():
try:
d[key] = getattr(self, attr)
except AttributeError:
pass
return d
class Param(Base):
"""
database object representing a parameter
the parameter object holds the name (or key) of a calculation parameter.
explicit creation of parameter objects can be avoided by using the mappings of the Model class.
"""
## @var id
# (int, primary key) database id of the parameter name
## @var key
# parameter name/key as used in calculations, should be very short, must be unique
## @var param_values
# collection of related ParamValue objects
#
# defines the relationship between Param and ParamValue objects.
__tablename__ = "Params"
id = Column(Integer, Sequence('param_id_seq'), primary_key=True)
key = Column(String(20, collation='NOCASE'), nullable=False, unique=True)
param_values = relationship('ParamValue', back_populates='param', cascade="all, delete, delete-orphan")
def __init__(self, key):
self.key = key
def __repr__(self):
return f'Param({repr(self.key)})'
class ParamValue(Base):
"""
association object class for parameter values
Model - Param is a many-to-many relationship built using this association class.
by using the dictionary-like Model.values and Model.deltas proxies,
explicit creation of association objects can be avoided.
the class applies the
[UniqueObjectValidateOnPending pattern](https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObjectValidatedOnPending)
to look up existing params in the database when a Param object is needed and only the key is given.
"""
## @var id
# (int, primary key) database id of the parameter value
## @var param_id
# (int, foreign key) database id of the related parameter name
## @var model_id
# (int, foreign key) database id of the related model
## @var value
# (float) numeric value of the parameter
## @var delta
# (float) numeric delta value of the parameter (reported by some optimizers)
## @var param
# associated Param object
#
# defines the relationship between ParamValue and Param objects
## @var model
# associated Model object
#
# defines the relationship between ParamValue and Model objects
## @var param_key
# key (name) of the asscoiated Param object
#
# this is an association proxy that provides direct access to param.key.
# it accesses or creates Param objects behind the scenes.
__tablename__ = "ParamValues"
id = Column(Integer, Sequence('paramvalue_id_seq'), primary_key=True)
param_id = Column(Integer, ForeignKey('Params.id'), index=True)
model_id = Column(Integer, ForeignKey('Models.id'), index=True)
value = Column(Float)
delta = Column(Float)
param = relationship("Param", back_populates="param_values")
model = relationship("Model", back_populates="param_values")
param_key = association_proxy('param', 'key')
def __init__(self, model=None, param=None, key=None, value=None, delta=None):
if model is not None:
self.model = model
if param is not None:
self.param = param
elif key is not None:
self.param_key = key
self.value = value
self.delta = delta
@validates("param")
def _validate_param(self, key, value):
"""
receive the event that occurs when `paramvalue.param` is set.
if the object is present in a Session, then make sure it's the Param
object that we looked up from the database.
otherwise, do nothing and we'll fix it later when the object is put into a Session.
"""
sess = object_session(self)
if sess is not None:
return _setup_param(sess, value)
else:
return value
@event.listens_for(Session, "transient_to_pending")
def _validate_param(session, object_):
"""
receive a ParamValue object when it gets attached to a Session to correct its unique Param relationship.
"""
if isinstance(object_, ParamValue):
if object_.param is not None and object_.param.id is None:
old_param = object_.param
new_param = _setup_param(session, object_.param)
if new_param is not old_param:
if old_param in session:
session.expunge(old_param)
object_.param = new_param
def _setup_param(session, param_object):
"""
given a Session and a Tag object, return the correct Tag object from the database.
"""
with session.no_autoflush:
try:
return session.query(Param).filter_by(key=param_object.key).one()
except NoResultFound:
return param_object
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
"""
set sqlite pragmas.
make sure sqlite enforces relational integrity.
@param dbapi_connection:
@param connection_record:
@return:
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def sqlite_link(path=None):
"""
format the sqlalchemy link to an sqlite3 database.
@param path: file path. if empty, an in-memory database is created.
@return: (str) database link for the sqlalchemy engine.
"""
if not path:
path = ':memory:'
return f'sqlite:///{path}'
def connect(db_link):
"""
connect to the database.
create the sqlalchemy engine and bind the session maker.
the database engine and session maker are global.
this function should be called only once in a process.
@param db_link: (str) database link expected by the sqlalchemy engine
@return: None
"""
global engine
engine = create_engine(db_link, echo=False)
Base.metadata.create_all(engine)
Session.configure(bind=engine)

158
pmsco/database/project.py Normal file
View File

@@ -0,0 +1,158 @@
"""
@package pmsco.database.project
wrapper class for project-specific database operations
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access here
# ...
# commit transaction
session.commit()
# continue in new transaction
# ...
# at the end of the context
# the session is closed and orm objects are detached from the database.
~~~~~~
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
import logging
import socket
from pmsco.database.access import DatabaseAccess
import pmsco.database.common as db_common
import pmsco.database.ingest as db_ingest
import pmsco.database.query as db_query
from pmsco.dispatch import mpi_size
logger = logging.getLogger(__name__)
class ProjectDatabase(DatabaseAccess):
"""
wrapper class for project specific database operations
the purpose of this class is to bundle all specific code and run-time information
for database access of a running calculation job.
after calling ingest_project_metadata(),
the class object stores the persistent project and job identifiers.
the other methods provide convenient wrappers so that database code can be kept minimal in the project.
usage:
~~~~~~{.py}
db = ProjectDatabase()
db.connect('file.db')
db.ingest_project_metadata(...)
for result in results:
db.ingest_result(result...)
~~~~~~
"""
def __init__(self):
super().__init__()
self.db_project_id = None
self.db_job_id = None
def ingest_project_metadata(self, project):
"""
ingest project metadata into the database
@param project: pmsco.project.Project object
@return: None
"""
with self.session() as session:
db_project = db_common.register_project(session=session,
name=project.project_name,
code=project.__module__,
allow_existing=True)
db_job = db_common.register_job(session=session,
project=db_project,
job_name=project.job_name,
allow_existing=False,
mode=project.mode,
machine=socket.gethostname(),
git_hash=project.git_hash,
datetime=datetime.datetime.now(),
processes=mpi_size,
hours=project.timedelta_limit.total_seconds() / 3600.,
description=project.description)
db_common.register_job_tags(session, db_job, project.job_tags)
db_common.register_params(session, project.model_space.start.keys())
session.commit()
self.db_project_id = db_project.id
self.db_job_id = db_job.id
def ingest_result(self, index, result, delta):
"""
add or update a result in the database.
the method updates the Models, Results and ParamValues tables.
the model is identified by self.job_id and index.model.
the result is identified by self.job_id and index.
if the model or result exists in the database, it is updated.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
"""
assert self.db_project_id is not None
assert self.db_job_id is not None
with self.session() as session:
job_obj = db_common.get_job(session, self.db_project_id, self.db_job_id)
model_obj = db_ingest.store_model(session, job_obj, index, result)
db_ingest.store_result_data(session, model_obj, index, result)
db_ingest.store_param_values(session, model_obj, result, delta)
session.commit()
def query_best_task_models(self, level, count):
"""
query N best models per task.
this is a wrapper for pmsco.database.query.query_best_task_models().
in addition to the wrapped function, it opens a session and uses the registered db_job_id.
this query is used by the file tracker to determine the models to keep.
@param level: level up to which to query.
the level can be specified by level name (str) or numeric index (0..4).
if it is scan (equivalent to 1), the method queries the model and scan levels.
@param count: number of models to query per task.
@return set of matching model numbers (model index, Models.model field).
"""
with self.session() as session:
models = db_query.query_best_task_models(session, self.db_job_id, level, count)
return models

470
pmsco/database/query.py Normal file
View File

@@ -0,0 +1,470 @@
"""
@package pmsco.database.query
specialized query functions for the pmsco database
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import numpy as np
from sqlalchemy import func
import pmsco.database.orm as orm
import pmsco.database.util as util
import pmsco.dispatch as dispatch
logger = logging.getLogger(__name__)
def query_newest_job(session):
"""
retrieve the entry of the newest job
the newest entry is determined by the datetime field.
@param session:
@return: pmsco.database.orm.Job object
"""
q = session.query(orm.Job)
q = q.order_by(orm.Job.datetime.desc(), orm.Job.id.desc())
job = q.first()
return job
def query_model(session, job_id=None, model_id=None, model=None):
"""
retrieve model parameters and control variables from the database.
@param model_id: id of the model in the database.
@return: (dict, dict) value dictionary and delta dictionary.
dictionary keys are parameter values.
the special value '_model' is included.
"""
query = session.query(orm.ParamValue)
if job_id is not None:
query = query.filter(orm.Job.id == job_id)
if model_id is not None:
query = query.filter(orm.Model.id == model_id)
if model is not None:
query = query.filter(orm.Model.model == model)
result = query.all()
param_value = {}
param_delta = {}
model_obj = None
for pv in result:
if model_obj is None:
model_obj = pv.model
param_value[pv.param.key] = pv.value
param_delta[pv.param.key] = pv.delta
param_value['_model_id'] = model_obj.id
param_value['_model'] = model_obj.model
param_value['_gen'] = model_obj.gen
param_value['_particle'] = model_obj.particle
param_delta['_model_id'] = model_obj.id
param_delta['_model'] = model_obj.model
param_delta['_gen'] = model_obj.gen
param_delta['_particle'] = model_obj.particle
return param_value, param_delta
def query_results(session, job_id):
query = session.query(orm.Result)
query = query.join(orm.Model)
query = query.filter(orm.Job == job_id)
return None
def query_tasks(session, job_id):
"""
query the task index used in a calculation job.
this query neglects the model index
and returns the unique tuples (-1, scan, domain, emit, region).
@param job_id: (int) id of the associated Jobs entry.
@return list of pmsco.dispatch.CalcID tuples of task indices.
the model attribute is -1 in all elements.
"""
query = session.query(orm.Result.scan, orm.Result.domain, orm.Result.emit, orm.Result.region)
query = query.join(orm.Model)
query = query.filter(orm.Model.job_id == job_id)
query = query.distinct()
query = query.order_by(orm.Result.scan, orm.Result.domain, orm.Result.emit, orm.Result.region)
results = query.all()
output = []
for row in results:
d = row._asdict()
d['model'] = -1
output.append(dispatch.CalcID(**d))
return output
def query_best_task_models(session, job_id, level, count):
"""
query N best models per task.
this query is used by the file tracker to determine the models to keep.
@param job_id: (int) id of the associated Jobs entry.
@param level: level up to which to query.
the level can be specified by level name (str) or numeric index (0..4).
if it is scan (equivalent to 1), the method queries the model and scan levels.
@param count: number of models to query per task.
@return set of matching model numbers (Models.model field).
"""
try:
level = int(level)
except ValueError:
level = dispatch.CALC_LEVELS.index(level)
assert 0 <= level < len(dispatch.CALC_LEVELS)
def _query_models(t):
query = session.query(orm.Model.model).join(orm.Job).join(orm.Result)
query = query.filter(orm.Job.id == job_id)
query = query.filter(orm.Result.scan == t.scan)
query = query.filter(orm.Result.domain == t.domain)
query = query.filter(orm.Result.emit == t.emit)
query = query.filter(orm.Result.region == t.region)
query = query.order_by(orm.Result.rfac)
results = query[0:count]
return set((row.model for row in results))
tasks = query_tasks(session, job_id)
models = set()
for task in tasks:
if task.numeric_level <= level:
q_models = _query_models(task)
models |= q_models
return models
def query_model_params_array(session, jobs=None, models=None, order=None, limit=None):
"""
query parameter values and return them in a numpy array
the models table can be filtered by job and/or model.
else, the whole database is returned (which might be huge!).
@param session:
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param models: filter by model.
the argument can be a singleton or sequence of orm.Model objects or their id.
@param order: ordering of results. this can be a sequence of orm.Model attributes.
the default order is by job_id and model.
@param limit: maximum number of models to return
@return: dict['values']: numpy values array, dict['deltas']: numpy deltas array
"""
count_query = session.query(orm.Model)
pn_query = session.query(orm.Param.key)
pv_query = session.query(orm.ParamValue)
if jobs:
try:
jobs = [int(jobs)]
except TypeError:
pass
job_ids = [j if isinstance(j, int) else j.id for j in jobs]
count_query = count_query.filter(orm.Model.job_id.in_(job_ids))
pn_query = pn_query.filter(orm.Model.job_id.in_(job_ids))
pv_query = pv_query.filter(orm.Model.job_id.in_(job_ids))
if models:
try:
models = [int(models)]
except TypeError:
pass
model_ids = [m if isinstance(m, int) else m.id for m in models]
count_query = count_query.filter(orm.ParamValue.model_id.in_(model_ids))
pn_query = pn_query.filter(orm.ParamValue.model_id.in_(model_ids))
pv_query = pv_query.filter(orm.ParamValue.model_id.in_(model_ids))
if order is not None:
pv_query = pv_query.order_by(*order)
else:
pv_query = pv_query.order_by(orm.Model.job_id, orm.Model.model)
if limit:
pv_query = pv_query[0:limit]
n_models = count_query.count()
param_names = pn_query.all()
param_values = pv_query.all()
special_names = orm.Model().as_dict().keys()
dt_names = special_names + param_names
dt = np.dtype([(n, util.field_to_numpy_type(n)) for n in sorted(dt_names, key=str.lower)])
values = np.zeros((n_models,), dtype=dt)
deltas = np.zeros((n_models,), dtype=dt)
for i, pv in enumerate(param_values):
for k, v in pv.model.as_dict():
values[i][k] = deltas[i][k] = v
values[i][pv.param_key] = pv.value
deltas[i][pv.param_key] = pv.delta
return {'values': values, 'deltas': deltas}
calc_id_props = {'model': orm.Model.model,
'scan': orm.Result.scan,
'domain': orm.Result.domain,
'emit': orm.Result.emit,
'region': orm.Result.region}
def query_model_results_array(session, jobs=None, models=None, order=None, limit=None,
query_hook=None, hook_data=None, include_params=False, **index):
"""
query a results table with flexible filtering options
the function returns a structured numpy array of the results and, optionally, parameter values.
the database is fully flattened, row of the array represents one result.
the jobs and models arguments filter for specific jobs and/or models.
custom filters can be added in a query hook function.
the hook function receives an sqlalchemy Query object of the Result table,
joined with the Model and Job tables.
other joins must be added explicitly.
the hook function can add more filters and return the modified query.
the hook function is called after the filters from the other function arguments
(job, models, index) have been applied,
and before the ordering and limit are applied.
@param session:
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param models: filter by model.
the argument can be a singleton or sequence of orm.Model objects or their id.
@param order: ordering of results. this can be a sequence of orm.Result attributes.
the default order is by `orm.Result.rfac`.
to override the default ascending order, append a modifier, e.g., `orm.Result.rfac.desc()`.
@param limit: maximum number of models to return
@param query_hook: hook function that modifies an sqlalchemy.orm.Query object.
the function receives the query as first argument, and any data from hook_data as keyword arguments.
it must return the modified query object.
@param hook_data: (dict) keyword arguments to be passed to the query_hook function.
@param include_params: include parameter values of each model in the result.
by default, only data from the Model and Result records is included.
@param index: filters the results list by scan, domain, emit, and/or region index.
for example, to get only the final results per model, specify `scan=-1`.
@return: numpy values array
"""
results_query = session.query(orm.Result).join(orm.Model).join(orm.Job)
if jobs:
results_query = filter_objects(results_query, orm.Job, jobs)
if models:
results_query = filter_objects(results_query, orm.Model, models)
for k, v in index.items():
results_query = results_query.filter(calc_id_props[k] == v)
if query_hook is not None:
results_query = query_hook(results_query, **hook_data)
if order is not None:
results_query = results_query.order_by(*order)
if limit:
results = results_query[0:limit]
else:
results = results_query.all()
n_results = len(results)
logger.debug(f"query_model_results_array: {results_query.statement} ({n_results} rows)")
dt_names = [n for n in util.DB_SPECIAL_PARAMS.values()]
if include_params:
model_ids = {r.model_id for r in results}
pn_query = session.query(orm.Param.key).join(orm.ParamValue)
pn_query = pn_query.filter(orm.ParamValue.model_id.in_(model_ids))
pn_query = pn_query.distinct()
pn_query = pn_query.order_by(orm.Param.key)
p_names = [r.key for r in pn_query.all()]
dt_names.extend(p_names)
logger.debug(f"query_model_results_array: {pn_query.statement} ({len(p_names)} rows)")
dt = []
v0 = []
for n in dt_names:
ft = util.field_to_numpy_type(n)
dt.append((n, ft))
v0.append(np.nan if ft[0] == 'f' else 0)
dt = np.dtype(dt)
v0 = np.array([tuple(v0)], dtype=dt)
values_array = np.full((n_results,), v0, dtype=dt)
deltas_array = np.full((n_results,), v0, dtype=dt)
for i, r in enumerate(results):
d = {**r.as_dict(), **r.model.as_dict()}
for k, v in d.items():
try:
values_array[i][k] = v
except TypeError:
values_array[i][k] = 0
deltas_array[i] = values_array[i]
if include_params:
for k, v in r.model.values.items():
values_array[i][k] = v
for k, v in r.model.deltas.items():
deltas_array[i][k] = v
return values_array, deltas_array
def query_best_models_per_job(session, projects=None, jobs=None, task_level='model', order=None, limit=None):
"""
return the best model (by rfac) of each selected job
the query gathers the R-factors of the selected jobs at the selected task levels
and, for each job, returns the (database) model id where the lowest R-factor is reported
among the gathered results.
this can be useful if you want to compile a report of the best model per job.
@param session:
@param projects: filter by project.
the argument can be a singleton or sequence of orm.Project objects or numeric id.
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param task_level: element of or index into @ref pmsco.dispatch.CALC_LEVELS.
deepest task_level to include in the query.
results on deeper levels are not considered.
e.g. if you pass 'scan', R-factors of individual scans are included in the query.
note that including deeper levels will not increase the number of results returned.
the lowest level that can be specified is `emit`.
@param order: ordering of results. this can be a sequence of orm.Result attributes.
the default order is by `orm.Result.rfac`.
@param limit: maximum number of models to return
@return sequence of (orm.Model, orm.Result) tuples.
the number of results corresponds to the number of jobs in the filter scope.
to find out details of the models, execute another query that filters on these model ids.
the method produces an SQL query similar to:
@code{.sql}
select Models.id from Models
join Results on Models.id = Results.model_id
join Jobs on Models.job_id = Jobs.id
where scan=-1
and project_id=1
and job_id in (1,2,3)
group by Models.job_id
having min(rfac)
order by rfac
@endcode
"""
try:
level = dispatch.CALC_LEVELS.index(task_level) + 1
except ValueError:
level = task_level + 1
try:
level_name = dispatch.CALC_LEVELS[level]
except IndexError:
level_name = dispatch.CALC_LEVELS[4]
query = session.query(orm.Model, orm.Result).join(orm.Result)
if projects:
query = filter_objects(query, orm.Project, projects)
if jobs:
query = filter_objects(query, orm.Job, jobs)
query = query.filter(getattr(orm.Result, level_name) == -1)
query = query.group_by(orm.Model.job_id)
query = query.having(func.min(orm.Result.rfac))
if order is not None:
query = query.order_by(*order)
else:
query = query.order_by(orm.Result.rfac)
if limit:
query = query[0:limit]
else:
query = query.all()
return query
def filter_objects(query, entity, objects):
"""
filter a query for the given objects
apply a simple object filter to a database query.
the criteria can be a single object or a sequence of objects.
the objects can be specified either by their object representation or numeric id.
the query is filtered by id.
thus, in the first case, the objects must have a valid id.
@param query: sqlalchemy.orm.Query object that queries a table that is linked to the entity table.
the function joins the entity table.
a table with a direct foreign key relationship to the entity table must already be in the query.
@param entity: orm entity class, e.g. pmsco.database.orm.Project.
@param objects: singleton or sequence of orm objects or their numeric ids.
@return: modified query
"""
# avoid duplicate joins
if str(query.statement).find(entity.__tablename__) < 0:
query = query.join(entity)
try:
objects = [p if isinstance(p, int) else p.id for p in objects]
query = query.filter(entity.id.in_(objects))
except TypeError:
object = objects if isinstance(objects, int) else objects.id
query = query.filter(entity.id == object)
return query
def filter_task_levels(query, level='model', include_parents=False):
"""
refine a query by filtering by task level.
@param query: sqlalchemy.orm.Query object that queries the Result table
(possibly joined with others).
@param level: element of or index into @ref pmsco.dispatch.CALC_LEVELS.
deepest task_level to include in the query.
results on deeper levels are not considered.
e.g. if you pass 'scan', R-factors of individual scans are included in the query.
the lowest level that can be specified is `emit`.
@param include_parents: by default, the query will return only results from the given level.
if True, combined results (parents) will be returned as well.
"""
try:
level = dispatch.CALC_LEVELS.index(level)
except ValueError:
level = int(level)
child_level = level + 1
try:
child_level_name = dispatch.CALC_LEVELS[child_level]
level_name = dispatch.CALC_LEVELS[level]
except IndexError:
child_level_name = dispatch.CALC_LEVELS[4]
level_name = dispatch.CALC_LEVELS[3]
query = query.filter(getattr(orm.Result, child_level_name) == -1)
if not include_parents:
query = query.filter(getattr(orm.Result, level_name) >= 0)
return query

161
pmsco/database/util.py Normal file
View File

@@ -0,0 +1,161 @@
import logging
import numpy as np
from pathlib import Path
import pmsco.dispatch as dispatch
logger = logging.getLogger(__name__)
## mapping of database fields to special parameter names
#
# `_db` parameters are returned by some query methods to identify the database records.
#
DB_SPECIAL_PARAMS = {"project_id": "_db_project_id",
"job_id": "_db_job_id",
"model_id": "_db_model_id",
"result_id": "_db_result_id",
"model": "_model",
"scan": "_scan",
"domain": "_domain",
"emit": "_emit",
"region": "_region",
"gen": "_gen",
"particle": "_particle",
"rfac": "_rfac",
"secs": "_secs",
"timestamp": "_timestamp"}
## numpy data types of special parameters by database field
#
# this dictionary helps to create a numpy array from a database record.
#
DB_SPECIAL_NUMPY_TYPES = {"_db_project_id": "i8",
"_db_job_id": "i8",
"_db_model_id": "i8",
"_db_result_id": "i8",
"_model": "i8",
"_scan": "i8",
"_domain": "i8",
"_emit": "i8",
"_region": "i8",
"_gen": "i8",
"_particle": "i8",
"_rfac": "f8",
"_secs": "f8",
"_timestamp": "f8"}
def regular_params(d):
"""
filter regular parameters from dictionary
returns a dictionary containing only the regular parameters (those not prefixed with an underscore).
@param d: dict or numpy.void or pmsco.dispatch.CalcID.
the param names must have no leading underscore.
the numpy.void type occurs when an element of a structured array is extracted.
the CalcID does not contain a regular parameter and will return an empty dictionary.
it is supported only for compatibility with special_params function.
a tuple or list is interpreted as a sequence of parameter names.
in this case the names representing special parameters are returned with underscore removed.
@return: dict for mapping types (numpy and dict) containing the regular key: value pairs of the original object.
list (tuple) of parameter names for sequence (tuple) types.
leading underscores are removed from key names.
"""
if isinstance(d, np.void):
d = {k: d[k] for k in d.dtype.names if k[0] != "_"}
elif isinstance(d, dispatch.CalcID):
d = {}
elif isinstance(d, tuple):
d = [k for k in d if k[0] != "_"]
d = tuple(d)
elif isinstance(d, dict):
d = {k: v for k, v in d.items() if k[0] != "_"}
else:
d = [k for k in d if k[0] != "_"]
return d
def special_params(d):
"""
filter special parameters from model dictionary, numpy record or sequence.
special parameters are those prefixed with an underscore.
the underscore is removed from the keys.
fields starting with '_db_' are removed.
@param d: dict or numpy.void or pmsco.dispatch.CalcID or sequence.
in the case of a dict or numpy.void,
the key names of the special parameters must have a leading underscore.
the numpy.void type occurs when an element of a structured array is extracted.
in the case of a CalcID, the attribute names become the key names.
a tuple or list is interpreted as a sequence of parameter names.
in this case the names representing special parameters are returned with underscore removed.
@return
the return type depends on the type of input `d`:
@arg in the case of a dict, numpy.void or CalcID it is a dictionary.
@arg in the case of a tuple or list the return type is the same as the input.
"""
if isinstance(d, np.void):
d = {k[1:]: d[k] for k in d.dtype.names if k[0] == "_" and k[0:4] != "_db_"}
elif isinstance(d, dispatch.CalcID):
d = d._asdict()
elif isinstance(d, tuple):
d = [k[1:] for k in d if k[0] == "_" and k[0:4] != "_db_"]
d = tuple(d)
elif isinstance(d, dict):
d = {k[1:]: v for k, v in d.items() if k[0] == "_" and k[0:4] != "_db_"}
else:
d = [k[1:] for k in d if k[0] == "_" and k[0:4] != "_db_"]
return d
def field_to_param(f):
"""
translate database field name to parameter name.
field names of optimization parameters are unchanged.
special parameters are prefixed by '_' or '_db_'.
@param f: (str) database field name.
@return: (str) parameter name as used in model dictionaries.
"""
try:
p = DB_SPECIAL_PARAMS[f]
except KeyError:
p = f
return p
def field_to_numpy_type(f):
"""
determine the numpy data type string of a database field.
@param f: (str) database field name.
@return: (str) numpy type description, e.g. 'f8'.
"""
try:
t = DB_SPECIAL_NUMPY_TYPES[f]
except KeyError:
t = 'f8'
return t
def is_sqlite3_file(path_like):
"""
test whether a file is an sqlite3 database file.
@param path_like: file path (str or pathlib.Path).
@return: (bool)
"""
try:
with Path(path_like).open("rb") as f:
s = f.read(16)
return s == b"SQLite format 3\000"
except OSError:
return False

272
pmsco/dispatch.py
View File

@@ -4,16 +4,13 @@ calculation dispatcher.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import os.path
import datetime
@@ -21,8 +18,18 @@ import signal
import collections
import copy
import logging
from attrdict import AttrDict
from mpi4py import MPI
try:
from mpi4py import MPI
mpi_comm = MPI.COMM_WORLD
mpi_size = mpi_comm.Get_size()
mpi_rank = mpi_comm.Get_rank()
except ImportError:
MPI = None
mpi_comm = None
mpi_size = 1
mpi_rank = 0
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@@ -53,7 +60,7 @@ TAG_ERROR_ABORTING = 4
## levels of calculation tasks
#
CALC_LEVELS = ('model', 'scan', 'sym', 'emit', 'region')
CALC_LEVELS = ('model', 'scan', 'domain', 'emit', 'region')
## intermediate sub-class of CalcID
#
@@ -159,13 +166,13 @@ class CalculationTask(object):
@arg @c id.model structure number or iteration (handled by the mode module)
@arg @c id.scan scan number (handled by the project)
@arg @c id.sym symmetry number (handled by the project)
@arg @c id.domain domain number (handled by the project)
@arg @c id.emit emitter number (handled by the project)
@arg @c id.region region number (handled by the region handler)
specified members must be greater or equal to zero.
-1 is the wildcard which is used in parent tasks,
where, e.g., no specific symmetry is chosen.
where, e.g., no specific domain is chosen.
the root task has the ID (-1, -1, -1, -1, -1).
"""
@@ -181,6 +188,15 @@ class CalculationTask(object):
# this is typically initialized to the parameters of the parent task,
# and varied at the level where the task ID was produced.
## @var delta (dict)
# dictionary containing a delta vector of the model parameters.
#
# this is a diagnostic value of the optimizer, it is not used by calculators.
# if defined, it is entered into the results database (ParamValue.delta field).
#
# the exact meaning depends on the optimizer.
# in particle swarm, e.g., it is the current velocity of the particle.
## @var file_root (string)
# file name without extension and index.
@@ -227,7 +243,7 @@ class CalculationTask(object):
# files generated by the task and their category
#
# dictionary key is the file name,
# value is the file category, e.g. 'cluster', 'phase', etc.
# value is the file category, e.g. 'cluster', 'atomic', etc.
#
# this information is used to automatically clean up unnecessary data files.
@@ -249,6 +265,7 @@ class CalculationTask(object):
self.id = CalcID(-1, -1, -1, -1, -1)
self.parent_id = self.id
self.model = {}
self.delta = {}
self.file_root = ""
self.file_ext = ""
self.result_filename = ""
@@ -311,7 +328,8 @@ class CalculationTask(object):
format input or output file name including calculation index.
@param overrides optional keyword arguments override object fields.
the following keywords are handled: @c root, @c model, @c scan, @c sym, @c emit, @c region, @c ext.
the following keywords are handled:
`root`, `model`, `scan`, `domain`, `emit`, `region`, `ext`.
@return a string consisting of the concatenation of the base name, the ID, and the extension.
"""
@@ -322,7 +340,7 @@ class CalculationTask(object):
for key in overrides.keys():
parts[key] = overrides[key]
filename = "{root}_{model}_{scan}_{sym}_{emit}_{region}{ext}".format(**parts)
filename = "{root}_{model}_{scan}_{domain}_{emit}_{region}{ext}".format(**parts)
return filename
def copy(self):
@@ -374,7 +392,7 @@ class CalculationTask(object):
this information is used to automatically clean up unnecessary data files.
@param name: file name (optionally including a path).
@param category: file category, e.g. 'cluster', 'phase', etc.
@param category: file category, e.g. 'cluster', 'atomic', etc.
@return: None
"""
self.files[name] = category
@@ -462,7 +480,7 @@ class CachedCalculationMethod(object):
def wrapped_func(inst, model, index):
# note: _replace returns a new instance of the namedtuple
index = index._replace(emit=-1, region=-1)
cache_index = (id(inst), index.model, index.scan, index.sym)
cache_index = (id(inst), index.model, index.scan, index.domain)
try:
result = self._cache[cache_index]
except KeyError:
@@ -490,6 +508,16 @@ class CachedCalculationMethod(object):
del self._cache[index]
class AttrDict(collections.UserDict):
def __getattr__(self, key):
return self.__getitem__(key)
def __setattr__(self, key, value):
if key == "data":
return super().__setattr__(key, value)
return self.__setitem__(key, value)
class MscoProcess(object):
"""
code shared by MscoMaster and MscoSlave.
@@ -518,10 +546,10 @@ class MscoProcess(object):
#
# the default is 2 days after start.
def __init__(self, comm):
self._comm = comm
def __init__(self):
self._project = None
self._calculator = None
self._atomic_scattering = None
self._multiple_scattering = None
self._running = False
self._finishing = False
self.stop_signal = False
@@ -529,7 +557,8 @@ class MscoProcess(object):
def setup(self, project):
self._project = project
self._calculator = project.calculator_class()
self._atomic_scattering = project.atomic_scattering_factory()
self._multiple_scattering = project.multiple_scattering_factory()
self._running = False
self._finishing = False
self.stop_signal = False
@@ -563,6 +592,8 @@ class MscoProcess(object):
"""
clean up after all calculations.
this method must be called after run() has finished.
@return: None
"""
pass
@@ -596,19 +627,18 @@ class MscoProcess(object):
scan = self._define_scan(task)
output_file = task.format_filename(ext="")
# check parameters and call the msc program
if clu.get_atom_count() < 2:
# check parameters and call the calculators
if clu.get_atom_count() >= 1:
self._calc_atomic(task, par, clu, scan, output_file)
else:
logger.error("empty cluster in calculation %s", s_id)
task.result_valid = False
elif clu.get_emitter_count() < 1:
if clu.get_emitter_count() >= 1:
self._calc_multiple(task, par, clu, scan, output_file)
else:
logger.error("no emitters in cluster of calculation %s.", s_id)
task.result_valid = False
else:
task.result_filename, files = self._calculator.run(par, clu, scan, output_file)
(root, ext) = os.path.splitext(task.result_filename)
task.file_ext = ext
task.result_valid = True
task.files.update(files)
task.time = datetime.datetime.now() - start_time
@@ -666,6 +696,8 @@ class MscoProcess(object):
"""
nem = self._project.cluster_generator.count_emitters(task.model, task.id)
clu = self._project.cluster_generator.create_cluster(task.model, task.id)
# overwrite atom classes only if they are at their default value
clu.init_atomclasses(field_or_value='t', default_only=True)
if task.id.region == 0:
file_index = task.id._replace(region=-1)
@@ -690,12 +722,65 @@ class MscoProcess(object):
parameters generation is delegated to the project's create_params method.
@param task: CalculationTask with all attributes set for the calculation.
@return: pmsco.project.Params object for the calculator.
@return: pmsco.project.CalculatorParams object for the calculator.
"""
par = self._project.create_params(task.model, task.id)
return par
def _calc_atomic(self, task, par, clu, scan, output_file):
"""
calculate the atomic scattering factors if necessary and link them to the cluster.
the method first calls the `before_atomic_scattering` project hook,
the atomic scattering calculator,
and finally the `after_atomic_scattering` hook.
this process updates the par and clu objects to link to the created files.
if any of the functions returns None, the par and clu objects are left unchanged.
@param task: CalculationTask with all attributes set for the calculation.
@param par: pmsco.project.CalculatorParams object for the calculator.
its phase_files attribute is updated with the created scattering files.
the radial matrix elements are not changed (but may be in a future version).
@param clu: pmsco.cluster.Cluster object for the calculator.
the cluster is overwritten with the one returned by the calculator,
so that atom classes match the phase_files.
@return: None
"""
_par = copy.deepcopy(par)
_clu = copy.deepcopy(clu)
_par, _clu = self._project.before_atomic_scattering(task, _par, _clu)
if _clu is not None:
filename, files = self._atomic_scattering.run(_par, _clu, scan, output_file)
if files:
task.files.update(files)
_par, _clu = self._project.after_atomic_scattering(task, _par, _clu)
if _clu is not None:
par.phase_files = _par.phase_files
clu.copy_from(_clu)
def _calc_multiple(self, task, par, clu, scan, output_file):
"""
calculate the multiple scattering intensity.
@param task: CalculationTask with all attributes set for the calculation.
@param par: pmsco.project.CalculatorParams object for the calculator.
@param clu: pmsco.cluster.Cluster object for the calculator.
@return: None
"""
task.result_filename, files = self._multiple_scattering.run(par, clu, scan, output_file)
if task.result_filename:
(root, ext) = os.path.splitext(task.result_filename)
task.file_ext = ext
task.result_valid = True
if files:
task.files.update(files)
class MscoMaster(MscoProcess):
"""
@@ -764,16 +849,16 @@ class MscoMaster(MscoProcess):
## @var task_handlers
# (AttrDict) dictionary of task handler objects
#
# the keys are the task levels 'model', 'scan', 'sym', 'emit' and 'region'.
# the keys are the task levels 'model', 'scan', 'domain', 'emit' and 'region'.
# the values are handlers.TaskHandler objects.
# the objects can be accessed in attribute or dictionary notation.
def __init__(self, comm):
super(MscoMaster, self).__init__(comm)
def __init__(self):
super().__init__()
self._pending_tasks = collections.OrderedDict()
self._running_tasks = collections.OrderedDict()
self._complete_tasks = collections.OrderedDict()
self._slaves = self._comm.Get_size() - 1
self._slaves = mpi_size - 1
self._idle_ranks = []
self.max_calculations = 1000000
self._calculations = 0
@@ -798,12 +883,18 @@ class MscoMaster(MscoProcess):
the method notifies the handlers of the number of available slave processes (slots).
some of the tasks handlers adjust their branching according to the number of slots.
this mechanism may be used to balance the load between the task levels.
however, the current implementation is very coarse in this respect.
it advertises all slots to the model handler but a reduced number to the remaining handlers
depending on the operation mode.
the region handler receives a maximum of 4 slots except in single calculation mode.
in single calculation mode, all slots can be used by all handlers.
this mechanism may be used to adjust the priorities of the task levels,
i.e., whether one slot handles all calculations of one model
so that all models of a generation finish around the same time,
or whether a model is finished completely before the next one is calculated
so that a result is returned as soon as possible.
the current algorithm tries to pass as many slots as available
down to the lowest level (region) in order to minimize wall time.
the lowest level is restricted to the minimum number of splits
only if the intermediate levels create a lot of branches,
in which case splitting scans would not offer a performance benefit.
"""
super(MscoMaster, self).setup(project)
@@ -812,8 +903,8 @@ class MscoMaster(MscoProcess):
self._idle_ranks = list(range(1, self._running_slaves + 1))
self._root_task = CalculationTask()
self._root_task.file_root = project.output_file
self._root_task.model = project.create_domain().start
self._root_task.file_root = str(project.output_file)
self._root_task.model = project.model_space.start
for level in self.task_levels:
self.task_handlers[level] = project.handler_classes[level]()
@@ -821,14 +912,22 @@ class MscoMaster(MscoProcess):
self.task_handlers.model.datetime_limit = self.datetime_limit
slaves_adj = max(self._slaves, 1)
self.task_handlers.model.setup(project, slaves_adj)
if project.mode != "single":
slaves_adj = max(slaves_adj / 2, 1)
self.task_handlers.scan.setup(project, slaves_adj)
self.task_handlers.sym.setup(project, slaves_adj)
self.task_handlers.emit.setup(project, slaves_adj)
if project.mode != "single":
n_models = self.task_handlers.model.setup(project, slaves_adj)
if n_models > 1:
slaves_adj = max(int(slaves_adj / 2), 1)
n_scans = self.task_handlers.scan.setup(project, slaves_adj)
if n_scans > 1:
slaves_adj = max(int(slaves_adj / 2), 1)
n_doms = self.task_handlers.domain.setup(project, slaves_adj)
if n_doms > 1:
slaves_adj = max(int(slaves_adj / 2), 1)
n_emits = self.task_handlers.emit.setup(project, slaves_adj)
if n_emits > 1:
slaves_adj = max(int(slaves_adj / 2), 1)
n_extra = max(n_scans, n_doms, n_emits)
if n_extra > slaves_adj * 2:
slaves_adj = min(slaves_adj, 4)
logger.debug(BMsg("{regions} slots available for region handler", regions=slaves_adj))
self.task_handlers.region.setup(project, slaves_adj)
project.setup(self.task_handlers)
@@ -855,6 +954,7 @@ class MscoMaster(MscoProcess):
else:
self._dispatch_tasks()
self._receive_result()
self._cleanup_tasks()
self._check_finish()
logger.debug("master exiting main loop")
@@ -862,12 +962,32 @@ class MscoMaster(MscoProcess):
self._save_report()
def cleanup(self):
"""
clean up after all calculations.
this method must be called after run() has finished.
in the master process, this calls cleanup() of each task handler and of the project.
@return: None
"""
logger.debug("master entering cleanup")
for level in reversed(self.task_levels):
self.task_handlers[level].cleanup()
self._project.cleanup()
super(MscoMaster, self).cleanup()
def _cleanup_tasks(self):
"""
periodic clean-up in the main loop.
once per iteration of the main loop, this method cleans up unnecessary files.
this is done by the project's cleanup_files() method.
@return: None
"""
self._project.cleanup_files()
def _dispatch_results(self):
"""
pass results through the post-processing modules.
@@ -937,7 +1057,7 @@ class MscoMaster(MscoProcess):
else:
logger.debug("assigning task %s to rank %u", str(task.id), rank)
self._running_tasks[task.id] = task
self._comm.send(task.get_mpi_message(), dest=rank, tag=TAG_NEW_TASK)
mpi_comm.send(task.get_mpi_message(), dest=rank, tag=TAG_NEW_TASK)
self._calculations += 1
else:
if not self._finishing:
@@ -959,7 +1079,7 @@ class MscoMaster(MscoProcess):
while self._idle_ranks:
rank = self._idle_ranks.pop()
logger.debug("send finish tag to rank %u", rank)
self._comm.send(None, dest=rank, tag=TAG_FINISH)
mpi_comm.send(None, dest=rank, tag=TAG_FINISH)
self._running_slaves -= 1
def _receive_result(self):
@@ -969,7 +1089,7 @@ class MscoMaster(MscoProcess):
if self._running_slaves > 0:
logger.debug("waiting for calculation result")
s = MPI.Status()
data = self._comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=s)
data = mpi_comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=s)
if s.tag == TAG_NEW_RESULT:
task_id = self._accept_task_done(data)
@@ -1025,19 +1145,19 @@ class MscoMaster(MscoProcess):
@return: self._finishing
"""
if not self._finishing and (self._model_done and not self._pending_tasks and not self._running_tasks):
logger.info("finish: model handler is done")
logger.warning("finish: model handler is done")
self._finishing = True
if not self._finishing and (self._calculations >= self.max_calculations):
logger.warning("finish: max. calculations (%u) exeeded", self.max_calculations)
self._finishing = True
if not self._finishing and self.stop_signal:
logger.info("finish: stop signal received")
logger.warning("finish: stop signal received")
self._finishing = True
if not self._finishing and (datetime.datetime.now() > self.datetime_limit):
logger.warning("finish: time limit exceeded")
self._finishing = True
if not self._finishing and os.path.isfile("finish_pmsco"):
logger.info("finish: finish_pmsco file detected")
logger.warning("finish: finish_pmsco file detected")
self._finishing = True
if self._finishing and not self._running_slaves and not self._running_tasks:
@@ -1066,9 +1186,9 @@ class MscoMaster(MscoProcess):
scan_tasks = self.task_handlers.scan.create_tasks(task)
for scan_task in scan_tasks:
sym_tasks = self.task_handlers.sym.create_tasks(scan_task)
for sym_task in sym_tasks:
emitter_tasks = self.task_handlers.emit.create_tasks(sym_task)
dom_tasks = self.task_handlers.domain.create_tasks(scan_task)
for dom_task in dom_tasks:
emitter_tasks = self.task_handlers.emit.create_tasks(dom_task)
for emitter_task in emitter_tasks:
region_tasks = self.task_handlers.region.create_tasks(emitter_task)
for region_task in region_tasks:
@@ -1089,8 +1209,8 @@ class MscoSlave(MscoProcess):
#
# typically, a task is aborted when an exception is encountered.
def __init__(self, comm):
super(MscoSlave, self).__init__(comm)
def __init__(self):
super().__init__()
self._errors = 0
self._max_errors = 5
@@ -1103,7 +1223,7 @@ class MscoSlave(MscoProcess):
self._running = True
while self._running:
logger.debug("waiting for message")
data = self._comm.recv(source=0, tag=MPI.ANY_TAG, status=s)
data = mpi_comm.recv(source=0, tag=MPI.ANY_TAG, status=s)
if s.tag == TAG_NEW_TASK:
logger.debug("received new task")
self.accept_task(data)
@@ -1133,17 +1253,17 @@ class MscoSlave(MscoProcess):
logger.exception(BMsg("unhandled exception in calculation task {0}", task.id))
self._errors += 1
if self._errors <= self._max_errors:
self._comm.send(data, dest=0, tag=TAG_INVALID_RESULT)
mpi_comm.send(data, dest=0, tag=TAG_INVALID_RESULT)
else:
logger.error("too many exceptions, aborting")
self._running = False
self._comm.send(data, dest=0, tag=TAG_ERROR_ABORTING)
mpi_comm.send(data, dest=0, tag=TAG_ERROR_ABORTING)
else:
logger.debug(BMsg("sending result of task {0} to master", result.id))
self._comm.send(result.get_mpi_message(), dest=0, tag=TAG_NEW_RESULT)
mpi_comm.send(result.get_mpi_message(), dest=0, tag=TAG_NEW_RESULT)
def run_master(mpi_comm, project):
def run_master(project):
"""
initialize and run the master calculation loop.
@@ -1155,25 +1275,25 @@ def run_master(mpi_comm, project):
if an unhandled exception occurs, this function aborts the MPI communicator, killing all MPI processes.
the caller will not have a chance to handle the exception.
@param mpi_comm: MPI communicator (mpi4py.MPI.COMM_WORLD).
@param project: project instance (sub-class of project.Project).
"""
try:
master = MscoMaster(mpi_comm)
master = MscoMaster()
master.setup(project)
master.run()
master.cleanup()
except (SystemExit, KeyboardInterrupt):
mpi_comm.Abort()
if mpi_comm:
mpi_comm.Abort()
raise
except Exception:
logger.exception("unhandled exception in master calculation loop.")
mpi_comm.Abort()
if mpi_comm:
mpi_comm.Abort()
raise
def run_slave(mpi_comm, project):
def run_slave(project):
"""
initialize and run the slave calculation loop.
@@ -1186,12 +1306,10 @@ def run_slave(mpi_comm, project):
unless it is a SystemExit or KeyboardInterrupt (where we expect that the master also receives the signal),
the MPI communicator is aborted, killing all MPI processes.
@param mpi_comm: MPI communicator (mpi4py.MPI.COMM_WORLD).
@param project: project instance (sub-class of project.Project).
"""
try:
slave = MscoSlave(mpi_comm)
slave = MscoSlave()
slave.setup(project)
slave.run()
slave.cleanup()
@@ -1199,7 +1317,8 @@ def run_slave(mpi_comm, project):
raise
except Exception:
logger.exception("unhandled exception in slave calculation loop.")
mpi_comm.Abort()
if mpi_comm:
mpi_comm.Abort()
raise
@@ -1211,12 +1330,9 @@ def run_calculations(project):
@param project: project instance (sub-class of project.Project).
"""
mpi_comm = MPI.COMM_WORLD
mpi_rank = mpi_comm.Get_rank()
if mpi_rank == 0:
logger.debug("MPI rank %u setting up master loop", mpi_rank)
run_master(mpi_comm, project)
run_master(project)
else:
logger.debug("MPI rank %u setting up slave loop", mpi_rank)
run_slave(mpi_comm, project)
run_slave(project)

2
pmsco/edac/.gitignore vendored
View File

@@ -1,2 +0,0 @@
edac_all_wrap.*
edac.py

1
pmsco/edac/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'muntwiler_m'

7
pmsco/edac/edac_all.i
View File

@@ -1,7 +0,0 @@
/* EDAC interface for other programs */
%module edac
%{
extern int run_script(char *scriptfile);
%}
extern int run_script(char *scriptfile);

47
pmsco/edac/makefile
View File

@@ -1,47 +0,0 @@
SHELL=/bin/sh
# makefile for EDAC program and module
#
# the EDAC source code is not included in the public distribution.
# please obtain it from the original author,
# copy it to this directory,
# and apply the edac_all.patch patch before compilation.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac
FC?=gfortran
FCCOPTS?=
F2PY?=f2py
F2PYOPTS?=
CXX?=g++
CXXOPTS?=-Wno-write-strings
PYTHON?=python
PYTHONOPTS?=
all: edac
edac: edac.exe _edac.so edac.py
edac.exe: edac_all.cpp
$(CXX) $(CXXOPTS) -o edac.exe edac_all.cpp
edac.py _edac.so: edac_all.cpp edac_all.i setup.py
$(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
revision.py: _edac.so
git log --pretty=format:"code_rev = 'Code revision %h, %ad'" --date=iso -1 > $@ || echo "code_rev = 'Code revision unknown, "`date +"%F %T %z"`"'" > $@
echo "" >> revision.py
revision.txt: _edac.so edac.exe
git log --pretty=format:"Code revision %h, %ad" --date=iso -1 > $@ || echo "Code revision unknown, "`date +"%F %T %z"` > $@
echo "" >> revision.txt
clean:
rm -f *.so *.o *.exe *.pyc
rm -f edac.py edac_all_wrap.*
rm -f revision.*

23
pmsco/edac/setup.py
View File

@@ -1,23 +0,0 @@
#!/usr/bin/env python
"""
setup.py file for EDAC
"""
from distutils.core import setup, Extension
edac_module = Extension('_edac',
sources=['edac_all.cpp', 'edac_all.i'],
swig_opts=['-c++']
)
setup (name = 'edac',
version = '0.1',
author = "Matthias Muntwiler",
description = """EDAC module in Python""",
ext_modules = [edac_module],
py_modules = ["edac"],
requires=['numpy']
)

1361
pmsco/elements/bindingenergy.json Normal file
View File

File diff suppressed because it is too large Load Diff

225
pmsco/elements/bindingenergy.py Normal file
View File

@@ -0,0 +1,225 @@
"""
@package pmsco.elements.bindingenergy
Electron binding energies of the elements
Extends the element table of the `periodictable` package
(https://periodictable.readthedocs.io/en/latest/index.html)
by the electron binding energies.
The binding energies are compiled from Gwyn Williams' web page
(https://userweb.jlab.org/~gwyn/ebindene.html).
Please refer to the original web page or the x-ray data booklet
for original sources, definitions and remarks.
Binding energies of gases are replaced by respective values of a common compound
from the 'handbook of x-ray photoelectron spectroscopy' (Physical Electronics, Inc., 1995).
Usage
-----
This module requires the periodictable package (https://pypi.python.org/pypi/periodictable).
~~~~~~{.py}
import periodictable as pt
import pmsco.elements.bindingenergy
# read any periodictable's element interfaces, e.g.
print(pt.gold.binding_energy['4f7/2'])
print(pt.elements.symbol('Au').binding_energy['4f7/2'])
print(pt.elements.name('gold').binding_energy['4f7/2'])
print(pt.elements[79].binding_energy['4f7/2'])
~~~~~~
The database is loaded from the accompanying bindingenergy.json file on first demand.
Attributes are writable, you may update the values in your run-time instance of the database.
Normally, the user will not need to call any functions in this module directly.
The query_binding_energy() function queries all terms with a particular binding energy.
@author Matthias Muntwiler
@copyright (c) 2020-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import json
import numpy as np
from pathlib import Path
import periodictable as pt
import periodictable.core
index_energy = np.zeros(0)
index_number = np.zeros(0)
index_term = []
default_data_path = Path(Path(__file__).parent, "bindingenergy.json")
def load_data(data_path=None):
"""
load binding energy data from json file
the data file must be in the same format as generated by save_data.
@param data_path file path of the data file. default: "bindingenergy.json" next to this module file
@return dictionary
"""
if data_path is None:
data_path = default_data_path
with open(data_path, "rt", encoding="utf8") as fp:
data = json.load(fp)
return data
def save_data(data_path=None):
"""
save binding energy data to json file
@param data_path file path of the data file. default: "bindingenergy.json" next to this module file
@return None
"""
if data_path is None:
data_path = default_data_path
data = {}
for element in pt.elements:
element_data = {}
for term, energy in element.binding_energy.items():
element_data[term] = energy
if element_data:
data[element.number] = element_data
with open(data_path, "w", encoding="utf8") as fp:
json.dump(data, fp, sort_keys=True, indent='\t')
def init(table, reload=False):
if 'binding_energy' in table.properties and not reload:
return
table.properties.append('binding_energy')
pt.core.Element.binding_energy = {}
pt.core.Element.binding_energy_units = "eV"
data = load_data()
for el_key, el_data in data.items():
try:
el = table[int(el_key)]
except ValueError:
el = table.symbol(el_key)
el.binding_energy = el_data
def build_index():
"""
build an index for query_binding_energy().
the index is kept in global variables of the module.
@return None
"""
global index_energy
global index_number
global index_term
n = 0
for element in pt.elements:
n += len(element.binding_energy)
index_energy = np.zeros(n)
index_number = np.zeros(n)
index_term = []
for element in pt.elements:
for term, energy in element.binding_energy.items():
index_term.append(term)
i = len(index_term) - 1
index_energy[i] = energy
index_number[i] = element.number
def query_binding_energy(energy, tol=1.0):
"""
search the periodic table for a specific binding energy and return all matching terms.
@param energy: binding energy in eV.
@param tol: tolerance in eV.
@return: list of dictionaries containing element and term specification.
the list is ordered arbitrarily.
each dictionary contains the following keys:
@arg 'number': element number
@arg 'symbol': element symbol
@arg 'term': spectroscopic term
@arg 'energy': actual binding energy
"""
if len(index_energy) == 0:
build_index()
sel = np.abs(index_energy - energy) < tol
idx = np.where(sel)
result = []
for i in idx[0]:
el_num = int(index_number[i])
d = {'number': el_num,
'symbol': pt.elements[el_num].symbol,
'term': index_term[i],
'energy': index_energy[i]}
result.append(d)
return result
def export_flat_text(f):
"""
export the binding energies to a flat general text file.
the file has four space-separated columns `number`, `symbol`, `term`, `energy`.
column names are included in the first row.
@param f: file path or open file object
@return: None
"""
if hasattr(f, "write") and callable(f.write):
f.write("number symbol term energy\n")
for element in pt.elements:
for term, energy in element.binding_energy.items():
f.write(f"{element.number} {element.symbol} {term} {energy}\n")
else:
with open(f, "w") as fi:
export_flat_text(fi)
def import_flat_text(f):
"""
import binding energies from a flat general text file.
data is in space-separated columns.
the first row contains column names.
at least the columns `number`, `term`, `energy` must be present.
the function updates existing entries and appends entries of non-existing terms.
existing terms that are not listed in the file remain unchanged.
@param f: file path or open file object
@return: None
"""
data = np.atleast_1d(np.genfromtxt(f, names=True, dtype=None, encoding="utf8"))
for d in data:
pt.elements[d['number']].binding_energy[d['term']] = d['energy']
def _load_binding_energy():
"""
delayed loading of the binding energy table.
"""
init(periodictable.core.default_table())
periodictable.core.delayed_load(['binding_energy'], _load_binding_energy)

BIN
pmsco/elements/cross-sections.dat Normal file
View File

Binary file not shown.

446
pmsco/elements/photoionization.py Normal file
View File

@@ -0,0 +1,446 @@
"""
@package pmsco.elements.photoionization
Photoionization cross-sections of the elements
Extends the element table of the `periodictable` package
(https://periodictable.readthedocs.io/en/latest/index.html)
by a table of photoionization cross-sections and asymmetry parameters.
The data is available from (https://vuo.elettra.eu/services/elements/)
or (https://figshare.com/articles/dataset/Digitisation_of_Yeh_and_Lindau_Photoionisation_Cross_Section_Tabulated_Data/12389750).
Both sources are based on the original atomic data tables by Yeh and Lindau (1985).
The Elettra data includes the cross section and asymmetry parameter and is interpolated at finer steps,
whereas the Kalha data contains only the cross sections at the photon energies calculated by Yeh and Lindau
plus an additional point at 8 keV.
The tables go up to 1500 eV photon energy and do not resolve spin-orbit splitting.
Usage
-----
This module adds the photoionization attribute to the elements database of the periodictable package (https://pypi.python.org/pypi/periodictable).
Python >= 3.6, numpy >= 1.15 and the periodictable package are required.
~~~~~~{.py}
import numpy as np
import periodictable as pt
import pmsco.elements.photoionization
# get a SubShellPhotoIonization object from any of periodictable's element interface:
sspi = pt.gold.photoionization['4f']
sspi = pt.elements.symbol('Au').photoionization['4f']
sspi = pt.elements.name('gold').photoionization['4f']
sspi = pt.elements[79].photoionization['4f']
# get the cross section, asymmetry parameter or differential cross section at 800 eV photon energy:
sspi.cross_section(800)
sspi.asymmetry_parameter(800)
sspi.diff_cross_section(800, gamma=30)
# with the j quantum number, the cross-section is weighted based on a full sub-shell:
sspi = pt.gold.photoionization['4f7/2']
print(sspi.weight)
print(pt.gold.photoionization['4f7/2'].cross_section(800) / pt.gold.photoionization['4f'].cross_section(800))
# the original data is contained in the data array (which is a numpy.recarray):
sspi.data.eph, sspi.data.cs, sspi.data.ap
~~~~~~
The data is loaded on demand from the cross-sections.dat file when the photoionization record is first accessed.
Normally, the user will not need to call any functions in this module directly.
The load_elettra_data()/load_kalha_data() and save_pickled_data() functions are provided
to import data from one of the sources referenced above and
to create the cross-sections.dat file.
@author Matthias Muntwiler
@copyright (c) 2020-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import copy
import numpy as np
from pathlib import Path
import periodictable as pt
import pickle
import urllib.request
import urllib.error
import periodictable.core
class PhotoIonization(dict):
"""
photo-ionization parameters of an element
this class provides the photo-ionization cross-section and asymmetry parameter of the sub-shells of an element.
it is, essentially, a dictionary, mapping 'nl' and 'nlj' terms to the corresponding SubShellPhotoIonization object.
examples of 'nl' and 'nlj' terms: '4f' and '4f7/2'
@note the dictionary actually contains raw data for 'nl' terms only.
for 'nlj' terms, the corresponding 'nl' object is copied,
and a weight according to the spin-orbit multiplicity is set.
@note 'nlj' terms are not considered by any methods or properties
except the bracket notation or __getitem__ method!
in particular, iteration or the keys() method will yield 'nl' terms only.
"""
def __init__(self, *args, **kwargs):
"""
dictionary constructor
the class accepts the same arguments as the Python built-in dict constructor.
keys are 'nl' terms, e.g. '4f', and values must be SubShellPhotoIonization() objects.
@param args:
@param kwargs:
"""
super().__init__(*args, **kwargs)
self.cross_section_units = "Mb"
def __getitem__(self, k):
"""
get sub-shell photo-ionization data by 'nl' or 'nlj' term.
@param k: dictionary key.
if this is an 'nl' term, the original object is returned.
if this is an 'nlj' term, a proxy of the corresponding 'nl' object
with shared data but weight based on j-branching is returned.
@return: SubShellPhotoIonization() object
@note whether the original or a proxy object is returned,
its data attribute always refers to the original data.
any modification will affect the original data (process memory).
"""
spi = super().__getitem__(k[0:2])
if len(k) > 2:
spi = copy.copy(spi)
spi.set_spin_orbit(k[1:5])
return spi
class SubShellPhotoIonization(object):
"""
Sub-shell photo-ionization parameters versus photon energy.
this class provides the photo-ionization cross-section and asymmetry parameter of one sub-shell.
it contains a three-column record array of photon energy, cross section and asymmetry parameter in self.data.
accessory functions provide high-level access to specific views and interpolated data.
a weighting factor self.weight is multiplied to the method results.
it is normally used to weight the spin-orbit peaks by calling set_spin_orbit().
"""
SPIN_ORBIT_WEIGHTS = {"p1/2": 1. / 3.,
"p3/2": 2. / 3.,
"d3/2": 2. / 5.,
"d5/2": 3. / 5.,
"f5/2": 3. / 7.,
"f7/2": 4. / 7.}
def __init__(self, photon_energy, cross_section, asymmetry_parameter):
"""
initialize a new object instance.
all arrays must have the same length.
@param photon_energy: (array-like) photon energies
@param cross_section: (array-like) cross-section values
@param asymmetry_parameter: (array-like) asymmetry parameter values
"""
super().__init__()
self.data = np.rec.fromarrays([photon_energy, cross_section, asymmetry_parameter], names='eph, cs, ap')
self.weight = 1.
def cross_section(self, photon_energy):
"""
interpolated sub-shell cross-section at a specific energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@param photon_energy: photon energy in eV.
can be scalar or numpy array.
@return: cross-section in Mb.
numpy.nan where photon_energy is off range.
"""
cs = np.interp(photon_energy, self.data.eph, self.data.cs, left=np.nan, right=np.nan) * self.weight
return cs
def asymmetry_parameter(self, photon_energy):
"""
interpolated asymmetry parameter at a specific energy.
@param photon_energy: photon energy in eV.
can be scalar or numpy array.
@return: asymmetry parameter (0..2).
numpy.nan where photon_energy is off range.
"""
ap = np.interp(photon_energy, self.data.eph, self.data.ap, left=np.nan, right=np.nan)
return ap
def diff_cross_section(self, photon_energy, gamma):
"""
differential cross-section for linear polarization.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@param photon_energy: photon energy in eV.
@param gamma: angle between polarization vector and electron propagation direction in degrees.
@return: differential cross-section in Mb.
"""
p2 = (3 * np.cos(gamma) ** 2 - 1) / 2
cs = self.cross_section(photon_energy)
ap = self.asymmetry_parameter(photon_energy)
dcs = cs / 4 / np.pi * (1 + ap * p2)
return dcs
def photon_energy_array(self):
"""
photon energy array.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return:
"""
return self.data.eph
def cross_section_array(self):
"""
sub-shell cross-section versus photon energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return: numpy.ndarray
"""
return self.data.cs * self.weight
def asymmetry_parameter_array(self):
"""
sub-shell asymmetry parameter versus photon energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return: numpy.ndarray
"""
return self.data.ap
def diff_cross_section_array(self, gamma):
"""
differential cross-section for linear polarization (full array).
@param gamma: angle between polarization vector and electron propagation direction in degrees.
@return: (np.ndarray) differential cross-section in Mb.
"""
p2 = (3 * np.cos(gamma) ** 2 - 1) / 2
dcs = self.data.cs / 4 / np.pi * (1 + self.data.ap * p2) * self.weight
return dcs
def set_spin_orbit(self, lj):
"""
set the weight according to the spin-orbit quantum number (based on full sub-shell).
the weight is stored in the self.weight attribute.
it is applied to the results of the cross-section methods, but not to the raw data in self.data!
@param lj: (str) 4-character lj term notation, e.g. 'f7/2'
@return: None
"""
self.weight = self.SPIN_ORBIT_WEIGHTS.get(lj, 1.)
def load_kalha_data():
"""
load all cross-sections from csv-files by Kalha et al.
the files must be placed in the 'kalha' directory next to this file.
@return: cross-section data in a nested dictionary, cf. load_pickled_data().
"""
data = {}
p = Path(Path(__file__).parent, "kalha")
for entry in p.glob('*_*.csv'):
if entry.is_file():
try:
element = int(entry.stem.split('_')[0])
except ValueError:
pass
else:
data[element] = load_kalha_file(entry)
return data
def load_kalha_file(path):
"""
load the cross-sections of an element from a csv-file by Kalha et al.
@param path: file path
@return: (dict) dictionary of 'nl' terms.
the data items are tuples (photon_energy, cross_sections) of 1-dimensional numpy arrays.
"""
a = np.genfromtxt(path, delimiter=',', names=True)
b = ~np.isnan(a['Photon_Energy__eV'])
a = a[b]
eph = a['Photon_Energy__eV'].copy()
data = {}
for n in range(1, 8):
for l in 'spdf':
col = f"{n}{l}"
try:
data[col] = SubShellPhotoIonization(eph, a[col].copy(), np.zeros_like(eph))
except ValueError:
pass
return data
def load_kalha_configuration(path):
"""
load the electron configuration from a csv-file by Kalha et al.
@param path: file path
@return: (dict) dictionary of 'nl' terms mapping to number of electrons in the sub-shell.
"""
p = Path(path)
subshells = []
electrons = []
config = {}
with p.open() as f:
for l in f.readlines():
s = l.split(',')
k_eph = "Photon Energy"
k_el = "#electrons"
if s[0][0:len(k_eph)] == k_eph:
subshells = s[1:]
elif s[0][0:len(k_el)] == k_el:
electrons = s[1:]
for i, sh in enumerate(subshells):
if sh:
config[sh] = electrons[i]
return config
def load_elettra_file(symbol, nl):
"""
download the cross sections of one level from the Elettra webelements web site.
@param symbol: (str) element symbol
@param nl: (str) nl term, e.g. '2p' (no spin-orbit)
@return: PhotoIonizationData(photon_energy, cross_section, asymmetry_parameter)
named tuple of 1-dimensional numpy arrays.
"""
spi = None
url = f"https://vuo.elettra.eu/services/elements/data/{symbol.lower()}{nl}.txt"
try:
data = urllib.request.urlopen(url)
except urllib.error.HTTPError:
pass
else:
a = np.genfromtxt(data)
try:
spi = SubShellPhotoIonization(a[:, 0], a[:, 1], a[:, 4])
except IndexError:
pass
return spi
def load_elettra_data():
"""
download the cross sections from the Elettra webelements web site.
@return: cross-section data in a nested dictionary, cf. load_pickled_data().
"""
data = {}
for element in pt.elements:
element_data = {}
for nlj in element.binding_energy:
nl = nlj[0:2]
eb = element.binding_energy[nlj]
if nl not in element_data and eb <= 2000:
spi = load_elettra_file(element.symbol, nl)
if spi is not None:
element_data[nl] = spi
if len(element_data):
data[element.symbol] = element_data
return data
def save_pickled_data(path, data):
"""
save a cross section data dictionary to a python-pickled file.
@param path: file path
@param data: cross-section data in a nested dictionary, cf. load_pickled_data().
@return: None
"""
with open(path, "wb") as f:
pickle.dump(data, f)
def load_pickled_data(path):
"""
load the cross section data from a python-pickled file.
the file can be generated by the save_pickled_data() function.
@param path: file path
@return: cross-section data in a nested dictionary.
the first-level keys are element symbols.
the second-level keys are 'nl' terms (e.g. '2p').
note that the Yeh and Lindau tables do not resolve spin-orbit splitting.
the data items are (photon_energy, cross_sections) tuples
of 1-dimensional numpy arrays holding the data table.
cross section values are given in Mb.
"""
with open(path, "rb") as f:
data = pickle.load(f)
return data
def init(table, reload=False):
"""
loads cross-section data into the periodic table.
this function is called by the periodictable to load the data on demand.
@param table:
@param reload:
@return:
"""
if 'photoionization' in table.properties and not reload:
return
table.properties.append('photoionization')
# default value
pt.core.Element.photoionization = PhotoIonization()
p = Path(Path(__file__).parent, "cross-sections.dat")
data = load_pickled_data(p)
for el_key, el_data in data.items():
# el_data is dict('nl': PhotoIonizationData)
try:
el = table[int(el_key)]
except ValueError:
el = table.symbol(el_key)
el.photoionization = PhotoIonization(el_data)
def _load_photoionization():
"""
delayed loading of the binding energy table.
"""
init(periodictable.core.default_table())
periodictable.core.delayed_load(['photoionization'], _load_photoionization)

209
pmsco/elements/spectrum.py Normal file
View File

@@ -0,0 +1,209 @@
"""
@package pmsco.elements.spectrum
photoelectron spectrum simulator
this module calculates the basic structure of a photoelectron spectrum.
it calculates positions and approximate amplitude of elastic peaks
based on photon energy, binding energy, photoionization cross section, and stoichiometry.
escape depth, photon flux, analyser transmission are not accounted for.
usage
-----
this module requires python 3.6, numpy, matplotlib and
the periodictable package (https://pypi.python.org/pypi/periodictable).
~~~~~~{.py}
import numpy as np
import periodictable as pt
import pmsco.elements.spectrum as spec
# for working with the data
labels, energy, intensity = spec.build_spectrum(800., {"Ti": 1, "O": 2})
# for plotting
spec.plot_spectrum(800., {"Ti": 1, "O": 2})
~~~~~~
@author Matthias Muntwiler
@copyright (c) 2020 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from matplotlib import pyplot as plt
import numpy as np
import periodictable as pt
from . import bindingenergy
from . import photoionization
def get_element(number_or_symbol):
"""
return the given Element object of the periodic table.
@param number_or_symbol: atomic number (int) or chemical symbol (str).
@return: Element object.
"""
try:
el = pt.elements[number_or_symbol]
except KeyError:
el = pt.elements.symbol(number_or_symbol)
return el
def get_binding_energy(photon_energy, element, nlj):
"""
look up the binding energy of a core level and check whether it is smaller than the photon energy.
@param photon_energy: photon energy in eV.
@param element: Element object of the periodic table.
@param nlj: (str) spectroscopic term, e.g. '4f7/2'.
@return: (float) binding energy or numpy.nan.
"""
try:
eb = element.binding_energy[nlj]
except KeyError:
return np.nan
if eb < photon_energy:
return eb
else:
return np.nan
def get_cross_section(photon_energy, element, nlj, gamma=None):
"""
look up the photo-ionization cross-section.
since the Yeh/Lindau tables do not resolve the spin-orbit splitting,
this function applies the normal relative weights of a full sub-shell.
the result is a linear interpolation between tabulated values.
@param photon_energy: photon energy in eV.
@param element: Element object of the periodic table.
@param nlj: (str) spectroscopic term, e.g. '4f7/2'.
the j-value can be left out, in which case the sum over all j-states is returned.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@return: (float) total (gamma=None) or differential (gamma not None) cross section in Mb.
"""
if not hasattr(element, "photoionization"):
element = get_element(element)
try:
pi = element.photoionization[nlj]
except KeyError:
return np.nan
if gamma is None:
cs = pi.cross_section(photon_energy)
else:
cs = pi.diff_cross_section(photon_energy, gamma)
return cs
def build_spectrum(photon_energy, elements, binding_energy=False, work_function=4.5, gamma=None):
"""
calculate the positions and amplitudes of core-level photoemission lines.
the function looks up the binding energies and cross sections of all photoemission lines in the energy range
given by the photon energy and returns an array of expected spectral lines.
@param photon_energy: (numeric) photon energy in eV.
@param elements: list or dictionary of elements.
elements are identified by their atomic number (int) or chemical symbol (str).
if a dictionary is given, the (float) values are stoichiometric weights of the elements.
@param binding_energy: (bool) return binding energies (True) rather than kinetic energies (False, default).
@param work_function: (float) work function of the instrument in eV.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@return: tuple (labels, positions, intensities) of 1-dimensional numpy arrays representing the spectrum.
labels are in the format {Symbol}{n}{l}{j}.
"""
ekin = []
ebind = []
intens = []
labels = []
for element in elements:
el = get_element(element)
for n in range(1, 8):
for l in "spdf":
for j in ['', '1/2', '3/2', '5/2', '7/2']:
nlj = f"{n}{l}{j}"
eb = get_binding_energy(photon_energy, el, nlj)
cs = get_cross_section(photon_energy, el, nlj, gamma=gamma)
try:
cs = cs * elements[element]
except (KeyError, TypeError):
pass
if not np.isnan(eb) and not np.isnan(cs):
ekin.append(photon_energy - eb - work_function)
ebind.append(eb)
intens.append(cs)
labels.append(f"{el.symbol}{nlj}")
ebind = np.array(ebind)
ekin = np.array(ekin)
intens = np.array(intens)
labels = np.array(labels)
if binding_energy:
return labels, ebind, intens
else:
return labels, ekin, intens
def plot_spectrum(photon_energy, elements, binding_energy=False, work_function=4.5, gamma=None, show_labels=True):
"""
plot a simple spectrum representation of a material.
the function looks up the binding energies and cross sections of all photoemission lines in the energy range
given by the photon energy and returns an array of expected spectral lines.
the spectrum is plotted using matplotlib.pyplot.stem.
@param photon_energy: (numeric) photon energy in eV.
@param elements: list or dictionary of elements.
elements are identified by their atomic number (int) or chemical symbol (str).
if a dictionary is given, the (float) values are stoichiometric weights of the elements.
@param binding_energy: (bool) return binding energies (True) rather than kinetic energies (False, default).
@param work_function: (float) work function of the instrument in eV.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@param show_labels: (bool) show peak labels (True, default) or not (False).
@return: (figure, axes)
"""
labels, energy, intensity = build_spectrum(photon_energy, elements, binding_energy=binding_energy,
work_function=work_function, gamma=gamma)
fig, ax = plt.subplots()
ax.stem(energy, intensity, basefmt=' ', use_line_collection=True)
if show_labels:
for sxy in zip(labels, energy, intensity):
ax.annotate(sxy[0], xy=(sxy[1], sxy[2]), textcoords='data')
ax.grid()
if binding_energy:
ax.set_xlabel('binding energy')
else:
ax.set_xlabel('kinetic energy')
ax.set_ylabel('intensity')
ax.set_title(elements)
return fig, ax
def plot_cross_section(el, nlj):
energy = np.arange(100, 1500, 140)
cs = get_cross_section(energy, el, nlj)
fig, ax = plt.subplots()
ax.set_yscale("log")
ax.plot(energy, cs)

121
pmsco/files.py
View File

@@ -27,28 +27,28 @@ logger = logging.getLogger(__name__)
#
# each string of this set marks a category of files.
#
# @arg @c 'input' : raw input files for calculator, including cluster and phase files in custom format
# @arg @c 'output' : raw output files from calculator
# @arg @c 'phase' : phase files in portable format for report
# @arg @c 'cluster' : cluster files in portable XYZ format for report
# @arg @c 'log' : log files
# @arg @c 'debug' : debug files
# @arg @c 'model': output files in ETPAI format: complete simulation (a_-1_-1_-1_-1)
# @arg @c 'scan' : output files in ETPAI format: scan (a_b_-1_-1_-1)
# @arg @c 'symmetry' : output files in ETPAI format: symmetry (a_b_c_-1_-1)
# @arg @c 'emitter' : output files in ETPAI format: emitter (a_b_c_d_-1)
# @arg @c 'region' : output files in ETPAI format: region (a_b_c_d_e)
# @arg @c 'report': final report of results
# @arg @c 'population': final state of particle population
# @arg @c 'rfac': files related to models which give bad r-factors (dynamic category, see below).
# @arg 'input' : raw input files for calculator, including cluster and atomic files in custom format
# @arg 'output' : raw output files from calculator
# @arg 'atomic' : atomic scattering (phase, emission) files in portable format
# @arg 'cluster' : cluster files in portable XYZ format for report
# @arg 'log' : log files
# @arg 'debug' : debug files
# @arg 'model': output files in ETPAI format: complete simulation (a_-1_-1_-1_-1)
# @arg 'scan' : output files in ETPAI format: scan (a_b_-1_-1_-1)
# @arg 'domain' : output files in ETPAI format: domain (a_b_c_-1_-1)
# @arg 'emitter' : output files in ETPAI format: emitter (a_b_c_d_-1)
# @arg 'region' : output files in ETPAI format: region (a_b_c_d_e)
# @arg 'report': final report of results
# @arg 'population': final state of particle population
# @arg 'rfac': files related to models which give bad r-factors (dynamic category, see below).
#
# @note @c 'rfac' is a dynamic category not connected to a particular file or content type.
# no file should be marked @c 'rfac'.
# the string is used only to specify whether bad models should be deleted or not.
# if so, all files related to bad models are deleted, regardless of their static category.
#
FILE_CATEGORIES = {'cluster', 'phase', 'input', 'output',
'report', 'region', 'emitter', 'scan', 'symmetry', 'model',
FILE_CATEGORIES = {'cluster', 'atomic', 'input', 'output',
'report', 'region', 'emitter', 'scan', 'domain', 'model',
'log', 'debug', 'population', 'rfac'}
## @var FILE_CATEGORIES_TO_KEEP
@@ -242,37 +242,52 @@ class FileTracker(object):
else:
self._complete_models.discard(model)
def delete_files(self, categories=None):
def delete_files(self, categories=None, incomplete_models=False):
"""
delete the files matching the list of categories.
delete all files matching a set of categories.
@version this method does not act on the 'rfac' category.
this function deletes all files that are tagged with one of the given categories.
tags are set by the code sections that create the files.
for a list of common categories, see FILE_CATEGORIES.
the categories can be given as an argument or taken from the categories_to_delete property.
files are deleted regardless of R-factor.
be sure to specify only categories that you don't need in the output at all.
by default, only files of complete models (cf. set_model_complete()) are deleted
to avoid interference with running calculations.
to clean up after calculations, the incomplete_models argument can override this.
@note this method does not act on the special 'rfac' category (see delete_bad_rfac()).
@param categories: set of file categories to delete.
defaults to self.categories_to_delete.
if the argument is None, it defaults to the categories_to_delete property.
@param incomplete_models: (bool) delete files of incomplete models as well.
by default (False), incomplete models are not deleted.
@return: None
"""
if categories is None:
categories = self.categories_to_delete
for cat in categories:
self.delete_category(cat)
self.delete_category(cat, incomplete_models=incomplete_models)
def delete_bad_rfac(self, keep=0, force_delete=False):
"""
delete the files of all models except a specified number of good models.
delete all files of all models except for a specified number of best ranking models.
the method first determines which models to keep.
models with R factor values of 0.0, without a specified R-factor, and
the specified number of best ranking non-zero models are kept.
the files belonging to the keeper models are kept, all others are deleted,
regardless of category.
files of incomplete models are also kept.
in addition, incomplete models, models with R factor = 0.0,
and those without a specified R-factor are kept.
all other files are deleted.
the method does not consider the file category.
the files are deleted from the list and the file system.
files are deleted only if 'rfac' is specified in self.categories_to_delete
or if force_delete is set to True.
the method executes only if 'rfac' is specified in self.categories_to_delete
or if force_delete is True.
otherwise the method does nothing.
@param keep: number of files to keep.
@@ -330,18 +345,32 @@ class FileTracker(object):
return len(del_models)
def delete_category(self, category):
def delete_category(self, category, incomplete_models=False):
"""
delete all files of a specified category from the list and the file system.
only files of complete models (cf. set_model_complete()) are deleted, but regardless of R-factor.
this function deletes all files that are tagged with the given category.
tags are set by the code sections that create the files.
for a list of common categories, see FILE_CATEGORIES.
files are deleted regardless of R-factor.
be sure to specify only categories that you don't need in the output at all.
by default, only files of complete models (cf. set_model_complete()) are deleted
to avoid interference with running calculations.
to clean up after calculations, the incomplete_models argument can override this.
@param category: (str) category.
should be one of FILE_CATEGORIES. otherwise, the function has no effect.
@param incomplete_models: (bool) delete files of incomplete models as well.
by default (False), incomplete models are not deleted.
@return: None
"""
del_names = {name for (name, cat) in self._file_category.items() if cat == category}
del_names &= {name for (name, model) in self._file_model.items() if model in self._complete_models}
if not incomplete_models:
del_names &= {name for (name, model) in self._file_model.items() if model in self._complete_models}
for name in del_names:
self.delete_file(name)
@@ -375,3 +404,33 @@ class FileTracker(object):
logger.warning("file system error deleting file {0}".format(path))
else:
logger.debug("delete file {0} ({1}, model {2})".format(path, cat, model))
def list_files_other_models(prefix, models):
"""
list input/output files except those of the given models.
this can be used to clean up all files except those belonging to the given models.
to delete the listed files:
for f in files:
os.remove(f)
@param prefix: file name prefix up to the first underscore.
only files starting with this prefix are listed.
@param models: sequence or set of model numbers that should not be listed.
@return: set of file names
"""
file_names = set([])
for entry in os.scandir():
if entry.is_file:
elements = entry.name.split('_')
try:
if len(elements) == 6 and elements[0] == prefix and int(elements[1]) not in models:
file_names.add(entry.name)
except (IndexError, ValueError):
pass
return file_names

0
pmsco/graphics/__init__.py
View File

178
pmsco/graphics/cluster.py Executable file
View File

@@ -0,0 +1,178 @@
#!/usr/bin/env python
"""
@package pmsco.graphics.cluster
graphics rendering module for clusters.
this module is experimental.
interface and implementation may change without notice.
at the moment we are evaluating rendering solutions.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2017 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import os
import numpy as np
import argparse
import logging
logger = logging.getLogger(__name__)
try:
import pymol2
except ImportError:
logger.warning("error importing pymol2. cluster rendering using pymol2 disabled.")
pymol2 = None
try:
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# from matplotlib.backends.backend_pdf import FigureCanvasPdf
# from matplotlib.backends.backend_svg import FigureCanvasSVG
except ImportError:
Axes3D = None
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. cluster rendering using matplotlib disabled.")
def render_file(spath, view):
sname = "cluster"
opath = spath + ".png"
pm = pymol2.PyMOL()
cmd = pm.cmd
pm.start()
try:
cmd.reinitialize()
cmd.load(spath, sname)
cmd.disable("all")
cmd.enable(sname)
cmd.set("orthoscopic", 1)
cmd.bg_color("white")
cmd.show_as("spheres")
cmd.alter("all", "vdw=0.8")
#cmd.show("sticks")
#zoom selection-expression # selection to fill the viewer
#orient selection-expression # largest dim horizontal, second-largest vertical
#cmd.orient() --- should stick to fixed orientation
#cmd.turn("x", -90)
#cmd.turn("x", 0)
#cmd.turn("y", 0)
#cmd.clip("slab", 5.0)
cmd.viewport(640, 640)
cmd.zoom(complete=1)
#pymol.cmd.rebuild() #--- necessary?
cmd.png(opath)
finally:
pm.stop()
def render_cluster(clu):
pass
def set_axes_equal(ax):
"""
Make axes of 3D plot have equal scale so that spheres appear as spheres,
cubes as cubes, etc.. This is one possible solution to Matplotlib's
ax.set_aspect('equal') and ax.axis('equal') not working for 3D.
@author https://stackoverflow.com/a/31364297
@param ax: a matplotlib axis, e.g., as output from plt.gca().
"""
x_limits = ax.get_xlim3d()
y_limits = ax.get_ylim3d()
z_limits = ax.get_zlim3d()
x_range = abs(x_limits[1] - x_limits[0])
x_middle = np.mean(x_limits)
y_range = abs(y_limits[1] - y_limits[0])
y_middle = np.mean(y_limits)
z_range = abs(z_limits[1] - z_limits[0])
z_middle = np.mean(z_limits)
# The plot bounding box is a sphere in the sense of the infinity
# norm, hence I call half the max range the plot radius.
plot_radius = 0.5*max([x_range, y_range, z_range])
ax.set_xlim3d([x_middle - plot_radius, x_middle + plot_radius])
ax.set_ylim3d([y_middle - plot_radius, y_middle + plot_radius])
ax.set_zlim3d([z_middle - plot_radius, z_middle + plot_radius])
def render_xyz_matplotlib(filename, data, canvas=None):
"""
produce a graphics file from an array of 3d coordinates in the matplotlib scatter style.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy array of shape (N,3).
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111, projection='3d')
# ax.set_aspect('equal')
try:
# method available in matplotlib 2.1 and later
ax.set_proj_type('ortho')
except AttributeError:
pass
ax.scatter(data[:, 0], data[:, 1], data[:, 2], c='r', marker='o')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
set_axes_equal(ax)
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def exec_cli():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--view', default='z')
parser.add_argument(dest='files', nargs='+')
args = parser.parse_args()
for fil in args.files:
render_file(fil, args.view)
if __name__ == '__main__':
exec_cli()
sys.exit(0)

2
pmsco/graphics/rfactor.py
View File

@@ -182,7 +182,7 @@ def render_results(results_file, data=None):
"""
if data is None:
data = np.genfromtxt(results_file, names=True)
data = np.atleast_1d(np.genfromtxt(results_file, names=True))
summary = evaluate_results(data)

45
pmsco/graphics/scan.py
View File

@@ -7,16 +7,13 @@ interface and implementation are subject to change.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
@copyright (c) 2018-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
@@ -36,7 +33,7 @@ except ImportError:
logger.warning("error importing matplotlib. graphics rendering disabled.")
def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False):
def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False, ref_data=None):
"""
produce a graphics file from a one-dimensional scan file.
@@ -47,14 +44,22 @@ def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False):
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of EI, ETPI or ETPAI data.
@param scan_mode: list containing the field name of the scanning axis of the data array.
it must contain one element exactly.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to set axis labels.
@param ref_data: numpy-structured array of EI, ETPI or ETPAI data.
this is reference data (e.g. experimental data) that should be plotted with the main dataset.
both datasets will be plotted on the same axis and should have similar data range.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@@ -66,6 +71,8 @@ def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False):
canvas(fig)
ax = fig.add_subplot(111)
if ref_data is not None:
ax.plot(ref_data[scan_mode[0]], ref_data['i'], 'k.')
ax.plot(data[scan_mode[0]], data['i'])
ax.set_xlabel(scan_mode[0])
@@ -125,9 +132,8 @@ def render_ea_scan(filename, data, scan_mode, canvas=None, is_modf=False):
im.set_cmap("RdBu_r")
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
im.set_clim((dlo, dhi))
im.set_clim((-1., 1.))
try:
# requires matplotlib 2.1.0
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
@@ -196,8 +202,10 @@ def render_tp_scan(filename, data, canvas=None, is_modf=False):
cb = fig.colorbar(pc, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(data['i'], 2)
dhi = np.nanpercentile(data['i'], 98)
clip = 2
dlo = np.nanpercentile(data['i'], clip)
dhi = np.nanpercentile(data['i'], 100 - clip)
if is_modf:
pc.set_cmap("RdBu_r")
# im.set_cmap("coolwarm")
@@ -205,7 +213,6 @@ def render_tp_scan(filename, data, canvas=None, is_modf=False):
dlo = -dhi
pc.set_clim((dlo, dhi))
try:
# requires matplotlib 2.1.0
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
@@ -216,16 +223,19 @@ def render_tp_scan(filename, data, canvas=None, is_modf=False):
# im.set_cmap("inferno")
# im.set_cmap("viridis")
pc.set_clim((dlo, dhi))
ti = cb.get_ticks()
ti = [min(ti), max(ti)]
cb.set_ticks(ti)
try:
ti = cb.get_ticks()
ti = [min(ti), max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scan(filename, data=None):
def render_scan(filename, data=None, ref_data=None):
"""
produce a graphics file from a scan file.
@@ -248,6 +258,11 @@ def render_scan(filename, data=None):
@param data: numpy-structured array of ETPI or ETPAI data.
if this argument is omitted, the data is loaded from the file referenced by the filename argument.
@param ref_data: numpy-structured array of ETPI or ETPAI data.
this is reference data (e.g. experimental data) that should be plotted with the main dataset.
this is supported for 1d scans only.
both datasets will be plotted on the same axis and should have similar data range.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
"""
@@ -258,7 +273,7 @@ def render_scan(filename, data=None):
try:
if len(scan_mode) == 1:
out_filename = render_1d_scan(filename, data, scan_mode, is_modf=is_modf)
out_filename = render_1d_scan(filename, data, scan_mode, is_modf=is_modf, ref_data=ref_data)
elif len(scan_mode) == 2 and 'e' in scan_mode:
out_filename = render_ea_scan(filename, data, scan_mode, is_modf=is_modf)
elif len(scan_mode) == 2 and 't' in scan_mode and 'p' in scan_mode:

210
pmsco/graphics/scattering.py Normal file
View File

@@ -0,0 +1,210 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
import scipy.interpolate
import scipy.special
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
class TMatrix(object):
def __init__(self):
"""
self.en.shape = (n_e,)
self.tl.shape = (n_e, n_l)
"""
self.en = None
self.tl = None
def load_test_data(self):
self.en = np.array([100.])
raw = [-0.052845, -0.003238, 0.478705, 0.672581, 0.137932, 0.981700, 0.323890, 0.805299, 0.291814, 0.776792,
0.369416, 0.351845, 0.199775, 0.113314, 0.062479, 0.025691, 0.013699, 0.005283]
re_tl = np.array(raw[0::2])
im_tl = np.array(raw[1::2])
self.tl = re_tl + 1j * im_tl
def load_edac_scattering(self, f, energy=math.nan):
"""
load T matrix from EDAC scattering file
currently, only the 'tl' format is supported.
@param f: file path
@param energy: kinetic energy in eV if none is defined in the file
@return: None
"""
with open(f, "r") as fi:
h = fi.readline().rstrip().split(' ')
ne = int(h[0])
if ne > 1:
assert h[1] == 'E(eV)'
del h[1]
lmax = int(h[1])
assert h[2] == 'regular'
assert h[3] == 'tl'
self.load_edac_tl(f, ne, lmax, energy=energy)
def load_edac_tl(self, f, ne, lmax, energy=math.nan):
"""
load T matrix from EDAC scattering file in 'tl' format
@param f: file path
@param ne: number of energies (rows)
@param lmax: maximum l number (columns = 2 * (lmax + 1))
@param energy: kinetic energy in eV if none is defined in the file
@return: None
"""
if ne > 1:
self.en = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=[0]))
start_col = 1
else:
self.en = np.asarray(energy)
start_col = 0
re_cols = range(start_col, start_col + (lmax + 1) * 2, 2)
im_cols = range(start_col + 1, start_col + (lmax + 1) * 2, 2)
re_tl = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=re_cols))
im_tl = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=im_cols))
self.tl = re_tl + 1j * im_tl
assert self.tl.shape == (ne, lmax + 1), "array shape mismatch"
def planewave_amplitude(self, energy, angle):
"""
total, complex plane wave scattering amplitude for given energy and angle
@param energy: kinetic energy in eV.
this can be a numeric value, a 1-dimensional numpy.ndarray,
or any value accepted by the numpy.asarray function.
@param angle: scattering angle in degrees (0..180).
this can be a numeric value, a 1-dimensional numpy.ndarray,
or any value accepted by the numpy.asarray function.
@return: 3 numpy arrays (amp, magnitude, phase) representing the scattering amplitude
versus energy and angle.
the shape of the three arrays is (n_energies, n_angles).
@arg amp: complex scattering amplitude.
@arg magnitude: magnitude (absolute value) of the scattering amplitude.
@arg phase: phase angle in radians of the scattering amplitude.
"""
if not isinstance(energy, np.ndarray):
energy = np.atleast_1d(np.asarray(energy))
ne = len(energy)
if not isinstance(angle, np.ndarray):
angle = np.atleast_1d(np.array(angle))
na = len(angle)
kinv = 1. / (0.513019932 * np.sqrt(energy))
f_tl = scipy.interpolate.interp1d(self.en, self.tl, axis=0, copy=False)
tl = f_tl(energy)
cos_angle = np.cos(np.radians(angle))
lmax = self.tl.shape[1] - 1
l = np.arange(0, lmax + 1)
amp = np.zeros((ne, na), dtype=complex)
for ia, ca in enumerate(cos_angle):
lpmn, __ = scipy.special.lpmn(0, lmax, ca)
fpart = np.outer(kinv, (2 * l + 1) * lpmn[0]) * tl
ftot = np.sum(fpart, axis=-1)
amp[:, ia] = ftot
mag = np.abs(amp)
pha = np.angle(amp)
return amp, mag, pha
def render_scattering_1d(filename, tmatrix, energy=None):
if energy is None:
en = tmatrix.en[0]
else:
en = energy
an = np.arange(0, 181, 2)
__, mag, pha = tmatrix.planewave_amplitude(en, an)
pha = pha / math.pi
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(211)
ax.plot(an, mag[0])
ax.set_xlabel('th (deg)')
ax.set_ylabel('mag (arb)')
ax = fig.add_subplot(212)
ax.plot(an, pha[0])
ax.set_xlabel('th (deg)')
ax.set_ylabel('pha (1/pi)')
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scattering_2d(filename, tmatrix):
en = tmatrix.en
an = np.arange(0, 181, 2)
__, mag, pha = tmatrix.planewave_amplitude(en, an)
pha = pha / math.pi
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(211)
im = ax.imshow(mag, origin='lower', aspect='auto', interpolation='none')
im.set_extent((an[0], an[-1], en[0], en[-1]))
im.set_cmap("magma")
ax.set_xlabel('th (deg)')
ax.set_ylabel('E (eV)')
# cb = ax.colorbar(im, shrink=0.4, pad=0.1)
# ti = cb.get_ticks()
# ti = [0., max(ti)]
# cb.set_ticks(ti)
ax = fig.add_subplot(212)
im = ax.imshow(pha, origin='lower', aspect='auto', interpolation='none')
im.set_extent((an[0], an[-1], en[0], en[-1]))
im.set_cmap("RdBu_r")
ax.set_xlabel('th (deg)')
ax.set_ylabel('E (eV)')
# cb = ax.colorbar(im, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(mag, 2)
dhi = np.nanpercentile(mag, 98)
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
im.set_clim((dlo, dhi))
# ti = cb.get_ticks()
# ti = [min(ti), 0., max(ti)]
# cb.set_ticks(ti)
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scattering_map(filename, energy):
tmatrix = TMatrix()
tmatrix.load_edac_scattering(filename, energy)
if tmatrix.tl.shape[0] == 1:
out_filename = render_scattering_1d(filename, tmatrix)
else:
out_filename = render_scattering_2d(filename, tmatrix)
return out_filename

132
pmsco/handlers.py
View File

@@ -1,6 +1,6 @@
"""
@package pmsco.handlers
project-independent task handlers for models, scans, symmetries, emitters and energies.
project-independent task handlers for models, scans, domains, emitters and energies.
calculation tasks are organized in a hierarchical tree.
at each node, a task handler (feel free to find a better name)
@@ -20,9 +20,9 @@ the handlers of the structural optimizers are declared in separate modules.
scans are defined by the project.
the actual merging step from multiple scans into one result dataset is delegated to the project class.
<em>symmetry handlers</em> split a task into one child per symmetry.
symmetries are defined by the project.
the actual merging step from multiple symmetries into one result dataset is delegated to the project class.
<em>domain handlers</em> split a task into one child per domain.
domains are defined by the project.
the actual merging step from multiple domains into one result dataset is delegated to the project class.
<em>emitter handlers</em> split a task into one child per emitter configuration (inequivalent sets of emitting atoms).
emitter configurations are defined by the project.
@@ -35,31 +35,28 @@ code inspection and tests have shown that per-emitter results from EDAC can be s
in order to take advantage of parallel processing.
while several classes of model handlers are available,
the default handlers for scans, symmetries, emitters and energies should be sufficient in most situations.
the scan and symmetry handlers call methods of the project class to invoke project-specific functionality.
the default handlers for scans, domains, emitters and energies should be sufficient in most situations.
the scan and domain handlers call methods of the project class to invoke project-specific functionality.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-18 by Paul Scherrer Institut @n
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import datetime
from functools import reduce
import logging
import math
import numpy as np
import os
from pathlib import Path
from pmsco.compat import open
import pmsco.data as md
import pmsco.dispatch as dispatch
import pmsco.graphics.scan as mgs
from pmsco.helpers import BraceMessage as BMsg
@@ -127,10 +124,14 @@ class TaskHandler(object):
for best efficiency the number of tasks generated should be greater or equal the number of slots.
it should not exceed N times the number of slots, where N is a reasonably small number.
@return None
@return (int) number of children that create_tasks() will generate on average.
the number does not need to be accurate, a rough estimate or order of magnitude if greater than 10 is fine.
it is used to distribute processing slots across task levels.
see pmsco.dispatch.MscoMaster.setup().
"""
self._project = project
self._slots = slots
return 1
def cleanup(self):
"""
@@ -344,7 +345,6 @@ class SingleModelHandler(ModelHandler):
modf_ext = ".modf" + parent_task.file_ext
parent_task.modf_filename = parent_task.file_root + modf_ext
assert not math.isnan(task.rfac)
self.result = task.model.copy()
self.result['_rfac'] = task.rfac
@@ -373,8 +373,8 @@ class SingleModelHandler(ModelHandler):
keys = [key for key in self.result]
keys.sort(key=lambda t: t[0].lower())
vals = (str(self.result[key]) for key in keys)
filename = self._project.output_file + ".dat"
with open(filename, "w") as outfile:
filename = Path(self._project.output_file).with_suffix(".dat")
with open(filename, "wt", encoding="latin1") as outfile:
outfile.write("# ")
outfile.write(" ".join(keys))
outfile.write("\n")
@@ -417,6 +417,8 @@ class ScanHandler(TaskHandler):
def setup(self, project, slots):
"""
initialize the scan task handler and save processed experimental scans.
@return (int) number of scans defined in the project.
"""
super(ScanHandler, self).setup(project, slots)
@@ -431,13 +433,15 @@ class ScanHandler(TaskHandler):
if project.combined_scan is not None:
ext = md.format_extension(project.combined_scan)
filename = project.output_file + ext
filename = Path(project.output_file).with_suffix(ext)
md.save_data(filename, project.combined_scan)
if project.combined_modf is not None:
ext = md.format_extension(project.combined_modf)
filename = project.output_file + ".modf" + ext
filename = Path(project.output_file).with_suffix(".modf" + ext)
md.save_data(filename, project.combined_modf)
return len(self._project.scans)
def create_tasks(self, parent_task):
"""
generate a calculation task for each scan of the given parent task.
@@ -527,7 +531,7 @@ class ScanHandler(TaskHandler):
return None
class SymmetryHandler(TaskHandler):
class DomainHandler(TaskHandler):
## @var _pending_ids_per_parent
# (dict) sets of child task IDs per parent
#
@@ -547,20 +551,29 @@ class SymmetryHandler(TaskHandler):
# the values are sets of all child CalculationTask.id belonging to the parent.
def __init__(self):
super(SymmetryHandler, self).__init__()
super(DomainHandler, self).__init__()
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the domain task handler.
@return (int) number of domains defined in the project.
"""
super(DomainHandler, self).setup(project, slots)
return len(self._project.domains)
def create_tasks(self, parent_task):
"""
generate a calculation task for each symmetry of the given parent task.
generate a calculation task for each domain of the given parent task.
all symmetries share the same model parameters.
all domains share the same model parameters.
@return list of CalculationTask objects, with one element per symmetry.
the symmetry index varies according to project.symmetries.
@return list of CalculationTask objects, with one element per domain.
the domain index varies according to project.domains.
"""
super(SymmetryHandler, self).create_tasks(parent_task)
super(DomainHandler, self).create_tasks(parent_task)
parent_id = parent_task.id
self._parent_tasks[parent_id] = parent_task
@@ -568,10 +581,10 @@ class SymmetryHandler(TaskHandler):
self._complete_ids_per_parent[parent_id] = set()
out_tasks = []
for (i_sym, sym) in enumerate(self._project.symmetries):
for (i_dom, domain) in enumerate(self._project.domains):
new_task = parent_task.copy()
new_task.parent_id = parent_id
new_task.change_id(sym=i_sym)
new_task.change_id(domain=i_dom)
child_id = new_task.id
self._pending_tasks[child_id] = new_task
@@ -580,25 +593,25 @@ class SymmetryHandler(TaskHandler):
out_tasks.append(new_task)
if not out_tasks:
logger.error("no symmetry tasks generated. your project must declare at least one symmetry.")
logger.error("no domain tasks generated. your project must declare at least one domain.")
return out_tasks
def add_result(self, task):
"""
collect and combine the calculation results versus symmetry.
collect and combine the calculation results versus domain.
* mark the task as complete
* store its result for later
* check whether this was the last pending task of the family (belonging to the same parent).
the actual merging of data is delegated to the project's combine_symmetries() method.
the actual merging of data is delegated to the project's combine_domains() method.
@param task: (CalculationTask) calculation task that completed.
@return parent task (CalculationTask) if the family is complete. None if the family is not complete yet.
"""
super(SymmetryHandler, self).add_result(task)
super(DomainHandler, self).add_result(task)
self._complete_tasks[task.id] = task
del self._pending_tasks[task.id]
@@ -608,7 +621,7 @@ class SymmetryHandler(TaskHandler):
family_pending.remove(task.id)
family_complete.add(task.id)
# all symmetries complete?
# all domains complete?
if len(family_pending) == 0:
parent_task = self._parent_tasks[task.parent_id]
@@ -625,11 +638,12 @@ class SymmetryHandler(TaskHandler):
parent_task.time = reduce(lambda a, b: a + b, child_times)
if parent_task.result_valid:
self._project.combine_symmetries(parent_task, child_tasks)
self._project.combine_domains(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'scan')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'scan')
graph_file = mgs.render_scan(parent_task.modf_filename)
graph_file = mgs.render_scan(parent_task.modf_filename,
ref_data=self._project.scans[parent_task.id.scan].modulation)
self._project.files.add_file(graph_file, parent_task.id.model, 'scan')
del self._pending_ids_per_parent[parent_task.id]
@@ -669,6 +683,19 @@ class EmitterHandler(TaskHandler):
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the emitter task handler.
@return (int) estimated number of emitter configurations that the cluster generator will generate.
the estimate is based on the start parameters, scan 0 and domain 0.
"""
super(EmitterHandler, self).setup(project, slots)
mock_model = self._project.model_space.start
mock_index = dispatch.CalcID(-1, 0, 0, -1, -1)
n_emitters = project.cluster_generator.count_emitters(mock_model, mock_index)
return n_emitters
def create_tasks(self, parent_task):
"""
generate a calculation task for each emitter configuration of the given parent task.
@@ -750,10 +777,11 @@ class EmitterHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_emitters(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'symmetry')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'symmetry')
graph_file = mgs.render_scan(parent_task.modf_filename)
self._project.files.add_file(graph_file, parent_task.id.model, 'symmetry')
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'domain')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'domain')
graph_file = mgs.render_scan(parent_task.modf_filename,
ref_data=self._project.scans[parent_task.id.scan].modulation)
self._project.files.add_file(graph_file, parent_task.id.model, 'domain')
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@@ -920,7 +948,7 @@ class EnergyRegionHandler(RegionHandler):
@param slots (int) number of calculation slots (processes).
@return None
@return (int) average number of child tasks
"""
super(EnergyRegionHandler, self).setup(project, slots)
@@ -933,6 +961,8 @@ class EnergyRegionHandler(RegionHandler):
logger.debug(BMsg("region handler: split scan {file} into {slots} chunks",
file=os.path.basename(scan.filename), slots=self._slots_per_scan[i]))
return max(int(sum(self._slots_per_scan) / len(self._slots_per_scan)), 1)
def create_tasks(self, parent_task):
"""
generate a calculation task for each energy region of the given parent task.
@@ -971,27 +1001,3 @@ class EnergyRegionHandler(RegionHandler):
logger.error("no region tasks generated. this is probably a bug.")
return out_tasks
def choose_region_handler_class(project):
"""
choose a suitable region handler for the project.
the function returns the EnergyRegionHandler class
if the project includes an energy scan with at least 10 steps.
Otherwise, it returns the SingleRegionHandler.
angle scans do not benefit from region splitting in EDAC.
@param project: Project instance.
@return: SingleRegionHandler or EnergyRegionHandler class.
"""
energy_scans = 0
for scan in project.scans:
if scan.energies.shape[0] >= 10:
energy_scans += 1
if energy_scans >= 1:
return EnergyRegionHandler
else:
return SingleRegionHandler

44
pmsco/helpers.py
View File

@@ -6,6 +6,13 @@ a collection of small and generic code bits mostly collected from the www.
"""
import contextlib
import ctypes
import io
import os
import sys
from typing import BinaryIO
class BraceMessage(object):
"""
@@ -22,3 +29,40 @@ class BraceMessage(object):
def __str__(self):
return self.fmt.format(*self.args, **self.kwargs)
libc = ctypes.CDLL(None)
c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout')
@contextlib.contextmanager
def stdout_redirected(dest_file: BinaryIO):
"""
A context manager to temporarily redirect stdout to a file.
Redirects all standard output from Python and the C library to the specified file.
This can be used, e.g., to capture output from Fortran code.
credit: https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/
@param dest_file: binary file open for writing ('wb' mode).
This function requires just the fileno function.
@return: None
"""
original_stdout_fd = sys.stdout.fileno()
def _redirect_stdout(to_fd):
"""Redirect stdout to the given file descriptor."""
libc.fflush(c_stdout)
sys.stdout.close()
os.dup2(to_fd, original_stdout_fd)
sys.stdout = io.TextIOWrapper(os.fdopen(original_stdout_fd, 'wb'))
saved_stdout_fd = os.dup(original_stdout_fd)
try:
_redirect_stdout(dest_file.fileno())
yield
_redirect_stdout(saved_stdout_fd)
finally:
os.close(saved_stdout_fd)

137
pmsco/igor.py Normal file
View File

@@ -0,0 +1,137 @@
"""
@package pmsco.igor
data exchange with wavemetrics igor pro.
this module provides functions for loading/saving pmsco data in igor pro.
@author Matthias Muntwiler
@copyright (c) 2019-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import numpy as np
def _escape_igor_string(s):
s = s.replace('\\', '\\\\')
s = s.replace('"', '\\"')
return s
def namefix_double(name):
"""
fix 1-character wave name by doubling
replaces length-1 string by a doubled version.
@param name: (str) proposed wave name
@return: corrected name
"""
return name*2 if len(name) == 1 else name
def namefix_etpais(name):
"""
fix 1-character wave name according to ETPAIS scheme
replaces 'e' by 'en' etc.
@param name: (str) proposed wave name
@return: corrected name
"""
name_map = {'e': 'en', 't': 'th', 'p': 'ph', 'i': 'in', 'm': 'mo', 's': 'si'}
try:
return name_map[name]
except KeyError:
return name
class IgorExport(object):
"""
class exports pmsco data to an Igor text (ITX) file.
usage:
1) create an object instance.
2) set @ref data.
3) set optional attributes: @ref prefix and @ref namefix.
4) call @ref export.
"""
def __init__(self):
super(IgorExport, self).__init__()
self.data = None
self.prefix = ""
self.namefix = namefix_double
def set_data(self, data):
"""
set the data array to export.
this must (currently) be a one-dimensional structured array.
the column names will become wave names.
@param data: numpy.ndarray
@return:
"""
self.data = data
def export(self, filename):
"""
write to igor file.
"""
with open(filename, 'wt', encoding="utf8") as f:
self._write_header(f)
self._write_data(f)
def _fix_name(self, name):
"""
fix a wave name.
this function first applies @ref namefix and @ref prefix to the proposed wave name.
@param name: (str) proposed wave name
@return: corrected name
"""
if self.namefix is not None:
name = self.namefix(name)
return self.prefix + name
def _write_header(self, f):
"""
write the header of the igor text file
@param f: open file or stream
@return: None
"""
f.write('IGOR' + '\n')
f.write('X // pmsco data export\n')
def _write_data(self, f):
"""
write a data section to the igor text file.
@param f: open file or stream
@return: None
"""
assert isinstance(self.data, np.ndarray)
assert len(self.data.shape) == 1
assert len(self.data.dtype.names[0]) >= 1
arr = self.data
shape = ",".join(map(str, arr.shape))
names = (self._fix_name(name) for name in arr.dtype.names)
names = ", ".join(names)
f.write('Waves/O/D/N=({shape}) {names}\n'.format(shape=shape, names=names))
f.write('BEGIN\n')
np.savetxt(f, arr, fmt='%g')
f.write('END\n')

1
pmsco/loess/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'matthias muntwiler'

74
pmsco/loess/makefile
View File

@@ -1,74 +0,0 @@
SHELL=/bin/sh
# makefile for the LOESS module
#
# required libraries: libblas, liblapack, libf2c
# (you may have to set soft links so that linker finds them)
#
# the makefile calls python-config to get the compilation flags and include path.
# you may override the corresponding variables on the command line or by environment variables:
#
# PYTHON_INC: specify additional include directories. each dir must start with -I prefix.
# PYTHON_CFLAGS: specify the C compiler flags.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .x
.PHONY: all loess test gas madeup ethanol air galaxy
OBJ=loessc.o loess.o predict.o misc.o loessf.o dqrsl.o dsvdc.o fix_main.o
FFLAGS?=-O
LIB=-lblas -lm -lf2c
LIBPATH?=
CC?=gcc
CCOPTS?=
SWIG?=swig
SWIGOPTS?=
PYTHON?=python
PYTHONOPTS?=
PYTHON_CONFIG = ${PYTHON}-config
#PYTHON_LIB ?= $(shell ${PYTHON_CONFIG} --libs)
#PYTHON_INC ?= $(shell ${PYTHON_CONFIG} --includes)
PYTHON_INC ?=
PYTHON_CFLAGS ?= $(shell ${PYTHON_CONFIG} --cflags)
#PYTHON_LDFLAGS ?= $(shell ${PYTHON_CONFIG} --ldflags)
all: loess
loess: _loess.so
loess.py _loess.so: loess.c loess.i
$(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
examples: gas madeup ethanol air galaxy
gas: gas.x
gas.x: gas.o $(OBJ)
$(CC) -o gas.x gas.o $(OBJ) $(LIB)
madeup: madeup.x
madeup.x: madeup.o $(OBJ)
$(CC) -o madeup.x madeup.o $(OBJ) $(LIB)
ethanol: ethanol.x
ethanol.x: ethanol.o $(OBJ)
$(CC) -o ethanol.x ethanol.o $(OBJ) $(LIB)
air: air.x
air.x: air.o $(OBJ)
$(CC) -o air.x air.o $(OBJ) $(LIB)
galaxy: galaxy.x
galaxy.x: galaxy.o $(OBJ)
$(CC) -o galaxy.x galaxy.o $(OBJ) $(LIB)
clean:
rm -f *.o *.so *.x core *.pyc
rm -f loess.py loess_wrap.c

63
pmsco/loess/setup.py
View File

@@ -1,63 +0,0 @@
#!/usr/bin/env python
"""
@package loess.setup
setup.py file for LOESS
the LOESS code included here was developed at Bell Labs by
William S. Cleveland, Eric Grosse, Ming-Jen Shyu,
and is dated 18 August 1992.
the code is available in the public domain
from http://www.netlib.org/a/dloess.
see the README file for details.
the Python wrapper was set up by M. Muntwiler
with the help of the SWIG toolkit
and other incredible goodies available in the Linux world.
@bug numpy.distutils.build_src in python 2.7 treats all Fortran files with f2py
so that they are compiled via both f2py and swig.
this produces extra object files which cause the linker to fail.
to fix this issue, this module hacks the build_src class.
this hack does not work with python 3. perhaps it's even unnecessary.
@author Matthias Muntwiler
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import numpy
try:
numpy_include = numpy.get_include()
except AttributeError:
numpy_include = numpy.get_numpy_include()
def configuration(parent_package='', top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('loess', parent_package, top_path)
lib = ['blas', 'm', 'f2c']
src = ['loess.c', 'loessc.c', 'predict.c', 'misc.c', 'loessf.f', 'dqrsl.f', 'dsvdc.f', 'fix_main.c', 'loess.i']
inc_dir = [numpy_include]
config.add_extension('_loess',
sources=src,
libraries=lib,
include_dirs=inc_dir
)
return config
def ignore_sources(self, sources, extension):
return sources
if __name__ == '__main__':
try:
from numpy.distutils.core import numpy_cmdclass
numpy_cmdclass['build_src'].f2py_sources = ignore_sources
except ImportError:
pass
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())

33
pmsco/makefile
View File

@@ -1,33 +0,0 @@
SHELL=/bin/sh
# makefile for EDAC, MSC, and MUFPOT programs and modules
#
# see the top-level makefile for additional information.
.PHONY: all clean edac loess msc mufpot
EDAC_DIR = edac
MSC_DIR = msc
MUFPOT_DIR = mufpot
LOESS_DIR = loess
all: edac loess
edac:
$(MAKE) -C $(EDAC_DIR)
loess:
$(MAKE) -C $(LOESS_DIR)
msc:
$(MAKE) -C $(MSC_DIR)
mufpot:
$(MAKE) -C $(MUFPOT_DIR)
clean:
$(MAKE) -C $(EDAC_DIR) clean
$(MAKE) -C $(LOESS_DIR) clean
$(MAKE) -C $(MSC_DIR) clean
$(MAKE) -C $(MUFPOT_DIR) clean
rm -f *.pyc

1
pmsco/msc/.gitignore vendored
View File

@@ -1 +0,0 @@
revision.f

1
pmsco/msc/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'muntwiler_m'

50
pmsco/msc/makefile
View File

@@ -1,50 +0,0 @@
SHELL=/bin/sh
# makefile for MSC program and module
#
# the MSC source code is not included in the public distribution.
# please obtain the MSC code from the original author,
# and copy it to this directory before compilation.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac msc mufpot
FC?=gfortran
FCCOPTS?=
F2PY?=f2py
F2PYOPTS?=
CC?=gcc
CCOPTS?=
SWIG?=swig
SWIGOPTS?=
PYTHON?=python
PYTHONOPTS?=
PYTHONINC?=
all: msc
msc: msc.exe msc.so
msc.exe: msc.f param.f common.f phases.f angles.f revision.f
$(FC) $(FCOPTS) -o msc.exe msc.f phases.f angles.f
#msc.pyf currently needs a manual edit before compiling.
#this target should execute only if it doesn't exist.
msc.pyf: | msc.f phases.f angles.f
$(F2PY) -h msc.pyf -m msc msc.f phases.f angles.f only: mscmain anglesarray anglesfile ps
$(error msc.pyf auto-generated - must be edited manually before build can continue!)
msc.so: msc.f param.f common.f phases.f angles.f revision.f msc.pyf
$(F2PY) -c $(F2PYOPTS) msc.pyf msc.f phases.f angles.f -m msc
revision.f: msc.f
echo " character*50 coderev" > revision.f
echo " parameter(coderev=" >> revision.f
git log --pretty=format:" ='Code revision %h, %ad')" --date=iso -1 $< >> $@ || echo " ='Code revision unknown, "`date +"%F %T %z"`"')" >> $@
clean:
rm -f *.so *.o *.exe
rm -f revision.f

1
pmsco/mufpot/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'muntwiler_m'

46
pmsco/mufpot/makefile
View File

@@ -1,46 +0,0 @@
SHELL=/bin/sh
# makefile for MUFPOT program and module
#
# the MUFPOT source code is not included in the public distribution.
# please obtain the MUFPOT code from the original author,
# and copy it to this directory before compilation.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac msc mufpot
FC=gfortran
FCCOPTS=
F2PY=f2py
F2PYOPTS=
CC=gcc
CCOPTS=
SWIG=swig
SWIGOPTS=
PYTHON=python2
PYTHONOPTS=
all: mufpot
mufpot: mufpot.exe mufpot.so
mufpot.exe: mufpot.f
$(FC) $(FCOPTS) -o mufpot.exe mufpot.f
mufpot.pyf: | mufpot.f
$(F2PY) -h mufpot.pyf -m mufpot mufpot.f only: mufpot
mufpot.so: mufpot.f mufpot.pyf
$(F2PY) -c $(F2PYOPTS) mufpot.pyf mufpot.f -m mufpot
revision.f: msc.f
echo " character*50 coderev" > revision.f
echo " parameter(coderev=" >> revision.f
git log --pretty=format:" ='Code revision %h, %ad')" --date=iso -1 $< >> $@ || echo " ='Code revision unknown, "`date +"%F %T %z"`"')" >> $@
clean:
rm -f *.so *.o *.exe
rm -f revision.f

Some files were not shown because too many files have changed in this diff Show More