pearl-public/pmsco-public
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: pearl-public:rev-distro-1.0.0
Branches Tags
pearl-public:distro
pearl-public:rev-distro-4.2.0 pearl-public:rev-distro-3.0.0 pearl-public:rev-distro-2.2.0 pearl-public:rev-distro-2.1.0 pearl-public:rev-distro-2.0.0 pearl-public:rev-distro-1.0.0
...
compare: pearl-public:rev-distro-4.2.0
Branches Tags
pearl-public:distro
pearl-public:rev-distro-4.2.0 pearl-public:rev-distro-3.0.0 pearl-public:rev-distro-2.2.0 pearl-public:rev-distro-2.1.0 pearl-public:rev-distro-2.0.0 pearl-public:rev-distro-1.0.0

6 Commits
rev-distro ... rev-distro

Author SHA1 Message Date
matthias muntwiler
b64beb694c public release 4.2.0 - see README.md and CHANGES.md for details 2026-01-08 19:10:45 +01:00
matthias muntwiler
ef781e2db4 public release 3.0.0 - see README and CHANGES for details 2021-02-09 12:46:20 +01:00
matthias muntwiler
2b3dbd8bac update README 2020-09-04 16:31:45 +02:00
matthias muntwiler
7c61eb1b41 public release 2.2.0 - see README.md and CHANGES.md for details 2020-09-04 16:22:42 +02:00
matthias muntwiler
fbd2d4fa8c public distro 2.1.0 2019-07-19 12:54:54 +02:00
matthias muntwiler
acea809e4e update public distribution 
based on internal repository c9a2ac8 2019-01-03 16:04:57 +0100
tagged rev-master-2.0.0
 2019-01-31 15:45:02 +01:00
218 changed files with 209211 additions and 146232 deletions
Expand all files Collapse all files

51
.gitea/workflows/build-test.yaml Normal file
View File

@@ -0,0 +1,51 @@
name: build and test the package
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: check runner environment
run: |
uname -a
lsb_release -a
echo "Runner home: $HOME"
- name: check out
uses: actions/checkout@v5
- name: set up compilers
run: |
sudo apt-get update
sudo apt-get -y install binutils build-essential g++ gcc gfortran libblas-dev liblapack-dev openmpi-bin openmpi-common sqlite3
- name: set up python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.9.18"
enable-cache: true
- name: lint with ruff
# configuration is in pyproject.toml
run: |
uvx ruff check --extend-exclude=.venv,build pmsco
- name: install dependencies
run: uv sync --locked --all-extras --dev
- name: tests
run: |
uv run nosetests

45
.gitea/workflows/deploy-pages.yaml Normal file
View File

@@ -0,0 +1,45 @@
name: build and deploy documentation
on:
push:
branches:
- master
jobs:
build-and-deploy:
runs-on: ubuntu-latest
container:
image: gitea.psi.ch/pearl/docs
credentials:
username: ${{ gitea.actor }}
password: ${{ secrets.package_token }}
steps:
- name: checkout
working-directory: /app
run: |
git clone --branch master --single-branch https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git
- name: build
working-directory: /app/pmsco/docs
run: |
export REVISION=$(shell git describe --always --tags --dirty --long || echo "unknown, "`date +"%F %T %z"`)
export OUTDIR=/app/build
doxygen config.dox
- name: configure git
working-directory: /app/pmsco
run: |
git config --global user.name "Gitea Actions"
git config --global user.email "actions@gitea.local"
- name: push to gitea-pages
working-directory: /app/pmsco
run: |
git checkout --orphan gitea-pages
git reset --hard
cp -r /app/build/html/* .
git add .
git commit -m "Deploy documentation to gitea"
git push -f https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git gitea-pages

12
.githooks/install-hooks.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
echo "Setting up Git hooks..."
cd "$(dirname "$0")"
cd ..
# Create symlinks
ln -sf ../../.githooks/pre-commit .git/hooks/pre-commit
ln -sf ../../.githooks/pre-push .git/hooks/pre-push
chmod +x .git/hooks/*
echo "Git hooks installed successfully!"

34
.githooks/pre-commit Executable file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
# .git/hooks/pre-commit
# requires uv
# Track overall status
PASS=true
# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${YELLOW}Running pre-commit checks...${NC}"
PY_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.py$')
# Python checks
if [ -n "$PY_FILES" ]; then
echo -e "${YELLOW}Checking Python files...${NC}"
if ! uvx ruff check --extend-exclude=.*,build*; then
PASS=false
fi
fi
# Final status
if [ "$PASS" = true ]; then
echo -e "${GREEN}All checks passed!${NC}"
exit 0
else
echo -e "${RED}Some checks failed. Please fix issues before committing.${NC}"
exit 1
fi

43
.githooks/pre-push Executable file
View File

@@ -0,0 +1,43 @@
#!/bin/bash
# .git/hooks/pre-push
# Track overall status
PASS=true
# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${YELLOW}Running pre-push checks...${NC}"
PY_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.py$')
# Python checks
echo -e "${YELLOW}Checking Python files...${NC}"
# Lint
if ! uvx ruff check --extend-exclude=.*,build*; then
PASS=false
fi
# Compile
uv sync
# Run different test suites based on changed files
echo -e "${YELLOW}Running Python tests...${NC}"
if ! uv run nosetests; then
echo -e "Tests failed. Push aborted."
PASS=false
fi
# Final status
if [ "$PASS" = true ]; then
echo -e "${GREEN}All checks passed!${NC}"
exit 0
else
echo -e "${RED}Some checks failed. Please fix issues before committing.${NC}"
exit 1
fi

5
.gitignore vendored
View File

@@ -1,6 +1,9 @@
work/*
debug/*
lib/*
dev/*
build/*
__pycache__/*
*.pyc
*.o
*.so
@@ -13,3 +16,5 @@ lib/*
.eric5project/*
.ropeproject/*
.fuse*
.trash
.wraplock

72
CHANGES.md Normal file
View File

@@ -0,0 +1,72 @@
Release 4.2.0 (2026-01-01)
==========================
- Switch to Astral-UV package manager
- CI lint, build, test workflow in gitea
- Automated documentation workflow in gitea
- Recommended Python version 3.12 (compatibility 3.10-3.13)
- Multipole expansion
- Table optimization mode
- Integrate phagen scattering amplitude calculator
- Select modulation and R-factor functions in runfile
- Parametric holo scan generator
- Namespace package installation, support for editable installation
- Simplified command line
- Meson build system
- Differential cross section in periodic table
- Configurable reports
- Path resolution
- Database interface for reports
- Runfile based job scheduling
Release 3.0.0 (2021-02-01)
==========================
- Compatibility with recent conda and singularity versions
- Installation: include plantuml.jar
- Documentation: replace doxypy by doxypypy
- Redefine output_file property
- Documentation of run file interface
- Introduce runfile interface
- Set legacy Fortran for compatibility with recent compiler
- Graphics: fixed color range for modulation functions
- Cluster: build_element accepts symbol or number
- Graphics: swarm plot
- Graphics: genetic chart
- Periodic table: use common binding energies in condensed matter XPS
- Periodic table: reformat bindingenergy.json, add more import/export functions
- Spectrum: add plot cross section function
Release 2.2.0 (2020-09-04)
==========================
- Demo project for arbitrary molecule (cluster file)
- Bugfix: DATA CORRUPTION in phagen translator (emitter mix-up)
- Bugfix: load native cluster file: wrong column order
- Bugfix: initial-state command line option not respected
- Photoionization cross sections and spectrum simulator
- Database: use local lock objects
- Database: create view on results and models
- REFACTORING: Domain -> ModelSpace, Params -> CalculatorParams
- REFACTORING: symmetry -> domain
- Possible conda/mpi4py conflict - changed installation procedure
- Cluster: new calc_scattering_angles function
- Include a periodic table of binding energies of the elements
- Clean up files in the main loop and in the end
- Bugfix: database ingestion overwrites results from previous jobs
- Time out at least 10 minutes before the hard time limit given on the command line
- Cluster: new get_center function
- Bugfix: type error in grid optimizer
- Bugfix: file extension in phagen/makefile
- Dispatch: new algo to distribute processing slots to task levels
- Bugfix: load single-line data files correctly!
- Cluster generator for zincblende crystal
- Phagen translator: fix phase convention (MAJOR), fix single-energy
- Dispatch: give more priority to mid-level tasks in single mode
- Improve performance of cluster add_bulk, add_layer and rotate
- Unit test for Cluster.translate, bugfix in translate and relax
- Fix compatibility with numpy >= 1.14, require numpy >= 1.13
- Database: introduce job-tags
- qpmsco: delete code after execution

201
LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2015-2020 Paul Scherrer Institut
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

4
NOTICE.md
View File

@@ -5,10 +5,10 @@ List of Contributors
Original Author
---------------
Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
Contributors
------------
- Frederik Schirdewahn, <mailto:frederik.schirdewahn@psi.ch>

111
README.md
View File

@@ -1,70 +1,129 @@
Introduction
============
PMSCO stands for PEARL multiple-scattering cluster calculations and structural optimization.
It is a collection of computer programs to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data.
PMSCO (PSI multiple-scattering cluster calculations and structural optimization)
is a Python-based workflow engine to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data using machine learning techniques.
PMSCO was developed at the [Paul Scherrer Institut (PSI)](https://www.psi.ch/)
by the team of the [PEARL beamline](https://www.psi.ch/en/sls/pearl).
The actual scattering calculation is done by code developed by other parties.
PMSCO wraps around that program and facilitates parameter handling, cluster building, structural optimization and parallel processing.
PMSCO wraps around those programs and facilitates parameter handling, cluster building, structural optimization and parallel processing.
In the current version, the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code
developed by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley (1999) is used for scattering calculations.
Other code can be integrated as well.
Instead of EDAC built-in routines, alternatively,
the PHAGEN program from [MsSpec-1.0](https://msspec.cnrs.fr/index.html) can be used to calculate atomic scattering factors.
Highlights
----------
- angle or energy scanned XPD.
- various scanning modes including energy, polar angle, azimuthal angle, analyser angle.
- averaging over multiple symmetries (domains or emitters).
- angle and energy scanned XPD.
- various scanning modes including energy, manipulator angle (polar/azimuthal), emission angle.
- averaging over multiple domains and emitters.
- global optimization of multiple scans.
- structural optimization algorithms: particle swarm optimization, grid search, gradient search.
- structural optimization algorithms: particle swarm optimization, genetic algorithm, grid scan, table scan.
- detailed reports and graphs of result files.
- calculation of the modulation function.
- calculation of the weighted R-factor.
- automatic parallel processing using OpenMPI.
- compatible with Slurm resource manager on Linux cluster machines.
Installation
============
PMSCO is written in Python 2.7.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a cluster with 20-50 available processor cores is recommended.
PMSCO is written in Python. The recommended Python version is 3.12.
Further requirements are the GNU compiler collection, BLAS/LAPACK libraries, OpenMPI and a package manager such as uv, pip or conda.
For optimization jobs, a cluster machine with 20-50 available processor cores is recommended.
Smaller jobs run on any recent Linux workstation.
The code requires about 2 GB of RAM per process.
Detailed installation instructions and dependencies can be found in the documentation
(docs/src/installation.dox).
A [Doxygen](http://www.stack.nl/~dimitri/doxygen/index.html) compiler with Doxypy is required to generate the documentation in HTML or LaTeX format.
The public distribution of PMSCO does not contain the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code.
Please obtain the EDAC source code from the original author, copy it to the pmsco/edac directory, and apply the edac_all.patch patch.
License
=======
The source code of PMSCO is licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
Please read and respect the license agreement.
This _does not include_ the calculation packages contained in the subprojects folder which are licensed separately.
Please share your extensions of the code with the original author.
The gitlab facility can be used to create forks and to submit pull requests.
Attribution notices for your contributions shall be added to the NOTICE.md file.
- Please read and respect the respective license agreements.
- Please acknowledge the use of the code.
- Please consider sharing your developments with the original author.
Due to different copyright terms, the third-party calculation programs are not contained in the public software repository.
These programs may not be used without an explicit agreement by the respective original authors.
Author
------
Authors
-------
Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
- Frederik Schirdewahn, <mailto:frederik.schirdewahn@psi.ch>
Copyright
---------
Copyright 2015-2017 by [Paul Scherrer Institut](http://www.psi.ch)
Copyright 2015-2025 by [Paul Scherrer Institut](http://www.psi.ch)
Release Notes
=============
For a detailed list of changes, see the CHANGES.md file.
4.2.0 (2026-01-01)
------------------
- Recommended Python version 3.12 (compatibility 3.10-3.13)
- Build system and package environment
- Switch to Astral-UV package manager
- Meson build system for Fortran, C and C++ extension modules
- Namespace package installation, support for editable installation
- CI lint, build, test workflow in gitea
- Automated documentation workflow in gitea
- User interface
- Simplified command line, all configuration via runfile and/or project class
- Select modulation and R-factor functions in runfile
- Parametric holo scan generator
- Configurable reports
- Path resolution in runfile
- Database interface for reports
- Runfile based job scheduling
- Calculation features
- Multipole expansion
- Table optimization mode
- Integrate phagen scattering amplitude calculator
- Differential cross section in periodic table
3.0.0 (2021-02-08)
------------------
- Run file interface replaces command line arguments:
- Specify all run-time parameters in a JSON-formatted text file.
- Override any public attribute of the project class.
- Only the name of the run file is needed on the command line.
- The command line interface is still available, some default values and the handling of directory paths have changed.
Check your code for compatibility.
- Integrated job scheduling with the Slurm resource manager:
- Declare all job arguments in the run file and have PMSCO submit the job.
- Graphics scripts for genetic chart and swarm population (experimental feature).
- Update for compatibility with recent Ubuntu (20.04), Anaconda (4.8) and Singularity (3.7).
- Drop compatibility with Python 2.7, minimum requirement is Python 3.6.
2.2.0 (2020-09-04)
------------------
This release breaks existing project code unless the listed refactorings are applied.
- Major refactoring: The 'symmetry' calculation level is renamed to 'domain'.
The previous Domain class is renamed to ModelSpace, Params to CalculatorParams.
The refactorings must be applied to project code as well.
- Included periodic table of elements with electron binding energies and scattering cross-sections.
- Various bug fixes in cluster routines, data file handling, and in the PHAGEN interface.
- Experimental sqlite3 database interface for optimization results.

157
bin/pmsco.ra.template
View File

@@ -1,157 +0,0 @@
#!/bin/bash
#
# Slurm script template for PMSCO calculations on the Ra cluster
# based on run_mpi_HPL_nodes-2.sl by V. Markushin 2016-03-01
#
# Use:
# - enter the appropriate parameters and save as a new file.
# - call the sbatch command to pass the job script.
# request a specific number of nodes and tasks.
# example:
# sbatch --nodes=2 --ntasks-per-node=24 --time=02:00:00 run_pmsco.sl
#
# PMSCO arguments
# copy this template to a new file, and set the arguments
#
# PMSCO_WORK_DIR
# path to be used as working directory.
# contains the script derived from this template.
# receives output and temporary files.
#
# PMSCO_PROJECT_FILE
# python module that declares the project and starts the calculation.
# must include the file path relative to $PMSCO_WORK_DIR.
#
# PMSCO_SOURCE_DIR
# path to the pmsco source directory
# (the directory which contains the bin, lib, pmsco sub-directories)
#
# PMSCO_SCAN_FILES
# list of scan files.
#
# PMSCO_OUT
# name of output file. should not include a path.
#
# all paths are relative to $PMSCO_WORK_DIR or (better) absolute.
#
#
# Further arguments
#
# PMSCO_JOBNAME (required)
# the job name is the base name for output files.
#
# PMSCO_WALLTIME_HR (integer, required)
# wall time limit in hours. must be integer, minimum 1.
# this value is passed to PMSCO.
# it should specify the same amount of wall time as requested from the scheduler.
#
# PMSCO_MODE (optional)
# calculation mode: single, swarm, grid, gradient
#
# PMSCO_CODE (optional)
# calculation code: edac, msc, test
#
# PMSCO_LOGLEVEL (optional)
# request log level: DEBUG, INFO, WARNING, ERROR
# create a log file based on the job name.
#
# PMSCO_PROJECT_ARGS (optional)
# extra arguments that are parsed by the project module.
#
#SBATCH --job-name="_PMSCO_JOBNAME"
#SBATCH --output="_PMSCO_JOBNAME.o.%j"
#SBATCH --error="_PMSCO_JOBNAME.e.%j"
PMSCO_WORK_DIR="_PMSCO_WORK_DIR"
PMSCO_JOBNAME="_PMSCO_JOBNAME"
PMSCO_WALLTIME_HR=_PMSCO_WALLTIME_HR
PMSCO_PROJECT_FILE="_PMSCO_PROJECT_FILE"
PMSCO_MODE="_PMSCO_MODE"
PMSCO_CODE="_PMSCO_CODE"
PMSCO_SOURCE_DIR="_PMSCO_SOURCE_DIR"
PMSCO_SCAN_FILES="_PMSCO_SCAN_FILES"
PMSCO_OUT="_PMSCO_JOBNAME"
PMSCO_LOGLEVEL="_PMSCO_LOGLEVEL"
PMSCO_PROJECT_ARGS="_PMSCO_PROJECT_ARGS"
module load psi-python27/2.4.1
module load gcc/4.8.5
module load openmpi/1.10.2
source activate pmsco
echo '================================================================================'
echo "=== Running $0 at the following time and place:"
date
/bin/hostname
cd $PMSCO_WORK_DIR
pwd
ls -lA
#the intel compiler is currently not compatible with mpi4py. -mm 170131
#echo
#echo '================================================================================'
#echo "=== Setting the environment to use Intel Cluster Studio XE 2016 Update 2 intel/16.2:"
#cmd="source /opt/psi/Programming/intel/16.2/bin/compilervars.sh intel64"
#echo $cmd
#$cmd
echo
echo '================================================================================'
echo "=== The environment is set as following:"
env
echo
echo '================================================================================'
echo "BEGIN test"
echo "=== Intel native mpirun will get the number of nodes and the machinefile from Slurm"
which mpirun
cmd="mpirun /bin/hostname"
echo $cmd
$cmd
echo "END test"
echo
echo '================================================================================'
echo "BEGIN mpirun pmsco"
echo "Intel native mpirun will get the number of nodes and the machinefile from Slurm"
echo
echo "code revision"
cd "$PMSCO_SOURCE_DIR"
git log --pretty=tformat:'%h %ai %d' -1
python -m compileall pmsco
python -m compileall projects
cd "$PMSCO_WORK_DIR"
echo
PMSCO_CMD="python $PMSCO_PROJECT_FILE"
PMSCO_ARGS="$PMSCO_PROJECT_ARGS"
if [ -n "$PMSCO_SCAN_FILES" ]; then
PMSCO_ARGS="-s $PMSCO_SCAN_FILES $PMSCO_ARGS"
fi
if [ -n "$PMSCO_CODE" ]; then
PMSCO_ARGS="-c $PMSCO_CODE $PMSCO_ARGS"
fi
if [ -n "$PMSCO_MODE" ]; then
PMSCO_ARGS="-m $PMSCO_MODE $PMSCO_ARGS"
fi
if [ -n "$PMSCO_OUT" ]; then
PMSCO_ARGS="-o $PMSCO_OUT $PMSCO_ARGS"
fi
if [ "$PMSCO_WALLTIME_HR" -ge 1 ]; then
PMSCO_ARGS="-t $PMSCO_WALLTIME_HR $PMSCO_ARGS"
fi
if [ -n "$PMSCO_LOGLEVEL" ]; then
PMSCO_ARGS="--log-level $PMSCO_LOGLEVEL --log-file $PMSCO_JOBNAME.log $PMSCO_ARGS"
fi
which mpirun
ls -l "$PMSCO_SOURCE_DIR"
ls -l "$PMSCO_PROJECT_FILE"
# Do no use the OpenMPI specific options, like "-x LD_LIBRARY_PATH", with the Intel mpirun.
cmd="mpirun $PMSCO_CMD $PMSCO_ARGS"
echo $cmd
$cmd
echo "END mpirun pmsco"
echo '================================================================================'
date
ls -lAtr
echo '================================================================================'
exit 0

178
bin/pmsco.sge.template
View File

@@ -1,178 +0,0 @@
#!/bin/bash
#
# SGE script template for MSC calculations
#
# This script uses the tight integration of openmpi-1.4.5-gcc-4.6.3 in SGE
# using the parallel environment (PE) "orte".
# This script must be used only with qsub command - do NOT run it as a stand-alone
# shell script because it will start all processes on the local node.
#
# PhD arguments
# copy this template to a new file, and set the arguments
#
# PHD_WORK_DIR
# path to be used as working directory.
# contains the SGE script derived from this template.
# receives output and temporary files.
#
# PHD_PROJECT_FILE
# python module that declares the project and starts the calculation.
# must include the file path relative to $PHD_WORK_DIR.
#
# PHD_SOURCE_DIR
# path to the pmsco source directory
# (the directory which contains the bin, lib, pmsco sub-directories)
#
# PHD_SCAN_FILES
# list of scan files.
#
# PHD_OUT
# name of output file. should not include a path.
#
# all paths are relative to $PHD_WORK_DIR or (better) absolute.
#
#
# Further arguments
#
# PHD_JOBNAME (required)
# the job name is the base name for output files.
#
# PHD_NODES (required)
# number of computing nodes (processes) to allocate for the job.
#
# PHD_WALLTIME_HR (required)
# wall time limit (hours)
#
# PHD_WALLTIME_MIN (required)
# wall time limit (minutes)
#
# PHD_MODE (optional)
# calculation mode: single, swarm, grid, gradient
#
# PHD_CODE (optional)
# calculation code: edac, msc, test
#
# PHD_LOGLEVEL (optional)
# request log level: DEBUG, INFO, WARNING, ERROR
# create a log file based on the job name.
#
# PHD_PROJECT_ARGS (optional)
# extra arguments that are parsed by the project module.
#
PHD_WORK_DIR="_PHD_WORK_DIR"
PHD_JOBNAME="_PHD_JOBNAME"
PHD_NODES=_PHD_NODES
PHD_WALLTIME_HR=_PHD_WALLTIME_HR
PHD_WALLTIME_MIN=_PHD_WALLTIME_MIN
PHD_PROJECT_FILE="_PHD_PROJECT_FILE"
PHD_MODE="_PHD_MODE"
PHD_CODE="_PHD_CODE"
PHD_SOURCE_DIR="_PHD_SOURCE_DIR"
PHD_SCAN_FILES="_PHD_SCAN_FILES"
PHD_OUT="_PHD_JOBNAME"
PHD_LOGLEVEL="_PHD_LOGLEVEL"
PHD_PROJECT_ARGS="_PHD_PROJECT_ARGS"
# Define your job name, parallel environment with the number of slots, and run time:
#$ -cwd
#$ -N _PHD_JOBNAME.job
#$ -pe orte _PHD_NODES
#$ -l ram=2G
#$ -l s_rt=_PHD_WALLTIME_HR:_PHD_WALLTIME_MIN:00
#$ -l h_rt=_PHD_WALLTIME_HR:_PHD_WALLTIME_MIN:30
#$ -V
###################################################
# Fix the SGE environment-handling bug (bash):
source /usr/share/Modules/init/sh
export -n -f module
# Load the environment modules for this job (the order may be important):
module load python/python-2.7.5
module load gcc/gcc-4.6.3
module load mpi/openmpi-1.4.5-gcc-4.6.3
module load blas/blas-20110419-gcc-4.6.3
module load lapack/lapack-3.4.2-gcc-4.6.3
export LD_LIBRARY_PATH=$PHD_SOURCE_DIR/lib/:$LD_LIBRARY_PATH
###################################################
# Set the environment variables:
MPIEXEC=$OPENMPI/bin/mpiexec
# OPENMPI is set by the mpi/openmpi-* module.
export OMP_NUM_THREADS=1
export OMPI_MCA_btl='openib,sm,self'
# export OMPI_MCA_orte_process_binding=core
##############
# BEGIN DEBUG
# Print the SGE environment on master host:
echo "================================================================"
echo "=== SGE job JOB_NAME=$JOB_NAME JOB_ID=$JOB_ID"
echo "================================================================"
echo DATE=`date`
echo HOSTNAME=`hostname`
echo PWD=`pwd`
echo "NSLOTS=$NSLOTS"
echo "PE_HOSTFILE=$PE_HOSTFILE"
cat $PE_HOSTFILE
echo "================================================================"
echo "Running environment:"
env
echo "================================================================"
echo "Loaded environment modules:"
module list 2>&1
echo
# END DEBUG
##############
##############
# Setup
cd "$PHD_SOURCE_DIR"
python -m compileall .
cd "$PHD_WORK_DIR"
ulimit -c 0
###################################################
# The command to run with mpiexec:
CMD="python $PHD_PROJECT_FILE"
ARGS="$PHD_PROJECT_ARGS"
if [ -n "$PHD_SCAN_FILES" ]; then
ARGS="-s $PHD_SCAN_FILES -- $ARGS"
fi
if [ -n "$PHD_CODE" ]; then
ARGS="-c $PHD_CODE $ARGS"
fi
if [ -n "$PHD_MODE" ]; then
ARGS="-m $PHD_MODE $ARGS"
fi
if [ -n "$PHD_OUT" ]; then
ARGS="-o $PHD_OUT $ARGS"
fi
if [ "$PHD_WALLTIME_HR" -ge 1 ]
then
ARGS="-t $PHD_WALLTIME_HR $ARGS"
else
ARGS="-t 0.5 $ARGS"
fi
if [ -n "$PHD_LOGLEVEL" ]; then
ARGS="--log-level $PHD_LOGLEVEL --log-file $PHD_JOBNAME.log $ARGS"
fi
# The MPI command to run:
MPICMD="$MPIEXEC --prefix $OPENMPI -x PATH -x LD_LIBRARY_PATH -x OMP_NUM_THREADS -x OMPI_MCA_btl -np $NSLOTS $CMD $ARGS"
echo "Command to run:"
echo "$MPICMD"
echo
exec $MPICMD
exit 0

145
bin/qpmsco.ra.sh
View File

@@ -1,145 +0,0 @@
#!/bin/sh
#
# submission script for PMSCO calculations on the Ra cluster
if [ $# -lt 1 ]; then
echo "Usage: $0 [NOSUB] JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]"
echo ""
echo " NOSUB (optional): do not submit the script to the queue. default: submit."
echo " JOBNAME (text): name of job. use only alphanumeric characters, no spaces."
echo " NODES (integer): number of computing nodes. (1 node = 24 or 32 processors)."
echo " do not specify more than 2."
echo " TASKS_PER_NODE (integer): 1...24, or 32."
echo " 24 or 32 for full-node allocation."
echo " 1...23 for shared node allocation."
echo " WALLTIME:HOURS (integer): requested wall time."
echo " 1...24 for day partition"
echo " 24...192 for week partition"
echo " 1...192 for shared partition"
echo " PROJECT: python module (file path) that declares the project and starts the calculation."
echo " MODE: PMSCO calculation mode (single|swarm|gradient|grid)."
echo " ARGS (optional): any number of further PMSCO or project arguments (except mode and time)."
echo ""
echo "the job script complete with the program code and input/output data is generated in ~/jobs/\$JOBNAME"
exit 1
fi
# location of the pmsco package is derived from the path of this script
SCRIPTDIR="$(dirname $(readlink -f $0))"
SOURCEDIR="$SCRIPTDIR/.."
PMSCO_SOURCE_DIR="$SOURCEDIR"
# read arguments
if [ "$1" == "NOSUB" ]; then
NOSUB="true"
shift
else
NOSUB="false"
fi
PMSCO_JOBNAME=$1
shift
PMSCO_NODES=$1
PMSCO_TASKS_PER_NODE=$2
PMSCO_TASKS=$(expr $PMSCO_NODES \* $PMSCO_TASKS_PER_NODE)
shift 2
PMSCO_WALLTIME_HR=$1
PMSCO_WALLTIME_MIN=$(expr $PMSCO_WALLTIME_HR \* 60)
shift
# select partition
if [ $PMSCO_WALLTIME_HR -ge 25 ]; then
PMSCO_PARTITION="week"
else
PMSCO_PARTITION="day"
fi
if [ $PMSCO_TASKS_PER_NODE -lt 24 ]; then
PMSCO_PARTITION="shared"
fi
PMSCO_PROJECT_FILE="$(readlink -f $1)"
shift
PMSCO_MODE="$1"
shift
PMSCO_PROJECT_ARGS="$*"
# use defaults, override explicitly in PMSCO_PROJECT_ARGS if necessary
PMSCO_SCAN_FILES=""
PMSCO_LOGLEVEL=""
PMSCO_CODE=""
# set up working directory
cd ~
if [ ! -d "jobs" ]; then
mkdir jobs
fi
cd jobs
if [ ! -d "$PMSCO_JOBNAME" ]; then
mkdir "$PMSCO_JOBNAME"
fi
cd "$PMSCO_JOBNAME"
WORKDIR="$(pwd)"
PMSCO_WORK_DIR="$WORKDIR"
# provide revision information, requires git repository
cd "$SOURCEDIR"
PMSCO_REV=$(git log --pretty=format:"Data revision %h, %ai" -1)
if [ $? -ne 0 ]; then
PMSCO_REV="Data revision unknown, "$(date +"%F %T %z")
fi
cd "$WORKDIR"
echo "$PMSCO_REV" > revision.txt
# generate job script from template
sed -e "s:_PMSCO_WORK_DIR:$PMSCO_WORK_DIR:g" \
-e "s:_PMSCO_JOBNAME:$PMSCO_JOBNAME:g" \
-e "s:_PMSCO_NODES:$PMSCO_NODES:g" \
-e "s:_PMSCO_WALLTIME_HR:$PMSCO_WALLTIME_HR:g" \
-e "s:_PMSCO_PROJECT_FILE:$PMSCO_PROJECT_FILE:g" \
-e "s:_PMSCO_PROJECT_ARGS:$PMSCO_PROJECT_ARGS:g" \
-e "s:_PMSCO_CODE:$PMSCO_CODE:g" \
-e "s:_PMSCO_MODE:$PMSCO_MODE:g" \
-e "s:_PMSCO_SOURCE_DIR:$PMSCO_SOURCE_DIR:g" \
-e "s:_PMSCO_SCAN_FILES:$PMSCO_SCAN_FILES:g" \
-e "s:_PMSCO_LOGLEVEL:$PMSCO_LOGLEVEL:g" \
"$SCRIPTDIR/pmsco.ra.template" > $PMSCO_JOBNAME.job
chmod u+x "$PMSCO_JOBNAME.job"
# request nodes and tasks
#
# The option --ntasks-per-node is meant to be used with the --nodes option.
# (For the --ntasks option, the default is one task per node, use the --cpus-per-task option to change this default.)
#
# sbatch options
# --cores-per-socket=16
# 32 cores per node
# --partition=[shared|day|week]
# --time=8-00:00:00
# override default time limit (2 days in long queue)
# time formats: "minutes", "minutes:seconds", "hours:minutes:seconds", "days-hours", "days-hours:minutes", "days-hours:minutes:seconds"
# --mail-type=ALL
# --test-only
# check script but do not submit
#
SLURM_ARGS="--nodes=$PMSCO_NODES --ntasks-per-node=$PMSCO_TASKS_PER_NODE"
if [ $PMSCO_TASKS_PER_NODE -gt 24 ]; then
SLURM_ARGS="--cores-per-socket=16 $SLURM_ARGS"
fi
SLURM_ARGS="--partition=$PMSCO_PARTITION $SLURM_ARGS"
SLURM_ARGS="--time=$PMSCO_WALLTIME_HR:00:00 $SLURM_ARGS"
CMD="sbatch $SLURM_ARGS $PMSCO_JOBNAME.job"
echo $CMD
if [ "$NOSUB" != "true" ]; then
$CMD
fi
exit 0

128
bin/qpmsco.sge
View File

@@ -1,128 +0,0 @@
#!/bin/sh
#
# submission script for PMSCO calculations on Merlin cluster
#
if [ $# -lt 1 ]; then
echo "Usage: $0 [NOSUB] JOBNAME NODES WALLTIME:HOURS PROJECT MODE [LOG_LEVEL]"
echo ""
echo " NOSUB (optional): do not submit the script to the queue. default: submit."
echo " WALLTIME:HOURS (integer): sets the wall time limits."
echo " soft limit = HOURS:00:00"
echo " hard limit = HOURS:00:30"
echo " for short.q: HOURS = 0 (-> MINUTES=30)"
echo " for all.q: HOURS <= 24"
echo " for long.q: HOURS <= 96"
echo " PROJECT: python module (file path) that declares the project and starts the calculation."
echo " MODE: PMSCO calculation mode (single|swarm|gradient|grid)."
echo " LOG_LEVEL (optional): one of DEBUG, INFO, WARNING, ERROR if log files should be produced."
echo ""
echo "the job script complete with the program code and input/output data is generated in ~/jobs/\$JOBNAME"
exit 1
fi
# location of the pmsco package is derived from the path of this script
SCRIPTDIR="$(dirname $(readlink -f $0))"
SOURCEDIR="$SCRIPTDIR/.."
PHD_SOURCE_DIR="$SOURCEDIR"
PHD_CODE="edac"
# read arguments
if [ "$1" == "NOSUB" ]; then
NOSUB="true"
shift
else
NOSUB="false"
fi
PHD_JOBNAME=$1
shift
PHD_NODES=$1
shift
PHD_WALLTIME_HR=$1
PHD_WALLTIME_MIN=0
shift
PHD_PROJECT_FILE="$(readlink -f $1)"
PHD_PROJECT_ARGS=""
shift
PHD_MODE="$1"
shift
PHD_LOGLEVEL=""
if [ "$1" == "DEBUG" ] || [ "$1" == "INFO" ] || [ "$1" == "WARNING" ] || [ "$1" == "ERROR" ]; then
PHD_LOGLEVEL="$1"
shift
fi
# ignore remaining arguments
PHD_SCAN_FILES=""
# select allowed queues
QUEUE=short.q,all.q,long.q
# for short queue (limit 30 minutes)
if [ "$PHD_WALLTIME_HR" -lt 1 ]; then
PHD_WALLTIME_HR=0
PHD_WALLTIME_MIN=30
fi
# set up working directory
cd ~
if [ ! -d "jobs" ]; then
mkdir jobs
fi
cd jobs
if [ ! -d "$PHD_JOBNAME" ]; then
mkdir "$PHD_JOBNAME"
fi
cd "$PHD_JOBNAME"
WORKDIR="$(pwd)"
PHD_WORK_DIR="$WORKDIR"
# provide revision information, requires git repository
cd "$SOURCEDIR"
PHD_REV=$(git log --pretty=format:"Data revision %h, %ad" --date=iso -1)
if [ $? -ne 0 ]; then
PHD_REV="Data revision unknown, "$(date +"%F %T %z")
fi
cd "$WORKDIR"
echo "$PHD_REV" > revision.txt
# generate job script from template
sed -e "s:_PHD_WORK_DIR:$PHD_WORK_DIR:g" \
-e "s:_PHD_JOBNAME:$PHD_JOBNAME:g" \
-e "s:_PHD_NODES:$PHD_NODES:g" \
-e "s:_PHD_WALLTIME_HR:$PHD_WALLTIME_HR:g" \
-e "s:_PHD_WALLTIME_MIN:$PHD_WALLTIME_MIN:g" \
-e "s:_PHD_PROJECT_FILE:$PHD_PROJECT_FILE:g" \
-e "s:_PHD_PROJECT_ARGS:$PHD_PROJECT_ARGS:g" \
-e "s:_PHD_CODE:$PHD_CODE:g" \
-e "s:_PHD_MODE:$PHD_MODE:g" \
-e "s:_PHD_SOURCE_DIR:$PHD_SOURCE_DIR:g" \
-e "s:_PHD_SCAN_FILES:$PHD_SCAN_FILES:g" \
-e "s:_PHD_LOGLEVEL:$PHD_LOGLEVEL:g" \
"$SCRIPTDIR/pmsco.sge.template" > $PHD_JOBNAME.job
chmod u+x "$PHD_JOBNAME.job"
if [ "$NOSUB" != "true" ]; then
# suppress bash error [stackoverflow.com/questions/10496758]
unset module
# submit the job script
# EMAIL must be defined in the environment
if [ -n "$EMAIL" ]; then
qsub -q $QUEUE -m ae -M $EMAIL $PHD_JOBNAME.job
else
qsub -q $QUEUE $PHD_JOBNAME.job
fi
fi
exit 0

644
docs/config.dox
View File

File diff suppressed because it is too large Load Diff

24
docs/makefile
View File

@@ -2,25 +2,33 @@ SHELL=/bin/sh
# makefile for PMSCO documentation
#
# requirements
#
# 1) doxygen
# 2) /usr/bin/doxypy
# 3) PLANTUML_JAR_PATH environment variable must point to plantUML jar.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .html
.PHONY: all docs clean
.PHONY: all docs html clean
DOX=doxygen
DOXOPTS=
LATEX_DIR=latex
all: docs
REVISION=$(shell git describe --always --tags --dirty --long || echo "unknown, "`date +"%F %T %z"`)
export REVISION
OUTDIR=
export OUTDIR
docs: doxygen pdf
all: html
docs: html
doxygen:
$(DOX) $(DOXOPTS) config.dox
pdf: doxygen
-$(MAKE) -C $(LATEX_DIR)
html: doxygen
clean:
-rm -rf latex/*
-rm -rf html/*
-rm -r html/*

2
docs/py_filter.sh Executable file
View File

@@ -0,0 +1,2 @@
#!/bin/bash
python -m doxypypy.doxypypy -a -c $1

28
docs/readme.md Normal file
View File

@@ -0,0 +1,28 @@
To compile the source code documentation in HTML format on Ubuntu, follow the instructions below.
~~~~~~{.sh}
apt-get update
apt-get install -y --no-install-recommends \
default-jre \
doxygen \
gawk \
git \
graphviz \
pandoc \
wget
pip install --no-cache-dir \
doxypypy \
meson \
meson-python \
ninja \
pynose
wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
export PLANTUML_JAR_PATH=/app/plantuml.jar
cd pmsco/docs
doxygen config.dox
~~~~~~
Open `pmsco/docs/html/index.html` in your browser.

7
docs/readme.txt
View File

@@ -1,7 +0,0 @@
to compile the source code documentation, you need the following packages (naming according to Debian):
doxygen
doxygen-gui (optional)
doxypy
graphviz
latex (optional)

162
docs/src/commandline.dox
View File

@@ -1,144 +1,48 @@
/*! @page pag_command Command Line
\section sec_command Command Line
This section describes the command line arguments for a direct call of PMSCO from the shell.
For batch job submission to Slurm see @ref sec_slurm.
Assuming that PMSCO has been installed in the active Python environment (@ref pag_install),
the basic command line of PMSCO is as follows:
Since PMSCO is started indirectly by a call of the specific project module,
the syntax of the command line arguments is defined by the project module.
However, to reduce the amount of custom code and documentation and to avoid confusion
it is recommended to adhere to the standard syntax described below.
~~~~~~{.sh}
[mpiexec -np NPROCESSES] python -m pmsco [options]
~~~~~~
The basic command line is as follows:
@code{.sh}
[mpiexec -np NPROCESSES] python path-to-project.py [common args] [project args]
@endcode
The first portion between square brackets is necessary for parallel execution using MPI.
Replace `NPROCESSES` by the number of processes.
Include the first portion between square brackets if you want to run parallel processes.
Specify the number of processes as the @c -np option.
@c path-to-project.py should be the path and name to your project module.
Common args and project args are described below.
The PMSCO main program has a limited number of `common arguments` that are described below.
Usually, all parameters should be declared in a @ref pag_runfile so that they can be archived with the results.
However, in some cases it may be necessary to override some common parameters, e.g. the job name, on the command line.
\subsection sec_common_args Common Arguments
\subsection sec_command_common Common Arguments
All common arguments are optional and default to more or less reasonable values if omitted.
They can be added to the command line in arbitrary order.
All common arguments can also be set in the project code or the run-file (recommended).
In that case, only the run-file is specified on the command line.
However, there are a number of options that override settings from the run-file.
The arguments can appear in arbitrary order.
The following table is ordered by importance.
| Option | Values | Description |
| --- | --- | --- |
| -h , --help | | Display a command line summary and exit. |
| -m , --mode | single (default), grid, swarm | Operation mode. |
| -d, --data-dir | file system path | Directory path for experimental data files (if required by project). Default: current working directory. |
| -o, --output-file | file system path | Base path and/or name for intermediate and output files. Default: pmsco_data |
| -t, --time-limit | decimal number | Wall time limit in hours. The optimizers try to finish before the limit. Default: 24.0. |
| -k, --keep-files | list of file categories | Output file categories to keep after the calculation. Multiple values can be specified and must be separated by spaces. By default, cluster and model (simulated data) of a limited number of best models are kept. See @ref sec_file_categories below. |
| --log-level | DEBUG, INFO, WARNING (default), ERROR, CRITICAL | Minimum level of messages that should be added to the log. |
| --log-file | file system path | Name of the main log file. Under MPI, the rank of the process is inserted before the extension. Default: output-file + log, or pmsco.log. |
| --log-disable | | Disable logging. By default, logging is on. |
| --pop-size | integer | Population size (number of particles) in swarm optimization mode. The default value is the greater of 4 or two times the number of calculation processes. |
| -c, --code | edac (default) | Scattering code. At the moment, only edac is supported. |
| Option | Values | Description | Run File |
| --- | --- | --- | --- |
| -r, --run-file | file path | JSON-formatted configuration file that defines run-time parameters. The format and content of a run file is described in a section @ref pag_runfile. | no |
| -o, --output-dir | file path | Base path and/or name for intermediate and output files. | see note below |
| -j , --job-name | string | Job name | job-name |
| -m, --module | file path | Project module | __module__ |
| -c, --project-class | string | Project class | __class__ |
| -h, --help | | Display a command line summary and exit. | no |
The job name is used as a prefix of output file names.
It is also registered in the `jobs` table of the results database (if used),
and it is used to identify the job with a job scheduling system.
\subsubsection sec_file_categories File Categories
\note It is important that the job name be unique within a project.
Specifically, you need to *provide a new job name each time you start pmsco*, otherwise the job may fail.
It may be more natural to specify the job name on the command line using the `-j` argument
than to change the run file every time.
Unfortunately, PMSCO cannot auto-generate, auto-increment or verify the job name.
The following category names can be used with the @c --keep-files option.
Multiple names can be specified and must be separated by spaces.
| Category | Description | Default Action |
| --- | --- | --- |
| input | raw input files for calculator, including cluster and phase files in custom format | delete |
| output | raw output files from calculator | delete |
| phase | phase files in portable format for report | delete |
| cluster | cluster files in portable XYZ format for report | keep |
| debug | debug files | delete |
| model | output files in ETPAI format: complete simulation (a_-1_-1_-1_-1) | keep |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | delete |
| symmetry | output files in ETPAI format: symmetry (a_b_c_-1_-1) | delete |
| emitter | output files in ETPAI format: emitter (a_b_c_d_-1) | delete |
| region | output files in ETPAI format: region (a_b_c_d_e) | delete |
| report| final report of results | keep |
| population | final state of particle population | keep |
| rfac | files related to models which give bad r-factors | delete |
\subsection sec_project_args Project Arguments
The following table lists a few recommended options that are handled by the project code.
Project options that are not listed here should use the long form to avoid conflicts in future versions.
| Option | Values | Description |
| --- | --- | --- |
| -s, --scans | project-dependent | Nick names of scans to use in calculation. The nick name selects the experimental data file and the initial state of the photoelectron. Multiple values can be specified and must be separated by spaces. |
\subsection sec_scanfile Experimental Scan Files
The recommended way of specifying experimental scan files is using nick names (dictionary keys) and the @c --scans option.
A dictionary in the module code defines the corresponding file name, chemical species of the emitter and initial state of the photoelectron.
The location of the files is selected using the common @c --data-dir option.
This way, the file names and photoelectron parameters are versioned with the code,
whereas command line arguments may easily get forgotten in the records.
\subsection sec_project_example Example Argument Handling
An example for handling the command line in a project module can be found in the twoatom.py demo project.
The following code snippet shows how the common and project arguments are separated and handled.
@code{.py}
def main():
# have the pmsco module parse the common arguments.
args, unknown_args = pmsco.pmsco.parse_cli()
# pass any arguments not handled by pmsco
# to the project-defined parse_project_args function.
# unknown_args can be passed to argparse.ArgumentParser.parse_args().
if unknown_args:
project_args = parse_project_args(unknown_args)
else:
project_args = None
# create the project object
project = create_project()
# apply the common arguments on the project
pmsco.pmsco.set_common_args(project, args)
# apply the specific arguments on the project
set_project_args(project, project_args)
# run the project
pmsco.pmsco.run_project(project)
@endcode
\section sec_slurm Slurm Job Submission
The command line of the Slurm job submission script for the Ra cluster at PSI is as follows.
This script is specific to the configuration of the Ra cluster but may be adapted to other Slurm-based queues.
@code{.sh}
qpmsco.sh [NOSUB] JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]
@endcode
Here, the first few arguments are positional and their order must be strictly adhered to.
After the positional arguments, optional arguments of the PMSCO project command line can be added in arbitrary order.
If you execute the script without arguments, it displays a short summary.
The job script is written to @c ~/jobs/\$JOBNAME.
| Argument | Values | Description |
| --- | --- | --- |
| NOSUB (optional) | NOSUB or omitted | If NOSUB is present as the first argument, create the job script but do not submit it to the queue. Otherwise, submit the job script. |
| JOBNAME | text | Name of job. Use only alphanumeric characters, no spaces. |
| NODES | integer | Number of computing nodes. (1 node = 24 or 32 processors). Do not specify more than 2. |
| TASKS_PER_NODE | 1...24, or 32 | Number of processes per node. 24 or 32 for full-node allocation. 1...23 for shared node allocation. |
| WALLTIME:HOURS | integer | Requested wall time. 1...24 for day partition, 24...192 for week partition, 1...192 for shared partition. This value is also passed on to PMSCO as the @c --time-limit argument. |
| PROJECT | file system path | Python module (file path) that declares the project and starts the calculation. |
| MODE | single, swarm, grid | PMSCO operation mode. This value is passed on to PMSCO as the @c --mode argument. |
| ARGS (optional) | | Any further arguments are passed on verbatim to PMSCO. You don't need to specify the mode and time limit here. |
*/
*/

114
docs/src/concepts-atomscat.dox Normal file
View File

@@ -0,0 +1,114 @@
/*! @page pag_concepts_atomscat Atomic scattering
\section sec_atomscat Atomic scattering
\subsection sec_atomscat_intro Introduction
The process of calculating atomic scattering factors (phase shifts) can be customized in several ways.
1. Internal processing.
Some multiple scattering programs, like EDAC, contain a built-in facility to calculate phase shifts.
This is the most simple and default behaviour.
2. Automatic calculation in a separate program.
PMSCO has an interface to run the PHAGEN program from
the [MsSpec-1.0 package](https://ipr.univ-rennes1.fr/msspec) to calculate scattering factors.
Note that the PHAGEN code is not included in the public distribution of PMSCO.
3. Manual calculation.
Scattering files created manually using an external program can be used by providing the file names.
The files must have the format required by the multiple scattering code,
and they must be linked to the corresponding atoms of the cluster.
In the case of automatic calculation, the project code can optionally hook into the process
and modify clusters before and after scattering factors are calculated.
For instance, it may provide an extended cluster in order to reduce boundary effects,
or it may modify the assignment of scattering files to cluster atoms
so that the scattering factors of selected atom classes are used
(cf. section \ref sec_atomscat_atomclass).
\subsection sec_atomscat_usage Usage
\subsubsection sec_atomscat_internal Internal processing
This is the default behaviour selected in the inherited pmsco.project.Project class.
Make sure not to override the `atomic_scattering_factory` attribute.
Its default value is pmsco.calculators.calculator.InternalAtomicCalculator.
\subsubsection sec_atomscat_external Automatic calculation in a separate program
To select the atomic scattering calculator,
assign its interface class to the project's `atomic_scattering_factory` attribute.
For example, to use PHAGEN, add the following code to your project's `__init__` constructor:
@code{.py}
from pmsco.calculators.phagen import PhagenCalculator
self.atomic_scattering_factory = PhagenCalculator
@endcode
\subsubsection sec_atomscat_manual Manual calculation
If you want to keep the scattering factors constant during an optimization,
you should run PMSCO in _single_ mode and provide the model parameters and cluster
that will return the desired scattering files.
In the `create_params` method of your project,
you should then set the `phase_files` attribute,
which is a dictionary that maps atom classes to the names of the scattering files.
Unless you set specific values in the cluster object, the atom class defaults to the element number.
The file names should include a path relative to the working directory.
\subsection sec_atomscat_implement Implementation
\subsubsection sec_atomscat_atomclass Atom classes
Atomic scattering programs classify atoms based on chemical element, charge state and symmetry of the local environment.
This means that two atoms of the same chemical element may have different scattering factors.
For example, if you have EDAC output the cluster after calculation of the muffin tin potential,
you will find that the chemical element number has been replaced by an arbitrary integer.
By default, PMSCO will do the linking of atom classes and scattering files transparently.
However, if you want to reduce the number of atom classes,
or if you have the scattering factors calculated on a reference cluster,
you will have to provide project code to do the assignment.
This is described further below.
\subsubsection sec_atomscat_calculator Atomic scattering calculator
The project selects the atomic scattering calculation mode by specifying its `atomic_scattering_factory` attributed.
This is the name of a class that inherits from @ref pmsco.calculators.calculator.AtomicCalculator.
The following calculators are currently implemented:
| Class | Description |
| --- | --- |
| pmsco.calculators.calculator.InternalAtomicCalculator | Calculate the atomic scattering factors in the multiple-scattering program. |
| pmsco.calculators.phagen.PhagenCalculator | Calculate the atomic scattering factors in the PHAGEN program. |
An atomic calculator class essentially defines a `run` method that operates on a cluster and scattering parameters object.
It generates the necessary scattering files, updates the cluster with the new atom classes
and updates the parameters with the file names of the scattering files.
Note that the scattering files have to be in the correct format for the multiple scattering calculator.
\subsubsection sec_atomscat_hooks Project hooks
Before and after calculation of the scattering factors,
the project's `before_atomic_scattering` and `after_atomic_scattering` methods are called
with the cluster and input parameters.
The _before_ method provides the cluster to be used for atomic scattering calculations.
It may,
1. just return the original cluster,
2. modify the provided cluster to include additional atoms or modify the charge state of the emitter,
3. create a completely different cluster,
4. return None to suppress the atomic scattering calculation.
The method is called once at the beginning of the PMSCO job with model -1,
where it may return the global reference cluster.
Later on it is called once for each calculation task with the specific task index.
Similarly, the _after_ method collects the results and updates the `phase_files` dictionary of the input parameters.
It is free to consolidate atom classes and remove unwanted atoms.
However, it must make sure that for each atom class in the cluster,
there is a corresponding link to a scattering file.
*/

185
docs/src/concepts-emitter.dox Normal file
View File

@@ -0,0 +1,185 @@
/*! @page pag_concepts_emitter Emitter configurations
\section sec_emitters Emitter configurations
\subsection sec_emit_intro Introduction
Since emitters contribute incoherently to the diffraction pattern,
it should make no difference how the emitters are grouped and calculated.
This fact can be used to distribute a calculation over multiple parallel processes
if each process calculates the diffraction pattern coming from one particular emitter atom.
In effect, some calculation codes are implemented for a single emitter per calculation.
With PMSCO, it is easy to distribute the emitters over parallel processes.
The project just declares the number of emitters and returns one specific cluster per emitter.
In the simplest case, this means that the emitter attribute of the cluster atoms is set differently,
while the atomic coordinates are the same for all clusters generated.
PMSCO takes care of dispatching the clusters to multiple calculation processes
depending on the number of allocated MPI processes
as well as summing up the resulting diffraction patterns.
In addition, the emitter framework also supports that clusters are tailored to a specific emitter configuration.
Suppose that the unit cell contains a large number of inequivalent emitters.
If all emitters had to be included in a single calculation,
the cluster would grow very large and the calculation would include many long scattering paths
that effectively did not contribute intensity to the final result.
Splitting a large cluster into small ones built locally around one emitter
can provide a significant performance gain in complex systems.
Note that the emitter framework does not require that an emitter _configuration_ contains only one emitter _atom_.
It is up to the project to define how many emitter configurations there are and what they encompass.
This should, however, normally not be necessary.
To avoid confusion, it is recommended to declare exactly one emitter atom per configuration.
\subsection sec_emit_implement Implementation
There are several implementation routes with varying complexity.
Which route to take can depend on the complexity of the system and/or the programming skills of the user.
The following class diagram illustrates the classes and packages involved in cluster generation.
@startuml "class diagram for cluster generation"
package pmsco {
class Project {
cluster_generator
export_cluster()
}
abstract class ClusterGenerator {
project
{abstract} count_emitters()
{abstract} create_cluster()
}
class LegacyClusterGenerator {
project
count_emitters()
create_cluster()
}
}
package "user project" {
class UserClusterGenerator {
project
count_emitters()
create_cluster()
}
note bottom : for complex cluster
class UserProject {
count_emitters()
create_cluster()
}
note bottom : for simple cluster
}
Project <|-- UserProject
ClusterGenerator <|-- LegacyClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
Project *-- ClusterGenerator
UserProject .> LegacyClusterGenerator
UserProject .> UserClusterGenerator
@enduml
In general, the cluster is generated by calls to the project's cluster_generator object.
This can be either a custom generator class derived from pmsco.cluster.ClusterGenerator
or the default pmsco.cluster.LegacyClusterGenerator which calls the UserProject.
For simple clusters, it may be sufficient to implement the cluster directly in the user project class
(UserProject in the diagram).
For more complex systems, it is recommended to implement a custom cluster generator class
(UserClusterGenerator).
\subsubsection sec_emit_implement_legacy Static cluster implemented in project methods
This is the most simple route as it requires the implementation of one or two methods of the user project class.
It can be used for single-emitter and multi-emitter problems.
This implementation is active while a pmsco.cluster.LegacyClusterGenerator
is assigned to the project's cluster_generator attribute.
1. Implement a count_emitters method in your project class
if the project uses more than one emitter configurations.
It must have same method contract as pmsco.cluster.ClusterGenerator.count_emitters.
Specifically, it must return the number of emitter configurations of a given model, scan and domain.
If there is only one configuration, the method does not need to be implemented.
2. Implement a create_cluster method in your project class.
It must have same method contract as pmsco.cluster.ClusterGenerator.create_cluster.
Specifically, it must return a cluster.Cluster object for the given model, scan, domain and emitter configuration.
The emitter atoms must be marked according to the emitter configuration specified by the index argument.
Note that, depending on the index.emit argument, all emitter atoms must be marked
or only the ones of the corresponding emitter configuration.
3. (Optionally) override the pmsco.project.Project.combine_emitters method
if the emitters should be added with non-uniform weights.
Although it's possible to produce emitter-dependent clusters using this approach,
this is usually not recommended.
Rather, the generator approach described below should be followed in this case.
\subsubsection sec_emit_implement_generator Static cluster implemented by generator class
The preferred way of creating clusters is to implement a _generator_ class
because it is the most scalable way from simple to complex systems.
In addition, one cluster generator class can be quickly exchanged for another
if there are multiple possibilities.
1. Implement a cluster generator class which inherits from pmsco.cluster.ClusterGenerator
in your project module.
2. Implement the create_cluster and count_emitters methods of the generator.
The method contracts are the same as the ones described in the previous paragraph,
just in the context of a separate class.
3. Initialize an instance of the generator and assign it to the project.cluster_generator attribute
in the initialization of your project.
\subsubsection sec_emit_implement_local Local clusters implemented by generator class
The basic method contract outlined in the previous paragraph is equally applicable to the case
where a local cluster is generated for each emitter configuration.
Again, the generator class with the two methods (count_emitters and create_cluster) is the minimum requirement.
However, for ease of code maintenance and/or for improved performance of large clusters,
some internal structure may be helpful.
Suppose that the system consists of a large supercell containing many emitters
and that a small cluster shall be built for each emitter configuration.
During the calculations, the generator will receive several calls to the count_emitters and create_cluster methods.
Every time the model and index are the same, the functions must return the same result.
Thus, most importantly, the implementation must make sure that the results are fully deterministic.
Second, depending on the complexity, it could be more efficient to cache a cluster for later use.
One way to reduce the complexity is to introduce a _master cluster_
from which the emitter configurations and individual clusters are derived.
1. Implement a master_cluster method with the same arguments and result types as create_cluster.
The method returns a full cluster of the supercell and its neighbouring cells.
All inequivalent emitters are marked (which determines the number of emitter configurations).
2. Decorate the master_cluster with pmsco.dispatch.CachedCalculationMethod.
This pre-defined decorator transparently caches the cluster
so that subsequent calls with the same arguments do not re-create the cluster but return the cached one.
3. The count_emitters method can simply return the emitter count of the master cluster.
4. The create_cluster method calls master_cluster() and extracts the region
corresponding to the requested emitter configuration.
\subsection sec_emit_report Reporting
The pmsco.project.Project class implements a method that saves a cluster to two XYZ files,
one containing the coordinates of all atoms
and one containing only the coordinates of the emitters.
The method is called for each cluster that is passed to the calculator, i.e., each emitter index.
You may override the method in your project to alter the reporting.
*/

3
docs/src/concepts-model.dox Normal file
View File

@@ -0,0 +1,3 @@
/*! @page pag_concepts_model Model
*/

3
docs/src/concepts-region.dox Normal file
View File

@@ -0,0 +1,3 @@
/*! @page pag_concepts_region Region
*/

31
docs/src/concepts-scan.dox Normal file
View File

@@ -0,0 +1,31 @@
/*! @page pag_concepts_scan Scans
\section sec_scanning Scanning
PMSCO with EDAC currently supports the following scan axes.
- kinetic energy E
- polar angle theta T
- azimuthal angle phi P
- analyser angle alpha A
The following combinations of these scan axes are allowed (see pmsco.data.SCANTYPES).
- E
- E-T
- E-A
- T-P (hemispherical or hologram scan)
@attention The T and A axes cannot be combined.
If a scan of one of them is specified, the other is assumed to be fixed at zero!
This assumption may change in the future,
so it is best to explicitly set the fixed angle to zero in the scan file.
@remark According to the measurement geometry at PEARL,
alpha scans are implemented in EDAC as theta scans at phi = 90 in fixed cluster mode.
The switch to fixed cluster mode is made by PMSCO internally,
no change of angles or other parameters is necessary in the scan or project files
besides filling the alpha instead of the theta column.
*/

32
docs/src/concepts-symmetry.dox Normal file
View File

@@ -0,0 +1,32 @@
/*! @page pag_concepts_domain Domain
\section sec_domain Domain Averaging
A _domain_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is derived from the same set of model parameters
and that contributes incoherently to the measured diffraction pattern.
A domain may be represented by special domain parameters that are not subject to optimization.
For instance, a real sample may have rotational domains that are not present in the cluster,
changing the symmetry from three-fold to six-fold.
Or, an adsorbate may be present in a number of different lateral configurations on the substrate.
In the first case, it may be sufficient to fold calculated data in the proper way to generate the same symmetry as in the measurement.
In the latter case, it may be necessary to execute a scattering calculation for each possible orientation or a representative number of possible orientations.
PMSCO provides the basic framework to spawn multiple calculations according to the number of domains (cf. \ref sec_tasks).
The actual data reduction from multiple domain to one measurement needs to be implemented on the project level.
This section explains the necessary steps.
1. Your project needs to populate the pmsco.project.Project.domains list.
For each domain, add a dictionary of domain parameters, e.g. <code>{'angle_azi': 15.0}</code>.
At least one domain must be declared in a project, otherwise no calculation is executed.
2. The project may use the domain index of a task to build the cluster and parameter file as necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular domain in addition to the model parameters.
3. The project combines the results of the calculations for the various domains into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with customizable weight.
It uses the special model parameters `wdom1`, `wdom2`, ... (if defined, default 1) to weight each domain.
If you need more control, override the pmsco.project.Project.combine_domains method and implement your own algorithm.
*/

306
docs/src/concepts-tasks.dox Normal file
View File

@@ -0,0 +1,306 @@
/*! @page pag_concepts_tasks Task concept
\section sec_tasks Calculation tasks
A _calculation task_ defines a concrete set of model parameters, atomic coordinates, emitter configuration,
experimental reference and meta-data (such as file names)
that completely defines how to produce the input data for the scattering program (the _calculator_).
For each task, the calculator is executed once and produces one result dataset.
In a typical optimization project, however, the calculator is executed multiple times for various reasons
mandated by the project but also efficient calculations in a multi-process environment:
1. The calculation must be repeated under variation of parameters.
A concrete set of parameters is called @ref sec_task_model.
2. The sample was measured multiple times or under different conditions (initial states, photon energy, emission angle).
Each contiguous measured dataset is called a @ref sec_task_scan.
3. The measurement averages over multiple inequivalent domains, cf. @ref sec_task_domain.
4. The measurement includes multiple geometrically inequivalent emitters, cf. @ref sec_task_emitter.
5. The calculation should be distributed over multiple processes that run in parallel to reduce the wall time, cf. @ref sec_task_region.
In PMSCO, these aspects are modelled as attributes of a calculation task
as shown schematically in the following diagram.
@startuml "attributes of a calculation task"
class CalculationTask {
model
scan
domain
emitter
region
..
files
}
class Model {
index
..
dlat
dAS
dS1S2
V0
Zsurf
Texp
rmax
}
class Scan {
index
..
filename
mode
initial_state
energies
thetas
phis
alphas
}
class Domain {
index
..
rotation
registry
}
class Emitter {
index
}
class Region {
index
..
range
}
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Domain
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
domains
model_handler
cluster_generator
}
class ClusterGenerator {
count_emitters()
create_cluster()
}
class ModelHandler {
create_tasks()
add_result()
}
Model ..> ModelHandler
Scan ..> Project
Domain ..> Project
Emitter ..> ClusterGenerator
Region ..> Project
Project *-left- ModelHandler
Project *- ClusterGenerator
hide empty members
@enduml
Although the attributes may have quite different types (as detailed below),
each instance is also given a unique (per attribute) integer index,
where -1 means that the attribute is undefined.
The indices of the five attributes together (pmsco.dispatch.CalcID tuple)
serve internally to identify a task and the data belonging it.
The identifier appears, for instance, in input and output file names.
Normally, data files are deleted after the calculation, and only a few top-level files are kept
(can be overridden at the command line or in the project code).
At the top level, only the model ID is set, the other ones are undefined (-1).
\subsection sec_task_model Model
The _model_ attribute is a dictionary of continuously variable parameters of the system such as lattice constants, relaxation constants, rotation angles, etc.
It may also define non-structural or non-physical parameters such as temperature, inner potential or cluster radius.
The dictionary contains key-value pairs where the keys are up to the user project (the figure shows some examples).
The values are floating-point numbers that are chosen by the model handler within the domain specified by the user project.
Models are generated by the chosen optimizer according to a particular algorithm or, in single mode, directly by the project.
Each specific instance of model parameters is given a unique index that allows to identify related input and output files.
Model parameters are reported with the corresponding R-factors during the optimization process.
\subsection sec_task_scan Scan
The _scan_ attribute is an index into the list of scans defined by the user project.
Each scan refers to one experimental data file and, thus, defines the initial and final states of the photoelectron.
PMSCO runs a separate calculation for each scan file and compares the combined results to the experimental data.
This is sometimes called a _global fit_.
\subsection sec_task_domain Domain
A _domain_ is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is independent of the _model_ and contributes incoherently to the measured diffraction pattern.
For instance, for a system that includes two inequivalent structural domains,
two separate clusters have to be generated and calculated for each model.
The domain parameter is not subject to optimization.
However, if the branching ratio is unknown a priori, a model parameter can be introduced
to control the relative contribution of a particular domain to the diffraction pattern.
The basic @ref pmsco.project.Project.combine_domains method reads the special model parameters `wdom1`, `wdom2`, etc. to weight the individual domains.
A domain is identified by its index which is an index into the project's domains table (pmsco.project.Project.domains).
It is up to the user project to give a physical description of the domain, e.g. a rotation angle,
by assigning a meaningful value (e.g. a dictionary with key-value pairs) to the domains table.
The cluster generator can then read the value from the table rather than from constants in the code.
The figure shows two examples of domain parameters.
The corresponding domains table could be set up like this:
@code{.py}
project.add_domain({'rotation': 0.0, 'registry': 0.0})
project.add_domain({'rotation': 30.0, 'registry': 0.0})
@endcode
\subsection sec_task_emitter Emitter
The _emitter_ component of the calculation task selects a specific emitter configuration of the cluster generator.
This is merely an index whose interpretation is up to the cluster generator.
The default emitter handler enumerates the emitter index from 1 to the emitter count reported by the cluster generator.
The emitter count and list of emitters may depend on model, scan and domain.
The cluster generator can tailor a cluster to the given model, scan, domain and emitter index.
For example, in a large unit cell with many inequivalent emitters,
the generator might return a small sub-cluster around the actual emitter for better calculation performance
since the distant atoms of the unit cell do not contribute to the diffraction pattern.
Emitter branching must be requested specifically by using a particular pattern in the code.
By default, it is disabled, which allows the cluster code to be written in a slightly easier way.
\subsection sec_task_region Region
The _region_ handler may split a scan region into several smaller chunks
so that the tasks can be distributed to multiple processes.
Chunking by energy regions is enabled automatically if the project contains an energy scan of at least 10 points
and the project is run in multiple processes.
It can be disabled by the user project.
\section sec_task_handler Task handlers
The previous section described the five important attributes of a calculation task.
These attributes span a five-dimensional index space
where each point maps to one task and, consequently, one calculation and one result dataset.
To populate the index space, however, calculation tasks are more adequately arranged in a tree-like hierarchy with five levels.
The code that defines attributes and processes results can then be separated into _handlers_.
Each level calls for a particular functional contract of the handler.
According to object-oriented principles the contracts at the five levels are defined by abstract base classes
which can be sub-classed for more specific behaviour.
For instance, the class of the model handler is chosen based on the execution mode (single, grid, swarm, etc.).
Though it is possible for a project to define its own handlers,
the PMSCO core declares handlers that should cover most calculation scenarios.
The following diagram shows the tree of calculation tasks and how handlers act on the task objects to populate the task attributes.
At the top of the tree, an empty task object (all attributes undefined) is fed into the model level handler which takes care of the model attribute.
The model handler generates a number of sub-tasks, one for each set of model parameters.
Each of these (incompletely defined) tasks is then passed to the next handler, and so on.
@startuml "calculation task hierarchy and task handler stack"
object "Root: CalculationTask" as Root {
index = (-1,-1,-1,-1,-1)
}
note right: all attributes undefined
object "Model: CalculationTask" as Model {
index = (i,-1,-1,-1,-1)
model
}
note right: model is defined\nother attributes undefined
object ModelHandler
object "Scan: CalculationTask" as Scan {
index = (i,j,-1,-1,-1)
model
scan
}
object ScanHandler
object "Domain: CalculationTask" as Domain {
index = (i,j,k,-1,-1)
model
scan
domain
}
object "DomainHandler" as DomainHandler
object "Emitter: CalculationTask" as Emitter {
index = (i,j,k,l,-1)
model
scan
domain
emitter
}
object EmitterHandler
object "Region: CalculationTask" as Region {
index = (i,j,k,l,m)
model
scan
domain
emitter
region
}
note right: all attributes well-defined
object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Domain
Domain "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Domain) .. DomainHandler
(Domain, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml
At the end of the stack, the tasks are fully specified and are passed to the calculation queue.
They are dispatched to the available processes of the MPI environment in which PMSCO was started,
which allows calculations to be run in parallel.
Only now that the model is broken down into multiple, fully specified tasks,
the cluster and input files are generated, and the calculation program is started.
At the end of a calculation, the output files are associated with their original task objects,
and the tasks are passed back through the task handler stack.
In this phase, each level joins the datasets from the sub-tasks to the data requested by the parent task.
For example, at the lowest level, one result file is present for each region.
The region handler gathers all files that correspond to the same parent task
(i.e. have the same emitter, domain, scan and model attributes),
joins them to one file which includes all regions,
links the file to the parent task and passes the result to the next higher level.
On the top level, the model handler compares the result to the experimental data.
Depending on the operation mode, it refines the model parameters and issues new tasks by passing them down the stack.
When the optimization is finished (according to a set of defined criteria),
The model handler returns the root task to the caller, which causes PMSCO to exit.
*/

192
docs/src/concepts.dox
View File

@@ -1,153 +1,83 @@
/*! @page pag_concepts Design Concepts
\section sec_tasks Tasks
/*! @page pag_concepts Design
In an optimization project, a number of optimizable, high-level parameters generated by the optimization algorithm
must be mapped to the input parameters and atomic coordinates before the calculation program is executed.
Possibly, the calculation program is executed multiple times for inequivalent domains, emitters or scan geometries.
After the calculation, the output is collected, compared to the experimental data, and the model is refined.
In PMSCO, the optimization is broken down into a set of _tasks_ and assigned to a stack of task _handlers_ according to the following figure.
Each invocation of the scattering program (EDAC) runs a specific task,
i.e. a calculation for a set of specific parameters, a fully-qualified cluster of atoms, and a specific angle and/or energy scan.
\section sec_components Components
\dotfile tasks.dot "PMSCO task stack"
The code for a PMSCO job consists of the following components.
At the root, the _model handler_ proposes models that need to be calculated according to the operation mode specified at the command line.
A _model_ is the minimum set of variable parameters in the context of a custom project.
Other parameters that will not vary under optimization are set directly by the project code.
The model handler may generate models based on a fixed scheme, e.g. on a grid, or based on R-factors of previous results.
@startuml "top-level components of scattering and optimization code"
For each model, one task is passed to the task handling chain, starting with the scan handler.
The _scan handler_ generates sub-tasks for each experimental scan dataset.
This way, the model can be optimized for multiple experimental scans in the same run (see Sec. \ref sec_scanning).
skinparam componentStyle uml2
The _symmetry handler_ generates sub-tasks based on the number of symmetries contained in the experimental data (see Sec. \ref sec_symmetry).
For instance, for a system that includes two inequivalent structural domains, two separate calculations have to be run for each model.
The symmetry handler is implemented on the project level and may be customized for a specific system.
component "PMSCO" as pmsco
component "project" as project
component "scattering code\n(calculator)" as calculator
The _emitter handler_ generates a sub-task for each inequivalent emitter atom
so that the tasks can be distributed to multiple processes (see Sec. \ref sec_emitters).
In a single-process environment, all emitters are calculated in one task.
interface "command line" as cli
interface "experimental data" as data
interface "results" as results
interface "output files" as output
The _region handler_ may split a scan region into several smaller chunks
so that the tasks can be distributed to multiple processes.
With EDAC, only energy scans can benefit from chunking
since it always calculates the full angular distribution.
This layer has to be enabled specifically in the project module.
It is disabled by default.
cli --> pmsco
data -> project
pmsco ..> project
pmsco ..> calculator
calculator -> output
pmsco -> results
At the end of the stack, the tasks are fully specified and are passed to the calculation queue.
They are dispatched to the available processes of the MPI environment in which PMSCO was started,
which allows calculations to be run in parallel.
Only now that the model is broken down into multiple tasks,
the cluster and input files are generated, and the calculation program is started.
@enduml
At the end of a calculation, the output is passed back through the task handler stack.
In this phase, each level gathers the datasets from the sub-tasks to the data requested by the parent task
and passes the result to the next higher level.
The main entry point is the _PMSCO_ module.
It implements a task loop to carry out the structural optimization
and provides an interface between calculation programs and project-specific code.
It also provides common utility classes and functions for the handling project data.
On the top level, the calculation is compared to the experimental data.
Depending on the operation mode, the model parameters are refined, and new tasks issued.
If the optimization is finished according to a set of defined criteria, PMSCO exits.
The _project_ consists of program code and parameters
that are specific to a particular experiment and calculation job.
The project code reads experimental data, defines the parameter dictionary of the model,
and contains code to generate the cluster, parameter and phase files for the scattering code.
The project is also the main entry point of process execution.
As an implentation detail, each task is given a unique _identifier_ consisting of five integer numbers
which correspond to the five levels model, scan, symmetry, emitter and region.
The identifier appears in the file names in the communication with the scattering program.
Normally, the data files are deleted after the calculation, and only a few top-level files are kept
(can be overridden at the command line or in the project code).
At the top level, only the model ID is set, the other ones are undefined (-1).
The _scattering code_ on the other hand is a static calculation engine
which accepts detailed input files
(parameters, atomic coordinates, emitter specification, scattering phases)
and outputs an intensity distribution of photoelectrons versus energy and/or angle.
\section sec_symmetry Symmetry and Domain Averaging
\section sec_control_flow Control flow
A _symmetry_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is derived from the same set of model parameters
and that contributes incoherently to the measured diffraction pattern.
A symmetry may be represented by a special symmetry parameter which is not subject to optimization.
The basic control flow of a optimization job is depicted schematically in the following figure.
For instance, a real sample may have additional rotational domains that are not present in the cluster,
increasing the symmetry from three-fold to six-fold.
Or, an adsorbate may be present in a number of different lateral configurations on the substrate.
In the first case, it may be sufficient to fold calculated data in the proper way to generate the same symmetry as in the measurement.
In the latter case, it may be necessary to execute a scattering calculation for each possible orientation or a representative number of possible orientations.
@startuml "top-level activity diagram"
PMSCO provides the basic framework to spawn multiple calculations according to the number of symmetries (cf. \ref sec_tasks).
The actual data reduction from multiple symmetries to one measurement needs to be implemented on the project level.
This section explains the necessary steps.
start
:initialize;
:import experimental data;
repeat
:define tasks;
fork
:calculate\ntask 1;
fork again
:calculate\ntask N;
end fork
:evaluate results;
repeat while
-> [finished];
:report results;
1. Your project needs to populate the pmsco.project.Project.symmetries list.
For each symmetry, add a dictionary of symmetry parameters, e.g. <code>{'angle_azi': 15.0}</code>.
There must be at least one symmetry in a project, otherwise no calculation is executed.
stop
2. The project may apply the symmetry of a task to the cluster and parameter file if necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular symmetry in addition to the model parameters.
@enduml
3. The project combines the results of the calculations for the various symmetries into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with equal weight.
If you need more control, you need to override the pmsco.project.Project.combine_symmetries method and implement your own algorithm.
After importing experimental data and setting up the model dictionary and job parameters,
the calculation tasks are defined depending on the execution mode and system setup.
Each task consists of a specific set of model, experimental and calculation parameters
that describe an independent calculation step,
while several steps may be required to produce a dataset that can be compared to the experimental data.
The idea is that tasks can be defined quickly
and that the time-consuming operations are dispatched to slave processes which can run in parallel.
\section sec_scanning Scanning
PMSCO with EDAC currently supports the following scan axes.
- kinetic energy E
- polar angle theta T
- azimuthal angle phi P
- analyser angle alpha A
The following combinations of these scan axes are allowed (see pmsco.data.SCANTYPES).
- E
- E-T
- E-A
- T-P (hemispherical or hologram scan)
@attention The T and A axes cannot be combined.
If a scan of one of them is specified, the other is assumed to be fixed at zero!
This assumption may change in the future,
so it is best to explicitly set the fixed angle to zero in the scan file.
@remark According to the measurement geometry at PEARL,
alpha scans are implemented in EDAC as theta scans at phi = 90 in fixed cluster mode.
The switch to fixed cluster mode is made by PMSCO internally,
no change of angles or other parameters is necessary in the scan or project files
besides filling the alpha instead of the theta column.
\section sec_emitters Emitter Configurations
Since emitters contribute incoherently to the diffraction pattern,
it should make no difference how the emitters are grouped and calculated.
EDAC allows to specify multiple emitters in one calculation.
However, running EDAC multiple times for a single-emitter configuration or simply summing up the results
gives the same final diffraction pattern with no significant difference of used CPU time.
It is, thus, easy to distribute the emitters over parallel processes in a multi-process environment.
PMSCO can handle this transparently with a minimal effort.
Within the same framework, PMSCO also supports that clusters are tailored to a specific emitter configuration.
Suppose that the unit cell contains a large number of inequivalent emitters.
If all emitters had to be included in a single calculation,
the cluster would grow very large and the calculation would take a long time
because it would include many long scattering paths
that effectively do not contribute intensity to the final result.
Using single-emitters, a cluster can be built locally around the emitter and kept to a reasonable size.
Even when using this feature, PMSCO does not require that each configuration contains only one emitter.
The term _emitter_ effectively means _emitter configuration_.
A configuration can include multiple emitters which will not be broken up further.
It is up to the project, what is included in a particular configuration.
To enable emitter handling,
1. override the count_emitters method of your cluster generator
and return the number of emitter configurations of a given model, scan and symmetry.
2. handle the emitter index in your create_cluster method.
3. (optionally) override the pmsco.project.Project.combine_emitters method
if the emitters should not be added with equal weights.
For implementation details see the respective method descriptions.
As soon as all necessary results are available they are combined into one dataset and compared to the experimental data.
Depending on the execution mode, the process of task definition and calculation repeats until the model has converged
or the calculations are stopped for another reason.
*/

22
docs/src/dataflow.dot
View File

@@ -10,7 +10,7 @@ digraph G {
create_params;
calc_modf;
calc_rfac;
comb_syms;
comb_doms;
comb_scans;
}
*/
@@ -24,11 +24,11 @@ digraph G {
model_handler -> model_creator [constraint=false, label="optimize"];
}
subgraph cluster_symmetry {
label = "symmetry handler";
subgraph cluster_domain {
label = "domain handler";
rank = same;
sym_creator [label="expand models", group=creators];
sym_handler [label="combine symmetries", group=handlers];
dom_creator [label="expand models", group=creators];
dom_handler [label="combine domains", group=handlers];
}
subgraph cluster_scan {
@@ -47,15 +47,15 @@ digraph G {
calculator [label="calculator (EDAC)", shape=box];
model_creator -> sym_creator [label="model", style=bold];
sym_creator -> scan_creator [label="models", style=bold];
model_creator -> dom_creator [label="model", style=bold];
dom_creator -> scan_creator [label="models", style=bold];
scan_creator -> calc_creator [label="models", style=bold];
calc_creator -> calculator [label="clusters,\rparameters", style=bold];
calculator -> calc_handler [label="output files", style=bold];
calc_handler -> scan_handler [label="raw data files", style=bold];
scan_handler -> sym_handler [label="combined scans", style=bold];
sym_handler -> model_handler [label="combined symmetries", style=bold];
scan_handler -> dom_handler [label="combined scans", style=bold];
dom_handler -> model_handler [label="combined domains", style=bold];
mode [shape=parallelogram];
mode -> model_creator [lhead="cluster_model"];
@@ -76,8 +76,8 @@ digraph G {
calc_rfac [shape=cds, label="R-factor function"];
calc_rfac -> model_handler [style=dashed];
comb_syms [shape=cds, label="symmetry combination rule"];
comb_syms -> sym_handler [style=dashed];
comb_doms [shape=cds, label="domain combination rule"];
comb_doms -> dom_handler [style=dashed];
comb_scans [shape=cds, label="scan combination rule"];
comb_scans -> scan_handler [style=dashed];

143
docs/src/execution.dox
View File

@@ -2,49 +2,44 @@
\section sec_run Running PMSCO
To run PMSCO you need the PMSCO code and its dependencies (cf. @ref pag_install),
a code module that contains the project-specific code,
and one or several files containing the scan parameters and experimental data.
a customized code module that contains the project-specific code,
one or several files containing the scan parameters and experimental data,
and a run-file specifying the calculation parameters.
Please check the <code>projects</code> folder for examples of project modules.
For a detailed description of the command line, see @ref pag_command.
\subsection sec_run_single Single Process
Run PMSCO from the command prompt:
The following instructions assume that PMSCO was installed as a Python site-package according to @ref pag_install.
To run PMSCO from the command prompt:
@code{.sh}
cd work-dir
python project-dir/project.py [pmsco-arguments] [project-arguments]
python -m pmsco -j job-name -r run-file
@endcode
where <code>work-dir</code> is the destination directory for output files,
<code>project.py</code> is the specific project module,
and <code>project-dir</code> is the directory where the project file is located.
PMSCO is run in one process which handles all calculations sequentially.
where:
The command line arguments are usually divided into common arguments interpreted by the main pmsco code (pmsco.py),
and project-specific arguments interpreted by the project module.
However, it is ultimately up to the project module how the command line is interpreted.
| `work-dir` | Destination directory for output files |
| `run-file` | JSON-formatted configuration file that defines run-time parameters. The format and content of a run file is described in a section @ref pag_runfile. |
| `job-name` | (optional) The job name appears mainly as the prefix of all output files but is also used in the database and other places. The job name can also be declared in the run file. |
In this form, PMSCO is run in one process which handles all calculations sequentially.
Example command line for a single EDAC calculation of the two-atom project:
@code{.sh}
cd work/twoatom
python pmsco/projects/twoatom/twoatom.py -s ea -o twoatom-demo -m single
python -m pmsco -j job0001 -r twoatom-hemi.json
@endcode
The project file <code>twoatom.py</code> takes the lead of the project execution.
Usually, it contains only project-specific code and delegates common tasks to the main pmsco code.
This command line executes the main pmsco module <code>pmsco.py</code>.
The information which project to load is contained in the <code>twoatom-hemi.json</code> file,
along with all common and specific project arguments.
In the command line above, the <code>-o twoatom-demo</code> and <code>-m single</code> arguments
are interpreted by the pmsco module.
<code>-o</code> sets the base name of output files,
and <code>-m</code> selects the operation mode to a single calculation.
The scan argument is interpreted by the project module.
It refers to a dictionary entry that declares the scan file, the emitting atomic species, and the initial state.
In this example, the project looks for the <code>twoatom_energy_alpha.etpai</code> scan file in the project directory,
and calculates the modulation function for a N 1s initial state.
The kinetic energy and emission angles are contained in the scan file.
This example can be run for testing.
All necessary parameters and data files are included in the code repository.
\subsection sec_run_parallel Parallel Processes
@@ -58,30 +53,104 @@ The slave processes will run the scattering calculations, while the master coord
and optimizes the model parameters (depending on the operation mode).
For optimum performance, the number of processes should not exceed the number of available processors.
To start a two-hour optimization job with multiple processes on an quad-core workstation with hyperthreading:
To start an optimization job with multiple processes on a quad-core workstation with hyperthreading:
@code{.sh}
cd work/my_project
mpiexec -np 8 project-dir/project.py -o my_job_0001 -t 2 -m swarm
mpiexec -np 8 --use-hwthread-cpus python -m pmsco -j my_job002 -r my_project.json
@endcode
The `--use-hwthread` option is necessary on certain hyperthreading architectures.
\subsection sec_run_hpc High-Performance Cluster
The script @c bin/qpmsco.ra.sh takes care of submitting a PMSCO job to the slurm queue of the Ra cluster at PSI.
The script can be adapted to other machines running the slurm resource manager.
The script generates a job script based on @c pmsco.ra.template,
substituting the necessary environment and parameters,
and submits it to the queue.
PMSCO is ready to run with resource managers on cluster machines.
Code for submitting jobs to Slurm queues is included and can be customized for many machines.
For example, code for the slurm queue of the Ra cluster at PSI is included in the pmsco.schedule module.
Other machines can be supported by sub-classing pmsco.schedule.JobSchedule or pmsco.schedule.SlurmSchedule.
Execute @c bin/qpmsco.ra.sh without arguments to see a summary of the arguments.
To have PMSCO submit a job, the arguments for the queue are entered in the schedule section of the run file,
cf. @ref pag_runfile.
Then, the same command as for starting a calculation directly will instead submit a job to the queue:
To submit a job to the PSI clusters (see also the PEARL-Wiki page MscCalcRa),
the analog command to the previous section would be:
@code{.sh}
bin/qpmsco.ra.sh my_job_0001 1 8 2 projects/my_project/project.py swarm
python -m pmsco -j job-name -r run-file.json
@endcode
The command creates a separate work directory with copies of the project source, the run-file and the job script.
This job directory will also receive the calculation results.
The full path of the job directory is _output-dir/job-name_.
The directory must not exist when you run the above command to prevent overwriting of previous data.
The job name can be declared in the run file or on the command line.
The command above also loads the project module and scan files.
Many parameter errors are caught this way and can be fixed before the job is submitted to the queue.
The run file offers an option to prepare a script file and not to submit the job immediately
so that you can inspect the job files and submit the job manually.
Be sure to consider the resource allocation policy of the cluster
before you decide on the number of processes.
Requesting less resources will prolong the run time but might increase the scheduling priority.
*/
\subsection sec_run_dirs Directories
Code and data files are typically located in different, possibly machine-specific locations.
This can make it difficult to port a project to another machine and to repeat calculations.
Ideally, a calculation job should be repeatable on different machines
with a minimum of changes to code, input data and parameter files.
Project code (which is under version control)
should never need modifications for porting to another machine.
Run-files (which are considered part of the data) can follow a project-specific or machine-specific directory structure.
PMSCO provides directory resolution at run-time to facilitate writing of portable code.
This is done by a number of directory aliases that can be included as shell-like placeholders, e.g. `${project}`, in file paths.
Some aliases are preset to system-based defaults,
further aliases can be added by the project code or declared in the run file.
Directory aliases can be used in Project.directories
as well as in other Project attributes that hold a file name.
The table below shows the aliases defined and/or required by PMSCO.
The paths are stored in Project.directories.
The aliases are resolved before the actual calculations start (in the Project.validate() method).
The resolved paths are printed to the log at warning level.
| Key | Description | Source | Use |
| --- | --- | --- | --- |
| work | Working directory at program start | PMSCO | |
| home | User's home directory | PMSCO | |
| project | Location of the project module. | PMSCO | Can be used to find auxiliary files that are part of the repository. |
| output | Intermediate and output files. | Must be set by the project or run file | The `output_file` property which serves as the basis of all output files is a concatenation of the `output` directory and `job_name`. |
| report | Directory for graphical output (reports) | Default: `${output}/report` | |
| data (optional) | Location of data (scan) files. | Project or run file | Usage is up to the project. |
| temp | Temporary files | | Reserved. Currently not supported |
| (job tag) | Any job_tags key that maps to a legal directory name can be included in a path | run file | project or run file |
| mode, job_name, project_name | These project attributes can be included in a path if they contain a valid directory name | | |
\subsection sec_run_stop Stopping a PMSCO job
A PMSCO optimization job stops on any one of the following events.
- The model handler is done.
Depending on the run mode, this happens when the optimization has converged or
the planned number of iterations or calculations has been reached.
- The number of calculation tasks exceeds the limit configured in `dispatch.MscoMaster.max_calculations`.
This is meant to prevent excessive and runaway jobs.
The default value is 1000000. It can be adjusted by the project code if necessary.
- The master process receives a SIGTERM, SIGUSR1 or SIGUSR2 from the operating system.
The signal can be sent, e.g., by the `kill` command on Linux.
This doesn´t work on all platforms.
- The time limit configured in `Project.timedelta_limit` is reached.
This is a soft limit and should be set shorter than the job reservation with the resource manager.
- A file named `finish_pmsco` is present in the output directory.
This is an easy way for a user to stop a running optimization.
The file doesn´t need any content.
It can be created by the `touch` command.
All these stop conditions cause graceful stops.
Running calculation tasks are waited for, but some results on the model level may not be complete.
Final reports of complete models are produced and the output folder is cleaned up.
Stops caused by resource managers such as Slurm are typically not graceful.
The results are in an undefined state, reports are not generated, and temporary files may be left over.
*/

352
docs/src/installation.dox
View File

@@ -3,166 +3,314 @@
\subsection sec_general General Remarks
The PMSCO code is maintained under git.
The central repository for PSI-internal projects is at https://git.psi.ch/pearl/pmsco,
the public repository at https://gitlab.psi.ch/pearl/pmsco.
The central repository for development and PSI-internal projects is at https://gitea.psi.ch/pearl/pmsco,
the public repository at https://gitea.psi.ch/pearl-public/pmsco.
For their own developments, users should clone the repository.
Changes to common code should be submitted via pull requests.
Scientific projects should be maintained in a separate directory tree, cf. @ref sec_project.
The program code of PMSCO and its external programs is written in Python, C++ and Fortran.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
For optimization jobs with parallel execution, a workstation with at least 4 processor cores
or cluster with 20-50 available processor cores is recommended.
The program requires about 2 GB of RAM per process.
The recommended IDE is [PyCharm (community edition)](https://www.jetbrains.com/pycharm).
[Spyder](https://docs.spyder-ide.org/index.html) is a good alternative with a focus on scientific data.
The documentation in [Doxygen](https://www.doxygen.nl/index.html) format is part of the source code.
The Doxygen compiler can generate documentation in HTML.
@attention Due to rapidly evolving computing environments
some of the installation instructions on this page may be outdated or incompatible with certain environments.
\subsection sec_requirements Requirements
The recommended IDE is [PyCharm (community edition)](https://www.jetbrains.com/pycharm).
The documentation in [Doxygen](http://www.stack.nl/~dimitri/doxygen/index.html) format is part of the source code.
The Doxygen compiler can generate separate documentation in HTML or LaTeX.
Please note that in some environments (particularly shared high-performance machines)
it may be important to choose specific compiler and library versions that are tailored to the hardware platform.
In order to maintain backward compatibility with older installations,
code that requires new versions of compilers and libraries should be introduced carefully.
The MSC and EDAC codes compile with the GNU Fortran and C++ compilers on Linux.
Other compilers may work but have not been tested.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a high-performance cluster with 20-50 available processor cores is recommended.
The code requires about 2 GB of RAM per process.
The following basic tools and libraries are required:
Please note that it may be important that the code remains compatible with earlier compiler and library versions.
Newer compilers or the latest versions of the libraries contain features that will break the compatibility.
The code can be used with newer versions as long they are backward compatible.
The code depends on the following libraries:
- GCC 4.8
- OpenMPI 1.10
- F2PY
- F2C
- SWIG
- Python 2.7 (incompatible with Python 3.0)
- Numpy 1.11 (incompatible with Numpy 1.13 and later)
- MPI4PY (from PyPI)
- GCC (C, C++, Fortran) >= 4.8
- BLAS
- LAPACK
- periodictable
- OpenMPI >= 1.10
- Git
Most of these requirements are available from the Linux distribution, or from PyPI (pip install), respectively.
If there are any issues with the packages installed by the distribution, try the ones from PyPI
(e.g. there is currently a bug in the Debian mpi4py package).
The F2C source code is contained in the repository for machines which don't have it installed.
On the PSI cluster machines, the environment must be set using the module system and conda (on Ra).
Details are explained in the PEARL Wiki.
Most of these requirements are available from the Linux distribution.
For the Python environment,
the [uv](https://docs.astral.sh/uv/) package and environment manager is recommended.
It can be installed by non-privileged users.
Other package managers like pip and conda may work as well but are not described here.
\subsubsection sec_install_ubuntu Installation on Ubuntu 16.04
The following tools are required to compile the documentation.
They are not needed in calculations.
The following instructions install the necessary dependencies on Ubuntu (or Lubuntu 16.04):
- doxygen
- doxypypy
- graphviz
- Java runtime environment (JRE)
- [plantUML](https://plantuml.com)
\subsection sec_install_instructions Instructions
Installation instructions are given for Ubuntu 24.04.
On managed HPC clusters use the compilers and libraries recommended by the administrator
(often provided by a module system).
\subsubsection sec_install_ubuntu Installation on Ubuntu
The following instructions install the necessary dependencies on Ubuntu, Debian or related distributions.
@code{.sh}
sudo apt-get update
sudo apt update
sudo apt-get install \
sudo apt install \
binutils \
build-essential \
doxygen \
doxypy \
f2c \
g++ \
gcc \
gfortran \
git \
graphviz \
ipython \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
python-all \
python-mock \
python-nose \
python-numpy \
python-pip \
python-scipy \
python2.7-dev \
swig
sudo pip install --system mpi4py periodictable
cd /usr/lib
sudo ln -s /usr/lib/libblas/libblas.so.3 libblas.so
python3 \
python3-venv \
sqlite3 \
wget
@endcode
The following instructions install the PyCharm IDE and a few other useful utilities:
In addition, download and install [uv](https://docs.astral.sh/uv/).
PSI users should configure uv to use PSI's PyPI package cache (cf. documentation on the intranet).
\subsubsection sec_install_extra Additional Applications
For working with the code and data, some other applications are recommended.
The PyCharm IDE can be installed from the Ubuntu software center.
The following commands install other useful helper applications:
@code{.sh}
sudo sh -c 'echo "deb http://archive.getdeb.net/ubuntu xenial-getdeb apps" >> /etc/apt/sources.list.d/getdeb.list'
wget -q -O - http://archive.getdeb.net/getdeb-archive.key | sudo apt-key add -
sudo apt-get update
sudo apt-get install \
sudo apt install \
avogadro \
gitg \
meld \
openjdk-9-jdk \
pycharm
meld
@endcode
To produce documentation in PDF format (not recommended on virtual machine), install LaTeX:
To compile the documentation install the following tools.
The basic documentation is in HTML format and can be opened in any internet browser.
@code{.sh}
sudo apt-get install texlive-latex-recommended
sudo apt install \
doxygen \
graphviz \
default-jre
wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
sudo mkdir /opt/plantuml/
sudo mv plantuml.jar /opt/plantuml/
echo "export PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar" | sudo tee /etc/profile.d/pmsco-env.sh
@endcode
\subsection sec_compile Compilation
\subsection sec_distro Download PMSCO Source Code
Make sure you have access to the PMSCO Git repository and set up your Git environment.
Depending on your setup, location and permissions, one of the following addresses may work.
Private key authentication is usually recommended except on shared computers.
Clone or download the code from one of these repository addresses:
| Repository | Access |
| --- | --- |
| `git@git.psi.ch:pearl/pmsco.git` | PSI intranet, SSH private key authentication |
| `https://git.psi.ch/pearl/pmsco.git` | PSI intranet, password prompt |
| `git@gitlab.psi.ch:pearl/pmsco.git` | Public repository, SSH private key authentication |
| `https://gitlab.psi.ch/pearl/pmsco.git` | Public repository, password prompt |
Clone the code repository using one of these repositiory addresses and switch to the desired branch:
| https://gitea.psi.ch/pearl/pmsco | PSI internal |
| https://gitea.psi.ch/pearl-public/pmsco-public | Public |
@code{.sh}
cd ~
git clone git@git.psi.ch:pearl/pmsco.git pmsco
git clone {repo-address see above} pmsco
cd pmsco
git checkout master
@endcode
These instructions download the base package of PMSCO.
The public repository does not contain external programs (EDAC, PHAGEN, LOESS).
You need to obtain the source code for these programs from their respective owners,
copy them to the respective subprojects directories and
apply the patches included in the PMSCO distribution.
Please respect the respective license terms and acknowledge the use of the codes.
\subsection sec_install_environment Set up the Python Environment
The following instructions are for the [uv](https://docs.astral.sh/uv/) package manager.
For other package managers, the pyproject.toml and requirements.txt files list the necessary dependencies.
\subsubsection sec_install_uv Virtual Environment with uv
By default, uv creates the virtual environment automatically in a `.venv` folder inside the source directory tree.
In this case, no explicit setup is necessary, and pmsco can be called by:
~~~~~~{.sh}
uv run pmsco -h
~~~~~~
On some platforms, however, it may be necessary to separate the environment from the code,
e.g. because of limited storage space or quota in the home directory.
In this case, create the environment as follows:
~~~~~~{.sh}
cd ~
mkdir envs
cd envs
uv venv --clear my_pmsco_env
~~~~~~
The `--clear` option resets an existing environment to empty.
To activate this environment, call this command once in every terminal:
~~~~~~{.sh}
source ~/envs/my_pmsco_env/bin/activate
~~~~~~
\subsubsection sec_normal_install Installing PMSCO
to install PMSCO and all dependencies into the active environment,
run the following commands in the top-level PMSCO directory (where `pyproject.toml` is located).
The commands compile the Fortran and C++ code of the calculation programs using the
[Meson build system](https://mesonbuild.com/meson-python/index.html)
and install the binaries and Python code in the site-packages folder of the active Python environment.
~~~~~~{.sh}
uv sync --active
~~~~~~
To use the default `.venv` environment, omit the `--active` option (also in the uv commands shown further below).
Now, run the unit tests to check the installation:
~~~~~~{.sh}
uv run --active nosetests
~~~~~~
And check the help page:
~~~~~~{.sh}
uv run --active pmsco -h
~~~~~~
In the explicit environment, these commands can alternatively be called directly:
~~~~~~{.sh}
nosetests
pmsco -h
~~~~~~
The PMSCO packages are now accessible in Python import statements.
Verify it by opening a Python shell and entering:
~~~~~~{.py}
import pmsco.project
dir(pmsco.project)
~~~~~~
Note: By default, uv installs the Python code in editable mode.
Changes in the PMSCO source tree are visible as soon as you start a new Python interpreter.
This does not apply to the subpackages, however.
After modifying the subpackages, you need to clear and re-sync the environment.
\subsection sec_test Test project
Run the twoatom project to check that everything is installed correctly:
~~~~~~{.py}
cd ~
mkdir -p work/twoatom
cd work/twoatom
nice python -m pmsco -r {path-to-pmsco}/projects/twoatom/twoatom-hemi.json
~~~~~~
You should get a number of result files whose names start with `twoatom0001` in `~/work/twoatom/`,
including a hologram plot of the modulation function.
To learn more about running PMSCO, see @ref pag_run.
\subsection sec_install_projects Installing Namespace Packages
Instructions on how to set up your own projects as namespace packages are given in section \ref sec_project.
To install them into the pmsco namespace, call uv with the `--inexact` option.
Without `--inexact`, uv would remove the previously installed packages (including PMSCO).
~~~~~~{.sh}
uv sync --active --inexact
~~~~~~
\subsection sec_install_singularity Installation in a Singularity container
Singularity containers are currently unmaintained.
The PMSCO source includes an install script for the [Singularity](https://sylabs.io/singularity/) container system
under `extras/singularity`.
To get started with Singularity,
download it from [sylabs.io](https://www.sylabs.io/singularity/) and install it according to their instructions.
On Windows, Singularity can be installed in a virtual machine using the [Vagrant](https://www.vagrantup.com/)
script included under `extras/vagrant`.
After installing Singularity,
check out PMSCO as explained in the @ref sec_compile section:
@code{.sh}
cd ~
mkdir containers
cd containers
git clone git@gitea.psi.ch:pearl-public/pmsco-public.git pmsco
cd pmsco
git checkout master
git checkout -b my_branch
@endcode
The compilation of the various modules is started by <code>make all</code>.
The compilation step is necessary only once after installation.
If the compilation of _loess.so failes due to a missing BLAS library,
try to set a link to the BLAS library as follows (the actual file names may vary due to the actual distribution or version):
@code{.sh}
cd /usr/lib
sudo ln -s /usr/lib/libblas/libblas.so.3 libblas.so
@endcode
\subsection sec_test Tests
Run the unit tests.
They should pass successfully.
Re-check from time to time.
Then, either copy a pre-built container into `~/containers`,
or build one from the definition file included under extras/singularity.
You may need to customize the definition file to match the host OS
or to install compatible OpenMPI libraries,
cf. [Singularity user guide](https://sylabs.io/guides/3.7/user-guide/mpi.html).
@code{.sh}
cd ~/pmsco
nosetests
cd ~/containers
sudo singularity build pmsco.sif ~/containers/pmsco/extras/singularity/singularity_python3
@endcode
Run the twoatom project to check the compilation of the calculation programs.
To work with PMSCO, start an interactive shell in the container and switch to the pmsco environment.
Note that the PMSCO code is outside the container and can be edited with the usual tools.
@code{.sh}
cd ~/pmsco
mkdir work
cd work
mkdir twoatom
cd twoatom/
nice python ~/pmsco/projects/twoatom/twoatom.py -s ~/pmsco/projects/twoatom/twoatom_energy_alpha.etpai -o twoatom_energy_alpha -m single
cd ~/containers
singularity shell pmsco.sif
. /opt/miniconda/etc/profile.d/conda.sh
conda activate pmsco
cd ~/containers/pmsco
meson setup build
meson compile -C build
meson install -C build
meson test -C build
@endcode
To learn more about running PMSCO, see @ref pag_run.
Or call PMSCO from outside:
@code{.sh}
cd ~/containers
mkdir output
cd output
singularity run -e ../pmsco.sif python -m pmsco -r path/to/your-runfile
@endcode
For parallel processing, prepend `mpirun -np X` to the singularity command as needed.
Note that this requires compatible OpenMPI versions on the host and container to avoid runtime errors.
*/

70
docs/src/introduction.dox
View File

@@ -1,61 +1,79 @@
/*! @mainpage Introduction
\section sec_intro Introduction
PMSCO stands for PEARL multiple-scattering cluster calculations and structural optimization.
It is a collection of computer programs to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data.
PMSCO (PSI multiple-scattering cluster calculations and structural optimization)
is a Python-based workflow engine to calculate photoelectron diffraction patterns,
and to optimize structural models based on measured data using machine learning techniques.
PMSCO was developed at the [Paul Scherrer Institut (PSI)](https://www.psi.ch/)
by the team of the [PEARL beamline](https://www.psi.ch/en/sls/pearl).
The actual scattering calculation is done by code developed by other parties.
While the scattering program typically calculates a diffraction pattern based on a set of static parameters and a specific coordinate file in a single process,
PMSCO wraps around that program to facilitate parameter handling, cluster building, structural optimization and parallel processing.
In the current version, the [EDAC](http://garciadeabajos-group.icfo.es/widgets/edac/) code
developed by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley (1999) is used for scattering calculations.
Other code can be integrated as well.
Initially, support for the MSC program by Kaduwela, Friedman, and Fadley was planned but is currently not maintained.
PMSCO is written in Python 2.7.
EDAC is written in C++, MSC in Fortran.
PMSCO interacts with the calculation programs through Python wrappers for C++ or Fortran.
In the current version, PMSCO can make use of the following programs.
Other programs may be integrated as well.
The MSC and EDAC source code is contained in the same software repository.
The PMSCO, MSC, and EDAC programs may not be used outside the PEARL group without an explicit agreement by the respective original authors.
Users of the PMSCO code are requested to coordinate and share the development of the code with the original author.
Please read and respect the respective license agreements.
- [EDAC](https://garciadeabajos-group.icfo.es/widgets/edac/)
by F. J. García de Abajo, M. A. Van Hove, and C. S. Fadley,
[Phys. Rev. B 63 (2001) 075404](http://dx.doi.org/10.1103/PhysRevB.63.075404)
- PHAGEN from the [MsSpec package](https://ipr.univ-rennes1.fr/msspec)
by C. R. Natoli and D. Sébilleau,
[Comp. Phys. Comm. 182 (2011) 2567](http://dx.doi.org/10.1016/j.cpc.2011.07.012)
\section sec_intro_highlights Highlights
- angle or energy scanned XPD.
- various scanning modes including energy, polar angle, azimuthal angle, analyser angle.
- averaging over multiple symmetries (domains or emitters).
- averaging over multiple domains and emitters.
- global optimization of multiple scans.
- structural optimization algorithms: particle swarm optimization, grid search, gradient search.
- structural optimization algorithms: genetic, particle swarm, grid search.
- calculation of the modulation function.
- calculation of the weighted R-factor.
- automatic parallel processing using OpenMPI.
- integrated and extensible reporting, database storage of results.
- automatic parallel processing using OpenMPI and job submission to scheduling systems.
\section sec_project Optimization Projects
\section sec_intro_project Optimization Projects
To set up a new optimization project, you need to:
- create a new directory under projects.
- create a new Python module in this directory, e.g., my_project.py.
- implement a sub-class of project.Project in my_project.py.
- override the create_cluster, create_params, and create_domain methods.
- optionally, override the combine_symmetries and combine_scans methods.
- add a global function create_project to my_project.py.
- provide experimental data files (intensity or modulation function).
- implement a sub-class of pmsco.project.Project in my_project.py.
- override the create_cluster, create_params, and create_model_space methods.
- optionally, override the combine_domains and combine_scans methods.
- add a global function create_project to my_project.py or create a @ref pag_runfile.
- prepare experimental data files (intensity or modulation function).
For details, see the documentation of the Project class,
and the example projects.
For details, see @ref pag_project, the documentation of the pmsco.project.Project class and the example projects.
\section sec_intro_start Getting Started
- @ref pag_concepts
- @ref pag_concepts_tasks
- @ref pag_concepts_emitter
- @ref pag_install
- @ref pag_project
- @ref pag_run
- @ref pag_command
- @ref pag_opt
\section sec_license License Information
The source code of PMSCO is licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0).
This _does not include_ the calculation packages contained in the subprojects folder which are licensed separately.
- Please read and respect the respective license agreements.
- Please acknowledge the use of the code.
- Please consider sharing your developments with the original author.
Due to different copyright terms, the third-party calculation programs are not contained in the public software repository.
These programs may not be used without an explicit agreement by the respective original authors.
\author Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
\version This documentation is compiled from version $(REVISION).
\copyright 2015-2025 by [Paul Scherrer Institut](http://www.psi.ch)
\copyright Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
*/

208
docs/src/optimizers.dox Normal file
View File

@@ -0,0 +1,208 @@
/*! @page pag_opt Model optimizers
\section sec_opt Model optimizers
\subsection sec_opt_swarm Particle swarm optimization (PSO)
The particle swarm optimization (PSO) algorithm seeks to find a global optimum in a multi-dimensional model space
by employing the _swarm intelligence_ of a number of particles traversing space,
each at its own velocity and direction,
but adjusting its trajectory based on its own experience and the results of its peers.
The PSO algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012).
It is implemented in the @ref pmsco.optimizers.swarm module.
The general parameters of the algorithm are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | Recommended 20..50 |
| position_constrain_mode | | default bounce | Resolution of domain limit violations. |
| seed_file | --seed-file | a file path, default none | |
| seed_limit | --seed-limit | 0..pop_size | |
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The model space attributes have the following meaning:
| Parameter | Description |
| --- | --- |
| start | Start value of particle 0 in first iteration. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Not used. |
\subsubsection sec_opt_seed Seeding a population
By default, one particle is initialized with the start value declared with the model space,
and the other ones are initialized at random positions in the model space.
You may initialize more particles of the population with specific values by providing a seed file.
The seed file must have a similar format as the result `.dat` files
with a header line specifying the column names and data rows containing the values for each particle.
A good practice is to use a previous `.dat` file and remove unwanted rows.
The `.dat` file from a previous optimization job can be used as is to continue the optimization,
also in a different optimization mode.
The seeding procedure can be tweaked by several optimizer parameters (see above).
PMSCO normally loads the first rows up to population size - 1 or up to the `seed_limit` parameter,
whichever is lower.
If an `_rfac` column is present, the file is first sorted by R-factor and only the best models are loaded.
Models that resulted in an R-factor above the `rfac_limit` parameter are ignored in any case.
In the first iteration of the optimization run, the models loaded from the seed file are re-calculated by default.
This may waste CPU time if the calculation is run under the same conditions
and would result in exactly the same R-factor,
as is the case if the seed is used to continue a previous optimization, for example.
In these situations, the `recalc_seed` parameter can be set to False,
and PMSCO will use the R-factor value from the seed file rather than calculating the model again.
\subsubsection sec_opt_patch Patching a running optimization
While an optimization job is running, the user can manually patch the population with arbitrary values,
for instance, to kick the population out of a local optimum or to drive it to a less sampled parameter region.
To patch a running population, prepare a population file named `pmsco_patch.pop` and copy it to the work directory.
The patch file must have the same format as the result `.dat` files
with a header line specifying the column names and data rows containing the values.
It should contain as many rows as particles to be patched but not more than the size of the population.
The columns must include a `_particle` column and the model parameters to be changed.
The `_particle` column specifies the index of the particle that is patched (ranging from 0 to population size - 1).
Parameters that should remain unaffected can be left out,
extra columns including `_gen`, `_rfac` etc. are ignored.
PMSCO checks the file for syntax errors and ignores it if errors are present.
Individual parameter values that lie outside the domain boundary are silently ignored.
Successful or failed patching is logged at warning level.
PMSCO keeps track of the time stamp of the file and re-applies the patch whenever the time stamp has changed.
\attention Since each change of time stamp may trigger patching,
do not edit the patch file in the working directory
to prevent it from being read in an unfinished state or multiple times!
\subsection sec_opt_genetic Genetic optimization
The genetic algorithm evolves a population of individuals
by a combination of inheritance, crossover, mutation
and selection in analogy to biological evolution.
The _genes_ are in this case the model parameters,
and selection occurs based on R-factor.
The genetic algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012).
It is implemented in the @ref pmsco.optimizers.genetic module.
The genetic optimization is helpful in the first stage of an optimization
where a large parameter space needs to be sampled
and fast convergence on a small part of the parameter space is less desirable
as it might catch on a local optimum.
On the other hand, convergence near the optimum is slower than in the particle swarm.
The genetic optimization should be run with a large number of iterations
rather than a large population size.
The general parameters of the genetic algorithm are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | Recommended 10..40 |
| mating_factor | | 1..pop_size, default 4 | |
| strong_mutation_probability | | 0..1, default 0.01 | Probability that a parameter undergoes a strong mutation. |
| weak_mutation_probability | | 0..1, default 1 | Probability that a parameter undergoes a weak mutation. This parameters should be left at 1. Lower values tend to produce discrete parameter values. Weak mutations can be tuned by the step domain parameters. |
| position_constrain_mode | | default random | Resolution of domain limit violations. |
| seed_file | --seed-file | a file path, default none | |
| seed_limit | --seed-limit | 0..pop_size | |
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The model space attributes have the following meaning:
| Parameter | Description |
| --- | --- |
| start | Seed model. The start values are copied into particle 0 of the initial population. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Standard deviation of the Gaussian distribution of weak mutations. The step should not be much lower than the the parameter range divided by the population size and not greater than one third of the parameter range. |
The population of the genetic optimizer can be seeded and patched in the same way as the particle swarm,
cf. sections @ref sec_opt_seed and @ref sec_opt_swarm.
\subsection sec_opt_grid Grid search
The grid search algorithm samples the parameter space at equidistant steps.
It is implemented in the @ref pmsco.optimizers.grid module.
The model space attributes have the following meaning.
The order of calculations is random so that results from different parts of the model space become available early.
| Parameter | Description |
| --- | --- |
| start | Values of fixed parameters. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. If abs(max - min) < step/2 , the parameter is kept constant. |
| step | Step size (distance between two grid points). If step <= 0, the parameter is kept constant. |
\subsection sec_opt_gradient Gradient search
Currently not implemented.
\subsection sec_opt_table Table scan
The table scan calculates models from an explicit table of model parameters.
It can be used to recalculate models from a previous optimization run on other experimental data,
as an interface to external optimizers,
or as a simple input of manually edited model parameters.
It is implemented in the @ref pmsco.optimizers.table module.
The table can be stored in an external file that is specified on the command line,
or supplied in one of several forms by the custom project class.
The table can be left unchanged during the calculations,
or new models can be added on the go.
Duplicate models are ignored.
@attention Because it is not easily possible to know when the table file is read,
if you do modify the table file while calculations are running,
1. Do not keep the file locked for longer than a second.
2. Append new models to the end of the table rather than overwriting previous ones.
3. Delete lines only if you're sure that they are not needed any more.
The general parameters of the table scan are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line or in the project class (depending on how the project class is implemented).
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | Number of models in a generation (calculated in parallel). In table mode, this parameter is not so important and can be left at the default. It has nothing to do with table size. |
| table_file | --table-file | a file path, default none | |
The model space attributes have the following meaning.
Models that violate the parameter range are not calculated.
| Parameter | Description |
| --- | --- |
| start | Not used. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Not used. |
\subsection sec_opt_single Single model
The single model optimizer calculates the model defined by domain.start.
| Parameter | Description |
| --- | --- |
| start | Values of model parameters. |
| min | Not used. |
| max | Not used. |
| step | Not used. |
*/

514
docs/src/project.dox Normal file
View File

@@ -0,0 +1,514 @@
/*! @page pag_project Setting up a new project
\section sec_project Setting up a new project
This topic guides you through the setup of a new project.
Be sure to check out the examples in the projects folder
and the code documentation as well.
The basic steps are:
1. Create a new package folder under `pmsco/projects`.
To keep your code and PMSCO separate, you are suggested to start your own pmsco/projects tree
in a convenient location separate from the PMSCO source code.
2. Add the parent directory of your pmsco/projects tree to the Python path.
3. In the new folder, create a Python module for the project (subsequently called _the project module_).
4. In the project module, define a cluster generator class which inherits from @ref pmsco.cluster.ClusterGenerator.
5. In the project module, define a project class which inherits from @ref pmsco.project.Project.
6. Create one or more run files.
The basic steps listed above are recommended and explained in the following.
In previous versions, other mechanisms of project invocation were available.
They are now obsolete.
\subsection sec_packages Namespace packages
[Python namespace packages](https://realpython.com/python-namespace-package/) provide an easy way
to inject project modules into the PMSCO namespace
while their source files are kept separate from the core PMSCO packages.
This way, PMSCO and the project modules can be under separate version control.
Namespace packages work by extending the Python module search path.
The module loader looks for packages in every entry of the search path
and does not stop at the first match as it would do for a regular package.
The recommended folder structure is:
~~~~~~
pmsco-projects/
+-- pyproject.toml
+-- pmsco/
+-- projects/
+-- project1/
+-- project1.py
+-- run1.json
+-- ...
+-- project2/
+-- ...
~~~~~~
In place of `pmsco-projects`, `project1`, `project2`, `run1`, you should use distinct names.
The two levels `pmsco` and `projects` should be left as is.
If you now include `pmsco-projects` in the Pyton path,
all of your projects become available within the `pmsco` namespace, i.e.,
you can `import pmsco.projects.project1.project1` in Python.
Furthermore, you can call the module in a run-file without specifying a file path.
You may install multiple project packages if needed.
The recommended way to add `pmsco-projects` to the Python path is by an editable installation.
This will allow you to keep editing your project sources in place.
1. Place your project files in a directory tree similar to `pmsco-projects/pmsco/projects/project1/`.
The `pmsco/projects` level is mandatory as a part of the path.
Replace `pmsco-projects` and `project1` by your own choice.
2. Be sure not to create any `__init__.py` files in this directory tree.
3. Copy the `pyproject.toml` file from the PMSCO source into your `pmsco-projects` and adjust its contents.
At least give the package a distinct name.
4. Select another build backend if necessary.
The default [uv_build](https://docs.astral.sh/uv/concepts/build-backend/) is recommended for pure Python projects.
5. 'Install' the project locally.
With uv, call `uv sync --active --inexact`
while you are in the directory that contains the `pyproject.toml` file.
In plain pip the corresponding command would be
`pip install --editable .`.
6. Check that you can `import pmsco.projects.project1.project1` (or whatever your project is called) in a Python shell.
If you encounter problems importing the pmsco modules, check the Python path in a Python shell.
It must contain the `site-packages` directory of your Python environment.
Make sure it does not contain any pmsco or project source directory explicitly.
Also make sure that you don't have any `__init__.py` files in your project tree,
and do not use explicit paths to pmsco or your project anywhere in your source code or shell configuration files.
Be careful not to install packages multiple times in different locations.
In case of trouble, set up a fresh environment.
\subsection sec_project_module Project module
A skeleton of the project module file (with some common imports) may look like this:
~~~~~~{.py}
import logging
import math
import numpy as np
import periodictable as pt
from pathlib import Path
import pmsco.cluster
import pmsco.data
import pmsco.dispatch
import pmsco.elements.bindingenergy
import pmsco.project
logger = logging.getLogger(__name__)
class MyClusterGenerator(pmsco.cluster.ClusterGenerator):
def create_cluster(self, model, index):
clu = pmsco.cluster.Cluster()
# ...
return clu
def count_emitters(self, model, index):
# ...
return 1
class MyProject(pmsco.project.Project):
def __init__(self):
super().__init__()
# ...
self.cluster_generator = MyClusterGenerator(self)
def create_model_space():
spa = pmsco.project.ModelSpace()
# ...
return spa
def create_params(self, model, index):
par = pmsco.project.CalculatorParams()
# ...
return par
~~~~~~
The main purpose of the `MyProject` class is to bundle the project-specific calculation parameters and code.
The purpose of the `MyClusterGenerator` class is to produce atomic clusters as a function of a number of model parameters.
For the project to be useful, some of the methods in the skeleton above need to be implemented.
The individual methods are discussed in the following.
Further descriptions can be found in the documentation of the code.
\subsection sec_project_cluster Cluster generator
The cluster generator is a project-specific Python object that produces a cluster, i.e., a list of atomic coordinates,
based on a small number of model parameters whenever PMSCO requires it.
The most important method of a cluster generator is `create_cluster`.
At least this method must be implemented for a functional cluster generator.
A generic `count_emitters` method is implemented in the base class.
It needs to be overridden if inequivalent emitters should be calculated in parallel.
\subsubsection sec_project_cluster_create Cluster definition
The `create_cluster` method takes the model parameters (a dictionary)
and the task index (a pmsco.dispatch.CalcID, cf. @ref pag_concepts_tasks) as arguments.
Given these arguments, it creates and fills a @ref pmsco.cluster.Cluster object.
See @ref pmsco.cluster.ClusterGenerator.create_cluster for details on the method contract.
As an example, have a look at the following simplified excerpt from the `twoatom` demo project.
~~~~~~{.py}
class TwoatomCluster(ClusterGenerator):
# ...
def create_cluster(self, model, index):
# access model parameters
# dAB - distance between atoms in Angstroms
# th - polar angle in degrees
# ph - azimuthal angle in degrees
r = model['dAB']
th = math.radians(model['th'])
ph = math.radians(model['ph'])
# prepare a cluster object
clu = pmsco.cluster.Cluster()
# the comment line is optional but can be useful
clu.comment = "{0} {1}".format(self.__class__, index)
# set the maximum radius of the cluster (outliers will be ignored)
clu.set_rmax(r * 2.0)
# calculate atomic vectors
dx = r * math.sin(th) * math.cos(ph)
dy = r * math.sin(th) * math.sin(ph)
dz = r * math.cos(th)
a_top = np.array((0.0, 0.0, 0.0))
a_bot = np.array((-dx, -dy, -dz))
# add an oxygen atom at a_top position and mark it as emitter
clu.add_atom('O', a_top, 1)
# add a copper atom at a_bot position
clu.add_atom('Cu', a_bot, 0)
# pass the created cluster to the calculator
return clu
~~~~~~
In this example, two atoms are added to the cluster.
The @ref pmsco.cluster.Cluster class provides several methods to simplify the task,
such as adding layers or bulk regions, rotation, translation, trim, emitter selection, etc.
Please refer to the documentation of its code for details.
It may also be instructive to have a look at the demo projects.
The main purposes of the cluster object are to store an array of atoms and to read/write cluster files in a variety of formats.
For each atom, the following properties are stored:
- sequential atom index (1-based, maintained by cluster code)
- atom type (chemical element number)
- chemical element symbol from periodic table
- x coordinate of the atom position
- y coordinate of the atom position
- z coordinate of the atom position
- emitter flag (0 = scatterer, 1 = emitter, default 0)
- charge/ionicity (units of elementary charge, default 0)
- scatterer class (default 0)
All of these properties except the scatterer class can be set by the `add_xxxx` methods of the cluster.
The scatterer class is used internally by the atomic scattering factor calculators.
Whether the charge/ionicity is used, depends on the particular calculators, EDAC does not use it, for instance.
\note You do not need to take care how many emitters a calculator allows,
or whether the emitter needs to be at the origin or the first place of the array.
These technical aspects are handled by PMSCO code transparently.
\subsubsection sec_project_cluster_domains Domains
Domains refer to regions of inequivalent structure in the probing region.
This may include regions of different orientation, different lattice constant, or even different structure.
The cluster methods read the requested domain from the `index.domain` argument.
This is an index into the @ref pmsco.project.Project.domains list where each item is a dictionary
that holds additional, invariable structural parameters.
A common case are rotational domains.
In this case, the list of domains may look like `[{"zrot": 0.0}, {"zrot": 60.0}]`, for example,
and the `create_cluster` method would include additional code to rotate the cluster:
~~~~~~{.py}
def create_cluster(self, model, index):
# filling atoms here
# ...
dom = self.domains[index.domain]
try:
z_rot = dom['zrot']
except KeyError:
z_rot = 0.0
if z_rot:
clu.rotate_z(z_rot)
# selecting emitters
# ...
return clu
~~~~~~
Depending on the complexity of the system, it is advisable to split the code into a separate method for each domain.
The @ref pmsco.project.Project class includes generic code to add intensities of domains incoherently
(cf. @ref pmsco.project.Project.combine_domains).
In this case, the model space should contain parameters 'wdom0', 'wdom1', etc.,
that define the weights of domain 0, 1, etc.
To avoid correlations between parameters, one domain must have a fixed weight:
Typically, 'wdom0' is left undefined and defaults to 1.
\subsubsection sec_project_cluster_emitters Emitter configurations
If a project uses a large cluster and/or many emitters,
it may be more efficient to generate emitter-specific cluster configurations,
for instance to leverage process parallelization,
or to produce small, local clusters around the emitter site.
This concept is called _emitter configurations_ and explained in detail in @ref pag_concepts_emitter.
To implement emitter configurations, override the `count_emitters` method to return the number of emitter configurations.
In the simplest case, this is the number of inequivalent emitters:
~~~~~~{.py}
def count_emitters(self, model, index):
index = index._replace(emit=-1)
clu = self.create_cluster(model, index)
return clu.get_emitter_count()
~~~~~~
Next, modify the `create_cluster` method to check the emitter index (`index.emit`).
If it is -1, the method must return the full cluster with all inequivalent emitters marked.
If it is positive, only the corresponding emitter configuration must be marked.
For example, if each emitting atom represents a separate emitter configuration:
~~~~~~{.py}
def create_cluster(self, model, index):
# filling atoms here
# ...
# select all possible emitters (atoms of a specific element) in a cylindrical volume
# idx_emit is an array of atom numbers (0-based atom index)
idx_emit = clu.find_index_cylinder(origin, r_xy, r_z, self.project.scans[index.scan].emitter)
# if PMSCO asks for a specific emitter, restrict the array index:
if index.emit >= 0:
idx_emit = idx_emit[index.emit]
# mark the selected emitters
clu.data['e'][idx_emit] = 1
return clu
~~~~~~
Now, the individual emitter configurations are calculated in separate tasks
which can run in parallel in a multi-process environment.
Note that the processing time of EDAC scales linearly with the number of emitters.
\subsection sec_project_project Project class
Most commonly, a project class overrides the `__init__`, `create_model_space` and `create_params` methods.
Most other inherited methods can be overridden optionally,
for instance `validate`, `setup`, `calc_modulation`, `rfactor`,
as well as the combine methods `combine_rfactors`, `combine_domains`, `combine_emitters`, etc.
This introduction shall focus on the three most important methods.
\subsubsection sec_project_project_init Initialization and defaults
The `__init__` method defines and initializes project properties with default values.
It may also redefine properties of the base class.
The following code is just an example to give some ideas.
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def __init__(self):
# call the inherited method first
super().__init__()
# re-define an inherited property
self.directories["data"] = Path("/home/pmsco/data")
# define a scan dictionary
self.scan_dict = {}
# fill the scan dictionary
self.build_scan_dict()
# create the cluster generator
self.cluster_generator = MyClusterGenerator(self)
# declare the list of domains (at least one is required)
self.domains = [{"zrot": 0.}]
def build_scan_dict(self):
self.scan_dict["empty"] = {"filename": "${pmsco}/projects/common/empty-hemiscan.etpi",
"emitter": "Si", "initial_state": "2p3/2"}
self.scan_dict["Si2p"] = {"filename": "${data}/xpd-Si2p.etpis",
"emitter": "Si", "initial_state": "2p3/2"}
~~~~~~
A scan dictionary is one way to specify locations and metadata of experimental files centrally in the project code.
The scan can then be selected by the dictionary key rather than copying file locations.
Note that all public attributes can be assigned from a run file.
This happens after the `__init__` method.
The values set by `__init__` serve as default values.
\subsubsection sec_project_project_space Model space
The model space defines the keys and value ranges of the model parameters.
There are three ways to declare the model space in order of priority:
1. Declare the model space in the run-file.
2. Assign a ModelSpace to the self.model_space property directly in the `__init__` method.
3. Implement the `create_model_space` method.
The third way may look like this:
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def create_model_space(self):
# create an empty model space
spa = pmsco.project.ModelSpace()
# add parameters
spa.add_param('dAB', 2.05, width=0.25, step=0.05)
spa.add_param('th', 15.00, 0.00, 30.00, 1.00)
spa.add_param('ph', 90.00)
spa.add_param('V0', 21.96, width=10.0, step=1.0)
spa.add_param('Zsurf', 1.50)
spa.add_param('wdom1', 0.5, 0.10, 10.00, 0.10)
# return the model space
return spa
~~~~~~
This code declares six model parameters: `dAB`, `th`, `ph`, `V0`, `Zsurf` and `wdom1`.
Three of them are structural parameters (used by the cluster generator above),
two are used by the `create_params` method (see below),
and `wdom1` is used in @ref pmsco.project.Project.combine_domains
while summing up contributions from different domains.
The values in the arguments list correspond to the start value (initial guess),
the lower and upper boundaries of the value range,
and the step size for optimizers that require it.
If just one value is given the parameter is held constant during the optimization.
The range can, alternatively, be specified by the `width` argument.
A similar declaration in a run-file could look like as follows (some parameters omitted for brevity).
Parameter values can be numeric constants,
or simple Python math expressions in double quotes.
~~~~~~{.py}
{
"project": {
// ...
"model_space": {
"dAB": {
"start": "2.0 / math.cos(math.radians(15.0))",
"width": 0.25,
"step": 0.05
},
"th": {
"start": 15.0,
"min": 0.0,
"max": 30.0,
"step": 1.0
},
"Zsurf": {
"start": 1.50
}
// ...
}
}
}
~~~~~~
\subsubsection sec_project_project_params Calculation parameters
Non-structural parameters that are needed for the input files of the calculators are passed
in a @ref pmsco.project.CalculatorParams object.
This object is created and filled in the `create_params` method of the project class.
The following example is from the `twoatoms` demo project:
~~~~~~{.py}
class MyProject(pmsco.project.Project):
def create_params(self, model, index):
params = pmsco.project.CalculatorParams()
# meta data
params.title = "two-atom demo"
params.comment = "{0} {1}".format(self.__class__, index)
# initial state and binding energy
initial_state = self.scans[index.scan].initial_state
params.initial_state = initial_state
emitter = self.scans[index.scan].emitter
params.binding_energy = pt.elements.symbol(emitter).binding_energy[initial_state]
# experimental setup
params.polarization = "H"
params.polar_incidence_angle = 60.0
params.azimuthal_incidence_angle = 0.0
params.experiment_temperature = 300.0
# material parameters
params.z_surface = model['Zsurf']
params.work_function = 4.5
params.inner_potential = model['V0']
params.debye_temperature = 356.0
# multiple-scattering parameters (EDAC)
params.emitters = []
params.lmax = 15
params.dmax = 5.0
params.orders = [25]
return params
~~~~~~
Most of the code is generic and can be copied to other projects.
Only the experimental and material parameters need to be adjusted.
Other properties can be changed as needed, see @ref pmsco.project.CalculatorParams.
\subsection sec_project_args Passing run-time parameters
The recommended way of passing calculation parameters is via @ref pag_runfile.
Run-files allow for a complete separation of code and data in a generic and flexible way.
Program code can be managed by a version control system,
and run-files can be stored along with the results.
This simplifies the reproduction of previous calculations and documentation of the workflow.
For testing and simple projects, it is possible to hard-code all parameters in the project class.
\subsubsection sec_project_args_runfile Setting up a run-file
The usage and format of run-files is described in detail under @ref pag_runfile.
\subsubsection sec_project_args_code Hard-coded arguments
Though it's normally recommended to declare all parameters in the run-file,
parameter values can also be hard-coded in the constructor and/or the validate method of the project class.
Which method to use depends on the processing stage.
The constructor can set default values for rarely changing parameters.
The declarations in the run-file override the defaults from the constructor.
If some parameters need adjusting _after_ the run-file has been loaded,
this can be done in the validate` method.
The call sequence of the methods is as follows.
1. `Project.__init__`:
The constructor is usually overridden by the project.
The constructor must call the superclass before applying its values.
2. `Project.set_properties`:
Sets the parameters from the run-file and resolves class names.
This method can be overridden if additional classes need resolving after loading the run-file.
It must call the superclass.
3. `Project.validate`: Parameters are validated, i.e., checked and made consistent.
Handler classes are resolved.
The `validate` method or its sub-methods can be overridden by the project.
The inherited method should be called.
*/

135
docs/src/reports.dox Normal file
View File

@@ -0,0 +1,135 @@
/*! @page pag_reports Reports
\section sec_reports Reports
The main output of PMSCO is the model parameters to R-factor mapping.
By default, it is produced in the form of a text file (.dat) as well as an sqlite3 database file (.db).
Graphical representations of the result data, called _reports_ in PMSCO, can be produced automatically at run-time or
manually after the calculation has ended.
PMSCO provides a number of pre-defined reports as well as an interface for custom reports.
Essentially, a report is defined by a Python class which derives from `pmsco.reports.base.ProjectReport`.
Instances of reports are added to the project's `reports` list during initialization of the calculation job.
They are called by the calculation handlers whenever a new model-level result is available in the database.
While reports typically produce graphics files for diagnostics,
report classes could basically produce any derived data including data files in different formats.
By default, no report is produced during a project run.
There are several ways to generate reports:
- Add instances of reports to the `reports` list of the project object.
This can be done in the project code or in the @ref pag_runfile.
One or multiple reports (of different classes) can be added and configured.
- Some report modules have their own command line interface.
This allows you to produce a report at any time during or after the project run.
- Lastly, all reports are Python classes and can be instantiated and executed in a Python shell.
The remainder of this page describes some of the pre-defined reports and their configuration parameters (attributes).
@note Reporting is still under development.
The configuration parameters and behaviour is subject to change, and the documentation may be partially outdated.
Be sure to check the in-line documentation as well as the source code for the latest information.
\subsection sec_reports_common Common Parameters
The reports share some common parameters which may, however, be used differently or ignored by some reports.
| Key | Values | Description |
| --- | --- | --- |
| `filename_format` | template string using `${key}`-type placeholders | Template string for file names of reports. Possible placeholders are listed below. |
| `title_format` | template string using `${key}`-type placeholders | Template string for graph titles. Possible placeholders are listed below. |
| `canvas` | string. default: `matplotlib.backends.backend_agg.FigureCanvasAgg` (PNG) | A matplotlib figure canvas such as FigureCanvasAgg, FigureCanvasPdf or FigureCanvasSVG. |
The `filename_format` and `title_format` attributes are template strings which can contain `${key}` type placeholders.
placeholders are replaced according to the following table.
Some of these values may not be available if you call the reports outside of an optimization run
(e.g., from the command line of a report module).
| Key | Description |
| --- | --- |
| `base` | Base file name. Default: job name |
| `mode` | optimization mode |
| `job_name` | job name |
| `project_name` | project name |
| any directories key | corresponding directories value |
| any job_tags key | corresponding job_tags value |
\subsection sec_reports_convergence Convergence Plot
The convergence plot is a violin plot where each violin represents the R-factor distribution of one generation.
The minimum, maximum and mean values are marked, and the distribution is indicated by the body.
Convergence plots are suitable for genetic or swarm optimizations.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.ConvergencePlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
\subsection sec_reports_genetic Genetic Chart
A genetic chart is a pseudo-colour representation of the coordinates of each individual in the model space.
The chart shows the amount of diversity in the population
and - by comparing charts of different generations - the changes due to mutation.
The axes are the model parameters (x) and particle number (y).
The colour is mapped from the relative parameter value within the parameter range.
Genetic charts are suitable for genetic or swarm optimizations.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.GeneticPlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
| cmap | string: 'viridis', 'plasma' (default), 'inferno', 'magma', 'cividis' | Name of colour map supported by matplotlib. |
| params | list of model parameter names | |
In addition to the common template substitutions,
the genetic chart report replaces the following placeholders
of the `filename_format` and `title_format` template strings.
| Key | Description |
| --- | --- |
| `gen` | Generation index (population reports only) |
\subsection sec_reports_swarm Particle Swarm Plot
The particle swarm plot shows the current positions and velocities of particles projected onto two dimensions.
The plot contains three elements:
- a pseudo-color scatter plot of all R-factors in the background,
- a scatter plot of particle positions.
- a quiver plot indicating the velocities of the particles.
Particle swarm plots are suitable in particle swarm optimization mode only.
| Key | Values | Description |
| --- | --- | --- |
| __class_name__ | pmsco.reports.population.SwarmPlot | |
| filename_format | template string using `${key}`-type placeholders | See common section. |
| title_format | template string using `${key}`-type placeholders | See common section. |
| cmap | string: 'viridis', 'plasma' (default), 'inferno', 'magma', 'cividis' | Name of colour map supported by matplotlib. |
| params | nested list of pairs of model parameter names | |
In addition to the common template substitutions,
the particle swarm plot report replaces the following placeholders
of the `filename_format` and `title_format` template strings.
| Key | Description |
| --- | --- |
| `gen` | Generation index (population reports only) |
| `param0` | Parameter name 0 (population reports only) |
| `param1` | Parameter name 1 (population reports only) |
\subsection sec_reports_misc Miscellaneous
To make a video from swarm or genetic plots, you may use ffmpeg on Linux:
~~~~~~{.sh}
ffmpeg -framerate 5 -i basename-%00d.geneticplot.png -c:v libx264 -profile:v high -crf 20 -pix_fmt yuv420p basename.geneticplot.mp4
~~~~~~
*/

506
docs/src/runfile.dox Normal file
View File

@@ -0,0 +1,506 @@
/*! @page pag_runfile Run File
\section sec_runfile Run File
This section describes the format of a run-file.
Run-files are a flexible way of passing arguments to a PMSCO process.
The benefits are:
- contain all essential parameters to repeat a calculation - no need to remember or record the command line
- avoid cluttering up the command line or frequent changes of source code
- can be versioned or stored separately from the code, maintain a single file or multiple files - up to the user
- any property and sub-property of the project object can be assigned in a generic way - even custom properties that are unknown to PMSCO
- no necessity for the project code to parse the command line
- schema validation can help to find syntax errors while editing
\subsection sec_runfile_how How It Works
Run-files are text files in machine and human readable [JSON](https://en.wikipedia.org/wiki/JSON) format.
In PMSCO, run-files contain dictionaries of parameters to be passed to the project object.
For the calculations, internally, the project object is the main container of calculation parameters, model objects and input data.
Upon launching PMSCO, a generic parser reads the run-file,
constructs the project object from the specified custom project class
and assigns the attributes defined in the run-file.
Run-files are a sort of script that assign data to the project.
The parser does not expect specific data types or classes.
It merely copies data items to the project attributes of the same name.
The validation and interpretation of the data is up to the project object.
The parser handles the following situations:
- Strings, numbers as well as dictionaries and lists of simple objects are assigned directly to project attributes.
If the project class declares a setter method for the attribute, the setter is called.
Else, the existing attribute is overwritten.
Setters can execute custom code to validate the data value.
- If specified in the run-file, the parser creates objects from classes in the namespace of the project module
and recursively assigns their properties.
\note There are no implicit checks of correctness of the assigned data objects!
The author of the run-file must make sure that the run-file is compatible with the project class,
else the calculation process might fail.
There are three ways to check assigned attributes before the calculations are started.
All have to be implemented explicitly by the project maintainer:
1. The run-file can be validated against a JSON schema before launching PMSCO (see below).
Schema validation may catch some obvious mistakes
but is not complete in the sense that it cannot guarantee error-free execution of the project code.
2. The classes used with run-files define property setters.
The setters can raise an exception or post an error in the log.
(The latter won't stop the calculation process.)
3. The project class implements a validation method to check and fix important or error-prone attributes.
It can write warnings and errors to the log, or raise an exception if the process should be aborted.
\subsection sec_runfile_general General File Format
Run-files must adhere to the [JSON](https://en.wikipedia.org/wiki/JSON) format.
Specifically, a JSON file can declare dictionaries, lists and simple objects
such as strings, numbers and `null`.
The syntax of these basic elements is similar to Python source code (there are some differences, though).
At the top level, a PMSCO run-file contains a dictionary with up to two items:
1. The _project_ item is the most important, it is described in the following under @ref sec_runfile_project.
2. The _schedule_ item is an optional section for passing the parameters to a job queue of a computing cluster.
See @ref sec_runfile_schedule .
\subsection sec_runfile_schema Schema
The structure of a JSON file can be described in a _schema_ file that can be used to check the syntax and structure programmatically.
The `schema/runfile.schema.json` file of the PMSCO distribution describes the structure of a run-file as well as common properties of the project.
The schema is, however, rather basic and does not cover all parameters, conditional cases or custom project properties.
A run-file can be easily validated against the schema while editing in the PyCharm IDE.
Alternatively, the jsonschema validator from the Python distribution can be used on the command line.
\subsection sec_runfile_project Project Specification
The following minimum run-file from the twoatom project demonstrates how to specify the project:
~~~~~~{.py}
{
"project": {
"__module__": "twoatom",
"__class__": "TwoatomProject",
"mode": "single",
"job_name": "twoatom0001"
}
}
~~~~~~
Here, the `project` keyword denotes the dictionary that is used to construct the project object.
Within the project dictionary, the `__module__` key selects the Python module file that contains the project code,
and `__class__` refers to the name of the actual project class.
Further dictionary items correspond to attributes of the project class.
The module name is the same as would be used in a Python import statement.
It must be findable on the Python path.
Alternatively, a file path may be specified.
PMSCO ensures that the directory containing the `pmsco` and `projects` sub-directories is on the Python path.
The class name must be in the namespace of the loaded module.
As PMSCO starts, it imports the specified module,
constructs an object of the specified project class,
and assigns any further items to project attributes.
In the example above, it creates an object of type `TwoatomProject` from the `twoatom` module
and assigns `single` to the `mode` property and `twoatom0001` to the `job_name` property.
Any attributes not specified in the run-file remain at their default values
that were set by the `__init__` constructor of the project class.
Note that parameter names must start with an alphabetic character, else they are ignored
(useful for comments as JSON does not have a syntax for comments).
Also note that PMSCO does not spell-check parameter names.
The parameter values are just written to the corresponding object attribute.
If a name is misspelled, the value will be written to the wrong attribute.
PMSCO carries out only some most important checks on the given parameter values.
Incorrect values may lead to improper operation or exceptions later in the calculations.
The project class can explicitly check and fix important or error-prone attributes, or report errors.
The following sub-sections describe the most common properties of the project class.
\subsubsection sec_runfile_common Common Arguments
The following table lists some important parameters controlling the calculations.
They are declared in the pmsco.projects.Project class.
| Key | Values | Description |
| --- | --- | --- |
| mode | `single` (default), `grid`, `swarm`, `genetic`, `table`, `test`, `validate` | Operation mode. `validate` can be used to check the syntax of the run-file, the process exits before starting calculations. |
| directories | dictionary | This dictionary lists common file paths used in the project. It contains keys such as `home`, `project`, `output` (see documentation of Project class in pmsco.project). |
| output_dir | path | Shortcut for directories["output"] |
| data_dir | path | Shortcut for directories["data"] |
| job_name | string, must be a valid and unique file name (see note below) | Base name for all produced output files. It is recommended to set a unique name for each calculation run. Do not include a path. The path can be set in _output_dir_. |
| cluster_generator | dictionary | Class name and attributes of the cluster generator. See below. |
| atomic_scattering_factory | string<br>Default: InternalAtomicCalculator from pmsco.calculators.calculator | Class name of the atomic scattering calculator. This name must be in the namespace of the project module. |
| multiple_scattering_factory | string<br>Default: EdacCalculator from pmsco.calculators.edac | Class name of the multiple scattering calculator. This name must be in the namespace of the project module. |
| model_space | dictionary | See @ref sec_runfile_space below. |
| domains | list of dictionaries | See @ref sec_runfile_domains below. |
| scans | list of dictionaries | See @ref sec_runfile_scans below. |
| optimizer_params | dictionary | See @ref sec_runfile_optimizer below. |
\note The *job name* parameter appears most visibly as the prefix of output file names.
It is also registered in the `jobs` table of the results database (if used),
and it is used to identify the job with a job scheduling system.
For these reasons, it is important that the job name be unique within the respective subsystem.
Specifically, you need to *provide a new job name each time you start pmsco*, otherwise the job may fail.
It may be more natural to specify the job name on the command line using the `-o` argument
rather than changing the run file every time.
Unfortunately, PMSCO cannot auto-generate, auto-increment or verify the job name.
File names specified in a runfile can include an explicit path or a placeholder.
Placeholders have the format `${key}` where `key` must be one of the keys of the `directories` dictionary.
The placeholder will then be replaced by the corresponding value before the calculation starts
(as a part of the pmsco.project.Project.validate method).
The `directories` dictionary can be filled by the project class or in the runfile.
In addition, a number of keys are defined by PMSCO and can be used as placeholders in other directories and file paths.
| Key | Type | Description |
| --- | --- | --- |
| data | absolute | Directory with experimental data. Must be set by user if needed. |
| home | absolute | Home directory of the current user |
| pmsco | absolute | Directory that contains the loaded pmsco.py module. Note: This may be in a site packages directory. |
| output | absolute | Output directory. Must be set by the user. |
| project | absolute | Directory where the project module is located. |
| project_name | relative | Name of the project. By default, the name of the project class. |
| job_name | relative | Name of the calculation job. |
| mode | relative | Calculation mode |
| report | absolute | Report directory. Defaults to `${output}/report`. |
| run | absolute | Directory where the runfile is located (if used). |
| temp | absolute | Directory for temporary files. Currently not used. |
| work | absolute | Current working directory |
Placeholders of absolute paths must be used at the beginning of a path.
Relative paths can be used at any position in a file path.
Some of the keys may have empty values if PMSCO was loaded in a non-standard way.
For verification of the path resolution, all directories are printed to the log file at WARNING level (default).
The following table lists some common control parameters and metadata
that affect the behaviour of the program but do not affect the calculation results.
The job metadata is used to identify and describe a job in the results database if requested.
| Key | Values | Description |
| --- | --- | --- |
| db_file | new or existing file path or `:memory:` | SQLite3 database file to receive the optimization results. If the database exists, results are inserted under the given job name. If it doesn't exist, a new file is created. If the attribute is `:memory:`, an in-memory database is used internally and flushed at the end of processing. |
| job_tags | dictionary of strings | User-specified job tags in key-value format (metadata). |
| description | string | Description of the calculation job (metadata) |
| time_limit | decimal number<br>Default: 24. | Wall time limit in hours. The optimizers try to finish before the limit. This cannot be guaranteed, however. |
| keep_files | list of file categories | Output file categories to keep after the calculation. Multiple values can be specified and must be separated by spaces. By default, cluster and model (simulated data) of a limited number of best models are kept. See @ref sec_runfile_files below. |
| keep_best | integer number<br>Default: 10 | number of best models for which result files should be kept. |
| keep_levels | integer number<br>Default: 1 | numeric task level down to which files are kept. 1 = scan, 2 = domain, 3 = emitter. |
| log_level | DEBUG, INFO, WARNING, ERROR, CRITICAL | Minimum level of messages that should be added to the log. Empty string turns off logging. |
| log_file | file system path<br>Default: job_name + ".log". | Name of the main log file. Under MPI, the rank of the process is inserted before the extension. The log name is created in the working directory. |
\subsubsection sec_runfile_space Model Space
The `model_space` parameter is a dictionary of model parameters.
The key is the name of the parameter as used by the cluster and input-formatting code,
the value is a dictionary holding the `start`, `min`, `max`, `step` values to be used by the optimizer.
Instead of `min` and `max` you may declare the `width`, which will center the space on the start value.
All parameter values can be declared as numbers or as simple Python expressions in double quotes.
Expressions are evaluated by the Python `eval` function.
All functions in the namespace of the project module.
Note that you have to import the `math` or `numpy` modules in your project module
if you want to use their functions.
~~~~~~{.py}
{
"project": {
// ...
"model_space": {
"dAB": {
"start": 2.109,
"min": "2.109 - 0.1",
"max": "2.109 + 0.1",
"step": 0.05
},
"pAB": {
"start": "4 * 3.56 / math.sqrt(3.0)",
"width": 4.0,
"step": 0.5
},
// ...
}
}
}
~~~~~~
Alternatively, the `model_space` can be declared as a `ModelSpace` object.
However, this shall not be described in detail here.
\subsubsection sec_runfile_domains Domains
Domains is a list of dictionaries.
Each dictionary holds keys describing the domain to the cluster and input-formatting code.
The meaning of these keys is up to the project.
An example:
~~~~~~{.py}
{
"project": {
// ...
"domains": [
{"surface": "Te", "doping": null, "zrot": 0.0},
{"surface": "Te", "doping": null, "zrot": 60.0}
],
}
}
~~~~~~
\subsection sec_runfile_scans Experimental Scan Files
The pmsco.scan.Scan objects used in the calculation cannot be instantiated from the run-file directly.
Instead, the scans object of the run-file is a list of scan creators/loaders which specify how to create a Scan object.
The pmsco.scan module defines four scan creators: `ScanLoader`, `ScanCreator`, `HoloScanCreator` and `ScanKey`.
The following code block shows examples:
~~~~~~{.py}
{
"project": {
// ...
"scans": [
{
"__class__": "ScanCreator",
"filename": "twoatom_energy_alpha.etpai",
"emitter": "N",
"initial_state": "1s",
"positions": {
"e": "np.arange(10, 400, 5)",
"t": "0",
"p": "0",
"a": "np.linspace(-30, 30, 31)"
}
},
{
"__class__": "ScanLoader",
"filename": "${project}/twoatom_hemi_250e.etpi",
"emitter": "N",
"initial_state": "1s",
"is_modf": false
},
{
"__class__": "HoloScanCreator",
"filename": "${project}/twoatom_scan3.etpi",
"emitter": "N",
"initial_state": "1s",
"generator": "pmsco.data.holo_grid",
"generator_args": {
"theta_start": 90,
"theta_step": 1,
"theta_range": 90,
"phi_start": 0,
"phi_range": 360,
"phi_refinement": 1
},
"other_positions": {"e": 250, "a": 0}
},
{
"__class__": "HoloScanCreator",
"filename": "${project}/twoatom_scan4.etpi",
"emitter": "N",
"initial_state": "1s",
"other_positions": {"e": 250, "a": 0}
}
]
}
}
~~~~~~
The class name must be specified as it would be called in the custom project module.
For the example shown above, the following import statements are necessary in the `pmsco.projects.twoatom.py` module.
(Other forms of the import statement can be used accordingly.)
~~~~~~{.py}
import numpy as np
import pmsco.data
from pmsco.scan import ScanKey, ScanLoader, ScanCreator, HoloScanCreator
~~~~~~
The *ScanCreator* object creates a scan using Numpy array constructors in `positions`.
In the example above, a two-dimensional rectangular energy-alpha scan grid is created.
The values of the positions axes are passed to Python's `eval` function
and must return a one-dimensional Numpy `ndarray`.
The `emitter` and `initial_state` keys define the probed core level.
The *HoloScanCreator* object creates a /holo scan/, i.e., an angle scan of the theta and phi axes.
The distribution of the grid points is defined by a separate generator function.
Usually, the default pmsco.data.holo_grid function is used
which generates the well-known Osterwalder holo scan
with constant point density in solid angle and equidistant polar steps.
The `generator` and `generator_args` properties have default values.
The two example holo scans above are equivalent,
as the first one above just uses default values explicitly.
If you want to specify a generator function explicitly,
you must import it into the namespace of your project.
E.g. for `pmsco.data.holo_grid` you have to `import pmsco.data`.
The *ScanLoader* object loads a data file, specified under `filename`.
The filename can include a placeholder which is replaced by the corresponding item from Project.directories.
Note that some of the directories (including `project`) are pre-set by PMSCO.
It is recommended to add a `data` key under `directories` in the run-file
if the data files are outside of the PMSCO directory tree.
The `is_modf` key indicates whether the file contains a modulation function (`true`) or intensity (`false`).
In the latter case, the modulation function is calculated after loading.
The *ScanKey* is the shortest scan specification in the run-file.
It should not be used in new projects as it uses hard-coded data links in program code.
ScanKey is a shortcut to a complete scan dictionary in the project object.
The `scan_dict` must be set up in the `__init__` method of the project class.
The `key` item specifies which key of `scan_dict` should be used to create the Scan object.
Each item of `scan_dict` holds a dictionary
that holds the attributes for either a `ScanCreator`, `HoloScanCreator` or a `ScanLoader`.
If it contains a `positions` (`other_positions`) key, it represents a `ScanCreator` (`HoloScanCreator`), else a `ScanLoader`.
\subsection sec_runfile_optimizer Optimizer Parameters
The `optimizer_params` is a dictionary holding one or more of the following items.
| Key | Values | Description |
| --- | --- | --- |
| pop_size | integer<br>The default value is the greater of 4 or the number of parallel calculation processes. | Population size (number of particles) in swarm and genetic optimization mode. |
| seed_file | file system path | Name of the population seed file. Population data of previous optimizations can be used to seed a new optimization. The file must have the same structure as the .pop or .dat files. See @ref pmsco.optimizers.population.Population.seed_from_file. |
| seed_limit | integer | Number of seed models to import. |
| recalc_seed | true or false | If true, the seed models are calculated. Otherwise, the R-factor from the seed file is used as result. Use true if the seed file contains no or outdated R-factors. |
| table_source | file system path | Name of the model table file in table scan mode. |
\subsubsection sec_runfile_files File Categories
The following category names can be used with the `keep_files` option.
Multiple names can be specified as a list.
| Category | Description | Default Action |
| --- | --- | --- |
| all | shortcut to include all categories | |
| input | raw input files for calculator, including cluster and phase files in custom format | delete |
| output | raw output files from calculator | delete |
| atomic | atomic scattering and emission files in portable format | delete |
| cluster | cluster files in portable XYZ format for report | keep |
| debug | debug files | delete |
| model | output files in ETPAI format: complete simulation (a_-1_-1_-1_-1) | keep |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | keep |
| domain | output files in ETPAI format: domain (a_b_c_-1_-1) | delete |
| emitter | output files in ETPAI format: emitter (a_b_c_d_-1) | delete |
| region | output files in ETPAI format: region (a_b_c_d_e) | delete |
| report| final report of results | keep always |
| population | final state of particle population | keep |
| rfac | files related to models which give bad r-factors, see warning below | delete |
\note
The `report` category is always kept and cannot be turned off.
The `model` category is always kept in single calculation mode.
\warning
If you want to specify `rfac` with the `keep_files` option,
you have to add the file categories that you want to keep, e.g.,
`"keep_files": ["rfac", "cluster", "model", "scan", "population"]`
(to return the default categories for all calculated models).
Do not specify `rfac` alone as this will effectively not return any file.
\subsection sec_runfile_reports Reports
Run-time graphical reports are configured in the `reports` section.
The section is organized as a list of dictionaries.
Each dictionary sets up a specific report.
For example:
~~~~~~{.py}
{
"project": {
// ...
"reports": [
{
"__class__": "ConvergencePlot",
"filename_format": "${base}.convergence",
"title_format": "my_calc"
},
{
"__class__": "SwarmPlot",
"filename_format": "${base}-${param0}-${param1}-${gen}.swarmplot",
"title_format": "my_calc ${param0}-${param1} gen ${gen}",
"params": [["A", "B"], ["C", "D"]]
}
]
}
}
~~~~~~
The class name must be specified as it would be called in the custom project module.
For the example above, the import section of the project must include:
~~~~~~{.py}
from pmsco.reports.convergence import ConvergencePlot
from pmsco.reports.swarm import SwarmPlot
~~~~~~
For details on reports and their configuration, see @ref sec_reports.
\subsection sec_runfile_schedule Job Scheduling
To submit a job to a resource manager such as Slurm, add a `schedule` section to the run file
(section ordering is not important):
~~~~~~{.py}
{
"schedule": {
"__module__": "pmsco.schedule",
"__class__": "PsiRaSchedule",
"nodes": 1,
"tasks_per_node": 24,
"walltime": "2:00",
"manual_run": true,
"enabled": true
},
"project": {
"__module__": "projects.twoatom.twoatom",
"__class__": "TwoatomProject",
"mode": "single",
"output_file": "${home}/pmsco/twoatom0001",
...
}
}
~~~~~~
In the same way as for the project, the `__module__` and `__class__` keys select the class that handles the job submission.
In this example, it is pmsco.schedule.PsiRaSchedule which is tied to the Ra cluster at PSI.
For other machines, you can sub-class one of the classes in the pmsco.schedule module and include it in your project package.
The derived job submission class must prepare the code, run file and job script, and submit the job to the queue.
It should copy the code to the calculation directory to avoid version conflicts if the user continues to edit the code.
Compilation of the code can be done before submission or as a part of the job script.
@note It is difficult to check the run file and code against errors that may abort job execution.
New code and run files should be tested with a modified, fast-running calculation.
The parameters of pmsco.schedule.PsiRaSchedule are as follows.
Information about the computing nodes and partitions can be printed by the `sinfo -Nel` and `sinfo --long` commands.
| Key | Values | Description |
| --- | --- | --- |
| nodes | integer: 1..2 | Number of compute nodes (main boards on Ra). The maximum number available for PEARL is 2. |
| tasks_per_node | integer: 1..24, 32 | Number of tasks (CPU cores on Ra) per node. Jobs with less than 24 tasks are assigned to the shared partition. |
| wall_time | string: [days-]hours[:minutes[:seconds]] <br> dict: with any combination of days, hours, minutes, seconds | Maximum run time (wall time) of the job. |
| manual | bool | Manual submission (true) or automatic submission (false). Manual submission allows you to inspect the job files before submission. |
| enabled | bool | Enable scheduling (true). Otherwise, the calculation is started directly (false). |
@note The calculation job may run in a different working directory than the current one.
It is important to specify absolute data and output directories in the run file (project/directories section).
Placeholders like `${home}` can be used to make run files portable, cf. @ref sec_run_dirs.
*/

22
docs/src/tasks.dot
View File

@@ -38,15 +38,15 @@ custom_scan [label="scan\nconfiguration", shape=note];
{rank=same; custom_scan; create_scan; combine_scan;}
custom_scan -> create_scan [lhead=cluster_scan];
subgraph cluster_symmetry {
label="symmetry handler";
subgraph cluster_domain {
label="domain handler";
rank=same;
create_symmetry [label="define\nsymmetry\ntasks"];
combine_symmetry [label="gather\nsymmetry\nresults"];
create_model_space [label="define\ndomain\ntasks"];
combine_domain [label="gather\ndomain\nresults"];
}
custom_symmetry [label="symmetry\ndefinition", shape=cds];
{rank=same; create_symmetry; combine_symmetry; custom_symmetry;}
custom_symmetry -> combine_symmetry [lhead=cluster_symmetry];
custom_domain [label="domain\ndefinition", shape=cds];
{rank=same; create_model_space; combine_domain; custom_domain;}
custom_domain -> combine_domain [lhead=cluster_domain];
subgraph cluster_emitter {
label="emitter handler";
@@ -80,11 +80,11 @@ create_cluster -> edac;
create_model -> create_scan [label="level 1 tasks"];
evaluate_model -> combine_scan [label="level 1 results", dir=back];
create_scan -> create_symmetry [label="level 2 tasks"];
combine_scan -> combine_symmetry [label="level 2 results", dir=back];
create_scan -> create_model_space [label="level 2 tasks"];
combine_scan -> combine_domain [label="level 2 results", dir=back];
create_symmetry -> create_emitter [label="level 3 tasks"];
combine_symmetry -> combine_emitter [label="level 3 results", dir=back];
create_model_space -> create_emitter [label="level 3 tasks"];
combine_domain -> combine_emitter [label="level 3 results", dir=back];
create_emitter -> create_region [label="level 4 tasks"];
combine_emitter -> combine_region [label="level 4 results", dir=back];

38
docs/src/uml/CalculationTask-class.puml Normal file
View File

@@ -0,0 +1,38 @@
@startuml
class CalculationTask {
id : CalcID
parent : CalcID
model : dict
file_root : str
file_ext : str
result_filename : str
modf_filename : str
result_valid : bool
time : datetime.timedelta
files : dict
region : dict
__init__()
__eq__()
__hash__()
copy()
change_id()
format_filename()
get_mpi_message()
set_mpi_message()
add_task_file()
rename_task_file()
remove_task_file()
}
class CalcID {
model
scan
domain
emit
region
}
CalculationTask *-- CalcID
@enduml

133
docs/src/uml/CalculationTask-objects.puml Normal file
View File

@@ -0,0 +1,133 @@
@startuml
object Root {
id = -1, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {}
}
Root o.. Model1
Root o.. Model2
object Model1 {
id = 1, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {'d': 5}
}
object Model2 {
id = 2, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {'d': 7}
}
Model1 o.. Scan11
Model1 o.. Scan12
Model2 o.. Scan21
object Scan11 {
id = 1, 1, -1, -1, -1
parent = 1, -1, -1, -1, -1
model = {'d': 5}
}
object Scan12 {
id = 1, 2, -1, -1, -1
parent = 1, -1, -1, -1, -1
model = {'d': 5}
}
object Scan21 {
id = 2, 1, -1, -1, -1
parent = 2, -1, -1, -1, -1
model = {'d': 7}
}
Scan11 o.. Dom111
object Dom111 {
id = 1, 1, 1, -1, -1
parent = 1, 1, -1, -1, -1
model = {'d': 5}
}
Dom111 o.. Emitter1111
object Emitter1111 {
id = 1, 1, 1, 1, -1
parent = 1, 1, 1, -1, -1
model = {'d': 5}
}
Emitter1111 o.. Region11111
object Region11111 {
id = 1, 1, 1, 1, 1
parent = 1, 1, 1, 1, -1
model = {'d': 5}
}
@enduml
@startuml
object "Root: CalculationTask" as Root {
}
note right: all attributes undefined
object "Model: CalculationTask" as Model {
model
}
note right: model is defined\nother attributes undefined
object ModelHandler
object "Scan: CalculationTask" as Scan {
model
scan
}
object ScanHandler
object "Domain: CalculationTask" as Domain {
model
scan
domain
}
object "DomainHandler" as DomainHandler
object "Emitter: CalculationTask" as Emitter {
model
scan
domain
emitter
}
object EmitterHandler
object "Region: CalculationTask" as Region {
model
scan
domain
emitter
region
}
note right: all attributes well-defined
object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Domain
Domain "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Domain) .. DomainHandler
(Domain, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml

90
docs/src/uml/calculation-task.puml Normal file
View File

@@ -0,0 +1,90 @@
@startuml
class CalculationTask {
model
scan
domain
emitter
region
..
files
}
class Model {
index
..
dlat
dAS
dS1S2
V0
Zsurf
Texp
rmax
}
class Scan {
index
..
filename
mode
initial_state
energies
thetas
phis
alphas
}
class Domain {
index
..
rotation
registry
}
class Emitter {
index
}
class Region {
index
..
range
}
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Domain
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
domains
model_handler
cluster_generator
}
class ClusterGenerator {
count_emitters()
create_cluster()
}
class ModelHandler {
create_tasks()
add_result()
}
Model ..> ModelHandler
Scan ..> Project
Domain ..> Project
Emitter ..> ClusterGenerator
Region ..> Project
Project *-left- ModelHandler
Project *- ClusterGenerator
hide empty members
@enduml

47
docs/src/uml/cluster-generator.puml Normal file
View File

@@ -0,0 +1,47 @@
@startuml
package pmsco {
class Project {
cluster_generator
export_cluster()
}
abstract class ClusterGenerator {
project
{abstract} count_emitters()
{abstract} create_cluster()
}
class LegacyClusterGenerator {
project
count_emitters()
create_cluster()
}
}
package "user project" {
class UserClusterGenerator {
project
count_emitters()
create_cluster()
}
note bottom : for complex cluster
class UserProject {
count_emitters()
create_cluster()
}
note bottom : for simple cluster
}
Project <|-- UserProject
ClusterGenerator <|-- LegacyClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
Project *-- ClusterGenerator
UserProject .> LegacyClusterGenerator
UserProject .> UserClusterGenerator
@enduml

93
docs/src/uml/database.puml Normal file
View File

@@ -0,0 +1,93 @@
@startuml
class Project << (T,orchid) >> {
id
..
..
name
code
}
class Job << (T,orchid) >> {
id
..
project_id
..
name
mode
machine
git_hash
datetime
description
}
class Tag << (T,orchid) >> {
id
..
..
key
}
class JobTag << (T,orchid) >> {
id
..
tag_id
job_id
..
value
}
class Model << (T,orchid) >> {
id
..
job_id
..
model
gen
particle
}
class Result << (T,orchid) >> {
id
..
model_id
..
scan
domain
emit
region
rfac
timestamp
secs
}
class Param << (T,orchid) >> {
id
..
..
key
}
class ParamValue << (T,orchid) >> {
id
..
param_id
model_id
..
value
delta
}
Project "1" *-- "*" Job
Job "1" *-- "*" JobTag
Tag "1" *-- "*" JobTag
Job "1" *-- "*" Model
Param "1" *-- "*" ParamValue
Model "1" *-- "*" ParamValue
Model "1" *-- "*" Result
hide empty members
@enduml

45
docs/src/uml/handler-activity.puml Normal file
View File

@@ -0,0 +1,45 @@
@startuml
start
repeat
:define model tasks;
:gather model results;
repeat while
stop
@enduml
@startuml
start
repeat
partition "generate tasks" {
:define model tasks;
:define scan tasks;
:define domain tasks;
:define emitter tasks;
:define region tasks;
}
fork
:calculate task 1;
fork again
:calculate task 2;
fork again
:calculate task N;
end fork
partition "collect results" {
:gather region results;
:gather emitter results;
:gather domain results;
:gather scan results;
:gather model results;
}
repeat while
stop
@enduml

24
docs/src/uml/master-slave-messages.puml Normal file
View File

@@ -0,0 +1,24 @@
@startuml{master-slave-messages.png}
== task execution ==
loop calculation tasks
hnote over Master : define task
Master -> Slave: TAG_NEW_TASK
activate Slave
hnote over Slave : calculation
alt successful
Slave --> Master: TAG_NEW_RESULT
else calculation failed
Slave --> Master: TAG_INVALID_RESULT
else critical error
Slave --> Master: TAG_ERROR_ABORTING
end
deactivate Slave
hnote over Master : collect results
end
...
== termination ==
Master -> Slave: TAG_FINISH
destroy Slave
@enduml

31
docs/src/uml/minimum-project-classes.puml Normal file
View File

@@ -0,0 +1,31 @@
@startuml
package pmsco {
abstract class Project {
mode
code
scans
domains
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_model_space()
}
}
package projects {
class UserProject {
__init__()
create_cluster()
create_params()
create_model_space()
}
}
Project <|-- UserProject
hide empty members
@enduml

66
docs/src/uml/mpi-processes.puml Normal file
View File

@@ -0,0 +1,66 @@
@startuml
participant rank0 as "rank 0 (master)"
participant rank1 as "rank 1 (slave)"
participant rank2 as "rank 2 (slave)"
participant rankN as "rank N (slave)"
== initialization ==
rank0 ->> rank0
activate rank0
rnote over rank0: initialize project
== task loop ==
rnote over rank0: specify tasks
rank0 ->> rank1: task 1
activate rank1
rnote over rank1: execute task 1
rank0 ->> rank2: task 2
activate rank2
rnote over rank2: execute task 2
rank0 ->> rankN: task N
deactivate rank0
activate rankN
rnote over rankN: execute task N
rank0 <<-- rank1: result 1
deactivate rank1
rnote over rank0: process results\nspecify tasks
activate rank0
rank0 ->> rank1: task N+1
deactivate rank0
activate rank1
rnote over rank1: execute task N+1
rank0 <<-- rank2: result 2
deactivate rank2
activate rank0
rank0 ->> rank2: task N+2
deactivate rank0
activate rank2
rnote over rank2: execute task N+2
rank0 <<-- rankN: result N
deactivate rankN
activate rank0
rank0 ->> rankN: task 2N
deactivate rank0
activate rankN
rnote over rankN: execute task 2N
rank0 <<-- rank1: result N+1
deactivate rank1
rank0 <<-- rank2: result N+2
deactivate rank2
rank0 <<-- rankN: result 2N
deactivate rankN
rnote over rank0: process results
activate rank0
hnote over rank0: calculations complete
== termination ==
rnote over rank0: report results
rank0 ->> rank1: finish
destroy rank1
rank0 ->> rank2: finish
destroy rank2
rank0 ->> rankN: finish
destroy rankN
deactivate rank0
@enduml

59
docs/src/uml/project-classes.puml Normal file
View File

@@ -0,0 +1,59 @@
@startuml
abstract class Project {
mode : str = "single"
code : str = "edac"
scans : Scan [1..*]
domains : dict [1..*]
cluster_generator : ClusterGenerator
handler_classes
files : FileTracker
{abstract} create_cluster() : Cluster
{abstract} create_params() : CalculatorParams
{abstract} create_model_space() : ModelSpace
}
class Scan {
filename
raw_data
dtype
modulation
mode
emitter
initial_state
energies
thetas
phis
alphas
import_scan_file()
}
class ModelSpace {
start : dict
min : dict
max : dict
step : dict
add_param(name, start, min, max, step)
get_param(name)
}
class CalculatorParams {
title
comment
cluster_file
output_file
scan_file
initial_state
polarization
angular_resolution
z_surface
inner_potential
work_function
polar_incidence_angle
azimuthal_incidence_angle
experiment_temperature
}
Project "1" *-- "1..*" Scan
@enduml

86
docs/src/uml/scan-classes.puml Normal file
View File

@@ -0,0 +1,86 @@
@startuml
'https://plantuml.com/class-diagram
class ConfigurableObject
class Scan {
filename: str
raw_data: numpy.ndarray
dtype: numpy.dtype
modulation: numpy.ndarray
modulation_func: Callable
modulation_args: Dict
rfactor_func: Callable
rfactor_args: Dict
mode: List
emitter: str
initial_state: str
positions: Dict
__init__()
copy()
load()
define_scan()
import_scan_file()
analyse_raw_data()
generate_holo_scan()
}
class ScanSpec {
filename: str
emitter: str
initial_state: str
modulation_func: Callable
modulation_args: Dict
rfactor_func: Callable
rfactor_args: Dict
__init__()
load()
}
class ScanKey {
project: pmsco.project.Project
key: str
__init__()
load()
}
class ScanLoader {
is_modf: bool
patch: Dict
__init__()
load()
}
class ScanCreator {
positions: Dict
__init__()
load()
}
class HoloScanCreator {
generator: Callable
generator_args: Dict
__init__()
load()
set_property()
}
ConfigurableObject <|-- ScanSpec
ConfigurableObject <|-- ScanKey
ScanSpec <|-- ScanCreator
ScanSpec <|-- ScanLoader
ScanSpec <|-- HoloScanCreator
ScanKey --> ScanCreator: creates
ScanKey --> HoloScanCreator: creates
ScanKey --> ScanLoader: creates
ScanLoader --> Scan: creates
ScanCreator --> Scan: creates
HoloScanCreator --> Scan: creates
@enduml

26
docs/src/uml/scan-tasks-activity.puml Normal file
View File

@@ -0,0 +1,26 @@
@startuml
:model task|
fork
partition "scan 0" {
:define scan;
:scan 0 task|
detach
:scan 0 result|
}
fork again
partition "scan 1" {
:define scan;
:scan 1 task|
detach
:scan 1 result|
}
fork again
partition "scan N" {
:define scan;
:scan N task|
detach
:scan N result|
}
end fork
:model result|
@enduml

42
docs/src/uml/top-activity-partitions.puml Normal file
View File

@@ -0,0 +1,42 @@
@startuml
|user|
start
:setup;
|pmsco|
:initialize;
:import experimental data;
repeat
:define task;
|calculator|
:calculate\ntask;
|pmsco|
:evaluate results;
repeat while
-> [finished];
:report results;
stop
@enduml
@startuml
|pmsco|
start
:define task (model, scan, domain, emitter, region);
|project|
:create cluster;
:create parameters;
|calculator|
:scattering calculation;
|pmsco|
:combine results;
|project|
:calculate modulation function;
:calculate R-factor;
stop
@enduml

21
docs/src/uml/top-activity.puml Normal file
View File

@@ -0,0 +1,21 @@
@startuml
start
:initialize;
:import experimental data|
repeat
:define tasks;
fork
:calculate\ntask 1;
fork again
:calculate\ntask N;
end fork
:evaluate results;
repeat while
-> [finished];
:report results|
stop
@enduml

21
docs/src/uml/top-components.puml Normal file
View File

@@ -0,0 +1,21 @@
@startuml
skinparam componentStyle uml2
component "PMSCO" as pmsco
component "project" as project
component "scattering code\n(calculator)" as calculator
interface "command line" as cli
interface "experimental data" as data
interface "results" as results
interface "output files" as output
cli --> pmsco
data -> project
pmsco ..> project
pmsco ..> calculator
calculator -> output
pmsco -> results
@enduml

55
docs/src/uml/user-project-classes.puml Normal file
View File

@@ -0,0 +1,55 @@
@startuml
package pmsco {
abstract class Project {
mode
code
scans
domains
cluster_generator
handler_classes
__
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_model_space()
..
combine_scans()
combine_domains()
combine_emitters()
calc_modulation()
calc_rfactor()
}
abstract class ClusterGenerator {
{abstract} count_emitters()
{abstract} create_cluster()
}
}
package projects {
class UserProject {
scan_dict
__
setup()
..
create_params()
create_model_space()
..
combine_domains()
}
class UserClusterGenerator {
count_emitters()
create_cluster()
}
}
Project <|-- UserProject
Project *-- ClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
hide empty members
@enduml

30
extras/docker-docs/Dockerfile Normal file
View File

@@ -0,0 +1,30 @@
FROM python:3.12
# docker container to build PMSCO documentation
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
default-jre \
doxygen \
gawk \
git \
graphviz \
pandoc \
wget \
&& rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir \
doxypypy \
meson \
meson-python \
ninja \
pynose
RUN wget -O plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
ENV PLANTUML_JAR_PATH=/app/plantuml.jar
COPY . .
CMD ["sh"]

137
extras/singularity/singularity_python3 Normal file
View File

@@ -0,0 +1,137 @@
BootStrap: debootstrap
OSVersion: focal
MirrorURL: http://ch.archive.ubuntu.com/ubuntu/
%help
A singularity container for PMSCO.
singularity run -e pmsco.sif path/to/pmsco -r path/to/your-runfile
path/to/pmsco must point to the directory that contains the __main__.py file.
#%setup
# executed on the host system outside of the container before %post
#
# this will be inside the container
# touch ${SINGULARITY_ROOTFS}/tacos.txt
# this will be on the host
# touch avocados.txt
#%files
# files are copied before %post
#
# this copies to root
# avocados.txt
# this copies to /opt
# avocados.txt /opt
#
# this does not work
# ~/.ssh/known_hosts /etc/ssh/ssh_known_hosts
# ~/.ssh/id_rsa /etc/ssh/id_rsa
%labels
Maintainer Matthias Muntwiler
Maintainer_Email matthias.muntwiler@psi.ch
Python_Version 3.8
%environment
export LC_ALL=C
export PYTHON_VERSION=3
export CONDA_ROOT=/opt/miniconda
export PLANTUML_JAR_PATH=/opt/plantuml/plantuml.jar
export SINGULAR_BRANCH="singular"
%post
export LC_ALL=C
export PYTHON_VERSION=3.8
export CONDA_ROOT=/opt/miniconda
export PLANTUML_ROOT=/opt/plantuml
sed -i 's/$/ universe/' /etc/apt/sources.list
apt-get update
apt-get -y install \
binutils \
build-essential \
default-jre \
doxygen \
f2c \
g++ \
gcc \
gfortran \
git \
graphviz \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
nano \
openmpi-bin \
openmpi-common \
sqlite3 \
wget
apt-get clean
wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -O ~/miniforge3.sh
bash ~/miniforge3.sh -b -p ${CONDA_ROOT}
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate base
conda create -q --yes -n pmsco python=${PYTHON_VERSION}
conda install -q --yes -n pmsco -c conda-forge \
commentjson \
fasteners \
future \
gitpython
ipython \
ipykernel \
jsonschema \
h5py \
matplotlib \
meson \
mock \
pynose \
"numpy>=1.13" \
pandas \
periodictable \
pip \
scikit-learn \
scipy \
seaborn \
sqlalchemy \
statsmodels \
swig
conda clean --all -y
${CONDA_ROOT}/envs/pmsco/bin/pip install meson-python mpi4py netgraph networkx doxypypy
mkdir ${PLANTUML_ROOT}
wget -O ${PLANTUML_ROOT}/plantuml.jar https://sourceforge.net/projects/plantuml/files/plantuml.jar/download
#%test
# test the image after build
%runscript
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
exec python "$@"
%apprun install
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
cd ~
git clone https://git.psi.ch/pearl/pmsco.git pmsco
cd pmsco
git checkout master
git checkout -b ${SINGULAR_BRANCH}
meson setup build
cd build
meson compile
meson install
meson test
%apprun compile
. ${CONDA_ROOT}/etc/profile.d/conda.sh
conda activate pmsco
cd build
meson compile
meson install
meson test

76
extras/vagrant/Vagrantfile vendored Normal file
View File

@@ -0,0 +1,76 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure("2") do |config|
# The most common configuration options are documented and commented below.
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "sylabs/singularity-3.7-ubuntu-bionic64"
config.vm.box_version = "3.7"
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs
# `vagrant box outdated`. This is not recommended.
# config.vm.box_check_update = false
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine. In the example below,
# accessing "localhost:8080" will access port 80 on the guest machine.
# NOTE: This will enable public access to the opened port
# config.vm.network "forwarded_port", guest: 80, host: 8080
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine and only allow access
# via 127.0.0.1 to disable public access
# config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
# Create a private network, which allows host-only access to the machine
# using a specific IP.
# config.vm.network "private_network", ip: "192.168.33.10"
# Create a public network, which generally matched to bridged network.
# Bridged networks make the machine appear as another physical device on
# your network.
# config.vm.network "public_network"
# Share an additional folder to the guest VM. The first argument is
# the path on the host to the actual folder. The second argument is
# the path on the guest to mount the folder. And the optional third
# argument is a set of non-required options.
# config.vm.synced_folder "../data", "/vagrant_data"
# Provider-specific configuration so you can fine-tune various
# backing providers for Vagrant. These expose provider-specific options.
# Example for VirtualBox:
#
config.vm.provider "virtualbox" do |vb|
# Display the VirtualBox GUI when booting the machine
# vb.gui = true
# Customize the amount of memory on the VM:
# Increase this number if you plan to run parallel processes.
vb.memory = "2048"
# Customize the number of CPUs:
# Increase as necessary but not more than physically available on the host.
vb.cpus = 2
end
#
# View the documentation for the provider you are using for more
# information on available options.
# Enable provisioning with a shell script. Additional provisioners such as
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
# documentation for more information about their specific syntax and use.
# config.vm.provision "shell", inline: <<-SHELL
# apt-get update
# apt-get install -y apache2
# SHELL
end

36
makefile
View File

@@ -1,36 +0,0 @@
SHELL=/bin/sh
# makefile for all programs, modules and documentation
#
# required libraries for LOESS module: libblas, liblapack, libf2c
# (you may have to set soft links so that linker finds them)
#
# on shared computing systems (high-performance clusters)
# you may have to switch the environment before running this script.
#
# note: the public distribution does not include third-party code
# (EDAC in particular) because of incompatible license terms.
# please obtain such code from the original authors
# and copy it to the proper directory before compilation.
#
# the MSC and MUFPOT programs are currently not used.
# they are not built by the top-level targets all and bin.
.PHONY: all bin docs clean edac loess msc mufpot
PMSCO_DIR = pmsco
DOCS_DIR = docs
all: edac loess docs
bin: edac loess
edac loess msc mufpot:
$(MAKE) -C $(PMSCO_DIR)
docs:
$(MAKE) -C $(DOCS_DIR)
clean:
$(MAKE) -C $(PMSCO_DIR) clean
$(MAKE) -C $(DOCS_DIR) clean

0
pmsco/__init__.py
View File

12
pmsco/__main__.py
View File

@@ -8,10 +8,14 @@ python pmsco [pmsco-arguments]
@endverbatim
"""
import pmsco
from pathlib import Path
import sys
pmsco_root = Path(__file__).resolve().parent.parent
if str(pmsco_root) not in sys.path:
sys.path.insert(0, str(pmsco_root))
if __name__ == '__main__':
args, unknown_args = pmsco.parse_cli()
pmsco.main_pmsco(args, unknown_args)
sys.exit(0)
import pmsco.pmsco
pmsco.pmsco.main()

71
pmsco/calculator.py → pmsco/calculators/calculator.py
View File

@@ -1,5 +1,5 @@
"""
@package pmsco.calculator
@package pmsco.calculators.calculator
abstract scattering program interface.
this module declares the basic interface to scattering programs.
@@ -11,17 +11,21 @@ TestCalcInterface is provided for testing the PMSCO code quickly without calling
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-19 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import time
import numpy as np
import data as md
import cluster as mc
import pmsco.data as md
__author__ = 'matthias muntwiler'
@@ -38,56 +42,37 @@ class Calculator(object):
or <code>output_file + '.etpai'</code> depending on scan mode.
all other intermediate files are deleted unless keep_temp_files is True.
@param params: a msco_project.Params() object with all necessary values except cluster and output files set.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param cluster: a msco_cluster.Cluster() object with all atom positions set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param scan: a msco_project.Scan() object describing the experimental scanning scheme.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param output_file: base name for all intermediate and output files
@return: result_file, files_cats
@arg result_file is the name of the main ETPI or ETPAI result file to be further processed.
@arg files_cats is a dictionary that lists the names of all created data files with their category.
@return: (str, dict) result_file, and dictionary of created files {filename: category}
@return: (str, dict) result_file, and dictionary of created files.
@arg the first element is the name of the main ETPI or ETPAI result file to be further processed.
@arg the second element is a dictionary that lists the names of all created data files with their category.
the dictionary key is the file name,
the value is the file category (cluster, phase, etc.).
the value is the file category (cluster, atomic, etc.).
"""
return None, None
def check_cluster(self, cluster, output_file):
"""
export the cluster in XYZ format for reference.
along with the complete cluster, the method also saves cuts in the xz (extension .y.xyz) and yz (.x.xyz) plane.
class AtomicCalculator(Calculator):
"""
abstract interface class to the atomic scattering calculation program.
"""
pass
@param cluster: a pmsco.cluster.Cluster() object with all atom positions set.
@param output_file: base name for all intermediate and output files
@return: dictionary listing the names of the created files with their category.
the dictionary key is the file name,
the value is the file category (cluster).
@warning experimental: this method may be moved elsewhere in a future version.
"""
xyz_filename = output_file + ".xyz"
cluster.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files = {xyz_filename: 'cluster'}
clucut = mc.Cluster()
clucut.copy_from(cluster)
clucut.trim_slab("x", 0.0, 0.1)
xyz_filename = output_file + ".x.xyz"
clucut.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files[xyz_filename] = 'cluster'
clucut.copy_from(cluster)
clucut.trim_slab("y", 0.0, 0.1)
xyz_filename = output_file + ".y.xyz"
clucut.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files[xyz_filename] = 'cluster'
return files
class InternalAtomicCalculator(AtomicCalculator):
"""
dummy atomic scattering class if scattering factors are calculated internally by the multiple scattering calculator.
"""
pass
class TestCalculator(Calculator):
@@ -127,5 +112,5 @@ class TestCalculator(Calculator):
md.save_data(etpi_filename, result_etpi)
files = {clu_filename: 'cluster', etpi_filename: 'energy'}
files = {clu_filename: 'cluster', etpi_filename: 'region'}
return etpi_filename, files

145
pmsco/edac_calculator.py → pmsco/calculators/edac.py
View File

@@ -1,28 +1,33 @@
"""
@package pmsco.edac_calculator
@package pmsco.calculators.edac
Garcia de Abajo EDAC program interface.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import division
import os
import logging
import math
import numpy as np
import calculator
import data as md
import cluster as mc
import edac.edac as edac
import os
import pmsco.calculators.calculator as calculator
import pmsco.data as md
import pmsco.cluster as mc
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
import edac
except (ImportError, ModuleNotFoundError) as e:
edac = None
logger.critical("Error importing the edac package.", exc_info=e)
class EdacCalculator(calculator.Calculator):
def write_input_file(self, params, scan, filepath):
@@ -44,18 +49,22 @@ class EdacCalculator(calculator.Calculator):
if alpha is defined, theta is implicitly set to normal emission! (to be generalized)
TODO: some parameters are still hard-coded.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param filepath: (str) name and path of the file to be created.
@return dictionary of created files {filename: category}
"""
with open(filepath, "w") as f:
files = {}
with open(filepath, "wt", encoding="latin1") as f:
f.write("verbose off\n")
f.write("cluster input %s\n" % (params.cluster_file))
f.write("emitters %u l(A)\n" % (len(params.emitters)))
f.write("cluster input {0}\n".format(params.cluster_file))
f.write("emitters {0:d} l(A)\n".format(len(params.emitters)))
for em in params.emitters:
f.write("%g %g %g %u\n" % em)
#for iat in range(params.atom_types):
#pf = params.phase_file[iat]
#pf = pf.replace(".pha", ".edac.pha")
#f.write("scatterer %u %s\n" % (params.atomic_number[iat], pf))
f.write("{0:f} {1:f} {2:f} {3:d}\n".format(*em))
en = scan.energies + params.work_function
en_min = en.min()
@@ -106,7 +115,7 @@ class EdacCalculator(calculator.Calculator):
assert th_num < th.shape[0] * 10, \
"linearization of theta scan causes excessive oversampling {0}/{1}".format(th_num, th.shape[0])
f.write("beta {0}\n".format(params.polar_incidence_angle, params.azimuthal_incidence_angle))
f.write("beta {0}\n".format(params.polar_incidence_angle))
f.write("incidence {0} {1}\n".format(params.polar_incidence_angle, params.azimuthal_incidence_angle))
f.write("emission angle theta {th0:f} {th1:f} {nth:d}\n".format(th0=th_min, th1=th_max, nth=th_num))
@@ -136,35 +145,67 @@ class EdacCalculator(calculator.Calculator):
f.write("initial state {0}\n".format(params.initial_state))
polarizations = {'H': 'LPx', 'V': 'LPy', 'L': 'LCP', 'R': 'RCP'}
f.write("polarization {0}\n".format(polarizations[params.polarization]))
f.write("muffin-tin\n")
f.write("V0 E(eV) {0}\n".format(params.inner_potential))
f.write("cluster surface l(A) {0}\n".format(params.z_surface))
scatterers = ["scatterer {at} {fi}\n".format(at=at, fi=fi)
for (at, fi) in params.phase_files.items()
if os.path.isfile(fi)]
rme = ["rmat {fi}\n".format(fi=fi)
for (at, fi) in params.rme_files.items()
if at == params.emitters[0][3] and os.path.isfile(fi)] or \
["rmat inline 1 regular1 {l0} {pv} {pd} {mv} {md}\n".format(l0=params.l_init,
pv=params.rme_plus_value, pd=params.rme_plus_shift,
mv=params.rme_minus_value, md=params.rme_minus_shift)]
if scatterers and rme:
for scat in scatterers:
f.write(scat)
f.write(rme[0])
else:
f.write("muffin-tin\n")
f.write("V0 E(eV) {0:f}\n".format(params.inner_potential))
f.write("cluster surface l(A) {0:f}\n".format(params.z_surface))
f.write("imfp SD-UC\n")
f.write("temperature %g %g\n" % (params.experiment_temperature, params.debye_temperature))
f.write("temperature {0:f} {1:f}\n".format(params.experiment_temperature, params.debye_temperature))
f.write("iteration recursion\n")
f.write("dmax l(A) %g\n" % (params.dmax))
f.write("lmax %u\n" % (params.lmax))
f.write("orders %u " % (len(params.orders)))
for order in params.orders:
f.write("%u " % (order))
f.write("\n")
f.write("emission angle window 1\n")
f.write("scan pd %s\n" % (params.output_file))
f.write("dmax l(A) {0:f}\n".format(params.dmax))
f.write("lmax {0:d}\n".format(params.lmax))
f.write("orders {0:d} ".format(len(params.orders)))
f.write(" ".join(format(order, "d") for order in params.orders) + "\n")
f.write("emission angle window {0:F}\n".format(params.angular_resolution / 2.0))
# scattering factor output (see project.CalculatorParams.phase_output_classes)
if params.phase_output_classes is not None:
fn = "{0}.clu".format(params.output_file)
f.write("cluster output l(A) {fn}\n".format(fn=fn))
files[fn] = "output"
try:
cls = (cl for cl in params.phase_output_classes)
except TypeError:
cls = range(params.phase_output_classes)
for cl in cls:
fn = "{of}.{cl}.scat".format(cl=cl, of=params.output_file)
f.write("scan scatterer {cl} phase-shifts {fn}\n".format(cl=cl, fn=fn))
files[fn] = "output"
f.write("scan pd {0}\n".format(params.output_file))
files[params.output_file] = "output"
f.write("end\n")
return files
def run(self, params, cluster, scan, output_file):
"""
run EDAC with the given parameters and cluster.
@param params: a msc_param.Params() object with all necessary values except cluster and output files set.
@param params: a pmsco.project.CalculatorParams object with all necessary values except cluster and output files set.
@param cluster: a msc_cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param scan: a msco_project.Scan() object describing the experimental scanning scheme.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param output_file: base name for all intermediate and output files
@return: result_file, files_cats
@return: (str, dict) result_file, and dictionary of created files {filename: category}
"""
# set up scan
@@ -178,20 +219,22 @@ class EdacCalculator(calculator.Calculator):
dat_filename = out_filename
if params.fixed_cluster:
etpi_filename = base_filename + ".etpai"
dtype = md.DTYPE_ETPAI
else:
etpi_filename = base_filename + ".etpi"
dtype = md.DTYPE_ETPI
# fix EDAC particularities
params.cluster_file = clu_filename
params.output_file = out_filename
params.data_file = dat_filename
params.emitters = cluster.get_emitters()
params.emitters = cluster.get_emitters(['x', 'y', 'z', 'c'])
# save parameter files
logger.debug("writing cluster file %s", clu_filename)
cluster.save_to_file(clu_filename, fmt=mc.FMT_EDAC)
logger.debug("writing input file %s", par_filename)
self.write_input_file(params, scan, par_filename)
files = self.write_input_file(params, scan, par_filename)
# run EDAC
logger.info("calling EDAC with input file %s", par_filename)
@@ -205,19 +248,25 @@ class EdacCalculator(calculator.Calculator):
result_etpi['e'] -= params.work_function
if 't' in scan.mode and 'p' in scan.mode:
hemi_tpi = scan.raw_data.copy()
hemi_tpi['i'] = 0.0
try:
hemi_tpi['s'] = 0.0
except ValueError:
pass
result_etpi = md.interpolate_hemi_scan(result_etpi, hemi_tpi)
dest_tpi = np.zeros(scan.raw_data.shape, dtype)
dest_tpi['t'] = scan.thetas
dest_tpi['p'] = scan.phis
result_etpi = md.interpolate_hemi_scan(result_etpi, dest_tpi)
if params.fixed_cluster:
expected_shape = max(scan.energies.shape[0], 1) * max(scan.alphas.shape[0], 1)
else:
expected_shape = max(scan.energies.shape[0], 1) * max(scan.phis.shape[0], scan.thetas.shape[0], 1)
if result_etpi.shape[0] != expected_shape:
logger.warning(BMsg("possible scan length mismatch: EDAC result: {result}, expected: {expected}",
result=result_etpi.shape[0], expected=expected_shape))
if result_etpi.shape[0] != scan.raw_data.shape[0]:
logger.error("scan length mismatch: EDAC result: %u, scan data: %u", result_etpi.shape[0], scan.raw_data.shape[0])
logger.debug("save result to file %s", etpi_filename)
md.save_data(etpi_filename, result_etpi)
files = {clu_filename: 'input', par_filename: 'input', dat_filename: 'output',
etpi_filename: 'region'}
files[clu_filename] = 'input'
files[par_filename] = 'input'
files[dat_filename] = 'output'
files[etpi_filename] = 'region'
return etpi_filename, files

63
pmsco/msc_calculator.py → pmsco/calculators/msc.py
View File

@@ -13,9 +13,12 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
import calculator
import data as md
import msc.msc as msc
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import pmsco.calculators.calculator as calculator
import pmsco.data as md
import subprojects.msc.msc as msc
import logging
logger = logging.getLogger(__name__)
@@ -24,42 +27,42 @@ logger = logging.getLogger(__name__)
class MscCalculator(calculator.Calculator):
def write_input_file(self, params, filepath):
with open(filepath, "w") as f:
f.write(" %s\n" % (params.title) )
f.write(" %s\n" % (params.comment) )
f.write(" %s\n" % (params.title))
f.write(" %s\n" % (params.comment))
l_init = "spdf".index(params.initial_state[1])
f.write(" %4u\n" % (l_init) )
f.write(" %4u\n" % (params.spherical_order) )
f.write(" %s\n" % (params.polarization) )
f.write(" %4u\n" % (params.scattering_level) )
f.write(" %7.2f%7.2f\n" % (params.fcut, params.cut) )
f.write(" %12.6f\n" % (params.angular_broadening) )
f.write(" %12.6f\n" % (params.lattice_constant) )
f.write(" %12.6f\n" % (params.z_surface) )
f.write(" %4u\n" % (params.atom_types) )
f.write(" %4u\n" % (l_init))
f.write(" %4u\n" % (params.spherical_order))
f.write(" %s\n" % (params.polarization))
f.write(" %4u\n" % (params.scattering_level))
f.write(" %7.2f%7.2f\n" % (params.fcut, params.cut))
f.write(" %12.6f\n" % (params.angular_resolution))
f.write(" %12.6f\n" % (params.lattice_constant))
f.write(" %12.6f\n" % (params.z_surface))
f.write(" %4u\n" % (params.atom_types))
for iat in range(params.atom_types):
f.write(" %4u %s\n" % (params.atomic_number[iat], "..."))
f.write(" %4u %s\n" % (params.atomic_number[iat], params.phase_file[iat]))
f.write(" %12.6f\n" % (params.msq_displacement[iat]) )
f.write(" %12.6f\n" % (params.planewave_attenuation) )
f.write(" %12.6f\n" % (params.inner_potential) )
f.write(" %12.6f\n" % (params.symmetry_range) )
f.write(" %12.6f\n" % (params.polar_incidence_angle) )
f.write(" %12.6f\n" % (params.azimuthal_incidence_angle) )
f.write(" %s\n" % (params.vibration_model) )
f.write(" %12.6f\n" % (params.substrate_atomic_mass) )
f.write(" %12.6f\n" % (params.experiment_temperature) )
f.write(" %12.6f\n" % (params.debye_temperature) )
f.write(" %12.6f\n" % (params.debye_wavevector) )
f.write(" %12.6f%7.3f\n" % (params.rme_minus_value, params.rme_minus_shift) )
f.write(" %12.6f%7.3f\n" % (params.rme_plus_value, params.rme_plus_shift) )
f.write(" %4u\n" % (1) )
f.write(" %4u %12.6f\n" % (1, 1.0) )
f.write(" %12.6f\n" % (params.msq_displacement[iat]))
f.write(" %12.6f\n" % (params.planewave_attenuation))
f.write(" %12.6f\n" % (params.inner_potential))
f.write(" %12.6f\n" % (params.symmetry_range))
f.write(" %12.6f\n" % (params.polar_incidence_angle))
f.write(" %12.6f\n" % (params.azimuthal_incidence_angle))
f.write(" %s\n" % (params.vibration_model))
f.write(" %12.6f\n" % (params.substrate_atomic_mass))
f.write(" %12.6f\n" % (params.experiment_temperature))
f.write(" %12.6f\n" % (params.debye_temperature))
f.write(" %12.6f\n" % (params.debye_wavevector))
f.write(" %12.6f%7.3f\n" % (params.rme_minus_value, params.rme_minus_shift))
f.write(" %12.6f%7.3f\n" % (params.rme_plus_value, params.rme_plus_shift))
f.write(" %4u\n" % (1))
f.write(" %4u %12.6f\n" % (1, 1.0))
def run(self, params, cluster, scan, output_file):
"""
run the MSC program with the given parameters and cluster.
@param params: a project.Params() object with all necessary values except cluster and output files set.
@param params: a project.CalculatorParams() object with all necessary values except cluster and output files set.
@param cluster: a cluster.Cluster(format=FMT_MSC) object with all atom positions set.

173
pmsco/calculators/phagen/runner.py Normal file
View File

@@ -0,0 +1,173 @@
"""
@package pmsco.calculators.phagen.runner
Natoli/Sebilleau PHAGEN interface
This module runs the PHAGEN program to calculate scattering factors and radial matrix elements.
Requires PHAGEN version 2.2 from https://git.ipr.univ-rennes.fr/epsi/msspec_python3.git (contained in subprojects).
@author Matthias Muntwiler
@copyright (c) 2015-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import os
import shutil
import tempfile
from pathlib import Path
import sys
from pmsco.calculators.calculator import AtomicCalculator
from pmsco.calculators.phagen.translator import Translator
import pmsco.cluster
from pmsco.helpers import stdout_redirected
import pmsco.project
logger = logging.getLogger(__name__)
try:
import phagen
except (ImportError, ModuleNotFoundError) as e:
phagen = None
logger.critical("Error importing the phagen package.", exc_info=e)
class PhagenCalculator(AtomicCalculator):
"""
use the PHAGEN program to calculate scattering factors and radial matrix element.
this produces scatterer, radial matrix element and cluster files for EDAC.
"""
def run(self,
params: pmsco.project.CalculatorParams,
cluster: pmsco.cluster.Cluster,
scan: pmsco.project.Scan,
output_file: str):
"""
create the input file, run PHAGEN, and translate the output to EDAC format.
the following files are created in the job work directory:
- scattering factor files in EDAC format.
their names are `output_file + "_{atomclass}.scat"`.
- radial matrix element file in EDAC format.
its name is `output_file + ".rme"`.
- cluster file in PMSCO format.
its name is `output_file + ".clu"`.
the cluster and params objects are updated and linked to the scattering files
so that they can be passed to EDAC without further modification.
the radial matrix element is currently not used.
note that the scattering files are numbered according to the atomic environment and not chemical element.
this means that the updated cluster (cluster object or ".clu" file)
must be used in the scattering calculation.
atomic index is not preserved - atoms in the input and output clusters can only be related by coordinate!
because PHAGEN generates a lot of files with hard-coded names,
the function creates a temporary directory for PHAGEN and deletes it before returning.
@param params: pmsco.project.CalculatorParams object.
the phase_files attribute is updated with the paths of the scattering files.
@param cluster: pmsco.cluster.Cluster object.
the cluster is updated with the one returned from PHAGEN.
the atom classes are linked to the scattering files.
@param scan: pmsco.project.Scan object.
the scan object is used to determine the kinetic energy range.
@param output_file: base path and name of the output files.
@return (None, dict) where dict is a list of output files with their category.
the category is "atomic" for all output files.
"""
assert cluster.get_emitter_count() == 1, "PHAGEN cannot handle more than one emitter at a time"
transl = Translator()
transl.params.set_params(params)
transl.params.set_cluster(cluster)
transl.params.set_scan(scan)
phagen_cluster = pmsco.cluster.Cluster()
files = {}
prev_wd = Path.cwd()
try:
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
in_path = temp_path / "input"
in_path.mkdir(exist_ok=True)
out_path = temp_path / "output"
out_path.mkdir(exist_ok=True)
infile = in_path / "input.ms"
try:
transl.write_input(infile)
report_infile = os.path.join(prev_wd, output_file + ".phagen.in")
shutil.copy(infile, report_infile)
files[report_infile] = "input"
except IOError:
logger.warning("error writing phagen input file {fi}.".format(fi=infile))
report_listfile = os.path.join(prev_wd, output_file + ".phagen.list")
files[report_listfile] = "log"
# call phagen, redirect stdout (unit 6)
os.chdir(out_path)
with open(report_listfile, "wb") as f:
with stdout_redirected(f):
phagen.phagen()
phafile = out_path / "div" / "phases.dat"
radfile = out_path / "fort.55"
# tlfile = out_path / "fort.35"
clufile = out_path / "clus" / "clus.out"
# collect results
try:
transl.parse_phagen_phase(phafile)
report_phafile = os.path.join(prev_wd, output_file + ".phagen.pha")
shutil.copy(phafile, report_phafile)
files[report_phafile] = "output"
except IOError:
logger.error("error loading phagen phase file {fi}".format(fi=phafile))
try:
transl.parse_radial_file(radfile)
report_radfile = os.path.join(prev_wd, output_file + ".phagen.rad")
shutil.copy(radfile, report_radfile)
files[report_radfile] = "output"
except IOError:
logger.error("error loading phagen radial file {fi}".format(fi=radfile))
try:
phagen_cluster.load_from_file(clufile, pmsco.cluster.FMT_PHAGEN_OUT)
except IOError:
logger.error("error loading phagen cluster file {fi}".format(fi=clufile))
finally:
os.chdir(prev_wd)
# write edac files
scatfile = output_file + "_{}.scat"
scatfiles = transl.write_edac_scattering(scatfile)
params.phase_files = scatfiles.copy()
files.update({f: "atomic" for f in params.phase_files.values()})
rmefile = output_file + "_{}.rme"
rmefiles = transl.write_edac_emission(rmefile)
params.rme_files = rmefiles.copy()
files.update({f: "atomic" for f in params.rme_files.values()})
cluster.update_atoms(phagen_cluster, {'c'})
clufile = output_file + ".pmsco.clu"
cluster.save_to_file(clufile, pmsco.cluster.FMT_PMSCO)
files[clufile] = "cluster"
return None, files

503
pmsco/calculators/phagen/translator.py Normal file
View File

@@ -0,0 +1,503 @@
"""
@package pmsco.calculators.phagen.translator
Natoli/Sebilleau PHAGEN interface
this module provides conversion between input/output files of PHAGEN and EDAC.
@author Matthias Muntwiler
@copyright (c) 2015-19 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import numpy as np
from pmsco.cluster import Cluster
logger = logging.getLogger(__name__)
## rydberg energy in electron volts
ERYDBERG = 13.6056923
def state_to_edge(state):
"""
translate spectroscopic notation to edge notation.
@param state: spectroscopic notation: "1s", "2s", "2p1/2", etc.
@return: edge notation: "k", "l1", "l2", etc.
note: if the j-value is not given, the lower j edge is returned.
"""
jshells = ['s', 'p1/2', 'p3/2', 'd3/2', 'd5/2', 'f5/2', 'f7/2']
lshells = [s[0] for s in jshells]
shell = int(state[0])
try:
subshell = jshells.index(state[1:]) + 1
except ValueError:
subshell = lshells.index(state[1]) + 1
except IndexError:
subshell = 1
edge = "klmnop"[shell-1]
if shell > 1:
edge += str(subshell)
return edge
class TranslationParams(object):
"""
project parameters needed for translation.
energy unit is eV.
"""
def __init__(self):
self.initial_state = "1s"
self.binding_energy = 0.
self.cluster = None
self.kinetic_energies = np.empty(0, dtype=float)
@property
def l_init(self):
return "spdf".index(self.initial_state[1])
@property
def edge(self):
return state_to_edge(self.initial_state)
def set_params(self, params):
"""
set the translation parameters.
@param params: a pmsco.project.CalculatorParams object or
a dictionary containing some or all public fields of this class.
@return: None
"""
try:
self.initial_state = params.initial_state
self.binding_energy = params.binding_energy
except AttributeError:
for key in params:
self.__setattr__(key, params[key])
def set_scan(self, scan):
"""
set the scan parameters.
@param scan: a pmsco.project.Scan object
@return: None
"""
try:
energies = scan.energies
except AttributeError:
try:
energies = scan['e']
except KeyError:
energies = scan
if not isinstance(energies, np.ndarray):
energies = np.array(energies)
self.kinetic_energies = np.resize(self.kinetic_energies, energies.shape)
self.kinetic_energies = energies
def set_cluster(self, cluster):
"""
set the initial cluster.
@param cluster: a pmsco.cluster.Cluster object
@return: None
"""
self.cluster = cluster
class Translator(object):
"""
data conversion to/from phagen input/output files.
usage:
1. set the translation parameters self.params.
2. call write_input_file to create the phagen input files.
3. call phagen on the input file.
4. call parse_phagen_phase.
5. call parse_radial_file.
6. call write_edac_scattering to produce the EDAC scattering matrix files.
7. call write_edac_emission to produce the EDAC emission matrix file.
"""
## @var params
#
# project parameters needed for translation.
#
# fill the attributes of this object before using any translator methods.
## @var scattering
#
# t-matrix storage
#
# the t-matrix is stored in a flat, one-dimensional numpy structured array consisting of the following fields:
# @arg e (float) energy (eV)
# @arg a (int) atom index (1-based)
# @arg l (int) angular momentum quantum number l
# @arg t (complex) scattering matrix element, t = exp(-i * delta) * sin delta
#
# @note PHAGEN uses the convention t = exp(-i * delta) * sin delta,
# whereas EDAC uses t = exp(i * delta) * sin delta (complex conjugate).
# this object stores the t-matrix according to the PHAGEN convention.
# the conversion to the EDAC convention occurs in write_edac_scattering_file().
## @var emission
#
# radial matrix element storage
#
# the radial matrix elemnts are stored in a flat, one-dimensional numpy structured array
# consisting of the following fields:
# @arg e (float) energy (eV)
# @arg dw (complex) matrix element for the transition to l-1
# @arg up (complex) matrix element for the transition to l+1
## @var cluster
#
# cluster object for PHAGEN
#
# this object is created by translate_cluster().
def __init__(self):
"""
initialize the object instance.
"""
self.params = TranslationParams()
dt = [('e', 'f4'), ('a', 'i4'), ('l', 'i4'), ('t', 'c16')]
self.scattering = np.empty(0, dtype=dt)
dt = [('e', 'f4'), ('dw', 'c16'), ('up', 'c16')]
self.emission = np.empty(0, dtype=dt)
self.cluster = None
def translate_cluster(self):
"""
translate the cluster into a form suitable for PHAGEN.
specifically, move the (first and hopefully only) emitter to the first atom position.
the method copies the cluster from self.params into a new object
and stores it under self.cluster.
@return: None
"""
self.cluster = Cluster()
self.cluster.copy_from(self.params.cluster)
ems = self.cluster.get_emitters(['i'])
self.cluster.move_to_first(idx=ems[0][0]-1)
def write_cluster(self, f):
"""
write the cluster section of the PHAGEN input file.
@param f: file or output stream (an object with a write method)
@return: None
"""
for atom in self.cluster.data:
d = {k: atom[k] for k in atom.dtype.names}
f.write("{s} {t} {x} {y} {z}\n".format(**d))
f.write("-1 -1 0. 0. 0.\n")
def write_ionicity(self, f):
"""
write the ionicity section of the PHAGEN input file.
ionicity is read from the 'q' column of the cluster.
all atoms of a chemical element must have the same charge state
because ionicity has to be specified per element.
this function writes the average of all charge states of an element.
@param f: file or output stream (an object with a write method)
@return: None
"""
data = self.cluster.data
elements = np.unique(data['t'])
for element in elements:
idx = np.where(data['t'] == element)
charge = np.mean(data['q'][idx])
f.write("{t} {q}\n".format(t=element, q=charge))
f.write("-1\n")
def write_input(self, f):
"""
write the PHAGEN input file.
@param f: file path or output stream (an object with a write method).
@return: None
"""
phagen_params = {}
self.translate_cluster()
phagen_params['absorber'] = 1
phagen_params['emin'] = self.params.kinetic_energies.min() / ERYDBERG
phagen_params['emax'] = self.params.kinetic_energies.max() / ERYDBERG
if self.params.kinetic_energies.shape[0] > 1:
phagen_params['delta'] = (phagen_params['emax'] - phagen_params['emin']) / \
(self.params.kinetic_energies.shape[0] - 1)
else:
phagen_params['delta'] = 0.1
phagen_params['edge'] = state_to_edge(self.params.initial_state)
phagen_params['edge1'] = 'm4' # auger not supported
phagen_params['edge2'] = 'm4' # auger not supported
phagen_params['cip'] = self.params.binding_energy / ERYDBERG
if phagen_params['cip'] < 0.001:
raise ValueError("binding energy parameter is zero.")
if np.sum(np.abs(self.cluster.data['q'])) > 0.:
phagen_params['ionzst'] = 'ionic'
else:
phagen_params['ionzst'] = 'neutral'
if hasattr(f, "write") and callable(f.write):
f.write("&job\n")
f.write("calctype='xpd',\n")
f.write("coor='angs',\n")
f.write("cip={cip},\n".format(**phagen_params))
f.write("absorber={absorber},\n".format(**phagen_params))
f.write("edge='{edge}',\n".format(**phagen_params))
f.write("edge1='{edge1}',\n".format(**phagen_params))
f.write("edge2='{edge1}',\n".format(**phagen_params))
f.write("gamma=0.03,\n")
f.write("lmax_mode=2,\n")
f.write("lmaxt=50,\n")
f.write("emin={emin},\n".format(**phagen_params))
f.write("emax={emax},\n".format(**phagen_params))
f.write("delta={delta},\n".format(**phagen_params))
f.write("potgen='in',\n")
f.write("potype='hedin',\n")
f.write("norman='stdcrm',\n")
f.write("ovlpfac=0.0,\n")
f.write("ionzst='{ionzst}',\n".format(**phagen_params))
f.write("charelx='ex',\n")
f.write("l2h=4\n")
f.write("&end\n")
f.write("comment 1\n")
f.write("comment 2\n")
f.write("\n")
self.write_cluster(f)
self.write_ionicity(f)
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_input(fi)
def parse_phagen_phase(self, f):
"""
parse the phase output file from PHAGEN.
the phase file is written to div/phases.dat.
it contains the following columns:
@arg e energy (Ry)
@arg x1 unknown 1
@arg x2 unknown 2
@arg na atom index (1-based)
@arg nl angular momentum quantum number l
@arg tr real part of the scattering matrix element
@arg ti imaginary part of the scattering matrix element
@arg ph phase shift
the data is translated into the self.scattering array.
@arg e energy (eV)
@arg a atom index (1-based)
@arg l angular momentum quantum number l
@arg t complex scattering matrix element
@note PHAGEN uses the convention t = exp(-i * delta) * sin delta,
whereas EDAC uses t = exp(i * delta) * sin delta (complex conjugate).
this class stores the t-matrix according to the PHAGEN convention.
the conversion to the EDAC convention occurs in write_edac_scattering_file().
@param f: file or path (any file-like or path-like object that can be passed to numpy.genfromtxt).
@return: None
"""
dt = [('e', 'f4'), ('x1', 'f4'), ('x2', 'f4'), ('na', 'i4'), ('nl', 'i4'),
('tr', 'f8'), ('ti', 'f8'), ('ph', 'f4')]
data = np.atleast_1d(np.genfromtxt(f, dtype=dt))
self.scattering = np.resize(self.scattering, data.shape)
scat = self.scattering
scat['e'] = data['e'] * ERYDBERG
scat['a'] = data['na']
scat['l'] = data['nl']
scat['t'] = data['tr'] + 1j * data['ti']
def write_edac_scattering(self, filename_format, phases=False):
"""
write scatterer files for EDAC.
produces one file for each atom class in self.scattering.
@param filename_format: file name including a placeholder {} for the atom class.
@param phases: write phase files instead of t-matrix files.
@return: dictionary that maps atom classes to file names
"""
if phases:
write = self.write_edac_phase_file
else:
write = self.write_edac_scattering_file
scat = self.scattering
atoms = np.unique(scat['a'])
files = {}
for atom in atoms:
f = filename_format.format(atom)
sel = scat['a'] == atom
idx = np.where(sel)
atom_scat = scat[idx]
write(f, atom_scat)
files[atom] = f
return files
def write_edac_scattering_file(self, f, scat):
"""
write a scatterer file for EDAC.
@param f: file path or output stream (an object with a write method).
@param scat: a slice of the self.scattering array belonging to the same atom class.
@return: None
"""
if hasattr(f, "write") and callable(f.write):
energies = np.unique(scat['e'])
ne = energies.shape[0]
lmax = scat['l'].max()
if ne == 1:
f.write("1 {lmax} regular tl\n".format(lmax=lmax))
else:
f.write("{nk} E(eV) {lmax} regular tl\n".format(nk=ne, lmax=lmax))
for energy in energies:
sel = scat['e'] == energy
idx = np.where(sel)
energy_scat = scat[idx]
if ne > 1:
f.write("{0:.3f} ".format(energy))
for item in energy_scat:
f.write(" {0:.6f} {1:.6f}".format(item['t'].real, -item['t'].imag))
for i in range(len(energy_scat), lmax + 1):
f.write(" 0 0")
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_edac_scattering_file(fi, scat)
def write_edac_phase_file(self, f, scat):
"""
write a phase file for EDAC.
@param f: file path or output stream (an object with a write method).
@param scat: a slice of the self.scattering array belonging to the same atom class.
@return: None
"""
if hasattr(f, "write") and callable(f.write):
energies = np.unique(scat['e'])
ne = energies.shape[0]
lmax = scat['l'].max()
if ne == 1:
f.write("1 {lmax} regular real\n".format(lmax=lmax))
else:
f.write("{nk} E(eV) {lmax} regular real\n".format(nk=ne, lmax=lmax))
for energy in energies:
sel = scat['e'] == energy
idx = np.where(sel)
energy_scat = scat[idx]
if ne > 1:
f.write("{0:.3f} ".format(energy))
for item in energy_scat:
pha = np.sign(item['t'].real) * np.arcsin(np.sqrt(np.abs(item['t'].imag)))
f.write(" {0:.6f}".format(pha))
for i in range(len(energy_scat), lmax + 1):
f.write(" 0")
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as fi:
self.write_edac_phase_file(fi, scat)
def parse_radial_file(self, f):
"""
parse the radial matrix element output file from phagen version 2.2.
the file contains 7 header lines and one data line per requested energy.
the data line contains real and imaginary parts of the matrix elements.
the first four columns contain the electric dipole transitions Rd(li --> li - 1) and Rd(li --> li + 1),
followed by higher orders that we do not use here.
@param f: file or path (any file-like or path-like object that can be passed to numpy.genfromtxt).
@return: None
@raise ValueError if the file is in a wrong format.
"""
data = np.atleast_2d(np.genfromtxt(f, skip_header=7))
if data.shape[0] != self.params.kinetic_energies.shape[0] or data.shape[1] < 4:
raise ValueError(f"Unexpected array size of Phagen radial matrix elements output: "
f"expected ({self.params.kinetic_energies.shape[0]}, >= 4), received {data.shape}")
self.emission = np.resize(self.emission, data.shape[0:1])
emission = self.emission
emission['e'] = self.params.kinetic_energies
emission['dw'] = data[:, 0] + 1j * data[:, 1]
emission['up'] = data[:, 2] + 1j * data[:, 3]
def write_edac_emission_file(self, f):
"""
write the radial photoemission matrix element in EDAC format.
requires self.emission, self.params.kinetic_energies and self.params.initial_state.
@param f: file path or output stream (an object with a write method).
@return: None
"""
if hasattr(f, "write") and callable(f.write):
l0 = self.params.l_init
energies = self.params.kinetic_energies
emission = self.emission
emission['e'] = energies
ne = energies.shape[0]
if ne == 1:
f.write("1 regular2 {l0}\n".format(l0=l0))
else:
f.write("{nk} E(eV) regular2 {l0}\n".format(nk=ne, l0=l0))
for item in emission:
if ne > 1:
f.write("{0:.3f} ".format(item['e']))
f.write(" {0:.6f} {1:.6f}".format(item['up'].real, item['up'].imag))
f.write(" {0:.6f} {1:.6f}".format(item['dw'].real, item['dw'].imag))
f.write("\n")
else:
with open(f, "wt", encoding="latin1") as of:
self.write_edac_emission_file(of)
def write_edac_emission(self, filename_format):
"""
write the radial photoemission matrix element in EDAC format.
requires self.scattering, self.emission, self.params.kinetic_energies and self.params.initial_state.
@param filename_format: file name including, optionally, a placeholder {} for the atom class.
since phagen calculates only one emitter, the placeholder is not necessary.
@return: dictionary that maps atom classes to file names.
since phagen calculates only one emitter, this dictionary will contain just one entry.
"""
scat = self.scattering
atom = scat['a'][0]
f = filename_format.format(atom)
self.write_edac_emission_file(f)
files = {atom: f}
return files

784
pmsco/cluster.py Normal file → Executable file
View File

File diff suppressed because it is too large Load Diff

163
pmsco/config.py Normal file
View File

@@ -0,0 +1,163 @@
"""
@package pmsco.config
infrastructure for configurable objects
@author Matthias Muntwiler
@copyright (c) 2021-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import collections.abc
import inspect
import logging
import os
from pathlib import Path
from string import Template
from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union
logger = logging.getLogger(__name__)
PathLike = Union[str, os.PathLike]
DataDict = Mapping[str, Union[str, int, float, Iterable, Mapping]]
def resolve_path(path: PathLike, dirs: Mapping[str, Any]):
"""
Resolve a file path by replacing placeholders.
Placeholders are enclosed in curly braces.
Values for all possible placeholders are provided in a dictionary.
@param path: str, Path or other path-like.
Example: '${work}/test/testfile.dat'.
@param dirs: Dictionary mapping placeholders to project paths.
The paths can be str, Path or other path-like
Example: {'work': '/home/user/work'}
@return: pathlib.Path object
"""
return Path(*(Template(p).substitute(dirs) for p in Path(path).parts))
class ConfigurableObject(object):
"""
Parent class for objects that can be configured from a runfile
The runfile is a JSON file that contains object data in a nested dictionary structure.
In the dictionary structure the keys are property or attribute names of the object to be initialized.
Keys starting with a non-alphabetic character (except for some special keys like __class__) are ignored.
These can be used as comments, or they protect private attributes.
The values can be numeric values, strings, lists or dictionaries.
Simple values are simply assigned using setattr.
This may call a property setter if defined.
Lists are iterated. Each item is appended to the attribute.
The attribute must implement an append method in this case.
If an item is a dictionary and contains the special key '__class__',
an object of that class is instantiated and recursively initialized with the dictionary elements.
This requires that the class can be found in the module scope passed to the parser methods,
and that the class inherits from this class.
Cases that can't be covered easily using this mechanism
should be implemented in a property setter.
Value-checking should also be done in a property setter (or the append method in sequence-like objects).
Attributes
----------
project_symbols: Dictionary of symbols that should be used to resolve class and function names.
This is usually the globals() dictionary of the project module.
"""
def __init__(self):
super().__init__()
self.project_symbols: Optional[Mapping[str, Any]] = None
def set_properties(self, symbols: Optional[Mapping[str, Any]],
data_dict: DataDict,
project: 'ConfigurableObject') -> None:
"""
Set properties from dictionary.
@param symbols: Dictionary of symbols that should be used to resolve class names.
This is usually the globals() dictionary of the project module.
Classes are resolved using the eval function.
@param data_dict: Dictionary of properties to set.
See the class description for details.
@param project: Reference to the project object.
@return: None
"""
self.project_symbols = symbols
for key in data_dict:
if key[0].isalpha():
self.set_property(symbols, key, data_dict[key], project)
def set_property(self, symbols: Optional[Mapping[str, Any]],
key: str,
value: DataDict,
project: 'ConfigurableObject') -> None:
"""
Set one property.
@param symbols: Dictionary of symbols that should be used to resolve class names.
This is usually the globals() dictionary of the project module.
Classes are resolved using the eval function.
@param key: Attribute name to set.
@param value: New value of the attribute.
@param project: Reference to the project object.
@return: None
"""
obj = self.parse_object(symbols, value, project)
if hasattr(self, key):
if obj is not None:
if isinstance(obj, collections.abc.MutableSequence):
attr = getattr(self, key)
for item in obj:
attr.append(item)
elif isinstance(obj, collections.abc.Mapping):
d = getattr(self, key)
if d is not None and isinstance(d, collections.abc.MutableMapping):
d.update(obj)
else:
setattr(self, key, obj)
else:
setattr(self, key, obj)
else:
setattr(self, key, obj)
else:
logger.warning(f"class {self.__class__.__name__} does not have attribute {key}.")
def parse_object(self, symbols: Optional[Mapping[str, Any]],
value: DataDict,
project: 'ConfigurableObject') -> object:
if isinstance(value, collections.abc.MutableMapping) and "__class__" in value:
cn = value["__class__"]
try:
c = eval(cn, symbols)
except (AttributeError, KeyError, NameError, ValueError):
logger.critical(f"can't resolve class name {cn}")
raise
s = inspect.signature(c)
if 'project' in s.parameters:
o = c(project=project)
else:
o = c()
o.set_properties(symbols, value, project)
elif isinstance(value, collections.abc.MutableSequence):
o = [self.parse_object(symbols, i, project) for i in value]
else:
o = value
return o

301
pmsco/data.py
View File

@@ -1,24 +1,36 @@
"""
@package pmsco.data
import, export, evaluation of msc data
Import, export, evaluation of msc data.
This module provides common functions for loading/saving and manipulating PED scan data sets.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import os
import logging
import math
import numpy as np
import numpy.typing as npt
import os
import scipy.special
import scipy.optimize as so
import loess.loess as loess
from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union
import h5py
logger = logging.getLogger(__name__)
try:
import loess
except (ModuleNotFoundError, ImportError) as e:
loess = None
logger.critical("Error importing the loess package.", exc_info=e)
## energy, intensity
DTYPE_EI = [('e', 'f4'), ('i', 'f4')]
## energy, theta, phi, intensity
@@ -35,9 +47,11 @@ DTYPE_TP = [('t', 'f4'), ('p', 'f4')]
DTYPE_TPI = [('t', 'f4'), ('p', 'f4'), ('i', 'f4')]
## theta, phi, intensity, sigma (standard deviation)
DTYPE_TPIS = [('t', 'f4'), ('p', 'f4'), ('i', 'f4'), ('s', 'f4')]
## intensity, theta, phi
DTYPE_ITP = [('i', 'f4'), ('t', 'f4'), ('p', 'f4')]
DTYPES = {'EI': DTYPE_EI, 'ETPI': DTYPE_ETPI, 'ETPIS': DTYPE_ETPIS, 'ETPAI': DTYPE_ETPAI, 'ETPAIS': DTYPE_ETPAIS,
'TP': DTYPE_TP, 'TPI': DTYPE_TPI, 'TPIS': DTYPE_TPIS, }
'TP': DTYPE_TP, 'TPI': DTYPE_TPI, 'TPIS': DTYPE_TPIS, 'ITP': DTYPE_ITP, }
DATATYPES = DTYPES.keys
## supportd scan types
@@ -47,8 +61,11 @@ DATATYPES = DTYPES.keys
# @arg @c 'TP' theta - phi (holo scan)
SCANTYPES = ['E', 'EA', 'ET', 'TP']
GenTextFileLike = Union[str, os.PathLike, Iterable[str], int]
OSFileLike = Union[str, os.PathLike, int]
def create_etpi(shape, sigma_column=True):
def create_etpi(shape: Tuple[int], sigma_column: bool = True) -> np.ndarray:
"""
create an ETPI array of a given size.
@@ -56,6 +73,7 @@ def create_etpi(shape, sigma_column=True):
the array is initialized with zeroes.
@param shape (tuple) shape of the array
@param sigma_column: whether the array should include a sigma field (ETPIS type instead of ETPI)
"""
if sigma_column:
data = np.zeros(shape, dtype=DTYPE_ETPIS)
@@ -64,7 +82,7 @@ def create_etpi(shape, sigma_column=True):
return data
def create_data(shape, datatype='', dtype=None):
def create_data(shape: Tuple[int], datatype: str = '', dtype: Optional[npt.DTypeLike] = None) -> np.ndarray:
"""
create a data array of a given size and type.
@@ -82,7 +100,108 @@ def create_data(shape, datatype='', dtype=None):
return data
def load_plt(filename, int_column=-1):
def holo_grid(theta_start: float = 90., theta_step: float = 1., theta_range: float = 90.,
phi_start: float = 0., phi_range: float = 360., phi_refinement: float = 1.):
"""
Generator of a holo grid with constant point density in solid angle.
The generator yields the polar coordinates of a hologram scan in the traditional Osterwalder fashion,
where the grid points are distributed evenly on the hemisphere by varying the azimuthal step size,
while the polar step size is constant.
The yield are tuples (theta, phi) in degrees.
Theta is the polar, phi the azimuthal coordinate.
@param theta_start Maximum polar angle in degrees, 0..90. Defaults to 90 (grazing emission).
@param theta_step Polar angle step in degrees, 1..90. Defaults to 1.
@param theta_range Polar angle range in degrees, 1..th_start. Defaults to 90.
@param phi_start Azimuthal start angle in degrees. Defaults to 0.
This azimuth is included at every polar step.
@param phi_range Azimuthal range in degrees. Defaults to 360.
@param phi_refinement Azimuthal refinement/oversampling (scalar). Defaults to 1.
A refinement of 2 yields a factor 2 more grid points in the azimuthal sub-scans.
@return yield tuples (theta, phi) in degrees
"""
deg2rad = 0.01745329
def calc_phi_step(th):
if th < 0.5 or int(phi_range * math.sin(th * deg2rad) * phi_refinement / theta_step) == 0:
phi_st = 0.0
else:
phi_st = phi_range / int(th / theta_start * phi_range / theta_step)
if abs(phi_st) < 0.001:
phi_st = 360.
return phi_st
for theta in np.arange(theta_range, -theta_step, -theta_step):
phi_step = calc_phi_step(theta)
for phi in np.arange(phi_start, phi_range, phi_step):
yield theta, phi
def holo_array(generator: Callable[..., Iterable[Tuple[float, float]]],
generator_args: Dict,
datatype: str = 'TP',
dtype: Optional[npt.DTypeLike] = None) -> np.ndarray:
"""
Create an hologram scan grid in a numpy array.
A holo data array is a numpy structured array containing at least
a column for theta (polar angle) and phi (azimuthal angle).
The theta and phi columns are filled with angles from the holo_grid (or custom generator) function.
The array can contain further columns for energy, intensity, etc. according to the data type specified.
These columns are initialized with zeroes.
@param generator Generator that yields tuples (theta, phi) for each grid point,
given the keyword arguments kwargs.
Defaults to holo_grid, the traditional Osterwalder holo scan.
@param generator_args Keyword arguments to be passed to the generator.
For arguments of the traditional holo scan, see the documentation of holo_grid.
@param datatype See DATATYPES. Must contain 'T' and 'P' dimensions. Defaults to 'TP'.
@param dtype See DTYPES. Must contain a 't' and 'p' column. Takes precedence over datatype.
Defaults to None (not specified).
"""
if not dtype:
dtype = DTYPES[datatype]
tp = np.fromiter(generator(**generator_args), dtype=DTYPES['TP'])
result = np.zeros(tp.shape, dtype=dtype)
result['t'] = tp['t']
result['p'] = tp['p']
return result
def analyse_holoscan_steps(holoscan: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Find the polar and azimuthal steps in a holoscan.
@param holoscan:
@return: thetas: unique theta angles. sorted.
dtheta: theta steps for each theta
dphi: phi step for each theta
"""
thetas, indices, counts = np.unique(holoscan['t'], return_index=True, return_counts=True)
dtheta = np.diff(thetas)
dtheta = np.append(dtheta, dtheta[-1])
adjusted_phis = np.append(holoscan['p'], holoscan['p'][-1])
phis0 = adjusted_phis[indices]
phis1 = adjusted_phis[indices+1]
dphi = phis1 - phis0
phi_range = counts[-1] * dphi[-1]
dphi[counts <= 1] = phi_range
return thetas, dtheta, dphi
def load_plt(filename: GenTextFileLike, int_column: int = -1) -> np.ndarray:
"""
loads ETPI data from an MSC output (plt) file
@@ -109,12 +228,13 @@ def load_plt(filename, int_column=-1):
data[i]['p'] = phi
data[i]['i'] = selected intensity column
"""
data = np.genfromtxt(filename, usecols=(0, 2, 3, int_column), dtype=DTYPE_ETPI)
data = np.atleast_1d(np.genfromtxt(filename, usecols=(0, 2, 3, int_column), dtype=DTYPE_ETPI))
sort_data(data)
return data
def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_cluster=False):
def load_edac_pd(filename: OSFileLike, int_column: int = -1,
energy: float = 0.0, theta: float = 0.0, phi: float = 0.0, fixed_cluster: bool = False) -> np.ndarray:
"""
load ETPI or ETPAI data from an EDAC PD output file.
@@ -149,7 +269,8 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
data[i]['i'] = selected intensity column
@endverbatim
"""
with open(filename, 'r') as f:
with open(filename, "rt", encoding="latin1") as f:
header1 = f.readline().strip()
header2 = f.readline().strip()
if not header1 == '--- scan PD':
@@ -181,7 +302,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
logger.warning("unexpected EDAC output file column name")
break
cols = tuple(cols)
raw = np.genfromtxt(filename, usecols=cols, dtype=dtype, skip_header=2)
raw = np.atleast_1d(np.genfromtxt(filename, usecols=cols, dtype=dtype, skip_header=2))
if fixed_cluster:
etpi = np.empty(raw.shape, dtype=DTYPE_ETPAI)
@@ -210,7 +331,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
return etpi
def load_etpi(filename):
def load_etpi(filename: GenTextFileLike) -> np.ndarray:
"""
loads ETPI or ETPIS data from a text file
@@ -245,7 +366,7 @@ def load_etpi(filename):
return data
def load_data(filename, dtype=None):
def load_data(filename: GenTextFileLike, dtype: Optional[npt.DTypeLike] = None):
"""
load column data (ETPI, and the like) from a text file.
@@ -259,20 +380,39 @@ def load_data(filename, dtype=None):
DTYPE_EI, DTYPE_ETPI, DTYPE_ETPIS, DTYPE_ETPAI, or DTYPE_ETPAIS.
by default, the function uses the extension to determine the data type.
the actual type can be read from the dtype attribute of the returned array.
if the extension is missing, DTYPE_EI is assumed.
@return one-dimensional numpy structured ndarray with data
@raise IOError if the file cannot be read.
@raise IndexError if the number of columns is lower than expected based on the dtype or extension.
"""
if not dtype:
(root, ext) = os.path.splitext(filename)
datatype = ext[1:].upper()
dtype = DTYPES[datatype]
ext_type = ext[1:].upper()
try:
dtype = DTYPES[ext_type]
except KeyError:
dtype = DTYPE_EI
data = np.loadtxt(filename, dtype=dtype)
sort_data(data)
return data
def save_data(filename, data):
def format_extension(data: np.ndarray) -> str:
"""
format the file extension based on the contents of an array.
@param data ETPI-like structured numpy.ndarray.
@return: file extension string including the leading period.
"""
return "." + "".join(data.dtype.names)
def save_data(filename: OSFileLike, data: npt.ArrayLike) -> None:
"""
save column data (ETPI, and the like) to a text file.
@@ -288,7 +428,7 @@ def save_data(filename, data):
np.savetxt(filename, data, fmt='%g')
def sort_data(data):
def sort_data(data: np.ndarray) -> None:
"""
sort scan data (ETPI and the like) in a consistent order.
@@ -311,7 +451,8 @@ def sort_data(data):
data.sort(kind='mergesort', order=sort_key)
def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
def restructure_data(data: np.ndarray, dtype: Optional[npt.DTypeLike] = None,
defaults: Optional[Mapping] = None) -> np.ndarray:
"""
restructure the type of a data array by adding or removing columns.
@@ -331,23 +472,29 @@ def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
undefined fields are initialized to zero.
if the parameter is unspecified, all fields are initialized to zero.
@return: re-structured numpy array
@return: re-structured numpy array or
@c data if the new and original data types are the same.
"""
new_data = np.zeros(data.shape, dtype=dtype)
fields = [dt[0] for dt in dtype if dt[0] in data.dtype.names]
if dtype is None:
dtype = DTYPE_ETPAIS
if data.dtype == dtype:
return data
else:
new_data = np.zeros(data.shape, dtype=dtype)
fields = [dt[0] for dt in dtype if dt[0] in data.dtype.names]
if defaults is not None:
for field, value in defaults.iteritems():
if field in new_data.dtype.names:
new_data[field] = value
if defaults is not None:
for field, value in defaults.items():
if field in new_data.dtype.names:
new_data[field] = value
for field in fields:
new_data[field] = data[field]
for field in fields:
new_data[field] = data[field]
return new_data
return new_data
def common_dtype(scans):
def common_dtype(scans: Iterable[Union[npt.ArrayLike, npt.DTypeLike]]) -> npt.DTypeLike:
"""
determine the common data type for a number of scans.
@@ -378,7 +525,7 @@ def common_dtype(scans):
return dtype
def detect_scan_mode(data):
def detect_scan_mode(data: np.ndarray) -> Tuple[List[str], Dict[str, np.ndarray]]:
"""
detect the scan mode and unique scan positions in a data array.
@@ -464,7 +611,7 @@ def detect_scan_mode(data):
return scan_mode, scan_positions
def filter_tp(data, filter):
def filter_tp(data: np.ndarray, _filter: np.ndarray) -> np.ndarray:
"""
select data points from an ETPI array that match theta and phi coordinates of another ETPI array.
@@ -472,7 +619,7 @@ def filter_tp(data, filter):
@param data ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
@param filter ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
@param _filter ETPI-like structured numpy.ndarray (ETPI, ETPIS, ETPAI, ETPAIS).
only 't' and 'p' columns are used.
@return filtered data (numpy.ndarray)
@@ -481,18 +628,19 @@ def filter_tp(data, filter):
"""
# copy theta,phi into separate structured arrays
data_tp = np.zeros_like(data, dtype=[('t', '<i4'), ('p', '<i4')])
filter_tp = np.zeros_like(filter, dtype=[('t', '<i4'), ('p', '<i4')])
filt_tp = np.zeros_like(_filter, dtype=[('t', '<i4'), ('p', '<i4')])
# multiply by 10, round to integer
data_tp['t'] = np.around(data['t'] * 10.0)
data_tp['p'] = np.around(data['p'] * 10.0)
filter_tp['t'] = np.around(filter['t'] * 10.0)
filter_tp['p'] = np.around(filter['p'] * 10.0)
filt_tp['t'] = np.around(_filter['t'] * 10.0)
filt_tp['p'] = np.around(_filter['p'] * 10.0)
# calculate intersection
idx = np.in1d(data_tp, filter_tp)
idx = np.in1d(data_tp, filt_tp)
result = data[idx]
return result
def interpolate_hemi_scan(rect_tpi, hemi_tpi):
def interpolate_hemi_scan(rect_tpi: np.ndarray, hemi_tpi: np.ndarray) -> np.ndarray:
"""
interpolate a hemispherical scan from a rectangular angle scan.
@@ -524,7 +672,9 @@ def interpolate_hemi_scan(rect_tpi, hemi_tpi):
hemi_tpi['i'][sel_theta] = result
return hemi_tpi
def reshape_2d(flat_data, axis_columns, return_column='i'):
def reshape_2d(flat_data: np.ndarray, axis_columns: Sequence[str], return_column: str = 'i') -> \
Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
reshape an ETPI-like array into a two-dimensional array according to the scan axes.
@@ -533,7 +683,9 @@ def reshape_2d(flat_data, axis_columns, return_column='i'):
the array must be sorted in the order of axis_labels.
@param axis_columns list of column names that designate the axes
@param return_column: name of field to return in two dimensions
@return the tuple (result_data, axis0, axis1), where
@arg result_data (ndarray) new two-dimensional ndarray of the scan
@arg axis0 (ndarray) scan positions along the first dimension
@@ -548,7 +700,7 @@ def reshape_2d(flat_data, axis_columns, return_column='i'):
return data.copy(), axis0, axis1
def calc_modfunc_mean(data):
def calc_modfunc_mean(data: np.ndarray) -> np.ndarray:
"""
calculates the modulation function using the mean value of data.
this is a simplified calculation method
@@ -584,7 +736,7 @@ def calc_modfunc_mean(data):
return modf
def calc_modfunc_loess(data):
def calc_modfunc_loess(data: np.ndarray, smth: float = 0.4) -> np.ndarray:
"""
calculate the modulation function using LOESS (locally weighted regression) smoothing.
@@ -609,9 +761,11 @@ def calc_modfunc_loess(data):
the modulation function is calculated for the finite-valued scan points.
NaNs are ignored and do not affect the finite values.
@return copy of the data array with the modulation function in the 'i' column.
@param smth: size of the smoothing window relative to the size of the scan.
reasonable values are between 0.2 and 0.5.
the default value 0.4 has been found to work in many cases.
@todo is a fixed smoothing factor of 0.5 okay?
@return copy of the data array with the modulation function in the 'i' column.
"""
sel = np.isfinite(data['i'])
_data = data[sel]
@@ -626,7 +780,7 @@ def calc_modfunc_loess(data):
factors = [_data[axis] for axis in scan_mode]
lo.set_x(np.hstack(tuple(factors)))
lo.set_y(_data['i'])
lo.model.span = 0.5
lo.model.span = smth
loess.loess(lo)
modf['i'][sel] = lo.get_fitted_residuals() / lo.get_fitted_values()
@@ -636,20 +790,27 @@ def calc_modfunc_loess(data):
return modf
def rfactor(experiment, theory):
def square_diff_rfactor(experiment: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a calculated modulation function.
Calculate the R-factor from the normalized sum of squared differences.
if the sigma column is present in experiment and non-zero,
If the sigma column is present in experiment and non-zero,
the R-factor terms are weighted by 1/sigma**2.
the input arrays must have the same shape and the coordinate columns must be identical (they are ignored).
the array elements are compared element-by-element.
terms having NaN intensity are ignored.
The input arrays must have the same shape and the coordinate columns must be identical.
The array elements are compared element-by-element.
The values of the coordinate arrays do not influence the result.
Terms having NaN intensity are ignored.
@param experiment: ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
This function can be specified in the Scan.rfactor_func parameter of the project.
@param theory: ETPI or ETPAI array containing the calculated modulation functions.
@param experiment: (numpy structured array)
ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
If an `s` field is present and non-zero,
the R-factor terms are weighted by 1/sigma**2.
@param theory: (numpy structured array)
ETPI or ETPAI array containing the theoretical function.
@return scalar R-factor in the range from 0.0 to 2.0.
@@ -669,7 +830,7 @@ def rfactor(experiment, theory):
return sum1 / sum2
def scaled_rfactor(scale, experiment, weights, theory):
def scaled_rfactor_func(scale: float, experiment: np.ndarray, weights: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a modulation function against the measurement with scaled amplitude.
@@ -699,6 +860,7 @@ def scaled_rfactor(scale, experiment, weights, theory):
@raise ValueError if all experiments and theory values or all weights are zero.
"""
difs = weights * (scale * experiment - theory) ** 2
sums = weights * (scale ** 2 * experiment ** 2 + theory ** 2)
sum1 = difs.sum(dtype=np.float64)
@@ -706,7 +868,7 @@ def scaled_rfactor(scale, experiment, weights, theory):
return sum1 / sum2
def optimize_rfactor(experiment, theory):
def optimize_rfactor(experiment: np.ndarray, theory: np.ndarray) -> float:
"""
calculate the R-factor of a calculated modulation function against the measurement, adjusting their amplitude.
@@ -717,13 +879,15 @@ def optimize_rfactor(experiment, theory):
this is useful if the amplitudes of the two functions do not match due to systematic effects
of the calculation or the measurement.
the optimization is done in a scipy.optimize.least_squares optimization of the scaled_rfactor() function.
the optimization is done in a scipy.optimize.least_squares optimization of the scaled_rfactor_func() function.
the initial guess of the scaling factor is 0.7, the constraining boundaries are 1/10 and 10.
the input arrays must have the same shape and the coordinate columns must be identical (they are ignored).
the array elements are compared element-by-element.
terms having NaN intensity are ignored.
This function can be specified in the Scan.rfactor_func parameter of the project.
@param experiment: ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
@param theory: ETPI or ETPAI array containing the calculated modulation functions.
@@ -740,13 +904,13 @@ def optimize_rfactor(experiment, theory):
else:
wgts = np.ones_like(experiment['i'])
result = so.least_squares(scaled_rfactor, 0.7, bounds=(0.1, 10.0), args=(experiment['i'], wgts, theory['i']))
result_r = scaled_rfactor(result.x, experiment['i'], wgts, theory['i'])
result = so.least_squares(scaled_rfactor_func, 0.7, bounds=(0.1, 10.0), args=(experiment['i'], wgts, theory['i']))
result_r = scaled_rfactor_func(result.x, experiment['i'], wgts, theory['i'])
return result_r
def alpha_average(data):
def alpha_average(data: np.ndarray) -> np.ndarray:
"""
average I(alpha, theta, phi) over alpha.
@@ -776,7 +940,7 @@ def alpha_average(data):
return result
def phi_average(data):
def phi_average(data: np.ndarray) -> np.ndarray:
"""
average I(theta, phi) over phi.
@@ -794,9 +958,9 @@ def phi_average(data):
names = list(data.dtype.names)
names.remove('p')
dtype = [(name, data.dtype[name].str) for name in names]
result = create_data((nt), dtype=dtype)
result = create_data((nt,), dtype=dtype)
for i,t in enumerate(t_axis):
for i, t in enumerate(t_axis):
sel = np.abs(scan_positions['t'] - t) < 0.01
for name in names:
result[name][i] = np.mean(data[name][sel], dtype=np.float64)
@@ -806,7 +970,7 @@ def phi_average(data):
return result
def alpha_mirror_average(data):
def alpha_mirror_average(data: np.ndarray) -> np.ndarray:
"""
calculate the average of I(alpha, theta, phi) and I(-alpha, theta, phi).
@@ -838,3 +1002,14 @@ def alpha_mirror_average(data):
logger.warning('asymmetric alpha scan. skipping alpha mirror average.')
return result1
if loess is not None:
default_modfunc = calc_modfunc_loess
logger.info("pmsco.data.default_modfunc = pmsco.data.calc_modfunc_loess")
else:
default_modfunc = calc_modfunc_mean
logger.warning("pmsco.data.default_modfunc = pmsco.data.calc_modfunc_mean")
default_rfactor = square_diff_rfactor
logger.info("pmsco.data.default_rfactor = pmsco.data.square_diff_rfactor")

169
pmsco/database/access.py Normal file
View File

@@ -0,0 +1,169 @@
"""
@package pmsco.database.access
wrapper classes for access to a pmsco database
the most import class to be used is DatabaseAccess.
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access here
# ...
# commit transaction
session.commit()
# continue in new transaction
# at the end of the context
# the session is closed and orm objects are detached from the database.
~~~~~~
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import fasteners
import logging
from pathlib import Path
import pmsco.database.orm as orm
logger = logging.getLogger(__name__)
class _DummyLock(object):
"""
dummy lock used for in memory database.
"""
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
class LockedSession(object):
"""
database session context manager
this context manager (to be used in a with statement)
acquires a lock on the database lock file
and provides a database session (orm.Session()).
the session is closed (and pending transactions committed) on exit.
if an exception occurs, pending transactions are rolled back before the session is closed.
@note the term _session_ refers to a session in sqlalchemy.
"""
def __init__(self, lock_file=None):
self.lock_file = lock_file
self._session = None
self._lock = None
def __enter__(self):
self._lock = self.lock()
self._lock.__enter__()
self._session = orm.Session()
return self._session
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self._session.close()
else:
self._session.rollback()
self._session.close()
self._lock.__exit__(exc_type, exc_val, exc_tb)
self._lock = None
def lock(self):
"""
create a file-lock context manager for the database.
this is either a fasteners.InterProcessLock object on self._lock_filename
or a _DummyLock object if the database is in memory.
InterprocessLock allows to serialize access to the database by means of a lock file.
this is necessary if multiple pmsco instances require access to the same database.
_DummyLock is used with an in-memory database which does not require locking.
the lock object can be used as context-manager in a with statement.
"""
if self.lock_file:
return fasteners.InterProcessLock(self.lock_file)
else:
return _DummyLock()
class DatabaseAccess(object):
"""
basic database connection
this class maintains a database connection and builds session objects.
a _session_ corresponds to an sqlalchemy session, which defines the lifecycle of mapped objects.
a session can open one or multiple (subsequent) transactions.
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access
session.commit()
~~~~~~
the session object is a context handler.
it commits the transaction and closes the session at the end of the context.
if an exception occurs, it rolls back the transaction and closes the session before passing the exception.
"""
def __init__(self):
self.db_file = ""
self.lock_file = ""
def connect(self, db_file, lock_file=""):
"""
connect to a new or existing database file.
if the file does not exist, or if it is empty, a new database schema is created.
@param db_file: name of a file or ":memory:" for an in-memory database.
@param lock_file: name of a file that is used to lock the database.
by default, the db_filename with a suffix of ".lock" is used.
for most uses, the default should be fine.
the argument is provided mainly for testing the locking functionality.
this must be a file that is not used for anything else.
the file does not need to exist.
it's best if the file is in the same directory as the database file.
all clients of a database must use the same lock file.
@return: None
"""
self.db_file = db_file
if lock_file:
self.lock_file = lock_file
elif db_file == ":memory:":
self.lock_file = ""
else:
self.lock_file = Path(str(db_file) + ".lock")
orm.connect(orm.sqlite_link(self.db_file))
def session(self):
"""
open a database session.
this function returns a pmsco.database.util.LockedSession object
which is a context handler that provides an sqlalchemy session
that is locked against concurrent access from other DatabaseAccess instances.
see the class description for an example usage pattern.
@return: pmsco.database.util.LockedSession() object.
"""
return LockedSession(self.lock_file)

329
pmsco/database/common.py Normal file
View File

@@ -0,0 +1,329 @@
"""
@package pmsco.database.common
common database operations
this module gathers a number of common database operations.
all functions require an open session object from pmsco.database.access.DatabaseAccess.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import sqlalchemy
import pmsco.database.orm as orm
logger = logging.getLogger(__name__)
def filter_project(query, project_or_name_or_id):
"""
filter a query by project
@param query: sqlalchemy query object
@param project_or_name_or_id: orm.Project object or project name or project id.
@return: modified query
"""
if isinstance(project_or_name_or_id, orm.Project):
query = query.filter(orm.Project == project_or_name_or_id)
elif isinstance(project_or_name_or_id, int):
query = query.filter(orm.Project.id == project_or_name_or_id)
else:
query = query.filter(orm.Project.name == project_or_name_or_id)
return query
def filter_job(query, job_or_name_or_id):
"""
filter a query by job
@param query: sqlalchemy query object
@param job_or_name_or_id: orm.Job object or job name or job id.
@return: modified query
"""
if isinstance(job_or_name_or_id, orm.Job):
query = query.filter(orm.Job == job_or_name_or_id)
elif isinstance(job_or_name_or_id, int):
query = query.filter(orm.Job.id == job_or_name_or_id)
else:
query = query.filter(orm.Job.name == job_or_name_or_id)
return query
def query_params(session, project=None, job=None):
"""
query parameter names and their associated objects from the database
the result is a dictionary of orm.Param objects mapped to their respective keys.
the parameters can be filtered by project and/or job.
if no arguments are given, parameters from all projects are returned.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: dictionary of parameters
"""
query = session.query(orm.Param).join(orm.ParamValue).join(orm.Model).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
params = query.all()
params = {param.key: param for param in params}
return params
def query_tags(session, project=None, job=None):
"""
query tag names and their associated objects from the database
the result is a dictionary of orm.Tag objects mapped to their respective keys.
the tags can be filtered by project and/or job.
if no arguments are given, tags from all projects are returned.
@note the orm.Job.tags mapping is an alternative way to access job tags.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: dictionary of tags
"""
query = session.query(orm.Tag).join(orm.JobTag).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
tags = query.all()
tags = {tag.key: tag for tag in tags}
return tags
def query_job_tags(session, project=None, job=None):
"""
query tags (keys and values) from the database
the result is a dictionary of tag values (str) mapped to their respective keys (str).
the tags can be filtered by project and/or job.
if no arguments are given, tags from all projects are returned.
@note for one specific job, this is equivalent to the orm.Job.tags mapping.
@note make sure previous changes have been committed. else the query may not find all records.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project: orm.Project object or project name or project id.
default: don't filter projects.
@param job: orm.Job object or job name or job id.
default: don't filter jobs
@return: tags dictionary {key: value}
"""
query = session.query(orm.JobTag).join(orm.Job).join(orm.Project)
if project is not None:
query = filter_project(query, project)
if job is not None:
query = filter_job(query, job)
job_tags = query.all()
job_tags = {jt.tag.key: jt.value for jt in job_tags}
return job_tags
def register_project(session, name, code, allow_existing=False):
"""
register (insert or query) a project with the database.
a new project record with the given parameters is inserted into the database.
if a project of the same name already exists, the existing record is returned.
@attention the orm.Project.id field is undefined until the session is committed!
it's better to identify a project by name or orm.Project object.
@note make sure previous changes have been committed. else the query may not find an existing project.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if a new project entry has been added.
@param name: project name. must be unique within the database.
@param code: name of the project module.
@param allow_existing: selects the behaviour if a project record exists in the database:
return the corresponding orm.Project (True) or raise an exception (False, default).
the exception is ValueError.
@return: orm.Project object.
the object can be used and modified as long as the session is active.
note that the id attribute is invalid until the session is committed!
@raise ValueError if the job exists and allow_existing is False.
"""
query = session.query(orm.Project)
query = query.filter(orm.Project.name == name)
project = query.one_or_none()
if project is None:
project = orm.Project(name=name, code=code)
session.add(project)
session.commit()
elif not allow_existing:
raise ValueError(f"project {project.name} exists")
return project
def get_project(session, project_or_name_or_id):
"""
resolve a project by name or id.
this function resolves a project specification to an orm.Project object.
if `project_or_name_or_id` is an orm.Project object, it just returns that object without any checks.
else, the project is looked up in the database.
@attention if `project_or_name_or_id` is an orm.Project object the function returns it without checks!
that means if the object is detached, you cannot use it to query results from the database.
if you need an object that is valid and in sync with the database,
resolve it by name or id!
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project_or_name_or_id: orm.Project object or project name or project id.
@return: orm.Project object
"""
if isinstance(project_or_name_or_id, orm.Project):
project = project_or_name_or_id
elif isinstance(project_or_name_or_id, int):
project = session.query(orm.Project).get(project_or_name_or_id)
else:
query = session.query(orm.Project)
query = query.filter(orm.Project.name == project_or_name_or_id)
project = query.one()
return project
def register_job(session, project, job_name, allow_existing=False, **job_attr):
"""
register (insert or query) a new job with the database.
a new job record with the given parameters is inserted into the database.
if a job of the same name exists within the given project, the existing record is returned
(without modifications!).
@note make sure previous changes have been committed. else the query may not find an existing project.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if a new job entry has been added.
@param project: orm.Project object or project name or project id.
@param job_name: name of job. unique in the project
@param job_attr: optional attributes of the job.
the keywords correspond to attribute names of the pmsco.database.Job object.
@param allow_existing: selects the behaviour if a job record exists in the database:
return the corresponding orm.Job (True) or raise an exception (False, default).
the exception is ValueError.
@return: orm.Job object.
the object can be used and modified as long as the session is active.
note that the id attribute is invalid until the session is committed!
@raise ValueError if the job exists and allow_existing is False.
"""
project = get_project(session, project)
query = session.query(orm.Job).join(orm.Project)
query = query.filter(orm.Project.name == project.name)
query = query.filter(orm.Job.name == job_name)
job = query.one_or_none()
if job is None:
job = orm.Job()
job.name = job_name
job.project = project
optional_args = {'mode', 'machine', 'git_hash', 'datetime', 'processes', 'hours', 'description'}
for name, value in job_attr.items():
if name in optional_args:
setattr(job, name, value)
session.add(job)
session.commit()
elif not allow_existing:
raise ValueError(f"a job {job_name} exists in project {project.name}")
return job
def get_job(session, project_or_name_or_id, job_or_name_or_id):
"""
resolve a job by name or id.
this function resolves any combination of project and job specification to an orm.Job object.
if `job_or_name_or_id` is an orm.Job object, it just returns that object without any checks.
else, the job is looked up in the database.
@attention if `job_or_name_or_id` is an orm.Job object the function returns it without checks!
that means if the object is detached, you cannot query results from the database.
if you need an object that is valid and in sync with the database,
query the job by name or id!
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param project_or_name_or_id: orm.Project object or project name or project id.
@param job_or_name_or_id: orm.Job object or job name or job id.
@return: orm.Job object
"""
if isinstance(job_or_name_or_id, orm.Job):
job = job_or_name_or_id
elif isinstance(job_or_name_or_id, int):
job = session.query(orm.Job).get(job_or_name_or_id)
else:
project = get_project(session, project_or_name_or_id)
query = session.query(orm.Job).join(orm.Project)
query = query.filter(orm.Project.name == project.name)
query = query.filter(sqlalchemy.or_(orm.Job.id == job_or_name_or_id,
orm.Job.name == job_or_name_or_id))
job = query.one()
return job
def register_job_tags(session, job, tags):
"""
insert or update key-value tags of a job
this is one of many options to populate the Tag and JobTag tables.
it is not required to use this function.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
@param job: orm.Job object
@param tags: dictionary of tags
@return: None
"""
for k, v in tags.items():
job.tags[k] = v
if tags:
session.commit()
def register_params(session, params):
"""
register (insert missing) parameter names
add new parameter names to the global list of parameter names.
this is one of many options to populate the Param table.
it is not required to use this function.
this function implies a session flush.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is committed if new parameters have been added
@param params: sequence of parameter names
param names with leading underscore are ignored.
@return: None
"""
existing_params = query_params(session).keys()
params = [param for param in params if param[0] != '_']
new_params = set(params) - set(existing_params)
for k in new_params:
session.add(orm.Param(key=k))
if new_params:
session.commit()

57
pmsco/database/git.py Normal file
View File

@@ -0,0 +1,57 @@
"""
@package pmsco.database.git
git metadata
this module retrieves the git hash of the running code for job metadata.
this requires that the code is run from a git repository
and that the gitpython package is installed.
gitpython is loaded on demand.
common errors (missing gitpython or invalid repository) are handled.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import importlib
def git():
"""
import the git module from GitPython
@return: git module or None if an error occurred
"""
try:
return importlib.import_module('git')
except ImportError:
return None
def get_git_hash(repo_path=None):
"""
get the git commit (hash) of the running code (HEAD)
the method looks for a git repository in the source tree of this module.
if successful, it returns the hash string of the HEAD commit.
@return: hexadecimal hash string.
empty string if the file is not in a git repository.
"""
if repo_path is None:
repo_path = __file__
_git = git()
if _git is not None:
try:
repo = _git.Repo(repo_path, search_parent_directories=True)
except _git.exc.InvalidGitRepositoryError:
return ""
else:
return repo.head.commit.hexsha
else:
return ""

406
pmsco/database/ingest.py Normal file
View File

@@ -0,0 +1,406 @@
"""
@package pmsco.database.ingest
ingest existing data such as flat results files (.dat or .tasks.dat) into a database.
the results file is a space-delimited, general text file
such as produced by pmsco.optimizers.population.Population.save_array().
each line contains one result dataset, the columns correspond to the regular and special parameters.
the first row contains the parameter names.
the main function is ingest_job_results().
the other functions require an open database session from pmsco.database.access.DatabaseAccess.session(),
and ingest the metadata and the actual results, respectively.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
import logging
import numpy as np
from pathlib import Path
from pmsco.database.access import DatabaseAccess
import pmsco.database.common as common
import pmsco.database.orm as orm
import pmsco.database.util as util
logger = logging.getLogger(__name__)
def insert_result(session, job, index, result, delta=None):
"""
add or update a calculation result including index and model to the database.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the model may not be found.
this function does not commit the transaction.
@param job: (orm.Job) job object.
use pmsco.database.common.get_object to retrieve by id or name.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
@return: (orm.Model, orm.Result) model and result objects
"""
model_obj = store_model(session, job, index, result)
result_obj = store_result_data(session, model_obj, index, result)
store_param_values(session, model_obj, result, delta)
return model_obj, result_obj
def store_model(session, job, index, result):
"""
add or update the model entry for a calculation result in the database.
the method updates the Models table.
the model is identified by job and index.model.
the result is identified by job and index.
if the model exists in the database, it is updated.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the model may not be found.
this function does not commit the transaction.
@param job: (orm.Job) job object.
use pmsco.database.common.get_object to retrieve by id or name.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen' and '_particle'.
'_gen' and '_particle' default to None if not present.
@return: (orm.Model) updated model object
"""
assert isinstance(job, orm.Job)
model_dict = {'gen': None, 'particle': None}
model_dict.update(util.special_params(result))
try:
model_dict['model'] = index.model
except AttributeError:
model_dict['model'] = index['_model']
q = session.query(orm.Model)
q = q.filter(orm.Model.job == job)
q = q.filter(orm.Model.model == model_dict['model'])
model_obj = q.one_or_none()
if model_obj is None:
model_obj = orm.Model()
model_obj.job = job
model_obj.model = model_dict['model']
session.add(model_obj)
model_obj.gen = model_dict['gen']
model_obj.particle = model_dict['particle']
return model_obj
def store_result_data(session, model_obj, index, result):
"""
add or update a result in the database.
the method updates the Results table.
the model is identified by model_id.
the result is identified by model_id and index.
if the result exists in the database, it is updated.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the result entry may not be found.
this function does not commit the transaction.
@param model_obj: (orm.Model) model object that is already part of the session.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@return: (orm.Result) updated Results object.
"""
assert isinstance(model_obj, orm.Model)
result_dict = util.special_params(result)
result_dict.update(util.special_params(index))
q = session.query(orm.Result)
q = q.filter(orm.Result.model == model_obj)
q = q.filter(orm.Result.scan == result_dict['scan'])
q = q.filter(orm.Result.domain == result_dict['domain'])
q = q.filter(orm.Result.emit == result_dict['emit'])
q = q.filter(orm.Result.region == result_dict['region'])
result_obj = q.one_or_none()
if result_obj is None:
result_obj = orm.Result()
result_obj.model = model_obj
result_obj.scan = result_dict['scan']
result_obj.domain = result_dict['domain']
result_obj.emit = result_dict['emit']
result_obj.region = result_dict['region']
session.add(result_obj)
result_obj.rfac = result_dict['rfac']
try:
result_obj.timestamp = result_dict['timestamp'].timestamp()
except KeyError:
result_obj.timestamp = datetime.datetime.now().timestamp()
except AttributeError:
result_obj.timestamp = result_dict['timestamp']
try:
result_obj.secs = result_dict['secs']
except KeyError:
pass
return result_obj
def store_param_values(session, model_obj, result, delta=None):
"""
add or update parameter values of a model in the database.
the method updates the ParamValues table.
@param session: (sqlalchemy.Session) database session.
when updating an existing model, previous changes must have been committed,
else the result entry may not be found.
this function flushes the session at the end.
it does not commit the transaction.
@param model_obj: (orm.Model) model object that is already part of the session.
@param result: (dict) dictionary containing the parameter values.
the parameter names must exist in the Params table and in the self._model_params dictionary.
special values (with a leading underscore) are ignored.
extra parameters may raise a KeyError.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
@return: None
@raise: KeyError if a parameter key is not registered.
"""
assert isinstance(model_obj, orm.Model)
for key in util.regular_params(result).keys():
pv = orm.ParamValue()
pv.model = model_obj
pv.param_key = key
pv.value = result[key]
try:
pv.delta = delta[key]
except (TypeError, KeyError):
pass
session.add(pv)
session.flush()
def ingest_results_file(session, project, job, filename):
"""
import a results file into the database.
this is a sub-method used by ingest().
a job entry with the given id must exist,
but there must be no model entries referencing the job.
it is not possible to update existing models, results or parameter values using this method.
instead, you have to delete the job (which also deletes all dependent entries)
and re-import the results.
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is flushed but not committed at the end of this function.
@param project: orm.Project object or project name or project id.
@param job: orm.Job object or job name or job id.
@param filename: path and name of the results file.
@return: None.
@raise ValueError if the job already has model entries.
"""
job = common.get_job(session, project, job)
assert isinstance(job, orm.Job)
data = np.atleast_1d(np.genfromtxt(filename, names=True))
try:
unique_models, unique_index = np.unique(data['_model'], True)
except ValueError:
unique_models = np.array([0])
unique_index = np.array([0])
unique_data = data[unique_index]
special_params = util.special_params(data.dtype.names)
model_objs = {}
# iterate on models
for _data in unique_data:
try:
_model = _data['_model']
except ValueError:
_model = unique_models[0]
model = orm.Model(job=job, model=_model)
if 'gen' in special_params:
model.gen = _data['_gen']
if 'particle' in special_params:
model.particle = _data['_particle']
session.add(model)
model_objs[_model] = model
for key, value in util.regular_params(_data).items():
model.values[key] = value
session.flush()
# iterate on results
for _data in data:
try:
_model = _data['_model']
except ValueError:
_model = unique_models[0]
result_entry = {'model': None,
'scan': -1,
'domain': -1,
'emit': -1,
'region': -1,
'rfac': None}
result_entry.update(util.special_params(_data))
result_entry['model'] = model_objs[_model]
result = orm.Result()
for key, value in result_entry.items():
setattr(result, key, value)
session.add(result)
session.flush()
def ingest_job_metadata(session, **kwargs):
"""
ingest job metadata
@param session: (sqlalchemy.Session) database session created by pmsco.database.access.DatabaseAccess.session()
the session is flushed but not committed at the end of this function.
@param kwargs: dictionary of function arguments.
the dictionary contains the following values.
all arguments are required unless noted.
@arg 'resultsfile' (required) name of the .tasks.dat results file.
@arg 'project' (required) unique name of the project.
@arg 'code' (optional) name of the project code.
@arg 'job' (required) name of the calculation job. job name must not exist for the project yet.
@arg 'mode' (required) pmsco optimization mode.
@arg 'machine' (optional) name of the machine where the job ran.
@arg 'processes' (optional) number of processes.
@arg 'hours' (optional) run time in hours (wall time).
@arg 'git_hash' (optional) git hash of the code revision.
@arg 'datetime' (datetime.datetime) time stamp (optional).
if not specified, the argument defaults to the time stamp of the results file.
hint: the constructor of a datetime object is
`datetime.datetime(year, month, day, hour, minute, second)`.
@arg 'description' (optional) meaningful description of the calculation job, up to the user.
@arg 'jobtags' (dict, optional) key=value tags to be associated with the job
@return (orm.Project, orm.Job) orm objects of the inserted records.
@raise sqlalchemy.exc.IntegrityError if the job already exists in the database.
"""
if 'datetime' not in kwargs:
rf = Path(kwargs['resultsfile'])
kwargs['datetime'] = datetime.datetime.fromtimestamp(rf.stat().st_mtime)
project = common.register_project(session, kwargs['project'], kwargs['code'])
job = common.register_job(session, project, kwargs['job'], **kwargs)
try:
common.register_job_tags(session, job, kwargs['jobtags'])
except KeyError:
pass
session.flush()
return project, job
def ingest_job_results(**kwargs):
"""
import results from a calculation job.
this function contains all steps necessary to import the results (tasks.dat)
from a calculation job into a database.
it registers the project and job, and imports the results data.
the project may exist in the database, the job must not exist (raises an exception).
arguments can be specified as dict (**d) or in keyword=value form.
@param kwargs: dictionary of function arguments.
the dictionary contains the following values.
all arguments are required unless noted.
@arg 'workdir' (optional) path to the working directory.
the working directory of the operating system is changed.
this is the root for relative paths of the database and results files.
if not specified, the working directory is unchanged.
@arg 'dbfile' (required) name of the database file.
@arg 'project' (required) unique name of the project.
@arg 'code' (optional) name of the project code.
@arg 'job' (required) name of the calculation job. job name must not exist for the project yet.
@arg 'mode' (required) pmsco optimization mode.
@arg 'machine' (optional) name of the machine where the job ran.
@arg 'processes' (optional) number of processes.
@arg 'hours' (optional) run time in hours (wall time).
@arg 'git_hash' (optional) git hash of the code revision.
@arg 'datetime' (datetime.datetime) time stamp (optional).
if not specified, the argument defaults to the time stamp of the results file.
hint: the constructor of a datetime object is
`datetime.datetime(year, month, day, hour, minute, second)`.
@arg 'description' (optional) meaningful description of the calculation job, up to the user.
@arg 'jobtags' (dict, optional) key=value tags to be associated with the job
@arg 'resultsfile' (required) name of the .tasks.dat results file.
@return dict with 'project_id' and 'job_id'.
these are the database ids of the project and job records.
@raise sqlalchemy.exc.IntegrityError if the job already exists in the database.
"""
try:
wd = Path(kwargs['workdir'])
except KeyError:
pass
else:
wd.cwd()
dba = DatabaseAccess()
dba.connect(kwargs['dbfile'])
with dba.session() as session:
project, job = ingest_job_metadata(session, **kwargs)
ingest_results_file(session, project, job, kwargs['resultsfile'])
session.commit()
ref = {'project_id': project.id, 'job_id': job.id}
return ref

746
pmsco/database/orm.py Normal file
View File

@@ -0,0 +1,746 @@
"""
@package pmsco.database.orm
pmsco results database object-relational mapper
this module declares the database schema and object mapping.
the object-relational mapping uses
the [sqlalchemy framework](https://docs.sqlalchemy.org/en/13/orm/tutorial.html).
the database backend is sqlite3.
for examples how to use the database, see the ingest module and the unit tests.
@author Matthias Muntwiler
@copyright (c) 2021 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
from sqlalchemy import create_engine
from sqlalchemy import event
from sqlalchemy import Column, Sequence, ForeignKey
from sqlalchemy import Boolean, Integer, Float, String, DateTime
from sqlalchemy.engine import Engine
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import object_session
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import validates
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm.exc import NoResultFound
import numpy as np
import sqlite3
from pmsco.dispatch import CalcID
import pmsco.database.util as db_util
# make sure sqlite understands numpy data types
sqlite3.register_adapter(np.float64, float)
sqlite3.register_adapter(np.float32, float)
sqlite3.register_adapter(np.int64, int)
sqlite3.register_adapter(np.int32, int)
Base = declarative_base()
engine = None
Session = sessionmaker()
class Project(Base):
"""
database object representing a project
@note there is an implicit constructor with keyword arguments that correspond to the attributes.
"""
## @var id
# (int, primary key) database id of the project
## @var name
# project name, should be short, must be unique within a project
## @var jobs
# collection of related jobs
#
# defines the relationship between Project and Job objects.
# the instance attribute maps job names (str) to Job objects.
__tablename__ = "Projects"
id = Column(Integer, Sequence('project_id_seq'), primary_key=True)
name = Column(String(50, collation='NOCASE'), nullable=False, unique=True)
code = Column(String(50, collation='NOCASE'))
jobs = relationship('Job', backref='project',
collection_class=attribute_mapped_collection('name'),
cascade="all, delete, delete-orphan", lazy='joined')
def __repr__(self):
return f'Project({repr(self.name), repr(self.code)})'
class Job(Base):
"""
database object representing a calculation job
a job object holds several descriptive values of a calculation job.
it also refers to a project.
tags are key-value pairs that describe the job in standardized terms.
they can provide a consistent classification scheme across jobs and projects.
for example, they can store special project arguments that may be important
to distinguish calculations in different stages or contexts.
the class also defines mapping and proxy objects that simplify the use of tags and models.
explicit creation of Tag and JobTag objects is then not necessary.
@attention after modifying the mapped collections job_tags, tags or models
make sure to call flush() or commit() on the session
before accessing those mappings in other objects
else integrity errors may occur!
"""
## @var id
# (int, primary key) database id of the job
## @var project_id
# (int, foreign key) database id of the related project
## @var name
# job name, should be short, must be unique within a project
## @var mode
# pmsco calculation mode
## @var machine
# name of the computing facility
## @var git_hash
# git hash of the used code if under version control
## @var datetime
# start date and time of the job, ISO format (yyyy-mm-dd hh:mm:ss)
## @var processes
# number of processes
## @var hours
# job run time (wall time) in hours
## @var description
# up to the user
## @var job_tags
# collection of related job tags
#
# defines the relationship between Job and JobTag objects.
# the instance attribute maps tag keys (str) to JobTag objects.
## @var tags
# collection of tags
#
# maps tag keys (str) to tag values (str).
# this is an association proxy of job_tags.
## @var models
# collection of related models
#
# defines the relationship between Job and Model objects.
# the instance attribute maps model numbers to Model objects
__tablename__ = "Jobs"
id = Column(Integer, Sequence('job_id_seq'), primary_key=True)
project_id = Column(Integer, ForeignKey('Projects.id'), index=True)
name = Column(String(50, collation='NOCASE'), nullable=False)
mode = Column(String(20, collation='NOCASE'))
machine = Column(String(50, collation='NOCASE'))
git_hash = Column(String(50, collation='NOCASE'))
datetime = Column(String(50))
processes = Column(Integer)
hours = Column(Float)
description = Column(String(200, collation='NOCASE'))
job_tags = relationship('JobTag', back_populates='job',
collection_class=attribute_mapped_collection('tag_key'),
cascade="all, delete, delete-orphan")
# mapping tag_key -> tag_value
tags = association_proxy('job_tags', 'value', creator=lambda k, v: JobTag(key=k, value=v))
models = relationship('Model', back_populates='job',
collection_class=attribute_mapped_collection('model'),
cascade="all, delete, delete-orphan")
def __repr__(self):
try:
project_name = repr(self.project.name)
except AttributeError:
project_name = None
try:
job_name = repr(self.name)
except AttributeError:
job_name = None
return f'Job({project_name}, {job_name}, {repr(self.mode)})'
class Tag(Base):
"""
database object representing a tag name
"""
## @var id
# (int, primary key) database id of the tag name
## @var key
# tag name/key, should be short, must be unique
## @var tag_jobs
# collection of related JobTag objects
#
# defines the relationship between Tag and JobTag objects.
__tablename__ = "Tags"
id = Column(Integer, Sequence('tag_id_seq'), primary_key=True)
key = Column(String(20, collation='NOCASE'), nullable=False, unique=True)
tag_jobs = relationship('JobTag', back_populates='tag', cascade="all, delete, delete-orphan")
def __init__(self, key):
self.key = key
def __repr__(self):
return f'Tag({repr(self.key)})'
class JobTag(Base):
"""
association object class for job tags
Job - Tag is a many-to-many relationship built using this association class.
by using the dictionary-like Job.tags proxy, explicit creation of association objects can be avoided.
the class applies the
[UniqueObjectValidateOnPending pattern](https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObjectValidatedOnPending)
to look up existing tags in the database when a Tag object is needed and only the key is given.
"""
## @var id
# (int, primary key) database id of the job tag
## @var tag_id
# (int, foreign key) database id of the related tag name
## @var job_id
# (int, foreign key) database id of the related job
## @var value
# value (str) of the job tag
## @var tag
# associated Tag object
#
# defines the relationship between JobTag and Tag objects
## @var job
# associated Job object
#
# defines the relationship between JobTag and Job objects
## @var tag_key
# key (name) of the asscoiated Tag object
#
# this is an association proxy that provides direct access to tag.key
# or links to or creates a Tag object behind the scenes.
__tablename__ = "JobTags"
id = Column(Integer, Sequence('jobtag_id_seq'), primary_key=True)
tag_id = Column(Integer, ForeignKey('Tags.id'), index=True)
job_id = Column(Integer, ForeignKey('Jobs.id'), index=True)
value = Column(String(200, collation='NOCASE'))
tag = relationship("Tag", back_populates="tag_jobs")
job = relationship("Job", back_populates="job_tags")
tag_key = association_proxy("tag", "key")
def __init__(self, key=None, value=None):
if key is not None:
self.tag_key = key
self.value = value
@validates("tag")
def _validate_tag(self, key, value):
"""
receive the event that occurs when `jobtag.tag` is set.
if the object is present in a Session, then make sure it's the Tag
object that we looked up from the database.
otherwise, do nothing and we'll fix it later when the object is
put into a Session.
@param key: attribute name, i.e., 'tag'
@param value: a JobTag object
"""
sess = object_session(self)
if sess is not None:
return _setup_tag(sess, value)
else:
return value
@event.listens_for(Session, "transient_to_pending")
def _validate_tag(session, object_):
"""
receive a JobTag object when it gets attached to a Session to correct its unique Tag relationship.
"""
if isinstance(object_, JobTag):
if object_.tag is not None and object_.tag.id is None:
old_tag = object_.tag
new_tag = _setup_tag(session, object_.tag)
if new_tag is not old_tag:
if old_tag in session:
session.expunge(old_tag)
object_.tag = new_tag
def _setup_tag(session, tag_object):
"""
given a Session and a Tag object, return the correct Tag object from the database.
"""
with session.no_autoflush:
try:
return session.query(Tag).filter_by(key=tag_object.key).one()
except NoResultFound:
return tag_object
class Model(Base):
"""
database object representing a model
the object holds the model number (which is unique within the context of a single job only),
the diagnostic generation and particle values, and refers to the job where the model is used.
the class also defines relationship properties that simplify access to referenced objects.
for instance, parameter values can be accessed via the values['param_key'] mapping proxy.
examples:
~~~~~~{.py}
model = Model(model=10, gen=5, particle=2)
model.job = job1_object
model.values['dA'] = 25.6
model.deltas['dA'] = 0.1
pv = ParamValue(value=39.0, delta=-0.3)
model.param_values['dB'] = pv
result = Result(calc_id=calc_id, rfac=0.77)
model.results.append(result)
~~~~~~
@attention after modifying the mapped collections param_values, values or deltas,
make sure to call flush() or commit() on the session
before accessing those mappings in another model
else integrity errors may occur!
"""
## @var id
# (int, primary key) database id of the model
## @var job_id
# (int, foreign key) database id of the related job
## @var model
# (int) model number as used in the task index of pmsco
#
# @note the model number is not unique in the database as multiple jobs can produce same task indices.
# the unique number, self.id is not used in pmsco code.
## @var gen
# (int) generation number assigned by some optimizers. defaults to None.
## @var particle
# (int) particle number assigned by some optimizers. defaults to None.
## @var job
# associated Job
#
# defines the relationship between Model and Job objects.
## @var results
# collection of Result objects
#
# defines the relationship between Model and Result objects.
## @var param_values
# collection of ParamValue objects
#
# defines the relationship between Model and ParamValue objects.
# the instance attribute maps parameter keys to ParamValue objects.
## @var values
# collection of parameter values
#
# this is an association proxy that maps parameter keys to parameter values (ParamValue.value).
# ParamValue objects are accessed and created behind the scene.
## @var deltas
# collection of delta values
#
# this is an association proxy that maps parameter keys to parameter deltas (ParamValue.delta.
# ParamValue objects are accessed and created behind the scene.
__tablename__ = "Models"
id = Column(Integer, Sequence('model_id_seq'), primary_key=True)
job_id = Column(Integer, ForeignKey('Jobs.id'), index=True)
model = Column(Integer, index=True)
gen = Column(Integer)
particle = Column(Integer)
job = relationship("Job", back_populates="models")
results = relationship('Result', back_populates='model', cascade="all, delete, delete-orphan")
# mapping param_key -> ParamValue object
param_values = relationship('ParamValue', back_populates='model',
collection_class=attribute_mapped_collection('param_key'),
cascade="all, delete, delete-orphan")
# mapping param_key -> param_value
values = association_proxy('param_values', 'value', creator=lambda k, v: ParamValue(key=k, value=v))
deltas = association_proxy('param_values', 'delta', creator=lambda k, v: ParamValue(key=k, delta=v))
def __repr__(self):
return f'Model(id={repr(self.id)}, job_id={repr(self.job_id)}, model={repr(self.model)})'
def as_dict(self):
"""
object properties in a dictionary.
the dictionary keys correspond to the column names of numpy arrays.
the mapping db_field -> column name is declared in pmsco.database.util.DB_SPECIAL_PARAMS
@return: (dict)
"""
d = {'_db_model_id': self.id}
for attr, key in db_util.DB_SPECIAL_PARAMS.items():
try:
d[key] = getattr(self, attr)
except AttributeError:
pass
return d
class Result(Base):
"""
database object representing a calculation result
the result object holds the calculated R-factor per job and calculation index.
the calculation index (CalcID) is not unique in the database because it may contain results from multiple jobs.
thus, the object links to a Model object which is unique.
the calc_id property can be used to reconstruct a CalcID.
"""
## @var id
# (int, primary key) database id of the result
## @var model_id
# (int, foreign key) database id of the related model
## @var model
# associated Model object
#
# defines the relationship between Result and Model objects.
#
# @attention do not confuse the Result.model and Model.model attributes of same name!
# to obtain the model number to which a result belongs, use Result.model.model.
## @var scan
# (int) scan index as used in the calculations
## @var domain
# (int) domain index as used in the calculations
## @var emit
# (int) emitter index as used in the calculations
## @var region
# (int) region index as used in the calculations
## @var rfac
# (float) calculated R-factor
## @var timestamp
# (float) end date and time of this calculation task
#
# the float value represents seconds since jan 1, 1970 (datetime.datetime.timestamp).
# the datetime proxy converts to and from python datetime.datetime.
## @var datetime
# (datetime.datetime) end date and time of this calculation task
#
# this is a conversion proxy for timestamp.
## @var secs
# (float) total duration of the calculation task in seconds
#
# total cpu time necessary to get this result (including child tasks) in seconds.
## @var calc_id
# (CalcID) calculation task index
#
# conversion proxy for the task index components.
#
# on assignment, the scan, domain, emit and region attributes are updated.
# it does not update the model index as it is not stored by this object!
# the model index must be set separately in the linked Model object.
__tablename__ = "Results"
id = Column(Integer, Sequence('result_id_seq'), primary_key=True)
model_id = Column(Integer, ForeignKey('Models.id'), index=True)
scan = Column(Integer, index=True)
domain = Column(Integer, index=True)
emit = Column(Integer, index=True)
region = Column(Integer, index=True)
rfac = Column(Float)
timestamp = Column(Float)
secs = Column(Float)
model = relationship("Model", back_populates="results")
def __init__(self, calc_id=None, scan=None, domain=None, emit=None, region=None,
rfac=None, timestamp=None, secs=None):
if calc_id is not None:
self.calc_id = calc_id
else:
self.scan = scan
self.domain = domain
self.emit = emit
self.region = region
self.rfac = rfac
self.timestamp = timestamp
self.secs = secs
def __repr__(self):
return f'Result(model_id={repr(self.model_id)}, calc_id={repr(self.calc_id)}, rfac={repr(self.rfac)})'
@property
def calc_id(self):
return CalcID(self.model.model, self.scan, self.domain, self.emit, self.region)
@calc_id.setter
def calc_id(self, calc_id):
self.scan = calc_id.scan
self.domain = calc_id.domain
self.emit = calc_id.emit
self.region = calc_id.region
@property
def datetime(self):
return datetime.datetime.fromtimestamp(self.timestamp)
@datetime.setter
def datetime(self, value):
self.timestamp = value.timestamp()
def as_dict(self):
"""
object properties in a dictionary.
the dictionary keys correspond to the column names of numpy arrays.
the mapping db_field -> column name is declared in pmsco.database.util.D.B_SPECIAL_PARAMS
@return: (dict)
"""
d = {'_db_result_id': self.id}
for attr, key in db_util.DB_SPECIAL_PARAMS.items():
try:
d[key] = getattr(self, attr)
except AttributeError:
pass
return d
class Param(Base):
"""
database object representing a parameter
the parameter object holds the name (or key) of a calculation parameter.
explicit creation of parameter objects can be avoided by using the mappings of the Model class.
"""
## @var id
# (int, primary key) database id of the parameter name
## @var key
# parameter name/key as used in calculations, should be very short, must be unique
## @var param_values
# collection of related ParamValue objects
#
# defines the relationship between Param and ParamValue objects.
__tablename__ = "Params"
id = Column(Integer, Sequence('param_id_seq'), primary_key=True)
key = Column(String(20, collation='NOCASE'), nullable=False, unique=True)
param_values = relationship('ParamValue', back_populates='param', cascade="all, delete, delete-orphan")
def __init__(self, key):
self.key = key
def __repr__(self):
return f'Param({repr(self.key)})'
class ParamValue(Base):
"""
association object class for parameter values
Model - Param is a many-to-many relationship built using this association class.
by using the dictionary-like Model.values and Model.deltas proxies,
explicit creation of association objects can be avoided.
the class applies the
[UniqueObjectValidateOnPending pattern](https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObjectValidatedOnPending)
to look up existing params in the database when a Param object is needed and only the key is given.
"""
## @var id
# (int, primary key) database id of the parameter value
## @var param_id
# (int, foreign key) database id of the related parameter name
## @var model_id
# (int, foreign key) database id of the related model
## @var value
# (float) numeric value of the parameter
## @var delta
# (float) numeric delta value of the parameter (reported by some optimizers)
## @var param
# associated Param object
#
# defines the relationship between ParamValue and Param objects
## @var model
# associated Model object
#
# defines the relationship between ParamValue and Model objects
## @var param_key
# key (name) of the asscoiated Param object
#
# this is an association proxy that provides direct access to param.key.
# it accesses or creates Param objects behind the scenes.
__tablename__ = "ParamValues"
id = Column(Integer, Sequence('paramvalue_id_seq'), primary_key=True)
param_id = Column(Integer, ForeignKey('Params.id'), index=True)
model_id = Column(Integer, ForeignKey('Models.id'), index=True)
value = Column(Float)
delta = Column(Float)
param = relationship("Param", back_populates="param_values")
model = relationship("Model", back_populates="param_values")
param_key = association_proxy('param', 'key')
def __init__(self, model=None, param=None, key=None, value=None, delta=None):
if model is not None:
self.model = model
if param is not None:
self.param = param
elif key is not None:
self.param_key = key
self.value = value
self.delta = delta
@validates("param")
def _validate_param(self, key, value):
"""
receive the event that occurs when `paramvalue.param` is set.
if the object is present in a Session, then make sure it's the Param
object that we looked up from the database.
otherwise, do nothing and we'll fix it later when the object is put into a Session.
"""
sess = object_session(self)
if sess is not None:
return _setup_param(sess, value)
else:
return value
@event.listens_for(Session, "transient_to_pending")
def _validate_param(session, object_):
"""
receive a ParamValue object when it gets attached to a Session to correct its unique Param relationship.
"""
if isinstance(object_, ParamValue):
if object_.param is not None and object_.param.id is None:
old_param = object_.param
new_param = _setup_param(session, object_.param)
if new_param is not old_param:
if old_param in session:
session.expunge(old_param)
object_.param = new_param
def _setup_param(session, param_object):
"""
given a Session and a Tag object, return the correct Tag object from the database.
"""
with session.no_autoflush:
try:
return session.query(Param).filter_by(key=param_object.key).one()
except NoResultFound:
return param_object
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
"""
set sqlite pragmas.
make sure sqlite enforces relational integrity.
@param dbapi_connection:
@param connection_record:
@return:
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def sqlite_link(path=None):
"""
format the sqlalchemy link to an sqlite3 database.
@param path: file path. if empty, an in-memory database is created.
@return: (str) database link for the sqlalchemy engine.
"""
if not path:
path = ':memory:'
return f'sqlite:///{path}'
def connect(db_link):
"""
connect to the database.
create the sqlalchemy engine and bind the session maker.
the database engine and session maker are global.
this function should be called only once in a process.
@param db_link: (str) database link expected by the sqlalchemy engine
@return: None
"""
global engine
engine = create_engine(db_link, echo=False)
Base.metadata.create_all(engine)
Session.configure(bind=engine)

158
pmsco/database/project.py Normal file
View File

@@ -0,0 +1,158 @@
"""
@package pmsco.database.project
wrapper class for project-specific database operations
usage:
~~~~~~{.py}
db = DatabaseAccess()
db.connect("file.db")
with db.session():
# database access here
# ...
# commit transaction
session.commit()
# continue in new transaction
# ...
# at the end of the context
# the session is closed and orm objects are detached from the database.
~~~~~~
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import datetime
import logging
import socket
from pmsco.database.access import DatabaseAccess
import pmsco.database.common as db_common
import pmsco.database.ingest as db_ingest
import pmsco.database.query as db_query
from pmsco.dispatch import mpi_size
logger = logging.getLogger(__name__)
class ProjectDatabase(DatabaseAccess):
"""
wrapper class for project specific database operations
the purpose of this class is to bundle all specific code and run-time information
for database access of a running calculation job.
after calling ingest_project_metadata(),
the class object stores the persistent project and job identifiers.
the other methods provide convenient wrappers so that database code can be kept minimal in the project.
usage:
~~~~~~{.py}
db = ProjectDatabase()
db.connect('file.db')
db.ingest_project_metadata(...)
for result in results:
db.ingest_result(result...)
~~~~~~
"""
def __init__(self):
super().__init__()
self.db_project_id = None
self.db_job_id = None
def ingest_project_metadata(self, project):
"""
ingest project metadata into the database
@param project: pmsco.project.Project object
@return: None
"""
with self.session() as session:
db_project = db_common.register_project(session=session,
name=project.project_name,
code=project.__module__,
allow_existing=True)
db_job = db_common.register_job(session=session,
project=db_project,
job_name=project.job_name,
allow_existing=False,
mode=project.mode,
machine=socket.gethostname(),
git_hash=project.git_hash,
datetime=datetime.datetime.now(),
processes=mpi_size,
hours=project.timedelta_limit.total_seconds() / 3600.,
description=project.description)
db_common.register_job_tags(session, db_job, project.job_tags)
db_common.register_params(session, project.model_space.start.keys())
session.commit()
self.db_project_id = db_project.id
self.db_job_id = db_job.id
def ingest_result(self, index, result, delta):
"""
add or update a result in the database.
the method updates the Models, Results and ParamValues tables.
the model is identified by self.job_id and index.model.
the result is identified by self.job_id and index.
if the model or result exists in the database, it is updated.
@param index: (pmsco.dispatch.CalcID or dict)
calculation index.
in case of dict, the keys must be the attribute names of CalcID prefixed with an underscore, i.e.,
'_model', '_scan', '_domain', '_emit', '_region'.
extra values in the dictionary are ignored.
undefined indices must be -1.
@param result: (dict) dictionary containing the parameter values and the '_rfac' result.
may also contain the special values '_gen', '_particle', '_timestamp'.
'_gen' and '_particle' are integers and default to None.
'_timestamp' can be numeric (seconds since jan 1, 1970)
or an object that implements a timestamp function like datetime.datetime.
it defaults to the current (local) time.
@param delta: (dict) dictionary containing the delta values.
the keys must correspond to model keys in the result dictionary.
this argument is optional.
"""
assert self.db_project_id is not None
assert self.db_job_id is not None
with self.session() as session:
job_obj = db_common.get_job(session, self.db_project_id, self.db_job_id)
model_obj = db_ingest.store_model(session, job_obj, index, result)
db_ingest.store_result_data(session, model_obj, index, result)
db_ingest.store_param_values(session, model_obj, result, delta)
session.commit()
def query_best_task_models(self, level, count):
"""
query N best models per task.
this is a wrapper for pmsco.database.query.query_best_task_models().
in addition to the wrapped function, it opens a session and uses the registered db_job_id.
this query is used by the file tracker to determine the models to keep.
@param level: level up to which to query.
the level can be specified by level name (str) or numeric index (0..4).
if it is scan (equivalent to 1), the method queries the model and scan levels.
@param count: number of models to query per task.
@return set of matching model numbers (model index, Models.model field).
"""
with self.session() as session:
models = db_query.query_best_task_models(session, self.db_job_id, level, count)
return models

470
pmsco/database/query.py Normal file
View File

@@ -0,0 +1,470 @@
"""
@package pmsco.database.query
specialized query functions for the pmsco database
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import numpy as np
from sqlalchemy import func
import pmsco.database.orm as orm
import pmsco.database.util as util
import pmsco.dispatch as dispatch
logger = logging.getLogger(__name__)
def query_newest_job(session):
"""
retrieve the entry of the newest job
the newest entry is determined by the datetime field.
@param session:
@return: pmsco.database.orm.Job object
"""
q = session.query(orm.Job)
q = q.order_by(orm.Job.datetime.desc(), orm.Job.id.desc())
job = q.first()
return job
def query_model(session, job_id=None, model_id=None, model=None):
"""
retrieve model parameters and control variables from the database.
@param model_id: id of the model in the database.
@return: (dict, dict) value dictionary and delta dictionary.
dictionary keys are parameter values.
the special value '_model' is included.
"""
query = session.query(orm.ParamValue)
if job_id is not None:
query = query.filter(orm.Job.id == job_id)
if model_id is not None:
query = query.filter(orm.Model.id == model_id)
if model is not None:
query = query.filter(orm.Model.model == model)
result = query.all()
param_value = {}
param_delta = {}
model_obj = None
for pv in result:
if model_obj is None:
model_obj = pv.model
param_value[pv.param.key] = pv.value
param_delta[pv.param.key] = pv.delta
param_value['_model_id'] = model_obj.id
param_value['_model'] = model_obj.model
param_value['_gen'] = model_obj.gen
param_value['_particle'] = model_obj.particle
param_delta['_model_id'] = model_obj.id
param_delta['_model'] = model_obj.model
param_delta['_gen'] = model_obj.gen
param_delta['_particle'] = model_obj.particle
return param_value, param_delta
def query_results(session, job_id):
query = session.query(orm.Result)
query = query.join(orm.Model)
query = query.filter(orm.Job == job_id)
return None
def query_tasks(session, job_id):
"""
query the task index used in a calculation job.
this query neglects the model index
and returns the unique tuples (-1, scan, domain, emit, region).
@param job_id: (int) id of the associated Jobs entry.
@return list of pmsco.dispatch.CalcID tuples of task indices.
the model attribute is -1 in all elements.
"""
query = session.query(orm.Result.scan, orm.Result.domain, orm.Result.emit, orm.Result.region)
query = query.join(orm.Model)
query = query.filter(orm.Model.job_id == job_id)
query = query.distinct()
query = query.order_by(orm.Result.scan, orm.Result.domain, orm.Result.emit, orm.Result.region)
results = query.all()
output = []
for row in results:
d = row._asdict()
d['model'] = -1
output.append(dispatch.CalcID(**d))
return output
def query_best_task_models(session, job_id, level, count):
"""
query N best models per task.
this query is used by the file tracker to determine the models to keep.
@param job_id: (int) id of the associated Jobs entry.
@param level: level up to which to query.
the level can be specified by level name (str) or numeric index (0..4).
if it is scan (equivalent to 1), the method queries the model and scan levels.
@param count: number of models to query per task.
@return set of matching model numbers (Models.model field).
"""
try:
level = int(level)
except ValueError:
level = dispatch.CALC_LEVELS.index(level)
assert 0 <= level < len(dispatch.CALC_LEVELS)
def _query_models(t):
query = session.query(orm.Model.model).join(orm.Job).join(orm.Result)
query = query.filter(orm.Job.id == job_id)
query = query.filter(orm.Result.scan == t.scan)
query = query.filter(orm.Result.domain == t.domain)
query = query.filter(orm.Result.emit == t.emit)
query = query.filter(orm.Result.region == t.region)
query = query.order_by(orm.Result.rfac)
results = query[0:count]
return set((row.model for row in results))
tasks = query_tasks(session, job_id)
models = set()
for task in tasks:
if task.numeric_level <= level:
q_models = _query_models(task)
models |= q_models
return models
def query_model_params_array(session, jobs=None, models=None, order=None, limit=None):
"""
query parameter values and return them in a numpy array
the models table can be filtered by job and/or model.
else, the whole database is returned (which might be huge!).
@param session:
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param models: filter by model.
the argument can be a singleton or sequence of orm.Model objects or their id.
@param order: ordering of results. this can be a sequence of orm.Model attributes.
the default order is by job_id and model.
@param limit: maximum number of models to return
@return: dict['values']: numpy values array, dict['deltas']: numpy deltas array
"""
count_query = session.query(orm.Model)
pn_query = session.query(orm.Param.key)
pv_query = session.query(orm.ParamValue)
if jobs:
try:
jobs = [int(jobs)]
except TypeError:
pass
job_ids = [j if isinstance(j, int) else j.id for j in jobs]
count_query = count_query.filter(orm.Model.job_id.in_(job_ids))
pn_query = pn_query.filter(orm.Model.job_id.in_(job_ids))
pv_query = pv_query.filter(orm.Model.job_id.in_(job_ids))
if models:
try:
models = [int(models)]
except TypeError:
pass
model_ids = [m if isinstance(m, int) else m.id for m in models]
count_query = count_query.filter(orm.ParamValue.model_id.in_(model_ids))
pn_query = pn_query.filter(orm.ParamValue.model_id.in_(model_ids))
pv_query = pv_query.filter(orm.ParamValue.model_id.in_(model_ids))
if order is not None:
pv_query = pv_query.order_by(*order)
else:
pv_query = pv_query.order_by(orm.Model.job_id, orm.Model.model)
if limit:
pv_query = pv_query[0:limit]
n_models = count_query.count()
param_names = pn_query.all()
param_values = pv_query.all()
special_names = orm.Model().as_dict().keys()
dt_names = special_names + param_names
dt = np.dtype([(n, util.field_to_numpy_type(n)) for n in sorted(dt_names, key=str.lower)])
values = np.zeros((n_models,), dtype=dt)
deltas = np.zeros((n_models,), dtype=dt)
for i, pv in enumerate(param_values):
for k, v in pv.model.as_dict():
values[i][k] = deltas[i][k] = v
values[i][pv.param_key] = pv.value
deltas[i][pv.param_key] = pv.delta
return {'values': values, 'deltas': deltas}
calc_id_props = {'model': orm.Model.model,
'scan': orm.Result.scan,
'domain': orm.Result.domain,
'emit': orm.Result.emit,
'region': orm.Result.region}
def query_model_results_array(session, jobs=None, models=None, order=None, limit=None,
query_hook=None, hook_data=None, include_params=False, **index):
"""
query a results table with flexible filtering options
the function returns a structured numpy array of the results and, optionally, parameter values.
the database is fully flattened, row of the array represents one result.
the jobs and models arguments filter for specific jobs and/or models.
custom filters can be added in a query hook function.
the hook function receives an sqlalchemy Query object of the Result table,
joined with the Model and Job tables.
other joins must be added explicitly.
the hook function can add more filters and return the modified query.
the hook function is called after the filters from the other function arguments
(job, models, index) have been applied,
and before the ordering and limit are applied.
@param session:
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param models: filter by model.
the argument can be a singleton or sequence of orm.Model objects or their id.
@param order: ordering of results. this can be a sequence of orm.Result attributes.
the default order is by `orm.Result.rfac`.
to override the default ascending order, append a modifier, e.g., `orm.Result.rfac.desc()`.
@param limit: maximum number of models to return
@param query_hook: hook function that modifies an sqlalchemy.orm.Query object.
the function receives the query as first argument, and any data from hook_data as keyword arguments.
it must return the modified query object.
@param hook_data: (dict) keyword arguments to be passed to the query_hook function.
@param include_params: include parameter values of each model in the result.
by default, only data from the Model and Result records is included.
@param index: filters the results list by scan, domain, emit, and/or region index.
for example, to get only the final results per model, specify `scan=-1`.
@return: numpy values array
"""
results_query = session.query(orm.Result).join(orm.Model).join(orm.Job)
if jobs:
results_query = filter_objects(results_query, orm.Job, jobs)
if models:
results_query = filter_objects(results_query, orm.Model, models)
for k, v in index.items():
results_query = results_query.filter(calc_id_props[k] == v)
if query_hook is not None:
results_query = query_hook(results_query, **hook_data)
if order is not None:
results_query = results_query.order_by(*order)
if limit:
results = results_query[0:limit]
else:
results = results_query.all()
n_results = len(results)
logger.debug(f"query_model_results_array: {results_query.statement} ({n_results} rows)")
dt_names = [n for n in util.DB_SPECIAL_PARAMS.values()]
if include_params:
model_ids = {r.model_id for r in results}
pn_query = session.query(orm.Param.key).join(orm.ParamValue)
pn_query = pn_query.filter(orm.ParamValue.model_id.in_(model_ids))
pn_query = pn_query.distinct()
pn_query = pn_query.order_by(orm.Param.key)
p_names = [r.key for r in pn_query.all()]
dt_names.extend(p_names)
logger.debug(f"query_model_results_array: {pn_query.statement} ({len(p_names)} rows)")
dt = []
v0 = []
for n in dt_names:
ft = util.field_to_numpy_type(n)
dt.append((n, ft))
v0.append(np.nan if ft[0] == 'f' else 0)
dt = np.dtype(dt)
v0 = np.array([tuple(v0)], dtype=dt)
values_array = np.full((n_results,), v0, dtype=dt)
deltas_array = np.full((n_results,), v0, dtype=dt)
for i, r in enumerate(results):
d = {**r.as_dict(), **r.model.as_dict()}
for k, v in d.items():
try:
values_array[i][k] = v
except TypeError:
values_array[i][k] = 0
deltas_array[i] = values_array[i]
if include_params:
for k, v in r.model.values.items():
values_array[i][k] = v
for k, v in r.model.deltas.items():
deltas_array[i][k] = v
return values_array, deltas_array
def query_best_models_per_job(session, projects=None, jobs=None, task_level='model', order=None, limit=None):
"""
return the best model (by rfac) of each selected job
the query gathers the R-factors of the selected jobs at the selected task levels
and, for each job, returns the (database) model id where the lowest R-factor is reported
among the gathered results.
this can be useful if you want to compile a report of the best model per job.
@param session:
@param projects: filter by project.
the argument can be a singleton or sequence of orm.Project objects or numeric id.
@param jobs: filter by job.
the argument can be a singleton or sequence of orm.Job objects or numeric id.
@param task_level: element of or index into @ref pmsco.dispatch.CALC_LEVELS.
deepest task_level to include in the query.
results on deeper levels are not considered.
e.g. if you pass 'scan', R-factors of individual scans are included in the query.
note that including deeper levels will not increase the number of results returned.
the lowest level that can be specified is `emit`.
@param order: ordering of results. this can be a sequence of orm.Result attributes.
the default order is by `orm.Result.rfac`.
@param limit: maximum number of models to return
@return sequence of (orm.Model, orm.Result) tuples.
the number of results corresponds to the number of jobs in the filter scope.
to find out details of the models, execute another query that filters on these model ids.
the method produces an SQL query similar to:
@code{.sql}
select Models.id from Models
join Results on Models.id = Results.model_id
join Jobs on Models.job_id = Jobs.id
where scan=-1
and project_id=1
and job_id in (1,2,3)
group by Models.job_id
having min(rfac)
order by rfac
@endcode
"""
try:
level = dispatch.CALC_LEVELS.index(task_level) + 1
except ValueError:
level = task_level + 1
try:
level_name = dispatch.CALC_LEVELS[level]
except IndexError:
level_name = dispatch.CALC_LEVELS[4]
query = session.query(orm.Model, orm.Result).join(orm.Result)
if projects:
query = filter_objects(query, orm.Project, projects)
if jobs:
query = filter_objects(query, orm.Job, jobs)
query = query.filter(getattr(orm.Result, level_name) == -1)
query = query.group_by(orm.Model.job_id)
query = query.having(func.min(orm.Result.rfac))
if order is not None:
query = query.order_by(*order)
else:
query = query.order_by(orm.Result.rfac)
if limit:
query = query[0:limit]
else:
query = query.all()
return query
def filter_objects(query, entity, objects):
"""
filter a query for the given objects
apply a simple object filter to a database query.
the criteria can be a single object or a sequence of objects.
the objects can be specified either by their object representation or numeric id.
the query is filtered by id.
thus, in the first case, the objects must have a valid id.
@param query: sqlalchemy.orm.Query object that queries a table that is linked to the entity table.
the function joins the entity table.
a table with a direct foreign key relationship to the entity table must already be in the query.
@param entity: orm entity class, e.g. pmsco.database.orm.Project.
@param objects: singleton or sequence of orm objects or their numeric ids.
@return: modified query
"""
# avoid duplicate joins
if str(query.statement).find(entity.__tablename__) < 0:
query = query.join(entity)
try:
objects = [p if isinstance(p, int) else p.id for p in objects]
query = query.filter(entity.id.in_(objects))
except TypeError:
object = objects if isinstance(objects, int) else objects.id
query = query.filter(entity.id == object)
return query
def filter_task_levels(query, level='model', include_parents=False):
"""
refine a query by filtering by task level.
@param query: sqlalchemy.orm.Query object that queries the Result table
(possibly joined with others).
@param level: element of or index into @ref pmsco.dispatch.CALC_LEVELS.
deepest task_level to include in the query.
results on deeper levels are not considered.
e.g. if you pass 'scan', R-factors of individual scans are included in the query.
the lowest level that can be specified is `emit`.
@param include_parents: by default, the query will return only results from the given level.
if True, combined results (parents) will be returned as well.
"""
try:
level = dispatch.CALC_LEVELS.index(level)
except ValueError:
level = int(level)
child_level = level + 1
try:
child_level_name = dispatch.CALC_LEVELS[child_level]
level_name = dispatch.CALC_LEVELS[level]
except IndexError:
child_level_name = dispatch.CALC_LEVELS[4]
level_name = dispatch.CALC_LEVELS[3]
query = query.filter(getattr(orm.Result, child_level_name) == -1)
if not include_parents:
query = query.filter(getattr(orm.Result, level_name) >= 0)
return query

161
pmsco/database/util.py Normal file
View File

@@ -0,0 +1,161 @@
import logging
import numpy as np
from pathlib import Path
import pmsco.dispatch as dispatch
logger = logging.getLogger(__name__)
## mapping of database fields to special parameter names
#
# `_db` parameters are returned by some query methods to identify the database records.
#
DB_SPECIAL_PARAMS = {"project_id": "_db_project_id",
"job_id": "_db_job_id",
"model_id": "_db_model_id",
"result_id": "_db_result_id",
"model": "_model",
"scan": "_scan",
"domain": "_domain",
"emit": "_emit",
"region": "_region",
"gen": "_gen",
"particle": "_particle",
"rfac": "_rfac",
"secs": "_secs",
"timestamp": "_timestamp"}
## numpy data types of special parameters by database field
#
# this dictionary helps to create a numpy array from a database record.
#
DB_SPECIAL_NUMPY_TYPES = {"_db_project_id": "i8",
"_db_job_id": "i8",
"_db_model_id": "i8",
"_db_result_id": "i8",
"_model": "i8",
"_scan": "i8",
"_domain": "i8",
"_emit": "i8",
"_region": "i8",
"_gen": "i8",
"_particle": "i8",
"_rfac": "f8",
"_secs": "f8",
"_timestamp": "f8"}
def regular_params(d):
"""
filter regular parameters from dictionary
returns a dictionary containing only the regular parameters (those not prefixed with an underscore).
@param d: dict or numpy.void or pmsco.dispatch.CalcID.
the param names must have no leading underscore.
the numpy.void type occurs when an element of a structured array is extracted.
the CalcID does not contain a regular parameter and will return an empty dictionary.
it is supported only for compatibility with special_params function.
a tuple or list is interpreted as a sequence of parameter names.
in this case the names representing special parameters are returned with underscore removed.
@return: dict for mapping types (numpy and dict) containing the regular key: value pairs of the original object.
list (tuple) of parameter names for sequence (tuple) types.
leading underscores are removed from key names.
"""
if isinstance(d, np.void):
d = {k: d[k] for k in d.dtype.names if k[0] != "_"}
elif isinstance(d, dispatch.CalcID):
d = {}
elif isinstance(d, tuple):
d = [k for k in d if k[0] != "_"]
d = tuple(d)
elif isinstance(d, dict):
d = {k: v for k, v in d.items() if k[0] != "_"}
else:
d = [k for k in d if k[0] != "_"]
return d
def special_params(d):
"""
filter special parameters from model dictionary, numpy record or sequence.
special parameters are those prefixed with an underscore.
the underscore is removed from the keys.
fields starting with '_db_' are removed.
@param d: dict or numpy.void or pmsco.dispatch.CalcID or sequence.
in the case of a dict or numpy.void,
the key names of the special parameters must have a leading underscore.
the numpy.void type occurs when an element of a structured array is extracted.
in the case of a CalcID, the attribute names become the key names.
a tuple or list is interpreted as a sequence of parameter names.
in this case the names representing special parameters are returned with underscore removed.
@return
the return type depends on the type of input `d`:
@arg in the case of a dict, numpy.void or CalcID it is a dictionary.
@arg in the case of a tuple or list the return type is the same as the input.
"""
if isinstance(d, np.void):
d = {k[1:]: d[k] for k in d.dtype.names if k[0] == "_" and k[0:4] != "_db_"}
elif isinstance(d, dispatch.CalcID):
d = d._asdict()
elif isinstance(d, tuple):
d = [k[1:] for k in d if k[0] == "_" and k[0:4] != "_db_"]
d = tuple(d)
elif isinstance(d, dict):
d = {k[1:]: v for k, v in d.items() if k[0] == "_" and k[0:4] != "_db_"}
else:
d = [k[1:] for k in d if k[0] == "_" and k[0:4] != "_db_"]
return d
def field_to_param(f):
"""
translate database field name to parameter name.
field names of optimization parameters are unchanged.
special parameters are prefixed by '_' or '_db_'.
@param f: (str) database field name.
@return: (str) parameter name as used in model dictionaries.
"""
try:
p = DB_SPECIAL_PARAMS[f]
except KeyError:
p = f
return p
def field_to_numpy_type(f):
"""
determine the numpy data type string of a database field.
@param f: (str) database field name.
@return: (str) numpy type description, e.g. 'f8'.
"""
try:
t = DB_SPECIAL_NUMPY_TYPES[f]
except KeyError:
t = 'f8'
return t
def is_sqlite3_file(path_like):
"""
test whether a file is an sqlite3 database file.
@param path_like: file path (str or pathlib.Path).
@return: (bool)
"""
try:
with Path(path_like).open("rb") as f:
s = f.read(16)
return s == b"SQLite format 3\000"
except OSError:
return False

668
pmsco/dispatch.py
View File

File diff suppressed because it is too large Load Diff

3
pmsco/edac/.gitignore vendored
View File

@@ -1,3 +0,0 @@
edac.py
edac_wrap.cxx
revision.py

1
pmsco/edac/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'muntwiler_m'

130
pmsco/edac/edac_all.patch
View File

@@ -1,130 +0,0 @@
*** /home/muntwiler_m/mnt/pearl_data/software/edac/edac_all.cpp 2011-04-14 23:38:44.000000000 +0200
--- edac_all.cpp 2016-02-11 12:15:45.322049772 +0100
***************
*** 10117,10122 ****
--- 10117,10123 ----
void scan_imfp(char *name);
void scan_imfp(FILE *fout);
numero iimfp_TPP(numero kr);
+ numero iimfp_SD(numero kr);
numero TPP_rho, TPP_Nv, TPP_Ep, TPP_Eg;
numero screening_length;
int scattering_so;
***************
*** 10230,10235 ****
--- 10231,10237 ----
int n_th;
int n_fi;
+ int n_ang;
numero *th, *fi;
numero *th_out,
***************
*** 10239,10244 ****
--- 10241,10247 ----
void free(void);
void init_th(numero thi, numero thf, int nth);
void init_phi(numero fii, numero fif, int nfi);
+ void read_angles(FILE *fin, char *my_file);
void init_refraction(
numero refraction);
void init_transmission(
***************
*** 12485,12490 ****
--- 12488,12494 ----
else {
kr=sqrt(sqr(calc.k[ik])+2*V0);
if(iimfp_flag==0) ki=iimfp.val(kr)/2;
+ else if(iimfp_flag==3) ki=iimfp_SD(kr)/2;
else ki=iimfp_TPP(kr)/2;
set_k(complex(kr,ki));
} } else if(calc.k_flag==2) set_k(calc.kc[ik]);
***************
*** 12507,12512 ****
--- 12511,12522 ----
numero imfp=E/(TPP_Ep*TPP_Ep*(beta*log(gamma*E)-C/E+D/(E*E)))/a0_au;
return 1/imfp;
}
+ numero propagation::iimfp_SD(numero kr)
+ {
+ numero E=sqr(kr)/2*au_eV;
+ numero imfp = (1.43e3/sqr(E) + 0.54*sqrt(E))/a0_au;
+ return 1/imfp;
+ }
void propagation::scan_imfp(char *name)
{
FILE *fout=NULL;
***************
*** 13202,13208 ****
}
final_state::final_state(void)
{
! n_th=n_fi=0;
n_1=n_2=0;
Ylm0_th_flag=Ylm0_fi_flag=0;
mesh_flag=0;
--- 13212,13218 ----
}
final_state::final_state(void)
{
! n_th=n_fi=n_ang=0;
n_1=n_2=0;
Ylm0_th_flag=Ylm0_fi_flag=0;
mesh_flag=0;
***************
*** 13233,13238 ****
--- 13243,13271 ----
if(n_fi==1) fi[0]=fii;
else for(j=0; j<n_fi; j++) fi[j]=fii+j*(fif-fii)/(n_fi-1);
} }
+ void final_state::read_angles(FILE *fin, char *my_file)
+ {
+ FILE *fang; int i, nang;
+ if(!strcmpC(my_file,"inline")) fang=fin;
+ else fang=open_file(foutput,my_file,"r");
+ nang=read_int(fang);
+ free_mesh();
+ if(nang>1) {
+ delete [] th; delete [] th_out; delete [] transmission; delete [] fi;
+ n_th=nang;
+ th=new numero [n_th];
+ th_out=new numero [n_th];
+ transmission=new numero [n_th];
+ n_fi=nang;
+ fi=new numero [n_fi];
+ for(i=0; i<nang; i++) {
+ th[i]=th_out[i]=read_numero(fang);
+ transmission[i]=1;
+ fi[i]=read_numero(fang);
+ }
+ }
+ if(strcmpC(my_file,"inline")) fclose(fang);
+ }
void final_state::init_refraction(numero refraction)
{
int i;
***************
*** 14743,14748 ****
--- 14776,14783 ----
|| scat.TPP_Ep<=0 || scat.TPP_Eg<0)
on_error(foutput,"(input) imfp TPP-2M", "wrong parameters");
scat.iimfp_flag=1;
+ } else if(!strcmpC(name,"SD-UC")) {
+ scat.iimfp_flag=3;
} else {
scat.read_imfp(fprog,name);
scat.iimfp_flag=0;
***************
*** 15162,15164 ****
--- 15197,15206 ----
fprintf(foutput,"That's all, folks!\n");
return 0;
}
+ int run_script(char *scriptfile)
+ {
+ particle_type=electrones;
+ init_fact();
+ electron.program(scriptfile);
+ return 0;
+ }

52
pmsco/edac/makefile
View File

@@ -1,52 +0,0 @@
SHELL=/bin/sh
# makefile for EDAC program and module
#
# the EDAC source code is not included in the public distribution.
# please obtain it from the original author,
# copy it to this directory,
# and apply the edac_all.patch patch before compilation.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac
FC=gfortran
FCCOPTS=
F2PY=f2py
F2PYOPTS=
CC=g++
CCOPTS=-Wno-write-strings
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHONOPTS=
all: edac
edac: edac.exe _edac.so edac.py
edac.exe: edac_all.cpp
$(CC) $(CCOPTS) -o edac.exe edac_all.cpp
edac_wrap.cxx: edac_all.cpp edac.i
$(SWIG) $(SWIGOPTS) -c++ -python edac.i
edac.py _edac.so: edac_wrap.cxx setup.py
$(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
revision.py: _edac.so
git log --pretty=format:"code_rev = 'Code revision %h, %ad'" --date=iso -1 > $@ || echo "code_rev = 'Code revision unknown, "`date +"%F %T %z"`"'" > $@
echo "" >> revision.py
revision.txt: _edac.so edac.exe
git log --pretty=format:"Code revision %h, %ad" --date=iso -1 > $@ || echo "Code revision unknown, "`date +"%F %T %z"` > $@
echo "" >> revision.txt
clean:
rm -f *.so *.o *.exe
rm -f *_wrap.cxx
rm -f revision.py
rm -f revision.txt

20
pmsco/edac/setup.py
View File

@@ -1,20 +0,0 @@
#!/usr/bin/env python
"""
setup.py file for EDAC
"""
from distutils.core import setup, Extension
edac_module = Extension('_edac',
sources=['edac_wrap.cxx', 'edac_all.cpp'],
)
setup (name = 'edac',
version = '0.1',
author = "Matthias Muntwiler",
description = """EDAC module in Python""",
ext_modules = [edac_module],
py_modules = ["edac"], requires=['numpy']
)

1361
pmsco/elements/bindingenergy.json Normal file
View File

File diff suppressed because it is too large Load Diff

225
pmsco/elements/bindingenergy.py Normal file
View File

@@ -0,0 +1,225 @@
"""
@package pmsco.elements.bindingenergy
Electron binding energies of the elements
Extends the element table of the `periodictable` package
(https://periodictable.readthedocs.io/en/latest/index.html)
by the electron binding energies.
The binding energies are compiled from Gwyn Williams' web page
(https://userweb.jlab.org/~gwyn/ebindene.html).
Please refer to the original web page or the x-ray data booklet
for original sources, definitions and remarks.
Binding energies of gases are replaced by respective values of a common compound
from the 'handbook of x-ray photoelectron spectroscopy' (Physical Electronics, Inc., 1995).
Usage
-----
This module requires the periodictable package (https://pypi.python.org/pypi/periodictable).
~~~~~~{.py}
import periodictable as pt
import pmsco.elements.bindingenergy
# read any periodictable's element interfaces, e.g.
print(pt.gold.binding_energy['4f7/2'])
print(pt.elements.symbol('Au').binding_energy['4f7/2'])
print(pt.elements.name('gold').binding_energy['4f7/2'])
print(pt.elements[79].binding_energy['4f7/2'])
~~~~~~
The database is loaded from the accompanying bindingenergy.json file on first demand.
Attributes are writable, you may update the values in your run-time instance of the database.
Normally, the user will not need to call any functions in this module directly.
The query_binding_energy() function queries all terms with a particular binding energy.
@author Matthias Muntwiler
@copyright (c) 2020-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import json
import numpy as np
from pathlib import Path
import periodictable as pt
import periodictable.core
index_energy = np.zeros(0)
index_number = np.zeros(0)
index_term = []
default_data_path = Path(Path(__file__).parent, "bindingenergy.json")
def load_data(data_path=None):
"""
load binding energy data from json file
the data file must be in the same format as generated by save_data.
@param data_path file path of the data file. default: "bindingenergy.json" next to this module file
@return dictionary
"""
if data_path is None:
data_path = default_data_path
with open(data_path, "rt", encoding="utf8") as fp:
data = json.load(fp)
return data
def save_data(data_path=None):
"""
save binding energy data to json file
@param data_path file path of the data file. default: "bindingenergy.json" next to this module file
@return None
"""
if data_path is None:
data_path = default_data_path
data = {}
for element in pt.elements:
element_data = {}
for term, energy in element.binding_energy.items():
element_data[term] = energy
if element_data:
data[element.number] = element_data
with open(data_path, "w", encoding="utf8") as fp:
json.dump(data, fp, sort_keys=True, indent='\t')
def init(table, reload=False):
if 'binding_energy' in table.properties and not reload:
return
table.properties.append('binding_energy')
pt.core.Element.binding_energy = {}
pt.core.Element.binding_energy_units = "eV"
data = load_data()
for el_key, el_data in data.items():
try:
el = table[int(el_key)]
except ValueError:
el = table.symbol(el_key)
el.binding_energy = el_data
def build_index():
"""
build an index for query_binding_energy().
the index is kept in global variables of the module.
@return None
"""
global index_energy
global index_number
global index_term
n = 0
for element in pt.elements:
n += len(element.binding_energy)
index_energy = np.zeros(n)
index_number = np.zeros(n)
index_term = []
for element in pt.elements:
for term, energy in element.binding_energy.items():
index_term.append(term)
i = len(index_term) - 1
index_energy[i] = energy
index_number[i] = element.number
def query_binding_energy(energy, tol=1.0):
"""
search the periodic table for a specific binding energy and return all matching terms.
@param energy: binding energy in eV.
@param tol: tolerance in eV.
@return: list of dictionaries containing element and term specification.
the list is ordered arbitrarily.
each dictionary contains the following keys:
@arg 'number': element number
@arg 'symbol': element symbol
@arg 'term': spectroscopic term
@arg 'energy': actual binding energy
"""
if len(index_energy) == 0:
build_index()
sel = np.abs(index_energy - energy) < tol
idx = np.where(sel)
result = []
for i in idx[0]:
el_num = int(index_number[i])
d = {'number': el_num,
'symbol': pt.elements[el_num].symbol,
'term': index_term[i],
'energy': index_energy[i]}
result.append(d)
return result
def export_flat_text(f):
"""
export the binding energies to a flat general text file.
the file has four space-separated columns `number`, `symbol`, `term`, `energy`.
column names are included in the first row.
@param f: file path or open file object
@return: None
"""
if hasattr(f, "write") and callable(f.write):
f.write("number symbol term energy\n")
for element in pt.elements:
for term, energy in element.binding_energy.items():
f.write(f"{element.number} {element.symbol} {term} {energy}\n")
else:
with open(f, "w") as fi:
export_flat_text(fi)
def import_flat_text(f):
"""
import binding energies from a flat general text file.
data is in space-separated columns.
the first row contains column names.
at least the columns `number`, `term`, `energy` must be present.
the function updates existing entries and appends entries of non-existing terms.
existing terms that are not listed in the file remain unchanged.
@param f: file path or open file object
@return: None
"""
data = np.atleast_1d(np.genfromtxt(f, names=True, dtype=None, encoding="utf8"))
for d in data:
pt.elements[d['number']].binding_energy[d['term']] = d['energy']
def _load_binding_energy():
"""
delayed loading of the binding energy table.
"""
init(periodictable.core.default_table())
periodictable.core.delayed_load(['binding_energy'], _load_binding_energy)

BIN
pmsco/elements/cross-sections.dat Normal file
View File

Binary file not shown.

446
pmsco/elements/photoionization.py Normal file
View File

@@ -0,0 +1,446 @@
"""
@package pmsco.elements.photoionization
Photoionization cross-sections of the elements
Extends the element table of the `periodictable` package
(https://periodictable.readthedocs.io/en/latest/index.html)
by a table of photoionization cross-sections and asymmetry parameters.
The data is available from (https://vuo.elettra.eu/services/elements/)
or (https://figshare.com/articles/dataset/Digitisation_of_Yeh_and_Lindau_Photoionisation_Cross_Section_Tabulated_Data/12389750).
Both sources are based on the original atomic data tables by Yeh and Lindau (1985).
The Elettra data includes the cross section and asymmetry parameter and is interpolated at finer steps,
whereas the Kalha data contains only the cross sections at the photon energies calculated by Yeh and Lindau
plus an additional point at 8 keV.
The tables go up to 1500 eV photon energy and do not resolve spin-orbit splitting.
Usage
-----
This module adds the photoionization attribute to the elements database of the periodictable package (https://pypi.python.org/pypi/periodictable).
Python >= 3.6, numpy >= 1.15 and the periodictable package are required.
~~~~~~{.py}
import numpy as np
import periodictable as pt
import pmsco.elements.photoionization
# get a SubShellPhotoIonization object from any of periodictable's element interface:
sspi = pt.gold.photoionization['4f']
sspi = pt.elements.symbol('Au').photoionization['4f']
sspi = pt.elements.name('gold').photoionization['4f']
sspi = pt.elements[79].photoionization['4f']
# get the cross section, asymmetry parameter or differential cross section at 800 eV photon energy:
sspi.cross_section(800)
sspi.asymmetry_parameter(800)
sspi.diff_cross_section(800, gamma=30)
# with the j quantum number, the cross-section is weighted based on a full sub-shell:
sspi = pt.gold.photoionization['4f7/2']
print(sspi.weight)
print(pt.gold.photoionization['4f7/2'].cross_section(800) / pt.gold.photoionization['4f'].cross_section(800))
# the original data is contained in the data array (which is a numpy.recarray):
sspi.data.eph, sspi.data.cs, sspi.data.ap
~~~~~~
The data is loaded on demand from the cross-sections.dat file when the photoionization record is first accessed.
Normally, the user will not need to call any functions in this module directly.
The load_elettra_data()/load_kalha_data() and save_pickled_data() functions are provided
to import data from one of the sources referenced above and
to create the cross-sections.dat file.
@author Matthias Muntwiler
@copyright (c) 2020-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import copy
import numpy as np
from pathlib import Path
import periodictable as pt
import pickle
import urllib.request
import urllib.error
import periodictable.core
class PhotoIonization(dict):
"""
photo-ionization parameters of an element
this class provides the photo-ionization cross-section and asymmetry parameter of the sub-shells of an element.
it is, essentially, a dictionary, mapping 'nl' and 'nlj' terms to the corresponding SubShellPhotoIonization object.
examples of 'nl' and 'nlj' terms: '4f' and '4f7/2'
@note the dictionary actually contains raw data for 'nl' terms only.
for 'nlj' terms, the corresponding 'nl' object is copied,
and a weight according to the spin-orbit multiplicity is set.
@note 'nlj' terms are not considered by any methods or properties
except the bracket notation or __getitem__ method!
in particular, iteration or the keys() method will yield 'nl' terms only.
"""
def __init__(self, *args, **kwargs):
"""
dictionary constructor
the class accepts the same arguments as the Python built-in dict constructor.
keys are 'nl' terms, e.g. '4f', and values must be SubShellPhotoIonization() objects.
@param args:
@param kwargs:
"""
super().__init__(*args, **kwargs)
self.cross_section_units = "Mb"
def __getitem__(self, k):
"""
get sub-shell photo-ionization data by 'nl' or 'nlj' term.
@param k: dictionary key.
if this is an 'nl' term, the original object is returned.
if this is an 'nlj' term, a proxy of the corresponding 'nl' object
with shared data but weight based on j-branching is returned.
@return: SubShellPhotoIonization() object
@note whether the original or a proxy object is returned,
its data attribute always refers to the original data.
any modification will affect the original data (process memory).
"""
spi = super().__getitem__(k[0:2])
if len(k) > 2:
spi = copy.copy(spi)
spi.set_spin_orbit(k[1:5])
return spi
class SubShellPhotoIonization(object):
"""
Sub-shell photo-ionization parameters versus photon energy.
this class provides the photo-ionization cross-section and asymmetry parameter of one sub-shell.
it contains a three-column record array of photon energy, cross section and asymmetry parameter in self.data.
accessory functions provide high-level access to specific views and interpolated data.
a weighting factor self.weight is multiplied to the method results.
it is normally used to weight the spin-orbit peaks by calling set_spin_orbit().
"""
SPIN_ORBIT_WEIGHTS = {"p1/2": 1. / 3.,
"p3/2": 2. / 3.,
"d3/2": 2. / 5.,
"d5/2": 3. / 5.,
"f5/2": 3. / 7.,
"f7/2": 4. / 7.}
def __init__(self, photon_energy, cross_section, asymmetry_parameter):
"""
initialize a new object instance.
all arrays must have the same length.
@param photon_energy: (array-like) photon energies
@param cross_section: (array-like) cross-section values
@param asymmetry_parameter: (array-like) asymmetry parameter values
"""
super().__init__()
self.data = np.rec.fromarrays([photon_energy, cross_section, asymmetry_parameter], names='eph, cs, ap')
self.weight = 1.
def cross_section(self, photon_energy):
"""
interpolated sub-shell cross-section at a specific energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@param photon_energy: photon energy in eV.
can be scalar or numpy array.
@return: cross-section in Mb.
numpy.nan where photon_energy is off range.
"""
cs = np.interp(photon_energy, self.data.eph, self.data.cs, left=np.nan, right=np.nan) * self.weight
return cs
def asymmetry_parameter(self, photon_energy):
"""
interpolated asymmetry parameter at a specific energy.
@param photon_energy: photon energy in eV.
can be scalar or numpy array.
@return: asymmetry parameter (0..2).
numpy.nan where photon_energy is off range.
"""
ap = np.interp(photon_energy, self.data.eph, self.data.ap, left=np.nan, right=np.nan)
return ap
def diff_cross_section(self, photon_energy, gamma):
"""
differential cross-section for linear polarization.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@param photon_energy: photon energy in eV.
@param gamma: angle between polarization vector and electron propagation direction in degrees.
@return: differential cross-section in Mb.
"""
p2 = (3 * np.cos(gamma) ** 2 - 1) / 2
cs = self.cross_section(photon_energy)
ap = self.asymmetry_parameter(photon_energy)
dcs = cs / 4 / np.pi * (1 + ap * p2)
return dcs
def photon_energy_array(self):
"""
photon energy array.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return:
"""
return self.data.eph
def cross_section_array(self):
"""
sub-shell cross-section versus photon energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return: numpy.ndarray
"""
return self.data.cs * self.weight
def asymmetry_parameter_array(self):
"""
sub-shell asymmetry parameter versus photon energy.
the weighting factor self.weight (e.g. spin-orbit) is included in the result.
@return: numpy.ndarray
"""
return self.data.ap
def diff_cross_section_array(self, gamma):
"""
differential cross-section for linear polarization (full array).
@param gamma: angle between polarization vector and electron propagation direction in degrees.
@return: (np.ndarray) differential cross-section in Mb.
"""
p2 = (3 * np.cos(gamma) ** 2 - 1) / 2
dcs = self.data.cs / 4 / np.pi * (1 + self.data.ap * p2) * self.weight
return dcs
def set_spin_orbit(self, lj):
"""
set the weight according to the spin-orbit quantum number (based on full sub-shell).
the weight is stored in the self.weight attribute.
it is applied to the results of the cross-section methods, but not to the raw data in self.data!
@param lj: (str) 4-character lj term notation, e.g. 'f7/2'
@return: None
"""
self.weight = self.SPIN_ORBIT_WEIGHTS.get(lj, 1.)
def load_kalha_data():
"""
load all cross-sections from csv-files by Kalha et al.
the files must be placed in the 'kalha' directory next to this file.
@return: cross-section data in a nested dictionary, cf. load_pickled_data().
"""
data = {}
p = Path(Path(__file__).parent, "kalha")
for entry in p.glob('*_*.csv'):
if entry.is_file():
try:
element = int(entry.stem.split('_')[0])
except ValueError:
pass
else:
data[element] = load_kalha_file(entry)
return data
def load_kalha_file(path):
"""
load the cross-sections of an element from a csv-file by Kalha et al.
@param path: file path
@return: (dict) dictionary of 'nl' terms.
the data items are tuples (photon_energy, cross_sections) of 1-dimensional numpy arrays.
"""
a = np.genfromtxt(path, delimiter=',', names=True)
b = ~np.isnan(a['Photon_Energy__eV'])
a = a[b]
eph = a['Photon_Energy__eV'].copy()
data = {}
for n in range(1, 8):
for l in 'spdf':
col = f"{n}{l}"
try:
data[col] = SubShellPhotoIonization(eph, a[col].copy(), np.zeros_like(eph))
except ValueError:
pass
return data
def load_kalha_configuration(path):
"""
load the electron configuration from a csv-file by Kalha et al.
@param path: file path
@return: (dict) dictionary of 'nl' terms mapping to number of electrons in the sub-shell.
"""
p = Path(path)
subshells = []
electrons = []
config = {}
with p.open() as f:
for l in f.readlines():
s = l.split(',')
k_eph = "Photon Energy"
k_el = "#electrons"
if s[0][0:len(k_eph)] == k_eph:
subshells = s[1:]
elif s[0][0:len(k_el)] == k_el:
electrons = s[1:]
for i, sh in enumerate(subshells):
if sh:
config[sh] = electrons[i]
return config
def load_elettra_file(symbol, nl):
"""
download the cross sections of one level from the Elettra webelements web site.
@param symbol: (str) element symbol
@param nl: (str) nl term, e.g. '2p' (no spin-orbit)
@return: PhotoIonizationData(photon_energy, cross_section, asymmetry_parameter)
named tuple of 1-dimensional numpy arrays.
"""
spi = None
url = f"https://vuo.elettra.eu/services/elements/data/{symbol.lower()}{nl}.txt"
try:
data = urllib.request.urlopen(url)
except urllib.error.HTTPError:
pass
else:
a = np.genfromtxt(data)
try:
spi = SubShellPhotoIonization(a[:, 0], a[:, 1], a[:, 4])
except IndexError:
pass
return spi
def load_elettra_data():
"""
download the cross sections from the Elettra webelements web site.
@return: cross-section data in a nested dictionary, cf. load_pickled_data().
"""
data = {}
for element in pt.elements:
element_data = {}
for nlj in element.binding_energy:
nl = nlj[0:2]
eb = element.binding_energy[nlj]
if nl not in element_data and eb <= 2000:
spi = load_elettra_file(element.symbol, nl)
if spi is not None:
element_data[nl] = spi
if len(element_data):
data[element.symbol] = element_data
return data
def save_pickled_data(path, data):
"""
save a cross section data dictionary to a python-pickled file.
@param path: file path
@param data: cross-section data in a nested dictionary, cf. load_pickled_data().
@return: None
"""
with open(path, "wb") as f:
pickle.dump(data, f)
def load_pickled_data(path):
"""
load the cross section data from a python-pickled file.
the file can be generated by the save_pickled_data() function.
@param path: file path
@return: cross-section data in a nested dictionary.
the first-level keys are element symbols.
the second-level keys are 'nl' terms (e.g. '2p').
note that the Yeh and Lindau tables do not resolve spin-orbit splitting.
the data items are (photon_energy, cross_sections) tuples
of 1-dimensional numpy arrays holding the data table.
cross section values are given in Mb.
"""
with open(path, "rb") as f:
data = pickle.load(f)
return data
def init(table, reload=False):
"""
loads cross-section data into the periodic table.
this function is called by the periodictable to load the data on demand.
@param table:
@param reload:
@return:
"""
if 'photoionization' in table.properties and not reload:
return
table.properties.append('photoionization')
# default value
pt.core.Element.photoionization = PhotoIonization()
p = Path(Path(__file__).parent, "cross-sections.dat")
data = load_pickled_data(p)
for el_key, el_data in data.items():
# el_data is dict('nl': PhotoIonizationData)
try:
el = table[int(el_key)]
except ValueError:
el = table.symbol(el_key)
el.photoionization = PhotoIonization(el_data)
def _load_photoionization():
"""
delayed loading of the binding energy table.
"""
init(periodictable.core.default_table())
periodictable.core.delayed_load(['photoionization'], _load_photoionization)

209
pmsco/elements/spectrum.py Normal file
View File

@@ -0,0 +1,209 @@
"""
@package pmsco.elements.spectrum
photoelectron spectrum simulator
this module calculates the basic structure of a photoelectron spectrum.
it calculates positions and approximate amplitude of elastic peaks
based on photon energy, binding energy, photoionization cross section, and stoichiometry.
escape depth, photon flux, analyser transmission are not accounted for.
usage
-----
this module requires python 3.6, numpy, matplotlib and
the periodictable package (https://pypi.python.org/pypi/periodictable).
~~~~~~{.py}
import numpy as np
import periodictable as pt
import pmsco.elements.spectrum as spec
# for working with the data
labels, energy, intensity = spec.build_spectrum(800., {"Ti": 1, "O": 2})
# for plotting
spec.plot_spectrum(800., {"Ti": 1, "O": 2})
~~~~~~
@author Matthias Muntwiler
@copyright (c) 2020 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from matplotlib import pyplot as plt
import numpy as np
import periodictable as pt
from . import bindingenergy
from . import photoionization
def get_element(number_or_symbol):
"""
return the given Element object of the periodic table.
@param number_or_symbol: atomic number (int) or chemical symbol (str).
@return: Element object.
"""
try:
el = pt.elements[number_or_symbol]
except KeyError:
el = pt.elements.symbol(number_or_symbol)
return el
def get_binding_energy(photon_energy, element, nlj):
"""
look up the binding energy of a core level and check whether it is smaller than the photon energy.
@param photon_energy: photon energy in eV.
@param element: Element object of the periodic table.
@param nlj: (str) spectroscopic term, e.g. '4f7/2'.
@return: (float) binding energy or numpy.nan.
"""
try:
eb = element.binding_energy[nlj]
except KeyError:
return np.nan
if eb < photon_energy:
return eb
else:
return np.nan
def get_cross_section(photon_energy, element, nlj, gamma=None):
"""
look up the photo-ionization cross-section.
since the Yeh/Lindau tables do not resolve the spin-orbit splitting,
this function applies the normal relative weights of a full sub-shell.
the result is a linear interpolation between tabulated values.
@param photon_energy: photon energy in eV.
@param element: Element object of the periodic table.
@param nlj: (str) spectroscopic term, e.g. '4f7/2'.
the j-value can be left out, in which case the sum over all j-states is returned.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@return: (float) total (gamma=None) or differential (gamma not None) cross section in Mb.
"""
if not hasattr(element, "photoionization"):
element = get_element(element)
try:
pi = element.photoionization[nlj]
except KeyError:
return np.nan
if gamma is None:
cs = pi.cross_section(photon_energy)
else:
cs = pi.diff_cross_section(photon_energy, gamma)
return cs
def build_spectrum(photon_energy, elements, binding_energy=False, work_function=4.5, gamma=None):
"""
calculate the positions and amplitudes of core-level photoemission lines.
the function looks up the binding energies and cross sections of all photoemission lines in the energy range
given by the photon energy and returns an array of expected spectral lines.
@param photon_energy: (numeric) photon energy in eV.
@param elements: list or dictionary of elements.
elements are identified by their atomic number (int) or chemical symbol (str).
if a dictionary is given, the (float) values are stoichiometric weights of the elements.
@param binding_energy: (bool) return binding energies (True) rather than kinetic energies (False, default).
@param work_function: (float) work function of the instrument in eV.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@return: tuple (labels, positions, intensities) of 1-dimensional numpy arrays representing the spectrum.
labels are in the format {Symbol}{n}{l}{j}.
"""
ekin = []
ebind = []
intens = []
labels = []
for element in elements:
el = get_element(element)
for n in range(1, 8):
for l in "spdf":
for j in ['', '1/2', '3/2', '5/2', '7/2']:
nlj = f"{n}{l}{j}"
eb = get_binding_energy(photon_energy, el, nlj)
cs = get_cross_section(photon_energy, el, nlj, gamma=gamma)
try:
cs = cs * elements[element]
except (KeyError, TypeError):
pass
if not np.isnan(eb) and not np.isnan(cs):
ekin.append(photon_energy - eb - work_function)
ebind.append(eb)
intens.append(cs)
labels.append(f"{el.symbol}{nlj}")
ebind = np.array(ebind)
ekin = np.array(ekin)
intens = np.array(intens)
labels = np.array(labels)
if binding_energy:
return labels, ebind, intens
else:
return labels, ekin, intens
def plot_spectrum(photon_energy, elements, binding_energy=False, work_function=4.5, gamma=None, show_labels=True):
"""
plot a simple spectrum representation of a material.
the function looks up the binding energies and cross sections of all photoemission lines in the energy range
given by the photon energy and returns an array of expected spectral lines.
the spectrum is plotted using matplotlib.pyplot.stem.
@param photon_energy: (numeric) photon energy in eV.
@param elements: list or dictionary of elements.
elements are identified by their atomic number (int) or chemical symbol (str).
if a dictionary is given, the (float) values are stoichiometric weights of the elements.
@param binding_energy: (bool) return binding energies (True) rather than kinetic energies (False, default).
@param work_function: (float) work function of the instrument in eV.
@param gamma: (float) angle in degrees between linear polarization vector and photoelectron emission direction.
By default (None), unpolarized light or magic angle (54.7 deg) geometry is assumed.
@param show_labels: (bool) show peak labels (True, default) or not (False).
@return: (figure, axes)
"""
labels, energy, intensity = build_spectrum(photon_energy, elements, binding_energy=binding_energy,
work_function=work_function, gamma=gamma)
fig, ax = plt.subplots()
ax.stem(energy, intensity, basefmt=' ', use_line_collection=True)
if show_labels:
for sxy in zip(labels, energy, intensity):
ax.annotate(sxy[0], xy=(sxy[1], sxy[2]), textcoords='data')
ax.grid()
if binding_energy:
ax.set_xlabel('binding energy')
else:
ax.set_xlabel('kinetic energy')
ax.set_ylabel('intensity')
ax.set_title(elements)
return fig, ax
def plot_cross_section(el, nlj):
energy = np.arange(100, 1500, 140)
cs = get_cross_section(energy, el, nlj)
fig, ax = plt.subplots()
ax.set_yscale("log")
ax.plot(energy, cs)

354
pmsco/files.py
View File

@@ -1,16 +1,19 @@
"""
@package pmsco.files
manage files produced by pmsco.
manage the lifetime of files produced by pmsco.
@author Matthias Muntwiler
@copyright (c) 2016 by Paul Scherrer Institut @n
@copyright (c) 2016-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import logging
import mpi4py
@@ -24,28 +27,28 @@ logger = logging.getLogger(__name__)
#
# each string of this set marks a category of files.
#
# @arg @c 'input' : raw input files for calculator, including cluster and phase files in custom format
# @arg @c 'output' : raw output files from calculator
# @arg @c 'phase' : phase files in portable format for report
# @arg @c 'cluster' : cluster files in portable XYZ format for report
# @arg @c 'log' : log files
# @arg @c 'debug' : debug files
# @arg @c 'model': output files in ETPAI format: complete simulation (a_-1_-1_-1_-1)
# @arg @c 'scan' : output files in ETPAI format: scan (a_b_-1_-1_-1)
# @arg @c 'symmetry' : output files in ETPAI format: symmetry (a_b_c_-1_-1)
# @arg @c 'emitter' : output files in ETPAI format: emitter (a_b_c_d_-1)
# @arg @c 'region' : output files in ETPAI format: region (a_b_c_d_e)
# @arg @c 'report': final report of results
# @arg @c 'population': final state of particle population
# @arg @c 'rfac': files related to models which give bad r-factors (dynamic category, see below).
# @arg 'input' : raw input files for calculator, including cluster and atomic files in custom format
# @arg 'output' : raw output files from calculator
# @arg 'atomic' : atomic scattering (phase, emission) files in portable format
# @arg 'cluster' : cluster files in portable XYZ format for report
# @arg 'log' : log files
# @arg 'debug' : debug files
# @arg 'model': output files in ETPAI format: complete simulation (a_-1_-1_-1_-1)
# @arg 'scan' : output files in ETPAI format: scan (a_b_-1_-1_-1)
# @arg 'domain' : output files in ETPAI format: domain (a_b_c_-1_-1)
# @arg 'emitter' : output files in ETPAI format: emitter (a_b_c_d_-1)
# @arg 'region' : output files in ETPAI format: region (a_b_c_d_e)
# @arg 'report': final report of results
# @arg 'population': final state of particle population
# @arg 'rfac': files related to models which give bad r-factors (dynamic category, see below).
#
# @note @c 'rfac' is a dynamic category not connected to a particular file or content type.
# no file should be marked @c 'rfac'.
# the string is used only to specify whether bad models should be deleted or not.
# if so, all files related to bad models are deleted, regardless of their static category.
#
FILE_CATEGORIES = {'cluster', 'phase', 'input', 'output',
'report', 'region', 'emitter', 'scan', 'symmetry', 'model',
FILE_CATEGORIES = {'cluster', 'atomic', 'input', 'output',
'report', 'region', 'emitter', 'scan', 'domain', 'model',
'log', 'debug', 'population', 'rfac'}
## @var FILE_CATEGORIES_TO_KEEP
@@ -53,7 +56,7 @@ FILE_CATEGORIES = {'cluster', 'phase', 'input', 'output',
#
# this constant defines the default set of file categories that are kept after the calculation.
#
FILE_CATEGORIES_TO_KEEP = {'cluster', 'model', 'report', 'population'}
FILE_CATEGORIES_TO_KEEP = {'cluster', 'model', 'scan', 'report', 'population'}
## @var FILE_CATEGORIES_TO_DELETE
# categories of files to be deleted.
@@ -67,13 +70,17 @@ FILE_CATEGORIES_TO_DELETE = FILE_CATEGORIES - FILE_CATEGORIES_TO_KEEP
class FileTracker(object):
"""
organize output files of calculations.
manage the lifetime of files produced by the calculations.
the file manager stores references to data files generated during calculations
and cleans up unused files according to a range of filter criteria.
this class identifies files by _file name_.
file names must therefore be unique over the whole calculation process.
it is possible to specify a full path that is used for communication with the operating system.
"""
## @var files_to_delete (set)
## @var categories_to_delete (set)
# categories of generated files that should be deleted after the calculation.
#
# each string of this set marks a category of files to be deleted.
@@ -93,96 +100,119 @@ class FileTracker(object):
#
# the default is 10.
## @var _last_id (int)
# last used file identification number (incremental)
## @var _path_by_id (dict)
# key = file id, value = file path
## @var _model_by_id (dict)
# key = file id, value = model number
## @var _category_by_id (dict)
# key = file id, value = category (str)
## @var _file_model (dict)
# key = file name, value = model number
## @var _file_category (dict)
# key = file name, value = category (str)
## @var _file_path (dict)
# key = file name, value = absolute file path (str)
## @var _rfac_by_model (dict)
# key = model number, value = file id
# key = model number, value = R-factor
## @var _complete_by_model (dict)
# key = model number, value (boolean) = all calculations complete, files can be deleted
## @var _complete_models (set)
# this set contains the model numbers of the models that have finished all calculations.
# files of these models can be considered for clean up.
def __init__(self):
self._id_by_path = {}
self._path_by_id = {}
self._model_by_id = {}
self._category_by_id = {}
self._file_model = {}
self._file_category = {}
self._file_path = {}
self._rfac_by_model = {}
self._complete_by_model = {}
self._last_id = 0
self._complete_models = set([])
self.categories_to_delete = FILE_CATEGORIES_TO_DELETE
self.keep_rfac = 10
def add_file(self, path, model, category='default'):
def get_file_count(self):
"""
return the number of tracked files.
@return: (int) number of tracked files.
"""
return len(self._file_path)
def get_complete_models_count(self):
"""
return the number of complete models.
@return: (int) number of complete models.
"""
return len(self._complete_models)
def add_file(self, name, model, category='default', path=''):
"""
add a new data file to the list.
@param path: (str) system path of the file relative to the working directory.
@param name: (str) unique identification of the file.
this can be the file name in the file system if file names are unique without path specification.
the name must be spelled identically
whenever the same file is referenced in a call to another method of this class.
the empty string is ignored.
@param model: (int) model number
@param category: (str) file category, e.g. 'output', etc.
@param path: (str) file system path of the file.
the file system path is used for communication with the operating system when the file is deleted.
by default, the path is the name argument expanded to a full path relative to the current working directory.
the path is expanded during the call of this method and will not change when the working directory changes.
@return: None
"""
self._last_id += 1
_id = self._last_id
self._id_by_path[path] = _id
self._path_by_id[_id] = path
self._model_by_id[_id] = model
self._category_by_id[_id] = category
if name:
self._file_model[name] = model
self._file_category[name] = category
self._file_path[name] = path if path else os.path.abspath(name)
def rename_file(self, old_path, new_path):
def rename_file(self, old_name, new_name, new_path=''):
"""
rename a data file in the list.
the method does not rename the file in the file system.
@param old_path: must match an existing file path identically.
if old_path is not in the list, the method does nothing.
@param old_name: name used in the original add_file() call.
if it is not in the list, the method does nothing.
@param new_path: new path.
@param new_name: new name of the file, see add_file().
if the file is already in the list, its model and category is overwritten by the values of the old file.
@param new_path: new file system path of the file, see add_file().
by default, the path is the name argument expanded to a full path relative to the current working directory.
@return: None
"""
try:
_id = self._id_by_path[old_path]
model = self._file_model[old_name]
cat = self._file_category[old_name]
except KeyError:
pass
else:
del self._id_by_path[old_path]
self._id_by_path[new_path] = _id
self._path_by_id[_id] = new_path
del self._file_model[old_name]
del self._file_category[old_name]
del self._file_path[old_name]
self.add_file(new_name, model, cat, new_path)
def remove_file(self, path):
def remove_file(self, name):
"""
remove a file from the list.
the method does not delete the file from the file system.
@param path: must match an existing file path identically.
if path is not in the list, the method does nothing.
@param name: must match an existing file name identically.
if the name is not found in the list, the method does nothing.
@return: None
"""
try:
_id = self._id_by_path[path]
del self._file_model[name]
del self._file_category[name]
del self._file_path[name]
except KeyError:
pass
else:
del self._id_by_path[path]
del self._path_by_id[_id]
del self._model_by_id[_id]
del self._category_by_id[_id]
def update_model_rfac(self, model, rfac):
"""
@@ -207,43 +237,57 @@ class FileTracker(object):
@param complete: (bool) True if all calculations of the model are complete (files can be deleted).
@return: None
"""
self._complete_by_model[model] = complete
if complete:
self._complete_models.add(model)
else:
self._complete_models.discard(model)
def delete_files(self, categories=None, keep_rfac=0):
def delete_files(self, categories=None, incomplete_models=False):
"""
delete the files matching the list of categories.
delete all files matching a set of categories.
this function deletes all files that are tagged with one of the given categories.
tags are set by the code sections that create the files.
for a list of common categories, see FILE_CATEGORIES.
the categories can be given as an argument or taken from the categories_to_delete property.
files are deleted regardless of R-factor.
be sure to specify only categories that you don't need in the output at all.
by default, only files of complete models (cf. set_model_complete()) are deleted
to avoid interference with running calculations.
to clean up after calculations, the incomplete_models argument can override this.
@note this method does not act on the special 'rfac' category (see delete_bad_rfac()).
@param categories: set of file categories to delete.
may include 'rfac' if bad r-factors should be deleted additionally (regardless of static category).
defaults to self.categories_to_delete.
if the argument is None, it defaults to the categories_to_delete property.
@param keep_rfac: number of best models to keep if bad r-factors are to be deleted.
the effective keep number is the greater of self.keep_rfac and this argument.
@param incomplete_models: (bool) delete files of incomplete models as well.
by default (False), incomplete models are not deleted.
@return: None
"""
if categories is None:
categories = self.categories_to_delete
for cat in categories:
self.delete_category(cat)
if 'rfac' in categories:
self.delete_bad_rfac(keep=keep_rfac)
self.delete_category(cat, incomplete_models=incomplete_models)
def delete_bad_rfac(self, keep=0, force_delete=False):
"""
delete the files of all models except a specified number of good models.
delete all files of all models except for a specified number of best ranking models.
the method first determines which models to keep.
models with R factor values of 0.0, without a specified R-factor, and
the specified number of best ranking non-zero models are kept.
the files belonging to the keeper models are kept, all others are deleted,
regardless of category.
files of incomplete models are also kept.
in addition, incomplete models, models with R factor = 0.0,
and those without a specified R-factor are kept.
all other files are deleted.
the method does not consider the file category.
the files are deleted from the list and the file system.
files are deleted only if 'rfac' is specified in self.categories_to_delete
or if force_delete is set to True.
the method executes only if 'rfac' is specified in self.categories_to_delete
or if force_delete is True.
otherwise the method does nothing.
@param keep: number of files to keep.
@@ -252,8 +296,6 @@ class FileTracker(object):
@param force_delete: delete the bad files even if 'rfac' is not selected in categories_to_delete.
@return: None
@todo should clean up rfac and model dictionaries from time to time.
"""
if force_delete or 'rfac' in self.categories_to_delete:
keep = max(keep, self.keep_rfac)
@@ -263,62 +305,132 @@ class FileTracker(object):
except IndexError:
return
complete_models = {_model for (_model, _complete) in self._complete_by_model.iteritems() if _complete}
del_models = {_model for (_model, _rfac) in self._rfac_by_model.iteritems() if _rfac > rfac_split}
del_models &= complete_models
del_ids = {_id for (_id, _model) in self._model_by_id.iteritems() if _model in del_models}
for _id in del_ids:
self.delete_file(_id)
keep_models = {model for (model, rfac) in self._rfac_by_model.items() if 0.0 <= rfac <= rfac_split}
del_models = self._complete_models - keep_models
del_names = {name for (name, model) in self._file_model.items() if model in del_models}
for name in del_names:
self.delete_file(name)
def delete_category(self, category):
def delete_models(self, keep=None, delete=None):
"""
delete all files by model.
this involves the following steps:
1. determine a list of complete models
(incomplete models are still being processed and must not be deleted).
2. intersect with the _delete_ list if specified.
3. subtract the _keep_ list if specified.
if neither the _keep_ nor the _delete_ list is specified,
or if the steps above resolve to the _complete_ list
the method considers it as an error and does nothing.
@param keep: (sequence) model numbers to keep, i.e., delete all others.
@param delete: (sequence) model numbers to delete.
@return (int) number of models deleted.
"""
del_models = self._complete_models.copy()
if delete:
del_models &= delete
if keep:
del_models -= keep
if not del_models or del_models == self._complete_models:
return 0
del_names = {name for (name, model) in self._file_model.items() if model in del_models}
for name in del_names:
self.delete_file(name)
return len(del_models)
def delete_category(self, category, incomplete_models=False):
"""
delete all files of a specified category from the list and the file system.
only files of complete models (cf. set_model_complete()) are deleted, but regardless of R-factor.
this function deletes all files that are tagged with the given category.
tags are set by the code sections that create the files.
for a list of common categories, see FILE_CATEGORIES.
files are deleted regardless of R-factor.
be sure to specify only categories that you don't need in the output at all.
by default, only files of complete models (cf. set_model_complete()) are deleted
to avoid interference with running calculations.
to clean up after calculations, the incomplete_models argument can override this.
@param category: (str) category.
should be one of FILE_CATEGORIES. otherwise, the function has no effect.
@param incomplete_models: (bool) delete files of incomplete models as well.
by default (False), incomplete models are not deleted.
@return: None
"""
complete_models = {_model for (_model, _complete) in self._complete_by_model.iteritems() if _complete}
del_ids = {_id for (_id, cat) in self._category_by_id.iteritems() if cat == category}
del_ids &= {_id for (_id, _model) in self._model_by_id.iteritems() if _model in complete_models}
for _id in del_ids:
self.delete_file(_id)
del_names = {name for (name, cat) in self._file_category.items() if cat == category}
if not incomplete_models:
del_names &= {name for (name, model) in self._file_model.items() if model in self._complete_models}
for name in del_names:
self.delete_file(name)
def delete_file(self, _id):
def delete_file(self, name):
"""
delete a specified file from the list and the file system.
the file is identified by ID number.
this method is unconditional. it does not consider category, completeness, nor R-factor.
@param _id: (int) ID number of the file to delete.
the method catches errors during file deletion and prints warnings to the logger.
@param name: must match an existing file path identically.
if it is not in the list, the method does nothing.
the method uses the associated path declared in add_file() to delete the file.
@return: None
"""
path = self._path_by_id[_id]
cat = self._category_by_id[_id]
model = self._model_by_id[_id]
del self._id_by_path[path]
del self._path_by_id[_id]
del self._model_by_id[_id]
del self._category_by_id[_id]
try:
self._os_delete_file(path)
except OSError:
logger.warning("error deleting file {0}".format(path))
cat = self._file_category[name]
model = self._file_model[name]
path = self._file_path[name]
except KeyError:
logger.warning("tried to delete untracked file {0}".format(name))
else:
logger.debug("delete file {0} ({1}, model {2})".format(path, cat, model))
del self._file_model[name]
del self._file_category[name]
del self._file_path[name]
try:
os.remove(path)
except OSError:
logger.warning("file system error deleting file {0}".format(path))
else:
logger.debug("delete file {0} ({1}, model {2})".format(path, cat, model))
@staticmethod
def _os_delete_file(path):
"""
have the operating system delete a file path.
this function is separate so that we can mock it in unit tests.
def list_files_other_models(prefix, models):
"""
list input/output files except those of the given models.
@param path: OS path
@return: None
"""
os.remove(path)
this can be used to clean up all files except those belonging to the given models.
to delete the listed files:
for f in files:
os.remove(f)
@param prefix: file name prefix up to the first underscore.
only files starting with this prefix are listed.
@param models: sequence or set of model numbers that should not be listed.
@return: set of file names
"""
file_names = set([])
for entry in os.scandir():
if entry.is_file:
elements = entry.name.split('_')
try:
if len(elements) == 6 and elements[0] == prefix and int(elements[1]) not in models:
file_names.add(entry.name)
except (IndexError, ValueError):
pass
return file_names

178
pmsco/graphics/cluster.py Executable file
View File

@@ -0,0 +1,178 @@
#!/usr/bin/env python
"""
@package pmsco.graphics.cluster
graphics rendering module for clusters.
this module is experimental.
interface and implementation may change without notice.
at the moment we are evaluating rendering solutions.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2017 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import os
import numpy as np
import argparse
import logging
logger = logging.getLogger(__name__)
try:
import pymol2
except ImportError:
logger.warning("error importing pymol2. cluster rendering using pymol2 disabled.")
pymol2 = None
try:
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# from matplotlib.backends.backend_pdf import FigureCanvasPdf
# from matplotlib.backends.backend_svg import FigureCanvasSVG
except ImportError:
Axes3D = None
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. cluster rendering using matplotlib disabled.")
def render_file(spath, view):
sname = "cluster"
opath = spath + ".png"
pm = pymol2.PyMOL()
cmd = pm.cmd
pm.start()
try:
cmd.reinitialize()
cmd.load(spath, sname)
cmd.disable("all")
cmd.enable(sname)
cmd.set("orthoscopic", 1)
cmd.bg_color("white")
cmd.show_as("spheres")
cmd.alter("all", "vdw=0.8")
#cmd.show("sticks")
#zoom selection-expression # selection to fill the viewer
#orient selection-expression # largest dim horizontal, second-largest vertical
#cmd.orient() --- should stick to fixed orientation
#cmd.turn("x", -90)
#cmd.turn("x", 0)
#cmd.turn("y", 0)
#cmd.clip("slab", 5.0)
cmd.viewport(640, 640)
cmd.zoom(complete=1)
#pymol.cmd.rebuild() #--- necessary?
cmd.png(opath)
finally:
pm.stop()
def render_cluster(clu):
pass
def set_axes_equal(ax):
"""
Make axes of 3D plot have equal scale so that spheres appear as spheres,
cubes as cubes, etc.. This is one possible solution to Matplotlib's
ax.set_aspect('equal') and ax.axis('equal') not working for 3D.
@author https://stackoverflow.com/a/31364297
@param ax: a matplotlib axis, e.g., as output from plt.gca().
"""
x_limits = ax.get_xlim3d()
y_limits = ax.get_ylim3d()
z_limits = ax.get_zlim3d()
x_range = abs(x_limits[1] - x_limits[0])
x_middle = np.mean(x_limits)
y_range = abs(y_limits[1] - y_limits[0])
y_middle = np.mean(y_limits)
z_range = abs(z_limits[1] - z_limits[0])
z_middle = np.mean(z_limits)
# The plot bounding box is a sphere in the sense of the infinity
# norm, hence I call half the max range the plot radius.
plot_radius = 0.5*max([x_range, y_range, z_range])
ax.set_xlim3d([x_middle - plot_radius, x_middle + plot_radius])
ax.set_ylim3d([y_middle - plot_radius, y_middle + plot_radius])
ax.set_zlim3d([z_middle - plot_radius, z_middle + plot_radius])
def render_xyz_matplotlib(filename, data, canvas=None):
"""
produce a graphics file from an array of 3d coordinates in the matplotlib scatter style.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy array of shape (N,3).
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111, projection='3d')
# ax.set_aspect('equal')
try:
# method available in matplotlib 2.1 and later
ax.set_proj_type('ortho')
except AttributeError:
pass
ax.scatter(data[:, 0], data[:, 1], data[:, 2], c='r', marker='o')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')
set_axes_equal(ax)
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def exec_cli():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--view', default='z')
parser.add_argument(dest='files', nargs='+')
args = parser.parse_args()
for fil in args.files:
render_file(fil, args.view)
if __name__ == '__main__':
exec_cli()
sys.exit(0)

198
pmsco/graphics/rfactor.py Normal file
View File

@@ -0,0 +1,198 @@
"""
@package pmsco.graphics.rfactor
graphics rendering module for r-factor optimization results.
this module is under development.
interface and implementation are subject to change.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
def render_param_rfac(filename, data, param_name, summary=None, canvas=None):
"""
render an r-factor versus one model parameter graph.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the results file.
this is used to derive the output file path by adding the parameter name and
the extension of the graphics file format.
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
@param param_name: name of the model parameter to display.
this must correspond to a field name of the data array.
@param summary: (dict) the dictionary returned by @ref evaluate_results.
this is used to mark the optimum value and the error limits.
if None, these values are not marked in the plot.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
ax.scatter(data[param_name], data['_rfac'], c='b', marker='o', s=4.0)
if summary is not None:
xval = summary['val'][param_name]
ymin = summary['vmin']['_rfac']
ymax = summary['vmax']['_rfac']
ax.plot((xval, xval), (ymin, ymax), ':k')
xmin = summary['vmin'][param_name]
xmax = summary['vmax'][param_name]
varr = summary['rmin'] + summary['rvar']
ax.plot((xmin, xmax), (varr, varr), ':k')
ax.grid(True)
ax.set_xlabel(param_name)
ax.set_ylabel('R-factor')
out_filename = "{0}.{1}.{2}".format(filename, param_name, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def evaluate_results(data, features=50.):
"""
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
@param features: number of independent features (pieces of information) in the data.
this quantity can be approximated as the scan range divided by the average width of a feature
which includes an intrinsic component and the instrumental resolution.
see Booth et al., Surf. Sci. 387 (1997), 152 for energy scans, and
Muntwiler et al., Surf. Sci. 472 (2001), 125 for angle scans.
the default value of 50 is a typical value.
@return dictionary of evaluation results.
the dictionary contains scalars and structured arrays as follows.
the structured arrays have the same data type as the input data and contain exactly one element.
@arg rmin: (scalar) minimum r-factor.
@arg rvar: (scalar) one-sigma variation of r-factor.
@arg imin: (scalar) array index where the minimum is located.
@arg val: (structured array) estimates of parameter values (parameter value at rmin).
@arg sig: (structured array) one-sigma error of estimated values.
@arg vmin: (structured array) minimum value of the parameter.
@arg vmax: (structured array) maximum value of the parameter.
"""
imin = data['_rfac'].argmin()
rmin = data['_rfac'][imin]
rvar = rmin * math.sqrt(2. / float(features))
val = np.zeros(1, dtype=data.dtype)
sig = np.zeros(1, dtype=data.dtype)
vmin = np.zeros(1, dtype=data.dtype)
vmax = np.zeros(1, dtype=data.dtype)
sel = data['_rfac'] <= rmin + rvar
for name in data.dtype.names:
val[name] = data[name][imin]
vmin[name] = data[name].min()
vmax[name] = data[name].max()
if name[0] != '_':
sig[name] = (data[name][sel].max() - data[name][sel].min()) / 2.
results = {'rmin': rmin, 'rvar': rvar, 'imin': imin, 'val': val, 'sig': sig, 'vmin': vmin, 'vmax': vmax}
return results
def render_results(results_file, data=None):
"""
produce a graphics file from optimization results.
the results can be passed in a file name or numpy array (see parameter section).
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function will log a warning message and return gracefully.
@param results_file: path and name of the result file.
result files are the ones written by swarm.SwarmPopulation.save_array, for instance.
the file contains columns of model parameters and optimization control values.
the first row must contain column names that identify the quantity.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
if the optional data parameter is present,
this is used only to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
if this argument is omitted, the data is loaded from the file referenced by the filename argument.
@return (list of str) path names of the generated graphics files.
empty if an error occurred.
the most common exceptions are caught and add a warning in the log file.
"""
if data is None:
data = np.atleast_1d(np.genfromtxt(results_file, names=True))
summary = evaluate_results(data)
out_files = []
try:
for name in data.dtype.names:
if name[0] != '_' and summary['sig'][name] > 0.:
graph_file = render_param_rfac(results_file, data, name, summary)
out_files.append(graph_file)
except (TypeError, AttributeError, IOError) as e:
logger.warning(BMsg("error rendering scan file {file}: {msg}", file=results_file, msg=str(e)))
return out_files

288
pmsco/graphics/scan.py Normal file
View File

@@ -0,0 +1,288 @@
"""
@package pmsco.graphics.scan
graphics rendering module for energy and angle scans.
this module is experimental.
interface and implementation are subject to change.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import logging
import math
import numpy as np
import pmsco.data as md
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# from matplotlib.backends.backend_pdf import FigureCanvasPdf
# from matplotlib.backends.backend_svg import FigureCanvasSVG
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False, ref_data=None):
"""
produce a graphics file from a one-dimensional scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of EI, ETPI or ETPAI data.
@param scan_mode: list containing the field name of the scanning axis of the data array.
it must contain one element exactly.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to set axis labels.
@param ref_data: numpy-structured array of EI, ETPI or ETPAI data.
this is reference data (e.g. experimental data) that should be plotted with the main dataset.
both datasets will be plotted on the same axis and should have similar data range.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
if ref_data is not None:
ax.plot(ref_data[scan_mode[0]], ref_data['i'], 'k.')
ax.plot(data[scan_mode[0]], data['i'])
ax.set_xlabel(scan_mode[0])
if is_modf:
ax.set_ylabel('chi')
else:
ax.set_ylabel('int')
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_ea_scan(filename, data, scan_mode, canvas=None, is_modf=False):
"""
produce a graphics file from an energy-angle scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of ETPI or ETPAI data.
@param scan_mode: list containing the field names of the scanning axes of the data array,
i.e. 'e' and one of the angle axes.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to select a suitable color scale.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
(data2d, axis0, axis1) = md.reshape_2d(data, scan_mode, 'i')
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
im = ax.imshow(data2d, origin='lower', aspect='auto', interpolation='none')
im.set_extent((axis1[0], axis1[-1], axis0[0], axis0[-1]))
ax.set_xlabel(scan_mode[1])
ax.set_ylabel(scan_mode[0])
cb = fig.colorbar(im, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(data['i'], 1)
dhi = np.nanpercentile(data['i'], 99)
if is_modf:
im.set_cmap("RdBu_r")
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
im.set_clim((-1., 1.))
try:
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
else:
im.set_cmap("magma")
im.set_clim((dlo, dhi))
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_tp_scan(filename, data, canvas=None, is_modf=False):
"""
produce a graphics file from an theta-phi (hemisphere) scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of TPI data.
the T and P columns describes a full or partial hemispherical scan.
the I column contains the intensity or modulation values.
other columns are ignored.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to select a suitable color scale.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111, projection='polar')
data = data[data['t'] <= 89.0]
# stereographic projection
rd = 2 * np.tan(np.radians(data['t']) / 2)
drdt = 1 + np.tan(np.radians(data['t']) / 2)**2
# http://matplotlib.org/api/collections_api.html#matplotlib.collections.PathCollection
pc = ax.scatter(data['p'] * math.pi / 180., rd, c=data['i'], lw=0, alpha=1.)
# interpolate marker size between 4 and 9 (for theta step = 1)
unique_theta = np.unique(data['t'])
theta_step = (np.max(unique_theta) - np.min(unique_theta)) / unique_theta.shape[0]
sz = np.ones_like(pc.get_sizes()) * drdt * 4.5 * theta_step**2
pc.set_sizes(sz)
# xticks = angles where grid lines are displayed (in radians)
ax.set_xticks([])
# rticks = radii where grid lines (circles) are displayed
ax.set_rticks([])
ax.set_rmax(2.0)
cb = fig.colorbar(pc, shrink=0.4, pad=0.1)
clip = 2
dlo = np.nanpercentile(data['i'], clip)
dhi = np.nanpercentile(data['i'], 100 - clip)
if is_modf:
pc.set_cmap("RdBu_r")
# im.set_cmap("coolwarm")
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
pc.set_clim((dlo, dhi))
try:
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
else:
pc.set_cmap("magma")
# im.set_cmap("inferno")
# im.set_cmap("viridis")
pc.set_clim((dlo, dhi))
try:
ti = cb.get_ticks()
ti = [min(ti), max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scan(filename, data=None, ref_data=None):
"""
produce a graphics file from a scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function will log a warning message and return gracefully.
@param filename: path and name of the scan file.
the file must have one of the formats supported by pmsco.data.load_data().
it must contain a single scan (not the combined scan from the model level of PMSCO).
supported are all one-dimensional linear scans,
and two-dimensional energy-angle scans (each axis must be linear).
hemispherical scans are currently not supported.
the filename should include ".modf" if the data contains a modulation function rather than intensity.
if the optional data parameter is present,
this is used only to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of ETPI or ETPAI data.
if this argument is omitted, the data is loaded from the file referenced by the filename argument.
@param ref_data: numpy-structured array of ETPI or ETPAI data.
this is reference data (e.g. experimental data) that should be plotted with the main dataset.
this is supported for 1d scans only.
both datasets will be plotted on the same axis and should have similar data range.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
"""
if data is None:
data = md.load_data(filename)
scan_mode, scan_positions = md.detect_scan_mode(data)
is_modf = filename.find(".modf") >= 0
try:
if len(scan_mode) == 1:
out_filename = render_1d_scan(filename, data, scan_mode, is_modf=is_modf, ref_data=ref_data)
elif len(scan_mode) == 2 and 'e' in scan_mode:
out_filename = render_ea_scan(filename, data, scan_mode, is_modf=is_modf)
elif len(scan_mode) == 2 and 't' in scan_mode and 'p' in scan_mode:
out_filename = render_tp_scan(filename, data, is_modf=is_modf)
else:
out_filename = ""
logger.warning(BMsg("no render function for scan file {file}", file=filename))
except (TypeError, AttributeError, IOError) as e:
out_filename = ""
logger.warning(BMsg("error rendering scan file {file}: {msg}", file=filename, msg=str(e)))
return out_filename

210
pmsco/graphics/scattering.py Normal file
View File

@@ -0,0 +1,210 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
import scipy.interpolate
import scipy.special
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
class TMatrix(object):
def __init__(self):
"""
self.en.shape = (n_e,)
self.tl.shape = (n_e, n_l)
"""
self.en = None
self.tl = None
def load_test_data(self):
self.en = np.array([100.])
raw = [-0.052845, -0.003238, 0.478705, 0.672581, 0.137932, 0.981700, 0.323890, 0.805299, 0.291814, 0.776792,
0.369416, 0.351845, 0.199775, 0.113314, 0.062479, 0.025691, 0.013699, 0.005283]
re_tl = np.array(raw[0::2])
im_tl = np.array(raw[1::2])
self.tl = re_tl + 1j * im_tl
def load_edac_scattering(self, f, energy=math.nan):
"""
load T matrix from EDAC scattering file
currently, only the 'tl' format is supported.
@param f: file path
@param energy: kinetic energy in eV if none is defined in the file
@return: None
"""
with open(f, "r") as fi:
h = fi.readline().rstrip().split(' ')
ne = int(h[0])
if ne > 1:
assert h[1] == 'E(eV)'
del h[1]
lmax = int(h[1])
assert h[2] == 'regular'
assert h[3] == 'tl'
self.load_edac_tl(f, ne, lmax, energy=energy)
def load_edac_tl(self, f, ne, lmax, energy=math.nan):
"""
load T matrix from EDAC scattering file in 'tl' format
@param f: file path
@param ne: number of energies (rows)
@param lmax: maximum l number (columns = 2 * (lmax + 1))
@param energy: kinetic energy in eV if none is defined in the file
@return: None
"""
if ne > 1:
self.en = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=[0]))
start_col = 1
else:
self.en = np.asarray(energy)
start_col = 0
re_cols = range(start_col, start_col + (lmax + 1) * 2, 2)
im_cols = range(start_col + 1, start_col + (lmax + 1) * 2, 2)
re_tl = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=re_cols))
im_tl = np.atleast_1d(np.genfromtxt(f, skip_header=1, usecols=im_cols))
self.tl = re_tl + 1j * im_tl
assert self.tl.shape == (ne, lmax + 1), "array shape mismatch"
def planewave_amplitude(self, energy, angle):
"""
total, complex plane wave scattering amplitude for given energy and angle
@param energy: kinetic energy in eV.
this can be a numeric value, a 1-dimensional numpy.ndarray,
or any value accepted by the numpy.asarray function.
@param angle: scattering angle in degrees (0..180).
this can be a numeric value, a 1-dimensional numpy.ndarray,
or any value accepted by the numpy.asarray function.
@return: 3 numpy arrays (amp, magnitude, phase) representing the scattering amplitude
versus energy and angle.
the shape of the three arrays is (n_energies, n_angles).
@arg amp: complex scattering amplitude.
@arg magnitude: magnitude (absolute value) of the scattering amplitude.
@arg phase: phase angle in radians of the scattering amplitude.
"""
if not isinstance(energy, np.ndarray):
energy = np.atleast_1d(np.asarray(energy))
ne = len(energy)
if not isinstance(angle, np.ndarray):
angle = np.atleast_1d(np.array(angle))
na = len(angle)
kinv = 1. / (0.513019932 * np.sqrt(energy))
f_tl = scipy.interpolate.interp1d(self.en, self.tl, axis=0, copy=False)
tl = f_tl(energy)
cos_angle = np.cos(np.radians(angle))
lmax = self.tl.shape[1] - 1
l = np.arange(0, lmax + 1)
amp = np.zeros((ne, na), dtype=complex)
for ia, ca in enumerate(cos_angle):
lpmn, __ = scipy.special.lpmn(0, lmax, ca)
fpart = np.outer(kinv, (2 * l + 1) * lpmn[0]) * tl
ftot = np.sum(fpart, axis=-1)
amp[:, ia] = ftot
mag = np.abs(amp)
pha = np.angle(amp)
return amp, mag, pha
def render_scattering_1d(filename, tmatrix, energy=None):
if energy is None:
en = tmatrix.en[0]
else:
en = energy
an = np.arange(0, 181, 2)
__, mag, pha = tmatrix.planewave_amplitude(en, an)
pha = pha / math.pi
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(211)
ax.plot(an, mag[0])
ax.set_xlabel('th (deg)')
ax.set_ylabel('mag (arb)')
ax = fig.add_subplot(212)
ax.plot(an, pha[0])
ax.set_xlabel('th (deg)')
ax.set_ylabel('pha (1/pi)')
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scattering_2d(filename, tmatrix):
en = tmatrix.en
an = np.arange(0, 181, 2)
__, mag, pha = tmatrix.planewave_amplitude(en, an)
pha = pha / math.pi
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(211)
im = ax.imshow(mag, origin='lower', aspect='auto', interpolation='none')
im.set_extent((an[0], an[-1], en[0], en[-1]))
im.set_cmap("magma")
ax.set_xlabel('th (deg)')
ax.set_ylabel('E (eV)')
# cb = ax.colorbar(im, shrink=0.4, pad=0.1)
# ti = cb.get_ticks()
# ti = [0., max(ti)]
# cb.set_ticks(ti)
ax = fig.add_subplot(212)
im = ax.imshow(pha, origin='lower', aspect='auto', interpolation='none')
im.set_extent((an[0], an[-1], en[0], en[-1]))
im.set_cmap("RdBu_r")
ax.set_xlabel('th (deg)')
ax.set_ylabel('E (eV)')
# cb = ax.colorbar(im, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(mag, 2)
dhi = np.nanpercentile(mag, 98)
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
im.set_clim((dlo, dhi))
# ti = cb.get_ticks()
# ti = [min(ti), 0., max(ti)]
# cb.set_ticks(ti)
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scattering_map(filename, energy):
tmatrix = TMatrix()
tmatrix.load_edac_scattering(filename, energy)
if tmatrix.tl.shape[0] == 1:
out_filename = render_scattering_1d(filename, tmatrix)
else:
out_filename = render_scattering_2d(filename, tmatrix)
return out_filename

241
pmsco/handlers.py
View File

@@ -1,6 +1,6 @@
"""
@package pmsco.handlers
project-independent task handlers for models, scans, symmetries, emitters and energies.
project-independent task handlers for models, scans, domains, emitters and energies.
calculation tasks are organized in a hierarchical tree.
at each node, a task handler (feel free to find a better name)
@@ -20,9 +20,9 @@ the handlers of the structural optimizers are declared in separate modules.
scans are defined by the project.
the actual merging step from multiple scans into one result dataset is delegated to the project class.
<em>symmetry handlers</em> split a task into one child per symmetry.
symmetries are defined by the project.
the actual merging step from multiple symmetries into one result dataset is delegated to the project class.
<em>domain handlers</em> split a task into one child per domain.
domains are defined by the project.
the actual merging step from multiple domains into one result dataset is delegated to the project class.
<em>emitter handlers</em> split a task into one child per emitter configuration (inequivalent sets of emitting atoms).
emitter configurations are defined by the project.
@@ -35,26 +35,30 @@ code inspection and tests have shown that per-emitter results from EDAC can be s
in order to take advantage of parallel processing.
while several classes of model handlers are available,
the default handlers for scans, symmetries, emitters and energies should be sufficient in most situations.
the scan and symmetry handlers call methods of the project class to invoke project-specific functionality.
the default handlers for scans, domains, emitters and energies should be sufficient in most situations.
the scan and domain handlers call methods of the project class to invoke project-specific functionality.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-17 by Paul Scherrer Institut @n
@copyright (c) 2015-21 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import division
import datetime
import os
from functools import reduce
import logging
import math
import numpy as np
import data as md
from helpers import BraceMessage as BMsg
import os
from pathlib import Path
import pmsco.data as md
import pmsco.dispatch as dispatch
import pmsco.graphics.scan as mgs
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@@ -66,10 +70,10 @@ class TaskHandler(object):
this class defines the common interface of task handlers.
"""
## @var project
## @var _project
# (Project) project instance.
## @var slots
## @var _slots
# (int) number of calculation slots (processes).
#
# for best efficiency the number of tasks generated should be greater or equal the number of slots.
@@ -93,7 +97,7 @@ class TaskHandler(object):
# the dictionary keys are the task identifiers CalculationTask.id,
# the values are the corresponding CalculationTask objects.
## @var invalid_count (int)
## @var _invalid_count (int)
# accumulated total number of invalid results received.
#
# the number is incremented by add_result if an invalid task is reported.
@@ -120,10 +124,14 @@ class TaskHandler(object):
for best efficiency the number of tasks generated should be greater or equal the number of slots.
it should not exceed N times the number of slots, where N is a reasonably small number.
@return None
@return (int) number of children that create_tasks() will generate on average.
the number does not need to be accurate, a rough estimate or order of magnitude if greater than 10 is fine.
it is used to distribute processing slots across task levels.
see pmsco.dispatch.MscoMaster.setup().
"""
self._project = project
self._slots = slots
return 1
def cleanup(self):
"""
@@ -188,21 +196,22 @@ class TaskHandler(object):
the id, model, and files attributes are required.
if model contains a '_rfac' value, the r-factor is
@return: None
@return None
"""
model_id = task.id.model
for path, cat in task.files.iteritems():
for path, cat in task.files.items():
self._project.files.add_file(path, model_id, category=cat)
def cleanup_files(self, keep=10):
def cleanup_files(self, keep=0):
"""
delete uninteresting files.
@param: number of best ranking models to keep.
@param keep: minimum number of models to keep.
0 (default): leave the decision to the project.
@return: None
@return None
"""
self._project.files.delete_files(keep_rfac=keep)
self._project.cleanup_files(keep=keep)
class ModelHandler(TaskHandler):
@@ -255,6 +264,22 @@ class ModelHandler(TaskHandler):
return None
def save_report(self, root_task):
"""
generate a final report of the optimization procedure.
detailed calculation results are usually saved as soon as they become available.
this method may be implemented in sub-classes to aggregate and summarize the results, generate plots, etc.
in this class, the method does nothing.
@note: implementations must add the path names of generated files to self._project.files.
@param root_task: (CalculationTask) task with initial model parameters.
@return: None
"""
pass
class SingleModelHandler(ModelHandler):
"""
@@ -263,6 +288,10 @@ class SingleModelHandler(ModelHandler):
this class runs a single calculation on the start parameters defined in the domain of the project.
"""
def __init__(self):
super(SingleModelHandler, self).__init__()
self.result = {}
def create_tasks(self, parent_task):
"""
start one task with the start parameters.
@@ -316,25 +345,17 @@ class SingleModelHandler(ModelHandler):
modf_ext = ".modf" + parent_task.file_ext
parent_task.modf_filename = parent_task.file_root + modf_ext
rfac = 1.0
if task.result_valid:
try:
rfac = self._project.calc_rfactor(task)
except ValueError:
task.result_valid = False
logger.warning(BMsg("calculation of model {0} resulted in an undefined R-factor.", task.id.model))
self.result = task.model.copy()
self.result['_rfac'] = task.rfac
task.model['_rfac'] = rfac
self.save_report_file(task.model)
self._project.files.update_model_rfac(task.id.model, rfac)
self._project.files.update_model_rfac(task.id.model, task.rfac)
self._project.files.set_model_complete(task.id.model, True)
parent_task.time = task.time
return parent_task
def save_report_file(self, result):
def save_report(self, root_task):
"""
save model parameters and r-factor to a file.
@@ -343,20 +364,25 @@ class SingleModelHandler(ModelHandler):
the first line contains the parameter names.
this is the same format as used by the swarm and grid handlers.
@param result: dictionary of results and parameters. the values should be scalars and strings.
@param root_task: (CalculationTask) the id.model attribute is used to register the generated files.
@return: None
"""
keys = [key for key in result]
super(SingleModelHandler, self).save_report(root_task)
keys = [key for key in self.result]
keys.sort(key=lambda t: t[0].lower())
vals = (str(result[key]) for key in keys)
with open(self._project.output_file + ".dat", "w") as outfile:
vals = (str(self.result[key]) for key in keys)
filename = Path(self._project.output_file).with_suffix(".dat")
with open(filename, "wt", encoding="latin1") as outfile:
outfile.write("# ")
outfile.write(" ".join(keys))
outfile.write("\n")
outfile.write(" ".join(vals))
outfile.write("\n")
self._project.files.add_file(filename, root_task.id.model, "report")
class ScanHandler(TaskHandler):
"""
@@ -388,6 +414,34 @@ class ScanHandler(TaskHandler):
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the scan task handler and save processed experimental scans.
@return (int) number of scans defined in the project.
"""
super(ScanHandler, self).setup(project, slots)
for (i_scan, scan) in enumerate(self._project.scans):
if scan.modulation is not None:
__, filename = os.path.split(scan.filename)
pre, ext = os.path.splitext(filename)
filename = "{pre}_{scan}.modf{ext}".format(pre=pre, ext=ext, scan=i_scan)
filepath = os.path.join(self._project.output_dir, filename)
md.save_data(filepath, scan.modulation)
mgs.render_scan(filepath, data=scan.modulation)
if project.combined_scan is not None:
ext = md.format_extension(project.combined_scan)
filename = Path(project.output_file).with_suffix(ext)
md.save_data(filename, project.combined_scan)
if project.combined_modf is not None:
ext = md.format_extension(project.combined_modf)
filename = Path(project.output_file).with_suffix(".modf" + ext)
md.save_data(filename, project.combined_modf)
return len(self._project.scans)
def create_tasks(self, parent_task):
"""
generate a calculation task for each scan of the given parent task.
@@ -464,6 +518,7 @@ class ScanHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_scans(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'model')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'model')
@@ -476,7 +531,7 @@ class ScanHandler(TaskHandler):
return None
class SymmetryHandler(TaskHandler):
class DomainHandler(TaskHandler):
## @var _pending_ids_per_parent
# (dict) sets of child task IDs per parent
#
@@ -496,20 +551,29 @@ class SymmetryHandler(TaskHandler):
# the values are sets of all child CalculationTask.id belonging to the parent.
def __init__(self):
super(SymmetryHandler, self).__init__()
super(DomainHandler, self).__init__()
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the domain task handler.
@return (int) number of domains defined in the project.
"""
super(DomainHandler, self).setup(project, slots)
return len(self._project.domains)
def create_tasks(self, parent_task):
"""
generate a calculation task for each symmetry of the given parent task.
generate a calculation task for each domain of the given parent task.
all symmetries share the same model parameters.
all domains share the same model parameters.
@return list of CalculationTask objects, with one element per symmetry.
the symmetry index varies according to project.symmetries.
@return list of CalculationTask objects, with one element per domain.
the domain index varies according to project.domains.
"""
super(SymmetryHandler, self).create_tasks(parent_task)
super(DomainHandler, self).create_tasks(parent_task)
parent_id = parent_task.id
self._parent_tasks[parent_id] = parent_task
@@ -517,10 +581,10 @@ class SymmetryHandler(TaskHandler):
self._complete_ids_per_parent[parent_id] = set()
out_tasks = []
for (i_sym, sym) in enumerate(self._project.symmetries):
for (i_dom, domain) in enumerate(self._project.domains):
new_task = parent_task.copy()
new_task.parent_id = parent_id
new_task.change_id(sym=i_sym)
new_task.change_id(domain=i_dom)
child_id = new_task.id
self._pending_tasks[child_id] = new_task
@@ -529,25 +593,25 @@ class SymmetryHandler(TaskHandler):
out_tasks.append(new_task)
if not out_tasks:
logger.error("no symmetry tasks generated. your project must declare at least one symmetry.")
logger.error("no domain tasks generated. your project must declare at least one domain.")
return out_tasks
def add_result(self, task):
"""
collect and combine the calculation results versus symmetry.
collect and combine the calculation results versus domain.
* mark the task as complete
* store its result for later
* check whether this was the last pending task of the family (belonging to the same parent).
the actual merging of data is delegated to the project's combine_symmetries() method.
the actual merging of data is delegated to the project's combine_domains() method.
@param task: (CalculationTask) calculation task that completed.
@return parent task (CalculationTask) if the family is complete. None if the family is not complete yet.
"""
super(SymmetryHandler, self).add_result(task)
super(DomainHandler, self).add_result(task)
self._complete_tasks[task.id] = task
del self._pending_tasks[task.id]
@@ -557,7 +621,7 @@ class SymmetryHandler(TaskHandler):
family_pending.remove(task.id)
family_complete.add(task.id)
# all symmetries complete?
# all domains complete?
if len(family_pending) == 0:
parent_task = self._parent_tasks[task.parent_id]
@@ -574,9 +638,13 @@ class SymmetryHandler(TaskHandler):
parent_task.time = reduce(lambda a, b: a + b, child_times)
if parent_task.result_valid:
self._project.combine_symmetries(parent_task, child_tasks)
self._project.combine_domains(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'scan')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'scan')
graph_file = mgs.render_scan(parent_task.modf_filename,
ref_data=self._project.scans[parent_task.id.scan].modulation)
self._project.files.add_file(graph_file, parent_task.id.model, 'scan')
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@@ -615,13 +683,26 @@ class EmitterHandler(TaskHandler):
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the emitter task handler.
@return (int) estimated number of emitter configurations that the cluster generator will generate.
the estimate is based on the start parameters, scan 0 and domain 0.
"""
super(EmitterHandler, self).setup(project, slots)
mock_model = self._project.model_space.start
mock_index = dispatch.CalcID(-1, 0, 0, -1, -1)
n_emitters = project.cluster_generator.count_emitters(mock_model, mock_index)
return n_emitters
def create_tasks(self, parent_task):
"""
generate a calculation task for each emitter configuration of the given parent task.
all emitters share the same model parameters.
@return list of @ref CalculationTask objects with one element per emitter configuration
@return list of @ref pmsco.dispatch.CalculationTask objects with one element per emitter configuration
if parallel processing is enabled.
otherwise the list contains a single CalculationTask object with emitter index 0.
the emitter index is used by the project's create_cluster method.
@@ -634,10 +715,7 @@ class EmitterHandler(TaskHandler):
self._complete_ids_per_parent[parent_id] = set()
n_emitters = self._project.cluster_generator.count_emitters(parent_task.model, parent_task.id)
if n_emitters > 1 and self._slots > 1:
emitters = range(1, n_emitters + 1)
else:
emitters = [0]
emitters = range(n_emitters)
out_tasks = []
for em in emitters:
@@ -698,8 +776,12 @@ class EmitterHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_emitters(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'symmetry')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'symmetry')
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'domain')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'domain')
graph_file = mgs.render_scan(parent_task.modf_filename,
ref_data=self._project.scans[parent_task.id.scan].modulation)
self._project.files.add_file(graph_file, parent_task.id.model, 'domain')
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@@ -776,15 +858,10 @@ class RegionHandler(TaskHandler):
parent_task.time = reduce(lambda a, b: a + b, child_times)
if parent_task.result_valid:
stack1 = [md.load_data(t.result_filename) for t in child_tasks]
dtype = md.common_dtype(stack1)
stack2 = [md.restructure_data(d, dtype) for d in stack1]
result_data = np.hstack(tuple(stack2))
md.sort_data(result_data)
md.save_data(parent_task.result_filename, result_data)
self._project.combine_regions(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, "emitter")
for t in child_tasks:
self._project.files.remove_file(t.result_filename)
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, "emitter")
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@@ -840,7 +917,7 @@ class EnergyRegionHandler(RegionHandler):
so that all child tasks of the same parent finish approximately in the same time.
pure angle scans are not split.
to use this feature, the project assigns this class to its @ref handler_classes['region'].
to use this feature, the project assigns this class to its @ref pmsco.project.Project.handler_classes['region'].
it is safe to use this handler for calculations that do not involve energy scans.
the handler is best used for single calculations.
in optimizations that calculate many models there is no advantage in using it
@@ -871,7 +948,7 @@ class EnergyRegionHandler(RegionHandler):
@param slots (int) number of calculation slots (processes).
@return None
@return (int) average number of child tasks
"""
super(EnergyRegionHandler, self).setup(project, slots)
@@ -884,6 +961,8 @@ class EnergyRegionHandler(RegionHandler):
logger.debug(BMsg("region handler: split scan {file} into {slots} chunks",
file=os.path.basename(scan.filename), slots=self._slots_per_scan[i]))
return max(int(sum(self._slots_per_scan) / len(self._slots_per_scan)), 1)
def create_tasks(self, parent_task):
"""
generate a calculation task for each energy region of the given parent task.
@@ -922,27 +1001,3 @@ class EnergyRegionHandler(RegionHandler):
logger.error("no region tasks generated. this is probably a bug.")
return out_tasks
def choose_region_handler_class(project):
"""
choose a suitable region handler for the project.
the function returns the EnergyRegionHandler class
if the project includes an energy scan with at least 10 steps.
Otherwise, it returns the SingleRegionHandler.
angle scans do not benefit from region splitting in EDAC.
@param project: Project instance.
@return: SingleRegionHandler or EnergyRegionHandler class.
"""
energy_scans = 0
for scan in project.scans:
if scan.energies.shape[0] >= 10:
energy_scans += 1
if energy_scans >= 1:
return EnergyRegionHandler
else:
return SingleRegionHandler

62
pmsco/helpers.py
View File

@@ -1,4 +1,27 @@
class BraceMessage:
"""
@package pmsco.helpers
helper classes
a collection of small and generic code bits mostly collected from the www.
"""
import contextlib
import ctypes
import io
import os
import sys
from typing import BinaryIO
class BraceMessage(object):
"""
a string formatting proxy class useful for logging and exceptions.
use BraceMessage("{0} {1}", "formatted", "message")
in place of "{0} {1}".format("formatted", "message").
the advantage is that the format method is called only if the string is actually used.
"""
def __init__(self, fmt, *args, **kwargs):
self.fmt = fmt
self.args = args
@@ -6,3 +29,40 @@ class BraceMessage:
def __str__(self):
return self.fmt.format(*self.args, **self.kwargs)
libc = ctypes.CDLL(None)
c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout')
@contextlib.contextmanager
def stdout_redirected(dest_file: BinaryIO):
"""
A context manager to temporarily redirect stdout to a file.
Redirects all standard output from Python and the C library to the specified file.
This can be used, e.g., to capture output from Fortran code.
credit: https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/
@param dest_file: binary file open for writing ('wb' mode).
This function requires just the fileno function.
@return: None
"""
original_stdout_fd = sys.stdout.fileno()
def _redirect_stdout(to_fd):
"""Redirect stdout to the given file descriptor."""
libc.fflush(c_stdout)
sys.stdout.close()
os.dup2(to_fd, original_stdout_fd)
sys.stdout = io.TextIOWrapper(os.fdopen(original_stdout_fd, 'wb'))
saved_stdout_fd = os.dup(original_stdout_fd)
try:
_redirect_stdout(dest_file.fileno())
yield
_redirect_stdout(saved_stdout_fd)
finally:
os.close(saved_stdout_fd)

137
pmsco/igor.py Normal file
View File

@@ -0,0 +1,137 @@
"""
@package pmsco.igor
data exchange with wavemetrics igor pro.
this module provides functions for loading/saving pmsco data in igor pro.
@author Matthias Muntwiler
@copyright (c) 2019-23 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import numpy as np
def _escape_igor_string(s):
s = s.replace('\\', '\\\\')
s = s.replace('"', '\\"')
return s
def namefix_double(name):
"""
fix 1-character wave name by doubling
replaces length-1 string by a doubled version.
@param name: (str) proposed wave name
@return: corrected name
"""
return name*2 if len(name) == 1 else name
def namefix_etpais(name):
"""
fix 1-character wave name according to ETPAIS scheme
replaces 'e' by 'en' etc.
@param name: (str) proposed wave name
@return: corrected name
"""
name_map = {'e': 'en', 't': 'th', 'p': 'ph', 'i': 'in', 'm': 'mo', 's': 'si'}
try:
return name_map[name]
except KeyError:
return name
class IgorExport(object):
"""
class exports pmsco data to an Igor text (ITX) file.
usage:
1) create an object instance.
2) set @ref data.
3) set optional attributes: @ref prefix and @ref namefix.
4) call @ref export.
"""
def __init__(self):
super(IgorExport, self).__init__()
self.data = None
self.prefix = ""
self.namefix = namefix_double
def set_data(self, data):
"""
set the data array to export.
this must (currently) be a one-dimensional structured array.
the column names will become wave names.
@param data: numpy.ndarray
@return:
"""
self.data = data
def export(self, filename):
"""
write to igor file.
"""
with open(filename, 'wt', encoding="utf8") as f:
self._write_header(f)
self._write_data(f)
def _fix_name(self, name):
"""
fix a wave name.
this function first applies @ref namefix and @ref prefix to the proposed wave name.
@param name: (str) proposed wave name
@return: corrected name
"""
if self.namefix is not None:
name = self.namefix(name)
return self.prefix + name
def _write_header(self, f):
"""
write the header of the igor text file
@param f: open file or stream
@return: None
"""
f.write('IGOR' + '\n')
f.write('X // pmsco data export\n')
def _write_data(self, f):
"""
write a data section to the igor text file.
@param f: open file or stream
@return: None
"""
assert isinstance(self.data, np.ndarray)
assert len(self.data.shape) == 1
assert len(self.data.dtype.names[0]) >= 1
arr = self.data
shape = ",".join(map(str, arr.shape))
names = (self._fix_name(name) for name in arr.dtype.names)
names = ", ".join(names)
f.write('Waves/O/D/N=({shape}) {names}\n'.format(shape=shape, names=names))
f.write('BEGIN\n')
np.savetxt(f, arr, fmt='%g')
f.write('END\n')

1
pmsco/loess/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'matthias muntwiler'

77
pmsco/loess/makefile
View File

@@ -1,77 +0,0 @@
SHELL=/bin/sh
# makefile for the LOESS module
#
# required libraries: libblas, liblapack, libf2c
# (you may have to set soft links so that linker finds them)
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .x
.PHONY: all loess test gas madeup ethanol air galaxy
HOST=$(shell hostname)
CFLAGS=-O
FFLAGS=-O
OBJ=loessc.o loess.o predict.o misc.o loessf.o dqrsl.o dsvdc.o fix_main.o
LIB=-lblas -lm -lf2c
LIBPATH=
CC=gcc
CCOPTS=
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHONOPTS=
ifneq (,$(filter merlin%,$(HOST)))
PYTHONINC=-I/usr/include/python2.7 -I/opt/python/python-2.7.5/include/python2.7/
else ifneq (,$(filter ra%,$(HOST)))
PYTHONINC=-I${PSI_PYTHON27_INCLUDE_DIR}/python2.7 -I${PSI_PYTHON27_LIBRARY_DIR}/python2.7/site-packages/numpy/core/include
else
PYTHONINC=-I/usr/include/python2.7
endif
all: loess
loess: _loess.so
loess_wrap.c: loess.c loess.i
$(SWIG) $(SWIGOPTS) -python loess.i
loess.py _loess.so: loess_wrap.c
# setuptools doesn't handle the fortran files correctly
# $(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
$(CC) $(CFLAGS) -fpic -c loessc.c loess.c predict.c misc.c loessf.f dqrsl.f dsvdc.f fix_main.c
$(CC) $(CFLAGS) -fpic -c loess_wrap.c $(PYTHONINC)
$(CC) -shared $(OBJ) $(LIB) $(LIBPATH) loess_wrap.o -o _loess.so
examples: gas madeup ethanol air galaxy
gas: gas.x
gas.x: gas.o $(OBJ)
$(CC) -o gas.x gas.o $(OBJ) $(LIB)
madeup: madeup.x
madeup.x: madeup.o $(OBJ)
$(CC) -o madeup.x madeup.o $(OBJ) $(LIB)
ethanol: ethanol.x
ethanol.x: ethanol.o $(OBJ)
$(CC) -o ethanol.x ethanol.o $(OBJ) $(LIB)
air: air.x
air.x: air.o $(OBJ)
$(CC) -o air.x air.o $(OBJ) $(LIB)
galaxy: galaxy.x
galaxy.x: galaxy.o $(OBJ)
$(CC) -o galaxy.x galaxy.o $(OBJ) $(LIB)
clean:
rm -f *.o *.so *.x core *.pyc
rm -f loess.py loess_wrap.c

55
pmsco/loess/setup.py
View File

@@ -1,55 +0,0 @@
#!/usr/bin/env python
__author__ = 'Matthias Muntwiler'
"""
@package loess.setup
setup.py file for LOESS
the LOESS code included here was developed at Bell Labs by
William S. Cleveland, Eric Grosse, Ming-Jen Shyu,
and is dated 18 August 1992.
the code is available in the public domain
from http://www.netlib.org/a/dloess.
see the README file for details.
the Python wrapper was set up by M. Muntwiler
with the help of the SWIG toolkit
and other incredible goodies available in the Linux world.
@bug this file is currently not used because
distutils does not compile the included Fortran files.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from distutils.core import setup, Extension
from distutils import sysconfig
import numpy
try:
numpy_include = numpy.get_include()
except AttributeError:
numpy_include = numpy.get_numpy_include()
loess_module = Extension('_loess',
sources=['loess.i', 'loess_wrap.c', 'loess.c', 'loessc.c', 'predict.c', 'misc.c', 'loessf.f',
'dqrsl.f', 'dsvdc.f'],
include_dirs = [numpy_include],
libraries=['blas', 'm', 'f2c'],
)
setup(name='loess',
version='0.1',
author=__author__,
author_email='matthias.muntwiler@psi.ch',
description="""LOESS module in Python""",
ext_modules=[loess_module],
py_modules=["loess"], requires=['numpy']
)

33
pmsco/makefile
View File

@@ -1,33 +0,0 @@
SHELL=/bin/sh
# makefile for EDAC, MSC, and MUFPOT programs and modules
#
# see the top-level makefile for additional information.
.PHONY: all clean edac loess msc mufpot
EDAC_DIR = edac
MSC_DIR = msc
MUFPOT_DIR = mufpot
LOESS_DIR = loess
all: edac loess
edac:
$(MAKE) -C $(EDAC_DIR)
loess:
$(MAKE) -C $(LOESS_DIR)
msc:
$(MAKE) -C $(MSC_DIR)
mufpot:
$(MAKE) -C $(MUFPOT_DIR)
clean:
$(MAKE) -C $(EDAC_DIR) clean
$(MAKE) -C $(LOESS_DIR) clean
$(MAKE) -C $(MSC_DIR) clean
$(MAKE) -C $(MUFPOT_DIR) clean
rm -f *.pyc

1
pmsco/msc/.gitignore vendored
View File

@@ -1 +0,0 @@
revision.f

1
pmsco/msc/__init__.py
View File

@@ -1 +0,0 @@
__author__ = 'muntwiler_m'

Some files were not shown because too many files have changed in this diff Show More