pearl-public/pmsco-public
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update public distribution

Browse Source 
based on internal repository c9a2ac8 2019-01-03 16:04:57 +0100
tagged rev-master-2.0.0
This commit is contained in:
muntwiler_m 2019-01-31 15:45:02 +01:00
parent bbd16d0f94
commit acea809e4e
92 changed files with 165828 additions and 143181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,6 +1,7 @@
work/*
debug/*
lib/*
dev/*
*.pyc
*.o
*.so
@ -13,3 +14,4 @@ lib/*
.eric5project/*
.ropeproject/*
.fuse*
.trash

4
README.md
View File

@ -27,7 +27,7 @@ Highlights
Installation
============
PMSCO is written in Python 2.7.
PMSCO is written in Python 3.6 and compatible with Python 2.7.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
@ -61,7 +61,7 @@ Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
Copyright
---------
Copyright 2015-2017 by [Paul Scherrer Institut](http://www.psi.ch)
Copyright 2015-2018 by [Paul Scherrer Institut](http://www.psi.ch)
Release Notes

2
bin/pmsco.ra.template
View File

@ -120,7 +120,7 @@ python -m compileall projects
cd "$PMSCO_WORK_DIR"
echo
PMSCO_CMD="python $PMSCO_PROJECT_FILE"
PMSCO_CMD="python $PMSCO_SOURCE_DIR/pmsco $PMSCO_PROJECT_FILE"
PMSCO_ARGS="$PMSCO_PROJECT_ARGS"
if [ -n "$PMSCO_SCAN_FILES" ]; then
PMSCO_ARGS="-s $PMSCO_SCAN_FILES $PMSCO_ARGS"

14
bin/qpmsco.ra.sh
View File

@ -3,9 +3,10 @@
# submission script for PMSCO calculations on the Ra cluster
if [ $# -lt 1 ]; then
echo "Usage: $0 [NOSUB] JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]"
echo "Usage: $0 [NOSUB] DESTDIR JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]"
echo ""
echo " NOSUB (optional): do not submit the script to the queue. default: submit."
echo " DESTDIR: destination directory. must exist. a sub-dir \$JOBNAME is created."
echo " JOBNAME (text): name of job. use only alphanumeric characters, no spaces."
echo " NODES (integer): number of computing nodes. (1 node = 24 or 32 processors)."
echo " do not specify more than 2."
@ -20,7 +21,7 @@ if [ $# -lt 1 ]; then
echo " MODE: PMSCO calculation mode (single|swarm|gradient|grid)."
echo " ARGS (optional): any number of further PMSCO or project arguments (except mode and time)."
echo ""
echo "the job script complete with the program code and input/output data is generated in ~/jobs/\$JOBNAME"
echo "the job script is written to \$DESTDIR/\$JOBNAME which is also the destination of calculation output."
exit 1
fi
@ -37,6 +38,9 @@ else
NOSUB="false"
fi
DEST_DIR="$1"
shift
PMSCO_JOBNAME=$1
shift
@ -73,11 +77,7 @@ PMSCO_LOGLEVEL=""
PMSCO_CODE=""
# set up working directory
cd ~
if [ ! -d "jobs" ]; then
mkdir jobs
fi
cd jobs
cd "$DEST_DIR"
if [ ! -d "$PMSCO_JOBNAME" ]; then
mkdir "$PMSCO_JOBNAME"
fi

23
docs/config.dox
View File

@ -38,13 +38,13 @@ PROJECT_NAME = "PEARL MSCO"
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER =
PROJECT_NUMBER = $(REVISION)
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
# quick idea about the purpose of the project. Keep the description short.
PROJECT_BRIEF = "PEARL multiple scattering calculations and optimizations"
PROJECT_BRIEF = "PEARL multiple scattering calculation and optimization"
# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
# in the documentation. The maximum height of the logo should not exceed 55
@ -228,7 +228,7 @@ TAB_SIZE = 4
# "Side Effects:". You can put \n's in the value part of an alias to insert
# newlines.
ALIASES =
ALIASES = "raise=@exception"
# This tag can be used to specify a number of word-keyword mappings (TCL only).
# A mapping has the form "name=value". For example adding "class=itcl::class"
@ -597,19 +597,19 @@ STRICT_PROTO_MATCHING = NO
# list. This list is created by putting \todo commands in the documentation.
# The default value is: YES.
GENERATE_TODOLIST = YES
GENERATE_TODOLIST = NO
# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
# list. This list is created by putting \test commands in the documentation.
# The default value is: YES.
GENERATE_TESTLIST = YES
GENERATE_TESTLIST = NO
# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
# list. This list is created by putting \bug commands in the documentation.
# The default value is: YES.
GENERATE_BUGLIST = YES
GENERATE_BUGLIST = NO
# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
# the deprecated list. This list is created by putting \deprecated commands in
@ -761,9 +761,12 @@ WARN_LOGFILE =
INPUT = \
src/introduction.dox \
src/concepts.dox \
src/concepts-tasks.dox \
src/concepts-emitter.dox \
src/installation.dox \
src/execution.dox \
src/commandline.dox \
src/optimizers.dox \
../pmsco \
../projects \
../tests
@ -859,7 +862,7 @@ EXAMPLE_RECURSIVE = NO
# that contain images that are to be included in the documentation (see the
# \image command).
IMAGE_PATH =
IMAGE_PATH = src/images
# The INPUT_FILTER tag can be used to specify a program that doxygen should
# invoke to filter for each input file. Doxygen will invoke the filter program
@ -876,7 +879,7 @@ IMAGE_PATH =
# code is scanned, but not when the output code is generated. If lines are added
# or removed, the anchors will not be placed correctly.
INPUT_FILTER = /usr/bin/doxypy
INPUT_FILTER =
# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
# basis. Doxygen will compare the file name with each pattern and apply the
@ -885,7 +888,7 @@ INPUT_FILTER = /usr/bin/doxypy
# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
# patterns match the file name, INPUT_FILTER is applied.
FILTER_PATTERNS =
FILTER_PATTERNS = *.py=/usr/bin/doxypy
# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
# INPUT_FILTER) will also be used to filter the input files that are used for
@ -2328,7 +2331,7 @@ DIAFILE_DIRS =
# generate a warning when it encounters a \startuml command in this case and
# will not generate output for the diagram.
PLANTUML_JAR_PATH =
PLANTUML_JAR_PATH = $(PLANTUML_JAR_PATH)
# When using plantuml, the specified paths are searched for files specified by
# the !include statement in a plantuml block.

13
docs/makefile
View File

@ -2,6 +2,11 @@ SHELL=/bin/sh
# makefile for PMSCO documentation
#
# requirements
#
# 1) doxygen
# 2) /usr/bin/doxypy
# 3) PLANTUML_JAR_PATH environment variable must point to plantUML jar.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .html
@ -11,6 +16,9 @@ DOX=doxygen
DOXOPTS=
LATEX_DIR=latex
REVISION=$(shell git describe --always --tags --dirty --long || echo "unknown, "`date +"%F %T %z"`)
export REVISION
all: docs
docs: doxygen pdf
@ -22,5 +30,6 @@ pdf: doxygen
-$(MAKE) -C $(LATEX_DIR)
clean:
-rm -rf latex/*
-rm -rf html/*
-rm -r latex/*
-rm -r html/*

56
docs/src/commandline.dox
View File

@ -11,14 +11,19 @@ it is recommended to adhere to the standard syntax described below.
The basic command line is as follows:
@code{.sh}
[mpiexec -np NPROCESSES] python path-to-project.py [common args] [project args]
[mpiexec -np NPROCESSES] python path/to/pmsco path/to/project.py [common args] [project args]
@endcode
Include the first portion between square brackets if you want to run parallel processes.
Specify the number of processes as the @c -np option.
@c path-to-project.py should be the path and name to your project module.
@c path/to/pmsco is the directory where <code>__main.py__</code> is located.
Do not include the extension <code>.py</code> or a trailing slash.
@c path/to/project.py should be the path and name to your project module.
Common args and project args are described below.
Note: In contrast to earlier versions, the project module is not executed directly any more.
Rather, it is loaded by the main pmsco module as a 'plug-in'.
\subsection sec_common_args Common Arguments
@ -30,7 +35,7 @@ The following table is ordered by importance.
| Option | Values | Description |
| --- | --- | --- |
| -h , --help | | Display a command line summary and exit. |
| -m , --mode | single (default), grid, swarm | Operation mode. |
| -m , --mode | single (default), grid, swarm, genetic | Operation mode. |
| -d, --data-dir | file system path | Directory path for experimental data files (if required by project). Default: current working directory. |
| -o, --output-file | file system path | Base path and/or name for intermediate and output files. Default: pmsco_data |
| -t, --time-limit | decimal number | Wall time limit in hours. The optimizers try to finish before the limit. Default: 24.0. |
@ -39,6 +44,8 @@ The following table is ordered by importance.
| --log-file | file system path | Name of the main log file. Under MPI, the rank of the process is inserted before the extension. Default: output-file + log, or pmsco.log. |
| --log-disable | | Disable logging. By default, logging is on. |
| --pop-size | integer | Population size (number of particles) in swarm optimization mode. The default value is the greater of 4 or two times the number of calculation processes. |
| --seed-file | file system path | Name of the population seed file. Population data of previous optimizations can be used to seed a new optimization. The file must have the same structure as the .pop or .dat files. See @ref pmsco.project.Project.seed_file. |
| --table-file | file system path | Name of the model table file in table scan mode. |
| -c, --code | edac (default) | Scattering code. At the moment, only edac is supported. |
@ -49,13 +56,14 @@ Multiple names can be specified and must be separated by spaces.
| Category | Description | Default Action |
| --- | --- | --- |
| all | shortcut to include all categories | |
| input | raw input files for calculator, including cluster and phase files in custom format | delete |
| output | raw output files from calculator | delete |
| phase | phase files in portable format for report | delete |
| cluster | cluster files in portable XYZ format for report | keep |
| debug | debug files | delete |
| model | output files in ETPAI format: complete simulation (a_-1_-1_-1_-1) | keep |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | delete |
| scan | output files in ETPAI format: scan (a_b_-1_-1_-1) | keep |
| symmetry | output files in ETPAI format: symmetry (a_b_c_-1_-1) | delete |
| emitter | output files in ETPAI format: emitter (a_b_c_d_-1) | delete |
| region | output files in ETPAI format: region (a_b_c_d_e) | delete |
@ -84,36 +92,11 @@ This way, the file names and photoelectron parameters are versioned with the cod
whereas command line arguments may easily get forgotten in the records.
\subsection sec_project_example Example Argument Handling
\subsection sec_project_example Argument Handling
An example for handling the command line in a project module can be found in the twoatom.py demo project.
The following code snippet shows how the common and project arguments are separated and handled.
@code{.py}
def main():
# have the pmsco module parse the common arguments.
args, unknown_args = pmsco.pmsco.parse_cli()
# pass any arguments not handled by pmsco
# to the project-defined parse_project_args function.
# unknown_args can be passed to argparse.ArgumentParser.parse_args().
if unknown_args:
project_args = parse_project_args(unknown_args)
else:
project_args = None
# create the project object
project = create_project()
# apply the common arguments on the project
pmsco.pmsco.set_common_args(project, args)
# apply the specific arguments on the project
set_project_args(project, project_args)
# run the project
pmsco.pmsco.run_project(project)
@endcode
To handle command line arguments in a project module,
the module must define a <code>parse_project_args</code> and a <code>set_project_args</code> function.
An example can be found in the twoatom.py demo project.
\section sec_slurm Slurm Job Submission
@ -122,23 +105,24 @@ The command line of the Slurm job submission script for the Ra cluster at PSI is
This script is specific to the configuration of the Ra cluster but may be adapted to other Slurm-based queues.
@code{.sh}
qpmsco.sh [NOSUB] JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]
qpmsco.sh [NOSUB] DESTDIR JOBNAME NODES TASKS_PER_NODE WALLTIME:HOURS PROJECT MODE [ARGS [ARGS [...]]]
@endcode
Here, the first few arguments are positional and their order must be strictly adhered to.
After the positional arguments, optional arguments of the PMSCO project command line can be added in arbitrary order.
If you execute the script without arguments, it displays a short summary.
The job script is written to @c ~/jobs/\$JOBNAME.
The job script is written to @c $DESTDIR/$JOBNAME which is also the destination of calculation output.
| Argument | Values | Description |
| --- | --- | --- |
| NOSUB (optional) | NOSUB or omitted | If NOSUB is present as the first argument, create the job script but do not submit it to the queue. Otherwise, submit the job script. |
| DESTDIR | file system path | destination directory. must exist. a sub-dir $JOBNAME is created. |
| JOBNAME | text | Name of job. Use only alphanumeric characters, no spaces. |
| NODES | integer | Number of computing nodes. (1 node = 24 or 32 processors). Do not specify more than 2. |
| TASKS_PER_NODE | 1...24, or 32 | Number of processes per node. 24 or 32 for full-node allocation. 1...23 for shared node allocation. |
| WALLTIME:HOURS | integer | Requested wall time. 1...24 for day partition, 24...192 for week partition, 1...192 for shared partition. This value is also passed on to PMSCO as the @c --time-limit argument. |
| PROJECT | file system path | Python module (file path) that declares the project and starts the calculation. |
| MODE | single, swarm, grid | PMSCO operation mode. This value is passed on to PMSCO as the @c --mode argument. |
| MODE | single, swarm, grid, genetic | PMSCO operation mode. This value is passed on to PMSCO as the @c --mode argument. |
| ARGS (optional) | | Any further arguments are passed on verbatim to PMSCO. You don't need to specify the mode and time limit here. |
*/

185
docs/src/concepts-emitter.dox Normal file
View File

@ -0,0 +1,185 @@
/*! @page pag_concepts_emitter Emitter configurations
\section sec_emitters Emitter configurations
\subsection sec_emit_intro Introduction
Since emitters contribute incoherently to the diffraction pattern,
it should make no difference how the emitters are grouped and calculated.
This fact can be used to distribute a calculation over multiple parallel processes
if each process calculates the diffraction pattern coming from one particular emitter atom.
In effect, some calculation codes are implemented for a single emitter per calculation.
With PMSCO, it is easy to distribute the emitters over parallel processes.
The project just declares the number of emitters and returns one specific cluster per emitter.
In the simplest case, this means that the emitter attribute of the cluster atoms is set differently,
while the atomic coordinates are the same for all clusters generated.
PMSCO takes care of dispatching the clusters to multiple calculation processes
depending on the number of allocated MPI processes
as well as summing up the resulting diffraction patterns.
In addition, the emitter framework also supports that clusters are tailored to a specific emitter configuration.
Suppose that the unit cell contains a large number of inequivalent emitters.
If all emitters had to be included in a single calculation,
the cluster would grow very large and the calculation would include many long scattering paths
that effectively did not contribute intensity to the final result.
Splitting a large cluster into small ones built locally around one emitter
can provide a significant performance gain in complex systems.
Note that the emitter framework does not require that an emitter _configuration_ contains only one emitter _atom_.
It is up to the project to define how many emitter configurations there are and what they encompass.
This should, however, normally not be necessary.
To avoid confusion, it is recommended to declare exactly one emitter atom per configuration.
\subsection sec_emit_implement Implementation
There are several implementation routes with varying complexity.
Which route to take can depend on the complexity of the system and/or the programming skills of the user.
The following class diagram illustrates the classes and packages involved in cluster generation.
@startuml "class diagram for cluster generation"
package pmsco {
class Project {
cluster_generator
export_cluster()
}
abstract class ClusterGenerator {
project
{abstract} count_emitters()
{abstract} create_cluster()
}
class LegacyClusterGenerator {
project
count_emitters()
create_cluster()
}
}
package "user project" {
class UserClusterGenerator {
project
count_emitters()
create_cluster()
}
note bottom : for complex cluster
class UserProject {
count_emitters()
create_cluster()
}
note bottom : for simple cluster
}
Project <|-- UserProject
ClusterGenerator <|-- LegacyClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
Project *-- ClusterGenerator
UserProject .> LegacyClusterGenerator
UserProject .> UserClusterGenerator
@enduml
In general, the cluster is generated by calls to the project's cluster_generator object.
This can be either a custom generator class derived from pmsco.cluster.ClusterGenerator
or the default pmsco.cluster.LegacyClusterGenerator which calls the UserProject.
For simple clusters, it may be sufficient to implement the cluster directly in the user project class
(UserProject in the diagram).
For more complex systems, it is recommended to implement a custom cluster generator class
(UserClusterGenerator).
\subsubsection sec_emit_implement_legacy Static cluster implemented in project methods
This is the most simple route as it requires the implementation of one or two methods of the user project class.
It can be used for single-emitter and multi-emitter problems.
This implementation is active while a pmsco.cluster.LegacyClusterGenerator
is assigned to the project's cluster_generator attribute.
1. Implement a count_emitters method in your project class
if the project uses more than one emitter configurations.
It must have same method contract as pmsco.cluster.ClusterGenerator.count_emitters.
Specifically, it must return the number of emitter configurations of a given model, scan and symmetry.
If there is only one configuration, the method does not need to be implemented.
2. Implement a create_cluster method in your project class.
It must have same method contract as pmsco.cluster.ClusterGenerator.create_cluster.
Specifically, it must return a cluster.Cluster object for the given model, scan, symmetry and emitter configuration.
The emitter atoms must be marked according to the emitter configuration specified by the index argument.
Note that, depending on the index.emit argument, all emitter atoms must be marked
or only the ones of the corresponding emitter configuration.
3. (Optionally) override the pmsco.project.Project.combine_emitters method
if the emitters should be added with non-uniform weights.
Although it's possible to produce emitter-dependent clusters using this approach,
this is usually not recommended.
Rather, the generator approach described below should be followed in this case.
\subsubsection sec_emit_implement_generator Static cluster implemented by generator class
The preferred way of creating clusters is to implement a _generator_ class
because it is the most scalable way from simple to complex systems.
In addition, one cluster generator class can be quickly exchanged for another
if there are multiple possibilities.
1. Implement a cluster generator class which inherits from pmsco.cluster.ClusterGenerator
in your project module.
2. Implement the create_cluster and count_emitters methods of the generator.
The method contracts are the same as the ones described in the previous paragraph,
just in the context of a separate class.
3. Initialize an instance of the generator and assign it to the project.cluster_generator attribute
in the initialization of your project.
\subsubsection sec_emit_implement_local Local clusters implemented by generator class
The basic method contract outlined in the previous paragraph is equally applicable to the case
where a local cluster is generated for each emitter configuration.
Again, the generator class with the two methods (count_emitters and create_cluster) is the minimum requirement.
However, for ease of code maintenance and/or for improved performance of large clusters,
some internal structure may be helpful.
Suppose that the system consists of a large supercell containing many emitters
and that a small cluster shall be built for each emitter configuration.
During the calculations, the generator will receive several calls to the count_emitters and create_cluster methods.
Every time the model and index are the same, the functions must return the same result.
Thus, most importantly, the implementation must make sure that the results are fully deterministic.
Second, depending on the complexity, it could be more efficient to cache a cluster for later use.
One way to reduce the complexity is to introduce a _master cluster_
from which the emitter configurations and individual clusters are derived.
1. Implement a master_cluster method with the same arguments and result types as create_cluster.
The method returns a full cluster of the supercell and its neighbouring cells.
All inequivalent emitters are marked (which determines the number of emitter configurations).
2. Decorate the master_cluster with pmsco.dispatch.CachedCalculationMethod.
This pre-defined decorator transparently caches the cluster
so that subsequent calls with the same arguments do not re-create the cluster but return the cached one.
3. The count_emitters method can simply return the emitter count of the master cluster.
4. The create_cluster method calls master_cluster() and extracts the region
corresponding to the requested emitter configuration.
\subsection sec_emit_report Reporting
The pmsco.project.Project class implements a method that saves a cluster to two XYZ files,
one containing the coordinates of all atoms
and one containing only the coordinates of the emitters.
The method is called for each cluster that is passed to the calculator, i.e., each emitter index.
You may override the method in your project to alter the reporting.
*/

3
docs/src/concepts-model.dox Normal file
View File

@ -0,0 +1,3 @@
/*! @page pag_concepts_model Model
*/

3
docs/src/concepts-region.dox Normal file
View File

@ -0,0 +1,3 @@
/*! @page pag_concepts_region Region
*/

31
docs/src/concepts-scan.dox Normal file
View File

@ -0,0 +1,31 @@
/*! @page pag_concepts_scan Scans
\section sec_scanning Scanning
PMSCO with EDAC currently supports the following scan axes.
- kinetic energy E
- polar angle theta T
- azimuthal angle phi P
- analyser angle alpha A
The following combinations of these scan axes are allowed (see pmsco.data.SCANTYPES).
- E
- E-T
- E-A
- T-P (hemispherical or hologram scan)
@attention The T and A axes cannot be combined.
If a scan of one of them is specified, the other is assumed to be fixed at zero!
This assumption may change in the future,
so it is best to explicitly set the fixed angle to zero in the scan file.
@remark According to the measurement geometry at PEARL,
alpha scans are implemented in EDAC as theta scans at phi = 90 in fixed cluster mode.
The switch to fixed cluster mode is made by PMSCO internally,
no change of angles or other parameters is necessary in the scan or project files
besides filling the alpha instead of the theta column.
*/

32
docs/src/concepts-symmetry.dox Normal file
View File

@ -0,0 +1,32 @@
/*! @page pag_concepts_symmetry Symmetry
\section sec_symmetry Symmetry and Domain Averaging
A _symmetry_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is derived from the same set of model parameters
and that contributes incoherently to the measured diffraction pattern.
A symmetry may be represented by a special symmetry parameter which is not subject to optimization.
For instance, a real sample may have additional rotational domains that are not present in the cluster,
increasing the symmetry from three-fold to six-fold.
Or, an adsorbate may be present in a number of different lateral configurations on the substrate.
In the first case, it may be sufficient to fold calculated data in the proper way to generate the same symmetry as in the measurement.
In the latter case, it may be necessary to execute a scattering calculation for each possible orientation or a representative number of possible orientations.
PMSCO provides the basic framework to spawn multiple calculations according to the number of symmetries (cf. \ref sec_tasks).
The actual data reduction from multiple symmetries to one measurement needs to be implemented on the project level.
This section explains the necessary steps.
1. Your project needs to populate the pmsco.project.Project.symmetries list.
For each symmetry, add a dictionary of symmetry parameters, e.g. <code>{'angle_azi': 15.0}</code>.
There must be at least one symmetry in a project, otherwise no calculation is executed.
2. The project may apply the symmetry of a task to the cluster and parameter file if necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular symmetry in addition to the model parameters.
3. The project combines the results of the calculations for the various symmetries into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with equal weight.
If you need more control, you need to override the pmsco.project.Project.combine_symmetries method and implement your own algorithm.
*/

306
docs/src/concepts-tasks.dox Normal file
View File

@ -0,0 +1,306 @@
/*! @page pag_concepts_tasks Task concept
\section sec_tasks Calculation tasks
A _calculation task_ defines a concrete set of model parameters, atomic coordinates, emitter configuration,
experimental reference and meta-data (such as file names)
that completely defines how to produce the input data for the scattering program (the _calculator_).
For each task, the calculator is executed once and produces one result dataset.
In a typical optimization project, however, the calculator is executed multiple times for various reasons
mandated by the project but also efficient calculations in a multi-process environment:
1. The calculation must be repeated under variation of parameters.
A concrete set of parameters is called @ref sec_task_model.
2. The sample was measured multiple times or under different conditions (initial states, photon energy, emission angle).
Each contiguous measured dataset is called a @ref sec_task_scan.
3. The measurement averages over multiple inequivalent domains, cf. @ref sec_task_symmetry.
4. The measurement includes multiple geometrically inequivalent emitters, cf. @ref sec_task_emitter.
5. The calculation should be distributed over multiple processes that run in parallel to reduce the wall time, cf. @ref sec_task_region.
In PMSCO, these aspects are modelled as attributes of a calculation task
as shown schematically in the following diagram.
@startuml "attributes of a calculation task"
class CalculationTask {
model
scan
symmetry
emitter
region
..
files
}
class Model {
index
..
dlat
dAS
dS1S2
V0
Zsurf
Texp
rmax
}
class Scan {
index
..
filename
mode
initial_state
energies
thetas
phis
alphas
}
class Symmetry {
index
..
rotation
registry
}
class Emitter {
index
}
class Region {
index
..
range
}
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Symmetry
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
symmetries
model_handler
cluster_generator
}
class ClusterGenerator {
count_emitters()
create_cluster()
}
class ModelHandler {
create_tasks()
add_result()
}
Model ..> ModelHandler
Scan ..> Project
Symmetry ..> Project
Emitter ..> ClusterGenerator
Region ..> Project
Project *-left- ModelHandler
Project *- ClusterGenerator
hide empty members
@enduml
Although the attributes may have quite different types (as detailed below),
each instance is also given a unique (per attribute) integer index,
where -1 means that the attribute is undefined.
The indices of the five attributes together (pmsco.dispatch.CalcID tuple)
serve internally to identify a task and the data belonging it.
The identifier appears, for instance, in input and output file names.
Normally, data files are deleted after the calculation, and only a few top-level files are kept
(can be overridden at the command line or in the project code).
At the top level, only the model ID is set, the other ones are undefined (-1).
\subsection sec_task_model Model
The _model_ attribute is a dictionary of continuously variable parameters of the system such as lattice constants, relaxation constants, rotation angles, etc.
It may also define non-structural or non-physical parameters such as temperature, inner potential or cluster radius.
The dictionary contains key-value pairs where the keys are up to the user project (the figure shows some examples).
The values are floating-point numbers that are chosen by the model handler within the domain specified by the user project.
Models are generated by the chosen optimizer according to a particular algorithm or, in single mode, directly by the project.
Each specific instance of model parameters is given a unique index that allows to identify related input and output files.
Model parameters are reported with the corresponding R-factors during the optimization process.
\subsection sec_task_scan Scan
The _scan_ attribute is an index into the list of scans defined by the user project.
Each scan refers to one experimental data file and, thus, defines the initial and final states of the photoelectron.
PMSCO runs a separate calculation for each scan file and compares the combined results to the experimental data.
This is sometimes called a _global fit_.
\subsection sec_task_symmetry Symmetry
A _symmetry_ is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is independent of the _model_ and contributes incoherently to the measured diffraction pattern.
For instance, for a system that includes two inequivalent structural domains,
two separate clusters have to be generated and calculated for each model.
The symmetry parameter is not subject to optimization.
However, if the branching ratio is unknown a priori, a model parameter can be introduced
to control the relative contribution of a particular symmetry to the diffraction pattern.
In that case, the @ref pmsco.project.Project.combine_symmetries method must be overridden.
A symmetry is identified by its index which is an index into the project's symmetries table (pmsco.project.Project.symmetries).
It is up to the user project to give a physical description of the symmetry, e.g. a rotation angle,
by assigning a meaningful value (e.g. a dictionary with key-value pairs) to the symmetries table.
The cluster generator can then read the value from the table rather than from constants in the code.
The figure shows two examples of symmetry parameters.
The corresponding symmetry table could be set up like this:
@code{.py}
project.add_symmetry = {'rotation': 0.0, 'registry': 0.0}
project.add_symmetry = {'rotation': 30.0, 'registry': 0.0}
@endcode
\subsection sec_task_emitter Emitter
The _emitter_ component of the calculation task selects a specific emitter configuration of the cluster generator.
This is merely an index whose interpretation is up to the cluster generator.
The default emitter handler enumerates the emitter index from 1 to the emitter count reported by the cluster generator.
The emitter count and list of emitters may depend on model, scan and symmetry.
The cluster generator can tailor a cluster to the given model, scan, symmetry and emitter index.
For example, in a large unit cell with many inequivalent emitters,
the generator might return a small sub-cluster around the actual emitter for better calculation performance
since the distant atoms of the unit cell do not contribute to the diffraction pattern.
Emitter branching must be requested specifically by using a particular pattern in the code.
By default, it is disabled, which allows the cluster code to be written in a slightly easier way.
\subsection sec_task_region Region
The _region_ handler may split a scan region into several smaller chunks
so that the tasks can be distributed to multiple processes.
Chunking by energy regions is enabled automatically if the project contains an energy scan of at least 10 points
and the project is run in multiple processes.
It can be disabled by the user project.
\section sec_task_handler Task handlers
The previous section described the five important attributes of a calculation task.
These attributes span a five-dimensional index space
where each point maps to one task and, consequently, one calculation and one result dataset.
To populate the index space, however, calculation tasks are more adequately arranged in a tree-like hierarchy with five levels.
The code that defines attributes and processes results can then be separated into _handlers_.
Each level calls for a particular functional contract of the handler.
According to object-oriented principles the contracts at the five levels are defined by abstract base classes
which can be sub-classed for more specific behaviour.
For instance, the class of the model handler is chosen based on the execution mode (single, grid, swarm, etc.).
Though it is possible for a project to define its own handlers,
the PMSCO core declares handlers that should cover most calculation scenarios.
The following diagram shows the tree of calculation tasks and how handlers act on the task objects to populate the task attributes.
At the top of the tree, an empty task object (all attributes undefined) is fed into the model level handler which takes care of the model attribute.
The model handler generates a number of sub-tasks, one for each set of model parameters.
Each of these (incompletely defined) tasks is then passed to the next handler, and so on.
@startuml "calculation task hierarchy and task handler stack"
object "Root: CalculationTask" as Root {
index = (-1,-1,-1,-1,-1)
}
note right: all attributes undefined
object "Model: CalculationTask" as Model {
index = (i,-1,-1,-1,-1)
model
}
note right: model is defined\nother attributes undefined
object ModelHandler
object "Scan: CalculationTask" as Scan {
index = (i,j,-1,-1,-1)
model
scan
}
object ScanHandler
object "Sym: CalculationTask" as Sym {
index = (i,j,k,-1,-1)
model
scan
symmetry
}
object "SymmetryHandler" as SymHandler
object "Emitter: CalculationTask" as Emitter {
index = (i,j,k,l,-1)
model
scan
symmetry
emitter
}
object EmitterHandler
object "Region: CalculationTask" as Region {
index = (i,j,k,l,m)
model
scan
symmetry
emitter
region
}
note right: all attributes well-defined
object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Sym
Sym "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Sym) .. SymHandler
(Sym, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml
At the end of the stack, the tasks are fully specified and are passed to the calculation queue.
They are dispatched to the available processes of the MPI environment in which PMSCO was started,
which allows calculations to be run in parallel.
Only now that the model is broken down into multiple, fully specified tasks,
the cluster and input files are generated, and the calculation program is started.
At the end of a calculation, the output files are associated with their original task objects,
and the tasks are passed back through the task handler stack.
In this phase, each level joins the datasets from the sub-tasks to the data requested by the parent task.
For example, at the lowest level, one result file is present for each region.
The region handler gathers all files that correspond to the same parent task
(i.e. have the same emitter, symmetry, scan and model attributes),
joins them to one file which includes all regions,
links the file to the parent task and passes the result to the next higher level.
On the top level, the model handler compares the result to the experimental data.
Depending on the operation mode, it refines the model parameters and issues new tasks by passing them down the stack.
When the optimization is finished (according to a set of defined criteria),
The model handler returns the root task to the caller, which causes PMSCO to exit.
*/

192
docs/src/concepts.dox
View File

@ -1,153 +1,85 @@
/*! @page pag_concepts Design Concepts
\section sec_tasks Tasks
/*! @page pag_concepts Design
In an optimization project, a number of optimizable, high-level parameters generated by the optimization algorithm
must be mapped to the input parameters and atomic coordinates before the calculation program is executed.
Possibly, the calculation program is executed multiple times for inequivalent domains, emitters or scan geometries.
After the calculation, the output is collected, compared to the experimental data, and the model is refined.
In PMSCO, the optimization is broken down into a set of _tasks_ and assigned to a stack of task _handlers_ according to the following figure.
Each invocation of the scattering program (EDAC) runs a specific task,
i.e. a calculation for a set of specific parameters, a fully-qualified cluster of atoms, and a specific angle and/or energy scan.
\section sec_components Components
\dotfile tasks.dot "PMSCO task stack"
The code for a PMSCO job consists of the following components.
At the root, the _model handler_ proposes models that need to be calculated according to the operation mode specified at the command line.
A _model_ is the minimum set of variable parameters in the context of a custom project.
Other parameters that will not vary under optimization are set directly by the project code.
The model handler may generate models based on a fixed scheme, e.g. on a grid, or based on R-factors of previous results.
@startuml "top-level components of scattering and optimization code"
For each model, one task is passed to the task handling chain, starting with the scan handler.
The _scan handler_ generates sub-tasks for each experimental scan dataset.
This way, the model can be optimized for multiple experimental scans in the same run (see Sec. \ref sec_scanning).
skinparam componentStyle uml2
The _symmetry handler_ generates sub-tasks based on the number of symmetries contained in the experimental data (see Sec. \ref sec_symmetry).
For instance, for a system that includes two inequivalent structural domains, two separate calculations have to be run for each model.
The symmetry handler is implemented on the project level and may be customized for a specific system.
component "project" as project
component "PMSCO" as pmsco
component "scattering code\n(calculator)" as calculator
The _emitter handler_ generates a sub-task for each inequivalent emitter atom
so that the tasks can be distributed to multiple processes (see Sec. \ref sec_emitters).
In a single-process environment, all emitters are calculated in one task.
interface "command line" as cli
interface "input files" as input
interface "output files" as output
interface "experimental data" as data
interface "results" as results
The _region handler_ may split a scan region into several smaller chunks
so that the tasks can be distributed to multiple processes.
With EDAC, only energy scans can benefit from chunking
since it always calculates the full angular distribution.
This layer has to be enabled specifically in the project module.
It is disabled by default.
data -> project
project ..> pmsco
pmsco ..> calculator
cli --> project
input -> calculator
calculator -> output
pmsco -> results
At the end of the stack, the tasks are fully specified and are passed to the calculation queue.
They are dispatched to the available processes of the MPI environment in which PMSCO was started,
which allows calculations to be run in parallel.
Only now that the model is broken down into multiple tasks,
the cluster and input files are generated, and the calculation program is started.
At the end of a calculation, the output is passed back through the task handler stack.
In this phase, each level gathers the datasets from the sub-tasks to the data requested by the parent task
and passes the result to the next higher level.
On the top level, the calculation is compared to the experimental data.
Depending on the operation mode, the model parameters are refined, and new tasks issued.
If the optimization is finished according to a set of defined criteria, PMSCO exits.
As an implentation detail, each task is given a unique _identifier_ consisting of five integer numbers
which correspond to the five levels model, scan, symmetry, emitter and region.
The identifier appears in the file names in the communication with the scattering program.
Normally, the data files are deleted after the calculation, and only a few top-level files are kept
(can be overridden at the command line or in the project code).
At the top level, only the model ID is set, the other ones are undefined (-1).
@enduml
\section sec_symmetry Symmetry and Domain Averaging
The _project_ consists of program code, system and experimental parameters
that are specific to a particular experiment and calculation job.
The project code reads experimental data, defines the parameter dictionary of the model,
and contains code to generate the cluster, parameter and phase files for the scattering code.
The project is also the main entry point of process execution.
A _symmetry_ under PMSCO is a discrete variant of a set of calculation parameters (including the atomic cluster)
that is derived from the same set of model parameters
and that contributes incoherently to the measured diffraction pattern.
A symmetry may be represented by a special symmetry parameter which is not subject to optimization.
The _scattering code_ on the other hand is a static calculation engine
which accepts detailed input files
(parameters, atomic coordinates, emitter specification, scattering phases)
and outputs an intensity distribution of photoelectrons versus energy and/or angle.
For instance, a real sample may have additional rotational domains that are not present in the cluster,
increasing the symmetry from three-fold to six-fold.
Or, an adsorbate may be present in a number of different lateral configurations on the substrate.
In the first case, it may be sufficient to fold calculated data in the proper way to generate the same symmetry as in the measurement.
In the latter case, it may be necessary to execute a scattering calculation for each possible orientation or a representative number of possible orientations.
PMSCO provides the basic framework to spawn multiple calculations according to the number of symmetries (cf. \ref sec_tasks).
The actual data reduction from multiple symmetries to one measurement needs to be implemented on the project level.
This section explains the necessary steps.
1. Your project needs to populate the pmsco.project.Project.symmetries list.
For each symmetry, add a dictionary of symmetry parameters, e.g. <code>{'angle_azi': 15.0}</code>.
There must be at least one symmetry in a project, otherwise no calculation is executed.
2. The project may apply the symmetry of a task to the cluster and parameter file if necessary.
The pmsco.project.Project.create_cluster and pmsco.project.Project.create_params methods receive the index of the particular symmetry in addition to the model parameters.
3. The project combines the results of the calculations for the various symmetries into one dataset that can be compared to the measurement.
The default method implemented in pmsco.project.Project just adds up all calculations with equal weight.
If you need more control, you need to override the pmsco.project.Project.combine_symmetries method and implement your own algorithm.
The _PMSCO core_ interfaces between the project and the calculator.
It carries out the structural optimization and manages the calculation tasks.
It generates and sends input files to the calculator and reads back the output.
\section sec_scanning Scanning
\section sec_control_flow Control flow
PMSCO with EDAC currently supports the following scan axes.
The basic control flow of a optimization job is depicted schematically in the following figure.
- kinetic energy E
- polar angle theta T
- azimuthal angle phi P
- analyser angle alpha A
@startuml "top-level activity diagram"
The following combinations of these scan axes are allowed (see pmsco.data.SCANTYPES).
start
:initialize;
:import experimental data;
repeat
:define tasks;
fork
:calculate\ntask 1;
fork again
:calculate\ntask N;
end fork
:evaluate results;
repeat while
-> [finished];
:report results;
- E
- E-T
- E-A
- T-P (hemispherical or hologram scan)
stop
@attention The T and A axes cannot be combined.
If a scan of one of them is specified, the other is assumed to be fixed at zero!
This assumption may change in the future,
so it is best to explicitly set the fixed angle to zero in the scan file.
@enduml
@remark According to the measurement geometry at PEARL,
alpha scans are implemented in EDAC as theta scans at phi = 90 in fixed cluster mode.
The switch to fixed cluster mode is made by PMSCO internally,
no change of angles or other parameters is necessary in the scan or project files
besides filling the alpha instead of the theta column.
After importing experimental data and setting up the model dictionary and job parameters,
the calculation tasks are defined depending on the execution mode and system setup.
Each task consists of a specific set of model, experimental and calculation parameters
that describe an independent calculation step,
while several steps may be required to produce a dataset that can be compared to the experimental data.
The idea is that tasks can be defined quickly
and that the time-consuming operations are dispatched to slave processes which can run in parallel.
\section sec_emitters Emitter Configurations
Since emitters contribute incoherently to the diffraction pattern,
it should make no difference how the emitters are grouped and calculated.
EDAC allows to specify multiple emitters in one calculation.
However, running EDAC multiple times for a single-emitter configuration or simply summing up the results
gives the same final diffraction pattern with no significant difference of used CPU time.
It is, thus, easy to distribute the emitters over parallel processes in a multi-process environment.
PMSCO can handle this transparently with a minimal effort.
Within the same framework, PMSCO also supports that clusters are tailored to a specific emitter configuration.
Suppose that the unit cell contains a large number of inequivalent emitters.
If all emitters had to be included in a single calculation,
the cluster would grow very large and the calculation would take a long time
because it would include many long scattering paths
that effectively do not contribute intensity to the final result.
Using single-emitters, a cluster can be built locally around the emitter and kept to a reasonable size.
Even when using this feature, PMSCO does not require that each configuration contains only one emitter.
The term _emitter_ effectively means _emitter configuration_.
A configuration can include multiple emitters which will not be broken up further.
It is up to the project, what is included in a particular configuration.
To enable emitter handling,
1. override the count_emitters method of your cluster generator
and return the number of emitter configurations of a given model, scan and symmetry.
2. handle the emitter index in your create_cluster method.
3. (optionally) override the pmsco.project.Project.combine_emitters method
if the emitters should not be added with equal weights.
For implementation details see the respective method descriptions.
As soon as all necessary results are available they are combined into one dataset and compared to the experimental data.
Depending on the execution mode, the process of task definition and calculation repeats until the model has converged
or the calculations are stopped for another reason.
*/

17
docs/src/execution.dox
View File

@ -14,26 +14,29 @@ Run PMSCO from the command prompt:
@code{.sh}
cd work-dir
python project-dir/project.py [pmsco-arguments] [project-arguments]
python pmsco-dir project-dir/project.py [pmsco-arguments] [project-arguments]
@endcode
where <code>work-dir</code> is the destination directory for output files,
<code>pmsco-dir</code> is the directory containing the <code>__main__.py</code> file,
<code>project.py</code> is the specific project module,
and <code>project-dir</code> is the directory where the project file is located.
PMSCO is run in one process which handles all calculations sequentially.
The command line arguments are usually divided into common arguments interpreted by the main pmsco code (pmsco.py),
The command line arguments are divided into common arguments interpreted by the main pmsco code (pmsco.py),
and project-specific arguments interpreted by the project module.
However, it is ultimately up to the project module how the command line is interpreted.
Example command line for a single EDAC calculation of the two-atom project:
@code{.sh}
cd work/twoatom
python pmsco/projects/twoatom/twoatom.py -s ea -o twoatom-demo -m single
python ../../pmsco ../../projects/twoatom/twoatom.py -s ea -o twoatom-demo -m single
@endcode
The project file <code>twoatom.py</code> takes the lead of the project execution.
Usually, it contains only project-specific code and delegates common tasks to the main pmsco code.
This command line executes the main pmsco module <code>pmsco.py</code>.
The main module loads the project file <code>twoatom.py</code> as a plug-in
and starts processing the common arguments.
The <code>twoatom.py</code> module contains only project-specific code
with several defined entry-points called from the main module.
In the command line above, the <code>-o twoatom-demo</code> and <code>-m single</code> arguments
are interpreted by the pmsco module.
@ -61,7 +64,7 @@ For optimum performance, the number of processes should not exceed the number of
To start a two-hour optimization job with multiple processes on an quad-core workstation with hyperthreading:
@code{.sh}
cd work/my_project
mpiexec -np 8 project-dir/project.py -o my_job_0001 -t 2 -m swarm
mpiexec -np 8 pmsco-dir/pmsco project-dir/project.py -o my_job_0001 -t 2 -m swarm
@endcode

194
docs/src/installation.dox
View File

@ -3,60 +3,71 @@
\subsection sec_general General Remarks
The PMSCO code is maintained under git.
The PMSCO code is maintained under [Git](https://git-scm.com/).
The central repository for PSI-internal projects is at https://git.psi.ch/pearl/pmsco,
the public repository at https://gitlab.psi.ch/pearl/pmsco.
For their own developments, users should clone the repository.
Changes to common code should be submitted via pull requests.
\subsection sec_requirements Requirements
The program code of PMSCO and its external programs is written in Python, C++ and Fortran.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a workstation with at least 4 processor cores
or cluster with 20-50 available processor cores is recommended.
The program requires about 2 GB of RAM per process.
The recommended IDE is [PyCharm (community edition)](https://www.jetbrains.com/pycharm).
The documentation in [Doxygen](http://www.stack.nl/~dimitri/doxygen/index.html) format is part of the source code.
The Doxygen compiler can generate separate documentation in HTML or LaTeX.
The MSC and EDAC codes compile with the GNU Fortran and C++ compilers on Linux.
Other compilers may work but have not been tested.
The code will run in any recent Linux environment on a workstation or in a virtual machine.
Scientific Linux, CentOS7, [Ubuntu](https://www.ubuntu.com/)
and [Lubuntu](http://lubuntu.net/) (recommended for virtual machine) have been tested.
For optimization jobs, a high-performance cluster with 20-50 available processor cores is recommended.
The code requires about 2 GB of RAM per process.
Please note that it may be important that the code remains compatible with earlier compiler and library versions.
Newer compilers or the latest versions of the libraries contain features that will break the compatibility.
The code can be used with newer versions as long they are backward compatible.
\subsection sec_requirements Requirements
Please note that in some environments (particularly shared high-performance machines)
it may be important to choose specific compiler and library versions.
In order to maintain backward compatibility with some of these older machines,
code that requires new versions of compilers and libraries should be introduced carefully.
The code depends on the following libraries:
- GCC 4.8
- OpenMPI 1.10
- GCC >= 4.8
- OpenMPI >= 1.10
- F2PY
- F2C
- SWIG
- Python 2.7 (incompatible with Python 3.0)
- Numpy 1.11 (incompatible with Numpy 1.13 and later)
- MPI4PY (from PyPI)
- BLAS
- LAPACK
- periodictable
- Python 2.7 or 3.6
- Numpy >= 1.11
- Python packages from PyPI listed in the requirements.txt file
Most of these requirements are available from the Linux distribution.
For an easily maintainable Python environment, [Miniconda](https://conda.io/miniconda.html) is recommended.
The Python environment distributed with the OS often contains outdated packages,
and it's difficult to switch between different Python versions.
Most of these requirements are available from the Linux distribution, or from PyPI (pip install), respectively.
If there are any issues with the packages installed by the distribution, try the ones from PyPI
(e.g. there is currently a bug in the Debian mpi4py package).
The F2C source code is contained in the repository for machines which don't have it installed.
On the PSI cluster machines, the environment must be set using the module system and conda (on Ra).
Details are explained in the PEARL Wiki.
\subsubsection sec_install_ubuntu Installation on Ubuntu 16.04
PMSCO runs under Python 2.7 or Python 3.6 or higher.
Since Python 2 is being deprecated, the code has been ported to Python 3.6.
Compatibility with Python 2.7 is maintained by using
the [future package](http://python-future.org/compatible_idioms.html).
New code should be written according to their guidelines.
The following instructions install the necessary dependencies on Ubuntu (or Lubuntu 16.04):
\subsection sec_install_instructions Instructions
\subsubsection sec_install_ubuntu Installation on Ubuntu
The following instructions install the necessary dependencies on Ubuntu, Debian or related distributions.
The Python environment is provided by [Miniconda](https://conda.io/miniconda.html).
@code{.sh}
sudo apt-get update
sudo apt update
sudo apt-get install \
sudo apt install \
binutils \
build-essential \
doxygen \
@ -67,38 +78,114 @@ gcc \
gfortran \
git \
graphviz \
ipython \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
python-all \
python-mock \
python-nose \
python-numpy \
python-pip \
python-scipy \
python2.7-dev \
swig
sqlite3 \
swig \
wget
@endcode
sudo pip install --system mpi4py periodictable
On systems where the link to libblas is missing (see @ref sec_compile below),
the following lines are necessary.
@code{.sh}
cd /usr/lib
sudo ln -s /usr/lib/libblas/libblas.so.3 libblas.so
@endcode
The following instructions install the PyCharm IDE and a few other useful utilities:
Install Miniconda according to their [instructions](https://conda.io/docs/user-guide/install/index.html),
then configure the Python environment:
@code{.sh}
sudo sh -c 'echo "deb http://archive.getdeb.net/ubuntu xenial-getdeb apps" >> /etc/apt/sources.list.d/getdeb.list'
wget -q -O - http://archive.getdeb.net/getdeb-archive.key | sudo apt-key add -
sudo apt-get update
sudo apt-get install \
conda create -q --yes -n pmsco python=2.7
source activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
future \
statsmodels
pip install periodictable attrdict fasteners
@endcode
\subsubsection sec_install_singularity Installation in Singularity container
A [Singularity](https://www.sylabs.io/guides/2.5/user-guide/index.html) container
contains all OS and Python dependencies for running PMSCO.
Besides the Singularity executable, nothing else needs to be installed in the host system.
This may be the fastest way to get PMSCO running.
For installation of Singularity,
see their [user guide](https://www.sylabs.io/guides/2.5/user-guide/installation.html).
On newer Linux systems (e.g. Ubuntu 18.04), Singularity is available from the package manager.
Installation in a virtual machine on Windows or Mac are straightforward
thanks to the [Vagrant system](https://www.vagrantup.com/).
After installing Singularity,
check out PMSCO as explained in the @ref sec_compile section:
@code{.sh}
cd ~
mkdir containers
git clone git@git.psi.ch:pearl/pmsco.git pmsco
cd pmsco
git checkout master
git checkout -b my_branch
@endcode
Then, either copy a pre-built container into `~/containers`,
or build one from a script provided by the PMSCO repository:
@code{.sh}
cd ~/containers
sudo singularity build pmsco.simg ~/containers/pmsco/extras/singularity/singularity_python2
@endcode
To work with PMSCO, start an interactive shell in the container and switch to the pmsco environment.
Note that the PMSCO code is outside the container and can be edited with the usual tools.
@code{.sh}
cd ~/containers
singularity shell pmsco.simg
source activate pmsco
cd ~/containers/pmsco
make all
nosetests -w tests/
@endcode
Or call PMSCO from outside:
@code{.sh}
cd ~/containers
mkdir output
cd output
singularity run ../pmsco.simg python ~/containers/pmsco/pmsco path/to/your-project.py arg1 arg2 ...
@endcode
For parallel processing, prepend `mpirun -np X` to the singularity command as needed.
\subsubsection sec_install_extra Additional Applications
For working with the code and data, some other applications are recommended.
The PyCharm IDE can be installed from the Ubuntu software center.
The following commands install other useful helper applications:
@code{.sh}
sudo apt install \
avogadro \
gitg \
meld \
openjdk-9-jdk \
pycharm
meld
@endcode
To produce documentation in PDF format (not recommended on virtual machine), install LaTeX:
@ -124,15 +211,18 @@ Private key authentication is usually recommended except on shared computers.
Clone the code repository using one of these repositiory addresses and switch to the desired branch:
@code{.sh}
cd ~
git clone git@git.psi.ch:pearl/pmsco.git pmsco
cd pmsco
git checkout master
git checkout -b my_branch
@endcode
The compilation of the various modules is started by <code>make all</code>.
The compilation step is necessary only once after installation.
Compile the code and run the unit tests to check that it worked.
@code{.sh}
make all
nosetests -w tests/
@endcode
If the compilation of _loess.so failes due to a missing BLAS library,
try to set a link to the BLAS library as follows (the actual file names may vary due to the actual distribution or version):
@ -150,7 +240,7 @@ Re-check from time to time.
@code{.sh}
cd ~/pmsco
nosetests
nosetests -w tests/
@endcode
Run the twoatom project to check the compilation of the calculation programs.
@ -161,8 +251,10 @@ mkdir work
cd work
mkdir twoatom
cd twoatom/
nice python ~/pmsco/projects/twoatom/twoatom.py -s ~/pmsco/projects/twoatom/twoatom_energy_alpha.etpai -o twoatom_energy_alpha -m single
nice python ~/pmsco/pmsco ~/pmsco/projects/twoatom/twoatom.py -s ea -o twoatom_energy_alpha -m single
@endcode
Runtime warnings may appear because the twoatom project does not contain experimental data.
To learn more about running PMSCO, see @ref pag_run.
*/

22
docs/src/introduction.dox
View File

@ -17,11 +17,6 @@ PMSCO is written in Python 2.7.
EDAC is written in C++, MSC in Fortran.
PMSCO interacts with the calculation programs through Python wrappers for C++ or Fortran.
The MSC and EDAC source code is contained in the same software repository.
The PMSCO, MSC, and EDAC programs may not be used outside the PEARL group without an explicit agreement by the respective original authors.
Users of the PMSCO code are requested to coordinate and share the development of the code with the original author.
Please read and respect the respective license agreements.
\section sec_intro_highlights Highlights
@ -54,8 +49,25 @@ and the example projects.
\section sec_intro_start Getting Started
- @ref pag_concepts
- @ref pag_concepts_tasks
- @ref pag_concepts_emitter
- @ref pag_install
- @ref pag_run
- @ref pag_command
\section sec_license License Information
An open distribution of PMSCO is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) at <https://gitlab.psi.ch/pearl-public/pmsco>.
- Please read and respect the respective license agreements.
- Please acknowledge the use of the code.
- Please share your development of the code with the original author.
Due to different copyright, the MSC and EDAC programs are not contained in the public software repository.
These programs may not be used without an explicit agreement by the respective original authors.
\author Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
\version This documentation is compiled from version $(REVISION).
\copyright 2015-2018 by [Paul Scherrer Institut](http://www.psi.ch)
\copyright Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
*/

141
docs/src/optimizers.dox Normal file
View File

@ -0,0 +1,141 @@
/*! @page pag_opt Model optimizers
\section sec_opt Model optimizers
\subsection sec_opt_swarm Particle swarm
The particle swarm algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012).
The general parameters of the genetic algorithm are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | |
| position_constrain_mode | | default bounce | Resolution of domain limit violations. |
| seed_file | --seed-file | a file path, default none | |
| seed_limit | --seed-limit | 0..pop_size | |
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The domain parameters have the following meanings:
| Parameter | Description |
| --- | --- |
| start | Seed model. The start values are copied into particle 0 of the initial population. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Not used. |
\subsection sec_opt_genetic Genetic optimization
The genetic algorithm evolves a population of individuals
by a combination of inheritance, crossover, mutation
and selection in analogy to biological evolution.
The _genes_ are in this case the model parameters,
and selection occurs based on R-factor.
The genetic algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012).
It is implemented in the @ref pmsco.optimizers.genetic module.
The genetic optimization is helpful in the first stage of an optimization
where a large parameter space needs to be sampled
and fast convergence on a small part of the parameter space is less desirable
as it might catch on a local optimum.
On the other hand, convergence near the optimum is slower than in the particle swarm.
The genetic optimization should be run with a large number of iterations
rather than a large population size.
The general parameters of the genetic algorithm are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line.
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | |
| mating_factor | | 1..pop_size, default 4 | |
| strong_mutation_probability | | 0..1, default 0.01 | Probability that a parameter undergoes a strong mutation. |
| weak_mutation_probability | | 0..1, default 1 | Probability that a parameter undergoes a weak mutation. This parameters should be left at 1. Lower values tend to produce discrete parameter values. Weak mutations can be tuned by the step domain parameters. |
| position_constrain_mode | | default random | Resolution of domain limit violations. |
| seed_file | --seed-file | a file path, default none | |
| seed_limit | --seed-limit | 0..pop_size | |
| rfac_limit | | 0..1, default 0.8 | Accept only seed values that have a lower R-factor. |
| recalc_seed | | True or False, default True | |
The domain parameters have the following meanings:
| Parameter | Description |
| --- | --- |
| start | Seed model. The start values are copied into particle 0 of the initial population. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Standard deviation of the Gaussian distribution of weak mutations. The step should not be much lower than the the parameter range divided by the population size and not greater than one third of the parameter range. |
\subsection sec_opt_grid Grid search
The grid search algorithm samples the parameter space at equidistant steps.
The order of calculations is randomized so that early results represent various parts of the parameter space.
| Parameter | Description |
| --- | --- |
| start | Values of fixed parameters. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. If abs(max - min) < step/2 , the parameter is kept constant. |
| step | Step size (distance between two grid points). If step <= 0, the parameter is kept constant. |
\subsection sec_opt_gradient Gradient search
Currently not implemented.
\subsection sec_opt_table Table scan
The table scan calculates models from an explicit table of model parameters.
It can be used to recalculate models from a previous optimization run on other experimental data,
as an interface to external optimizers,
or as a simple input of manually edited model parameters.
The table can be stored in an external file that is specified on the command line,
or supplied in one of several forms by the custom project class.
The table can be left unchanged during the calculations,
or new models can be added on the go.
@attention Because it is not easily possible to know when and which models have been read from the table file, if you do modify the table file during processing, pay attention to the following hints:
1. The file on disk must not be locked for more than a second. Do not keep the file open unnecessarily.
2. _Append_ new models to the end of the table rather than overwriting previous ones. Otherwise, some models may be lost before they have been calculated.
The general parameters of the table scan are specified in the @ref Project.optimizer_params dictionary.
Some of them can be changed on the command line or in the project class (depending on how the project class is implemented).
| Parameter | Command line | Range | Description |
| --- | --- | --- | --- |
| pop_size | --pop-size | &ge; 1 | Number of models in a generation (calculated in parallel). In table mode, this parameter is not so important and can be left at the default. It has nothing to do with table size. |
| table_file | --table-file | a file path, default none | |
The domain parameters have the following meanings.
Models that violate the parameter range are not calculated.
| Parameter | Description |
| --- | --- |
| start | Not used. |
| min | Lower limit of the parameter range. |
| max | Upper limit of the parameter range. |
| step | Not used. |
\subsection sec_opt_single Single model
The single model optimizer calculates the model defined by domain.start.
| Parameter | Description |
| --- | --- |
| start | Values of model parameters. |
| min | Not used. |
| max | Not used. |
| step | Not used. |
*/

38
docs/src/uml/CalculationTask-class.puml Normal file
View File

@ -0,0 +1,38 @@
@startuml
class CalculationTask {
id : CalcID
parent : CalcID
model : dict
file_root : str
file_ext : str
result_filename : str
modf_filename : str
result_valid : bool
time : datetime.timedelta
files : dict
region : dict
__init__()
__eq__()
__hash__()
copy()
change_id()
format_filename()
get_mpi_message()
set_mpi_message()
add_task_file()
rename_task_file()
remove_task_file()
}
class CalcID {
model
scan
sym
emit
region
}
CalculationTask *-- CalcID
@enduml

133
docs/src/uml/CalculationTask-objects.puml Normal file
View File

@ -0,0 +1,133 @@
@startuml
object Root {
id = -1, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {}
}
Root o.. Model1
Root o.. Model2
object Model1 {
id = 1, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {'d': 5}
}
object Model2 {
id = 2, -1, -1, -1, -1
parent = -1, -1, -1, -1, -1
model = {'d': 7}
}
Model1 o.. Scan11
Model1 o.. Scan12
Model2 o.. Scan21
object Scan11 {
id = 1, 1, -1, -1, -1
parent = 1, -1, -1, -1, -1
model = {'d': 5}
}
object Scan12 {
id = 1, 2, -1, -1, -1
parent = 1, -1, -1, -1, -1
model = {'d': 5}
}
object Scan21 {
id = 2, 1, -1, -1, -1
parent = 2, -1, -1, -1, -1
model = {'d': 7}
}
Scan11 o.. Sym111
object Sym111 {
id = 1, 1, 1, -1, -1
parent = 1, 1, -1, -1, -1
model = {'d': 5}
}
Sym111 o.. Emitter1111
object Emitter1111 {
id = 1, 1, 1, 1, -1
parent = 1, 1, 1, -1, -1
model = {'d': 5}
}
Emitter1111 o.. Region11111
object Region11111 {
id = 1, 1, 1, 1, 1
parent = 1, 1, 1, 1, -1
model = {'d': 5}
}
@enduml
@startuml
object "Root: CalculationTask" as Root {
}
note right: all attributes undefined
object "Model: CalculationTask" as Model {
model
}
note right: model is defined\nother attributes undefined
object ModelHandler
object "Scan: CalculationTask" as Scan {
model
scan
}
object ScanHandler
object "Sym: CalculationTask" as Sym {
model
scan
symmetry
}
object "SymmetryHandler" as SymHandler
object "Emitter: CalculationTask" as Emitter {
model
scan
symmetry
emitter
}
object EmitterHandler
object "Region: CalculationTask" as Region {
model
scan
symmetry
emitter
region
}
note right: all attributes well-defined
object RegionHandler
Root "1" o.. "1..*" Model
Model "1" o.. "1..*" Scan
Scan "1" o.. "1..*" Sym
Sym "1" o.. "1..*" Emitter
Emitter "1" o.. "1..*" Region
(Root, Model) .. ModelHandler
(Model, Scan) .. ScanHandler
(Scan, Sym) .. SymHandler
(Sym, Emitter) .. EmitterHandler
(Emitter, Region) .. RegionHandler
@enduml

90
docs/src/uml/calculation-task.puml Normal file
View File

@ -0,0 +1,90 @@
@startuml
class CalculationTask {
model
scan
symmetry
emitter
region
..
files
}
class Model {
index
..
dlat
dAS
dS1S2
V0
Zsurf
Texp
rmax
}
class Scan {
index
..
filename
mode
initial_state
energies
thetas
phis
alphas
}
class Symmetry {
index
..
rotation
registry
}
class Emitter {
index
}
class Region {
index
..
range
}
CalculationTask *-- Model
CalculationTask *-- Scan
CalculationTask *-- Symmetry
CalculationTask *-- Emitter
CalculationTask *-- Region
class Project {
scans
symmetries
model_handler
cluster_generator
}
class ClusterGenerator {
count_emitters()
create_cluster()
}
class ModelHandler {
create_tasks()
add_result()
}
Model ..> ModelHandler
Scan ..> Project
Symmetry ..> Project
Emitter ..> ClusterGenerator
Region ..> Project
Project *-left- ModelHandler
Project *- ClusterGenerator
hide empty members
@enduml

47
docs/src/uml/cluster-generator.puml Normal file
View File

@ -0,0 +1,47 @@
@startuml
package pmsco {
class Project {
cluster_generator
export_cluster()
}
abstract class ClusterGenerator {
project
{abstract} count_emitters()
{abstract} create_cluster()
}
class LegacyClusterGenerator {
project
count_emitters()
create_cluster()
}
}
package "user project" {
class UserClusterGenerator {
project
count_emitters()
create_cluster()
}
note bottom : for complex cluster
class UserProject {
count_emitters()
create_cluster()
}
note bottom : for simple cluster
}
Project <|-- UserProject
ClusterGenerator <|-- LegacyClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
Project *-- ClusterGenerator
UserProject .> LegacyClusterGenerator
UserProject .> UserClusterGenerator
@enduml

81
docs/src/uml/database.puml Normal file
View File

@ -0,0 +1,81 @@
@startuml
class Project << (T,orchid) >> {
id
..
..
name
code
}
class Scan << (T,orchid) >> {
id
..
job_id
..
name
}
class Job << (T,orchid) >> {
id
..
project_id
..
name
mode
machine
git_hash
datetime
description
}
class Model << (T,orchid) >> {
id
..
job_id
..
model
gen
particle
}
class Result << (T,orchid) >> {
id
..
model_id
..
scan
sym
emit
region
rfac
}
class Param << (T,orchid) >> {
id
..
..
key
}
class ParamValue << (T,orchid) >> {
id
..
param_id
model_id
..
value
}
Project "1" *-- "*" Job
Job "1" *-- "*" Model
Job "1" *-- "*" Scan
Param "1" *-- "*" ParamValue
Model "1" *-- "*" ParamValue
Model "1" *-- "*" Result
hide empty members
@enduml

45
docs/src/uml/handler-activity.puml Normal file
View File

@ -0,0 +1,45 @@
@startuml
start
repeat
:define model tasks;
:gather model results;
repeat while
stop
@enduml
@startuml
start
repeat
partition "generate tasks" {
:define model tasks;
:define scan tasks;
:define symmetry tasks;
:define emitter tasks;
:define region tasks;
}
fork
:calculate task 1;
fork again
:calculate task 2;
fork again
:calculate task N;
end fork
partition "collect results" {
:gather region results;
:gather emitter results;
:gather symmetry results;
:gather scan results;
:gather model results;
}
repeat while
stop
@enduml

24
docs/src/uml/master-slave-messages.puml Normal file
View File

@ -0,0 +1,24 @@
@startuml{master-slave-messages.png}
== task execution ==
loop calculation tasks
hnote over Master : define task
Master -> Slave: TAG_NEW_TASK
activate Slave
hnote over Slave : calculation
alt successful
Slave --> Master: TAG_NEW_RESULT
else calculation failed
Slave --> Master: TAG_INVALID_RESULT
else critical error
Slave --> Master: TAG_ERROR_ABORTING
end
deactivate Slave
hnote over Master : collect results
end
...
== termination ==
Master -> Slave: TAG_FINISH
destroy Slave
@enduml

31
docs/src/uml/minimum-project-classes.puml Normal file
View File

@ -0,0 +1,31 @@
@startuml
package pmsco {
abstract class Project {
mode
code
scans
symmetries
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_domain()
}
}
package projects {
class UserProject {
__init__()
create_cluster()
create_params()
create_domain()
}
}
Project <|-- UserProject
hide empty members
@enduml

66
docs/src/uml/mpi-processes.puml Normal file
View File

@ -0,0 +1,66 @@
@startuml
participant rank0 as "rank 0 (master)"
participant rank1 as "rank 1 (slave)"
participant rank2 as "rank 2 (slave)"
participant rankN as "rank N (slave)"
== initialization ==
rank0 ->> rank0
activate rank0
rnote over rank0: initialize project
== task loop ==
rnote over rank0: specify tasks
rank0 ->> rank1: task 1
activate rank1
rnote over rank1: execute task 1
rank0 ->> rank2: task 2
activate rank2
rnote over rank2: execute task 2
rank0 ->> rankN: task N
deactivate rank0
activate rankN
rnote over rankN: execute task N
rank0 <<-- rank1: result 1
deactivate rank1
rnote over rank0: process results\nspecify tasks
activate rank0
rank0 ->> rank1: task N+1
deactivate rank0
activate rank1
rnote over rank1: execute task N+1
rank0 <<-- rank2: result 2
deactivate rank2
activate rank0
rank0 ->> rank2: task N+2
deactivate rank0
activate rank2
rnote over rank2: execute task N+2
rank0 <<-- rankN: result N
deactivate rankN
activate rank0
rank0 ->> rankN: task 2N
deactivate rank0
activate rankN
rnote over rankN: execute task 2N
rank0 <<-- rank1: result N+1
deactivate rank1
rank0 <<-- rank2: result N+2
deactivate rank2
rank0 <<-- rankN: result 2N
deactivate rankN
rnote over rank0: process results
activate rank0
hnote over rank0: calculations complete
== termination ==
rnote over rank0: report results
rank0 ->> rank1: finish
destroy rank1
rank0 ->> rank2: finish
destroy rank2
rank0 ->> rankN: finish
destroy rankN
deactivate rank0
@enduml

59
docs/src/uml/project-classes.puml Normal file
View File

@ -0,0 +1,59 @@
@startuml
abstract class Project {
mode : str = "single"
code : str = "edac"
scans : Scan [1..*]
symmetries : dict [1..*]
cluster_generator : ClusterGenerator
handler_classes
files : FileTracker
{abstract} create_cluster() : Cluster
{abstract} create_params() : Params
{abstract} create_domain() : Domain
}
class Scan {
filename
raw_data
dtype
modulation
mode
emitter
initial_state
energies
thetas
phis
alphas
set_scan()
}
class Domain {
start : dict
min : dict
max : dict
step : dict
add_param(name, start, min, max, step)
get_param(name)
}
class Params {
title
comment
cluster_file
output_file
scan_file
initial_state
polarization
angular_resolution
z_surface
inner_potential
work_function
polar_incidence_angle
azimuthal_incidence_angle
experiment_temperature
}
Project "1" *-- "1..*" Scan
@enduml

26
docs/src/uml/scan-tasks-activity.puml Normal file
View File

@ -0,0 +1,26 @@
@startuml
:model task|
fork
partition "scan 0" {
:define scan;
:scan 0 task|
detach
:scan 0 result|
}
fork again
partition "scan 1" {
:define scan;
:scan 1 task|
detach
:scan 1 result|
}
fork again
partition "scan N" {
:define scan;
:scan N task|
detach
:scan N result|
}
end fork
:model result|
@enduml

42
docs/src/uml/top-activity-partitions.puml Normal file
View File

@ -0,0 +1,42 @@
@startuml
|user|
start
:setup;
|pmsco|
:initialize;
:import experimental data;
repeat
:define task;
|calculator|
:calculate\ntask;
|pmsco|
:evaluate results;
repeat while
-> [finished];
:report results;
stop
@enduml
@startuml
|pmsco|
start
:define task (model, scan, symmetry, emitter, region);
|project|
:create cluster;
:create parameters;
|calculator|
:scattering calculation;
|pmsco|
:combine results;
|project|
:calculate modulation function;
:calculate R-factor;
stop
@enduml

21
docs/src/uml/top-activity.puml Normal file
View File

@ -0,0 +1,21 @@
@startuml
start
:initialize;
:import experimental data|
repeat
:define tasks;
fork
:calculate\ntask 1;
fork again
:calculate\ntask N;
end fork
:evaluate results;
repeat while
-> [finished];
:report results|
stop
@enduml

23
docs/src/uml/top-components.puml Normal file
View File

@ -0,0 +1,23 @@
@startuml
skinparam componentStyle uml2
component "project" as project
component "PMSCO" as pmsco
component "scattering code\n(calculator)" as calculator
interface "command line" as cli
interface "input files" as input
interface "output files" as output
interface "experimental data" as data
interface "results" as results
data -> project
project ..> pmsco
pmsco ..> calculator
cli --> project
input -> calculator
calculator -> output
pmsco -> results
@enduml

55
docs/src/uml/user-project-classes.puml Normal file
View File

@ -0,0 +1,55 @@
@startuml
package pmsco {
abstract class Project {
mode
code
scans
symmetries
cluster_generator
handler_classes
__
{abstract} create_cluster()
{abstract} create_params()
{abstract} create_domain()
..
combine_scans()
combine_symmetries()
combine_emitters()
calc_modulation()
calc_rfactor()
}
abstract class ClusterGenerator {
{abstract} count_emitters()
{abstract} create_cluster()
}
}
package projects {
class UserProject {
scan_dict
__
setup()
..
create_params()
create_domain()
..
combine_symmetries()
}
class UserClusterGenerator {
count_emitters()
create_cluster()
}
}
Project <|-- UserProject
Project *-- ClusterGenerator
ClusterGenerator <|-- UserClusterGenerator
hide empty members
@enduml

118
extras/singularity/singularity_python2 Normal file
View File

@ -0,0 +1,118 @@
BootStrap: debootstrap
OSVersion: bionic
MirrorURL: http://ch.archive.ubuntu.com/ubuntu/
%help
a singularity container for PMSCO.
git clone requires an ssh key for git.psi.ch.
try agent forwarding (-A option to ssh).
#%setup
# executed on the host system outside of the container before %post
#
# this will be inside the container
# touch ${SINGULARITY_ROOTFS}/tacos.txt
# this will be on the host
# touch avocados.txt
#%files
# files are copied before %post
#
# this copies to root
# avocados.txt
# this copies to /opt
# avocados.txt /opt
#
# this does not work
# ~/.ssh/known_hosts /etc/ssh/ssh_known_hosts
# ~/.ssh/id_rsa /etc/ssh/id_rsa
%labels
Maintainer Matthias Muntwiler
Maintainer_Email matthias.muntwiler@psi.ch
Python_Version 2.7
%environment
export PATH="/usr/local/miniconda3/bin:$PATH"
export PYTHON_VERSION=2.7
export SINGULAR_BRANCH="singular"
export LC_ALL=C
%post
export PYTHON_VERSION=2.7
export LC_ALL=C
sed -i 's/$/ universe/' /etc/apt/sources.list
apt-get update
apt-get -y install \
binutils \
build-essential \
doxygen \
doxypy \
f2c \
g++ \
gcc \
gfortran \
git \
graphviz \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
nano \
openmpi-bin \
openmpi-common \
sqlite3 \
swig \
wget
apt-get clean
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
bash ~/miniconda.sh -b -p /usr/local/miniconda3
export PATH="/usr/local/miniconda3/bin:$PATH"
conda create -q --yes -n pmsco python=${PYTHON_VERSION}
. /usr/local/miniconda3/bin/activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
future \
statsmodels
conda clean --all -y
/usr/local/miniconda3/envs/pmsco/bin/pip install periodictable attrdict fasteners
#%test
# test the image after build
%runscript
# executes command from command line
. /usr/local/miniconda3/bin/activate pmsco
exec echo "$@"
%apprun install
. /usr/local/miniconda3/bin/activate pmsco
cd ~
git clone https://git.psi.ch/pearl/pmsco.git pmsco
cd pmsco
git checkout develop
git checkout -b ${SINGULAR_BRANCH}
make all
nosetests
%apprun python
. /usr/local/miniconda3/bin/activate pmsco
exec python "${@}"
%apprun conda
. /usr/local/miniconda3/bin/activate pmsco
exec conda "${@}"

117
extras/singularity/singularity_python3 Normal file
View File

@ -0,0 +1,117 @@
BootStrap: debootstrap
OSVersion: bionic
MirrorURL: http://ch.archive.ubuntu.com/ubuntu/
%help
a singularity container for PMSCO.
git clone requires an ssh key for git.psi.ch.
try agent forwarding (-A option to ssh).
#%setup
# executed on the host system outside of the container before %post
#
# this will be inside the container
# touch ${SINGULARITY_ROOTFS}/tacos.txt
# this will be on the host
# touch avocados.txt
#%files
# files are copied before %post
#
# this copies to root
# avocados.txt
# this copies to /opt
# avocados.txt /opt
#
# this does not work
# ~/.ssh/known_hosts /etc/ssh/ssh_known_hosts
# ~/.ssh/id_rsa /etc/ssh/id_rsa
%labels
Maintainer Matthias Muntwiler
Maintainer_Email matthias.muntwiler@psi.ch
Python_Version 3
%environment
export PATH="/usr/local/miniconda3/bin:$PATH"
export PYTHON_VERSION=3
export SINGULAR_BRANCH="singular"
export LC_ALL=C
%post
export PYTHON_VERSION=3
export LC_ALL=C
sed -i 's/$/ universe/' /etc/apt/sources.list
apt-get update
apt-get -y install \
binutils \
build-essential \
doxygen \
doxypy \
f2c \
g++ \
gcc \
gfortran \
git \
graphviz \
libblas-dev \
liblapack-dev \
libopenmpi-dev \
make \
openmpi-bin \
openmpi-common \
sqlite3 \
swig \
wget
apt-get clean
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
bash ~/miniconda.sh -b -p /usr/local/miniconda3
export PATH="/usr/local/miniconda3/bin:$PATH"
conda create -q --yes -n pmsco python=${PYTHON_VERSION}
. /usr/local/miniconda3/bin/activate pmsco
conda install -q --yes -n pmsco \
pip \
numpy \
scipy \
ipython \
mpi4py \
matplotlib \
nose \
mock \
future \
statsmodels
conda clean --all -y
/usr/local/miniconda3/envs/pmsco/bin/pip install periodictable attrdict fasteners
#%test
# test the image after build
%runscript
# executes command from command line
source /usr/local/miniconda3/bin/activate pmsco
exec echo "$@"
%apprun install
source /usr/local/miniconda3/bin/activate pmsco
cd ~
git clone https://git.psi.ch/pearl/pmsco.git pmsco
cd pmsco
git checkout develop
git checkout -b ${SINGULAR_BRANCH}
make all
nosetests
%apprun python
source /usr/local/miniconda3/bin/activate pmsco
exec python "${@}"
%apprun conda
source /usr/local/miniconda3/bin/activate pmsco
exec conda "${@}"

76
extras/vagrant/Vagrantfile vendored Normal file
View File

@ -0,0 +1,76 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure("2") do |config|
# The most common configuration options are documented and commented below.
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "singularityware/singularity-2.4"
config.vm.box_version = "2.4"
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs
# `vagrant box outdated`. This is not recommended.
# config.vm.box_check_update = false
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine. In the example below,
# accessing "localhost:8080" will access port 80 on the guest machine.
# NOTE: This will enable public access to the opened port
# config.vm.network "forwarded_port", guest: 80, host: 8080
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine and only allow access
# via 127.0.0.1 to disable public access
# config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
# Create a private network, which allows host-only access to the machine
# using a specific IP.
# config.vm.network "private_network", ip: "192.168.33.10"
# Create a public network, which generally matched to bridged network.
# Bridged networks make the machine appear as another physical device on
# your network.
# config.vm.network "public_network"
# Share an additional folder to the guest VM. The first argument is
# the path on the host to the actual folder. The second argument is
# the path on the guest to mount the folder. And the optional third
# argument is a set of non-required options.
# config.vm.synced_folder "../data", "/vagrant_data"
# Provider-specific configuration so you can fine-tune various
# backing providers for Vagrant. These expose provider-specific options.
# Example for VirtualBox:
#
config.vm.provider "virtualbox" do |vb|
# Display the VirtualBox GUI when booting the machine
# vb.gui = true
# Customize the amount of memory on the VM:
# Increase this number if you plan to run parallel processes.
vb.memory = "2048"
# Customize the number of CPUs:
# Increase as necessary but not more than physically available on the host.
vb.cpus = 2
end
#
# View the documentation for the provider you are using for more
# information on available options.
# Enable provisioning with a shell script. Additional provisioners such as
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
# documentation for more information about their specific syntax and use.
# config.vm.provision "shell", inline: <<-SHELL
# apt-get update
# apt-get install -y apache2
# SHELL
end

19
makefile
View File

@ -15,6 +15,25 @@ SHELL=/bin/sh
#
# the MSC and MUFPOT programs are currently not used.
# they are not built by the top-level targets all and bin.
#
# the make system uses the compiler executables of the current environment.
# to override the executables, you may set the following variables.
# to switch between python versions, however, the developers recommend miniconda.
#
# PYTHON = python executable (default: python)
# PYTHONOPTS = python options (default: none)
# CC = C and Fortran compiler executable (default: gcc)
# CCOPTS = C compiler options (default: none)
# CXX = C++ compiler executable (default: g++)
# CXXOPTS = C++ compiler options (default: none)
#
# make all PYTHON=/usr/bin/python2.7
#
# or:
#
# export PYTHON=/usr/bin/python2.7
# make all
#
.PHONY: all bin docs clean edac loess msc mufpot

15
pmsco/__main__.py
View File

@ -8,10 +8,17 @@ python pmsco [pmsco-arguments]
@endverbatim
"""
import pmsco
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import os.path
file_dir = os.path.dirname(__file__) or '.'
root_dir = os.path.join(file_dir, '..')
root_dir = os.path.abspath(root_dir)
sys.path[0] = root_dir
if __name__ == '__main__':
args, unknown_args = pmsco.parse_cli()
pmsco.main_pmsco(args, unknown_args)
sys.exit(0)
import pmsco.pmsco
pmsco.pmsco.main()

0
pmsco/calculators/__init__.py Normal file
View File

63
pmsco/calculator.py → pmsco/calculators/calculator.py
View File

@ -1,5 +1,5 @@
"""
@package pmsco.calculator
@package pmsco.calculators.calculator
abstract scattering program interface.
this module declares the basic interface to scattering programs.
@ -11,17 +11,21 @@ TestCalcInterface is provided for testing the PMSCO code quickly without calling
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import time
import numpy as np
import data as md
import cluster as mc
import pmsco.data as md
__author__ = 'matthias muntwiler'
@ -38,57 +42,24 @@ class Calculator(object):
or <code>output_file + '.etpai'</code> depending on scan mode.
all other intermediate files are deleted unless keep_temp_files is True.
@param params: a msco_project.Params() object with all necessary values except cluster and output files set.
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param cluster: a msco_cluster.Cluster() object with all atom positions set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param scan: a msco_project.Scan() object describing the experimental scanning scheme.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param output_file: base name for all intermediate and output files
@return: result_file, files_cats
@arg result_file is the name of the main ETPI or ETPAI result file to be further processed.
@arg files_cats is a dictionary that lists the names of all created data files with their category.
@return: (str, dict) result_file, and dictionary of created files {filename: category}
@return: (str, dict) result_file, and dictionary of created files.
@arg the first element is the name of the main ETPI or ETPAI result file to be further processed.
@arg the second element is a dictionary that lists the names of all created data files with their category.
the dictionary key is the file name,
the value is the file category (cluster, phase, etc.).
"""
return None, None
def check_cluster(self, cluster, output_file):
"""
export the cluster in XYZ format for reference.
along with the complete cluster, the method also saves cuts in the xz (extension .y.xyz) and yz (.x.xyz) plane.
@param cluster: a pmsco.cluster.Cluster() object with all atom positions set.
@param output_file: base name for all intermediate and output files
@return: dictionary listing the names of the created files with their category.
the dictionary key is the file name,
the value is the file category (cluster).
@warning experimental: this method may be moved elsewhere in a future version.
"""
xyz_filename = output_file + ".xyz"
cluster.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files = {xyz_filename: 'cluster'}
clucut = mc.Cluster()
clucut.copy_from(cluster)
clucut.trim_slab("x", 0.0, 0.1)
xyz_filename = output_file + ".x.xyz"
clucut.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files[xyz_filename] = 'cluster'
clucut.copy_from(cluster)
clucut.trim_slab("y", 0.0, 0.1)
xyz_filename = output_file + ".y.xyz"
clucut.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
files[xyz_filename] = 'cluster'
return files
class TestCalculator(Calculator):
"""
@ -127,5 +98,5 @@ class TestCalculator(Calculator):
md.save_data(etpi_filename, result_etpi)
files = {clu_filename: 'cluster', etpi_filename: 'energy'}
files = {clu_filename: 'cluster', etpi_filename: 'region'}
return etpi_filename, files

94
pmsco/edac_calculator.py → pmsco/calculators/edac.py
View File

@ -1,25 +1,30 @@
"""
@package pmsco.edac_calculator
@package pmsco.calculators.edac
Garcia de Abajo EDAC program interface.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
import os
from __future__ import print_function
import logging
import math
import numpy as np
import calculator
import data as md
import cluster as mc
import edac.edac as edac
import os
import pmsco.calculators.calculator as calculator
from pmsco.compat import open
import pmsco.data as md
import pmsco.cluster as mc
import pmsco.edac.edac as edac
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -44,18 +49,19 @@ class EdacCalculator(calculator.Calculator):
if alpha is defined, theta is implicitly set to normal emission! (to be generalized)
TODO: some parameters are still hard-coded.
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param filepath: (str) name and path of the file to be created.
"""
with open(filepath, "w") as f:
f.write("verbose off\n")
f.write("cluster input %s\n" % (params.cluster_file))
f.write("emitters %u l(A)\n" % (len(params.emitters)))
f.write("cluster input {0}\n".format(params.cluster_file))
f.write("emitters {0:d} l(A)\n".format(len(params.emitters)))
for em in params.emitters:
f.write("%g %g %g %u\n" % em)
#for iat in range(params.atom_types):
#pf = params.phase_file[iat]
#pf = pf.replace(".pha", ".edac.pha")
#f.write("scatterer %u %s\n" % (params.atomic_number[iat], pf))
f.write("{0:f} {1:f} {2:f} {3:d}\n".format(*em))
en = scan.energies + params.work_function
en_min = en.min()
@ -106,7 +112,7 @@ class EdacCalculator(calculator.Calculator):
assert th_num < th.shape[0] * 10, \
"linearization of theta scan causes excessive oversampling {0}/{1}".format(th_num, th.shape[0])
f.write("beta {0}\n".format(params.polar_incidence_angle, params.azimuthal_incidence_angle))
f.write("beta {0}\n".format(params.polar_incidence_angle))
f.write("incidence {0} {1}\n".format(params.polar_incidence_angle, params.azimuthal_incidence_angle))
f.write("emission angle theta {th0:f} {th1:f} {nth:d}\n".format(th0=th_min, th1=th_max, nth=th_num))
@ -136,35 +142,49 @@ class EdacCalculator(calculator.Calculator):
f.write("initial state {0}\n".format(params.initial_state))
polarizations = {'H': 'LPx', 'V': 'LPy', 'L': 'LCP', 'R': 'RCP'}
f.write("polarization {0}\n".format(polarizations[params.polarization]))
scatterers = ["scatterer {at} {fi}\n".format(at=at, fi=fi)
for (at, fi) in params.phase_files.items()
if os.path.isfile(fi)]
if scatterers:
for scat in scatterers:
f.write(scat)
else:
f.write("muffin-tin\n")
f.write("V0 E(eV) {0}\n".format(params.inner_potential))
f.write("cluster surface l(A) {0}\n".format(params.z_surface))
f.write("V0 E(eV) {0:f}\n".format(params.inner_potential))
f.write("cluster surface l(A) {0:f}\n".format(params.z_surface))
f.write("imfp SD-UC\n")
f.write("temperature %g %g\n" % (params.experiment_temperature, params.debye_temperature))
f.write("temperature {0:f} {1:f}\n".format(params.experiment_temperature, params.debye_temperature))
f.write("iteration recursion\n")
f.write("dmax l(A) %g\n" % (params.dmax))
f.write("lmax %u\n" % (params.lmax))
f.write("orders %u " % (len(params.orders)))
for order in params.orders:
f.write("%u " % (order))
f.write("\n")
f.write("emission angle window 1\n")
f.write("scan pd %s\n" % (params.output_file))
f.write("dmax l(A) {0:f}\n".format(params.dmax))
f.write("lmax {0:d}\n".format(params.lmax))
f.write("orders {0:d} ".format(len(params.orders)))
f.write(" ".join(format(order, "d") for order in params.orders) + "\n")
f.write("emission angle window {0:F}\n".format(params.angular_resolution / 2.0))
# f.write("cluster output l(A) out.clu")
# problems:
# - muffin-tin relabels atoms
# - there can be multiple atom types for the same chemical element
# - we have to compare coordinates to find the mapping between input and output cluster
# f.write("scan scatterer i phase-shifts i.pha")
# f.write("scan scatterer i potential i.pot")
f.write("scan pd {0}\n".format(params.output_file))
f.write("end\n")
def run(self, params, cluster, scan, output_file):
"""
run EDAC with the given parameters and cluster.
@param params: a msc_param.Params() object with all necessary values except cluster and output files set.
@param params: a pmsco.project.Params object with all necessary values except cluster and output files set.
@param cluster: a msc_cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param cluster: a pmsco.cluster.Cluster(format=FMT_EDAC) object with all atom positions set.
@param scan: a msco_project.Scan() object describing the experimental scanning scheme.
@param scan: a pmsco.project.Scan() object describing the experimental scanning scheme.
@param output_file: base name for all intermediate and output files
@return: result_file, files_cats
@return: (str, dict) result_file, and dictionary of created files {filename: category}
"""
# set up scan
@ -213,8 +233,14 @@ class EdacCalculator(calculator.Calculator):
pass
result_etpi = md.interpolate_hemi_scan(result_etpi, hemi_tpi)
if result_etpi.shape[0] != scan.raw_data.shape[0]:
logger.error("scan length mismatch: EDAC result: %u, scan data: %u", result_etpi.shape[0], scan.raw_data.shape[0])
if params.fixed_cluster:
expected_shape = max(scan.energies.shape[0], 1) * max(scan.alphas.shape[0], 1)
else:
expected_shape = max(scan.energies.shape[0], 1) * max(scan.phis.shape[0], scan.thetas.shape[0], 1)
if result_etpi.shape[0] != expected_shape:
logger.warning(BMsg("possible scan length mismatch: EDAC result: {result}, expected: {expected}",
result=result_etpi.shape[0], expected=expected_shape))
logger.debug("save result to file %s", etpi_filename)
md.save_data(etpi_filename, result_etpi)

11
pmsco/msc_calculator.py → pmsco/calculators/msc.py
View File

@ -13,9 +13,12 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
import calculator
import data as md
import msc.msc as msc
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import pmsco.calculators.calculator as calculator
import pmsco.data as md
import pmsco.msc.msc as msc
import logging
logger = logging.getLogger(__name__)
@ -32,7 +35,7 @@ class MscCalculator(calculator.Calculator):
f.write(" %s\n" % (params.polarization) )
f.write(" %4u\n" % (params.scattering_level) )
f.write(" %7.2f%7.2f\n" % (params.fcut, params.cut) )
f.write(" %12.6f\n" % (params.angular_broadening) )
f.write(" %12.6f\n" % (params.angular_resolution) )
f.write(" %12.6f\n" % (params.lattice_constant) )
f.write(" %12.6f\n" % (params.z_surface) )
f.write(" %4u\n" % (params.atom_types) )

247
pmsco/cluster.py
View File

@ -14,12 +14,17 @@ pip install --user periodictable
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut
@copyright (c) 2015-18 by Paul Scherrer Institut
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import math
import numpy as np
import periodictable as pt
import sys
## default file format identifier
FMT_DEFAULT = 0
@ -30,8 +35,15 @@ FMT_EDAC = 2
## XYZ file format identifier
FMT_XYZ = 3
# python version dependent type of chemical symbol
if sys.version_info[0] >= 3:
_SYMBOL_TYPE = 'U2'
else:
_SYMBOL_TYPE = 'S2'
## numpy.array datatype of Cluster.data array
DTYPE_CLUSTER_INTERNAL = [('i','i4'), ('t','i4'), ('s','a2'), ('x','f4'), ('y','f4'), ('z','f4'), ('e','u1')]
DTYPE_CLUSTER_INTERNAL = [('i', 'i4'), ('t', 'i4'), ('s', _SYMBOL_TYPE), ('x', 'f4'), ('y', 'f4'), ('z', 'f4'),
('e', 'u1')]
## file format of internal Cluster.data array
FMT_CLUSTER_INTERNAL = ["%5u", "%2u", "%s", "%7.3f", "%7.3f", "%7.3f", "%1u"]
## field (column) names of internal Cluster.data array
@ -52,7 +64,7 @@ FMT_CLUSTER_EDAC = ["%5u", "%2u", "%7.3f", "%7.3f", "%7.3f"]
FIELDS_CLUSTER_EDAC = ['i', 't', 'x', 'y', 'z']
## numpy.array datatype of cluster for XYZ file input/output
DTYPE_CLUSTER_XYZ= [('s','a2'), ('x','f4'), ('y','f4'), ('z','f4')]
DTYPE_CLUSTER_XYZ= [('s', _SYMBOL_TYPE), ('x', 'f4'), ('y', 'f4'), ('z', 'f4')]
## file format of XYZ cluster file
FMT_CLUSTER_XYZ = ["%s", "%10.5f", "%10.5f", "%10.5f"]
## field (column) names of XYZ cluster file
@ -137,7 +149,7 @@ class Cluster(object):
"""
Copy the data from another cluster.
@param cluster (Cluster): other Cluster object.
@param cluster: (Cluster) other Cluster object.
"""
self.data = cluster.data.copy()
@ -164,10 +176,10 @@ class Cluster(object):
@param x, y, z: (float) atom coordinates in the cluster
@param emitter: (uint) 1 = emitter, 0 = regular
@param emitter: (int or bool) True = emitter, False = scatterer
"""
symbol = pt.elements[element_number].symbol
element = (index, element_number, symbol, x, y, z, emitter)
element = (index, element_number, symbol, x, y, z, int(emitter))
return element
def add_atom(self, atomtype, v_pos, is_emitter):
@ -178,10 +190,10 @@ class Cluster(object):
@param v_pos: (numpy.ndarray, shape = (3)) position vector
@param is_emitter: (uint) 1 = emitter, 0 = regular
@param is_emitter: (int or bool) True = emitter, False = scatterer
"""
n0 = self.data.shape[0] + 1
element = self.build_element(n0, atomtype, v_pos[0], v_pos[1], v_pos[2], is_emitter)
element = self.build_element(n0, atomtype, v_pos[0], v_pos[1], v_pos[2], int(is_emitter))
self.data = np.append(self.data, np.array(element,
dtype=self.data.dtype))
@ -342,6 +354,29 @@ class Cluster(object):
return idx
def translate(self, vector, element=0):
"""
translate the cluster or all atoms of a specified element.
@param vector: (numpy.ndarray) 3-dimensional displacement vector.
@param element: (int) chemical element number if atoms of a specific element should be affected.
by default (element = 0), all atoms are moved.
@return: (numpy.ndarray) indices of the atoms that have been shifted.
"""
if element:
try:
sel = self.data['t'] == int(element)
except ValueError:
sel = self.data['s'] == element
else:
sel = np.ones_like(self.data['t'])
idx = np.where(sel)
self.data['x'][idx] += vector[0]
self.data['y'][idx] += vector[1]
self.data['z'][idx] += vector[2]
return idx
def matrix_transform(self, matrix):
"""
apply a transformation matrix to each atom of the cluster.
@ -474,6 +509,31 @@ class Cluster(object):
self.data = self.data[idx]
self.update_index()
def trim_paraboloid(self, rxy, z0):
"""
remove atoms outside a given paraboloid.
the paraboloid is defined by z(r) = z0 * (1 - r**2 / rxy**2),
where r**2 = x**2 + y**2.
its apex is at (0, 0, z0).
z(r) = 0 for x**2 + y**2 = rxy**2.
the coordinates (x,y,z) of atoms to keep must match z >= z(r).
@param rxy: (float) radius of the paraboloid at z = 0.
@param z0: (float) vertical coordinate of the apex (at x = y = 0).
in the usual cluster layout where the surface is near z = 0 and the atoms at lower z,
this value is negative.
@return: None
"""
rsq = self.data['x']**2 + self.data['y']**2
pz = z0 * (1. - rsq / rxy**2)
idx = np.where(self.data['z'] >= pz)
self.data = self.data[idx]
self.update_index()
def trim_sphere(self, radius):
"""
remove atoms outside a given sphere.
@ -637,7 +697,7 @@ class Cluster(object):
"""
idx = self.data['e'] != 0
ems = self.data[['x', 'y', 'z', 't']][idx]
return map(tuple, ems)
return [tuple(em) for em in ems]
def get_emitter_count(self):
"""
@ -702,7 +762,7 @@ class Cluster(object):
else:
self.data['e'] = 0
pos = self.positions()
pos = self.get_positions()
# note: np.linalg.norm does not accept axis argument in version 1.7
# (check np.version.version)
norm = np.sqrt(np.sum(pos**2, axis=1))
@ -739,13 +799,14 @@ class Cluster(object):
"""
self.data['i'] = np.arange(1, self.data.shape[0] + 1)
def save_to_file(self, f, fmt=FMT_DEFAULT, comment=""):
def save_to_file(self, f, fmt=FMT_DEFAULT, comment="", emitters_only=False):
"""
save the cluster to a file which can be read by the scattering program.
the method updates the atom index because some file formats require an index column.
@param f: (string/handle) path name or open file handle of the cluster file.
if the filename ends in .gz, the file is saved in compressed gzip format
@param fmt: (int) file format.
must be one of the FMT_ constants.
@ -755,7 +816,10 @@ class Cluster(object):
not used in other file formats.
by default, self.comment is used.
@remark if the filename ends in .gz, the file is saved in compressed gzip format
@param emitters_only: (bool) if True, only atoms marked as emitters are saved.
by default, all atoms are saved.
@return None
"""
if fmt == FMT_DEFAULT:
fmt = self.file_format
@ -763,6 +827,13 @@ class Cluster(object):
if not comment:
comment = self.comment
self.update_index()
if emitters_only:
idx = self.data['e'] != 0
data = self.data[idx]
else:
data = self.data
if fmt == FMT_MSC:
file_format = FMT_CLUSTER_MSC
fields = FIELDS_CLUSTER_MSC
@ -770,16 +841,158 @@ class Cluster(object):
elif fmt == FMT_EDAC:
file_format = FMT_CLUSTER_EDAC
fields = FIELDS_CLUSTER_EDAC
header = "%u l(A)" % (self.data.shape[0])
header = "{nat} l(A)".format(nat=data.shape[0])
elif fmt == FMT_XYZ:
file_format = FMT_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
header = "{0}\n{1}".format(self.data.shape[0], comment)
header = "{nat}\n{com}".format(nat=data.shape[0], com=comment)
else:
file_format = FMT_CLUSTER_XYZ
fields = FIELDS_CLUSTER_XYZ
header = "{0}\n{1}".format(self.data.shape[0], comment)
header = "{nat}\n{com}".format(nat=data.shape[0], com=comment)
self.update_index()
data = self.data[fields]
data = data[fields]
np.savetxt(f, data, fmt=file_format, header=header, comments="")
class ClusterGenerator(object):
"""
cluster generator class.
this class bundles the cluster methods in one place
so that it's easier to exchange them for different kinds of clusters.
the project must override at least the create_cluster method.
if emitters should be run in parallel tasks, the count_emitters method must be implemented as well.
"""
def __init__(self, project):
"""
initialize the cluster generator.
@param project: reference to the project object.
cluster generators may need to look up project parameters.
"""
self.project = project
def count_emitters(self, model, index):
"""
return the number of emitter configurations for a particular model, scan and symmetry.
the number of emitter configurations may depend on the model parameters, scan index and symmetry index.
by default, the method returns 1, which means that there is only one emitter configuration.
emitter configurations are mainly a way to distribute the calculations to multiple processes
since the resulting diffraction patterns add up incoherently.
for this to work, the create_cluster() method must pay attention to the emitter index
and generate either a full cluster with all emitters (single process)
or a cluster with only a subset of the emitters according to the emitter index (multiple processes).
whether all emitters are calculated in one or multiple processes is decided at run-time
based on the available resources.
note that this function returns the number of _configurations_ not _atoms_.
an emitter configuration (declared in a Cluster) may include more than one atom.
it is up to the project, what is included in a particular configuration.
to enable multiple emitter configurations, the derived project class must override this method
and return a number greater than 1.
@note in some cases it may be most efficient to call create_cluster and
return Cluster.get_emitter_count() of the generated cluster.
this is possible because the method is called with emitter index -1.
model and index can be passed unchanged to create_cluster.
@param model (dictionary) model parameters to be used in the calculation.
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index must be -1.
@return number of emitter configurations.
this implementation returns the default value of 1.
"""
return 1
def create_cluster(self, model, index):
"""
create a Cluster object given the model parameters and calculation index.
the generated cluster will typically depend on the model parameters.
depending on the project, it may also depend on the scan index, symmetry index and emitter index.
the scan index can be used to generate a different cluster for different scan geometry,
e.g., if some atoms can be excluded due to a longer mean free path.
if this is not the case for the specific project, the scan index can be ignored.
the symmetry index may select a particular domain that has a different atomic arrangement.
in this case, depending on the value of index.sym, the function must generate a cluster corresponding
to the particular domain/symmetry.
the method can ignore the symmetry index if the project defines only one symmetry,
or if the symmetry does not correspond to a different atomic structure.
the emitter index selects a particular emitter configuration.
depending on the value of the emitter index, the method must react differently:
1. if the value is -1, return the full cluster and mark all inequivalent emitter atoms.
emitters which are reproduced by a symmetry expansion in combine_emitters() should not be marked.
the full diffraction scan will be calculated in one calculation.
2. if the value is greater or equal to zero, generate the cluster with the emitter configuration
selected by the emitter index.
the index is in the range between 0 and the return value of count_emitters() minus 1.
the results of the individual emitter calculations are summed up in combine_emitters().
the code should ideally be written such that either case yields the same diffraction result.
if count_emitters() always returns 1 (default), the second case does not have to be implemented,
and the method can ignore the emitter index.
the method must ignore the region index.
if the creation of a cluster fails due to invalid model parameters,
rather than raising an exception or constraining values,
the method should send an error message to the logger and return an empty cluster or None.
@param model (dictionary) model parameters to be used in the calculation.
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index.
if -1, generate the full cluster and mark all emitters.
if greater or equal to zero, the value is a zero-based index of the emitter configuration.
@return None.
sub-classes must return a valid Cluster object or None in case of failure.
"""
return None
class LegacyClusterGenerator(ClusterGenerator):
"""
cluster generator class for projects that don't declare a generator.
in previous versions, the create_cluster and count_emitters methods were implemented by the project class.
this class redirects generator calls to the project methods
providing compatibility to older project code.
"""
def __init__(self, project):
super(LegacyClusterGenerator, self).__init__(project)
def count_emitters(self, model, index):
"""
redirect the call to the corresponding project method if implemented.
"""
try:
return self.project.count_emitters(model, index)
except AttributeError:
return 1
def create_cluster(self, model, index):
"""
redirect the call to the corresponding project method.
"""
return self.project.create_cluster(model, index)

40
pmsco/compat.py Normal file
View File

@ -0,0 +1,40 @@
"""
@package pmsco.compat
compatibility code
code bits to provide compatibility for different python versions.
currently supported 2.7 and 3.6.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from io import open as io_open
def open(fname, mode='r', encoding='latin1'):
"""
open a data file for read/write/append using the default str type
this is a drop-in for io.open
where data is exchanged via the built-in str type of python,
whether this is a byte string (python 2) or unicode string (python 3).
the file is assumed to be a latin-1 encoded binary file.
@param fname: file name and path
@param mode: 'r', 'w' or 'a'
@param encoding: 'latin1' (default), 'ascii' or 'utf-8'
@return file handle
"""
if isinstance(b'b', str):
# python 2
mode += 'b'
kwargs = {}
else:
# python 3
mode += 't'
kwargs = {'encoding': encoding}
return io_open(fname, mode, **kwargs)

59
pmsco/data.py
View File

@ -1,21 +1,29 @@
"""
@package pmsco.data
import, export, evaluation of msc data
import, export, evaluation of msc data.
this module provides common functions for loading/saving and manipulating PED scan data sets.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-17 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import os
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import numpy as np
import os
import scipy.optimize as so
import loess.loess as loess
from pmsco.compat import open
import pmsco.loess.loess as loess
logger = logging.getLogger(__name__)
@ -149,7 +157,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
data[i]['i'] = selected intensity column
@endverbatim
"""
with open(filename, 'r') as f:
with open(filename, "r") as f:
header1 = f.readline().strip()
header2 = f.readline().strip()
if not header1 == '--- scan PD':
@ -259,19 +267,38 @@ def load_data(filename, dtype=None):
DTYPE_EI, DTYPE_ETPI, DTYPE_ETPIS, DTYPE_ETPAI, or DTYPE_ETPAIS.
by default, the function uses the extension to determine the data type.
the actual type can be read from the dtype attribute of the returned array.
if the extension is missing, DTYPE_EI is assumed.
@return one-dimensional numpy structured ndarray with data
@raise IOError if the file cannot be read.
@raise IndexError if the number of columns is lower than expected based on the dtype or extension.
"""
if not dtype:
(root, ext) = os.path.splitext(filename)
datatype = ext[1:].upper()
dtype = DTYPES[datatype]
ext_type = ext[1:].upper()
try:
dtype = DTYPES[ext_type]
except KeyError:
dtype = DTYPE_EI
data = np.loadtxt(filename, dtype=dtype)
sort_data(data)
return data
def format_extension(data):
"""
format the file extension based on the contents of an array.
@param data ETPI-like structured numpy.ndarray.
@return: file extension string including the leading period.
"""
return "." + "".join(data.dtype.names)
def save_data(filename, data):
"""
save column data (ETPI, and the like) to a text file.
@ -331,13 +358,17 @@ def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
undefined fields are initialized to zero.
if the parameter is unspecified, all fields are initialized to zero.
@return: re-structured numpy array
@return: re-structured numpy array or
@c data if the new and original data types are the same.
"""
if data.dtype == dtype:
return data
else:
new_data = np.zeros(data.shape, dtype=dtype)
fields = [dt[0] for dt in dtype if dt[0] in data.dtype.names]
if defaults is not None:
for field, value in defaults.iteritems():
for field, value in defaults.items():
if field in new_data.dtype.names:
new_data[field] = value
@ -584,7 +615,7 @@ def calc_modfunc_mean(data):
return modf
def calc_modfunc_loess(data):
def calc_modfunc_loess(data, smth=0.4):
"""
calculate the modulation function using LOESS (locally weighted regression) smoothing.
@ -609,9 +640,11 @@ def calc_modfunc_loess(data):
the modulation function is calculated for the finite-valued scan points.
NaNs are ignored and do not affect the finite values.
@return copy of the data array with the modulation function in the 'i' column.
@param smth: size of the smoothing window relative to the size of the scan.
reasonable values are between 0.2 and 0.5.
the default value 0.4 has been found to work in many cases.
@todo is a fixed smoothing factor of 0.5 okay?
@return copy of the data array with the modulation function in the 'i' column.
"""
sel = np.isfinite(data['i'])
_data = data[sel]
@ -626,7 +659,7 @@ def calc_modfunc_loess(data):
factors = [_data[axis] for axis in scan_mode]
lo.set_x(np.hstack(tuple(factors)))
lo.set_y(_data['i'])
lo.model.span = 0.5
lo.model.span = smth
loess.loess(lo)
modf['i'][sel] = lo.get_fitted_residuals() / lo.get_fitted_values()

1312
pmsco/database.py Normal file
View File

File diff suppressed because it is too large Load Diff

442
pmsco/dispatch.py
View File

@ -11,7 +11,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import os.path
import datetime
@ -19,8 +21,9 @@ import signal
import collections
import copy
import logging
from attrdict import AttrDict
from mpi4py import MPI
from helpers import BraceMessage as BMsg
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -48,7 +51,99 @@ TAG_INVALID_RESULT = 3
## the message is empty
TAG_ERROR_ABORTING = 4
CalcID = collections.namedtuple('CalcID', ['model', 'scan', 'sym', 'emit', 'region'])
## levels of calculation tasks
#
CALC_LEVELS = ('model', 'scan', 'sym', 'emit', 'region')
## intermediate sub-class of CalcID
#
# this class should not be instantiated.
# instead, use CalcID which provides some useful helper methods.
#
_CalcID = collections.namedtuple('_CalcID', CALC_LEVELS)
class CalcID(_CalcID):
"""
named tuple class to uniquely identify a calculation task.
this is a 5-tuple of indices, one index per task level.
a positive index refers to a specific instance in the task hierarchy.
the ID is defined as a named tuple so that it can be used as key of a dictionary.
cf. @ref CalculationTask for further detail.
compared to a plain named tuple, the CalcID class provides additional helper methods and properties.
example constructor: CalcID(1, 2, 3, 4, 5).
"""
@property
def levels(self):
"""
level names.
this property returns the defined level names in a tuple.
this is the same as @ref CALC_LEVELS.
@return: tuple of level names
"""
return self._fields
@property
def level(self):
"""
specific level of a task, dictionary key form.
this corresponds to the name of the last positive component.
@return: attribute name corresponding to the task level.
empty string if all members are negative (the root task).
"""
for k in reversed(self._fields):
if self.__getattribute__(k) >= 0:
return k
return ''
@property
def numeric_level(self):
"""
specific level of a task, numeric form.
this corresponds to the last positive value in the sequence of indices.
@return: index corresponding to the significant task level component of the id.
the value ranges from -1 to len(CalcID) - 1.
it is -1 if all indices are negative (root task).
"""
for k in reversed(range(len(self))):
if self[k] >= 0:
return k
return -1
def collapse_levels(self, level):
"""
return a new CalcID that is collapsed at a specific level.
the method returns a new CalcID object where the indices below the given level are -1 (undefined).
this can be seen as collapsing the tree at the specified node (level).
@note because a CalcID is immutable, this method returns a new instance.
@param level: level at which to collapse.
the index at this level remains unchanged, lower ones are set to -1.
the level can be specified by attribute name (str) or numeric index (-1..4).
@raise ValueError if level is not numeric and not in CALC_LEVELS.
@return: new CalcID instance.
"""
try:
level = int(level)
except ValueError:
level = CALC_LEVELS.index(level)
assert -1 <= level < len(CALC_LEVELS)
mask = {l: -1 for (i, l) in enumerate(CALC_LEVELS) if i > level}
return self._replace(**mask)
class CalculationTask(object):
@ -71,11 +166,11 @@ class CalculationTask(object):
specified members must be greater or equal to zero.
-1 is the wildcard which is used in parent tasks,
where, e.g., no specific symmetry is chosen.
the root task has the ID (-1, -1, -1, -1).
the root task has the ID (-1, -1, -1, -1, -1).
"""
## @var id (CalcID)
# named tuple CalcID containing the 4-part calculation task identifier.
# named tuple CalcID containing the 5-part calculation task identifier.
## @var parent_id (CalcID)
# named tuple CalcID containing the task identifier of the parent task.
@ -105,6 +200,21 @@ class CalculationTask(object):
## @var modf_filename (string)
# name of the ETPI or ETPAI file that contains the resulting modulation function.
## @var result_valid (bool)
# validity status of the result file @ref result_filename.
#
# if True, the file must exist and contain valid data according to the task specification.
# the value is set True when a calculation task completes successfully.
# it may be reset later to invalidate the data if an error occurs during processing.
#
# validity of a parent task requires validity of all child tasks.
## @var rfac (float)
# r-factor value of the task result.
#
# the rfac field is written by @ref pmsco.project.Project.evaluate_result.
# the initial value is Not A Number.
## @var time (timedelta)
# execution time of the task.
#
@ -125,7 +235,7 @@ class CalculationTask(object):
# scan positions to substitute the ones from the original scan.
#
# this is used to distribute scans over multiple calculator processes,
# cf. e.g. @ref EnergyRegionHandler.
# cf. e.g. @ref pmsco.handlers.EnergyRegionHandler.
#
# dictionary key must be the scan dimension 'e', 't', 'p', 'a'.
# the value is a numpy.ndarray containing the scan positions.
@ -147,6 +257,7 @@ class CalculationTask(object):
self.time = datetime.timedelta()
self.files = {}
self.region = {}
self.rfac = float('nan')
def __eq__(self, other):
"""
@ -192,7 +303,7 @@ class CalculationTask(object):
msg['id'] = CalcID(**msg['id'])
if isinstance(msg['parent_id'], dict):
msg['parent_id'] = CalcID(**msg['parent_id'])
for k, v in msg.iteritems():
for k, v in msg.items():
self.__setattr__(k, v)
def format_filename(self, **overrides):
@ -204,13 +315,8 @@ class CalculationTask(object):
@return a string consisting of the concatenation of the base name, the ID, and the extension.
"""
parts = {}
parts = self.id._asdict()
parts['root'] = self.file_root
parts['model'] = self.id.model
parts['scan'] = self.id.scan
parts['sym'] = self.id.sym
parts['emit'] = self.id.emit
parts['region'] = self.id.region
parts['ext'] = self.file_ext
for key in overrides.keys():
@ -237,6 +343,30 @@ class CalculationTask(object):
"""
self.id = self.id._replace(**kwargs)
@property
def level(self):
"""
specific level of a task, dictionary key form.
this corresponds to the name of the last positive component of self.id.
@return: attribute name corresponding to the task level.
empty string for the root task.
"""
return self.id.level
@property
def numeric_level(self):
"""
specific level of a task, numeric form.
this corresponds to the index of the last positive component of self.id.
@return: index corresponding to the significant task level component of the id.
-1 for the root task.
"""
return self.id.numeric_level
def add_task_file(self, name, category):
"""
register a file that was generated by the calculation task.
@ -284,6 +414,82 @@ class CalculationTask(object):
logger.warning("CalculationTask.remove_task_file: could not remove file {0}".format(filename))
class CachedCalculationMethod(object):
"""
decorator to cache results of expensive calculation functions.
this decorator can be used to transparently cache any expensive calculation result
that depends in a deterministic way on the calculation index.
for example, each cluster gets a unique calculation index.
if a cluster needs to be calculated repeatedly, it may be more efficient to cache it.
the method to decorate must have the following signature:
result = func(self, model, index).
the index (neglecting emitter and region) identifies the result (completely and uniquely).
the number of cached results is limited by the ttl (time to live) attribute.
the items' ttl values are decreased each time a requested calculation is not found in the cache (miss).
on a cache hit, the corresponding item's ttl is reset.
the target ttl (time to live) can be specified as an optional parameter of the decorator.
time increases with every cache miss.
"""
## @var _cache (dict)
#
# key = calculation index,
# value = function result
## @var _ttl (dict)
#
# key = calculation index,
# value (int) = remaining time to live
# where time is the number of subsequent cache misses.
## @var ttl (int)
#
# target time to live of cache items.
# time is given by the number cache misses.
def __init__(self, ttl=10):
super(CachedCalculationMethod, self).__init__()
self._cache = {}
self._ttl = {}
self.ttl = ttl
def __call__(self, func):
def wrapped_func(inst, model, index):
# note: _replace returns a new instance of the namedtuple
index = index._replace(emit=-1, region=-1)
cache_index = (id(inst), index.model, index.scan, index.sym)
try:
result = self._cache[cache_index]
except KeyError:
result = func(inst, model, index)
self._expire()
self._cache[cache_index] = result
self._ttl[cache_index] = self.ttl
return result
return wrapped_func
def _expire(self):
"""
decrease the remaining ttl of cache items and delete items whose ttl has fallen below 0.
@return: None
"""
for index in self._ttl:
self._ttl[index] -= 1
old_items = [index for index in self._ttl if self._ttl[index] < 0]
for index in old_items:
del self._ttl[index]
del self._cache[index]
class MscoProcess(object):
"""
code shared by MscoMaster and MscoSlave.
@ -357,11 +563,9 @@ class MscoProcess(object):
"""
clean up after all calculations.
this method calls the clean up function of the project.
@return: None
"""
self._project.cleanup()
pass
def calc(self, task):
"""
@ -387,14 +591,36 @@ class MscoProcess(object):
logger.info("model %s", s_model)
start_time = datetime.datetime.now()
# create parameter and cluster structures
clu = self._project.cluster_generator.create_cluster(task.model, task.id)
par = self._project.create_params(task.model, task.id)
# generate file names
clu = self._create_cluster(task)
par = self._create_params(task)
scan = self._define_scan(task)
output_file = task.format_filename(ext="")
# determine scan range
# check parameters and call the msc program
if clu.get_atom_count() < 2:
logger.error("empty cluster in calculation %s", s_id)
task.result_valid = False
elif clu.get_emitter_count() < 1:
logger.error("no emitters in cluster of calculation %s.", s_id)
task.result_valid = False
else:
task.result_filename, files = self._calculator.run(par, clu, scan, output_file)
(root, ext) = os.path.splitext(task.result_filename)
task.file_ext = ext
task.result_valid = True
task.files.update(files)
task.time = datetime.datetime.now() - start_time
return task
def _define_scan(self, task):
"""
define the scan range.
@param task: CalculationTask with all attributes set for the calculation.
@return: pmsco.project.Scan object for the calculator.
"""
scan = self._project.scans[task.id.scan]
if task.region:
scan = scan.copy()
@ -419,26 +645,56 @@ class MscoProcess(object):
except KeyError:
pass
# check parameters and call the msc program
if clu.get_atom_count() < 2:
logger.error("empty cluster in calculation %s", s_id)
task.result_valid = False
elif clu.get_emitter_count() < 1:
logger.error("no emitters in cluster of calculation %s.", s_id)
task.result_valid = False
else:
files = self._calculator.check_cluster(clu, output_file)
return scan
def _create_cluster(self, task):
"""
generate the cluster for the given calculation task.
cluster generation is delegated to the project's cluster_generator object.
if the current task has region == 0,
the method also exports diagnostic clusters via the project's export_cluster() method.
the file name is formatted with the given task index except that region is -1.
if (in addition to region == 0) the current task has emit == 0 and cluster includes multiple emitters,
the method also exports the master cluster and full emitter list.
the file name is formatted with the given task index except that emitter and region are -1.
@param task: CalculationTask with all attributes set for the calculation.
@return: pmsco.cluster.Cluster object for the calculator.
"""
nem = self._project.cluster_generator.count_emitters(task.model, task.id)
clu = self._project.cluster_generator.create_cluster(task.model, task.id)
if task.id.region == 0:
file_index = task.id._replace(region=-1)
filename = task.format_filename(region=-1)
files = self._project.export_cluster(file_index, filename, clu)
task.files.update(files)
task.result_filename, files = self._calculator.run(par, clu, scan, output_file)
(root, ext) = os.path.splitext(task.result_filename)
task.file_ext = ext
task.result_valid = True
# master cluster
if nem > 1 and task.id.emit == 0:
master_index = task.id._replace(emit=-1, region=-1)
filename = task.format_filename(emit=-1, region=-1)
master_cluster = self._project.cluster_generator.create_cluster(task.model, master_index)
files = self._project.export_cluster(master_index, filename, master_cluster)
task.files.update(files)
task.time = datetime.datetime.now() - start_time
return clu
return task
def _create_params(self, task):
"""
generate the parameters list.
parameters generation is delegated to the project's create_params method.
@param task: CalculationTask with all attributes set for the calculation.
@return: pmsco.project.Params object for the calculator.
"""
par = self._project.create_params(task.model, task.id)
return par
class MscoMaster(MscoProcess):
@ -505,20 +761,12 @@ class MscoMaster(MscoProcess):
# it defines the initial model and the output file name.
# it is passed to the model handler during the main loop.
# @var _model_handler
# (ModelHandler) model handler instance
# @var _scan_handler
# (ScanHandler) scan handler instance
# @var _symmetry_handler
# (SymmetryHandler) symmetry handler instance
# @var _emitter_handler
# (EmitterHandler) emitter handler instance
# @var _region_handler
# (RegionHandler) region handler instance
## @var task_handlers
# (AttrDict) dictionary of task handler objects
#
# the keys are the task levels 'model', 'scan', 'sym', 'emit' and 'region'.
# the values are handlers.TaskHandler objects.
# the objects can be accessed in attribute or dictionary notation.
def __init__(self, comm):
super(MscoMaster, self).__init__(comm)
@ -534,11 +782,8 @@ class MscoMaster(MscoProcess):
self._min_queue_len = self._slaves + 1
self._root_task = None
self._model_handler = None
self._scan_handler = None
self._symmetry_handler = None
self._emitter_handler = None
self._region_handler = None
self.task_levels = list(CalcID._fields)
self.task_handlers = AttrDict()
def setup(self, project):
"""
@ -564,30 +809,29 @@ class MscoMaster(MscoProcess):
logger.debug("master entering setup")
self._running_slaves = self._slaves
self._idle_ranks = range(1, self._running_slaves + 1)
self._idle_ranks = list(range(1, self._running_slaves + 1))
self._root_task = CalculationTask()
self._root_task.file_root = project.output_file
self._root_task.model = project.create_domain().start
self._model_handler = project.handler_classes['model']()
self._scan_handler = project.handler_classes['scan']()
self._symmetry_handler = project.handler_classes['symmetry']()
self._emitter_handler = project.handler_classes['emitter']()
self._region_handler = project.handler_classes['region']()
for level in self.task_levels:
self.task_handlers[level] = project.handler_classes[level]()
self._model_handler.datetime_limit = self.datetime_limit
self.task_handlers.model.datetime_limit = self.datetime_limit
slaves_adj = max(self._slaves, 1)
self._model_handler.setup(project, slaves_adj)
self.task_handlers.model.setup(project, slaves_adj)
if project.mode != "single":
slaves_adj = max(slaves_adj / 2, 1)
self._scan_handler.setup(project, slaves_adj)
self._symmetry_handler.setup(project, slaves_adj)
self._emitter_handler.setup(project, slaves_adj)
self.task_handlers.scan.setup(project, slaves_adj)
self.task_handlers.sym.setup(project, slaves_adj)
self.task_handlers.emit.setup(project, slaves_adj)
if project.mode != "single":
slaves_adj = min(slaves_adj, 4)
self._region_handler.setup(project, slaves_adj)
self.task_handlers.region.setup(project, slaves_adj)
project.setup(self.task_handlers)
def run(self):
"""
@ -615,14 +859,13 @@ class MscoMaster(MscoProcess):
logger.debug("master exiting main loop")
self._running = False
self._save_report()
def cleanup(self):
logger.debug("master entering cleanup")
self._region_handler.cleanup()
self._emitter_handler.cleanup()
self._symmetry_handler.cleanup()
self._scan_handler.cleanup()
self._model_handler.cleanup()
for level in reversed(self.task_levels):
self.task_handlers[level].cleanup()
self._project.cleanup()
super(MscoMaster, self).cleanup()
def _dispatch_results(self):
@ -632,29 +875,25 @@ class MscoMaster(MscoProcess):
logger.debug("dispatching results of %u tasks", len(self._complete_tasks))
while self._complete_tasks:
__, task = self._complete_tasks.popitem(last=False)
self._dispatch_result(task)
logger.debug("passing task %s to region handler", str(task.id))
task = self._region_handler.add_result(task)
def _dispatch_result(self, task):
"""
pass a result through the post-processing modules.
@param task: a CalculationTask object.
@return None
"""
level = task.level
if level:
logger.debug(BMsg("passing task {task} to {level} handler", task=str(task.id), level=level))
task = self.task_handlers[level].add_result(task)
if task:
logger.debug("passing task %s to emitter handler", str(task.id))
task = self._emitter_handler.add_result(task)
if task:
logger.debug("passing task %s to symmetry handler", str(task.id))
task = self._symmetry_handler.add_result(task)
if task:
logger.debug("passing task %s to scan handler", str(task.id))
task = self._scan_handler.add_result(task)
if task:
logger.debug("passing task %s to model handler", str(task.id))
task = self._model_handler.add_result(task)
if task:
logger.debug("root task %s complete", str(task.id))
self._dispatch_result(task)
else:
self._finishing = True
logger.debug(BMsg("root task {task} complete", task=str(task.id)))
def _create_tasks(self):
"""
@ -668,7 +907,7 @@ class MscoMaster(MscoProcess):
"""
logger.debug("creating new tasks from root")
while len(self._pending_tasks) < self._min_queue_len:
tasks = self._model_handler.create_tasks(self._root_task)
tasks = self.task_handlers.model.create_tasks(self._root_task)
logger.debug("model handler returned %u new tasks", len(tasks))
if not tasks:
self._model_done = True
@ -807,6 +1046,17 @@ class MscoMaster(MscoProcess):
return self._finishing
def _save_report(self):
"""
generate a final report.
this method is called at the end of the master loop.
it passes the call to @ref pmsco.handlers.ModelHandler.save_report.
@return: None
"""
self.task_handlers.model.save_report(self._root_task)
def add_model_task(self, task):
"""
add a new model task including all of its children to the task queue.
@ -814,13 +1064,13 @@ class MscoMaster(MscoProcess):
@param task (CalculationTask) task identifier and model parameters.
"""
scan_tasks = self._scan_handler.create_tasks(task)
scan_tasks = self.task_handlers.scan.create_tasks(task)
for scan_task in scan_tasks:
sym_tasks = self._symmetry_handler.create_tasks(scan_task)
sym_tasks = self.task_handlers.sym.create_tasks(scan_task)
for sym_task in sym_tasks:
emitter_tasks = self._emitter_handler.create_tasks(sym_task)
emitter_tasks = self.task_handlers.emit.create_tasks(sym_task)
for emitter_task in emitter_tasks:
region_tasks = self._region_handler.create_tasks(emitter_task)
region_tasks = self.task_handlers.region.create_tasks(emitter_task)
for region_task in region_tasks:
self._pending_tasks[region_task.id] = region_task

3
pmsco/edac/.gitignore vendored
View File

@ -1,3 +1,2 @@
edac_all_wrap.*
edac.py
edac_wrap.cxx
revision.py

15224
pmsco/edac/edac_all.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

7
pmsco/edac/edac_all.i Normal file
View File

@ -0,0 +1,7 @@
/* EDAC interface for other programs */
%module edac
%{
extern int run_script(char *scriptfile);
%}
extern int run_script(char *scriptfile);

143
pmsco/edac/edac_all.patch
View File

@ -1,8 +1,18 @@
*** /home/muntwiler_m/mnt/pearl_data/software/edac/edac_all.cpp 2011-04-14 23:38:44.000000000 +0200
--- edac_all.cpp 2016-02-11 12:15:45.322049772 +0100
--- edac_all.cpp 2018-02-05 17:30:17.347373088 +0100
***************
*** 3085,3090 ****
--- 3085,3091 ----
numero Vxc_Barth_Hedin(numero den, numero denup)
{
if(den<1e-10) return 0;
+ if(denup<0) denup=0;
numero rs=1/pow(4*pi/3*den, 1/3.0);
numero x=denup/den;
numero alpha0=pow(4/(9*pi), 1/3.0);
***************
*** 10117,10122 ****
--- 10117,10123 ----
--- 10118,10124 ----
void scan_imfp(char *name);
void scan_imfp(FILE *fout);
numero iimfp_TPP(numero kr);
@ -12,7 +22,7 @@
int scattering_so;
***************
*** 10230,10235 ****
--- 10231,10237 ----
--- 10232,10238 ----
int n_th;
int n_fi;
@ -22,7 +32,7 @@
numero *th_out,
***************
*** 10239,10244 ****
--- 10241,10247 ----
--- 10242,10248 ----
void free(void);
void init_th(numero thi, numero thf, int nth);
void init_phi(numero fii, numero fif, int nfi);
@ -31,8 +41,121 @@
numero refraction);
void init_transmission(
***************
*** 10905,10942 ****
}
numero calculation::IIIthfi(int no, numero th, numero fi)
{
! numero ii,ii0, xth,xfi;
! numero th0=final.th[0], th1=final.th[final.n_th-1];
numero fi0=final.fi[0], fi1=final.fi[final.n_fi-1];
int ith,ifi,ij;
while(fi<0) fi+=2*pi;
while(fi>2*pi) fi-=2*pi;
! xth=(final.n_th-1)*(th-th0)/(th1-th0); ith=int(floor(xth-0.001));
! xfi=(final.n_fi-1)*(fi-fi0)/(fi1-fi0); ifi=int(floor(xfi-0.001));
if(ifi==-1) ifi=0;
if(0<=ith && ith<final.n_th-1 && 0<=ifi && ifi<final.n_fi-1) {
ij=no*n_ang+ith*final.n_fi+ifi;
! ii0=III0[ij];
! ii=ii0 + (xth-ith)*(III0[ij+final.n_fi]-ii0) + (xfi-ifi)*(III0[ij+1]-ii0);
} else ii=0;
if(ii<0) ii=0;
return ii;
}
! numero calculation::IIIave(int no, numero th, numero fi)
{
! if(thave<=1e-6) return IIIthfi(no,th,fi);
int i,j, nn=10, mm=50;
! numero tth,ffi,val=0, r,f;
for(i=0; i<nn; i++)
for(j=0; j<mm; j++) {
! r=i*thave/nn;
! f=j*2*pi/mm;
! tth=th+r*cos(f);
! ffi=fi+r*sin(f)/cos(pi*th/180);
! val+=IIIthfi(no,tth,ffi);
}
! return val/(nn*mm);
}
void calculation::write_ang(FILE *fout_, int ik)
{
int no,nno,i,j;
--- 10909,10963 ----
}
numero calculation::IIIthfi(int no, numero th, numero fi)
{
! numero ii,xth,xfi;
! numero th0=final.th_out[0], th1=final.th_out[final.n_th-1];
numero fi0=final.fi[0], fi1=final.fi[final.n_fi-1];
int ith,ifi,ij;
while(fi<0) fi+=2*pi;
while(fi>2*pi) fi-=2*pi;
! xth=(final.n_th-1)*(th-th0)/(th1-th0); ith=int(floor(xth-0.0001));
! xfi=(final.n_fi-1)*(fi-fi0)/(fi1-fi0); ifi=int(floor(xfi-0.0001));
if(ifi==-1) ifi=0;
+ if(ith==-1) ith=0;
if(0<=ith && ith<final.n_th-1 && 0<=ifi && ifi<final.n_fi-1) {
ij=no*n_ang+ith*final.n_fi+ifi;
! xth = xth-ith;
! xfi = xfi-ifi;
! ii=III0[ij]*(1-xth)*(1-xfi) + III0[ij+final.n_fi]*xth*(1-xfi) + III0[ij+1]*(1-xth)*xfi + III0[ij+final.n_fi+1]*xth*xfi;
} else ii=0;
if(ii<0) ii=0;
return ii;
}
! numero calculation::IIIave(int no, numero th, numero ph)
{
! if(thave<=1e-6) return IIIthfi(no,th,ph);
int i,j, nn=10, mm=50;
! numero tth, ffi, val=0, th1, ph1, cf, nw=0, x0, y0, z0, x1, y1, th2, ph2;
for(i=0; i<nn; i++)
for(j=0; j<mm; j++) {
! th1=i*2*thave/(nn-1); //2*sigma range
! ph1=j*2*pi/mm;
! //rotation of (001) around Y by th1 and around Z by ph1
! x0 = cos(ph1)*sin(th1);
! y1 = sin(ph1)*sin(th1);
! z0 = cos(th1);
! //rotation around Y by th
! x1 = x0*cos(th) + z0*sin(th);
! z0 = -x0*sin(th) + z0*cos(th);
! //rotation around Z by ph
! x0 = x1*cos(ph) - y1*sin(ph);
! y0 = x1*sin(ph) + y1*cos(ph);
!
! th2 = acos(z0);
! ph2 = atan2(y0,x0);
!
! cf = exp(-(th1*th1/thave*thave))*(i>0.1?i:0.1); //gauss weight * radial weight
! nw += cf; //sum of weights
! val+=IIIthfi(no,th2,ph2)*cf;
}
! return val/nw;
}
+
void calculation::write_ang(FILE *fout_, int ik)
{
int no,nno,i,j;
***************
*** 10961,10967 ****
for(no=0; no<nno; no++)
for(i=0; i<final.n_th; i++)
for(j=0; j<final.n_fi; j++)
! III[no*n_ang+i*final.n_fi+j]=IIIave(no, final.th[i], final.fi[j]);
delete [] III0;
}
for(i=0; i<final.n_th; i++)
--- 10982,10988 ----
for(no=0; no<nno; no++)
for(i=0; i<final.n_th; i++)
for(j=0; j<final.n_fi; j++)
! III[no*n_ang+i*final.n_fi+j]=IIIave(no, final.th_out[i], final.fi[j]);
delete [] III0;
}
for(i=0; i<final.n_th; i++)
***************
*** 12485,12490 ****
--- 12488,12494 ----
--- 12506,12512 ----
else {
kr=sqrt(sqr(calc.k[ik])+2*V0);
if(iimfp_flag==0) ki=iimfp.val(kr)/2;
@ -42,7 +165,7 @@
} } else if(calc.k_flag==2) set_k(calc.kc[ik]);
***************
*** 12507,12512 ****
--- 12511,12522 ----
--- 12529,12540 ----
numero imfp=E/(TPP_Ep*TPP_Ep*(beta*log(gamma*E)-C/E+D/(E*E)))/a0_au;
return 1/imfp;
}
@ -64,7 +187,7 @@
n_1=n_2=0;
Ylm0_th_flag=Ylm0_fi_flag=0;
mesh_flag=0;
--- 13212,13218 ----
--- 13230,13236 ----
}
final_state::final_state(void)
{
@ -74,7 +197,7 @@
mesh_flag=0;
***************
*** 13233,13238 ****
--- 13243,13271 ----
--- 13261,13289 ----
if(n_fi==1) fi[0]=fii;
else for(j=0; j<n_fi; j++) fi[j]=fii+j*(fif-fii)/(n_fi-1);
} }
@ -106,7 +229,7 @@
int i;
***************
*** 14743,14748 ****
--- 14776,14783 ----
--- 14794,14801 ----
|| scat.TPP_Ep<=0 || scat.TPP_Eg<0)
on_error(foutput,"(input) imfp TPP-2M", "wrong parameters");
scat.iimfp_flag=1;
@ -117,7 +240,7 @@
scat.iimfp_flag=0;
***************
*** 15162,15164 ****
--- 15197,15206 ----
--- 15215,15224 ----
fprintf(foutput,"That's all, folks!\n");
return 0;
}

33
pmsco/edac/makefile
View File

@ -13,28 +13,23 @@ SHELL=/bin/sh
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac
FC=gfortran
FCCOPTS=
F2PY=f2py
F2PYOPTS=
CC=g++
CCOPTS=-Wno-write-strings
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHONOPTS=
FC?=gfortran
FCCOPTS?=
F2PY?=f2py
F2PYOPTS?=
CXX?=g++
CXXOPTS?=-Wno-write-strings
PYTHON?=python
PYTHONOPTS?=
all: edac
edac: edac.exe _edac.so edac.py
edac.exe: edac_all.cpp
$(CC) $(CCOPTS) -o edac.exe edac_all.cpp
$(CXX) $(CXXOPTS) -o edac.exe edac_all.cpp
edac_wrap.cxx: edac_all.cpp edac.i
$(SWIG) $(SWIGOPTS) -c++ -python edac.i
edac.py _edac.so: edac_wrap.cxx setup.py
edac.py _edac.so: edac_all.cpp edac_all.i setup.py
$(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
revision.py: _edac.so
@ -46,7 +41,7 @@ revision.txt: _edac.so edac.exe
echo "" >> revision.txt
clean:
rm -f *.so *.o *.exe
rm -f *_wrap.cxx
rm -f revision.py
rm -f revision.txt
rm -f *.so *.o *.exe *.pyc
rm -f edac.py edac_all_wrap.*
rm -f revision.*

7
pmsco/edac/setup.py
View File

@ -8,7 +8,8 @@ from distutils.core import setup, Extension
edac_module = Extension('_edac',
sources=['edac_wrap.cxx', 'edac_all.cpp'],
sources=['edac_all.cpp', 'edac_all.i'],
swig_opts=['-c++']
)
setup (name = 'edac',
@ -16,5 +17,7 @@ setup (name = 'edac',
author = "Matthias Muntwiler",
description = """EDAC module in Python""",
ext_modules = [edac_module],
py_modules = ["edac"], requires=['numpy']
py_modules = ["edac"],
requires=['numpy']
)

241
pmsco/files.py
View File

@ -1,16 +1,19 @@
"""
@package pmsco.files
manage files produced by pmsco.
manage the lifetime of files produced by pmsco.
@author Matthias Muntwiler
@copyright (c) 2016 by Paul Scherrer Institut @n
@copyright (c) 2016-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import logging
import mpi4py
@ -53,7 +56,7 @@ FILE_CATEGORIES = {'cluster', 'phase', 'input', 'output',
#
# this constant defines the default set of file categories that are kept after the calculation.
#
FILE_CATEGORIES_TO_KEEP = {'cluster', 'model', 'report', 'population'}
FILE_CATEGORIES_TO_KEEP = {'cluster', 'model', 'scan', 'report', 'population'}
## @var FILE_CATEGORIES_TO_DELETE
# categories of files to be deleted.
@ -67,13 +70,17 @@ FILE_CATEGORIES_TO_DELETE = FILE_CATEGORIES - FILE_CATEGORIES_TO_KEEP
class FileTracker(object):
"""
organize output files of calculations.
manage the lifetime of files produced by the calculations.
the file manager stores references to data files generated during calculations
and cleans up unused files according to a range of filter criteria.
this class identifies files by _file name_.
file names must therefore be unique over the whole calculation process.
it is possible to specify a full path that is used for communication with the operating system.
"""
## @var files_to_delete (set)
## @var categories_to_delete (set)
# categories of generated files that should be deleted after the calculation.
#
# each string of this set marks a category of files to be deleted.
@ -93,96 +100,119 @@ class FileTracker(object):
#
# the default is 10.
## @var _last_id (int)
# last used file identification number (incremental)
## @var _file_model (dict)
# key = file name, value = model number
## @var _path_by_id (dict)
# key = file id, value = file path
## @var _file_category (dict)
# key = file name, value = category (str)
## @var _model_by_id (dict)
# key = file id, value = model number
## @var _category_by_id (dict)
# key = file id, value = category (str)
## @var _file_path (dict)
# key = file name, value = absolute file path (str)
## @var _rfac_by_model (dict)
# key = model number, value = file id
# key = model number, value = R-factor
## @var _complete_by_model (dict)
# key = model number, value (boolean) = all calculations complete, files can be deleted
## @var _complete_models (set)
# this set contains the model numbers of the models that have finished all calculations.
# files of these models can be considered for clean up.
def __init__(self):
self._id_by_path = {}
self._path_by_id = {}
self._model_by_id = {}
self._category_by_id = {}
self._file_model = {}
self._file_category = {}
self._file_path = {}
self._rfac_by_model = {}
self._complete_by_model = {}
self._last_id = 0
self._complete_models = set([])
self.categories_to_delete = FILE_CATEGORIES_TO_DELETE
self.keep_rfac = 10
def add_file(self, path, model, category='default'):
def get_file_count(self):
"""
return the number of tracked files.
@return: (int) number of tracked files.
"""
return len(self._file_path)
def get_complete_models_count(self):
"""
return the number of complete models.
@return: (int) number of complete models.
"""
return len(self._complete_models)
def add_file(self, name, model, category='default', path=''):
"""
add a new data file to the list.
@param path: (str) system path of the file relative to the working directory.
@param name: (str) unique identification of the file.
this can be the file name in the file system if file names are unique without path specification.
the name must be spelled identically
whenever the same file is referenced in a call to another method of this class.
the empty string is ignored.
@param model: (int) model number
@param category: (str) file category, e.g. 'output', etc.
@param path: (str) file system path of the file.
the file system path is used for communication with the operating system when the file is deleted.
by default, the path is the name argument expanded to a full path relative to the current working directory.
the path is expanded during the call of this method and will not change when the working directory changes.
@return: None
"""
self._last_id += 1
_id = self._last_id
self._id_by_path[path] = _id
self._path_by_id[_id] = path
self._model_by_id[_id] = model
self._category_by_id[_id] = category
if name:
self._file_model[name] = model
self._file_category[name] = category
self._file_path[name] = path if path else os.path.abspath(name)
def rename_file(self, old_path, new_path):
def rename_file(self, old_name, new_name, new_path=''):
"""
rename a data file in the list.
the method does not rename the file in the file system.
@param old_path: must match an existing file path identically.
if old_path is not in the list, the method does nothing.
@param old_name: name used in the original add_file() call.
if it is not in the list, the method does nothing.
@param new_path: new path.
@param new_name: new name of the file, see add_file().
if the file is already in the list, its model and category is overwritten by the values of the old file.
@param new_path: new file system path of the file, see add_file().
by default, the path is the name argument expanded to a full path relative to the current working directory.
@return: None
"""
try:
_id = self._id_by_path[old_path]
model = self._file_model[old_name]
cat = self._file_category[old_name]
except KeyError:
pass
else:
del self._id_by_path[old_path]
self._id_by_path[new_path] = _id
self._path_by_id[_id] = new_path
del self._file_model[old_name]
del self._file_category[old_name]
del self._file_path[old_name]
self.add_file(new_name, model, cat, new_path)
def remove_file(self, path):
def remove_file(self, name):
"""
remove a file from the list.
the method does not delete the file from the file system.
@param path: must match an existing file path identically.
if path is not in the list, the method does nothing.
@param name: must match an existing file name identically.
if the name is not found in the list, the method does nothing.
@return: None
"""
try:
_id = self._id_by_path[path]
del self._file_model[name]
del self._file_category[name]
del self._file_path[name]
except KeyError:
pass
else:
del self._id_by_path[path]
del self._path_by_id[_id]
del self._model_by_id[_id]
del self._category_by_id[_id]
def update_model_rfac(self, model, rfac):
"""
@ -207,18 +237,19 @@ class FileTracker(object):
@param complete: (bool) True if all calculations of the model are complete (files can be deleted).
@return: None
"""
self._complete_by_model[model] = complete
if complete:
self._complete_models.add(model)
else:
self._complete_models.discard(model)
def delete_files(self, categories=None, keep_rfac=0):
def delete_files(self, categories=None):
"""
delete the files matching the list of categories.
@param categories: set of file categories to delete.
may include 'rfac' if bad r-factors should be deleted additionally (regardless of static category).
defaults to self.categories_to_delete.
@version this method does not act on the 'rfac' category.
@param keep_rfac: number of best models to keep if bad r-factors are to be deleted.
the effective keep number is the greater of self.keep_rfac and this argument.
@param categories: set of file categories to delete.
defaults to self.categories_to_delete.
@return: None
"""
@ -226,8 +257,6 @@ class FileTracker(object):
categories = self.categories_to_delete
for cat in categories:
self.delete_category(cat)
if 'rfac' in categories:
self.delete_bad_rfac(keep=keep_rfac)
def delete_bad_rfac(self, keep=0, force_delete=False):
"""
@ -252,8 +281,6 @@ class FileTracker(object):
@param force_delete: delete the bad files even if 'rfac' is not selected in categories_to_delete.
@return: None
@todo should clean up rfac and model dictionaries from time to time.
"""
if force_delete or 'rfac' in self.categories_to_delete:
keep = max(keep, self.keep_rfac)
@ -263,12 +290,45 @@ class FileTracker(object):
except IndexError:
return
complete_models = {_model for (_model, _complete) in self._complete_by_model.iteritems() if _complete}
del_models = {_model for (_model, _rfac) in self._rfac_by_model.iteritems() if _rfac > rfac_split}
del_models &= complete_models
del_ids = {_id for (_id, _model) in self._model_by_id.iteritems() if _model in del_models}
for _id in del_ids:
self.delete_file(_id)
keep_models = {model for (model, rfac) in self._rfac_by_model.items() if 0.0 <= rfac <= rfac_split}
del_models = self._complete_models - keep_models
del_names = {name for (name, model) in self._file_model.items() if model in del_models}
for name in del_names:
self.delete_file(name)
def delete_models(self, keep=None, delete=None):
"""
delete all files by model.
this involves the following steps:
1. determine a list of complete models
(incomplete models are still being processed and must not be deleted).
2. intersect with the _delete_ list if specified.
3. subtract the _keep_ list if specified.
if neither the _keep_ nor the _delete_ list is specified,
or if the steps above resolve to the _complete_ list
the method considers it as an error and does nothing.
@param keep: (sequence) model numbers to keep, i.e., delete all others.
@param delete: (sequence) model numbers to delete.
@return (int) number of models deleted.
"""
del_models = self._complete_models.copy()
if delete:
del_models &= delete
if keep:
del_models -= keep
if not del_models or del_models == self._complete_models:
return 0
del_names = {name for (name, model) in self._file_model.items() if model in del_models}
for name in del_names:
self.delete_file(name)
return len(del_models)
def delete_category(self, category):
"""
@ -280,45 +340,38 @@ class FileTracker(object):
@return: None
"""
complete_models = {_model for (_model, _complete) in self._complete_by_model.iteritems() if _complete}
del_ids = {_id for (_id, cat) in self._category_by_id.iteritems() if cat == category}
del_ids &= {_id for (_id, _model) in self._model_by_id.iteritems() if _model in complete_models}
for _id in del_ids:
self.delete_file(_id)
del_names = {name for (name, cat) in self._file_category.items() if cat == category}
del_names &= {name for (name, model) in self._file_model.items() if model in self._complete_models}
for name in del_names:
self.delete_file(name)
def delete_file(self, _id):
def delete_file(self, name):
"""
delete a specified file from the list and the file system.
the file is identified by ID number.
this method is unconditional. it does not consider category, completeness, nor R-factor.
@param _id: (int) ID number of the file to delete.
the method catches errors during file deletion and prints warnings to the logger.
@param name: must match an existing file path identically.
if it is not in the list, the method does nothing.
the method uses the associated path declared in add_file() to delete the file.
@return: None
"""
path = self._path_by_id[_id]
cat = self._category_by_id[_id]
model = self._model_by_id[_id]
del self._id_by_path[path]
del self._path_by_id[_id]
del self._model_by_id[_id]
del self._category_by_id[_id]
try:
self._os_delete_file(path)
cat = self._file_category[name]
model = self._file_model[name]
path = self._file_path[name]
except KeyError:
logger.warning("tried to delete untracked file {0}".format(name))
else:
del self._file_model[name]
del self._file_category[name]
del self._file_path[name]
try:
os.remove(path)
except OSError:
logger.warning("error deleting file {0}".format(path))
logger.warning("file system error deleting file {0}".format(path))
else:
logger.debug("delete file {0} ({1}, model {2})".format(path, cat, model))
@staticmethod
def _os_delete_file(path):
"""
have the operating system delete a file path.
this function is separate so that we can mock it in unit tests.
@param path: OS path
@return: None
"""
os.remove(path)

0
pmsco/graphics/__init__.py Normal file
View File

198
pmsco/graphics/rfactor.py Normal file
View File

@ -0,0 +1,198 @@
"""
@package pmsco.graphics.rfactor
graphics rendering module for r-factor optimization results.
this module is under development.
interface and implementation are subject to change.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
def render_param_rfac(filename, data, param_name, summary=None, canvas=None):
"""
render an r-factor versus one model parameter graph.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the results file.
this is used to derive the output file path by adding the parameter name and
the extension of the graphics file format.
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
@param param_name: name of the model parameter to display.
this must correspond to a field name of the data array.
@param summary: (dict) the dictionary returned by @ref evaluate_results.
this is used to mark the optimum value and the error limits.
if None, these values are not marked in the plot.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
ax.scatter(data[param_name], data['_rfac'], c='b', marker='o', s=4.0)
if summary is not None:
xval = summary['val'][param_name]
ymin = summary['vmin']['_rfac']
ymax = summary['vmax']['_rfac']
ax.plot((xval, xval), (ymin, ymax), ':k')
xmin = summary['vmin'][param_name]
xmax = summary['vmax'][param_name]
varr = summary['rmin'] + summary['rvar']
ax.plot((xmin, xmax), (varr, varr), ':k')
ax.grid(True)
ax.set_xlabel(param_name)
ax.set_ylabel('R-factor')
out_filename = "{0}.{1}.{2}".format(filename, param_name, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def evaluate_results(data, features=50.):
"""
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
@param features: number of independent features (pieces of information) in the data.
this quantity can be approximated as the scan range divided by the average width of a feature
which includes an intrinsic component and the instrumental resolution.
see Booth et al., Surf. Sci. 387 (1997), 152 for energy scans, and
Muntwiler et al., Surf. Sci. 472 (2001), 125 for angle scans.
the default value of 50 is a typical value.
@return dictionary of evaluation results.
the dictionary contains scalars and structured arrays as follows.
the structured arrays have the same data type as the input data and contain exactly one element.
@arg rmin: (scalar) minimum r-factor.
@arg rvar: (scalar) one-sigma variation of r-factor.
@arg imin: (scalar) array index where the minimum is located.
@arg val: (structured array) estimates of parameter values (parameter value at rmin).
@arg sig: (structured array) one-sigma error of estimated values.
@arg vmin: (structured array) minimum value of the parameter.
@arg vmax: (structured array) maximum value of the parameter.
"""
imin = data['_rfac'].argmin()
rmin = data['_rfac'][imin]
rvar = rmin * math.sqrt(2. / float(features))
val = np.zeros(1, dtype=data.dtype)
sig = np.zeros(1, dtype=data.dtype)
vmin = np.zeros(1, dtype=data.dtype)
vmax = np.zeros(1, dtype=data.dtype)
sel = data['_rfac'] <= rmin + rvar
for name in data.dtype.names:
val[name] = data[name][imin]
vmin[name] = data[name].min()
vmax[name] = data[name].max()
if name[0] != '_':
sig[name] = (data[name][sel].max() - data[name][sel].min()) / 2.
results = {'rmin': rmin, 'rvar': rvar, 'imin': imin, 'val': val, 'sig': sig, 'vmin': vmin, 'vmax': vmax}
return results
def render_results(results_file, data=None):
"""
produce a graphics file from optimization results.
the results can be passed in a file name or numpy array (see parameter section).
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function will log a warning message and return gracefully.
@param results_file: path and name of the result file.
result files are the ones written by swarm.SwarmPopulation.save_array, for instance.
the file contains columns of model parameters and optimization control values.
the first row must contain column names that identify the quantity.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
if the optional data parameter is present,
this is used only to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of results (one-dimensional).
the field names identify the model parameters and optimization control values.
model parameters can have any name not including a leading underscore and are evaluated as is.
the names of the special optimization control values begin with an underscore.
of these, at least _rfac must be provided.
if this argument is omitted, the data is loaded from the file referenced by the filename argument.
@return (list of str) path names of the generated graphics files.
empty if an error occurred.
the most common exceptions are caught and add a warning in the log file.
"""
if data is None:
data = np.genfromtxt(results_file, names=True)
summary = evaluate_results(data)
out_files = []
try:
for name in data.dtype.names:
if name[0] != '_' and summary['sig'][name] > 0.:
graph_file = render_param_rfac(results_file, data, name, summary)
out_files.append(graph_file)
except (TypeError, AttributeError, IOError) as e:
logger.warning(BMsg("error rendering scan file {file}: {msg}", file=results_file, msg=str(e)))
return out_files

273
pmsco/graphics/scan.py Normal file
View File

@ -0,0 +1,273 @@
"""
@package pmsco.graphics.scan
graphics rendering module for energy and angle scans.
this module is experimental.
interface and implementation are subject to change.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import math
import numpy as np
import pmsco.data as md
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
try:
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# from matplotlib.backends.backend_pdf import FigureCanvasPdf
# from matplotlib.backends.backend_svg import FigureCanvasSVG
except ImportError:
Figure = None
FigureCanvas = None
logger.warning("error importing matplotlib. graphics rendering disabled.")
def render_1d_scan(filename, data, scan_mode, canvas=None, is_modf=False):
"""
produce a graphics file from a one-dimensional scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of EI, ETPI or ETPAI data.
@param scan_mode: list containing the field name of the scanning axis of the data array.
it must contain one element exactly.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to set axis labels.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
ax.plot(data[scan_mode[0]], data['i'])
ax.set_xlabel(scan_mode[0])
if is_modf:
ax.set_ylabel('chi')
else:
ax.set_ylabel('int')
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_ea_scan(filename, data, scan_mode, canvas=None, is_modf=False):
"""
produce a graphics file from an energy-angle scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of ETPI or ETPAI data.
@param scan_mode: list containing the field names of the scanning axes of the data array,
i.e. 'e' and one of the angle axes.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to select a suitable color scale.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
(data2d, axis0, axis1) = md.reshape_2d(data, scan_mode, 'i')
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111)
im = ax.imshow(data2d, origin='lower', aspect='auto', interpolation='none')
im.set_extent((axis1[0], axis1[-1], axis0[0], axis0[-1]))
ax.set_xlabel(scan_mode[1])
ax.set_ylabel(scan_mode[0])
cb = fig.colorbar(im, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(data['i'], 1)
dhi = np.nanpercentile(data['i'], 99)
if is_modf:
im.set_cmap("RdBu_r")
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
im.set_clim((dlo, dhi))
try:
# requires matplotlib 2.1.0
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
else:
im.set_cmap("magma")
im.set_clim((dlo, dhi))
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_tp_scan(filename, data, canvas=None, is_modf=False):
"""
produce a graphics file from an theta-phi (hemisphere) scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function raises an error.
@param filename: path and name of the scan file.
this is used to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of TPI data.
the T and P columns describes a full or partial hemispherical scan.
the I column contains the intensity or modulation values.
other columns are ignored.
@param canvas: a FigureCanvas class reference from a matplotlib backend.
if None, the default FigureCanvasAgg is used which produces a bitmap file in PNG format.
@param is_modf: whether data contains a modulation function (True) or intensity (False, default).
this parameter is used to select a suitable color scale.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
@raise TypeError if matplotlib is not available.
"""
if canvas is None:
canvas = FigureCanvas
fig = Figure()
canvas(fig)
ax = fig.add_subplot(111, projection='polar')
data = data[data['t'] <= 89.0]
# stereographic projection
rd = 2 * np.tan(np.radians(data['t']) / 2)
drdt = 1 + np.tan(np.radians(data['t']) / 2)**2
# http://matplotlib.org/api/collections_api.html#matplotlib.collections.PathCollection
pc = ax.scatter(data['p'] * math.pi / 180., rd, c=data['i'], lw=0, alpha=1.)
# interpolate marker size between 4 and 9 (for theta step = 1)
unique_theta = np.unique(data['t'])
theta_step = (np.max(unique_theta) - np.min(unique_theta)) / unique_theta.shape[0]
sz = np.ones_like(pc.get_sizes()) * drdt * 4.5 * theta_step**2
pc.set_sizes(sz)
# xticks = angles where grid lines are displayed (in radians)
ax.set_xticks([])
# rticks = radii where grid lines (circles) are displayed
ax.set_rticks([])
ax.set_rmax(2.0)
cb = fig.colorbar(pc, shrink=0.4, pad=0.1)
dlo = np.nanpercentile(data['i'], 2)
dhi = np.nanpercentile(data['i'], 98)
if is_modf:
pc.set_cmap("RdBu_r")
# im.set_cmap("coolwarm")
dhi = max(abs(dlo), abs(dhi))
dlo = -dhi
pc.set_clim((dlo, dhi))
try:
# requires matplotlib 2.1.0
ti = cb.get_ticks()
ti = [min(ti), 0., max(ti)]
cb.set_ticks(ti)
except AttributeError:
pass
else:
pc.set_cmap("magma")
# im.set_cmap("inferno")
# im.set_cmap("viridis")
pc.set_clim((dlo, dhi))
ti = cb.get_ticks()
ti = [min(ti), max(ti)]
cb.set_ticks(ti)
out_filename = "{0}.{1}".format(filename, canvas.get_default_filetype())
fig.savefig(out_filename)
return out_filename
def render_scan(filename, data=None):
"""
produce a graphics file from a scan file.
the default file format is PNG.
this function requires the matplotlib module.
if it is not available, the function will log a warning message and return gracefully.
@param filename: path and name of the scan file.
the file must have one of the formats supported by pmsco.data.load_data().
it must contain a single scan (not the combined scan from the model level of PMSCO).
supported are all one-dimensional linear scans,
and two-dimensional energy-angle scans (each axis must be linear).
hemispherical scans are currently not supported.
the filename should include ".modf" if the data contains a modulation function rather than intensity.
if the optional data parameter is present,
this is used only to derive the output file path by adding the extension of the graphics file format.
@param data: numpy-structured array of ETPI or ETPAI data.
if this argument is omitted, the data is loaded from the file referenced by the filename argument.
@return (str) path and name of the generated graphics file.
empty string if an error occurred.
"""
if data is None:
data = md.load_data(filename)
scan_mode, scan_positions = md.detect_scan_mode(data)
is_modf = filename.find(".modf") >= 0
try:
if len(scan_mode) == 1:
out_filename = render_1d_scan(filename, data, scan_mode, is_modf=is_modf)
elif len(scan_mode) == 2 and 'e' in scan_mode:
out_filename = render_ea_scan(filename, data, scan_mode, is_modf=is_modf)
elif len(scan_mode) == 2 and 't' in scan_mode and 'p' in scan_mode:
out_filename = render_tp_scan(filename, data, is_modf=is_modf)
else:
out_filename = ""
logger.warning(BMsg("no render function for scan file {file}", file=filename))
except (TypeError, AttributeError, IOError) as e:
out_filename = ""
logger.warning(BMsg("error rendering scan file {file}: {msg}", file=filename, msg=str(e)))
return out_filename

135
pmsco/handlers.py
View File

@ -40,21 +40,28 @@ the scan and symmetry handlers call methods of the project class to invoke proje
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-17 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import datetime
import os
from functools import reduce
import logging
import math
import numpy as np
import data as md
from helpers import BraceMessage as BMsg
import os
from pmsco.compat import open
import pmsco.data as md
import pmsco.graphics.scan as mgs
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -66,10 +73,10 @@ class TaskHandler(object):
this class defines the common interface of task handlers.
"""
## @var project
## @var _project
# (Project) project instance.
## @var slots
## @var _slots
# (int) number of calculation slots (processes).
#
# for best efficiency the number of tasks generated should be greater or equal the number of slots.
@ -93,7 +100,7 @@ class TaskHandler(object):
# the dictionary keys are the task identifiers CalculationTask.id,
# the values are the corresponding CalculationTask objects.
## @var invalid_count (int)
## @var _invalid_count (int)
# accumulated total number of invalid results received.
#
# the number is incremented by add_result if an invalid task is reported.
@ -188,21 +195,22 @@ class TaskHandler(object):
the id, model, and files attributes are required.
if model contains a '_rfac' value, the r-factor is
@return: None
@return None
"""
model_id = task.id.model
for path, cat in task.files.iteritems():
for path, cat in task.files.items():
self._project.files.add_file(path, model_id, category=cat)
def cleanup_files(self, keep=10):
def cleanup_files(self, keep=0):
"""
delete uninteresting files.
@param: number of best ranking models to keep.
@param keep: minimum number of models to keep.
0 (default): leave the decision to the project.
@return: None
@return None
"""
self._project.files.delete_files(keep_rfac=keep)
self._project.cleanup_files(keep=keep)
class ModelHandler(TaskHandler):
@ -255,6 +263,22 @@ class ModelHandler(TaskHandler):
return None
def save_report(self, root_task):
"""
generate a final report of the optimization procedure.
detailed calculation results are usually saved as soon as they become available.
this method may be implemented in sub-classes to aggregate and summarize the results, generate plots, etc.
in this class, the method does nothing.
@note: implementations must add the path names of generated files to self._project.files.
@param root_task: (CalculationTask) task with initial model parameters.
@return: None
"""
pass
class SingleModelHandler(ModelHandler):
"""
@ -263,6 +287,10 @@ class SingleModelHandler(ModelHandler):
this class runs a single calculation on the start parameters defined in the domain of the project.
"""
def __init__(self):
super(SingleModelHandler, self).__init__()
self.result = {}
def create_tasks(self, parent_task):
"""
start one task with the start parameters.
@ -316,25 +344,18 @@ class SingleModelHandler(ModelHandler):
modf_ext = ".modf" + parent_task.file_ext
parent_task.modf_filename = parent_task.file_root + modf_ext
rfac = 1.0
if task.result_valid:
try:
rfac = self._project.calc_rfactor(task)
except ValueError:
task.result_valid = False
logger.warning(BMsg("calculation of model {0} resulted in an undefined R-factor.", task.id.model))
assert not math.isnan(task.rfac)
self.result = task.model.copy()
self.result['_rfac'] = task.rfac
task.model['_rfac'] = rfac
self.save_report_file(task.model)
self._project.files.update_model_rfac(task.id.model, rfac)
self._project.files.update_model_rfac(task.id.model, task.rfac)
self._project.files.set_model_complete(task.id.model, True)
parent_task.time = task.time
return parent_task
def save_report_file(self, result):
def save_report(self, root_task):
"""
save model parameters and r-factor to a file.
@ -343,20 +364,25 @@ class SingleModelHandler(ModelHandler):
the first line contains the parameter names.
this is the same format as used by the swarm and grid handlers.
@param result: dictionary of results and parameters. the values should be scalars and strings.
@param root_task: (CalculationTask) the id.model attribute is used to register the generated files.
@return: None
"""
keys = [key for key in result]
super(SingleModelHandler, self).save_report(root_task)
keys = [key for key in self.result]
keys.sort(key=lambda t: t[0].lower())
vals = (str(result[key]) for key in keys)
with open(self._project.output_file + ".dat", "w") as outfile:
vals = (str(self.result[key]) for key in keys)
filename = self._project.output_file + ".dat"
with open(filename, "w") as outfile:
outfile.write("# ")
outfile.write(" ".join(keys))
outfile.write("\n")
outfile.write(" ".join(vals))
outfile.write("\n")
self._project.files.add_file(filename, root_task.id.model, "report")
class ScanHandler(TaskHandler):
"""
@ -388,6 +414,30 @@ class ScanHandler(TaskHandler):
self._pending_ids_per_parent = {}
self._complete_ids_per_parent = {}
def setup(self, project, slots):
"""
initialize the scan task handler and save processed experimental scans.
"""
super(ScanHandler, self).setup(project, slots)
for (i_scan, scan) in enumerate(self._project.scans):
if scan.modulation is not None:
__, filename = os.path.split(scan.filename)
pre, ext = os.path.splitext(filename)
filename = "{pre}_{scan}.modf{ext}".format(pre=pre, ext=ext, scan=i_scan)
filepath = os.path.join(self._project.output_dir, filename)
md.save_data(filepath, scan.modulation)
mgs.render_scan(filepath, data=scan.modulation)
if project.combined_scan is not None:
ext = md.format_extension(project.combined_scan)
filename = project.output_file + ext
md.save_data(filename, project.combined_scan)
if project.combined_modf is not None:
ext = md.format_extension(project.combined_modf)
filename = project.output_file + ".modf" + ext
md.save_data(filename, project.combined_modf)
def create_tasks(self, parent_task):
"""
generate a calculation task for each scan of the given parent task.
@ -464,6 +514,7 @@ class ScanHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_scans(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'model')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'model')
@ -575,8 +626,11 @@ class SymmetryHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_symmetries(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'scan')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'scan')
graph_file = mgs.render_scan(parent_task.modf_filename)
self._project.files.add_file(graph_file, parent_task.id.model, 'scan')
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@ -621,7 +675,7 @@ class EmitterHandler(TaskHandler):
all emitters share the same model parameters.
@return list of @ref CalculationTask objects with one element per emitter configuration
@return list of @ref pmsco.dispatch.CalculationTask objects with one element per emitter configuration
if parallel processing is enabled.
otherwise the list contains a single CalculationTask object with emitter index 0.
the emitter index is used by the project's create_cluster method.
@ -634,10 +688,7 @@ class EmitterHandler(TaskHandler):
self._complete_ids_per_parent[parent_id] = set()
n_emitters = self._project.cluster_generator.count_emitters(parent_task.model, parent_task.id)
if n_emitters > 1 and self._slots > 1:
emitters = range(1, n_emitters + 1)
else:
emitters = [0]
emitters = range(n_emitters)
out_tasks = []
for em in emitters:
@ -698,8 +749,11 @@ class EmitterHandler(TaskHandler):
if parent_task.result_valid:
self._project.combine_emitters(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, 'symmetry')
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, 'symmetry')
graph_file = mgs.render_scan(parent_task.modf_filename)
self._project.files.add_file(graph_file, parent_task.id.model, 'symmetry')
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@ -776,15 +830,10 @@ class RegionHandler(TaskHandler):
parent_task.time = reduce(lambda a, b: a + b, child_times)
if parent_task.result_valid:
stack1 = [md.load_data(t.result_filename) for t in child_tasks]
dtype = md.common_dtype(stack1)
stack2 = [md.restructure_data(d, dtype) for d in stack1]
result_data = np.hstack(tuple(stack2))
md.sort_data(result_data)
md.save_data(parent_task.result_filename, result_data)
self._project.combine_regions(parent_task, child_tasks)
self._project.evaluate_result(parent_task, child_tasks)
self._project.files.add_file(parent_task.result_filename, parent_task.id.model, "emitter")
for t in child_tasks:
self._project.files.remove_file(t.result_filename)
self._project.files.add_file(parent_task.modf_filename, parent_task.id.model, "emitter")
del self._pending_ids_per_parent[parent_task.id]
del self._complete_ids_per_parent[parent_task.id]
@ -840,7 +889,7 @@ class EnergyRegionHandler(RegionHandler):
so that all child tasks of the same parent finish approximately in the same time.
pure angle scans are not split.
to use this feature, the project assigns this class to its @ref handler_classes['region'].
to use this feature, the project assigns this class to its @ref pmsco.project.Project.handler_classes['region'].
it is safe to use this handler for calculations that do not involve energy scans.
the handler is best used for single calculations.
in optimizations that calculate many models there is no advantage in using it

18
pmsco/helpers.py
View File

@ -1,4 +1,20 @@
class BraceMessage:
"""
@package pmsco.helpers
helper classes
a collection of small and generic code bits mostly collected from the www.
"""
class BraceMessage(object):
"""
a string formatting proxy class useful for logging and exceptions.
use BraceMessage("{0} {1}", "formatted", "message")
in place of "{0} {1}".format("formatted", "message").
the advantage is that the format method is called only if the string is actually used.
"""
def __init__(self, fmt, *args, **kwargs):
self.fmt = fmt
self.args = args

49
pmsco/loess/makefile
View File

@ -5,45 +5,42 @@ SHELL=/bin/sh
# required libraries: libblas, liblapack, libf2c
# (you may have to set soft links so that linker finds them)
#
# the makefile calls python-config to get the compilation flags and include path.
# you may override the corresponding variables on the command line or by environment variables:
#
# PYTHON_INC: specify additional include directories. each dir must start with -I prefix.
# PYTHON_CFLAGS: specify the C compiler flags.
#
# see the top-level makefile for additional information.
.SUFFIXES:
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so .x
.PHONY: all loess test gas madeup ethanol air galaxy
HOST=$(shell hostname)
CFLAGS=-O
FFLAGS=-O
OBJ=loessc.o loess.o predict.o misc.o loessf.o dqrsl.o dsvdc.o fix_main.o
FFLAGS?=-O
LIB=-lblas -lm -lf2c
LIBPATH=
CC=gcc
CCOPTS=
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHONOPTS=
ifneq (,$(filter merlin%,$(HOST)))
PYTHONINC=-I/usr/include/python2.7 -I/opt/python/python-2.7.5/include/python2.7/
else ifneq (,$(filter ra%,$(HOST)))
PYTHONINC=-I${PSI_PYTHON27_INCLUDE_DIR}/python2.7 -I${PSI_PYTHON27_LIBRARY_DIR}/python2.7/site-packages/numpy/core/include
else
PYTHONINC=-I/usr/include/python2.7
endif
LIBPATH?=
CC?=gcc
CCOPTS?=
SWIG?=swig
SWIGOPTS?=
PYTHON?=python
PYTHONOPTS?=
PYTHON_CONFIG = ${PYTHON}-config
#PYTHON_LIB ?= $(shell ${PYTHON_CONFIG} --libs)
#PYTHON_INC ?= $(shell ${PYTHON_CONFIG} --includes)
PYTHON_INC ?=
PYTHON_CFLAGS ?= $(shell ${PYTHON_CONFIG} --cflags)
#PYTHON_LDFLAGS ?= $(shell ${PYTHON_CONFIG} --ldflags)
all: loess
loess: _loess.so
loess_wrap.c: loess.c loess.i
$(SWIG) $(SWIGOPTS) -python loess.i
loess.py _loess.so: loess_wrap.c
# setuptools doesn't handle the fortran files correctly
# $(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
$(CC) $(CFLAGS) -fpic -c loessc.c loess.c predict.c misc.c loessf.f dqrsl.f dsvdc.f fix_main.c
$(CC) $(CFLAGS) -fpic -c loess_wrap.c $(PYTHONINC)
$(CC) -shared $(OBJ) $(LIB) $(LIBPATH) loess_wrap.o -o _loess.so
loess.py _loess.so: loess.c loess.i
$(PYTHON) $(PYTHONOPTS) setup.py build_ext --inplace
examples: gas madeup ethanol air galaxy

50
pmsco/loess/setup.py
View File

@ -1,7 +1,5 @@
#!/usr/bin/env python
__author__ = 'Matthias Muntwiler'
"""
@package loess.setup
setup.py file for LOESS
@ -17,39 +15,49 @@ the Python wrapper was set up by M. Muntwiler
with the help of the SWIG toolkit
and other incredible goodies available in the Linux world.
@bug this file is currently not used because
distutils does not compile the included Fortran files.
@bug numpy.distutils.build_src in python 2.7 treats all Fortran files with f2py
so that they are compiled via both f2py and swig.
this produces extra object files which cause the linker to fail.
to fix this issue, this module hacks the build_src class.
this hack does not work with python 3. perhaps it's even unnecessary.
@author Matthias Muntwiler
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from distutils.core import setup, Extension
from distutils import sysconfig
import numpy
try:
numpy_include = numpy.get_include()
except AttributeError:
numpy_include = numpy.get_numpy_include()
loess_module = Extension('_loess',
sources=['loess.i', 'loess_wrap.c', 'loess.c', 'loessc.c', 'predict.c', 'misc.c', 'loessf.f',
'dqrsl.f', 'dsvdc.f'],
include_dirs = [numpy_include],
libraries=['blas', 'm', 'f2c'],
def configuration(parent_package='', top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('loess', parent_package, top_path)
lib = ['blas', 'm', 'f2c']
src = ['loess.c', 'loessc.c', 'predict.c', 'misc.c', 'loessf.f', 'dqrsl.f', 'dsvdc.f', 'fix_main.c', 'loess.i']
inc_dir = [numpy_include]
config.add_extension('_loess',
sources=src,
libraries=lib,
include_dirs=inc_dir
)
return config
def ignore_sources(self, sources, extension):
return sources
if __name__ == '__main__':
try:
from numpy.distutils.core import numpy_cmdclass
numpy_cmdclass['build_src'].f2py_sources = ignore_sources
except ImportError:
pass
from numpy.distutils.core import setup
setup(**configuration(top_path='').todict())
setup(name='loess',
version='0.1',
author=__author__,
author_email='matthias.muntwiler@psi.ch',
description="""LOESS module in Python""",
ext_modules=[loess_module],
py_modules=["loess"], requires=['numpy']
)

21
pmsco/msc/makefile
View File

@ -12,16 +12,17 @@ SHELL=/bin/sh
.SUFFIXES: .c .cpp .cxx .exe .f .h .i .o .py .pyf .so
.PHONY: all clean edac msc mufpot
FC=gfortran
FCCOPTS=
F2PY=f2py
F2PYOPTS=
CC=gcc
CCOPTS=
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHONOPTS=
FC?=gfortran
FCCOPTS?=
F2PY?=f2py
F2PYOPTS?=
CC?=gcc
CCOPTS?=
SWIG?=swig
SWIGOPTS?=
PYTHON?=python
PYTHONOPTS?=
PYTHONINC?=
all: msc

2
pmsco/mufpot/makefile
View File

@ -20,7 +20,7 @@ CC=gcc
CCOPTS=
SWIG=swig
SWIGOPTS=
PYTHON=python
PYTHON=python2
PYTHONOPTS=
all: mufpot

0
pmsco/optimizers/__init__.py Normal file
View File

308
pmsco/optimizers/genetic.py Normal file
View File

@ -0,0 +1,308 @@
"""
@package pmsco.optimizers.genetic
genetic optimization algorithm.
this module implements a genetic algorithm for structural optimization.
the genetic algorithm is adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012)
the genetic algorithm evolves a population of individuals
by a combination of inheritance, crossover and mutation
and R-factor based selection.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import numpy as np
import random
import pmsco.optimizers.population as population
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
class GeneticPopulation(population.Population):
"""
population implementing a genetic optimization algorithm.
the genetic algorithm implements the following principles:
1. inheritance: two children of a new generation are generated from the genes (i.e. model parameters)
of two parents of the old generation.
2. elitism: individuals with similar r-factors are more likely to mate.
3. crossover: the genes of the parents are randomly distributed to their children.
4. mutation: a gene may mutate at random.
5. selection: the globally best individual is added to a parent population (and replaces the worst).
the main tuning parameter of the algorithm is the mutation_step which is copied from the domain.step.
it defines the width of a gaussian distribution of change under a weak mutation.
it should be large enough so that the whole parameter space can be probed,
but small enough that a frequent mutation does not throw the individual out of the convergence region.
typically, the step should be of the order of the parameter range divided by the population size.
other tunable parameters are the mating_factor, the weak_mutation_probability and the strong_mutation_probability.
the defaults should normally be fine.
"""
## @var weak_mutation_probability
#
# probability (between 0 and 1) that a parameter changes in the mutate_weak() method.
#
# the default is 1.0, i.e., each parameter mutates in each generation.
#
# 1.0 has shown better coverage of the continuous parameter space and faster finding of the optimum.
## @var strong_mutation_probability
#
# probability (between 0 and 1) that a parameter changes in the mutate_strong() method.
#
# the default is 0.01, i.e., on average, every hundredth probed parameter is affected by a strong mutation.
# if the model contains 10 parameters, for example,
# every tenth particle would see a mutation of at least one of its parameters.
#
# too high value may disturb convergence,
# too low value may trap the algorithm in a local optimum.
## @var mating_factor
#
# inverse width of the mating preference distribution.
#
# the greater this value, the more similar partners are mated by the mate_parents() method.
#
# the default value 4.0 results in a probability of about 0.0025
# that the best particle mates the worst.
## @var position_constrain_mode
#
# the position constrain mode selects what to do if a particle violates the parameter limits.
#
# the default is "random" which resets the parameter to a random value.
## @var mutation_step
#
# standard deviations of the exponential distribution function used in the mutate_weak() method.
# the variable is a dictionary with the same keys as model_step (the parameter domain).
#
# it is initialized from the domain.step
# or set to a default value based on the parameter range and population size.
def __init__(self):
"""
initialize the population object.
"""
super(GeneticPopulation, self).__init__()
self.weak_mutation_probability = 1.0
self.strong_mutation_probability = 0.01
self.mating_factor = 4.
self.position_constrain_mode = 'random'
self.mutation_step = {}
def setup(self, size, domain, **kwargs):
"""
@copydoc Population.setup()
in addition to the inherited behaviour, this method initializes self.mutation_step.
mutation_step of a parameter is set to its domain.step if non-zero.
otherwise it is set to the parameter range divided by the population size.
"""
super(GeneticPopulation, self).setup(size, domain, **kwargs)
for key in self.model_step:
val = self.model_step[key]
self.mutation_step[key] = val if val != 0 else (self.model_max[key] - self.model_min[key]) / size
def randomize(self, pos=True, vel=True):
"""
initializes a "random" population.
this implementation is a new proposal.
the distribution is not completely random.
rather, a position vector (by parameter) is initialized with a linear function
that covers the parameter domain.
the linear function is then permuted randomly.
the method does not update the particle info fields.
@param pos: randomize positions. if False, the positions are not changed.
@param vel: randomize velocities. if False, the velocities are not changed.
"""
if pos:
for key in self.model_start:
self.pos[key] = np.random.permutation(np.linspace(self.model_min[key], self.model_max[key],
self.pos.shape[0]))
if vel:
for key in self.model_start:
d = (self.model_max[key] - self.model_min[key]) / 8
self.vel[key] = np.random.permutation(np.linspace(-d, d, self.vel.shape[0]))
def advance_population(self):
"""
advance the population by one generation.
the population is advanced in several steps:
1. replace the worst individual by the best found so far.
2. mate the parents in pairs of two.
3. produce children by crossover from the parents.
4. apply weak mutations.
5. apply strong mutations.
if generation is lower than zero, the method increases the generation number but does not advance the particles.
@return: None
"""
if not self._hold_once:
self.generation += 1
pop = self.pos.copy()
pop.sort(order='_rfac')
elite = self.best.copy()
elite.sort(order='_rfac')
if elite[0]['_model'] not in pop['_model']:
elite[0]['_particle'] = pop[-1]['_particle']
pop[-1] = elite[0]
pop.sort(order='_rfac')
parents = self.mate_parents(pop)
children = []
for x, y in parents:
a, b = self.crossover(x, y)
children.append(a)
children.append(b)
for child in children:
index = child['_particle']
self.mutate_weak(child, self.weak_mutation_probability)
self.mutate_strong(child, self.strong_mutation_probability)
self.mutate_duplicate(child)
for key in self.model_start:
vel = child[key] - self.pos[index][key]
child[key], vel, self.model_min[key], self.model_max[key] = \
self.constrain_position(child[key], vel, self.model_min[key], self.model_max[key],
self.position_constrain_mode)
self.pos[index] = child
self.update_particle_info(index)
super(GeneticPopulation, self).advance_population()
def mate_parents(self, positions):
"""
group the population in pairs of two.
to mate two individuals, the first individual of the (remaining) population selects one of the following
with an exponential preference of earlier ones.
the process is repeated until all individuals are mated.
@param positions: original population (numpy structured array)
the population should be ordered with best model first.
@return: sequence of pairs (tuples) of structured arrays holding one model each.
"""
seq = [model for model in positions]
parents = []
while len(seq) >= 2:
p1 = seq.pop(0)
ln = len(seq)
i = min(int(random.expovariate(self.mating_factor / ln) * ln), ln - 1)
p2 = seq.pop(i)
parents.append((p1, p2))
return parents
def crossover(self, parent1, parent2):
"""
crossover two parents to create two children.
for each model parameter, the parent's value is randomly assigned to either one of the children.
@param parent1: numpy structured array holding the model of the first parent.
@param parent2: numpy structured array holding the model of the second parent.
@return: tuple of the two crossed children.
these are two new ndarray instances that are independent of their parents.
"""
child1 = parent1.copy()
child2 = parent2.copy()
for key in self.model_start:
if random.random() >= 0.5:
child1[key], child2[key] = parent2[key], parent1[key]
return child1, child2
def mutate_weak(self, model, probability):
"""
apply a weak mutation to a model.
each parameter is changed to a different value in the domain of the parameter at the given probability.
the amount of change has a gaussian distribution with a standard deviation of mutation_step.
@param[in,out] model: structured numpy.ndarray holding the model parameters.
model is modified in place.
@param probability: probability between 0 and 1 at which to change a parameter.
0 = no change, 1 = force change.
@return: model (same instance as the @c model input argument).
"""
for key in self.model_start:
if random.random() < probability:
model[key] += random.gauss(0, self.mutation_step[key])
return model
def mutate_strong(self, model, probability):
"""
apply a strong mutation to a model.
each parameter is changed to a random value in the domain of the parameter at the given probability.
@param[in,out] model: structured numpy.ndarray holding the model parameters.
model is modified in place.
@param probability: probability between 0 and 1 at which to change a parameter.
0 = no change, 1 = force change.
@return: model (same instance as the @c model input argument).
"""
for key in self.model_start:
if random.random() < probability:
model[key] = (self.model_max[key] - self.model_min[key]) * random.random() + self.model_min[key]
return model
def mutate_duplicate(self, model):
"""
mutate a model if it is identical to a previously calculated one.
if the model was calculated before, the mutate_weak mutation is applied with probability 1.
@param[in,out] model: structured numpy.ndarray holding the model parameters.
model is modified in place.
@return: model (same instance as the @c model input argument).
"""
try:
self.find_model(model)
self.mutate_weak(model, 1.0)
except ValueError:
pass
return model
class GeneticOptimizationHandler(population.PopulationHandler):
"""
model handler which implements a genetic algorithm.
"""
def __init__(self):
super(GeneticOptimizationHandler, self).__init__()
self._pop = GeneticPopulation()

0
pmsco/gradient.py → pmsco/optimizers/gradient.py
View File

58
pmsco/grid.py → pmsco/optimizers/grid.py
View File

@ -13,14 +13,19 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
import copy
import os
from __future__ import print_function
import datetime
import math
import numpy as np
import logging
import handlers
from helpers import BraceMessage as BMsg
from pmsco.compat import open
import pmsco.handlers as handlers
import pmsco.graphics as graphics
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -114,16 +119,14 @@ class GridPopulation(object):
@param domain: definition of initial and limiting model parameters
expected by the cluster and parameters functions.
@param domain.start: values of the fixed parameters.
@param domain.min: minimum values allowed.
@param domain.max: maximum values allowed.
the attributes have the following meanings:
@arg start: values of the fixed parameters.
@arg min: minimum values allowed.
@arg max: maximum values allowed.
if abs(max - min) < step/2 , the parameter is kept constant.
@param domain.step: step size (distance between two grid points).
@arg step: step size (distance between two grid points).
if step <= 0, the parameter is kept constant.
"""
self.model_start = domain.start
self.model_min = domain.min
@ -374,17 +377,10 @@ class GridSearchHandler(handlers.ModelHandler):
del self._pending_tasks[task.id]
parent_task = self._parent_tasks[task.parent_id]
rfac = 1.0
if task.result_valid:
try:
rfac = self._project.calc_rfactor(task)
except ValueError:
task.result_valid = False
self._invalid_count += 1
logger.warning(BMsg("calculation of model {0} resulted in an undefined R-factor.", task.id.model))
task.model['_rfac'] = rfac
self._pop.add_result(task.model, rfac)
assert not math.isnan(task.rfac)
task.model['_rfac'] = task.rfac
self._pop.add_result(task.model, task.rfac)
if self._outfile:
s = (str(task.model[name]) for name in self._pop.positions.dtype.names)
@ -392,12 +388,14 @@ class GridSearchHandler(handlers.ModelHandler):
self._outfile.write("\n")
self._outfile.flush()
self._project.files.update_model_rfac(task.id.model, rfac)
self._project.files.update_model_rfac(task.id.model, task.rfac)
self._project.files.set_model_complete(task.id.model, True)
if task.result_valid:
if task.time > self._model_time:
self._model_time = task.time
else:
self._invalid_count += 1
# grid search complete?
if len(self._pending_tasks) == 0:
@ -407,3 +405,17 @@ class GridSearchHandler(handlers.ModelHandler):
self.cleanup_files()
return parent_task
def save_report(self, root_task):
"""
generate a graphical summary of the optimization.
@param root_task: (CalculationTask) the id.model attribute is used to register the generated files.
@return: None
"""
super(GridSearchHandler, self).save_report(root_task)
files = graphics.rfactor.render_results(self._project.output_file + ".dat", self._pop.positions)
for f in files:
self._project.files.add_file(f, root_task.id.model, "report")

1358
pmsco/optimizers/population.py Normal file
View File

File diff suppressed because it is too large Load Diff

139
pmsco/optimizers/swarm.py Normal file
View File

@ -0,0 +1,139 @@
"""
@package pmsco.optimizers.swarm
particle swarm optimization handler.
the module starts multiple MSC calculations and optimizes the model parameters
according to the particle swarm optimization algorithm.
Particle swarm optimization adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012)
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import numpy as np
import pmsco.optimizers.population as population
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
class SwarmPopulation(population.Population):
"""
particle swarm population.
"""
## @var friends
# number of other particles that each particle consults for the global best fit.
# default = 3.
## @var momentum
# momentum of the particle.
# default = 0.689343.
## @var attract_local
# preference for returning to the local best fit
# default = 1.92694.
## @var attract_global
# preference for heading towards the global best fit.
# default = 1.92694
def __init__(self):
"""
initialize the population object.
"""
super(SwarmPopulation, self).__init__()
self.friends = 3
self.momentum = 0.689343
self.attract_local = 1.92694
self.attract_global = 1.92694
self.position_constrain_mode = 'default'
self.velocity_constrain_mode = 'default'
def advance_population(self):
"""
advance the population by one step.
this method just calls advance_particle() for each particle of the population.
if generation is lower than zero, the method increases the generation number but does not advance the particles.
@return: None
"""
if not self._hold_once:
self.generation += 1
for index, __ in enumerate(self.pos):
self.advance_particle(index)
super(SwarmPopulation, self).advance_population()
def advance_particle(self, index):
"""
advance a particle by one step.
@param index: index of the particle in the population.
"""
# note: the following two identifiers are views,
# assignment will modify the original array
pos = self.pos[index]
vel = self.vel[index]
# best fit that this individual has seen
xl = self.best[index]
# best fit that a group of others have seen
xg = self.best_friend(index)
for key in self.model_start:
# update velocity
dxl = xl[key] - pos[key]
dxg = xg[key] - pos[key]
pv = np.random.random()
pl = np.random.random()
pg = np.random.random()
vel[key] = (self.momentum * pv * vel[key] +
self.attract_local * pl * dxl +
self.attract_global * pg * dxg)
pos[key], vel[key], self.model_min[key], self.model_max[key] = \
self.constrain_velocity(pos[key], vel[key], self.model_min[key], self.model_max[key],
self.velocity_constrain_mode)
# update position
pos[key] += vel[key]
pos[key], vel[key], self.model_min[key], self.model_max[key] = \
self.constrain_position(pos[key], vel[key], self.model_min[key], self.model_max[key],
self.position_constrain_mode)
self.update_particle_info(index)
# noinspection PyUnusedLocal
def best_friend(self, index):
"""
select the best fit out of a random set of particles
returns the "best friend"
"""
friends = np.random.choice(self.best, self.friends, replace=False)
index = np.argmin(friends['_rfac'])
return friends[index]
class ParticleSwarmHandler(population.PopulationHandler):
"""
model handler which implements the particle swarm optimization algorithm.
"""
def __init__(self):
super(ParticleSwarmHandler, self).__init__()
self._pop = SwarmPopulation()

155
pmsco/optimizers/table.py Normal file
View File

@ -0,0 +1,155 @@
"""
@package pmsco.table
table scan optimization handler
the table scan scans through an explicit table of model parameters.
it can be used to recalculate models from a previous optimization run on different scans,
or as an interface to external optimizers.
new elements can be added to the table while the calculation loop is in progress.
though the concepts _population_ and _optimization_ are not intrinsic to a table scan,
the classes defined here inherit from the generic population class and optimization handler.
this is done to share as much code as possible between the different optimizers.
the only difference is that the table optimizer does not generate models internally.
instead, it loads them (possibly repeatedly) from a file or asks the project code to provide the data.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import numpy as np
import pmsco.optimizers.population as population
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
class TablePopulation(population.Population):
"""
population generated from explicit values.
this class maintains a population that is updated from a table of explicit values.
the table can be static (defined at the start of the optimization process)
or dynamic (new models appended during the optimization process).
for each generation, the table is read and the next models are imported into the population.
the class de-duplicates the table, i.e. models with equal parameters as a previous one are not calculated again.
it is, thus, perfectly fine that new models are appended to the table rather than overwrite previous entries.
the table can be built from the following data sources:
@arg (numpy.ndarray): structured array that can be added to self.positions,
having at least the columns defining the model parameters.
@arg (sequence of dict, numpy.ndarray, numpy.void, named tuple):
each element must be syntactically compatible with a dict
that holds the model parameters.
@arg (str): file name that contains a table in the same format as
@ref pmsco.optimizers.population.Population.save_array produces.
@arg (callable): a function that returns one of the above objects
(or None to mark the end of the table).
the data source is passed as an argument to the self.setup() method.
structured arrays and sequences cannot be modified after they are passed to `setup`.
this means that the complete table must be known at the start of the process.
the most flexible way is to pass a function that generates a structured array in each call.
this would even allow to include a non-standard optimization algorithm.
the function is best defined in the custom project class.
the population calls it every time before a new generation starts.
to end the optimization process, it simply returns None.
the table can also be defined in an external file, e.g. as calculated by other programs or edited manually.
the table file can either remain unchanged during the optimization process,
or new models can be added while the optimization is in progress.
in the latter case, note that there is no reliable synchronization of file access.
first, writing to the file must be as short as possible.
the population class has a read timeout of ten seconds.
second, because it is impossible to know whether the file has been read or not,
new models should be _appended_ rather than _overwrite_ previous ones.
the population class automatically skips models that have already been read.
this class supports does not support seeding.
although, a seed file is accepted, it is not used.
patching is allowed, but there is normally no advantage over modifying the table.
the domain is used to define the model parameters and the parameter range.
models violating the parameter domain are ignored.
"""
## @var table_source
# data source of the model table
#
# this can be any object accepted by @ref pmsco.optimizers.population.Population.import_positions,
# e.g. a file name, a numpy structured array, or a function returning a structured array.
# see the class description for details.
def __init__(self):
"""
initialize the population object.
"""
super(TablePopulation, self).__init__()
self.table_source = None
self.position_constrain_mode = 'error'
def setup(self, size, domain, **kwargs):
"""
set up the population arrays, parameter domain and data source.
@param size: requested number of particles.
this does not need to correspond to the number of table entries.
on each generation the population loads up to this number of new entries from the table source.
@param domain: definition of initial and limiting model parameters
expected by the cluster and parameters functions.
@arg domain.start: not used.
@arg domain.min: minimum values allowed.
@arg domain.max: maximum values allowed.
@arg domain.step: not used.
the following arguments are keyword arguments.
the method also accepts the inherited arguments for seeding. they do not have an effect, however.
@param table_source: data source of the model table.
this can be any object accepted by @ref pmsco.optimizers.population.Population.import_positions,
e.g. a file name, a numpy structured array, or a function returning a structured array.
see the class description for details.
@return: None
"""
super(TablePopulation, self).setup(size, domain, **kwargs)
self.table_source = kwargs['table_source']
def advance_population(self):
"""
advance the population by one step.
this methods re-imports the table file
and copies the table to current population.
@return: None
"""
self.import_positions(self.table_source)
self.advance_from_import()
super(TablePopulation, self).advance_population()
class TableModelHandler(population.PopulationHandler):
"""
model handler which implements the table algorithm.
"""
def __init__(self):
super(TableModelHandler, self).__init__()
self._pop = TablePopulation()

191
pmsco/pmsco.py
View File

@ -4,13 +4,9 @@
@package pmsco.pmsco
PEARL Multiple-Scattering Calculation and Structural Optimization
this is the main entry point and top-level interface of the PMSCO package.
all calculations (any mode, any project) start by calling the main_pmsco() function of this module.
the module also provides a command line parser.
command line usage: call with -h option to see the list of arguments.
python usage: call main_pmsco() with suitable arguments.
this is the top-level interface of the PMSCO package.
all calculations (any mode, any project) start by calling the run_project() function of this module.
the module also provides a command line parser for common options.
for parallel execution, prefix the command line with mpi_exec -np NN, where NN is the number of processes to use.
note that in parallel mode, one process takes the role of the coordinator (master).
@ -24,48 +20,39 @@ PMSCO serializes the calculations automatically.
the code of the main module is independent of a particular calculation project.
all project-specific code must be in a separate python module.
the project module must implement a class derived from pmsco.project.Project,
and a global function create_project which returns a new instance of the derived project class.
and call run_project() with an instance of the project class.
refer to the projects folder for examples.
@pre
* python 2.7, including python-pip
* numpy
* nose from Debian python-nose
* statsmodels from Debian python-statsmodels, or PyPI (https://pypi.python.org/pypi/statsmodels)
* periodictable from PyPI (https://pypi.python.org/pypi/periodictable)
* mpi4py from PyPI (the Debian package may have a bug causing the program to crash)
* OpenMPI, including libopenmpi-dev
* SWIG from Debian swig
to install a PyPI package, e.g. periodictable, do
@code{.sh}
pip install --user periodictable
@endcode
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
from builtins import range
import datetime
import logging
import importlib
import os.path
import sys
import datetime
import argparse
import logging
import cluster
import dispatch
import handlers
import files
import calculator
import swarm
import grid
# import gradient
from mpi4py import MPI
import pmsco.calculators.calculator as calculator
import pmsco.cluster as cluster
import pmsco.dispatch as dispatch
import pmsco.files as files
import pmsco.handlers as handlers
from pmsco.optimizers import genetic, swarm, grid, table
# the module-level logger
logger = logging.getLogger(__name__)
@ -152,13 +139,20 @@ def set_common_args(project, args):
logger.debug("creating project")
mode = args.mode.lower()
if mode in {'single', 'grid', 'swarm'}:
if mode in {'single', 'grid', 'swarm', 'genetic', 'table'}:
project.mode = mode
else:
logger.error("invalid optimization mode '%s'.", mode)
if args.pop_size:
project.pop_size = args.pop_size
project.optimizer_params['pop_size'] = args.pop_size
if args.seed_file:
project.optimizer_params['seed_file'] = args.seed_file
if args.seed_limit:
project.optimizer_params['seed_limit'] = args.seed_limit
if args.table_file:
project.optimizer_params['table_file'] = args.table_file
code = args.code.lower()
if code in {'edac', 'msc', 'test'}:
@ -178,6 +172,10 @@ def set_common_args(project, args):
if mode == 'single':
cats -= {'model'}
project.files.categories_to_delete = cats
if args.keep_levels > project.keep_levels:
project.keep_levels = args.keep_levels
if args.keep_best > project.keep_best:
project.keep_best = args.keep_best
def log_project_args(project):
@ -190,8 +188,19 @@ def log_project_args(project):
try:
logger.info("scattering code: {0}".format(project.code))
logger.info("optimization mode: {0}".format(project.mode))
logger.info("minimum swarm size: {0}".format(project.pop_size))
try:
logger.info("minimum population size: {0}".format(project.optimizer_params['pop_size']))
except KeyError:
pass
try:
logger.info("seed file: {0}".format(project.optimizer_params['seed_file']))
logger.info("seed limit: {0}".format(project.optimizer_params['seed_limit']))
except KeyError:
pass
try:
logger.info("table file: {0}".format(project.optimizer_params['table_file']))
except KeyError:
pass
logger.info("data directory: {0}".format(project.data_dir))
logger.info("output file: {0}".format(project.output_file))
@ -217,10 +226,14 @@ def run_project(project):
optimizer_class = grid.GridSearchHandler
elif project.mode == 'swarm':
optimizer_class = swarm.ParticleSwarmHandler
elif project.mode == 'genetic':
optimizer_class = genetic.GeneticOptimizationHandler
elif project.mode == 'gradient':
logger.error("gradient search not implemented")
# TODO: implement gradient search
# optimizer_class = gradient.GradientSearchHandler
elif project.mode == 'table':
optimizer_class = table.TableModelHandler
else:
logger.error("invalid optimization mode '%s'.", project.mode)
project.handler_classes['model'] = optimizer_class
@ -230,14 +243,14 @@ def run_project(project):
calculator_class = None
if project.code == 'edac':
logger.debug("importing EDAC interface")
import edac_calculator
from pmsco.calculators import edac
project.cluster_format = cluster.FMT_EDAC
calculator_class = edac_calculator.EdacCalculator
calculator_class = edac.EdacCalculator
elif project.code == 'msc':
logger.debug("importing MSC interface")
import msc_calculator
from pmsco.calculators import msc
project.cluster_format = cluster.FMT_MSC
calculator_class = msc_calculator.MscCalculator
calculator_class = msc.MscCalculator
elif project.code == 'test':
logger.debug("importing TEST interface")
project.cluster_format = cluster.FMT_EDAC
@ -273,7 +286,7 @@ class Args(object):
values as the command line parser.
"""
def __init__(self, mode="single", code="edac", output_file=""):
def __init__(self, mode="single", code="edac", output_file="pmsco_data"):
"""
constructor.
@ -284,14 +297,19 @@ class Args(object):
"""
self.mode = mode
self.pop_size = 0
self.seed_file = ""
self.seed_limit = 0
self.code = code
self.data_dir = os.getcwd()
self.output_file = output_file
self.time_limit = 24.0
self.keep_files = []
self.keep_files = files.FILE_CATEGORIES_TO_KEEP
self.keep_best = 10
self.keep_levels = 1
self.log_level = "WARNING"
self.log_file = ""
self.log_enable = True
self.table_file = ""
def get_cli_parser(default_args=None):
@ -324,26 +342,47 @@ def get_cli_parser(default_args=None):
# for simplicity, the parser does not check these requirements.
# all parameters are optional and accepted regardless of mode.
# errors may occur if implicit requirements are not met.
parser.add_argument('-m', '--mode', default='single',
choices=['single', 'grid', 'swarm', 'gradient'],
parser.add_argument('project_module',
help="path to custom module that defines the calculation project")
parser.add_argument('-m', '--mode', default=default_args.mode,
choices=['single', 'grid', 'swarm', 'genetic', 'table'],
help='calculation mode')
parser.add_argument('--pop-size', type=int, default=0,
help='population size (number of particles) in swarm optimization mode. ' +
parser.add_argument('--pop-size', type=int, default=default_args.pop_size,
help='population size (number of particles) in swarm or genetic optimization mode. ' +
'default is the greater of 4 or two times the number of calculation processes.')
parser.add_argument('-c', '--code', choices=['msc', 'edac', 'test'], default="edac",
parser.add_argument('--seed-file',
help='path and name of population seed file. ' +
'population data of previous optimizations can be used to seed a new optimization. ' +
'the file must have the same structure as the .pop or .dat files.')
parser.add_argument('--seed-limit', type=int, default=default_args.seed_limit,
help='maximum number of models to use from the seed file. ' +
'the models with the best R-factors are selected.')
parser.add_argument('-c', '--code', choices=['msc', 'edac', 'test'], default=default_args.code,
help='scattering code (default: edac)')
parser.add_argument('-d', '--data-dir', default=os.getcwd(),
parser.add_argument('-d', '--data-dir', default=default_args.data_dir,
help='directory path for experimental data files (if required by project). ' +
'default: working directory')
parser.add_argument('-o', '--output-file',
parser.add_argument('-o', '--output-file', default=default_args.output_file,
help='base path for intermediate and output files.' +
'default: pmsco_data')
parser.add_argument('-k', '--keep-files', nargs='*', default=files.FILE_CATEGORIES_TO_KEEP,
parser.add_argument('--table-file',
help='path and name of population table file for table optimization mode. ' +
'the file must have the same structure as the .pop or .dat files.')
parser.add_argument('-k', '--keep-files', nargs='*', default=default_args.keep_files,
choices=KEEP_FILES_CHOICES,
help='output file categories to keep after the calculation. '
'by default, cluster and model (simulated data) '
'of a limited number of best models are kept.')
parser.add_argument('-t', '--time-limit', type=float, default=24.0,
parser.add_argument('--keep-best', type=int, default=default_args.keep_best,
help='number of best models for which to keep result files '
'(at each node from root down to keep-levels). '
'default 10 (project can define higher default).')
parser.add_argument('--keep-levels', type=int, choices=range(5),
default=default_args.keep_levels,
help='task level down to which result files of best models are kept. '
'0 = model, 1 = scan, 2 = symmetry, 3 = emitter, 4 = region. '
'default 1 (project can define higher default).')
parser.add_argument('-t', '--time-limit', type=float, default=default_args.time_limit,
help='wall time limit in hours. the optimizers try to finish before the limit. default: 24.')
parser.add_argument('--log-file', default=default_args.log_file,
help='name of the main log file. ' +
@ -375,7 +414,47 @@ def parse_cli():
return args, unknown_args
def import_project_module(path):
"""
import the custom project module.
imports the project module given its file path.
the path is expanded to its absolute form and appended to the python path.
@param path: path and name of the module to be loaded.
path is optional and defaults to the python path.
if the name includes an extension, it is stripped off.
@return: the loaded module as a python object
"""
path, name = os.path.split(path)
name, __ = os.path.splitext(name)
path = os.path.abspath(path)
sys.path.append(path)
project_module = importlib.import_module(name)
return project_module
def main():
args, unknown_args = parse_cli()
if args:
module = import_project_module(args.project_module)
try:
project_args = module.parse_project_args(unknown_args)
except NameError:
project_args = None
project = module.create_project()
set_common_args(project, args)
try:
module.set_project_args(project, project_args)
except NameError:
pass
run_project(project)
if __name__ == '__main__':
main_parser = get_cli_parser()
main_parser.print_help()
main()
sys.exit(0)

763
pmsco/project.py
View File

@ -26,16 +26,27 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import collections
import copy
import datetime
import logging
import numpy as np
import collections
import data as md
import cluster as mc
import files
import handlers
import os.path
import socket
import sys
import pmsco.cluster as mc
from pmsco.compat import open
import pmsco.data as md
import pmsco.database as database
import pmsco.dispatch as dispatch
import pmsco.files as files
import pmsco.handlers as handlers
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -95,7 +106,7 @@ class Domain(object):
self.max = {}
self.step = {}
def add_param(self, name, start, min, max, step):
def add_param(self, name, start, min=None, max=None, step=None, width=None):
"""
set the domain of one parameter with all necessary values at once.
@ -107,15 +118,29 @@ class Domain(object):
@param start (float) start value.
@param min (float) lower bound of the parameter interval.
must be lower or equal to start.
if None, the field is set to start.
@param max (float) upper bound of the parameter interval.
must be greater or equal to start.
if None, the field is set to start.
@param width (float) width of the parameter interval.
instead of min and max, the interval can be set centered around the start value.
this is equivalent to min = start - width/2, max = start + width/2.
this argument overrides min and max. don't use both arguments.
@param step (float) step size.
must be greater or equal to zero.
if None, the field is set to zero.
"""
self.start[name] = start
self.min[name] = min
self.max[name] = max
self.step[name] = step
self.min[name] = min if min is not None else start
self.max[name] = max if max is not None else start
if width is not None:
self.min[name] = start - width / 2.
self.max[name] = start + width / 2.
self.step[name] = step if step is not None else 0.0
def get_param(self, name):
"""
@ -144,9 +169,27 @@ class Params(object):
objects of this class are created by the implementation of the create_params() method
of the actual project class.
"""
## @var angular_resolution (float)
# FWHM angular resolution of the detector.
#
# maps to:
# @arg emission angle window (EDAC)
# @arg angular_broadening (MSC)
## @var phase_files (dict)
# dictionary of phase files.
#
# the keys are atomic numbers, the values file names.
# if the dictionary is empty or the files don't exist, the phases are computed internally (EDAC only).
#
# maps to:
# @arg scatterer (EDAC)
# @arg atomic_number, phase_file (MSC)
def __init__(self):
self.title = "MSC default parameters"
self.comment = "from msc_project.Params()"
self.title = "default parameters"
self.comment = "set by project.Params()"
self.cluster_file = ""
self.output_file = ""
self.scan_file = ""
@ -154,7 +197,7 @@ class Params(object):
self.initial_state = "1s"
# MSC convention: H, V, L, R, U
self.polarization = "H"
self.angular_broadening = 0.0
self.angular_resolution = 1.0
self.z_surface = 0.0
self.inner_potential = 10.0
# the energy scale of EDAC is referenced to the vacuum level
@ -167,16 +210,14 @@ class Params(object):
self.experiment_temperature = 300.0
self.debye_temperature = 400.0
self.debye_wavevector = 1.0
self.phase_files = {}
# used by MSC only
self.spherical_order = 2
self.scattering_level = 5
self.fcut = 15.0
self.cut = 15.0
self.lattice_constant = 1.0
self.atom_types = 0
self.atomic_number = [1, 2, 3, 4]
self.phase_file = ["1.pha", "2.pha", "3.pha", "4.pha"]
self.msq_displacement = [0.1, 0.1, 0.1, 0.1]
self.msq_displacement = {}
self.planewave_attenuation = 1.0
self.vibration_model = "N"
self.substrate_atomic_mass = 1.0
@ -216,9 +257,12 @@ class Scan(object):
# example: ['t','p']
## @var emitter (string)
# chemical symbol of emitter atom
# chemical symbol and, optionally following, further specification (chemical state, environment, ...)
# of photo-emitting atoms.
# the interpretation of this string is up to the project and its cluster generator.
# it should, however, always start with a chemical element symbol.
#
# example: 'Cu'
# examples: 'Ca' (calcium), 'CA' (carbon A), 'C a' (carbon a), 'C 1' (carbon one), 'N=O', 'FeIII'.
## @var initial_state (string)
# nl term of initial state
@ -342,142 +386,6 @@ class Scan(object):
self.alphas = np.zeros((1))
class ClusterGenerator(object):
"""
cluster generator class.
this class bundles the cluster methods in one place
so that it's easier to exchange them for different kinds of clusters.
the project must override at least the create_cluster method.
if emitters should be run in parallel tasks, the count_emitters method must be implemented as well.
"""
def __init__(self, project):
"""
initialize the cluster generator.
@param project: reference to the project object.
cluster generators may need to look up project parameters.
"""
self.project = project
def count_emitters(self, model, index):
"""
return the number of emitter configurations for a particular model.
the number of emitter configurations may depend on the model parameters, scan index and symmetry index.
by default, the method returns 1, which means that there is only one emitter configuration.
emitter configurations are mainly a way to distribute the calculations to multiple processes
based on emitters since the resulting diffraction patterns add up incoherently.
for this to work, the create_cluster() method must pay attention to the emitter index
and generate either a full cluster with all emitters (single process)
or a cluster with only a subset of the emitters according to the emitter index (multiple processes).
whether all emitters are calculated in one or multiple processes is decided at run-time
based on the available resources.
note that this function returns the number of _configurations_ not _atoms_.
an emitter configuration (declared in a Cluster) may include more than one atom.
it is up to the project, what is included in a particular configuration.
to enable multiple emitter configurations, the derived project class must override this method
and return a number greater than 1.
@note in some cases it may be most efficient to call create_cluster and
return Cluster.get_emitter_count() of the generated cluster.
this is possible because the method is called with emitter index -1.
model and index can be passed unchanged to create_cluster.
@param model (dictionary) model parameters to be used in the calculation.
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index is -1 if called by the emitter handler.
@return number of emitter configurations.
this implementation returns the default value of 1.
"""
return 1
def create_cluster(self, model, index):
"""
create a Cluster object given the model parameters and calculation index.
the generated cluster will typically depend on the model parameters.
depending on the project, it may also depend on the scan index, symmetry index and emitter index.
the scan index can be used to generate a different cluster for different scan geometry,
e.g., if some atoms can be excluded due to a longer mean free path.
if this is not the case for the specific project, the scan index can be ignored.
the symmetry index may select a particular domain that has a different atomic arrangement.
in this case, depending on the value of index.sym, the function must generate a cluster corresponding
to the particular domain/symmetry.
the method can ignore the symmetry index if the project defines only one symmetry,
or if the symmetry does not correspond to a different atomic structure.
the emitter index selects a particular emitter configuration.
depending on the value of the emitter index, the method must react differently:
1. if the value lower or equal to zero, return the full cluster and mark all inequivalent emitter atoms.
emitters which are reproduced by a symmetry expansion in combine_emitters() should not be marked.
the full diffraction scan will be calculated in one calculation.
2. if the value is greater than zero, generate the cluster with the emitter configuration
selected by the emitter index.
the index is in the range between 1 and the return value of count_emitters().
the results of the individual emitter calculations are summed up in combine_emitters().
the code should ideally be written such that either case yields the same diffraction result.
if count_emitters() always returns 1 (default), the second case does not have to be implemented,
and the method can ignore the emitter index.
the method must ignore the model and energy index.
@param model (dictionary) model parameters to be used in the calculation.
@param index (named tuple CalcID) calculation index.
the method should consider only the following attributes:
@arg @c scan scan index (index into Project.scans)
@arg @c sym symmetry index (index into Project.symmetries)
@arg @c emit emitter index.
if lower or equal to zero, generate the full cluster and mark all emitters.
if greater than zero, the value is a 1-based index of the emitter configuration.
"""
return None
class LegacyClusterGenerator(ClusterGenerator):
"""
cluster generator class for projects that don't declare a generator.
in previous versions, the create_cluster and count_emitters methods were implemented by the project class.
this class redirects generator calls to the project methods
providing compatibility to older project code.
"""
def __init__(self, project):
super(LegacyClusterGenerator, self).__init__(project)
def count_emitters(self, model, index):
"""
redirect the call to the corresponding project method if implemented.
"""
try:
return self.project.count_emitters(model, index)
except AttributeError:
return 1
def create_cluster(self, model, index):
"""
redirect the call to the corresponding project method.
"""
return self.project.create_cluster(model, index)
# noinspection PyMethodMayBeStatic
class Project(object):
"""
@ -549,37 +457,25 @@ class Project(object):
# the initial value is a LegacyClusterGenerator object
# which routes cluster calls back to the project for compatibility with older project code.
## @var pop_size (int)
# population size (number of particles) in the particle swarm optimization.
## @var optimizer_params (dict)
# optional parameters of the model optimizer.
#
# this is a dictionary that can have (among others) the following values.
# for a detailed list, see the documentation of the respective model handler.
#
# @arg @c 'pop_size' (int)
# population size (number of particles) in the swarm or genetic optimization mode.
# by default, the ParticleSwarmHandler chooses the population size depending on the number of parallel processes.
# you may want to override the default value in cases where the automatic choice is not appropriate, e.g.:
# - the calculation of a model takes a long time compared to the available computing time.
# - the calculation of a model spawns many sub-tasks due to complex symmetry.
# - you want to increase the number of generations compared to the number of particles.
#
# the default value is 0.
#
# you may want to override the default value in cases where the automatic choice is not appropriate.
# the value can be set by the command line.
## @var history_file (string)
# @arg @c 'seed_file' (string)
# name of a file containing the results from previous optimization runs.
# this can be used to resume a swarm optimization where it was interrupted before.
#
# the history file is a space-delimited, multi-column, text file.
# output files of a previous optimization run can be used as is.
# there must be one column for each model parameter, and one column of R factors.
# the first row must contain the names of the model parameters.
# the name of th R factor column must be '_rfac'.
# additional columns may be included and are ignored.
#
# by default, no history is loaded.
## @var recalc_history (bool)
# select whether the R-factors of the historic models are calculated again.
#
# this is useful if the historic data was calculated for a different cluster, different set of parameters,
# or different experimental data, and if the R-factors of the new optimization may be systematically greater.
# this can be used to resume a swarm or genetic optimization where it was interrupted before.
# the seed file is a space-delimited, multi-column, text file,
# e.g., the output file of a previous optimization.
# by default, no seed is loaded.
# @arg @c 'recalc_seed' (bool)
# select whether the R-factors of the seed models are calculated again.
# set this argument to False only if the calculation is a continuation of a previous one
# without any changes to the code.
@ -594,9 +490,17 @@ class Project(object):
# if the location of the files may depend on the machine or user account,
# the user may want to specify the data path on the command line.
## @var output_dir (string)
# directory path for data files produced during the calculation, including intermediate files.
#
# output_dir and output_file are set at once by @ref set_output.
## @var output_file (string)
# file name root for data files produced during the calculation, including intermediate files.
#
# the file name should include the path. the path must also be set in @ref output_dir.
#
# output_dir and output_file are set at once by @ref set_output.
## @var timedelta_limit (datetime.timedelta)
# wall time after which no new calculations should be started.
@ -604,11 +508,11 @@ class Project(object):
# the actual wall time may be longer by the remaining time of running calculations.
# running calculations will not be aborted.
## @var _combined_scan
## @var combined_scan
# combined raw data from scans.
# updated by add_scan().
## @var _combined_modf
## @var combined_modf
# combined modulation function from scans.
# updated by add_scan().
@ -618,30 +522,55 @@ class Project(object):
#
# files.categories_to_delete determines which files can be deleted.
## @var keep_best
# number of best models for which result files should be kept.
#
# this attribute determines how many models are kept based on R-factor ranking at each node of the task tree
# (up to keep_levels).
## @var keep_levels
# numeric task level down to which R-factors are considered when model files are cleaned up.
#
# @arg 0 = model level: combined results only.
# @arg 1 = scan level: scan nodes in addition to combined results (level 0).
# @arg 2 = symmetry level: symmetry nodes in addition to level 1.
# @arg 3 = emitter level: emitter nodes in addition to level 1.
# @arg 4 = region level: region nodes in addition to level 1.
def __init__(self):
self.mode = "single"
self.code = "edac"
self.features = {}
self.cluster_format = mc.FMT_EDAC
self.cluster_generator = LegacyClusterGenerator(self)
self.cluster_generator = mc.LegacyClusterGenerator(self)
self.scans = []
self.symmetries = []
self.pop_size = 0
self.history_file = ""
self.recalc_history = True
self.optimizer_params = {
'pop_size': 0,
'seed_file': "",
'seed_limit': 0,
'recalc_seed': True,
'table_file': ""
}
self.data_dir = ""
self.output_dir = ""
self.output_file = "pmsco_data"
self.timedelta_limit = datetime.timedelta(days=1)
self._combined_scan = None
self._combined_modf = None
self.combined_scan = None
self.combined_modf = None
self.files = files.FileTracker()
self.handler_classes = {}
self.handler_classes['model'] = handlers.SingleModelHandler
self.handler_classes['scan'] = handlers.ScanHandler
self.handler_classes['symmetry'] = handlers.SymmetryHandler
self.handler_classes['emitter'] = handlers.EmitterHandler
self.handler_classes['region'] = handlers.SingleRegionHandler
self.keep_levels = 1
self.keep_best = 10
self.handler_classes = {
'model': handlers.SingleModelHandler,
'scan': handlers.ScanHandler,
'sym': handlers.SymmetryHandler,
'emit': handlers.EmitterHandler,
'region': handlers.SingleRegionHandler
}
self.calculator_class = None
self._tasks_fields = []
self._db = database.ResultsDatabase()
def create_domain(self):
"""
@ -676,8 +605,8 @@ class Project(object):
@return: None
"""
self.scans = []
self._combined_scan = None
self._combined_modf = None
self.combined_scan = None
self.combined_modf = None
def add_scan(self, filename, emitter, initial_state, is_modf=False, modf_model=None):
"""
@ -695,7 +624,7 @@ class Project(object):
* intensity vs theta and phi (hemisphere or hologram scan)
the method calculates the modulation function if @c is_modf is @c False.
it also updates @c _combined_scan and @c _combined_modf which may be used as R-factor comparison targets.
it also updates @c combined_scan and @c combined_modf which may be used as R-factor comparison targets.
@param filename: (string) file name of the experimental data, possibly including a path.
@ -732,22 +661,26 @@ class Project(object):
scan.modulation = None
if scan.raw_data is not None:
if self._combined_scan is not None:
dtype = md.common_dtype((self._combined_scan, scan.raw_data))
self._combined_scan = np.hstack((self._combined_scan, md.restructure_data(scan.raw_data, dtype)))
if self.combined_scan is not None:
dt = md.common_dtype((self.combined_scan, scan.raw_data))
d1 = md.restructure_data(self.combined_scan, dt)
d2 = md.restructure_data(scan.raw_data, dt)
self.combined_scan = np.hstack((d1, d2))
else:
self._combined_scan = scan.raw_data.copy()
self.combined_scan = scan.raw_data.copy()
else:
self._combined_scan = None
self.combined_scan = None
if scan.modulation is not None:
if self._combined_modf is not None:
dtype = md.common_dtype((self._combined_modf, scan.modulation))
self._combined_modf = np.hstack((self._combined_modf, md.restructure_data(scan.modulation, dtype)))
if self.combined_modf is not None:
dt = md.common_dtype((self.combined_modf, scan.modulation))
d1 = md.restructure_data(self.combined_modf, dt)
d2 = md.restructure_data(scan.modulation, dt)
self.combined_modf = np.hstack((d1, d2))
else:
self._combined_modf = scan.modulation.copy()
self.combined_modf = scan.modulation.copy()
else:
self._combined_modf = None
self.combined_modf = None
return scan
@ -783,9 +716,16 @@ class Project(object):
def set_output(self, filename):
"""
set base name of output file
set path and base name of output file.
path and name are copied to the output_file attribute.
path is copied to the output_dir attribute.
if the path is missing, the destination is the current working directory.
"""
self.output_file = filename
path, name = os.path.split(filename)
self.output_dir = path
def set_timedelta_limit(self, timedelta):
"""
@ -797,12 +737,15 @@ class Project(object):
def combine_symmetries(self, parent_task, child_tasks):
"""
combine results of different symmetry into one result. calculate the modulation function.
combine results of different symmetry into one result and calculate the modulation function.
the symmetry results are read from the file system using the indices defined by the child_tasks,
and the combined result is written to the file system with the index defined by parent_task.
by default, this method adds all symmetries with equal weight.
weights can be defined in the model dictionary with keys 'wsym0', 'wsym1', etc.
missing weights default to 1.
note: to avoid correlated parameters, one symmetry must always have a fixed weight.
@param parent_task: (CalculationTask) parent task of the symmetry tasks.
the method must write the results to the files indicated
@ -817,19 +760,26 @@ class Project(object):
@raise IndexError if child_tasks is empty
@raise KeyError if a filename is missing
@raise IOError if a filename is missing
@note the weights of the symmetries (in derived classes) can be part of the optimizable model parameters.
the model parameters are available as the @c model attribute of the calculation tasks.
"""
result_data = None
sum_weights = 0.
for task in child_tasks:
data = md.load_data(task.result_filename)
if result_data is not None:
result_data['i'] += data['i']
else:
result_data = data
if result_data is None:
result_data = data.copy()
result_data['i'] = 0.
try:
weight = task.model['wsym{}'.format(task.id.sym)]
except KeyError:
weight = 1.
result_data['i'] += weight * data['i']
sum_weights += weight
result_data['i'] /= sum_weights
md.save_data(parent_task.result_filename, result_data)
@ -865,7 +815,7 @@ class Project(object):
@raise IndexError if child_tasks is empty
@raise KeyError if a filename is missing
@raise IOError if a filename is missing
@note the weights of the emitters (in derived classes) can be part of the optimizable model parameters.
the model parameters are available as the @c model attribute of the calculation tasks.
@ -898,7 +848,7 @@ class Project(object):
the datasets of the scans are appended.
this is done for intensity and modulation data independently.
@param parent_task: (CalculationTask) parent task of the symmetry tasks.
@param parent_task: (CalculationTask) parent task of the scan tasks.
the method must write the results to the files indicated
by the @c result_filename and @c modf_filename attributes.
@ -910,14 +860,12 @@ class Project(object):
@return: None
@raise IndexError if child_tasks is empty.
@raise KeyError if a filename is missing.
"""
# intensity
try:
stack1 = [md.load_data(task.result_filename) for task in child_tasks]
except (KeyError, IOError):
except IOError:
parent_task.result_filename = ""
else:
dtype = md.common_dtype(stack1)
@ -928,7 +876,7 @@ class Project(object):
# modulation
try:
stack1 = [md.load_data(task.modf_filename) for task in child_tasks]
except (KeyError, IOError):
except IOError:
parent_task.modf_filename = ""
else:
dtype = md.common_dtype(stack1)
@ -936,6 +884,142 @@ class Project(object):
result_modf = np.hstack(tuple(stack2))
md.save_data(parent_task.modf_filename, result_modf)
def combine_regions(self, parent_task, child_tasks):
"""
combine results from different regions into one result, for intensity and modulation.
the scan results are read from the file system using the indices defined by the child_tasks,
and the combined result is written to the file system with the index defined by parent_task.
the datasets of the regions are appended and sorted in the standard order of the data module.
if the resulting length differs from the corresponding experimental scan,
an error is printed to the logger, but the calculation continues.
the modulation function is calculated by calling @ref calc_modulation.
@param parent_task: (CalculationTask) parent task of the region tasks.
the method writes the results to the file names
given by the @c result_filename and @c modf_filename attributes.
@param child_tasks: (sequence of CalculationTask) tasks which identify each region.
the reads the source data from the files
indicated by the @c result_filename attributes.
the sequence is sorted by task ID, i.e., essentially, by region index.
@return: None
@raise IndexError if child_tasks is empty.
"""
# intensity
try:
stack1 = [md.load_data(task.result_filename) for task in child_tasks]
except IOError:
parent_task.result_valid = False
parent_task.result_filename = ""
else:
dtype = md.common_dtype(stack1)
stack2 = [md.restructure_data(data, dtype) for data in stack1]
result_data = np.hstack(tuple(stack2))
md.sort_data(result_data)
md.save_data(parent_task.result_filename, result_data)
scan = self.scans[parent_task.id.scan]
if result_data.shape[0] != scan.raw_data.shape[0]:
logger.error(BMsg("scan length mismatch: combined result: {result}, experimental data: {expected}",
result=result_data.shape[0], expected=scan.raw_data.shape[0]))
# modulation
try:
data = md.load_data(parent_task.result_filename)
modf = self.calc_modulation(data, parent_task.model)
except IOError:
parent_task.modf_filename = ""
else:
md.save_data(parent_task.modf_filename, modf)
def setup(self, handlers):
"""
prepare for calculations.
this method is called in the master process before starting the task loop.
at this point the task handlers have been created and set up.
if the project needs to change settings of task handlers it can do so in this method.
this instance writes the header of the tasks.dat file
that will receive sub-task evaluation results from the evaluate_result() method.
@param handlers: dictionary listing the initialized task handler instances.
the dictionary keys are the attribute names of pmsco.dispatch.CalcID:
'model', 'scan', 'sym', 'emit' and 'region'.
@return: None
"""
fields = ["rfac"]
fields.extend(dispatch.CalcID._fields)
fields = ["_" + f for f in fields]
dom = self.create_domain()
model_fields = dom.start.keys()
model_fields.sort(key=lambda name: name.lower())
fields.extend(model_fields)
self._tasks_fields = fields
with open(self.output_file + ".tasks.dat", "w") as outfile:
outfile.write("# ")
outfile.write(" ".join(fields))
outfile.write("\n")
# todo : fill in the descriptive fields, change to file-database
self._db.connect(":memory:")
project_id = self._db.register_project(self.__class__.__name__, sys.argv[0])
job_id = self._db.register_job(project_id,
"job-name",
self.mode,
socket.gethostname(),
"git-hash",
datetime.datetime.now(),
"description")
self._db.register_params(model_fields)
self._db.create_models_view()
def evaluate_result(self, parent_task, child_tasks):
"""
evaluate the result of a calculation task.
this method is called from the add_result of the task handlers at each level.
it gives the project a hook to check the progress of a model at any level of the task tree.
the method calculates the r-factor by calling the Project.calc_rfactor method.
the result is written to the task.rfac field and to the .tasks.dat file.
invalid and region-level results are skipped.
this method is called in the master process only.
@param parent_task: (CalculationTask) a calculation task.
@param child_tasks: (sequence of CalculationTask) tasks which identify each scan.
the sequence must be sorted by task ID.
@return: None
"""
if parent_task.result_valid and parent_task.id.region == -1:
try:
parent_task.rfac = self.calc_rfactor(parent_task, child_tasks)
except ValueError:
parent_task.result_valid = False
logger.warning(BMsg("calculation {0} resulted in an undefined R-factor.", parent_task.id))
else:
values_dict = parent_task.id._asdict()
values_dict = {"_" + k: v for k, v in values_dict.items()}
values_dict.update(parent_task.model)
values_dict['_rfac'] = parent_task.rfac
values_list = [values_dict[field] for field in self._tasks_fields]
with open(self.output_file + ".tasks.dat", "a") as outfile:
outfile.write(" ".join(format(value) for value in values_list) + "\n")
self._db.insert_result(parent_task.id, values_dict)
return None
# noinspection PyUnusedLocal
def calc_modulation(self, data, model):
"""
@ -965,31 +1049,246 @@ class Project(object):
return md.calc_modfunc_loess(data)
def calc_rfactor(self, task):
def calc_rfactor(self, parent_task, child_tasks):
"""
calculate the R-factor of a task.
calculate the r-factor of a task.
the method calculates the R-factor over the combined scans.
the corresponding experimental data is taken from self._combined_modf.
the r-factor is calculated on the experimental and simulated modulation functions.
the algorithm differs for the model level and the lower task levels.
at the model level, the calculation is delegated to Project.combine_rfactors.
at all other levels, the r-factor is calculated by Project.rfactor,
where the simulated is loaded from the file specified by parent_task.modf_filename
and the experimental data from Project.scan.
this method is called by the model handler.
this method is called by the task handlers.
all child tasks belonging to the parent task must be complete.
by default, the R-factor is calculated by data.rfactor() over the combined scans.
override this method in your project to use a different R-factor algorithm.
to select or implement a specific R-factor algorithm,
the project sub-class should override Project.rfactor.
to combine scan r-factors, it should override or patch Project.combine_rfactors.
@param task: (CalculationTask) a model task.
@version in earlier versions,
projects had to override this method to implement their algorithm.
this has lead to duplication of common code.
the r-factor algorithm is now distributed over several methods,
and the method signature has changed.
new projects should override Project.rfactor and/or Project.combine_rfactors.
@return (int) calculated R-factor.
@param parent_task: (CalculationTask) a calculation task.
@param child_tasks: (sequence of CalculationTask) tasks which identify each scan.
the sequence must be sorted by task ID.
@return (float) calculated R-factor.
@raise ValueError if the function fails (e.g. division by zero or all elements non-finite).
"""
task_data = md.load_data(task.modf_filename)
result_r = md.rfactor(self._combined_modf, task_data)
if parent_task.id.scan >= 0:
task_data = md.load_data(parent_task.modf_filename)
exp_data = self.scans[parent_task.id.scan].modulation
result_r = self.rfactor(exp_data, task_data)
else:
result_r = self.combine_rfactors(parent_task, child_tasks)
return result_r
def rfactor(self, exp_data, theo_data):
"""
calculate the r-factor of simulated diffraction data.
in this class, the method calls the data.rfactor function to calculate the r-factor.
override this method in your project to use a different R-factor algorithm.
the input arrays must have the same shape,
and the coordinate columns must be identical (they are ignored, however).
the array elements are compared element-by-element.
terms having NaN intensity are ignored.
if the sigma column is present in experiment and non-zero,
the R-factor terms are weighted.
@param exp_data: (numpy structured array)
ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
if an @c s field is present and non-zero,
the R-factor terms are weighted by 1/sigma**2.
@param theo_data: (numpy structured array)
ETPI or ETPAI array containing the calculated modulation functions.
@return: (float) scalar R-factor
@raise ValueError if the function fails (e.g. division by zero or all elements non-finite).
"""
return md.rfactor(exp_data, theo_data)
def opt_rfactor(self, exp_data, theo_data):
"""
calculate the r-factor of simulated diffraction data, adjusting their amplitude.
this is an alternative r-factor calculation algorithm
using the pmsco.data.optimize_rfactor() function.
to activate this method (replacing the default one), assign it to Project.rfactor
in the overriding __init__ or setup method:
@code{.py}
self.rfactor = self.opt_rfactor
@endcode
@param exp_data: (numpy structured array)
ETPI, ETPIS, ETPAI or ETPAIS array containing the experimental modulation function.
if an @c s field is present and non-zero,
the R-factor terms are weighted by 1/sigma**2.
@param theo_data: (numpy structured array)
ETPI or ETPAI array containing the calculated modulation functions.
@return: (float) scalar R-factor
@raise ValueError if the function fails (e.g. division by zero or all elements non-finite).
"""
return md.optimize_rfactor(exp_data, theo_data)
def combine_rfactors(self, parent_task, child_tasks):
"""
combine r-factors of child tasks.
the r-factors are taken from the rfac attribute of the child_tasks.
the result is an average of the child r-rfactors.
to produce a balanced result, every child dataset must contain a similar amount of information.
if this is not the case, the child r-factors must be weighted.
weighting is currently not implemented but may be introduced in a future version.
the method is intended to be used at the model level (children are scans).
though it can technically be used at any level where child r-factors are available.
@param parent_task: (CalculationTask) parent task for which the r-factor is calculated,
i.e. a model task.
@param child_tasks: (sequence of CalculationTask) child tasks of parent_tasks
that may be consulted for calculating the r-factor.
@return: (float) r-factor, NaN if parent task is invalid
@raise ValueError or IndexError if child_tasks is empty.
"""
if parent_task.result_valid:
rsum = 0.
for task in child_tasks:
rsum += task.rfac
return rsum / len(child_tasks)
else:
return float('nan')
def alt_combine_rfactors(self, parent_task, child_tasks):
"""
combine r-factors of child tasks by explicit calculation on the combined result.
this is an alternative implementation of combine_rfactors.
instead of using the r-factors from child tasks,
it re-calculates the r-factor for the combined dataset.
this method avoids the issue of weighting
but can introduce bias if the amplitudes of the child datasets differ substantially.
the experimental dataset is loaded from the file specified by the parent task,
the corresponding experimental data is taken from self.combined_modf.
to activate this method, assign it to combine_rfactors.
in the overriding __init__ or setup method:
@code{.py}
self.combine_rfactors = self.alt_combine_rfactors
@endcode
@param parent_task: (CalculationTask) parent task for which the r-factor is calculated,
i.e. a model task.
@param child_tasks: (sequence of CalculationTask) child tasks of parent_tasks
that may be consulted for calculating the r-factor.
@return: (float) r-factor, NaN if parent task is invalid
"""
if parent_task.result_valid:
task_data = md.load_data(parent_task.modf_filename)
exp_data = self.combined_modf
return self.rfactor(exp_data, task_data)
else:
return float('nan')
def export_cluster(self, index, filename, cluster):
"""
export the cluster of a calculation task in XYZ format for diagnostics and reporting.
this method is called with the final cluster just before it is handed over to the calculator.
it saves the atom coordinates in XYZ format for future reference (e.g. graphics).
the method creates two files:
@arg a file with extension '.xyz' contains the whole cluster in XYZ format.
@arg a file with extension '.emit.xyz' contains only emitter atoms in XYZ format.
the first part of the file name is formatted with the output name and the complete task identification.
the file is registered with the file tracker in the 'cluster' category
so that it will be deleted unless the cluster category is selected for keeping.
derived project class may override or extend this method
to carry out further diagnostics or reporting on the cluster.
@param index: (CalcID) calculation index to which the cluster belongs.
region may be -1 if only one cluster is exported for all regions
(clusters do not depend on the scan region).
emit may be -1 if the cluster is a master from which emitter-related child clusters are derived.
@param filename: (str) base file name for the output files.
the filename should be formatted using pmsco.dispatch.CalculationTask.format_filename().
extensions are appended by this method.
@param cluster: a pmsco.cluster.Cluster() object with all atom positions and emitters.
@return: dictionary listing the names of the created files with their category.
the dictionary key is the file name,
the value is the file category (cluster).
"""
_files = {}
xyz_filename = filename + ".xyz"
cluster.save_to_file(xyz_filename, fmt=mc.FMT_XYZ)
_files[xyz_filename] = 'cluster'
xyz_filename = filename + ".emit.xyz"
cluster.save_to_file(xyz_filename, fmt=mc.FMT_XYZ, emitters_only=True)
_files[xyz_filename] = 'cluster'
return _files
def cleanup(self):
"""
delete unwanted files at the end of a project.
@return: None
"""
self.cleanup_files()
self._db.disconnect()
def cleanup_files(self, keep=0):
"""
delete uninteresting files.
these are all files that
belong to one of the self.files.categories_to_delete categories or
do not belong to one of the "best" models.
"best" models are a number (self.keep_best) of models that gave the lowest R-factors
at each task level from root to self.keep_levels.
for example if `keep_best = 10` and `keep_levels = 1`
the 10 best models at the top level, and the 10 best at the scan level are kept.
this means that in total up to `n = 10 + 10 * n_scans` models may be kept,
where n_scans is the number of scan files in the job.
@param keep: minimum number of best models to keep.
0 (default): use the project parameter self.keep_best.
@return None
"""
self.files.delete_files()
if 'rfac' in self.files.categories_to_delete:
keep = max(keep, self.keep_best)
keepers = self._db.query_best_task_models(self.keep_levels, keep)
self.files.delete_models(keep=keepers)

909
pmsco/swarm.py
View File

@ -1,909 +0,0 @@
"""
@package pmsco.swarm
particle swarm optimization handler.
the module starts multiple MSC calculations and optimizes the model parameters
according to the particle swarm optimization algorithm.
Particle swarm optimization adapted from
D. A. Duncan et al., Surface Science 606, 278 (2012)
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import division
import copy
import os
import datetime
import logging
import numpy as np
import handlers
from helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
CONSTRAIN_MODES = {'re-enter', 'bounce', 'scatter', 'stick', 'expand'}
class Population(object):
"""
particle swarm population.
"""
## @var size_req
# requested number of particles.
# read-only. call setup() to change this attribute.
## @var model_start
# (dict) initial model parameters.
# read-only. call setup() to change this attribute.
## @var model_min
# (dict) low limits of the model parameters.
# read-only. call setup() to change this attribute.
## @var model_max
# (dict) high limits of the model parameters.
# if min == max, the parameter is kept constant.
# read-only. call setup() to change this attribute.
## @var model_max
# (dict) high limits of the model parameters.
# read-only. call setup() to change this attribute.
## @var model_step
# (dict) initial velocity (difference between two steps) of the particle.
# read-only. call setup() to change this attribute.
## @var friends
# number of other particles that each particle consults for the global best fit.
# default = 3.
## @var momentum
# momentum of the particle.
# default = 0.689343.
## @var attract_local
# preference for returning to the local best fit
# default = 1.92694.
## @var attract_global
# preference for heading towards the global best fit.
# default = 1.92694
## @var generation
# generation number. the counter is incremented by advance_population().
# initial value = 0.
## @var model_count
# model number.
# the counter is incremented by advance_particle() each time a particle position is changed.
# initial value = 0.
## @var pos
# (numpy.ndarray) current positions of each particle.
#
# the column names include the names of the model parameters, taken from domain.start,
# and the special names @c '_particle', @c '_model', @c '_rfac'.
# the special fields have the following meanings:
#
# * @c '_particle': index of the particle in the array.
# the particle index is used to match a calculation result and its original particle.
# it must be preserved during the calculation process.
#
# * @c '_gen': generation number.
# the generation number counts the number of calls to advance_population().
# this field is not used internally.
# the first population is generation 0.
#
# * @c '_model': model number.
# the model number counts the number of calls to advance_particle().
# the field is filled with the current value of model_count whenever the position is changed.
# this field is not used internally.
# the model handlers use it to derive their model ID.
#
# * @c '_rfac': calculated R-factor for this position.
# this field is meaningful in the best and results arrays only
# where it is set by the add_result() method.
# in the pos and vel arrays, the field value is arbitrary.
#
# @note if your read a single element, e.g. pos[0], from the array, you will get a numpy.void object.
# this object is a <em>view</em> of the original array item
## @var vel
# (numpy.ndarray) current the velocities of each particle.
# the structure is the same as for the pos array.
## @var best
# (numpy.ndarray) best positions found by each particle so far.
# the structure is the same as for the pos array.
## @var results
# (numpy.ndarray) all positions and resulting R-factors calculated.
# the structure is the same as for the pos array.
## @var _hold_once
# (bool) hold the population once during the next update.
# if _hold_once is True, advance_population() will skip the update process once.
# this flag is set by setup() because it sets up a valid initial population.
# the caller then doesn't have to care whether to skip advance_population() after setup.
def __init__(self):
"""
initialize the population object.
"""
self.size_req = 0
self.model_start = {}
self.model_min = {}
self.model_max = {}
self.model_step = {}
self.friends = 3
self.momentum = 0.689343
self.attract_local = 1.92694
self.attract_global = 1.92694
self.position_constrain_mode = 'default'
self.velocity_constrain_mode = 'default'
self.generation = 0
self.model_count = 0
self._hold_once = False
self.pos = None
self.vel = None
self.best = None
self.results = None
def pos_gen(self):
"""
generator for dictionaries of the pos array.
the generator can be used to loop over the array.
on each iteration, it yields a dictionary of the position at the current index.
for example,
@code{.py}
for pos in pop.pos_gen():
print pos['_index'], pos['_rfac']
@endcode
"""
return ({name: pos[name] for name in pos.dtype.names} for pos in self.pos)
def vel_gen(self):
"""
generator for dictionaries of the vel array.
@see pos_gen() for details.
"""
return ({name: vel[name] for name in vel.dtype.names} for vel in self.vel)
def best_gen(self):
"""
generator for dictionaries of the best array.
@see pos_gen() for details.
"""
return ({name: best[name] for name in best.dtype.names} for best in self.best)
def results_gen(self):
"""
generator for dictionaries of the results array.
@see pos_gen() for details.
"""
return ({name: results[name] for name in results.dtype.names} for results in self.results)
@staticmethod
def get_model_dtype(model_params):
"""
get numpy array data type for model parameters and swarm control variables.
@param model_params: dictionary of model parameters or list of parameter names.
@return: dtype for use with numpy array constructors.
this is a sorted list of (name, type) tuples.
"""
dt = []
for key in model_params:
dt.append((key, 'f4'))
dt.append(('_particle', 'i4'))
dt.append(('_gen', 'i4'))
dt.append(('_model', 'i4'))
dt.append(('_rfac', 'f4'))
dt.sort(key=lambda t: t[0].lower())
return dt
def setup(self, size, domain, history_file="", recalc_history=True):
"""
set up the population arrays seeded with previous results and the start model.
* set the population parameters and allocate the data arrays.
* set one particle to the initial guess, and the others to positions from a previous results file.
if the file contains less particles than allocated, the remaining particles are initialized randomly.
seeding from a history file can be used to continue an interrupted optimization process.
the method loads the results into the best and position arrays,
and updates the other arrays and variables
so that the population can be advanced and calculated.
by default, the calculations of the previous parameters are repeated.
this is recommended whenever the code, the experimental input, or the project arguments change
because all of them may have an influence on the R-factor.
re-calculation can be turned off by setting recalc_history to false.
this is recommended only if the calculation is a direct continuation of a previous one
without any changes to the code or input.
in that case, the previous results are marked as generation -1 with a negative model number.
upon the first iteration before running the scattering calculations,
new parameters will be derived by the swarm algorithm.
@param size: requested number of particles.
@param domain: definition of initial and limiting model parameters
expected by the cluster and parameters functions.
@arg domain.start: initial guess.
@arg domain.min: minimum values allowed.
@arg domain.max: maximum values allowed. if min == max, the parameter is kept constant.
@arg domain.step: initial velocity (difference between two steps) for particle swarm.
@param history_file: name of the results history file.
this can be a file created by the @ref save_array or @ref save_results methods.
the columns of the plain-text file contain model parameters and
the _rfac values of a previous calculation.
additional columns are ignored.
the first row must contain the column names.
if a parameter column is missing,
the corresponding parameter is seeded with a random value within the domain.
in this case, a warning is added to the log file.
the number of rows does not need to be equal to the population size.
if it is lower, the remaining particles are initialized randomly.
if it is higher, only the ones with the lowest R factors are used.
results with R >= 1.0 are ignored in any case.
@param recalc_history: select whether the R-factors of the historic models are calculated again.
this is useful if the historic data was calculated for a different cluster, different set of parameters,
or different experimental data, and if the R-factors of the new optimization may be systematically greater.
set this argument to False only if the calculation is a continuation of a previous one
without any changes to the code.
@return: None
"""
self.size_req = size
self.model_start = domain.start
self.model_min = domain.min
self.model_max = domain.max
self.model_step = domain.step
# allocate arrays
dt = self.get_model_dtype(self.model_start)
self.pos = np.zeros(self.size_req, dtype=dt)
self.vel = np.zeros(self.size_req, dtype=dt)
self.results = np.empty((0), dtype=dt)
# randomize population
self.generation = 0
self.randomize()
self.pos['_particle'] = np.arange(self.size_req)
self.pos['_gen'] = self.generation
self.pos['_model'] = np.arange(self.size_req)
self.pos['_rfac'] = 2.1
self.model_count = self.size_req
# add previous results
if history_file:
hist = np.genfromtxt(history_file, names=True)
hist = hist[hist['_rfac'] < 1.0]
hist.sort(order='_rfac')
hist_size = min(hist.shape[0], self.size_req - 1)
discarded_fields = {'_particle', '_gen', '_model'}
source_fields = set(hist.dtype.names) - discarded_fields
dest_fields = set(self.pos.dtype.names) - discarded_fields
common_fields = source_fields & dest_fields
if len(common_fields) < len(dest_fields):
logger.warning(BMsg("missing columns in history file {hf} default to random seed value.",
hf=history_file))
for name in common_fields:
self.pos[name][0:hist_size] = hist[name][0:hist_size]
self.pos['_particle'] = np.arange(self.size_req)
logger.info(BMsg("seeding swarm population with {hs} models from history file {hf}.",
hs=hist_size, hf=history_file))
if recalc_history:
self.pos['_gen'] = self.generation
self.pos['_model'] = np.arange(self.size_req)
self.pos['_rfac'] = 2.1
logger.info("historic models will be re-calculated.")
else:
self.pos['_gen'][0:hist_size] = -1
self.pos['_model'][0:hist_size] = -np.arange(hist_size) - 1
self.model_count = self.size_req - hist_size
self.pos['_model'][hist_size:] = np.arange(self.model_count)
logger.info("historic models will not be re-calculated.")
# seed last particle with start parameters
self.seed(self.model_start, index=-1)
# initialize best array
self.best = self.pos.copy()
self._hold_once = True
def randomize(self, pos=True, vel=True):
"""
initializes a random population.
the position array is filled with random values (uniform distribution) from the parameter domain.
velocity values are randomly chosen between -1/8 to 1/8 times the width (max - min) of the parameter domain.
the method does not update the particle info fields.
@param pos: randomize positions. if False, the positions are not changed.
@param vel: randomize velocities. if False, the velocities are not changed.
"""
if pos:
for key in self.model_start:
self.pos[key] = ((self.model_max[key] - self.model_min[key]) *
np.random.random_sample(self.pos.shape) + self.model_min[key])
if vel:
for key in self.model_start:
self.vel[key] = ((self.model_max[key] - self.model_min[key]) *
(np.random.random_sample(self.pos.shape) - 0.5) / 4.0)
def seed(self, params, index=0):
"""
set the one of the particles to the specified seed values.
the method does not update the particle info fields.
@param params: dictionary of model parameters.
the keys must match the ones of domain.start.
@param index: index of the particle that is seeded.
the index must be in the allowed range of the self.pos array.
0 is the first, -1 the last particle.
"""
for key in params:
self.pos[key][index] = params[key]
def update_particle_info(self, index, inc_model=True):
"""
set the internal particle info fields.
the fields @c _particle, @c _gen, and @c _model are updated with the current values.
@c _rfac is set to the default value 2.1.
this method must be called after each change of particle position.
@param index: (int) particle index.
@param inc_model: (bool) if True, increment the model count afterwards.
@return: None
"""
self.pos['_particle'][index] = index
self.pos['_gen'][index] = self.generation
self.pos['_model'][index] = self.model_count
self.pos['_rfac'][index] = 2.1
if inc_model:
self.model_count += 1
def advance_population(self):
"""
advance the population by one step.
this method just calls advance_particle() for each particle of the population.
if generation is lower than zero, the method increases the generation number but does not advance the particles.
@return: None
"""
if not self._hold_once:
self.generation += 1
for index, __ in enumerate(self.pos):
self.advance_particle(index)
self._hold_once = False
def advance_particle(self, index):
"""
advance a particle by one step.
@param index: index of the particle in the population.
"""
# note: the following two identifiers are views,
# assignment will modify the original array
pos = self.pos[index]
vel = self.vel[index]
# best fit that this individual has seen
xl = self.best[index]
# best fit that a group of others have seen
xg = self.best_friend(index)
for key in self.model_start:
# update velocity
dxl = xl[key] - pos[key]
dxg = xg[key] - pos[key]
pv = np.random.random()
pl = np.random.random()
pg = np.random.random()
vel[key] = (self.momentum * pv * vel[key] +
self.attract_local * pl * dxl +
self.attract_global * pg * dxg)
pos[key], vel[key], self.model_min[key], self.model_max[key] = \
self.constrain_velocity(pos[key], vel[key], self.model_min[key], self.model_max[key],
self.velocity_constrain_mode)
# update position
pos[key] += vel[key]
pos[key], vel[key], self.model_min[key], self.model_max[key] = \
self.constrain_position(pos[key], vel[key], self.model_min[key], self.model_max[key],
self.position_constrain_mode)
self.update_particle_info(index)
@staticmethod
def constrain_velocity(_pos, _vel, _min, _max, _mode='default'):
"""
constrain a velocity to the given bounds.
@param _pos: current position of the particle.
@param _vel: new velocity of the particle, i.e. distance to move.
@param _min: lower position boundary.
@param _max: upper position boundary.
_max must be greater or equal to _min.
@param _mode: what to do if a boundary constraint is violated.
reserved for future use. should be set to 'default'.
@return: tuple (new position, new velocity, new lower boundary, new upper boundary).
in the current implementation only the velocity may change.
however, in future versions any of these values may change.
"""
d = abs(_max - _min) / 2.0
if d > 0.0:
while abs(_vel) >= d:
_vel /= 2.0
else:
_vel = 0.0
return _pos, _vel, _min, _max
@staticmethod
def constrain_position(_pos, _vel, _min, _max, _mode='default'):
"""
constrain a position to the given bounds.
@param _pos: new position of the particle, possible out of bounds.
@param _vel: velocity of the particle, i.e. distance from the previous position.
_vel must be lower than _max - _min.
@param _min: lower boundary.
@param _max: upper boundary.
_max must be greater or equal to _min.
@param _mode: what to do if a boundary constraint is violated:
@arg 're-enter': re-enter from the opposite side of the parameter interval.
@arg 'bounce': fold the motion vector at the boundary and move the particle back into the domain.
@arg 'scatter': place the particle at a random place between its old position and the violated boundary.
@arg 'stick': place the particle at the violated boundary.
@arg 'expand': move the boundary so that the particle fits.
@arg 'random': place the particle at a random position between the lower and upper boundaries.
@arg 'default': the default mode is 'bounce'. this may change in future versions.
@return: tuple (new position, new velocity, new lower boundary, new upper boundary).
depending on the mode, any of these values may change.
the velocity is adjusted to be consistent with the change of position.
"""
_rng = max(_max - _min, 0.0)
_old = _pos - _vel
# prevent undershoot
if _vel > 0.0 and _pos < _min:
_pos = _min
_vel = _pos - _old
if _vel < 0.0 and _pos > _max:
_pos = _max
_vel = _pos - _old
assert abs(_vel) <= _rng, \
"velocity: pos = {0}, min = {1}, max = {2}, vel = {3}, _rng = {4}".format(_pos, _min, _max, _vel, _rng)
assert (_vel >= 0 and _pos >= _min) or (_vel <= 0 and _pos <= _max), \
"undershoot: pos = {0}, min = {1}, max = {2}, vel = {3}, _rng = {4}".format(_pos, _min, _max, _vel, _rng)
if _rng > 0.0:
while _pos > _max:
if _mode == 're-enter':
_pos -= _rng
elif _mode == 'bounce' or _mode == 'default':
_pos = _max - (_pos - _max)
_vel = -_vel
elif _mode == 'scatter':
_pos = _old + (_max - _old) * np.random.random()
_vel = _pos - _old
elif _mode == 'stick':
_pos = _max
_vel = _pos - _old
elif _mode == 'expand':
_max = _pos
elif _mode == 'random':
_pos = _min + _rng * np.random.random()
_vel = _pos - _old
else:
raise ValueError('invalid constrain mode')
while _pos < _min:
if _mode == 're-enter':
_pos += _rng
elif _mode == 'bounce' or _mode == 'default':
_pos = _min - (_pos - _min)
_vel = -_vel
elif _mode == 'scatter':
_pos = _old + (_min - _old) * np.random.random()
_vel = _pos - _old
elif _mode == 'stick':
_pos = _min
_vel = _pos - _old
elif _mode == 'expand':
_min = _pos
elif _mode == 'random':
_pos = _min + _rng * np.random.random()
_vel = _pos - _old
else:
raise ValueError('invalid constrain mode')
else:
_pos = _max
_vel = 0.0
return _pos, _vel, _min, _max
# noinspection PyUnusedLocal
def best_friend(self, index):
"""
select the best fit out of a random set of particles
returns the "best friend"
"""
friends = np.random.choice(self.best, self.friends, replace=False)
index = np.argmin(friends['_rfac'])
return friends[index]
def add_result(self, particle, rfac):
"""
add a calculation particle to the results array, and update the best fit array.
@param particle: dictionary of model parameters and particle values.
the keys must correspond to the columns of the pos array,
i.e. the names of the model parameters plus the _rfac, _particle, and _model fields.
@param rfac: calculated R-factor.
the R-factor is written to the '_rfac' field.
@return better (bool): True if the new R-factor is better than the particle's previous best mark.
"""
particle['_rfac'] = rfac
l = [particle[n] for n in self.results.dtype.names]
t = tuple(l)
a = np.asarray(t, dtype=self.results.dtype)
self.results = np.append(self.results, a)
index = particle['_particle']
better = particle['_rfac'] < self.best['_rfac'][index]
if better:
self.best[index] = a
return better
def is_converged(self, tol=0.01):
"""
check whether the population has converged.
convergence is reached when the R-factors of the N latest results,
do not vary more than tol, where N is the size of the population.
@param tol: max. difference allowed between greatest and lowest value of the R factor in the population.
"""
nres = self.results.shape[0]
npop = self.pos.shape[0]
if nres >= npop:
rfac1 = np.min(self.results['_rfac'][-npop:])
rfac2 = np.max(self.results['_rfac'][-npop:])
converg = rfac2 - rfac1 < tol
return converg
else:
return False
def save_array(self, filename, array):
"""
save a population array to a text file.
the columns are space-delimited.
the first line contains the column names.
@param filename: name of destination file, optionally including a path.
@param array: population array to save.
must be one of self.pos, self.vel, self.best, self.results
"""
header = " ".join(self.results.dtype.names)
np.savetxt(filename, array, fmt='%g', header=header)
def load_array(self, filename, array):
"""
load a population array from a text file.
the array to load must be compatible with the current population
(same number of rows, same columns).
the first row must contain column names.
the ordering of columns may be different.
the returned array is ordered according to the array argument.
@param filename: name of source file, optionally including a path.
@param array: population array to load.
must be one of self.pos, self.vel, self.results.
@return array with loaded data.
this may be the same instance as on input.
@raise AssertionError if the number of rows of the two files differ.
"""
data = np.genfromtxt(filename, names=True)
assert data.shape == array.shape
for name in data.dtype.names:
array[name] = data[name]
return array
def save_population(self, base_filename):
"""
save the population array to a set of text files.
the file name extensions are .pos, .vel, and .best
"""
self.save_array(base_filename + ".pos", self.pos)
self.save_array(base_filename + ".vel", self.vel)
self.save_array(base_filename + ".best", self.best)
def load_population(self, base_filename):
"""
load the population array from a set of previously saved text files.
this can be used to continue an optimization job.
the file name extensions are .pos, .vel, and .best.
the files must have the same format as produced by save_population.
the files must have the same number of rows.
"""
self.pos = self.load_array(base_filename + ".pos", self.pos)
self.vel = self.load_array(base_filename + ".vel", self.vel)
self.best = self.load_array(base_filename + ".best", self.best)
def save_results(self, filename):
"""
saves the complete list of calculations results.
"""
self.save_array(filename, self.results)
class ParticleSwarmHandler(handlers.ModelHandler):
"""
model handler which implements the particle swarm optimization algorithm.
"""
## @var _pop (Population)
# holds the population object.
## @var _pop_size (int)
# number of particles in the swarm.
## @var _outfile (file)
# output file for model parametes and R factor.
# the file is open during calculations.
# each calculation result adds one line.
## @var _model_time (timedelta)
# estimated CPU time to calculate one model.
# this value is the maximum time measured of the completed calculations.
# it is used to determine when the optimization should be finished so that the time limit is not exceeded.
## @var _converged (bool)
# indicates that the population has converged.
# convergence is detected by calling Population.is_converged().
# once convergence has been reached, this flag is set, and further convergence tests are skipped.
## @var _timeout (bool)
# indicates when the handler has run out of time,
# i.e. time is up before convergence has been reached.
# if _timeout is True, create_tasks() will not create further tasks,
# and add_result() will signal completion when the _pending_tasks queue becomes empty.
## @var _invalid_limit (int)
# maximum tolerated number of invalid calculations.
#
# if the number of invalid calculations (self._invalid_count) exceeds this limit,
# the optimization is aborted.
# the variable is initialized by self.setup() to 10 times the population size.
def __init__(self):
super(ParticleSwarmHandler, self).__init__()
self._pop = None
self._pop_size = 0
self._outfile = None
self._model_time = datetime.timedelta()
self._converged = False
self._timeout = False
self._invalid_limit = 10
def setup(self, project, slots):
"""
initialize the particle swarm and open an output file.
the population size is set to project.pop_size if it is defined and greater than 4.
otherwise, it defaults to <code>max(2 * slots, 4)</code>.
for good efficiency the population size (number of particles) should be
greater or equal to the number of available processing slots,
otherwise the next generation is created before all particles have been calculated
which may slow down convergence.
if calculations take a long time compared to the available computation time
or spawn a lot of sub-tasks due to complex symmetry,
and you prefer to allow for a good number of generations,
you should override the population size.
@param project: project instance.
@param slots: number of calculation processes available through MPI.
@return: None
"""
super(ParticleSwarmHandler, self).setup(project, slots)
_min_size = 4
if project.pop_size:
self._pop_size = max(project.pop_size, _min_size)
else:
self._pop_size = max(self._slots * 2, _min_size)
self._pop = Population()
self._pop.setup(self._pop_size, self._project.create_domain(), self._project.history_file,
self._project.recalc_history)
self._invalid_limit = self._pop_size * 10
self._outfile = open(self._project.output_file + ".dat", "w")
self._outfile.write("# ")
self._outfile.write(" ".join(self._pop.results.dtype.names))
self._outfile.write("\n")
return None
def cleanup(self):
self._outfile.close()
super(ParticleSwarmHandler, self).cleanup()
def create_tasks(self, parent_task):
"""
develop the particle population and create a calculation task per particle.
this method advances the population by one step.
it generates one task for each particle if its model number is positive.
negative model numbers indicate that the particle is used for seeding
and does not need to be calculated in the first generation.
if the time limit is approaching, no new tasks are created.
the process loop calls this method every time the length of the task queue drops
below the number of calculation processes (slots).
this means in particular that a population will not be completely calculated
before the next generation starts.
for efficiency reasons, we do not wait until a population is complete.
this will cause a certain mixing of generations and slow down convergence
because the best peer position in the generation may not be known yet.
the effect can be reduced by making the population larger than the number of processes.
@return list of generated tasks. empty list if the optimization has converged (see Population.is_converged()).
"""
super(ParticleSwarmHandler, self).create_tasks(parent_task)
# this is the top-level handler, we expect just one parent: root.
parent_id = parent_task.id
assert parent_id == (-1, -1, -1, -1, -1)
self._parent_tasks[parent_id] = parent_task
time_pending = self._model_time * len(self._pending_tasks)
time_avail = (self.datetime_limit - datetime.datetime.now()) * max(self._slots, 1)
out_tasks = []
if not self._timeout and not self._converged:
self._pop.advance_population()
for pos in self._pop.pos_gen():
time_pending += self._model_time
if time_pending > time_avail:
self._timeout = True
logger.info("time limit reached")
break
if pos['_model'] >= 0:
new_task = parent_task.copy()
new_task.parent_id = parent_id
new_task.model = pos
new_task.change_id(model=pos['_model'])
child_id = new_task.id
self._pending_tasks[child_id] = new_task
out_tasks.append(new_task)
return out_tasks
def add_result(self, task):
"""
calculate the R factor of the result and add it to the results list of the population.
* save the current population.
* append the result to the result output file.
* update the execution time statistics.
* remove temporary files if requested.
* check whether the population has converged.
@return parent task (CalculationTask) if the optimization has converged, @c None otherwise.
"""
super(ParticleSwarmHandler, self).add_result(task)
self._complete_tasks[task.id] = task
del self._pending_tasks[task.id]
parent_task = self._parent_tasks[task.parent_id]
rfac = 1.0
if task.result_valid:
try:
rfac = self._project.calc_rfactor(task)
except ValueError:
task.result_valid = False
self._invalid_count += 1
logger.warning(BMsg("calculation of model {0} resulted in an undefined R-factor.", task.id.model))
task.model['_rfac'] = rfac
self._pop.add_result(task.model, rfac)
self._pop.save_population(self._project.output_file + ".pop")
if self._outfile:
s = (str(task.model[name]) for name in self._pop.results.dtype.names)
self._outfile.write(" ".join(s))
self._outfile.write("\n")
self._outfile.flush()
self._project.files.update_model_rfac(task.id.model, rfac)
self._project.files.set_model_complete(task.id.model, True)
if task.result_valid:
if self._pop.is_converged() and not self._converged:
logger.info("population converged")
self._converged = True
if task.time > self._model_time:
self._model_time = task.time
else:
if self._invalid_count >= self._invalid_limit:
logger.error("number of invalid calculations (%u) exceeds limit", self._invalid_count)
self._converged = True
# optimization complete?
if (self._timeout or self._converged) and len(self._pending_tasks) == 0:
del self._parent_tasks[parent_task.id]
else:
parent_task = None
self.cleanup_files(keep=self._pop_size)
return parent_task

36
projects/demo/fcc.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2
"""
@package pmsco.projects.fcc
@ -13,24 +13,19 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
import sys
import os
from __future__ import print_function
import math
import numpy as np
import os.path
import periodictable as pt
import argparse
import logging
base_dir = os.path.dirname(__file__) or '.'
package_dir = os.path.join(base_dir, '../..')
package_dir = os.path.abspath(package_dir)
sys.path.append(package_dir)
import pmsco.pmsco
import pmsco.cluster as mc
import pmsco.project as mp
import pmsco.data as md
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
@ -82,7 +77,7 @@ class FCC111Project(mp.Project):
clu.add_layer(self.element, a_l1, a1, a2)
clu.add_layer(self.element, a_l2, a1, a2)
clu.add_layer(self.element, a_l3, a1, a2)
clu.add_bulk(self.element, a_bulk, a1, a2, a3)
clu.add_bulk(self.element, a_bulk, a1, a2, a3, a_bulk[2] + 0.01)
clu.set_emitter(a_l1)
@ -110,7 +105,7 @@ class FCC111Project(mp.Project):
params.scattering_level = 5
params.fcut = 15.0
params.cut = 15.0
params.angular_broadening = 0.0
params.angular_resolution = 0.0
params.lattice_constant = 1.0
params.z_surface = model['Zsurf']
params.atom_types = 3
@ -265,20 +260,3 @@ def parse_project_args(_args):
parsed_args = parser.parse_known_args(_args)
return parsed_args
def main():
args, unknown_args = pmsco.pmsco.parse_cli()
if unknown_args:
project_args = parse_project_args(unknown_args)
else:
project_args = None
project = create_project(project_args.element)
pmsco.pmsco.set_common_args(project, args)
set_project_args(project, project_args)
pmsco.pmsco.run_project(project)
if __name__ == '__main__':
main()
sys.exit(0)

206
projects/twoatom/twoatom.py
View File

@ -1,5 +1,3 @@
#!/usr/bin/env python
"""
@package projects.twoatom
Two-atom demo scattering calculation project
@ -8,30 +6,129 @@ this file is specific to the project and the state of the data analysis,
as it contains particular parameter values.
"""
from __future__ import absolute_import
from __future__ import division
import sys
import os
import math
import numpy as np
import periodictable as pt
from __future__ import print_function
import argparse
import logging
import math
import numpy as np
import os.path
import periodictable as pt
# adjust the system path so that the main PMSCO code is found
base_dir = os.path.dirname(__file__) or '.'
package_dir = os.path.join(base_dir, '../..')
package_dir = os.path.abspath(package_dir)
sys.path.append(package_dir)
import pmsco.pmsco
import pmsco.cluster as mc
import pmsco.project as mp
import pmsco.data as md
from pmsco.helpers import BraceMessage as BMsg
logger = logging.getLogger(__name__)
class TwoatomCluster(mc.ClusterGenerator):
"""
cluster of two atoms.
atom A (top) is set at position (0, 0, 0), atom B (bottom) at (-dx, -dy, -dz)
where dx, dy and dz are calculated from model parameters.
the type of the atoms is set upon construction.
the model parameters are:
@arg @c model['dAB'] : distance between the two atoms in Angstrom.
@arg @c model['th'] : polar angle of the connection line, 0 = on top geometry.
@arg @c model['ph'] : azimuthal angle of the connection line, 0 = polar angle affects X coordinate.
the class is designed to be reusable in various projects.
object attributes refine the atom types and the mapping of project-specific model parameters.
"""
## @var atom_types (dict)
# chemical element numbers of the cluster atoms.
#
# atom 'A' is the top atom, 'B' the bottom one.
# upon construction both atoms are set to oxygen.
# to customize, call @ref set_atom_type.
## @var model_dict (dict)
# mapping of model parameters to cluster parameters
#
# the default model parameters used by the cluster are 'dAB', 'th' and 'ph'.
# if the project uses other parameter names, e.g. 'dCO' instead of 'dAB',
# the project-specific names can be declared here.
# in the example, set model_dict['dAB'] = 'dCO'.
def __init__(self, project):
"""
initialize the cluster generator.
the atoms and model dictionary are given default values.
see @ref set_atom_type and @ref model_dict for customization.
@param project: project instance.
"""
super(TwoatomCluster, self).__init__(project)
self.atom_types = {'A': pt.O.number, 'B': pt.O.number}
self.model_dict = {'dAB': 'dAB', 'th': 'th', 'ph': 'ph'}
def set_atom_type(self, atom, element):
"""
set the type (chemical element) of an atom.
@param atom: atom key, 'A' (top) or 'B' (bottom).
@param element: chemical element number or symbol.
"""
try:
self.atom_types[atom] = int(element)
except ValueError:
self.atom_types[atom] = pt.elements.symbol(element.strip()).number
def count_emitters(self, model, index):
"""
return the number of emitter configurations.
this cluster supports only one configuration.
@param model:
@param index:
@return 1
"""
return 1
def create_cluster(self, model, index):
"""
create a cluster given the model parameters and index.
@param model:
@param index:
@return a pmsco.cluster.Cluster object containing the atomic coordinates.
"""
r = model[self.model_dict['dAB']]
try:
th = math.radians(model[self.model_dict['th']])
except KeyError:
th = 0.
try:
ph = math.radians(model[self.model_dict['ph']])
except KeyError:
ph = 0.
dx = r * math.sin(th) * math.cos(ph)
dy = r * math.sin(th) * math.sin(ph)
dz = r * math.cos(th)
clu = mc.Cluster()
clu.comment = "{0} {1}".format(self.__class__, index)
clu.set_rmax(r * 2.0)
a_top = np.array((0.0, 0.0, 0.0))
a_bot = np.array((-dx, -dy, -dz))
clu.add_atom(self.atom_types['A'], a_top, 1)
clu.add_atom(self.atom_types['B'], a_bot, 0)
return clu
class TwoatomProject(mp.Project):
"""
two-atom calculation project class.
@ -49,31 +146,12 @@ class TwoatomProject(mp.Project):
def __init__(self):
super(TwoatomProject, self).__init__()
self.scan_dict = {}
def create_cluster(self, model, index):
"""
calculate a specific set of atom positions given the optimizable parameters.
the cluster contains a nitrogen in the top layer,
and a nickel atom in the second layer.
The layer distance and the angle can be adjusted by parameters.
@param model: (dict) optimizable parameters
"""
clu = mc.Cluster()
clu.comment = "{0} {1}".format(self.__class__, index)
clu.set_rmax(10.0)
a_N = np.array((0.0, 0.0, 0.0))
rad_pNNi = math.radians(model['pNNi'])
a_Ni1 = np.array((0.0,
-model['dNNi'] * math.sin(rad_pNNi),
-model['dNNi'] * math.cos(rad_pNNi)))
clu.add_atom(pt.N.number, a_N, 1)
clu.add_atom(pt.Ni.number, a_Ni1, 0)
return clu
self.cluster_generator = TwoatomCluster(self)
self.cluster_generator.set_atom_type('A', 'N')
self.cluster_generator.set_atom_type('B', 'Ni')
self.cluster_generator.model_dict['dAB'] = 'dNNi'
self.cluster_generator.model_dict['th'] = 'pNNi'
self.cluster_generator.model_dict['ph'] = 'aNNi'
def create_params(self, model, index):
"""
@ -93,13 +171,13 @@ class TwoatomProject(mp.Project):
params.scattering_level = 5
params.fcut = 15.0
params.cut = 15.0
params.angular_broadening = 0.0
params.angular_resolution = 0.0
params.lattice_constant = 1.0
params.z_surface = model['Zsurf']
params.atom_types = 3
params.atomic_number = [7, 28]
params.phase_file = ["hbn_n.pha", "ni.pha"]
params.msq_displacement = [0.01, 0.01, 0.00]
params.phase_files = {self.cluster_generator.atom_types['A']: "",
self.cluster_generator.atom_types['B']: ""}
params.msq_displacement = {self.cluster_generator.atom_types['A']: 0.01,
self.cluster_generator.atom_types['B']: 0.0}
params.planewave_attenuation = 1.0
params.inner_potential = model['V0']
params.work_function = 3.6
@ -153,6 +231,27 @@ class TwoatomProject(mp.Project):
return dom
def example_intensity(e, t, p, a):
"""
arbitrary intensity pattern for example data
this function can be used to calculate the intensity in example scan files.
the function implements an arbitrary modulation function
@param e: energy
@param t: theta
@param p: phi
@param a: alpha
@return intensity
"""
i = np.random.random() * 1e6 * \
np.cos(np.radians(t)) ** 2 * \
np.cos(np.radians(a)) ** 2 * \
np.cos(np.radians(p)) ** 2 * \
np.sin(e / 1000. * np.pi * 0.1 / np.sqrt(e)) ** 2
return i
def create_project():
"""
create a new TwoatomProject calculation project.
@ -230,20 +329,3 @@ def parse_project_args(_args):
parsed_args = parser.parse_args(_args)
return parsed_args
def main():
args, unknown_args = pmsco.pmsco.parse_cli()
if unknown_args:
project_args = parse_project_args(unknown_args)
else:
project_args = None
project = create_project()
pmsco.pmsco.set_common_args(project, args)
set_project_args(project, project_args)
pmsco.pmsco.run_project(project)
if __name__ == '__main__':
main()
sys.exit(0)

72038
projects/twoatom/twoatom_energy_alpha.etpai
View File

File diff suppressed because it is too large Load Diff

72038
projects/twoatom/twoatom_energy_theta_0p.etpi
View File

File diff suppressed because it is too large Load Diff

72038
projects/twoatom/twoatom_energy_theta_180p.etpi
View File

File diff suppressed because it is too large Load Diff

32752
projects/twoatom/twoatom_hemi_215e.etpi
View File

File diff suppressed because it is too large Load Diff

32752
projects/twoatom/twoatom_hemi_250e.etpi
View File

File diff suppressed because it is too large Load Diff

4
requirements.txt
View File

@ -1,3 +1,5 @@
attrdict
fasteners
numpy
periodictable
statsmodels
@ -5,3 +7,5 @@ mpi4py
nose
mock
scipy
matplotlib
future

65
tests/test_cluster.py
View File

@ -17,9 +17,14 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
import unittest
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from io import BytesIO
import math
import numpy as np
import unittest
import pmsco.cluster as mc
@ -153,7 +158,7 @@ class TestClusterFunctions(unittest.TestCase):
self.assertEqual(2, clu.get_emitter_count())
result = clu.get_emitters()
expect = [(0., 0., 0., 1), (1., 0., 1., 10)]
self.assertItemsEqual(expect, result)
self.assertEqual(expect, result)
def test_get_z_layers(self):
clu = mc.Cluster()
@ -288,13 +293,21 @@ class TestClusterFunctions(unittest.TestCase):
v_lat3 = np.asarray([0, 0, 1])
clu.add_bulk(7, v_pos, v_lat1, v_lat2, v_lat3)
clu.set_emitter(pos=v_pos)
clu.trim_cylinder(2.3, 4.2)
r0 = 2.3
z0 = 4.2
clu.trim_cylinder(r0, z0)
self.assertEqual(clu.data.dtype, clu.dtype)
self.assertEqual(clu.data.shape[0], 21 * 5)
self.assertEqual(clu.data[1]['i'], 2)
self.assertEqual(clu.data[1]['s'], 'N')
self.assertEqual(clu.data[1]['t'], 7)
self.assertEqual(clu.get_emitter_count(), 1)
n_low = np.sum(clu.data['z'] < -z0)
self.assertEqual(0, n_low)
n_high = np.sum(clu.data['z'] > z0)
self.assertEqual(0, n_high)
n_out = np.sum(clu.data['x']**2 + clu.data['y']**2 > r0**2)
self.assertEqual(0, n_out)
def test_trim_sphere(self):
clu = mc.Cluster()
@ -305,13 +318,39 @@ class TestClusterFunctions(unittest.TestCase):
v_lat3 = np.asarray([0, 0, 1])
clu.add_bulk(7, v_pos, v_lat1, v_lat2, v_lat3)
clu.set_emitter(pos=v_pos)
clu.trim_sphere(2.3)
r0 = 2.3
clu.trim_sphere(r0)
self.assertEqual(clu.data.dtype, clu.dtype)
self.assertEqual(clu.data.shape[0], 39)
self.assertEqual(clu.data[1]['i'], 2)
self.assertEqual(clu.data[1]['s'], 'N')
self.assertEqual(clu.data[1]['t'], 7)
self.assertEqual(clu.get_emitter_count(), 1)
n_out = np.sum(clu.data['x']**2 + clu.data['y']**2 + clu.data['z'] > r0**2)
self.assertEqual(0, n_out)
def test_trim_paraboloid(self):
clu = mc.Cluster()
clu.set_rmax(10.0)
v_pos = np.asarray([0, 0, 0])
v_lat1 = np.asarray([1, 0, 0])
v_lat2 = np.asarray([0, 1, 0])
v_lat3 = np.asarray([0, 0, 1])
clu.add_bulk(7, v_pos, v_lat1, v_lat2, v_lat3)
clu.set_emitter(pos=v_pos)
r0 = 3.5
z0 = -2.3
clu.trim_paraboloid(r0, z0)
self.assertEqual(clu.data.dtype, clu.dtype)
self.assertEqual(63, clu.data.shape[0])
self.assertEqual(2, clu.data[1]['i'])
self.assertEqual('N', clu.data[1]['s'])
self.assertEqual(7, clu.data[1]['t'])
self.assertEqual(1, clu.get_emitter_count())
n_low = np.sum(clu.data['z'] < z0)
self.assertEqual(0, n_low)
n_out = np.sum(clu.data['x']**2 + clu.data['y']**2 > r0**2)
self.assertEqual(0, n_out)
def test_trim_slab(self):
clu = self.create_cube()
@ -319,3 +358,21 @@ class TestClusterFunctions(unittest.TestCase):
self.assertEqual(clu.data.dtype, clu.dtype)
self.assertEqual(clu.data.shape[0], 9 * 2)
self.assertEqual(clu.get_emitter_count(), 1)
def test_save_to_file(self):
clu = self.create_cube()
f = BytesIO()
pos = np.asarray((-1, -1, 0))
clu.set_emitter(pos=pos)
clu.save_to_file(f, mc.FMT_XYZ, "qwerty", emitters_only=True)
f.seek(0)
line = f.readline()
self.assertEqual(line, b"2\n", b"line 1: " + line)
line = f.readline()
self.assertEqual(line, b"qwerty\n", b"line 2: " + line)
line = f.readline()
self.assertRegexpMatches(line, b"H +[0.]+ +[0.]+ +[0.]+", b"line 3: " + line)
line = f.readline()
self.assertRegexpMatches(line, b"Si +[01.-]+ +[01.-]+ +[0.]+", b"line 4: " + line)
line = f.readline()
self.assertEqual(line, b"", b"end of file")

30
tests/test_data.py
View File

@ -20,6 +20,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import math
import numpy as np
@ -80,7 +83,7 @@ class TestDataFunctions(unittest.TestCase):
shape = (10, )
data = md.create_data(shape, dtype=md.DTYPE_ETPAIS)
expected_names = ('e', 't', 'p', 'a', 'i', 's')
self.assertItemsEqual(data.dtype.names, expected_names)
self.assertEqual(data.dtype.names, expected_names)
self.assertEqual(data.shape, shape)
def test_detect_scan_mode_1d(self):
@ -88,10 +91,11 @@ class TestDataFunctions(unittest.TestCase):
expected_mode = ['e']
expected_positions = {}
expected_positions['e'] = np.linspace(0.0, 10.0, 10)
expected_positions['e'] = np.linspace(100.0, 200.0, 10)
self.assertItemsEqual(scan_mode, expected_mode)
self.assertItemsEqual(scan_positions, expected_positions)
self.assertEqual(scan_mode, expected_mode)
for dim in expected_positions:
np.testing.assert_almost_equal(scan_positions[dim], expected_positions[dim], decimal=3)
def test_detect_scan_mode_2d(self):
scan_mode, scan_positions = md.detect_scan_mode(self.ea_scan)
@ -103,8 +107,9 @@ class TestDataFunctions(unittest.TestCase):
expected_positions['p'] = np.zeros((1))
expected_positions['a'] = np.asarray((-1.0, 0.0, 1.0, 2.0))
self.assertItemsEqual(scan_mode, expected_mode)
self.assertItemsEqual(scan_positions, expected_positions)
self.assertEqual(scan_mode, expected_mode)
for dim in expected_positions:
np.testing.assert_almost_equal(scan_positions[dim], expected_positions[dim], decimal=3)
def test_detect_scan_mode_holo(self):
scan_mode, scan_positions = md.detect_scan_mode(self.holo_scan)
@ -115,8 +120,9 @@ class TestDataFunctions(unittest.TestCase):
expected_positions['t'] = self.holo_scan['t']
expected_positions['p'] = self.holo_scan['p']
self.assertItemsEqual(scan_mode, expected_mode)
self.assertItemsEqual(scan_positions, expected_positions)
self.assertEqual(scan_mode, expected_mode)
for dim in expected_positions:
np.testing.assert_almost_equal(scan_positions[dim], expected_positions[dim], decimal=3)
def test_detect_scan_mode_theta(self):
scan = self.holo_scan
@ -130,8 +136,9 @@ class TestDataFunctions(unittest.TestCase):
expected_positions['t'] = np.linspace(1.0, 2.0, scan.shape[0])
expected_positions['p'] = np.ones((1)) * 3.3
self.assertItemsEqual(scan_mode, expected_mode)
self.assertItemsEqual(scan_positions, expected_positions)
self.assertEqual(scan_mode, expected_mode)
for dim in expected_positions:
np.testing.assert_almost_equal(scan_positions[dim], expected_positions[dim], decimal=3)
def test_calc_modfunc_mean_1d(self):
modf = md.calc_modfunc_mean(self.e_scan)
@ -178,6 +185,7 @@ class TestDataFunctions(unittest.TestCase):
exp_modf['i'] = +1.000001
np.testing.assert_array_less(modf['i'], exp_modf['i'])
@unittest.skip("test_calc_modfunc_loess_1d_nan fails and kills the whole test process")
def test_calc_modfunc_loess_1d_nan(self):
"""
check that data.calc_modfunc_loess() ignores NaNs gracefully.
@ -229,7 +237,7 @@ class TestDataFunctions(unittest.TestCase):
# the R factor should be sensitive enough to detect mixed-up axes.
exp_modf['i'] = 0.03 * np.sin((scan['e'] - 150) / 50 * math.pi)
rf = md.rfactor(modf, exp_modf)
print rf
print(rf)
self.assertLessEqual(rf, 0.50)
def test_alpha_mirror_average(self):

456
tests/test_database.py Normal file
View File

@ -0,0 +1,456 @@
"""
@package tests.test_database
unit tests for pmsco.database
the purpose of these tests is to help debugging the code.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@pre nose must be installed (python-nose package on Debian).
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2016 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import datetime
import os.path
import tempfile
import shutil
import numpy as np
import pmsco.database as db
import pmsco.dispatch as dispatch
import pmsco.optimizers.population as population
class TestDatabase(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self.lock_filename = os.path.join(self.test_dir, "test_database.lock")
self.db = db.ResultsDatabase()
self.db.connect(":memory:", lock_filename=self.lock_filename)
def tearDown(self):
self.db.disconnect()
shutil.rmtree(self.test_dir)
@classmethod
def setup_class(cls):
# before any methods in this class
pass
@classmethod
def teardown_class(cls):
# teardown_class() after any methods in this class
pass
def test_regular_params(self):
d1 = {'parA': 1.234, 'par_B': 5.678, '_model': 91, '_rfac': 0.534}
d2 = db.regular_params(d1)
d3 = {'parA': d1['parA'], 'par_B': d1['par_B']}
self.assertEqual(d2, d3)
self.assertIsNot(d2, d1)
def test_special_params(self):
d1 = {'parA': 1.234, 'par_B': 5.678, '_model': 91, '_rfac': 0.534, '_db_model': 99}
d2 = db.special_params(d1)
d3 = {'model': d1['_model'], 'rfac': d1['_rfac']}
self.assertEqual(d2, d3)
self.assertIsNot(d2, d1)
dt = [('parA', 'f4'), ('par_B', 'f4'), ('_model', 'i4'), ('_rfac', 'f4'), ('_db_model', 'f4')]
arr = np.zeros(1, dtype=dt)
for k, v in d1.items():
arr[0][k] = v
d4 = db.special_params(arr[0])
self.assertEqual(d4.keys(), d3.keys())
for k in d4:
self.assertAlmostEqual(d4[k], d3[k])
cid1 = dispatch.CalcID(1, 2, 3, 4, -1)
cid2 = db.special_params(cid1)
cid3 = {'model': 1, 'scan': 2, 'sym': 3, 'emit': 4, 'region': -1}
self.assertEqual(cid2, cid3)
l1 = d1.keys()
l2 = db.special_params(l1)
l3 = d3.keys()
self.assertEqual(list(l2), list(l3))
t1 = tuple(l1)
t2 = db.special_params(t1)
t3 = tuple(l3)
self.assertEqual(t2, t3)
def setup_sample_database(self):
self.db.register_project("unittest", "testcode")
self.db.register_job(self.db.project_id, "testjob", "testmode", "testhost", None, datetime.datetime.now())
self.ex_model = {'parA': 1.234, 'parB': 5.678, '_model': 91, '_rfac': 0.534}
self.db.register_params(self.ex_model)
self.db.insert_model(self.ex_model)
self.db.create_models_view()
def test_register_project(self):
id1 = self.db.register_project("unittest1", "Atest")
self.assertIsInstance(id1, int)
id2 = self.db.register_project("unittest2", "Btest")
self.assertIsInstance(id2, int)
id3 = self.db.register_project("unittest1", "Ctest")
self.assertIsInstance(id3, int)
self.assertNotEqual(id1, id2)
self.assertEqual(id1, id3)
c = self.db._conn.cursor()
c.execute("select count(*) from Projects")
count = c.fetchone()
self.assertEqual(count[0], 2)
c.execute("select name, code from Projects where id=:id", {'id': id1})
row = c.fetchone()
self.assertIsNotNone(row)
self.assertEqual(len(row), 2)
self.assertEqual(row[0], "unittest1")
self.assertEqual(row[1], "Atest")
self.assertEqual(row['name'], "unittest1")
self.assertEqual(row['code'], "Atest")
def test_register_params(self):
self.setup_sample_database()
model5 = {'parA': 2.341, 'parC': 6.785, '_model': 92, '_rfac': 0.453}
self.db.register_params(model5)
expected = ['parA', 'parB', 'parC']
c = self.db._conn.cursor()
c.execute("select * from Params order by key")
results = c.fetchall()
self.assertEqual(len(results), 3)
result_params = [row['key'] for row in results]
self.assertEqual(result_params, expected)
def test_query_project_params(self):
self.setup_sample_database()
project1 = self.db.project_id
self.db.register_project("unittest2", "testcode2")
self.db.register_job(self.db.project_id, "testjob2", "test", "localhost", None, datetime.datetime.now())
model5 = {'parA': 2.341, 'parC': 6.785, '_model': 92, '_rfac': 0.453}
self.db.register_params(model5)
self.db.insert_model(model5)
results = self.db.query_project_params(project_id=project1)
expected = ['parA', 'parB']
self.assertEqual(expected, sorted(results.keys()))
def test_insert_model(self):
self.setup_sample_database()
c = self.db._conn.cursor()
c.execute("select count(*) from Models")
count = c.fetchone()
self.assertEqual(count[0], 1)
c.execute("select * from Models")
row = c.fetchone()
model_id = row['id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['job_id'], self.db.job_id)
self.assertEqual(row['model'], self.ex_model['_model'])
self.assertIsNone(row['gen'])
self.assertIsNone(row['particle'])
sql = "select key, value from ParamValues " + \
"join Params on ParamValues.param_id = Params.id " + \
"where model_id = :model_id"
c.execute(sql, {'model_id': model_id})
result = c.fetchall() # list of Row objects
self.assertEqual(len(result), 2)
for row in result:
self.assertAlmostEqual(row['value'], self.ex_model[row['key']])
def test_query_model(self):
self.setup_sample_database()
c = self.db._conn.cursor()
c.execute("select * from Models")
row = c.fetchone()
model_id = row['id']
model = self.db.query_model(model_id)
del self.ex_model['_model']
del self.ex_model['_rfac']
self.assertEqual(model, self.ex_model)
def test_query_model_array(self):
self.setup_sample_database()
index = {'_scan': -1, '_sym': -1, '_emit': -1, '_region': -1}
model2 = {'parA': 4.123, 'parB': 8.567, '_model': 92, '_rfac': 0.654}
model3 = {'parA': 3.412, 'parB': 7.856, '_model': 93, '_rfac': 0.345}
model4 = {'parA': 4.123, 'parB': 8.567, '_model': 94, '_rfac': 0.354}
model5 = {'parA': 2.341, 'parC': 6.785, '_model': 95, '_rfac': 0.453}
model6 = {'parA': 4.123, 'parB': 8.567, '_model': 96, '_rfac': 0.354}
self.db.register_params(model5)
self.db.create_models_view()
model2.update(index)
model3.update(index)
model4.update(index)
model5.update(index)
model6.update(index)
self.db.insert_result(model2, model2)
self.db.insert_result(model3, model3)
self.db.insert_result(model4, model4)
self.db.insert_result(model5, model5)
self.db.insert_result(model6, model6)
# only model3, model4 and model5 fulfill all conditions and limits
fil = ['mode = "testmode"', 'rfac <= 0.6']
lim = 3
result = self.db.query_model_array(filter=fil, limit=lim)
template = ['parA', 'parB', 'parC', '_model', '_rfac', '_gen', '_particle']
dt = population.Population.get_pop_dtype(template)
expected = np.zeros((lim,), dtype=dt)
expected['parA'] = np.array([3.412, 4.123, 2.341])
expected['parB'] = np.array([7.856, 8.567, None])
expected['parC'] = np.array([None, None, 6.785])
expected['_model'] = np.array([93, 94, 95])
expected['_rfac'] = np.array([0.345, 0.354, 0.453])
expected['_gen'] = np.array([0, 0, 0])
expected['_particle'] = np.array([0, 0, 0])
self.assertEqual(result.shape, expected.shape)
np.testing.assert_array_almost_equal(result['parA'], expected['parA'])
np.testing.assert_array_almost_equal(result['parB'], expected['parB'])
np.testing.assert_array_almost_equal(result['parC'], expected['parC'])
np.testing.assert_array_almost_equal(result['_model'], expected['_model'])
np.testing.assert_array_almost_equal(result['_gen'], expected['_gen'])
np.testing.assert_array_almost_equal(result['_particle'], expected['_particle'])
def test_query_best_results(self):
self.setup_sample_database()
model2 = {'parA': 4.123, 'parB': 8.567, '_model': 92, '_rfac': 0.654, '_gen': 1, '_particle': 2}
model3 = {'parA': 3.412, 'parB': 7.856, '_model': 93, '_rfac': 0.345, '_gen': 1, '_particle': 3}
model4 = {'parA': 4.123, 'parB': 8.567, '_model': 94, '_rfac': 0.354, '_gen': 1, '_particle': 4}
model5 = {'parA': 2.341, 'parC': 6.785, '_model': 95, '_rfac': 0.453, '_gen': 1, '_particle': 5}
model6 = {'parA': 4.123, 'parB': 8.567, '_model': 96, '_rfac': 0.354, '_gen': 1, '_particle': 6}
model7 = {'parA': 5.123, 'parB': 6.567, '_model': 97, '_rfac': 0.154, '_gen': 1, '_particle': 7}
self.db.register_params(model5)
self.db.create_models_view()
model2.update({'_scan': -1, '_sym': 11, '_emit': 21, '_region': 31})
model3.update({'_scan': 1, '_sym': 12, '_emit': 22, '_region': 32})
model4.update({'_scan': 2, '_sym': 11, '_emit': 23, '_region': 33})
model5.update({'_scan': 3, '_sym': 11, '_emit': 24, '_region': 34})
model6.update({'_scan': 4, '_sym': 11, '_emit': 25, '_region': 35})
model7.update({'_scan': 5, '_sym': -1, '_emit': -1, '_region': -1})
self.db.insert_result(model2, model2)
self.db.insert_result(model3, model3)
self.db.insert_result(model4, model4)
self.db.insert_result(model5, model5)
self.db.insert_result(model6, model6)
self.db.insert_result(model7, model7)
# only model3, model4 and model5 fulfill all conditions and limits
fil = ['mode = "testmode"', 'sym = 11']
lim = 3
result = self.db.query_best_results(filter=fil, limit=lim)
ifields = ['_db_job', '_db_model', '_db_result',
'_model', '_scan', '_sym', '_emit', '_region',
'_gen', '_particle']
ffields = ['_rfac']
dt = [(f, 'i8') for f in ifields]
dt.extend([(f, 'f8') for f in ffields])
expected = np.zeros((lim,), dtype=dt)
expected['_rfac'] = np.array([0.354, 0.354, 0.453])
expected['_model'] = np.array([94, 96, 95])
expected['_scan'] = np.array([2, 4, 3])
expected['_sym'] = np.array([11, 11, 11])
expected['_emit'] = np.array([23, 25, 24])
expected['_region'] = np.array([33, 35, 34])
expected['_gen'] = np.array([1, 1, 1])
expected['_particle'] = np.array([4, 6, 5])
self.assertEqual(result.shape, expected.shape)
np.testing.assert_array_almost_equal(result['_rfac'], expected['_rfac'])
np.testing.assert_array_equal(result['_model'], expected['_model'])
np.testing.assert_array_equal(result['_scan'], expected['_scan'])
np.testing.assert_array_equal(result['_sym'], expected['_sym'])
np.testing.assert_array_equal(result['_emit'], expected['_emit'])
np.testing.assert_array_equal(result['_region'], expected['_region'])
np.testing.assert_array_equal(result['_gen'], expected['_gen'])
np.testing.assert_array_equal(result['_particle'], expected['_particle'])
def test_insert_result(self):
self.setup_sample_database()
index = dispatch.CalcID(15, 16, 17, 18, -1)
result = {'parA': 4.123, 'parB': 8.567, '_rfac': 0.654, '_particle': 21}
result_id = self.db.insert_result(index, result)
c = self.db._conn.cursor()
c.execute("select count(*) from Results")
count = c.fetchone()
self.assertEqual(count[0], 1)
c.execute("select * from Results")
row = c.fetchone()
self.assertIsInstance(row['id'], int)
self.assertEqual(row['id'], result_id)
model_id = row['model_id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['scan'], index.scan)
self.assertEqual(row['sym'], index.sym)
self.assertEqual(row['emit'], index.emit)
self.assertEqual(row['region'], index.region)
self.assertEqual(row['rfac'], result['_rfac'])
c.execute("select * from Models where id = :model_id", {'model_id': model_id})
row = c.fetchone()
model_id = row['id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['job_id'], self.db.job_id)
self.assertEqual(row['model'], index.model)
self.assertIsNone(row['gen'])
self.assertEqual(row['particle'], result['_particle'])
sql = "select key, value from ParamValues " + \
"join Params on ParamValues.param_id = Params.id " + \
"where model_id = :model_id"
c.execute(sql, {'model_id': model_id})
rows = c.fetchall() # list of Row objects
self.assertEqual(len(rows), 2)
for row in rows:
self.assertAlmostEqual(row['value'], result[row['key']])
def test_update_result(self):
self.setup_sample_database()
index = dispatch.CalcID(15, 16, 17, 18, -1)
result1 = {'parA': 4.123, 'parB': 8.567, '_rfac': 0.654, '_particle': 21}
result_id1 = self.db.insert_result(index, result1)
result2 = {'parA': 5.456, '_rfac': 0.254, '_particle': 11}
result_id2 = self.db.insert_result(index, result2)
result3 = result1.copy()
result3.update(result2)
self.assertEqual(result_id1, result_id2)
c = self.db._conn.cursor()
c.execute("select count(*) from Results")
count = c.fetchone()
self.assertEqual(count[0], 1)
c.execute("select * from Results")
row = c.fetchone()
self.assertIsInstance(row['id'], int)
self.assertEqual(row['id'], result_id1)
model_id = row['model_id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['scan'], index.scan)
self.assertEqual(row['sym'], index.sym)
self.assertEqual(row['emit'], index.emit)
self.assertEqual(row['region'], index.region)
self.assertEqual(row['rfac'], result2['_rfac'])
c.execute("select * from Models where id = :model_id", {'model_id': model_id})
row = c.fetchone()
model_id = row['id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['job_id'], self.db.job_id)
self.assertEqual(row['model'], index.model)
self.assertIsNone(row['gen'])
self.assertEqual(row['particle'], result2['_particle'])
sql = "select key, value from ParamValues " + \
"join Params on ParamValues.param_id = Params.id " + \
"where model_id = :model_id"
c.execute(sql, {'model_id': model_id})
rows = c.fetchall() # list of Row objects
self.assertEqual(len(rows), 2)
for row in rows:
self.assertAlmostEqual(row['value'], result3[row['key']])
def test_update_result_dict(self):
"""
test update result with index as dictionary
@return:
"""
self.setup_sample_database()
index = {'_model': 15, '_scan': 16, '_sym': 17, '_emit': 18, '_region': -1}
result1 = {'parA': 4.123, 'parB': 8.567, '_rfac': 0.654, '_particle': 21}
result_id1 = self.db.insert_result(index, result1)
result2 = {'parA': 5.456, '_rfac': 0.254, '_particle': 11}
result_id2 = self.db.insert_result(index, result2)
result3 = result1.copy()
result3.update(result2)
self.assertEqual(result_id1, result_id2)
c = self.db._conn.cursor()
c.execute("select count(*) from Results")
count = c.fetchone()
self.assertEqual(count[0], 1)
c.execute("select * from Results")
row = c.fetchone()
self.assertIsInstance(row['id'], int)
self.assertEqual(row['id'], result_id1)
model_id = row['model_id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['scan'], index['_scan'])
self.assertEqual(row['sym'], index['_sym'])
self.assertEqual(row['emit'], index['_emit'])
self.assertEqual(row['region'], index['_region'])
self.assertEqual(row['rfac'], result2['_rfac'])
c.execute("select * from Models where id = :model_id", {'model_id': model_id})
row = c.fetchone()
model_id = row['id']
self.assertIsInstance(model_id, int)
self.assertEqual(row['job_id'], self.db.job_id)
self.assertEqual(row['model'], index['_model'])
self.assertIsNone(row['gen'])
self.assertEqual(row['particle'], result2['_particle'])
sql = "select key, value from ParamValues " + \
"join Params on ParamValues.param_id = Params.id " + \
"where model_id = :model_id"
c.execute(sql, {'model_id': model_id})
rows = c.fetchall() # list of Row objects
self.assertEqual(len(rows), 2)
for row in rows:
self.assertAlmostEqual(row['value'], result3[row['key']])
def test_query_best_task_models(self):
self.setup_sample_database()
model0xxx = {'_model': 0, '_scan': -1, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4., 'parB': 8.567, '_rfac': 0.01}
model00xx = {'_model': 1, '_scan': 0, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4., 'parB': 8.567, '_rfac': 0.02}
model000x = {'_model': 2, '_scan': 0, '_sym': 0, '_emit': -1, '_region': -1, 'parA': 4., 'parB': 8.567, '_rfac': 0.03}
model01xx = {'_model': 3, '_scan': 1, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4., 'parB': 8.567, '_rfac': 0.04}
model010x = {'_model': 4, '_scan': 1, '_sym': 0, '_emit': -1, '_region': -1, 'parA': 4., 'parB': 8.567, '_rfac': 0.05}
model1xxx = {'_model': 5, '_scan': -1, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.09}
model10xx = {'_model': 6, '_scan': 0, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.08}
model100x = {'_model': 7, '_scan': 0, '_sym': 0, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.07}
model11xx = {'_model': 8, '_scan': 1, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.06}
model110x = {'_model': 9, '_scan': 1, '_sym': 0, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.05}
model2xxx = {'_model': 10, '_scan': -1, '_sym': -1, '_emit': -1, '_region': -1, 'parA': 4.123, 'parB': 8.567, '_rfac': 0.01}
self.db.insert_result(model0xxx, model0xxx)
self.db.insert_result(model00xx, model00xx)
self.db.insert_result(model000x, model000x)
self.db.insert_result(model01xx, model01xx)
self.db.insert_result(model010x, model010x)
self.db.insert_result(model1xxx, model1xxx)
self.db.insert_result(model10xx, model10xx)
self.db.insert_result(model100x, model100x)
self.db.insert_result(model11xx, model11xx)
self.db.insert_result(model110x, model110x)
self.db.insert_result(model2xxx, model2xxx)
result = self.db.query_best_task_models(level=1, count=2)
expected = {0, 1, 3, 6, 8, 10}
self.assertEqual(result, expected)
if __name__ == '__main__':
unittest.main()

117
tests/test_dispatch.py Normal file
View File

@ -0,0 +1,117 @@
"""
@package tests.test_dispatch
unit tests for pmsco.dispatch
the purpose of these tests is to mainly to check the syntax, and correct data types,
i.e. anything that could cause a run-time error.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import six
import copy
import unittest
import pmsco.dispatch as dispatch
class TestCalcID(unittest.TestCase):
def setUp(self):
# before each test method
pass
def tearDown(self):
# after each test method
pass
def test_levels(self):
cid = dispatch.CalcID(1, 2, 3, -1, -1)
result = cid.levels
expected = dispatch.CALC_LEVELS
self.assertIsInstance(cid, tuple)
self.assertEqual(result, expected)
def test_level(self):
cid = dispatch.CalcID(1, 2, 3, -1, -1)
result = cid.level
expected = dispatch.CALC_LEVELS[2]
self.assertEqual(result, expected)
cid = dispatch.CalcID(-1, -1, -1, -1, -1)
result = cid.level
expected = ''
self.assertEqual(result, expected)
def test_numeric_level(self):
cid = dispatch.CalcID(1, 2, 3, -1, -1)
result = cid.numeric_level
expected = 2
self.assertEqual(result, expected)
cid = dispatch.CalcID(-1, -1, -1, -1, -1)
result = cid.numeric_level
expected = -1
self.assertEqual(result, expected)
def test_collapse_levels(self):
cid = dispatch.CalcID(1, 2, 3, 4, 5)
result = cid.collapse_levels(dispatch.CALC_LEVELS[2])
expected = dispatch.CalcID(1, 2, 3, -1, -1)
self.assertEqual(result, expected)
result = cid.collapse_levels(-1)
expected = dispatch.CalcID(-1, -1, -1, -1, -1)
self.assertEqual(result, expected)
class TestCalculationTask(unittest.TestCase):
def setUp(self):
self.sample = dispatch.CalculationTask()
self.sample.id = dispatch.CalcID(11, 12, 13, 14, 15)
self.sample.parent_id = dispatch.CalcID(21, 22, 23, 24, 25)
self.sample.model = {'A': 31, 'B': 32}
self.sample.file_root = "testfile"
self.sample.file_ext = ".ext"
self.sample.result_filename = "resultfile"
self.sample.modf_filename = "modffile"
self.sample.result_valid = True
# self.sample.time = datetime.timedelta()
# self.sample.files = {}
# self.sample.region = {}
self.sample.rfac = 0.123456
def tearDown(self):
pass
def test_change_id(self):
result = copy.deepcopy(self.sample)
self.sample.id = dispatch.CalcID(11, 92, 13, 14, 15)
result.change_id(scan=92)
self.assertEqual(result, self.sample)
def test_get_mpi_message(self):
result = self.sample.get_mpi_message()
expected = {'model': 11, 'scan': 12, 'sym': 13, 'emit': 14, 'region': 15}
self.assertEqual(result['id'], expected)
self.assertEqual(result['model'], self.sample.model)
self.assertEqual(result['result_filename'], self.sample.result_filename)
self.assertEqual(result['result_valid'], self.sample.result_valid)
def test_set_mpi_message(self):
result = dispatch.CalculationTask()
msg = self.sample.get_mpi_message()
result.set_mpi_message(msg)
self.assertEqual(result, self.sample)
def test_format_filename(self):
result = self.sample.format_filename(emit=94)
expected = "testfile_11_12_13_94_15.ext"
self.assertEqual(result, expected)

97
tests/test_files.py
View File

@ -10,24 +10,34 @@ to run the tests, change to the directory which contains the tests directory, an
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015 by Paul Scherrer Institut @n
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import mock
import os
import pmsco.files as files
class TestFileTracker(unittest.TestCase):
"""
unit tests of pmsco.files.FileTracker
for a description of the mock and patch mechanism, cf.
- http://blog.thedigitalcatonline.com/blog/2016/03/06/python-mocks-a-gentle-introduction-part-1
- http://blog.thedigitalcatonline.com/blog/2016/09/27/python-mocks-a-gentle-introduction-part-2
"""
def setUp(self):
# before each test method
self.files = files.FileTracker()
self.files.keep_rfac = 1
self.files._os_delete_file = mock.Mock(return_value=None)
self.files.add_file("model 1 file 1 cluster K", 1, 'cluster')
self.files.add_file("model 1 file 2 output D", 1, 'output')
@ -66,39 +76,76 @@ class TestFileTracker(unittest.TestCase):
pass
def test_add_file(self):
pass
self.assertEqual(self.files.get_file_count(), 10, "file count after setup")
# add a new file of an existing model
self.files.add_file("model 2 file 1 input K", 2, 'input')
self.assertEqual(self.files.get_file_count(), 11, "file count after add")
# change the category of an existing file
self.files.add_file("model 2 file 2 output D", 2, 'report')
self.assertEqual(self.files.get_file_count(), 11, "file count after change")
def test_rename_file(self):
pass
self.files.rename_file("model 2 file 2 output D", "renamed file")
self.assertEqual(self.files.get_file_count(), 10, "file count after rename")
self.files.rename_file("inexistant file", "renamed file")
self.assertEqual(self.files.get_file_count(), 10, "file count after rename")
def test_remove_file(self):
pass
@mock.patch('os.remove')
def test_remove_file(self, os_remove_mock):
self.files.remove_file("model 3 file 1 cluster K")
self.assertEqual(self.files.get_file_count(), 9, "file count after remove")
self.assertFalse(os_remove_mock.called, "file deleted")
def test_update_model_rfac(self):
pass
def test_delete_files(self):
def test_set_model_complete(self):
self.assertEqual(self.files.get_complete_models_count(), 3, "complete model count after setup")
self.files.set_model_complete(3, True)
self.assertEqual(self.files.get_complete_models_count(), 4, "complete model count after add")
self.files.set_model_complete(2, False)
self.assertEqual(self.files.get_complete_models_count(), 3, "complete model count after remove")
self.files.set_model_complete(5, True)
self.assertEqual(self.files.get_complete_models_count(), 3, "complete model count after no change")
@mock.patch('os.remove')
def test_delete_files(self, os_remove_mock):
self.files.keep_rfac = 10
self.files.delete_files()
self.files._os_delete_file.assert_any_call("model 1 file 2 output D")
self.files._os_delete_file.assert_any_call("model 2 file 2 output D")
self.files._os_delete_file.assert_any_call("model 5 file 2 output D")
self.assertEqual(len(self.files._id_by_path), 5+2)
self.assertEqual(len(self.files._path_by_id), 5+2)
self.assertEqual(len(self.files._model_by_id), 5+2)
self.assertEqual(len(self.files._category_by_id), 5+2)
os_remove_mock.assert_any_call(os.path.abspath("model 1 file 2 output D"))
os_remove_mock.assert_any_call(os.path.abspath("model 2 file 2 output D"))
os_remove_mock.assert_any_call(os.path.abspath("model 5 file 2 output D"))
self.assertEqual(self.files.get_file_count(), 5+2)
def test_delete_file(self):
pass
@mock.patch('os.remove')
def test_delete_file(self, os_remove_mock):
self.files.delete_file("model 3 file 1 cluster K")
self.assertEqual(self.files.get_file_count(), 9, "file count after remove")
self.assertTrue(os_remove_mock.called, "file not deleted")
def test_delete_bad_rfac(self):
@mock.patch('os.remove')
def test_delete_bad_rfac(self, os_remove_mock):
self.files.delete_bad_rfac(keep=2, force_delete=True)
self.files._os_delete_file.assert_any_call("model 1 file 1 cluster K")
self.files._os_delete_file.assert_any_call("model 5 file 1 cluster K")
self.assertEqual(len(self.files._id_by_path), 6)
self.assertEqual(len(self.files._path_by_id), 6)
self.assertEqual(len(self.files._model_by_id), 6)
self.assertEqual(len(self.files._category_by_id), 6)
self.assertEqual(self.files.get_file_count(), 6)
os_remove_mock.assert_any_call(os.path.abspath("model 1 file 1 cluster K"))
os_remove_mock.assert_any_call(os.path.abspath("model 5 file 1 cluster K"))
def test_delete_category(self):
pass
@mock.patch('os.remove')
def test_delete_category(self, os_remove_mock):
self.files.delete_category('cluster')
self.assertEqual(self.files.get_file_count(), 7)
os_remove_mock.assert_any_call(os.path.abspath("model 1 file 1 cluster K"))
os_remove_mock.assert_any_call(os.path.abspath("model 2 file 1 cluster K"))
os_remove_mock.assert_any_call(os.path.abspath("model 5 file 1 cluster K"))
@mock.patch('os.remove')
def test_delete_models(self, os_remove_mock):
n = self.files.delete_models()
self.assertEqual(n, 0)
n = self.files.delete_models(keep={2, 5})
self.assertEqual(n, 1)
self.assertEqual(self.files.get_file_count(), 8, "file count after remove")
self.assertTrue(os_remove_mock.called, "file not deleted")
n = self.files.delete_models(delete={2, 3})
self.assertEqual(n, 1)
self.assertEqual(self.files.get_file_count(), 6, "file count after remove")

381
tests/test_genetic.py Normal file
View File

@ -0,0 +1,381 @@
"""
@package tests.test_genetic
unit tests for pmsco.optimizers.genetic
the purpose of these tests is to help debugging the code.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@pre nose must be installed (python-nose package on Debian).
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2018 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import math
import numpy as np
import os
import os.path
import random
import shutil
import tempfile
import unittest
import pmsco.optimizers.genetic as mo
import pmsco.project as mp
POP_SIZE = 6
class TestPopulation(unittest.TestCase):
def setUp(self):
random.seed(0)
self._test_dir = ""
self.domain = mp.Domain()
self.domain.add_param('A', 1.5, 1.0, 2.0, 0.1)
self.domain.add_param('B', 2.5, 2.0, 3.0, 0.1)
self.domain.add_param('C', 3.5, 3.0, 4.0, 0.1)
self.expected_names = ('_gen', '_model', '_particle', '_rfac', 'A', 'B', 'C')
self.size = POP_SIZE
self.pop = mo.GeneticPopulation()
self.optimum1 = {'A': 1.045351, 'B': 2.346212, 'C': 3.873627}
self.optimum2 = {'A': 1.045351, 'B': 2.346212, 'C': 4.873627}
def tearDown(self):
# after each test method
self.pop = None
if self._test_dir:
shutil.rmtree(self._test_dir)
@property
def test_dir(self):
if not self._test_dir:
self._test_dir = tempfile.mkdtemp()
return self._test_dir
@classmethod
def setup_class(cls):
# before any methods in this class
pass
@classmethod
def teardown_class(cls):
# teardown_class() after any methods in this class
pass
def rfactor1(self, pos):
r = (pos['A'] - self.optimum1['A']) ** 2 \
+ (pos['B'] - self.optimum1['B']) ** 2 \
+ (pos['C'] - self.optimum1['C']) ** 2
r /= 3.0
return r
def rfactor2(self, pos):
"""
R-factor function with multiple local minima
global minimum R = 0.0138591 at A = 1.745, B = 2.395, C = 3.755.
domain A = 1.0:2.0, B = 2.0:3.0, C = 3.0:4.0
@param pos: dict-like position with keys 'A', 'B' and 'C'.
@return: R-factor
"""
xa = (pos['A'] - pos['B']) * 12
xb = (pos['B'] - pos['C']) * 15
xc = (pos['C'] - pos['A']) * 18
da = pos['A'] - 1.8
db = pos['B'] - 2.3
dc = pos['C'] - 3.8
aa = 1.0
ab = 1.0
ac = 1.0
ba = 0.4
bb = 0.8
bc = 1.2
r = aa * math.sin(xa) + ab * math.sin(xb) + ac * math.sin(xc)
r += ba * da**2 + bb * db**2 + bc * dc**2 + 3.0
return r
def test_setup(self):
self.pop.setup(self.size, self.domain)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
np.testing.assert_array_equal(np.arange(POP_SIZE), self.pop.pos['_particle'])
np.testing.assert_array_equal(np.zeros(POP_SIZE), self.pop.pos['_gen'])
np.testing.assert_array_equal(np.arange(POP_SIZE), self.pop.pos['_model'])
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertEqual(0, self.pop.generation)
self.assertEqual(POP_SIZE, self.pop.model_count)
def test_setup_with_results(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, seed_file=data_file, recalc_seed=False)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
np.testing.assert_array_equal(self.pop.pos['_particle'], np.arange(POP_SIZE))
np.testing.assert_array_equal(self.pop.pos['_gen'], [0, 0, -1, 0, 0, 0])
np.testing.assert_array_equal(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(0.6, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][2], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][2], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][0], 3)
self.assertGreaterEqual(4.0, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][2], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][0], 3)
def test_setup_with_results_recalc(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, seed_file=data_file, recalc_seed=True)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
np.testing.assert_array_equal(self.pop.pos['_particle'], np.arange(POP_SIZE))
np.testing.assert_array_equal(self.pop.pos['_gen'], [0, 0, 0, 0, 0, 0])
np.testing.assert_array_equal(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][2], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][2], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][0], 3)
self.assertGreaterEqual(4.0, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][2], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][0], 3)
def test_pos_gen(self):
self.pop.setup(self.size, self.domain)
for index, item in enumerate(self.pop.pos_gen()):
self.assertIsInstance(item, dict)
self.assertEqual(set(item.keys()), set(self.expected_names))
self.assertEqual(item['_particle'], index)
def test_randomize(self):
self.pop.setup(self.size, self.domain)
self.pop.randomize()
self.assertTrue(np.all(self.pop.pos['A'] >= self.domain.min['A']))
self.assertTrue(np.all(self.pop.pos['A'] <= self.domain.max['A']))
self.assertGreater(np.std(self.pop.pos['A']), self.domain.step['A'])
def test_seed(self):
self.pop.setup(self.size, self.domain)
self.pop.seed(self.domain.start)
self.assertAlmostEqual(self.pop.pos['A'][0], self.domain.start['A'], delta=0.001)
def test_add_result(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
i_result = 0
result = self.pop.pos[i_sample]
self.pop.add_result(result, 0.0)
self.assertEqual(self.pop.results.shape[0], 1)
self.assertEqual(self.pop.results[i_result], result)
self.assertEqual(self.pop.best[i_sample], result)
def test_is_converged(self):
self.pop.setup(self.size, self.domain)
self.assertFalse(self.pop.is_converged())
i_sample = 0
result = self.pop.pos[i_sample]
for i in range(POP_SIZE):
rfac = 1.0 - float(i) / POP_SIZE
self.pop.add_result(result, rfac)
self.assertFalse(self.pop.is_converged())
for i in range(POP_SIZE):
rfac = (1.0 - float(i) / POP_SIZE) / 1000.0
self.pop.add_result(result, rfac)
self.assertTrue(self.pop.is_converged())
def test_save_population(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_population.pop")
self.pop.save_population(filename)
def test_save_results(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
result = self.pop.pos[i_sample]
self.pop.add_result(result, 1.0)
filename = os.path.join(self.test_dir, "test_save_results.dat")
self.pop.save_results(filename)
def test_save_array(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_array.pos")
self.pop.save_array(filename, self.pop.pos)
def test_load_array(self):
n = 3
filename = os.path.join(self.test_dir, "test_load_array")
self.pop.setup(self.size, self.domain)
# expected array
dt_exp = self.pop.get_pop_dtype(self.domain.start)
a_exp = np.zeros((n,), dtype=dt_exp)
a_exp['A'] = np.linspace(0, 1, n)
a_exp['B'] = np.linspace(1, 2, n)
a_exp['C'] = np.linspace(3, 4, n)
a_exp['_rfac'] = np.linspace(5, 6, n)
a_exp['_gen'] = np.array([3, 4, 7])
a_exp['_particle'] = np.array([1, 0, 2])
a_exp['_model'] = np.array([3, 6, 1])
# test array is a expected array with different column order
dt_test = [('A', 'f4'), ('_particle', 'i4'), ('_rfac', 'f4'), ('C', 'f4'), ('_gen', 'i4'), ('B', 'f4'),
('_model', 'i4')]
names_test = [a[0] for a in dt_test]
a_test = np.zeros((n,), dtype=dt_test)
for name in names_test:
a_test[name] = a_exp[name]
header = " ".join(names_test)
np.savetxt(filename, a_test, fmt='%g', header=header)
result = np.zeros((n,), dtype=dt_exp)
result = self.pop.load_array(filename, result)
self.assertEqual(result.dtype.names, a_exp.dtype.names)
for name in a_exp.dtype.names:
np.testing.assert_almost_equal(result[name], a_exp[name], err_msg=name)
def test_mate_parents(self):
self.pop.setup(self.size, self.domain)
pos1 = self.pop.pos.copy()
parents = self.pop.mate_parents(pos1)
self.assertEqual(len(parents), pos1.shape[0] / 2)
def test_crossover(self):
self.pop.setup(self.size, self.domain)
p1 = self.pop.pos[2].copy()
p2 = self.pop.pos[3].copy()
c1, c2 = self.pop.crossover(p1, p2)
self.assertIsInstance(c1, np.void)
self.assertIsInstance(c2, np.void)
self.assertEqual(c1['_particle'], p1['_particle'])
self.assertEqual(c2['_particle'], p2['_particle'])
for name in self.domain.start:
self.assertAlmostEqual(c1[name] + c2[name], p1[name] + p2[name], msg=name)
def test_mutate_weak(self):
self.pop.setup(self.size, self.domain)
p1 = self.pop.pos[3].copy()
c1 = p1.copy()
self.pop.mutate_weak(c1, 1.0)
self.assertEqual(c1['_particle'], p1['_particle'])
self.assertNotAlmostEqual(c1['A'], p1['A'])
self.assertNotAlmostEqual(c1['B'], p1['B'])
self.assertNotAlmostEqual(c1['C'], p1['C'])
def test_mutate_strong(self):
self.pop.setup(self.size, self.domain)
p1 = self.pop.pos[3].copy()
c1 = p1.copy()
self.pop.mutate_strong(c1, 1.0)
self.assertEqual(c1['_particle'], p1['_particle'])
self.assertNotAlmostEqual(c1['A'], p1['A'])
self.assertNotAlmostEqual(c1['B'], p1['B'])
self.assertNotAlmostEqual(c1['C'], p1['C'])
def test_advance_population(self):
self.pop.setup(self.size, self.domain)
p1 = {'A': np.linspace(1.0, 2.0, POP_SIZE),
'B': np.linspace(2.0, 3.0, POP_SIZE),
'C': np.linspace(3.0, 4.0, POP_SIZE)}
self.pop.pos['A'] = p1['A']
self.pop.pos['B'] = p1['B']
self.pop.pos['C'] = p1['C']
for pos in self.pop.pos:
pos['_rfac'] = self.rfactor1(pos)
self.pop._hold_once = False
self.pop.weak_mutation_probability = 1.
self.pop.strong_mutation_probability = 0.
self.pop.advance_population()
for name, value in p1.items():
self.assertTrue(np.any(abs(self.pop.pos[name] - value) >= 0.001), msg=name)
def test_convergence_1(self):
self.pop.setup(self.size, self.domain)
self.pop.pos['A'] = np.linspace(1.0, 2.0, POP_SIZE)
self.pop.pos['B'] = np.linspace(2.0, 3.0, POP_SIZE)
self.pop.pos['C'] = np.linspace(3.0, 4.0, POP_SIZE)
self.pop.pos['_rfac'] = np.linspace(2.0, 1.0, POP_SIZE)
best_rfactors = []
for i in range(10):
self.pop.advance_population()
for pos in self.pop.pos:
self.pop.add_result(pos, self.rfactor1(pos))
best_rfactors.append(self.pop.best['_rfac'].min())
self.assertLess(best_rfactors[-1], best_rfactors[0])
def optimize_rfactor_2(self, pop_size, iterations):
self.size = pop_size
self.pop.setup(self.size, self.domain)
for i in range(iterations):
self.pop.advance_population()
for pos in self.pop.pos:
self.pop.add_result(pos, self.rfactor2(pos))
@unittest.skip("test_convergence_2 is unreliable")
def test_convergence_2(self):
"""
there is a certain probability that this test fails.
@return:
"""
self.pop.weak_mutation_probability = 1.
self.pop.strong_mutation_probability = 0.01
self.optimize_rfactor_2(10, 200)
ibest = self.pop.results['_rfac'].argmin()
best = self.pop.results[ibest]
self.assertLess(best['_rfac'], 0.2)
self.assertAlmostEqual(best['A'], 1.745, delta=0.1)
self.assertAlmostEqual(best['B'], 2.395, delta=0.1)
self.assertAlmostEqual(best['C'], 3.755, delta=0.1)
if __name__ == '__main__':
unittest.main()

48
tests/test_hbncu.py
View File

@ -1,48 +0,0 @@
"""
@package tests.test_hbncu
unit tests for projects.hbncu
the purpose of these tests is to help debugging the code.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@pre nose must be installed (python-nose package on Debian).
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
import unittest
import os.path
import tempfile
import shutil
import numpy as np
import projects.hbncu.hbncu as hbncu
import pmsco.data as data
import pmsco.dispatch as dispatch
class TestHbncuProject(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self.project = hbncu.HbncuProject()
def tearDown(self):
# after each test method
self.project = None
shutil.rmtree(self.test_dir)
@classmethod
def setup_class(cls):
# before any methods in this class
pass
@classmethod
def teardown_class(cls):
# teardown_class() after any methods in this class
pass

616
tests/test_population.py Normal file
View File

@ -0,0 +1,616 @@
"""
@package tests.test_population
unit tests for @ref pmsco.optimizers.population
the purpose of these tests is to help debugging the code.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@pre nose must be installed (python-nose package on Debian).
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import six
import numpy as np
import os
import os.path
import random
import shutil
import tempfile
import unittest
import pmsco.optimizers.population as population
import pmsco.project as project
from pmsco.helpers import BraceMessage as BMsg
POP_SIZE = 5
class TestPopulation(unittest.TestCase):
def setUp(self):
random.seed(0)
self.test_dir = tempfile.mkdtemp()
self.domain = project.Domain()
self.domain.add_param('A', 1.5, 1.0, 2.0, 0.1)
self.domain.add_param('B', 2.5, 2.0, 3.0, 0.1)
self.domain.add_param('C', 3.5, 3.0, 4.0, 0.1)
self.expected_names = ('_gen', '_model', '_particle', '_rfac', 'A', 'B', 'C')
self.size = POP_SIZE
self.pop = population.Population()
self.optimum1 = {'A': 1.045351, 'B': 2.346212, 'C': 3.873627}
def tearDown(self):
# after each test method
self.pop = None
shutil.rmtree(self.test_dir)
@classmethod
def setup_class(cls):
# before any methods in this class
pass
@classmethod
def teardown_class(cls):
# teardown_class() after any methods in this class
pass
def reorder_pop_array(self, x):
"""
re-order the columns of a population-like array.
this makes it easier to create constant arrays with arbitrary column order
since population arrays are ordered alphabetically.
@param x: numpy structured array
@return: numpy structured array having the same dtype as the population arrays.
"""
y = np.zeros(x.shape, dtype=self.pop.get_pop_dtype(self.pop.model_start))
for name in y.dtype.names:
y[name] = x[name]
return y
def assert_pop_array_equal(self, x, y, msg=""):
"""
array equality test for population-like structured numpy arrays.
- check for same column names and kinds
- check for same shape
- check float columns using np.testing.assert_array_almost_equal
- check all other columns using np.testing.assert_array_equal
@param x: structured numpy array.
@param y: structured numpy array.
@param msg: optional error message.
@return: None
@raise AssertionError if arrays are not equal.
"""
self.assertEqual(x.dtype.names, y.dtype.names, msg=BMsg("{0} (dtype)", msg))
self.assertEqual(x.shape, y.shape, msg=BMsg("{0} (shape)", msg))
for n, t in x.dtype.fields.items():
self.assertEqual(t[0].kind, y.dtype.fields[n][0].kind, msg=BMsg("{0} (column {1} kind)", msg, n))
if t[0].kind in {'f', 'c'}:
np.testing.assert_array_almost_equal(x[n], y[n], err_msg="{0} (column {1})".format(msg, n))
else:
np.testing.assert_array_equal(x[n], y[n], err_msg="{0} (column {1})".format(msg, n))
def rfactor1(self, pos):
r = (pos['A'] - self.optimum1['A']) ** 2 \
+ (pos['B'] - self.optimum1['B']) ** 2 \
+ (pos['C'] - self.optimum1['C']) ** 2
r /= 3.0
return r
def test_setup(self):
self.pop.setup(self.size, self.domain)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
np.testing.assert_array_equal(np.arange(POP_SIZE), self.pop.pos['_particle'])
np.testing.assert_array_equal(np.zeros(POP_SIZE), self.pop.pos['_gen'])
np.testing.assert_array_equal(np.arange(POP_SIZE), self.pop.pos['_model'])
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertEqual(0, self.pop.generation)
self.assertEqual(POP_SIZE, self.pop.model_count)
def test_setup_with_results(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, seed_file=data_file, recalc_seed=False)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
np.testing.assert_array_equal(self.pop.pos['_particle'], np.arange(POP_SIZE))
np.testing.assert_array_equal(self.pop.pos['_gen'], [0, 0, -1, 0, 0])
np.testing.assert_array_equal(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(0.6, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][2], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][2], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][0], 3)
self.assertGreaterEqual(4.0, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][2], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][0], 3)
def test_setup_with_results_recalc(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, seed_file=data_file, recalc_seed=True)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
np.testing.assert_array_equal(self.pop.pos['_particle'], np.arange(POP_SIZE))
np.testing.assert_array_equal(self.pop.pos['_gen'], [0, 0, 0, 0, 0])
np.testing.assert_array_equal(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][2], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][2], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][0], 3)
self.assertGreaterEqual(4.0, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][2], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][0], 3)
def test_setup_with_partial_results(self):
self.domain.add_param('D', 4.5, 4.0, 5.0, 0.1)
self.expected_names = ('_gen', '_model', '_particle', '_rfac', 'A', 'B', 'C', 'D')
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, seed_file=data_file, recalc_seed=False)
self.assertEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
np.testing.assert_array_equal(self.pop.pos['_particle'], np.arange(POP_SIZE))
np.testing.assert_array_equal(self.pop.pos['_gen'], [0, 0, -1, 0, 0])
np.testing.assert_array_equal(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(0.6, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][2], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][2], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][0], 3)
self.assertGreaterEqual(4.0, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][2], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][0], 3)
def test_pos_gen(self):
self.pop.setup(self.size, self.domain)
for index, item in enumerate(self.pop.pos_gen()):
self.assertIsInstance(item, dict)
self.assertEqual(set(item.keys()), set(self.expected_names))
self.assertEqual(item['_particle'], index)
def test_get_pop_dtype(self):
result = population.Population.get_pop_dtype({'A': 1.045, 'C': 3.873, '_D': 4.381, 'B': 2.346})
expected = [('_gen', 'i8'), ('_model', 'i8'), ('_particle', 'i8'), ('_rfac', 'f8'),
('A', 'f8'), ('B', 'f8'), ('C', 'f8')]
self.assertEqual(result, expected)
result = population.Population.get_pop_dtype(['A', 'C', '_D', 'B'])
self.assertEqual(result, expected)
def test_get_model_dtype(self):
result = population.Population.get_model_dtype({'A': 1.045, 'C': 3.873, '_D': 4.381, 'B': 2.346})
expected = [('A', 'f8'), ('B', 'f8'), ('C', 'f8')]
self.assertEqual(result, expected)
result = population.Population.get_model_dtype(['A', 'C', '_D', 'B'])
self.assertEqual(result, expected)
def test_get_model_array(self):
result = population.Population.get_model_array({'A': 1.045, 'C': 3.873, '_D': 4.381, 'B': 2.346})
dt = [('A', 'f8'), ('B', 'f8'), ('C', 'f8')]
expected = np.array([(1.045, 2.346, 3.873)], dtype=dt)
np.testing.assert_array_equal(result, expected)
def test_randomize(self):
self.pop.setup(self.size, self.domain)
self.pop.randomize()
m = np.mean(self.pop.pos['A'])
self.assertGreaterEqual(m, self.domain.min['A'])
self.assertLessEqual(m, self.domain.max['A'])
def test_seed(self):
self.pop.setup(self.size, self.domain)
self.pop.seed(self.domain.start)
self.assertAlmostEqual(self.pop.pos['A'][0], self.domain.start['A'], delta=0.001)
def test_add_result(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
i_result = 0
result = self.pop.pos[i_sample]
self.pop.add_result(result, 0.0)
self.assertEqual(self.pop.results.shape[0], 1)
self.assertEqual(self.pop.results[i_result], result)
self.assertEqual(self.pop.best[i_sample], result)
def test_save_population(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_population.pop")
self.pop.save_population(filename)
def test_save_results(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
result = self.pop.pos[i_sample]
self.pop.add_result(result, 1.0)
filename = os.path.join(self.test_dir, "test_save_results.dat")
self.pop.save_results(filename)
def test_save_array(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_array.pos")
self.pop.save_array(filename, self.pop.pos)
def test_load_array(self):
n = 3
filename = os.path.join(self.test_dir, "test_load_array")
self.pop.setup(self.size, self.domain)
# expected array
dt_exp = self.pop.get_pop_dtype(self.domain.start)
a_exp = np.zeros((n,), dtype=dt_exp)
a_exp['A'] = np.linspace(0, 1, n)
a_exp['B'] = np.linspace(1, 2, n)
a_exp['C'] = np.linspace(3, 4, n)
a_exp['_rfac'] = np.linspace(5, 6, n)
a_exp['_gen'] = np.array([3, 4, 7])
a_exp['_particle'] = np.array([1, 0, 2])
a_exp['_model'] = np.array([3, 6, 1])
# test array is a expected array with different column order
dt_test = [('A', 'f4'), ('_particle', 'i4'), ('_rfac', 'f4'), ('C', 'f4'), ('_gen', 'i4'), ('B', 'f4'),
('_model', 'i4')]
names_test = [a[0] for a in dt_test]
a_test = np.zeros((n,), dtype=dt_test)
for name in names_test:
a_test[name] = a_exp[name]
header = " ".join(names_test)
np.savetxt(filename, a_test, fmt='%g', header=header)
result = np.zeros((n,), dtype=dt_exp)
result = self.pop.load_array(filename, result)
self.assert_pop_array_equal(result, a_exp)
def test_constrain_position(self):
# upper
pos1 = 11.0
vel1 = 5.0
min1 = 0.0
max1 = 10.0
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 're-enter')
self.assertAlmostEqual(pos2, 1.0)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'bounce')
self.assertAlmostEqual(pos2, 9.0)
self.assertAlmostEqual(vel2, -vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'scatter')
self.assertGreaterEqual(pos2, 6.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, 0.0)
self.assertLessEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'stick')
self.assertAlmostEqual(pos2, max1)
self.assertAlmostEqual(vel2, max1 - (pos1 - vel1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'random')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, -(max1 - min1))
self.assertLessEqual(vel2, (max1 - min1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'expand')
self.assertAlmostEqual(pos2, pos1)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max2, pos1)
self.assertRaises(ValueError, population.Population.constrain_position, pos1, vel1, min1, max1, 'error')
# lower
pos1 = -1.0
vel1 = -5.0
min1 = 0.0
max1 = 10.0
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 're-enter')
self.assertAlmostEqual(pos2, 9.0)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'bounce')
self.assertAlmostEqual(pos2, 1.0)
self.assertAlmostEqual(vel2, -vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'scatter')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 4.0)
self.assertGreaterEqual(vel2, vel1)
self.assertLessEqual(vel2, 0.0)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'stick')
self.assertAlmostEqual(pos2, min1)
self.assertAlmostEqual(vel2, min1 - (pos1 - vel1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'random')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, -(max1 - min1))
self.assertLessEqual(vel2, max1 - min1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = population.Population.constrain_position(pos1, vel1, min1, max1, 'expand')
self.assertAlmostEqual(pos2, pos1)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min2, pos1)
self.assertAlmostEqual(max2, max1)
self.assertRaises(ValueError, population.Population.constrain_position, pos1, vel1, min1, max1, 'error')
def test_patch_from_file(self):
self.pop.setup(self.size, self.domain)
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
patch_size = 4
expected_pos = {'A': [1.1, 1.2, 1.3, 1.4], 'B': [2.1, 2.2, 2.3, 2.4], 'C': [3.1, 3.2, 4.3, 3.4]}
self.pop.patch_from_file(data_file)
self.assertIsInstance(self.pop.pos_patch, np.ndarray)
self.assertEqual(self.pop.pos_patch.shape, (patch_size,))
np.testing.assert_array_equal(self.pop.pos_patch['_particle'], [1, 2, 3, 4])
np.testing.assert_array_equal(self.pop.pos_patch['_gen'], [1, 2, 3, 4])
np.testing.assert_array_almost_equal(self.pop.pos_patch['A'], expected_pos['A'])
np.testing.assert_array_almost_equal(self.pop.pos_patch['B'], expected_pos['B'])
np.testing.assert_array_almost_equal(self.pop.pos_patch['C'], expected_pos['C'])
def test_apply_patch(self):
self.pop.setup(self.size, self.domain)
expected_pos = self.pop.pos.copy()
dt_test = [('A', 'f4'), ('_particle', 'i4'), ('_rfac', 'f4'), ('C', 'f4'), ('_model', 'i4')]
patch_size = 3
self.pop.pos_patch = np.zeros(patch_size, dtype=dt_test)
self.pop.pos_patch['_particle'] = np.array([1, 2, 3])
self.pop.pos_patch['_rfac'] = np.random.random_sample(patch_size)
self.pop.pos_patch['_model'] = np.arange(patch_size)
self.pop.pos_patch['A'] = np.array([1.1, 2.2, 3.3])
self.pop.pos_patch['C'] = np.array([1.1, 2.2, 3.3])
expected_pos['A'][1] = 1.1
expected_pos['C'][3] = 3.3
expected_pos['_particle'] = np.arange(POP_SIZE)
expected_pos['_gen'] = 0
expected_pos['_model'] = np.arange(POP_SIZE)
expected_pos['_rfac'] = 2.1
self.pop._apply_patch()
self.assertIsNone(self.pop.pos_patch)
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
self.assert_pop_array_equal(self.pop.pos, expected_pos)
def test_find_result(self):
self.domain.min['A'] = -0.1
self.domain.max['A'] = 0.1
self.domain.min['B'] = 0.0
self.domain.max['B'] = 1000.
self.domain.min['C'] = 9.
self.domain.max['C'] = 9.001
self.size = 100
self.pop.setup(self.size, self.domain)
self.pop.results = self.pop.pos.copy()
expected_index = 77
names = ['A', 'B', 'C']
target0 = {key: self.pop.results[key][expected_index] for key in names}
target1 = self.pop.results[names].copy()
target1 = target1[expected_index]
target2 = {key: target0[key] for key in ['C', 'B']}
target3 = target2.copy()
target3['C'] += 9.e-6
target4 = {'A': random.random(), 'B': random.random(), 'C': 9.002}
result0 = self.pop.find_model(target0)
self.assertEqual(result0, expected_index)
result1 = self.pop.find_model(target1)
self.assertEqual(result1, expected_index)
result2 = self.pop.find_model(target2)
self.assertEqual(result2, expected_index)
result3 = self.pop.find_model(target3)
self.assertEqual(result3, expected_index)
self.assertRaises(ValueError, self.pop.find_model, target4)
def test_import_positions(self):
"""
test the population.Population.import_positions method.
check the different type conversions.
the main work is in test_import_positions_array.
"""
self.pop.setup(self.size, self.domain)
source_type = [('A', 'f4'), ('B', 'f4'), ('C', 'f4'), ('D', 'f4'),
('_model', 'i4'), ('_particle', 'i4'), ('_gen', 'i4'), ('_rfac', 'f4')]
source = np.array([(1.5, 3.5, 3.5, 4.5, 51, 52, 53, 0.5),
(1.6, 2.6, 2.6, 4.6, 61, 62, 63, 0.6),
(1.7, 2.7, 3.7, 4.7, 71, 72, 73, 0.7),
(1.8, 2.8, 3.8, 4.8, 81, 82, 83, 0.8)], dtype=source_type)
expected_type = [('_gen', 'i8'), ('_model', 'i8'), ('_particle', 'i8'), ('_rfac', 'f8'),
('A', 'f8'), ('B', 'f8'), ('C', 'f8')]
expected = np.array([(0, 0, 0, 0.0, 1.5, 3.5, 3.5),
(0, 0, 0, 0.0, 1.6, 2.6, 2.6),
(0, 0, 0, 0.0, 1.7, 2.7, 3.7),
(0, 0, 0, 0.0, 1.8, 2.8, 3.8)], dtype=expected_type)
expected_single = np.array([expected[0]], dtype=expected_type)
# None
self.pop.clear_import()
source_none = None
retval = self.pop.import_positions(source_none)
self.assertEqual(retval, 0)
# np.ndarray
self.pop.clear_import()
source_ndarray = source
retval = self.pop.import_positions(source_ndarray)
self.assertEqual(retval, expected.shape[0])
self.assert_pop_array_equal(self.pop.pos_import, expected, msg="numpy.ndarray")
# np.void
self.pop.clear_import()
source_void = source[0]
retval = self.pop.import_positions(source_void)
self.assertEqual(retval, 1)
self.assert_pop_array_equal(self.pop.pos_import, expected_single, msg="numpy.void")
# dict
self.pop.clear_import()
source_dict = {k: source[0][k] for k in source.dtype.names}
retval = self.pop.import_positions(source_dict)
self.assertEqual(retval, 1)
self.assert_pop_array_equal(self.pop.pos_import, expected_single, msg="dict")
# sequence
self.pop.clear_import()
source_sequence = [row for row in source]
retval = self.pop.import_positions(source_sequence)
self.assertEqual(retval, expected.shape[0])
self.assert_pop_array_equal(self.pop.pos_import, expected, msg="sequence")
# file
filename = os.path.join(self.test_dir, "test_import_positions")
self.pop.save_array(filename, source)
self.pop.clear_import()
retval = self.pop.import_positions(filename)
self.assertEqual(retval, expected.shape[0])
self.assert_pop_array_equal(self.pop.pos_import, expected, msg="file")
# callable
def source_callable():
return expected.copy()
self.pop.clear_import()
retval = self.pop.import_positions(source_callable)
self.assertEqual(retval, expected.shape[0])
self.assert_pop_array_equal(self.pop.pos_import, expected, msg="callable")
def test_import_positions_array(self):
"""
test the population.Population.import_positions_array method.
- import a data array, check type and size.
- add to previous import.
- ignore control fields.
- no range or duplicate checking.
- missing parameter.
"""
self.pop.setup(self.size, self.domain)
source_type = [('A', 'f4'), ('B', 'f4'), ('C', 'f4'), ('D', 'f4'),
('_model', 'i4'), ('_particle', 'i4'), ('_gen', 'i4'), ('_rfac', 'f4')]
source = np.array([(1.0, 0.0, 0.0, 0.0, 0, 0, 0, 0.0),
(1.5, 3.5, 3.5, 4.5, 51, 52, 53, 0.5),
(1.6, 2.6, 2.6, 4.6, 61, 62, 63, 0.6),
(1.7, 2.7, 3.7, 4.7, 71, 72, 73, 0.7),
(1.8, 2.8, 3.8, 4.8, 81, 82, 83, 0.8)], dtype=source_type)
self.pop.pos_import.resize(1)
self.pop.pos_import[0]['A'] = 1.0
expected_type = [('_gen', 'i8'), ('_model', 'i8'), ('_particle', 'i8'), ('_rfac', 'f8'),
('A', 'f8'), ('B', 'f8'), ('C', 'f8')]
expected = np.array([(0, 0, 0, 0.0, 1.0, 0.0, 0.0),
(0, 0, 0, 0.0, 1.0, 0.0, 0.0),
(0, 0, 0, 0.0, 1.5, 3.5, 3.5),
(0, 0, 0, 0.0, 1.6, 2.6, 2.6),
(0, 0, 0, 0.0, 1.7, 2.7, 3.7),
(0, 0, 0, 0.0, 1.8, 2.8, 3.8)], dtype=expected_type)
retval = self.pop.import_positions_array(source)
self.assertEqual(retval, source.shape[0])
self.assertEqual(self.pop.pos_import.dtype.names, self.pop.pos.dtype.names)
self.assert_pop_array_equal(self.pop.pos_import, expected, msg="imported array")
source_type = [('B', 'f4'), ('C', 'f4'), ('D', 'f4'), ('E', 'f4'),
('_model', 'i4'), ('_particle', 'i4'), ('_gen', 'i4'), ('_rfac', 'f4')]
source = np.array([(1.0, 0.0, 0.0, 0.0, 0, 0, 0, 0.0),
(1.8, 2.8, 3.8, 4.8, 81, 82, 83, 0.8)], dtype=source_type)
self.assertRaises(ValueError, self.pop.import_positions_array, source)
def test_advance_from_import(self):
"""
test the population.Population.advance_from_import method.
- array type and size.
- range checks.
- de-duplication.
"""
self.pop.setup(self.size, self.domain)
self.pop.position_constrain_mode = 'error'
source_type = [('A', 'f8'), ('B', 'f8'), ('C', 'f8'),
('_model', 'i8'), ('_particle', 'i8'), ('_gen', 'i8'), ('_rfac', 'f8')]
source = np.array([(1.3, 2.3, 3.3, 0, 0, 0, 0.0),
(1.3, 2.3, 3.3, 0, 0, 0, 0.0),
(1.4, 2.4, 3.4, 0, 0, 0, 0.0),
(1.5, 3.5, 3.5, 0, 0, 0, 0.0),
(1.6, 2.6, 2.6, 0, 0, 0, 0.0),
(1.7, 2.7, 3.7, 0, 0, 0, 0.0),
(1.8, 2.8, 3.8, 0, 0, 0, 0.0)], dtype=source_type)
source = self.reorder_pop_array(source)
results = np.array([(1.4, 2.4, 3.4, 0, 0, 0, 0.4)], dtype=source_type)
results = self.reorder_pop_array(results)
expected = np.array([(1.3, 2.3, 3.3, POP_SIZE + 0, 0, 0, 2.1),
(1.7, 2.7, 3.7, POP_SIZE + 1, 1, 0, 2.1),
(1.8, 2.8, 3.8, POP_SIZE + 2, 2, 0, 2.1)], dtype=source_type)
expected = self.reorder_pop_array(expected)
self.pop.pos_import = source
self.pop.results = results
retval = self.pop.advance_from_import()
self.assertEqual(retval, expected.shape[0])
self.assertEqual(self.pop.size_act, expected.shape[0])
pos_act = self.pop.pos[0:self.pop.size_act]
self.assert_pop_array_equal(pos_act, expected)
if __name__ == '__main__':
unittest.main()

82
tests/test_project.py Normal file
View File

@ -0,0 +1,82 @@
"""
@package tests.test_project
unit tests for pmsco.project.
the purpose of these tests is to help debugging the code.
to run the tests, change to the directory which contains the tests directory, and execute =nosetests=.
@pre nose and mock must be installed.
@author Matthias Muntwiler, matthias.muntwiler@psi.ch
@copyright (c) 2015-18 by Paul Scherrer Institut @n
Licensed under the Apache License, Version 2.0 (the "License"); @n
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import mock
import numpy as np
import pmsco.data as data
import pmsco.dispatch as dispatch
import pmsco.project as project
class TestProject(unittest.TestCase):
def setUp(self):
# before each test method
self.project = project.Project()
def tearDown(self):
# after each test method
pass
@classmethod
def setup_class(cls):
# before any methods in this class
pass
@classmethod
def teardown_class(cls):
# teardown_class() after any methods in this class
pass
@mock.patch('pmsco.data.load_data')
@mock.patch('pmsco.data.save_data')
def test_combine_symmetries(self, save_data_mock, load_data_mock):
self.project.scans.append(project.Scan())
parent_task = dispatch.CalculationTask()
parent_task.change_id(model=0, scan=0)
parent_task.model['wsym1'] = 0.5
child_tasks = [parent_task.copy()] * 2
for idx, task in enumerate(child_tasks):
task.change_id(sym=idx)
data1 = data.create_data(5, datatype='EI')
data1['e'] = np.arange(5)
data1['i'] = 10.
data2 = data1.copy()
data2['i'] = 10.
load_data_mock.side_effect = [data1, data2]
data3 = data1.copy()
data3['i'] = (10. + 0.5 * 10.) / 1.5
self.project.combine_symmetries(parent_task, child_tasks)
save_data_mock.assert_called()
args, kwargs = save_data_mock.call_args
result_data = args[1]
np.testing.assert_array_almost_equal(result_data['e'], data3['e'])
np.testing.assert_array_almost_equal(result_data['i'], data3['i'])
if __name__ == '__main__':
unittest.main()

262
tests/test_swarm.py
View File

@ -1,6 +1,6 @@
"""
@package tests.test_swarm
unit tests for pmsco.swarm
unit tests for @ref pmsco.optimizers.swarm
the purpose of these tests is to help debugging the code.
@ -17,29 +17,36 @@ Licensed under the Apache License, Version 2.0 (the "License"); @n
http://www.apache.org/licenses/LICENSE-2.0
"""
import unittest
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import os
import os.path
import tempfile
import random
import shutil
import numpy as np
import pmsco.swarm as mo
import pmsco.project as mp
import tempfile
import unittest
import pmsco.optimizers.swarm as swarm
import pmsco.project as project
POP_SIZE = 5
class TestPopulation(unittest.TestCase):
class TestSwarmPopulation(unittest.TestCase):
def setUp(self):
random.seed(0)
self.test_dir = tempfile.mkdtemp()
self.domain = mp.Domain()
self.domain = project.Domain()
self.domain.add_param('A', 1.5, 1.0, 2.0, 0.1)
self.domain.add_param('B', 2.5, 2.0, 3.0, 0.1)
self.domain.add_param('C', 3.5, 3.0, 4.0, 0.1)
self.expected_names = ('A', 'B', 'C', '_particle', '_gen', '_model', '_rfac')
self.expected_names = ('_gen', '_model', '_particle', '_rfac', 'A', 'B', 'C')
self.size = POP_SIZE
self.pop = mo.Population()
self.pop = swarm.SwarmPopulation()
self.optimum1 = {'A': 1.045351, 'B': 2.346212, 'C': 3.873627}
@ -65,100 +72,12 @@ class TestPopulation(unittest.TestCase):
r /= 3.0
return r
def test_setup(self):
self.pop.setup(self.size, self.domain)
self.assertItemsEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertItemsEqual(np.arange(POP_SIZE), self.pop.pos['_particle'])
self.assertItemsEqual(np.zeros((POP_SIZE)), self.pop.pos['_gen'])
self.assertItemsEqual(np.arange(POP_SIZE), self.pop.pos['_model'])
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertEqual(0, self.pop.generation)
self.assertEqual(POP_SIZE, self.pop.model_count)
def test_setup_with_results(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, data_file, False)
self.assertItemsEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(0, self.pop.generation)
self.assertEqual(3, self.pop.model_count)
self.assertItemsEqual(np.arange(POP_SIZE), self.pop.pos['_particle'])
self.assertItemsEqual([-1, -1, 0, 0, 0], self.pop.pos['_gen'])
self.assertItemsEqual([-1, -2, 0, 1, 2], self.pop.pos['_model'])
self.assertAlmostEqual(0.3, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(0.6, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][4], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][4], 3)
self.assertAlmostEqual(3.3, self.pop.pos['C'][0], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][4], 3)
def test_setup_with_results_recalc(self):
data_dir = os.path.dirname(os.path.abspath(__file__))
data_file = os.path.join(data_dir, "test_swarm.setup_with_results.1.dat")
self.pop.setup(self.size, self.domain, data_file, True)
self.assertItemsEqual(self.pop.pos.dtype.names, self.expected_names)
self.assertEqual(self.pop.pos.shape, (POP_SIZE,))
self.assertEqual(self.pop.generation, 0)
self.assertEqual(self.pop.model_count, POP_SIZE)
self.assertItemsEqual(self.pop.pos['_particle'], np.arange(POP_SIZE))
self.assertItemsEqual(self.pop.pos['_gen'], [0, 0, 0, 0, 0])
self.assertItemsEqual(self.pop.pos['_model'], np.arange(POP_SIZE))
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][1], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][2], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][3], 3)
self.assertAlmostEqual(2.1, self.pop.pos['_rfac'][4], 3)
self.assertAlmostEqual(1.3, self.pop.pos['A'][0], 3)
self.assertAlmostEqual(1.1, self.pop.pos['A'][1], 3)
self.assertAlmostEqual(1.5, self.pop.pos['A'][4], 3)
self.assertAlmostEqual(2.3, self.pop.pos['B'][0], 3)
self.assertAlmostEqual(2.1, self.pop.pos['B'][1], 3)
self.assertAlmostEqual(2.5, self.pop.pos['B'][4], 3)
self.assertAlmostEqual(3.3, self.pop.pos['C'][0], 3)
self.assertAlmostEqual(3.1, self.pop.pos['C'][1], 3)
self.assertAlmostEqual(3.5, self.pop.pos['C'][4], 3)
def test_pos_gen(self):
self.pop.setup(self.size, self.domain)
for index, item in enumerate(self.pop.pos_gen()):
self.assertIsInstance(item, dict)
self.assertItemsEqual(item.keys(), self.expected_names)
self.assertEqual(item['_particle'], index)
def test_randomize(self):
self.pop.setup(self.size, self.domain)
self.pop.randomize()
m = np.mean(self.pop.pos['A'])
self.assertGreaterEqual(m, self.domain.min['A'])
self.assertLessEqual(m, self.domain.max['A'])
def test_seed(self):
self.pop.setup(self.size, self.domain)
self.pop.seed(self.domain.start)
self.assertAlmostEqual(self.pop.pos['A'][0], self.domain.start['A'], delta=0.001)
def test_best_friend(self):
self.pop.setup(self.size, self.domain)
self.pop.best['_rfac'] = np.arange(self.size)
friend = self.pop.best_friend(0)
self.assertNotIsInstance(friend, np.ndarray)
self.assertItemsEqual(friend.dtype.names, self.expected_names)
self.assertEqual(friend.dtype.names, self.expected_names)
def test_advance_particle(self):
self.pop.setup(self.size, self.domain)
@ -182,16 +101,6 @@ class TestPopulation(unittest.TestCase):
self.assertGreaterEqual(pos, self.domain.min[key])
self.assertLessEqual(pos, self.domain.max[key])
def test_add_result(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
i_result = 0
result = self.pop.pos[i_sample]
self.pop.add_result(result, 0.0)
self.assertEqual(self.pop.results.shape[0], 1)
self.assertItemsEqual(self.pop.results[i_result], result)
self.assertItemsEqual(self.pop.best[i_sample], result)
def test_is_converged(self):
self.pop.setup(self.size, self.domain)
self.assertFalse(self.pop.is_converged())
@ -206,139 +115,6 @@ class TestPopulation(unittest.TestCase):
self.pop.add_result(result, rfac)
self.assertTrue(self.pop.is_converged())
def test_save_population(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_population.pop")
self.pop.save_population(filename)
def test_save_results(self):
self.pop.setup(self.size, self.domain)
i_sample = 1
result = self.pop.pos[i_sample]
self.pop.add_result(result, 1.0)
filename = os.path.join(self.test_dir, "test_save_results.dat")
self.pop.save_results(filename)
def test_save_array(self):
self.pop.setup(self.size, self.domain)
filename = os.path.join(self.test_dir, "test_save_array.pos")
self.pop.save_array(filename, self.pop.pos)
def test_load_array(self):
n = 3
filename = os.path.join(self.test_dir, "test_load_array")
self.pop.setup(self.size, self.domain)
# expected array
dt_exp = self.pop.get_model_dtype(self.domain.start)
a_exp = np.zeros((n,), dtype=dt_exp)
a_exp['A'] = np.linspace(0, 1, n)
a_exp['B'] = np.linspace(1, 2, n)
a_exp['C'] = np.linspace(3, 4, n)
a_exp['_rfac'] = np.linspace(5, 6, n)
a_exp['_gen'] = np.array([3, 4, 7])
a_exp['_particle'] = np.array([1, 0, 2])
a_exp['_model'] = np.array([3, 6, 1])
# test array is a expected array with different column order
dt_test = [('A', 'f4'), ('_particle', 'i4'), ('_rfac', 'f4'), ('C', 'f4'), ('_gen', 'i4'), ('B', 'f4'),
('_model', 'i4')]
names_test = [a[0] for a in dt_test]
a_test = np.zeros((n,), dtype=dt_test)
for name in names_test:
a_test[name] = a_exp[name]
header = " ".join(names_test)
np.savetxt(filename, a_test, fmt='%g', header=header)
result = np.zeros((n,), dtype=dt_exp)
result = self.pop.load_array(filename, result)
self.assertItemsEqual(result.dtype.names, a_exp.dtype.names)
for name in a_exp.dtype.names:
np.testing.assert_almost_equal(result[name], a_exp[name], err_msg=name)
def test_constrain_position(self):
# upper
pos1 = 11.0
vel1 = 5.0
min1 = 0.0
max1 = 10.0
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 're-enter')
self.assertAlmostEqual(pos2, 1.0)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'bounce')
self.assertAlmostEqual(pos2, 9.0)
self.assertAlmostEqual(vel2, -vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'scatter')
self.assertGreaterEqual(pos2, 6.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, 0.0)
self.assertLessEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'stick')
self.assertAlmostEqual(pos2, max1)
self.assertAlmostEqual(vel2, max1 - (pos1 - vel1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'random')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, -(max1 - min1))
self.assertLessEqual(vel2, (max1 - min1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'expand')
self.assertAlmostEqual(pos2, pos1)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max2, pos1)
self.assertRaises(ValueError, mo.Population.constrain_position, pos1, vel1, min1, max1, 'undefined')
# lower
pos1 = -1.0
vel1 = -5.0
min1 = 0.0
max1 = 10.0
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 're-enter')
self.assertAlmostEqual(pos2, 9.0)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'bounce')
self.assertAlmostEqual(pos2, 1.0)
self.assertAlmostEqual(vel2, -vel1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'scatter')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 4.0)
self.assertGreaterEqual(vel2, vel1)
self.assertLessEqual(vel2, 0.0)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'stick')
self.assertAlmostEqual(pos2, min1)
self.assertAlmostEqual(vel2, min1 - (pos1 - vel1))
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'random')
self.assertGreaterEqual(pos2, 0.0)
self.assertLessEqual(pos2, 10.0)
self.assertGreaterEqual(vel2, -(max1 - min1))
self.assertLessEqual(vel2, max1 - min1)
self.assertAlmostEqual(min1, min2)
self.assertAlmostEqual(max1, max2)
pos2, vel2, min2, max2 = mo.Population.constrain_position(pos1, vel1, min1, max1, 'expand')
self.assertAlmostEqual(pos2, pos1)
self.assertAlmostEqual(vel2, vel1)
self.assertAlmostEqual(min2, pos1)
self.assertAlmostEqual(max2, max1)
self.assertRaises(ValueError, mo.Population.constrain_position, pos1, vel1, min1, max1, 'undefined')
def test_convergence_1(self):
self.pop.setup(self.size, self.domain)
@ -359,6 +135,6 @@ class TestPopulation(unittest.TestCase):
for pos in self.pop.pos:
self.assertLess(pos['_rfac'], 0.2)
if __name__ == '__main__':
unittest.main()

10
tests/test_swarm.setup_with_results.1.dat
View File

@ -1,5 +1,5 @@
# A B C _gen _model _rfac
1.1 2.1 3.1 1 1111 0.6
1.2 2.2 3.2 2 2222 1.5
1.3 2.3 3.3 3 3333 0.3
1.4 2.4 3.4 4 4444 1.0
# A B C E _gen _model _rfac _particle
1.1 2.1 3.1 5.1 1 1111 0.6 1
1.2 2.2 3.2 5.2 2 2222 1.5 2
1.3 2.3 4.3 5.3 3 3333 0.3 3
1.4 2.4 3.4 5.4 4 4444 1.0 4