usov_i-org/bec
1
0
Fork 0
mirror of https://github.com/ivan-usov-org/bec.git synced 2025-04-20 01:40:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

ci: drop python/3.9

Browse Source
This commit is contained in:
usov_i 2024-02-27 15:47:54 +01:00 committed by wakonig_k
parent a4adb64f5f
commit d16268ce81
25 changed files with 77 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.gitlab-ci.yml
View File

@ -1,7 +1,7 @@
# This file is a template, and might need editing before it works on your project.
# Official language image. Look for the different tagged releases at:
# https://hub.docker.com/r/library/python/tags/
image: $CI_DOCKER_REGISTRY/python:3.9
image: $CI_DOCKER_REGISTRY/python:3.10
#commands to run in the Docker container before starting each job.
variables:
DOCKER_TLS_CERTDIR: ""
@ -72,7 +72,7 @@ pylint:
artifacts:
paths:
- ./pylint/
expire_in: 1 week
expire_in: 1 week
pylint-check:
stage: Formatter
@ -128,11 +128,11 @@ tests:
junit: report.xml
coverage_report:
coverage_format: cobertura
path: coverage.xml
path: coverage.xml
tests-3.10:
tests-3.11:
stage: AdditionalTests
image: $CI_DOCKER_REGISTRY/python:3.10
image: $CI_DOCKER_REGISTRY/python:3.11
needs: ["tests"]
allow_failure: true
script:
@ -145,14 +145,8 @@ tests-3.10:
- *install-bec-services-dev
- pytest -v --junitxml=report.xml --random-order ./data_processing/tests ./bec_lib/tests ./scan_server/tests ./device_server/tests ./scan_bundler/tests ./bec_client/tests/client_tests ./file_writer/tests ./scihub/tests
tests-3.11:
extends: "tests-3.10"
stage: AdditionalTests
image: $CI_DOCKER_REGISTRY/python:3.11
allow_failure: true
tests-3.12:
extends: "tests-3.10"
extends: "tests-3.11"
stage: AdditionalTests
image: $CI_DOCKER_REGISTRY/python:3.12
allow_failure: true
@ -173,7 +167,7 @@ end-2-end:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
- if: '$CI_PIPELINE_SOURCE == "pipeline"'
- if: '$CI_PIPELINE_SOURCE == "pipeline"'
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "production"'
script:
@ -185,17 +179,17 @@ end-2-end:
- git clone --branch $OPHYD_DEVICES_BRANCH https://oauth2:$CI_OPHYD_DEVICES_KEY@gitlab.psi.ch/bec/ophyd_devices.git
# initialize the database with the test config
- docker build -t init_config:test -f ./ci/config_init_dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t init_config:test -f ./ci/config_init_dockerfile --build-arg PY_VERSION=3.10 .
- docker run --network=host --name init_config init_config:test
- docker build -t scihub:test -f ./scihub/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t scihub:test -f ./scihub/Dockerfile --build-arg PY_VERSION=3.10 .
- docker run --network=host -d --name scihub scihub:test
# build scan_bundler, scan_server, device_server and file_writer
- docker build -t scan_bundler:test -f ./scan_bundler/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t scan_server:test -f ./scan_server/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t device_server:test -f ./device_server/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t file_writer:test -f ./file_writer/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t data_processing:test -f ./data_processing/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t scan_bundler:test -f ./scan_bundler/Dockerfile --build-arg PY_VERSION=3.10 .
- docker build -t scan_server:test -f ./scan_server/Dockerfile --build-arg PY_VERSION=3.10 .
- docker build -t device_server:test -f ./device_server/Dockerfile --build-arg PY_VERSION=3.10 .
- docker build -t file_writer:test -f ./file_writer/Dockerfile --build-arg PY_VERSION=3.10 .
- docker build -t data_processing:test -f ./data_processing/Dockerfile --build-arg PY_VERSION=3.10 .
# run scan_bundler, scan_server, device_server and file_writer
- docker run --network=host -d --name device_server device_server:test
@ -205,7 +199,7 @@ end-2-end:
- docker run --network=host -d --name data_processing data_processing:test
# build and run the tests
- docker build -t en2end_client:test -f ./bec_client/tests/Dockerfile --build-arg PY_VERSION=3.9 .
- docker build -t en2end_client:test -f ./bec_client/tests/Dockerfile --build-arg PY_VERSION=3.10 .
- docker run --network=host --name end2end_client en2end_client:test
after_script:
@ -265,7 +259,7 @@ end-2-end-conda:
- apt-get update
- apt-get install -y tmux
- conda config --set always_yes yes --set changeps1 no
- conda create -q -n test-environment python=3.9
- conda create -q -n test-environment python=3.10
- conda init bash
- source ~/.bashrc
- conda activate test-environment
@ -300,17 +294,17 @@ end-2-end-conda:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
- if: '$CI_PIPELINE_SOURCE == "pipeline"'
- if: '$CI_PIPELINE_SOURCE == "pipeline"'
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "production"'
end-2-end-conda-39:
end-2-end-conda-310:
stage: End2End
needs: []
image: continuumio/miniconda3
allow_failure: false
variables:
PYTHON_VERSION: "3.9"
PYTHON_VERSION: "3.10"
script:
- apt-get update
- apt-get install -y tmux
@ -352,30 +346,23 @@ end-2-end-conda-39:
rules:
- if: '$E2E_FULL == "1"'
end-2-end-conda-310:
stage: End2End
extends: "end-2-end-conda-39"
allow_failure: false
variables:
PYTHON_VERSION: "3.10"
end-2-end-conda-311:
stage: End2End
extends: "end-2-end-conda-39"
extends: "end-2-end-conda-310"
allow_failure: false
variables:
PYTHON_VERSION: "3.11"
end-2-end-conda-312:
stage: End2End
extends: "end-2-end-conda-39"
extends: "end-2-end-conda-310"
allow_failure: false
variables:
PYTHON_VERSION: "3.12"
end-2-end-conda-313:
stage: End2End
extends: "end-2-end-conda-39"
extends: "end-2-end-conda-310"
allow_failure: true
variables:
PYTHON_VERSION: "3.13"
@ -454,4 +441,3 @@ pages:
script:
- curl -X POST -d "branches=$CI_COMMIT_REF_NAME" -d "token=$RTD_TOKEN" https://readthedocs.org/api/v2/webhook/beamline-experiment-control/221870/
- curl -X POST -d "branches=$CI_COMMIT_REF_NAME" -d "token=$RTD_TOKEN_BEC" https://readthedocs.org/api/v2/webhook/bec/246899/

2
.pylintrc
View File

@ -52,7 +52,7 @@ persistent=yes
# Minimum Python version to use for version dependent checks. Will default to
# the version used to run pylint.
py-version=3.9
py-version=3.10
# When enabled, pylint would attempt to guess common misconfiguration and emit
# user-friendly hints instead of false-positive error messages.

2
.readthedocs.yaml
View File

@ -9,7 +9,7 @@ version: 2
build:
os: ubuntu-20.04
tools:
python: "3.9"
python: "3.10"
# Build documentation in the docs/ directory with Sphinx
sphinx:

2
bec_client/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
bec_client/tests/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
bec_lib/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

4
bec_lib/util_scripts/plugin_setup_files/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .
where = .

2
bec_server/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
bin/install_bec_dev.sh
View File

@ -48,7 +48,7 @@ while getopts "hsc:rt" o; do
esac
done
conda_deps=(python=3.9)
conda_deps=(python=3.10)
# check if tmux should be installed
if [ "$skip_tmux" = false ]; then

2
ci/config_init_dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
data_processing/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
data_processing/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
device_server/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
device_server/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

19
docs/source/developer/bec_plugins.md
View File

@ -1,5 +1,5 @@
(developer.bec_plugins)=
# BEC Plugins
# BEC Plugins
BEC plugins are a way to extend the functionality of BEC. They are written in Python and can be used to add new features to BEC or to modify existing ones. This enables beamlines to customize BEC to their needs without having to modify the core code. Plugins can be used for various purposes but the most common ones are:
* Adding new scan types
@ -12,7 +12,7 @@ Plugins are commonly provided to BEC by installing them as a Python package `bec
## Plugin Structure
The following sections describe the structure of a BEC plugin. As plugins typically live on gitlab, we will use the following example structure of a "beamline_XX_plugins" repository to explain the different parts of BEC plugins. Instead of creating the structure manually, you can also use the script located in BEC library to create the structure for you.
The following sections describe the structure of a BEC plugin. As plugins typically live on gitlab, we will use the following example structure of a "beamline_XX_plugins" repository to explain the different parts of BEC plugins. Instead of creating the structure manually, you can also use the script located in BEC library to create the structure for you.
```bash
python ./<path_to_bec>/bec/bec_lib/util_scripts/create_plugin_structure.py <path_to_new_plugin>
```
@ -57,7 +57,7 @@ beamline_XX_plugins/
└── setup.py
```
<!-- done with https://tree.nathanfriend.io -->
<!--
<!--
beamline_XX_plugins
bec_plugins
bec_client
@ -100,9 +100,9 @@ beamline_XX_plugins
Within the root directory of your repository, place the [setup files](#setup_files) and the plugins folder. The plugins folder contains the actual plugins and is structured as follows:
* `bec_client`: Contains plugins that are used by the BEC client. This includes plugins to customize the startup procedure, adding helper classes and functions to the CLI or adding aliases to the CLI to simplify the usage of BEC.
* `device_server`: Contains plugins that are used by the device server.
* `scan_server`: Contains plugins that are used by the scan server. This includes plugins to add new scan types.
* `scan_server`: Contains plugins that are used by the scan server. This includes plugins to add new scan types.
* `devices`: Contains plugins that are used to add support for new devices that are not covered by the shared ophyd_devices package.
* `device_configs`: Contains the configuration files for the devices.
* `device_configs`: Contains the configuration files for the devices.
{#setup_files}
@ -118,10 +118,10 @@ from setuptools import setup
if __name__ == "__main__":
setup(
# specify the additional dependencies
install_requires=["pyyaml", "pyepics"],
install_requires=["pyyaml", "pyepics"],
# if you have additional dependencies for development, specify them here
extras_require={"dev": ["pytest", "pytest-random-order", "coverage"]},
extras_require={"dev": ["pytest", "pytest-random-order", "coverage"]},
)
```
@ -146,7 +146,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .
@ -157,6 +157,3 @@ where = .
``` {note}
While the `setup.py` file can (and probably should) be modified to fit your needs, the `setup.cfg` file and especially the name of the package ("bec_plugins") should not be changed. This is because the BEC services and clients look for plugins in a package called "bec_plugins".
```

29
docs/source/developer/install_developer_env.md
View File

@ -1,7 +1,7 @@
(developer.install_developer_env)=
# Install developer environment
If your goal is to install BEC in an environment for code development purposes, this section will guide you through the steps.
If your goal is to install BEC in an environment for code development purposes, this section will guide you through the steps.
In contrast to a deployed production system of BEC, this installation will allow you to edit the code base of BEC actively while you are operating the system.
In that sense, installing BEC in _[dev]_ mode, is the right choice in case you like to:
@ -12,13 +12,13 @@ In that sense, installing BEC in _[dev]_ mode, is the right choice in case you l
**Requirements:**
---
- [python](https://www.python.org) (>=3.9)
- [python](https://www.python.org) (>=3.10)
- [redis](https://redis.io)
- [tmux](https://github.com/tmux/tmux/wiki) (=3.2)
---
On a PSI-system, requirements are available via pmodules. If you run BEC on your own system, make sure to install the required packages.
On a PSI-system, requirements are available via pmodules. If you run BEC on your own system, make sure to install the required packages.
**Step-by-Step Guide**
The first step is to clone the repository
@ -37,10 +37,11 @@ cd bec
2. Satisfy requirements
On PSI-maintained systems with pmodules, you can simply load psi-python39/2021.11 and tmux/3.2 via
On PSI-maintained systems with pmodules, you can simply load psi-python311/2024.02 and tmux/3.2 via
```{code-block} bash
module add psi-python39/2021.11
```{code-block} bash
module use unstable
module add psi-python311/2024.02
module add tmux/3.2
```
@ -55,7 +56,7 @@ source ./bec_venv/bin/activate
4. Install BEC
To keep things simple, we have compiled all dependencies within the `setup.py` from `bec_server`.
To keep things simple, we have compiled all dependencies within the `setup.py` from `bec_server`.
Note, you need to install the package in editable mode (with `-e` flag), to allow changes to the code base.
```bash
@ -63,14 +64,14 @@ pip install wheel
pip install -e './bec_server/.[dev]'
```
```{warning}
On newer versions of pip (pip>22.3.1) wheel must be installed manually before installing the BEC server! If not, it will crash with an import error within pip. Should you run into problems, please run `pip cache purge` twice before repeating the commands above.
On newer versions of pip (pip>22.3.1) wheel must be installed manually before installing the BEC server! If not, it will crash with an import error within pip. Should you run into problems, please run `pip cache purge` twice before repeating the commands above.
```
```{note}
The extension [dev] will install additional dependencies, which are useful for code development such as for instance `pytest`, `black`.
```
5. Start Redis
Open a new terminal, and start Redis.
Open a new terminal, and start Redis.
Make sure that you've either loaded the pmodule or installed Redis on your system.
```
module add redis/7.0.12
@ -86,7 +87,7 @@ Redis will create a `dump.rdb`, where it regularly stores data on disk. Make sur
6. Start BEC server
Now we can start the BEC server.
Make sure that you activate the `bec_venv` created above, and that `tmux/3.2` is availabe, e.g. loaded via pmodule.
Make sure that you activate the `bec_venv` created above, and that `tmux/3.2` is availabe, e.g. loaded via pmodule.
Then you can start the BEC server
```bash
bec-server start
@ -97,7 +98,7 @@ You may open the tmux session to look at the different BEC services via
```bash
tmux attach -t bec
```
and exit the tmux session again via `CTRL+B+D`.
and exit the tmux session again via `CTRL+B+D`.
Both commands are also highlighted in your command-line interface.
```{note}
@ -110,19 +111,19 @@ Strictly speaking, you do not need to install tmux. However, if you do not use t
bec
```
You are now ready to load your first device configuration.
You are now ready to load your first device configuration.
To this end, please follow the instructions given in [bec_config](#developer.bec_config).
8. Start services with different port
It could be the case, that port `6379` is already occupied or that you have to run multiple Redis server on the same system.
It could be the case, that port `6379` is already occupied or that you have to run multiple Redis server on the same system.
If this is the case, you can also spin up the system with a modified configuration, e.g. on port `xxxx`.
The redis-server can be passed on a specific port.
```bash
redis-server --port xxxx
```
In addition, you will have to start the bec-server with a customized config.
In addition, you will have to start the bec-server with a customized config.
Please check the example file ``bec/bec_config_template.yaml`` to create a custom config and specify port `xxxx` and pass it to the bec-server upon start
``` bash

19
docs/source/user/installation.md
View File

@ -2,20 +2,21 @@
## Installation
If you are using BEC at the beamline, there is a good chance that BEC is already installed.
Please contact your beamline responsible for further information.
Please contact your beamline responsible for further information.
If you need to install BEC yourself, the following section will guide you through this.
**Requirements:**
---
- [python](https://www.python.org) (>=3.9)
- [python](https://www.python.org) (>=3.10)
- [redis](https://redis.io)
- [tmux](https://github.com/tmux/tmux/wiki) (=3.2)
---
On a PSI-system, requirements are available via pmodules. If you run BEC on your own system, make sure to install the required packages.
```{code-block} bash
module add psi-python39/2021.11
On a PSI-system, requirements are available via pmodules. If you run BEC on your own system, make sure to install the required packages.
```{code-block} bash
module use unstable
module add psi-python311/2024.02
module add redis/7.0.12
module add tmux/3.2
```
@ -38,8 +39,8 @@ pip install bec-server
```{code-block} bash
redis-server
```
BEC services are connected via Redis, a message broker sitting at the core of all BEC services.
Thus, Redis needs to be started on your system.
BEC services are connected via Redis, a message broker sitting at the core of all BEC services.
Thus, Redis needs to be started on your system.
If the pmodule is loaded (or Redis installed on your system), open a new terminal and start a redis server.
Redis will automatically dump data on disk into the file `dump.rdb`, up to a few GB, and should therefore be started in a location with sufficient storage.
@ -51,7 +52,7 @@ Now you can go back to the terminal where the bec_venv is still activated and st
```{code-block} bash
bec-server start
```
The BEC server will automatically start in a tmux session.
The BEC server will automatically start in a tmux session.
More detailed information about Redis and the BEC server can be found in [architecture](#developer.architecture) and [developer install guide](#developer.install_developer_env)
5. Start BEC client
@ -59,5 +60,5 @@ More detailed information about Redis and the BEC server can be found in [archit
```{code-block} bash
bec
```
BEC is running now and you would be ready to load your first device configuration.
BEC is running now and you would be ready to load your first device configuration.
To this end, please follow the instructions given in the section [devices](#user.devices).

2
file_writer/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
file_writer/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
scan_bundler/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
scan_bundler/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
scan_server/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
scan_server/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .

2
scihub/Dockerfile
View File

@ -1,5 +1,5 @@
# set base image (host OS)
ARG PY_VERSION=3.9
ARG PY_VERSION=3.10
FROM morgana-harbor.psi.ch/bec/python:${PY_VERSION}
# set the working directory in the container

2
scihub/setup.cfg
View File

@ -15,7 +15,7 @@ classifiers =
package_dir =
= .
packages = find:
python_requires = >=3.9
python_requires = >=3.10
[options.packages.find]
where = .