HPCE/Pmodules_tools
HPCE/Pmodules_tools
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring + adding weekly-checker tool

Browse Source
This commit is contained in:
germann_e 2023-02-03 07:48:22 +00:00
parent eeec149e8c
commit d6870030bf
8 changed files with 315 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
__pycache__/

3
.gitlab-ci.yml
View File

@ -3,6 +3,7 @@ stages: [check]
variables:
COLUMNS: 9000
TERM: 'xterm'
GIT_STRATEGY: clone
check:
tags:
@ -13,4 +14,4 @@ check:
- module load Python
- export COLUMNS=$COLUMNS
- export TERM=$TERM
- python3 dependency-checker.py
- python3 pmodules_tools/pmodules_tools.py --deps-check

39
README.md Normal file
View File

@ -0,0 +1,39 @@
# Pmodules tools
[![Dependency checker](https://git.psi.ch/Pmodules/dependency-checker/badges/main/pipeline.svg)](https://git.psi.ch/Pmodules/dependency-checker)
## Intro
A python project to analyse Pmodules modules and their status changes.
There are for the moment two possible checks:
1) Dependency checker <br>
Check for module dependencies status and change their status accordingly.
2) Weekly Pmodules checker <br>
Report new, deleted or changed modules and write it to a pmodules_changes.md file.
## Usage
On Merlin need to first load a Python3:
```sh
module load Python
```
### CLI help interface
```sh
python3 pmodules_tools/pmodules_tools.py --help
```
### Dependency checker
```sh
python3 pmodules_tools/pmodules_tools.py --deps-check # or -d
```
### Weekly Pmodules checker
```sh
python3 pmodules_tools/pmodules_tools.py --weekly-check # or -w
```

0
pmodules_tools/deps_status/__init__.py Normal file
View File

92
dependency-checker.py → pmodules_tools/deps_status/check.py
View File

@ -1,15 +1,22 @@
import re
from functools import cache
import subprocess
import sys
from functools import cache
# Global parameter
failed_at_least_once = False
assert sys.version_info >= (3, 7), 'Python version is too low, please load a python >= 3.7'
failed_at_least_once=False
def deps_status_check(module_cmd_process):
for module in module_cmd_process.stderr.splitlines():
if len(module) != 0:
package = module.split()
Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
Pmodule_pckg.check_correct_status(module)
if failed_at_least_once:
sys.exit(1)
def subprocess_cmd(cmd):
return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, shell=True)
def failure_handler(module, given_status):
if len(module) != 0:
@ -17,19 +24,33 @@ def failure_handler(module, given_status):
global failed_at_least_once
failed_at_least_once = True
class PmodulePackage():
def subprocess_cmd(cmd):
return subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=True,
shell=True,
)
class PmodulePackage:
def __init__(self, name, status, deps=[]):
self.name = name
self.given_status = status
self.true_status = [status]
if len(deps) != 0 and self.given_status != 'deprecated':
if len(deps) != 0 and self.given_status != "deprecated":
compiler = deps[0]
if len(deps) > 1 and 'mpi' in deps[1]:
if len(deps) > 1 and "mpi" in deps[1]:
mpi_provider = deps[1]
else:
mpi_provider = ''
self.deps = [PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps]
mpi_provider = ""
self.deps = [
PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps
]
self.true_status += self.check_deps_status()
def check_deps_status(self):
@ -38,27 +59,27 @@ class PmodulePackage():
deps_status += d.true_status
return deps_status
def check_correct_status(self, module=''):
if (self.given_status != 'deprecated') and len(self.true_status) > 1:
if 'deprecated' in self.true_status:
self.given_status = 'deprecated'
def check_correct_status(self, module=""):
if (self.given_status != "deprecated") and len(self.true_status) > 1:
if "deprecated" in self.true_status:
self.given_status = "deprecated"
failure_handler(module, self.given_status)
elif self.given_status == 'stable' and 'unstable' in self.true_status:
self.given_status = 'unstable'
elif self.given_status == "stable" and "unstable" in self.true_status:
self.given_status = "unstable"
failure_handler(module, self.given_status)
self.true_status = [self.given_status]
@staticmethod
@cache
def Pmoduliser(pckg_name='', compiler='', mpi_provider=''):
default_module_cmd = 'module search ' + pckg_name + ' -a --all-deps --no-header'
def Pmoduliser(pckg_name="", compiler="", mpi_provider=""):
default_module_cmd = "module search " + pckg_name + " -a --all-deps --no-header"
# try to precise the module search if compiler and/or mpi_provider are given
module_cmd = default_module_cmd
if compiler != '' and pckg_name != compiler:
module_cmd = module_cmd + ' --with=' + compiler
if mpi_provider != '' and pckg_name != mpi_provider:
module_cmd = module_cmd + ' --with=' + mpi_provider
if compiler != "" and pckg_name != compiler:
module_cmd = module_cmd + " --with=" + compiler
if mpi_provider != "" and pckg_name != mpi_provider:
module_cmd = module_cmd + " --with=" + mpi_provider
module_cmd_process = subprocess_cmd(module_cmd)
# take the default command if the dependency wasn't compiled with the same compiler and/or mpi_provider
@ -73,20 +94,11 @@ class PmodulePackage():
Pmodule_pckg.check_correct_status()
return Pmodule_pckg
else:
print('Warning: "' + pckg_name + '" could not be found using "' + module_cmd + '"')
return PmodulePackage('', '')
def main():
module_cmd = 'module search -a --all-deps --no-header'
module_cmd_process = subprocess_cmd(module_cmd)
for module in module_cmd_process.stderr.splitlines():
if len(module) != 0:
package = module.split()
Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
Pmodule_pckg.check_correct_status(module)
if(failed_at_least_once):
sys.exit(1)
if __name__=="__main__":
main()
print(
'Warning: "'
+ pckg_name
+ '" could not be found using "'
+ module_cmd
+ '"'
)
return PmodulePackage("", "")

64
pmodules_tools/pmodules_tools.py Normal file
View File

@ -0,0 +1,64 @@
import argparse
import sys
from datetime import date
from deps_status.check import deps_status_check, subprocess_cmd
from weekly_diff.check import weekly_check_diff
assert sys.version_info >= (
3,
7,
), "Python version is too low, please load a python >= 3.7"
def parse_args():
parser = argparse.ArgumentParser(
description="A python project to analyse Pmodules modules and their status changes."
)
parser.add_argument(
"-d",
"--deps-check",
action="store_true",
help="Check for module dependencies status and change their status accordingly.",
)
parser.add_argument(
"-w",
"--weekly-check",
action="store_true",
help="Report new, deleted or changed modules and print it to pmodules_changes.md.",
)
parser.add_argument(
"--select-module",
default="",
help="Select a subtype of module which should be checked for. Default is all.",
)
args = parser.parse_args()
return args
def main():
# Analyse CLI args
cli_args = parse_args()
# Global variables
Pmodules_db_path = "/afs/psi.ch/sys/spack-rhel7/test/"
Pmodules_states = ["deprecated", "stable", "unstable"]
# Main search to analyse
module_cmd = (
"module search -a " + cli_args.select_module + " --all-deps --no-header"
)
module_cmd_process = subprocess_cmd(module_cmd)
# Check for Pmodules addition, deletion or changes of state
if cli_args.weekly_check:
weekly_check_diff(module_cmd_process.stderr, Pmodules_db_path, Pmodules_states)
# Check for module dependencies status and change the main module status accordingly
if cli_args.deps_check:
deps_status_check(module_cmd_process)
if __name__ == "__main__":
main()

0
pmodules_tools/weekly_diff/__init__.py Normal file
View File

157
pmodules_tools/weekly_diff/check.py Normal file
View File

@ -0,0 +1,157 @@
import difflib
import hashlib
import os
import re
import shutil
def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
if (len(old_pmodule_state)) != 0:
old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
if current_sha != old_sha:
return True
else:
return False
def print_to_markdown(new_module_list, deleted_module_list, changed_module_list):
with open("pmodules_changes.md", "w") as md_file:
standard_print(md_file, new_module_list, "New modules:")
md_file.write("\n# Changed modules: \n")
if len(changed_module_list) > 1:
for index in range(0, len(changed_module_list) - 1, 2):
md_file.write(
changed_module_list[index]
+ "<br>Changed state from "
+ changed_module_list[index + 1].split()[1]
+ " to ---> "
+ changed_module_list[index].split()[1]
+ "<br>"
)
standard_print(md_file, deleted_module_list, "Deleted modules:")
def print_pmodules_differences(
current_pmodule_state, old_pmodule_state, Pmodules_states
):
# Make sure the whitespaces in between the module descriptions are always one space only.
current_db = set(" ".join(i.split()) for i in current_pmodule_state.splitlines())
old_db = set(" ".join(i.split()) for i in old_pmodule_state.splitlines())
new_module_list = list(current_db - old_db)
deleted_module_list = list(old_db - current_db)
changed_module_list = []
# Replace the state of the modules with * for Regex comparison
new_module_list_no_state = new_module_list.copy()
deleted_module_list_no_state = deleted_module_list.copy()
for state in Pmodules_states:
new_module_list_no_state = [
diff.replace(state, ".*") for diff in new_module_list_no_state
]
deleted_module_list_no_state = [
diff.replace(state, ".*") for diff in deleted_module_list_no_state
]
# Check if the state of the module is the only thing that changed
# Append changed_module_list if yes
if len(deleted_module_list_no_state) >= len(new_module_list_no_state):
set_changed_module_list(
new_module_list_no_state,
deleted_module_list_no_state,
new_module_list,
deleted_module_list,
changed_module_list,
)
else:
set_changed_module_list(
deleted_module_list_no_state,
new_module_list_no_state,
new_module_list,
deleted_module_list,
changed_module_list,
)
print_to_markdown(new_module_list, deleted_module_list, changed_module_list)
def set_changed_module_list(
small_module_list_no_state,
big_module_list_no_state,
new_module_list,
deleted_module_list,
changed_module_list,
):
for diff in small_module_list_no_state:
if diff in big_module_list_no_state:
new_module_index = [
idx
for idx, string in enumerate(new_module_list)
if re.search(diff, string)
][0]
deleted_module_index = [
idx
for idx, string in enumerate(deleted_module_list)
if re.search(diff, string)
][0]
changed_module_list += [
new_module_list[new_module_index],
deleted_module_list[deleted_module_index],
]
new_module_list[new_module_index] = ""
deleted_module_list[deleted_module_index] = ""
def standard_print(file, module_list, string):
file.write("\n# " + string + " \n")
for diff in module_list:
if diff != "":
file.write(diff + "<br>")
def weekly_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
current_sha = hashlib.sha256(current_pmodule_state.encode()).hexdigest()
no_current_db = False
# Retrieve old state for comparison
try:
old_pmodule_file = sorted(
[Pmodules_db_path + f for f in os.listdir(Pmodules_db_path)],
key=os.path.getctime,
)[0]
old_pmodule_state = open(old_pmodule_file, "r").read()
except:
print(
"There is no old Pmodule database available on path "
+ Pmodules_db_path
+ "... Writing current one"
)
no_current_db = True
# There is a database and we have to check the differences between the pmodule states.
if not no_current_db and compare_states_sha(
current_pmodule_state, old_pmodule_state, current_sha
):
print_pmodules_differences(
current_pmodule_state, old_pmodule_state, Pmodules_states
)
# There is no database available or there are differences with the old pmodule state, writing current state.
if no_current_db or compare_states_sha(
current_pmodule_state, old_pmodule_state, current_sha
):
write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path)
def write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path):
# Emptying Pmodules database first
if os.path.exists(Pmodules_db_path):
shutil.rmtree(Pmodules_db_path)
# Recreating dir and writing new database
os.makedirs(Pmodules_db_path)
with open(Pmodules_db_path + current_sha, "w") as current_pmodule_file:
current_pmodule_file.write(current_pmodule_state)