diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..ba0430d2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+__pycache__/
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index df2aa8fa..d8fa8bed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,7 @@ stages: [check]
variables:
COLUMNS: 9000
TERM: 'xterm'
+ GIT_STRATEGY: clone
check:
tags:
@@ -13,4 +14,4 @@ check:
- module load Python
- export COLUMNS=$COLUMNS
- export TERM=$TERM
- - python3 dependency-checker.py
+ - python3 pmodules_tools/pmodules_tools.py --deps-check
diff --git a/README.md b/README.md
new file mode 100644
index 00000000..0f57e43c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,39 @@
+# Pmodules tools
+
+[](https://git.psi.ch/Pmodules/dependency-checker)
+
+## Intro
+A python project to analyse Pmodules modules and their status changes.
+
+There are for the moment two possible checks:
+
+1) Dependency checker
+ Check for module dependencies status and change their status accordingly.
+
+2) Weekly Pmodules checker
+ Report new, deleted or changed modules and write it to a pmodules_changes.md file.
+
+## Usage
+On Merlin need to first load a Python3:
+
+```sh
+module load Python
+```
+
+### CLI help interface
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --help
+```
+
+### Dependency checker
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --deps-check # or -d
+```
+
+### Weekly Pmodules checker
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --weekly-check # or -w
+```
diff --git a/pmodules_tools/deps_status/__init__.py b/pmodules_tools/deps_status/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/dependency-checker.py b/pmodules_tools/deps_status/check.py
similarity index 56%
rename from dependency-checker.py
rename to pmodules_tools/deps_status/check.py
index 96e39dbf..5f768c8e 100644
--- a/dependency-checker.py
+++ b/pmodules_tools/deps_status/check.py
@@ -1,15 +1,22 @@
-import re
+from functools import cache
+
import subprocess
import sys
-from functools import cache
+# Global parameter
+failed_at_least_once = False
-assert sys.version_info >= (3, 7), 'Python version is too low, please load a python >= 3.7'
-failed_at_least_once=False
+def deps_status_check(module_cmd_process):
+ for module in module_cmd_process.stderr.splitlines():
+ if len(module) != 0:
+ package = module.split()
+ Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
+ Pmodule_pckg.check_correct_status(module)
+
+ if failed_at_least_once:
+ sys.exit(1)
-def subprocess_cmd(cmd):
- return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, shell=True)
def failure_handler(module, given_status):
if len(module) != 0:
@@ -17,19 +24,33 @@ def failure_handler(module, given_status):
global failed_at_least_once
failed_at_least_once = True
-class PmodulePackage():
+
+def subprocess_cmd(cmd):
+ return subprocess.run(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ check=True,
+ shell=True,
+ )
+
+
+class PmodulePackage:
def __init__(self, name, status, deps=[]):
self.name = name
self.given_status = status
self.true_status = [status]
- if len(deps) != 0 and self.given_status != 'deprecated':
+ if len(deps) != 0 and self.given_status != "deprecated":
compiler = deps[0]
- if len(deps) > 1 and 'mpi' in deps[1]:
+ if len(deps) > 1 and "mpi" in deps[1]:
mpi_provider = deps[1]
else:
- mpi_provider = ''
- self.deps = [PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps]
+ mpi_provider = ""
+ self.deps = [
+ PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps
+ ]
self.true_status += self.check_deps_status()
def check_deps_status(self):
@@ -38,27 +59,27 @@ class PmodulePackage():
deps_status += d.true_status
return deps_status
- def check_correct_status(self, module=''):
- if (self.given_status != 'deprecated') and len(self.true_status) > 1:
- if 'deprecated' in self.true_status:
- self.given_status = 'deprecated'
+ def check_correct_status(self, module=""):
+ if (self.given_status != "deprecated") and len(self.true_status) > 1:
+ if "deprecated" in self.true_status:
+ self.given_status = "deprecated"
failure_handler(module, self.given_status)
- elif self.given_status == 'stable' and 'unstable' in self.true_status:
- self.given_status = 'unstable'
+ elif self.given_status == "stable" and "unstable" in self.true_status:
+ self.given_status = "unstable"
failure_handler(module, self.given_status)
self.true_status = [self.given_status]
@staticmethod
@cache
- def Pmoduliser(pckg_name='', compiler='', mpi_provider=''):
- default_module_cmd = 'module search ' + pckg_name + ' -a --all-deps --no-header'
+ def Pmoduliser(pckg_name="", compiler="", mpi_provider=""):
+ default_module_cmd = "module search " + pckg_name + " -a --all-deps --no-header"
# try to precise the module search if compiler and/or mpi_provider are given
module_cmd = default_module_cmd
- if compiler != '' and pckg_name != compiler:
- module_cmd = module_cmd + ' --with=' + compiler
- if mpi_provider != '' and pckg_name != mpi_provider:
- module_cmd = module_cmd + ' --with=' + mpi_provider
+ if compiler != "" and pckg_name != compiler:
+ module_cmd = module_cmd + " --with=" + compiler
+ if mpi_provider != "" and pckg_name != mpi_provider:
+ module_cmd = module_cmd + " --with=" + mpi_provider
module_cmd_process = subprocess_cmd(module_cmd)
# take the default command if the dependency wasn't compiled with the same compiler and/or mpi_provider
@@ -73,20 +94,11 @@ class PmodulePackage():
Pmodule_pckg.check_correct_status()
return Pmodule_pckg
else:
- print('Warning: "' + pckg_name + '" could not be found using "' + module_cmd + '"')
- return PmodulePackage('', '')
-
-def main():
- module_cmd = 'module search -a --all-deps --no-header'
- module_cmd_process = subprocess_cmd(module_cmd)
- for module in module_cmd_process.stderr.splitlines():
- if len(module) != 0:
- package = module.split()
- Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
- Pmodule_pckg.check_correct_status(module)
-
- if(failed_at_least_once):
- sys.exit(1)
-
-if __name__=="__main__":
- main()
\ No newline at end of file
+ print(
+ 'Warning: "'
+ + pckg_name
+ + '" could not be found using "'
+ + module_cmd
+ + '"'
+ )
+ return PmodulePackage("", "")
diff --git a/pmodules_tools/pmodules_tools.py b/pmodules_tools/pmodules_tools.py
new file mode 100644
index 00000000..1c81704a
--- /dev/null
+++ b/pmodules_tools/pmodules_tools.py
@@ -0,0 +1,64 @@
+import argparse
+import sys
+
+from datetime import date
+from deps_status.check import deps_status_check, subprocess_cmd
+from weekly_diff.check import weekly_check_diff
+
+assert sys.version_info >= (
+ 3,
+ 7,
+), "Python version is too low, please load a python >= 3.7"
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="A python project to analyse Pmodules modules and their status changes."
+ )
+ parser.add_argument(
+ "-d",
+ "--deps-check",
+ action="store_true",
+ help="Check for module dependencies status and change their status accordingly.",
+ )
+ parser.add_argument(
+ "-w",
+ "--weekly-check",
+ action="store_true",
+ help="Report new, deleted or changed modules and print it to pmodules_changes.md.",
+ )
+ parser.add_argument(
+ "--select-module",
+ default="",
+ help="Select a subtype of module which should be checked for. Default is all.",
+ )
+
+ args = parser.parse_args()
+ return args
+
+
+def main():
+ # Analyse CLI args
+ cli_args = parse_args()
+
+ # Global variables
+ Pmodules_db_path = "/afs/psi.ch/sys/spack-rhel7/test/"
+ Pmodules_states = ["deprecated", "stable", "unstable"]
+
+ # Main search to analyse
+ module_cmd = (
+ "module search -a " + cli_args.select_module + " --all-deps --no-header"
+ )
+ module_cmd_process = subprocess_cmd(module_cmd)
+
+ # Check for Pmodules addition, deletion or changes of state
+ if cli_args.weekly_check:
+ weekly_check_diff(module_cmd_process.stderr, Pmodules_db_path, Pmodules_states)
+
+ # Check for module dependencies status and change the main module status accordingly
+ if cli_args.deps_check:
+ deps_status_check(module_cmd_process)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pmodules_tools/weekly_diff/__init__.py b/pmodules_tools/weekly_diff/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/pmodules_tools/weekly_diff/check.py b/pmodules_tools/weekly_diff/check.py
new file mode 100644
index 00000000..77ee2d7b
--- /dev/null
+++ b/pmodules_tools/weekly_diff/check.py
@@ -0,0 +1,157 @@
+import difflib
+import hashlib
+import os
+import re
+import shutil
+
+
+def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
+ if (len(old_pmodule_state)) != 0:
+ old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
+ if current_sha != old_sha:
+ return True
+ else:
+ return False
+
+
+def print_to_markdown(new_module_list, deleted_module_list, changed_module_list):
+ with open("pmodules_changes.md", "w") as md_file:
+ standard_print(md_file, new_module_list, "New modules:")
+
+ md_file.write("\n# Changed modules: \n")
+ if len(changed_module_list) > 1:
+ for index in range(0, len(changed_module_list) - 1, 2):
+ md_file.write(
+ changed_module_list[index]
+ + "
Changed state from "
+ + changed_module_list[index + 1].split()[1]
+ + " to ---> "
+ + changed_module_list[index].split()[1]
+ + "
"
+ )
+
+ standard_print(md_file, deleted_module_list, "Deleted modules:")
+
+
+def print_pmodules_differences(
+ current_pmodule_state, old_pmodule_state, Pmodules_states
+):
+ # Make sure the whitespaces in between the module descriptions are always one space only.
+ current_db = set(" ".join(i.split()) for i in current_pmodule_state.splitlines())
+ old_db = set(" ".join(i.split()) for i in old_pmodule_state.splitlines())
+
+ new_module_list = list(current_db - old_db)
+ deleted_module_list = list(old_db - current_db)
+ changed_module_list = []
+
+ # Replace the state of the modules with * for Regex comparison
+ new_module_list_no_state = new_module_list.copy()
+ deleted_module_list_no_state = deleted_module_list.copy()
+ for state in Pmodules_states:
+ new_module_list_no_state = [
+ diff.replace(state, ".*") for diff in new_module_list_no_state
+ ]
+ deleted_module_list_no_state = [
+ diff.replace(state, ".*") for diff in deleted_module_list_no_state
+ ]
+
+ # Check if the state of the module is the only thing that changed
+ # Append changed_module_list if yes
+ if len(deleted_module_list_no_state) >= len(new_module_list_no_state):
+ set_changed_module_list(
+ new_module_list_no_state,
+ deleted_module_list_no_state,
+ new_module_list,
+ deleted_module_list,
+ changed_module_list,
+ )
+ else:
+ set_changed_module_list(
+ deleted_module_list_no_state,
+ new_module_list_no_state,
+ new_module_list,
+ deleted_module_list,
+ changed_module_list,
+ )
+
+ print_to_markdown(new_module_list, deleted_module_list, changed_module_list)
+
+
+def set_changed_module_list(
+ small_module_list_no_state,
+ big_module_list_no_state,
+ new_module_list,
+ deleted_module_list,
+ changed_module_list,
+):
+ for diff in small_module_list_no_state:
+ if diff in big_module_list_no_state:
+ new_module_index = [
+ idx
+ for idx, string in enumerate(new_module_list)
+ if re.search(diff, string)
+ ][0]
+ deleted_module_index = [
+ idx
+ for idx, string in enumerate(deleted_module_list)
+ if re.search(diff, string)
+ ][0]
+ changed_module_list += [
+ new_module_list[new_module_index],
+ deleted_module_list[deleted_module_index],
+ ]
+ new_module_list[new_module_index] = ""
+ deleted_module_list[deleted_module_index] = ""
+
+
+def standard_print(file, module_list, string):
+ file.write("\n# " + string + " \n")
+ for diff in module_list:
+ if diff != "":
+ file.write(diff + "
")
+
+
+def weekly_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
+ current_sha = hashlib.sha256(current_pmodule_state.encode()).hexdigest()
+
+ no_current_db = False
+
+ # Retrieve old state for comparison
+ try:
+ old_pmodule_file = sorted(
+ [Pmodules_db_path + f for f in os.listdir(Pmodules_db_path)],
+ key=os.path.getctime,
+ )[0]
+ old_pmodule_state = open(old_pmodule_file, "r").read()
+ except:
+ print(
+ "There is no old Pmodule database available on path "
+ + Pmodules_db_path
+ + "... Writing current one"
+ )
+ no_current_db = True
+
+ # There is a database and we have to check the differences between the pmodule states.
+ if not no_current_db and compare_states_sha(
+ current_pmodule_state, old_pmodule_state, current_sha
+ ):
+ print_pmodules_differences(
+ current_pmodule_state, old_pmodule_state, Pmodules_states
+ )
+
+ # There is no database available or there are differences with the old pmodule state, writing current state.
+ if no_current_db or compare_states_sha(
+ current_pmodule_state, old_pmodule_state, current_sha
+ ):
+ write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path)
+
+
+def write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path):
+ # Emptying Pmodules database first
+ if os.path.exists(Pmodules_db_path):
+ shutil.rmtree(Pmodules_db_path)
+
+ # Recreating dir and writing new database
+ os.makedirs(Pmodules_db_path)
+ with open(Pmodules_db_path + current_sha, "w") as current_pmodule_file:
+ current_pmodule_file.write(current_pmodule_state)