Refactoring + adding weekly-checker tool

.gitignore

@@ -0,0 +1 @@
+__pycache__/

.gitlab-ci.yml

@@ -3,6 +3,7 @@ stages: [check]
 variables:
   COLUMNS: 9000
   TERM: 'xterm'
+  GIT_STRATEGY: clone
 
 check:
   tags:
@@ -13,4 +14,4 @@ check:
     - module load Python
     - export COLUMNS=$COLUMNS
     - export TERM=$TERM
-    - python3 dependency-checker.py
+    - python3 pmodules_tools/pmodules_tools.py --deps-check

README.md

@@ -0,0 +1,39 @@
+# Pmodules tools
+
+[![Dependency checker](https://git.psi.ch/Pmodules/dependency-checker/badges/main/pipeline.svg)](https://git.psi.ch/Pmodules/dependency-checker)
+
+## Intro
+A python project to analyse Pmodules modules and their status changes.
+
+There are for the moment two possible checks:
+
+1) Dependency checker <br>
+   Check for module dependencies status and change their status accordingly.
+
+2) Weekly Pmodules checker <br> 
+   Report new, deleted or changed modules and write it to a pmodules_changes.md file.
+
+## Usage
+On Merlin need to first load a Python3:
+
+```sh
+module load Python
+```
+
+### CLI help interface
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --help
+```
+
+### Dependency checker
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --deps-check # or -d
+```
+
+### Weekly Pmodules checker
+
+```sh
+python3 pmodules_tools/pmodules_tools.py --weekly-check # or -w
+```

pmodules_tools/deps_status/check.py

@@ -1,15 +1,22 @@
-import re
+from functools import cache
+
 import subprocess
 import sys
 
-from functools import cache
+# Global parameter
+failed_at_least_once = False
 
-assert sys.version_info >= (3, 7), 'Python version is too low, please load a python >= 3.7'
 
-failed_at_least_once=False
+def deps_status_check(module_cmd_process):
+    for module in module_cmd_process.stderr.splitlines():
+        if len(module) != 0:
+            package = module.split()
+            Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
+            Pmodule_pckg.check_correct_status(module)
+
+    if failed_at_least_once:
+        sys.exit(1)
 
-def subprocess_cmd(cmd):
-    return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True, shell=True)
 
 def failure_handler(module, given_status):
     if len(module) != 0:
@@ -17,19 +24,33 @@ def failure_handler(module, given_status):
         global failed_at_least_once
         failed_at_least_once = True
 
-class PmodulePackage():
+
+def subprocess_cmd(cmd):
+    return subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        check=True,
+        shell=True,
+    )
+
+
+class PmodulePackage:
     def __init__(self, name, status, deps=[]):
         self.name = name
         self.given_status = status
         self.true_status = [status]
 
-        if len(deps) != 0 and self.given_status != 'deprecated':
+        if len(deps) != 0 and self.given_status != "deprecated":
             compiler = deps[0]
-            if len(deps) > 1 and 'mpi' in deps[1]:
+            if len(deps) > 1 and "mpi" in deps[1]:
                 mpi_provider = deps[1]
             else:
-                mpi_provider = ''
+                mpi_provider = ""
-            self.deps = [PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps]
+            self.deps = [
+                PmodulePackage.Pmoduliser(d, compiler, mpi_provider) for d in deps
+            ]
             self.true_status += self.check_deps_status()
 
     def check_deps_status(self):
@@ -38,27 +59,27 @@ class PmodulePackage():
             deps_status += d.true_status
         return deps_status
 
-    def check_correct_status(self, module=''):
+    def check_correct_status(self, module=""):
-        if (self.given_status != 'deprecated') and len(self.true_status) > 1:
+        if (self.given_status != "deprecated") and len(self.true_status) > 1:
-            if 'deprecated' in self.true_status:
+            if "deprecated" in self.true_status:
-                self.given_status = 'deprecated'
+                self.given_status = "deprecated"
                 failure_handler(module, self.given_status)
-            elif self.given_status == 'stable' and 'unstable' in self.true_status:
+            elif self.given_status == "stable" and "unstable" in self.true_status:
-                self.given_status = 'unstable'
+                self.given_status = "unstable"
                 failure_handler(module, self.given_status)
             self.true_status = [self.given_status]
 
     @staticmethod
     @cache
-    def Pmoduliser(pckg_name='', compiler='', mpi_provider=''):
+    def Pmoduliser(pckg_name="", compiler="", mpi_provider=""):
-        default_module_cmd = 'module search ' + pckg_name + ' -a --all-deps --no-header'
+        default_module_cmd = "module search " + pckg_name + " -a --all-deps --no-header"
 
         # try to precise the module search if compiler and/or mpi_provider are given
         module_cmd = default_module_cmd
-        if compiler != '' and pckg_name != compiler:
+        if compiler != "" and pckg_name != compiler:
-            module_cmd = module_cmd + ' --with=' + compiler
+            module_cmd = module_cmd + " --with=" + compiler
-        if mpi_provider != '' and pckg_name != mpi_provider:
+        if mpi_provider != "" and pckg_name != mpi_provider:
-            module_cmd = module_cmd + ' --with=' + mpi_provider
+            module_cmd = module_cmd + " --with=" + mpi_provider
         module_cmd_process = subprocess_cmd(module_cmd)
 
         # take the default command if the dependency wasn't compiled with the same compiler and/or mpi_provider
@@ -73,20 +94,11 @@ class PmodulePackage():
             Pmodule_pckg.check_correct_status()
             return Pmodule_pckg
         else:
-            print('Warning: "' + pckg_name + '" could not be found using "' + module_cmd + '"')
+            print(
-            return PmodulePackage('', '')
+                'Warning: "'
-
+                + pckg_name
-def main():
+                + '" could not be found using "'
-    module_cmd = 'module search -a --all-deps --no-header'
+                + module_cmd
-    module_cmd_process = subprocess_cmd(module_cmd)
+                + '"'
-    for module in module_cmd_process.stderr.splitlines():
+            )
-        if len(module) != 0:
+            return PmodulePackage("", "")
-            package = module.split()
-            Pmodule_pckg = PmodulePackage(package[0], package[1], package[3:])
-            Pmodule_pckg.check_correct_status(module)
-
-    if(failed_at_least_once):
-        sys.exit(1)
-
-if __name__=="__main__":
-    main()

pmodules_tools/pmodules_tools.py

@@ -0,0 +1,64 @@
+import argparse
+import sys
+
+from datetime import date
+from deps_status.check import deps_status_check, subprocess_cmd
+from weekly_diff.check import weekly_check_diff
+
+assert sys.version_info >= (
+    3,
+    7,
+), "Python version is too low, please load a python >= 3.7"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="A python project to analyse Pmodules modules and their status changes."
+    )
+    parser.add_argument(
+        "-d",
+        "--deps-check",
+        action="store_true",
+        help="Check for module dependencies status and change their status accordingly.",
+    )
+    parser.add_argument(
+        "-w",
+        "--weekly-check",
+        action="store_true",
+        help="Report new, deleted or changed modules and print it to pmodules_changes.md.",
+    )
+    parser.add_argument(
+        "--select-module",
+        default="",
+        help="Select a subtype of module which should be checked for. Default is all.",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    # Analyse CLI args
+    cli_args = parse_args()
+
+    # Global variables
+    Pmodules_db_path = "/afs/psi.ch/sys/spack-rhel7/test/"
+    Pmodules_states = ["deprecated", "stable", "unstable"]
+
+    # Main search to analyse
+    module_cmd = (
+        "module search -a " + cli_args.select_module + " --all-deps --no-header"
+    )
+    module_cmd_process = subprocess_cmd(module_cmd)
+
+    # Check for Pmodules addition, deletion or changes of state
+    if cli_args.weekly_check:
+        weekly_check_diff(module_cmd_process.stderr, Pmodules_db_path, Pmodules_states)
+
+    # Check for module dependencies status and change the main module status accordingly
+    if cli_args.deps_check:
+        deps_status_check(module_cmd_process)
+
+
+if __name__ == "__main__":
+    main()

pmodules_tools/weekly_diff/check.py

@@ -0,0 +1,157 @@
+import difflib
+import hashlib
+import os
+import re
+import shutil
+
+
+def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
+    if (len(old_pmodule_state)) != 0:
+        old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
+        if current_sha != old_sha:
+            return True
+        else:
+            return False
+
+
+def print_to_markdown(new_module_list, deleted_module_list, changed_module_list):
+    with open("pmodules_changes.md", "w") as md_file:
+        standard_print(md_file, new_module_list, "New modules:")
+
+        md_file.write("\n# Changed modules: \n")
+        if len(changed_module_list) > 1:
+            for index in range(0, len(changed_module_list) - 1, 2):
+                md_file.write(
+                    changed_module_list[index]
+                    + "<br>Changed state from "
+                    + changed_module_list[index + 1].split()[1]
+                    + " to ---> "
+                    + changed_module_list[index].split()[1]
+                    + "<br>"
+                )
+
+        standard_print(md_file, deleted_module_list, "Deleted modules:")
+
+
+def print_pmodules_differences(
+    current_pmodule_state, old_pmodule_state, Pmodules_states
+):
+    # Make sure the whitespaces in between the module descriptions are always one space only.
+    current_db = set(" ".join(i.split()) for i in current_pmodule_state.splitlines())
+    old_db = set(" ".join(i.split()) for i in old_pmodule_state.splitlines())
+
+    new_module_list = list(current_db - old_db)
+    deleted_module_list = list(old_db - current_db)
+    changed_module_list = []
+
+    # Replace the state of the modules with * for Regex comparison
+    new_module_list_no_state = new_module_list.copy()
+    deleted_module_list_no_state = deleted_module_list.copy()
+    for state in Pmodules_states:
+        new_module_list_no_state = [
+            diff.replace(state, ".*") for diff in new_module_list_no_state
+        ]
+        deleted_module_list_no_state = [
+            diff.replace(state, ".*") for diff in deleted_module_list_no_state
+        ]
+
+    # Check if the state of the module is the only thing that changed
+    # Append changed_module_list if yes
+    if len(deleted_module_list_no_state) >= len(new_module_list_no_state):
+        set_changed_module_list(
+            new_module_list_no_state,
+            deleted_module_list_no_state,
+            new_module_list,
+            deleted_module_list,
+            changed_module_list,
+        )
+    else:
+        set_changed_module_list(
+            deleted_module_list_no_state,
+            new_module_list_no_state,
+            new_module_list,
+            deleted_module_list,
+            changed_module_list,
+        )
+
+    print_to_markdown(new_module_list, deleted_module_list, changed_module_list)
+
+
+def set_changed_module_list(
+    small_module_list_no_state,
+    big_module_list_no_state,
+    new_module_list,
+    deleted_module_list,
+    changed_module_list,
+):
+    for diff in small_module_list_no_state:
+        if diff in big_module_list_no_state:
+            new_module_index = [
+                idx
+                for idx, string in enumerate(new_module_list)
+                if re.search(diff, string)
+            ][0]
+            deleted_module_index = [
+                idx
+                for idx, string in enumerate(deleted_module_list)
+                if re.search(diff, string)
+            ][0]
+            changed_module_list += [
+                new_module_list[new_module_index],
+                deleted_module_list[deleted_module_index],
+            ]
+            new_module_list[new_module_index] = ""
+            deleted_module_list[deleted_module_index] = ""
+
+
+def standard_print(file, module_list, string):
+    file.write("\n# " + string + " \n")
+    for diff in module_list:
+        if diff != "":
+            file.write(diff + "<br>")
+
+
+def weekly_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
+    current_sha = hashlib.sha256(current_pmodule_state.encode()).hexdigest()
+
+    no_current_db = False
+
+    # Retrieve old state for comparison
+    try:
+        old_pmodule_file = sorted(
+            [Pmodules_db_path + f for f in os.listdir(Pmodules_db_path)],
+            key=os.path.getctime,
+        )[0]
+        old_pmodule_state = open(old_pmodule_file, "r").read()
+    except:
+        print(
+            "There is no old Pmodule database available on path "
+            + Pmodules_db_path
+            + "... Writing current one"
+        )
+        no_current_db = True
+
+    # There is a database and we have to check the differences between the pmodule states.
+    if not no_current_db and compare_states_sha(
+        current_pmodule_state, old_pmodule_state, current_sha
+    ):
+        print_pmodules_differences(
+            current_pmodule_state, old_pmodule_state, Pmodules_states
+        )
+
+    # There is no database available or there are differences with the old pmodule state, writing current state.
+    if no_current_db or compare_states_sha(
+        current_pmodule_state, old_pmodule_state, current_sha
+    ):
+        write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path)
+
+
+def write_curent_state(current_pmodule_state, current_sha, Pmodules_db_path):
+    # Emptying Pmodules database first
+    if os.path.exists(Pmodules_db_path):
+        shutil.rmtree(Pmodules_db_path)
+
+    # Recreating dir and writing new database
+    os.makedirs(Pmodules_db_path)
+    with open(Pmodules_db_path + current_sha, "w") as current_pmodule_file:
+        current_pmodule_file.write(current_pmodule_state)