Merge branch 'html_templating' into 'master'

ADD: html json database

See merge request Pmodules/Pmodules_tools!11

doc/deleted_modules.adoc

@@ -0,0 +1 @@
+No modules have been removed!

doc/index.adoc

@@ -25,4 +25,4 @@ include::changed_modules.adoc[]
 
 === Removed Modules
 
-include::removed_modules.adoc[]
+include::deleted_modules.adoc[]

pmodules_tools/db_diff/check.py

@@ -1,15 +1,33 @@
 import difflib
 import hashlib
+import json
 import os
 import re
 import shutil
 
-from datetime import date
+from datetime import date, datetime, timedelta
 from pytablewriter import AsciiDocTableWriter
 
 
-def asciidoc_dump(list, name, steps):
+def asciidoc_json_dump(list, name, steps, Pmodules_db_path):
     matrix = []
+    matrix_db = []
+
+    # Check the existing database
+    db_file = os.path.join(Pmodules_db_path, name + "_modules.json")
+    if os.path.exists(db_file):
+        with open(db_file, "r") as f:
+            matrix_db = json.load(f)
+        temp_matrix_db = matrix_db.copy()
+        for elem in matrix_db:
+            if (
+                datetime.strptime(elem[-1], "%Y-%m-%d") + timedelta(days=30)
+                < datetime.now()
+            ):
+                temp_matrix_db.remove(elem)
+        matrix_db = temp_matrix_db
+
+    # Update the database for new changes
     if len(list) > 1:
         list_range = len(list) - steps + 1
         for index in range(0, list_range, steps):
@@ -26,12 +44,28 @@ def asciidoc_dump(list, name, steps):
                 str(date.today()),
             ]
             matrix.append(matrix_row)
-        writer = AsciiDocTableWriter(
-            table_name=name.capitalize() + " modules",
-            headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"],
-            value_matrix=matrix,
-        )
-        writer.dump(os.path.join("doc", name + "_modules.adoc"))
+
+    # Update database file
+    matrix += matrix_db
+    with open(db_file, "w") as f:
+        json.dump(matrix, f)
+
+    # Dump asciidoc file
+    writer = AsciiDocTableWriter(
+        table_name=name.capitalize() + " modules",
+        headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"],
+        value_matrix=matrix,
+    )
+    writer.dump(os.path.join("doc", name + "_modules.adoc"))
+
+
+def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
+    if (len(old_pmodule_state)) != 0:
+        old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
+        if current_sha != old_sha:
+            return True
+        else:
+            return False
 
 
 def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
@@ -41,11 +75,12 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
 
     # Retrieve old state for comparison
     try:
-        old_pmodule_file = sorted(
-            [Pmodules_db_path + f for f in os.listdir(Pmodules_db_path)],
-            key=os.path.getctime,
-        )[0]
-        old_pmodule_state = open(old_pmodule_file, "r").read()
+        old_pmodule_file = [
+            f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")
+        ][0]
+        old_pmodule_state = open(
+            os.path.join(Pmodules_db_path, old_pmodule_file), "r"
+        ).read()
     except:
         print(
             "There is no old Pmodule database available on path "
@@ -59,7 +94,7 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
         current_pmodule_state, old_pmodule_state, current_sha
     ):
         print_pmodules_differences(
-            current_pmodule_state, old_pmodule_state, Pmodules_states
+            current_pmodule_state, old_pmodule_state, Pmodules_states, Pmodules_db_path
         )
 
     # There is no database available or there are differences with the old pmodule state, writing current state.
@@ -69,23 +104,16 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
         write_current_state(current_pmodule_state, current_sha, Pmodules_db_path)
 
 
-def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
-    if (len(old_pmodule_state)) != 0:
-        old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
-        if current_sha != old_sha:
-            return True
-        else:
-            return False
-
-
-def print_to_asciidoc(new_module_list, deleted_module_list, changed_module_list):
-    asciidoc_dump(new_module_list, "new", 1)
-    asciidoc_dump(deleted_module_list, "deleted", 1)
-    asciidoc_dump(changed_module_list, "changed", 2)
+def print_to_asciidoc(
+    new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path
+):
+    asciidoc_json_dump(new_module_list, "new", 1, Pmodules_db_path)
+    asciidoc_json_dump(deleted_module_list, "deleted", 1, Pmodules_db_path)
+    asciidoc_json_dump(changed_module_list, "changed", 2, Pmodules_db_path)
 
 
 def print_pmodules_differences(
-    current_pmodule_state, old_pmodule_state, Pmodules_states
+    current_pmodule_state, old_pmodule_state, Pmodules_states, Pmodules_db_path
 ):
     # Make sure the whitespaces in between the module descriptions are always one space only.
     current_db = set(" ".join(i.split()) for i in current_pmodule_state.splitlines())
@@ -125,7 +153,9 @@ def print_pmodules_differences(
             changed_module_list,
         )
 
-    print_to_asciidoc(new_module_list, deleted_module_list, changed_module_list)
+    print_to_asciidoc(
+        new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path
+    )
 
 
 def set_changed_module_list(
@@ -155,19 +185,12 @@ def set_changed_module_list(
             deleted_module_list[deleted_module_index] = ""
 
 
-def standard_print(file, module_list, string):
-    file.write("\n# " + string + " \n")
-    for diff in module_list:
-        if diff != "":
-            file.write(diff + "<br/>")
-
-
 def write_current_state(current_pmodule_state, current_sha, Pmodules_db_path):
     # Emptying Pmodules database first
-    if os.path.exists(Pmodules_db_path):
-        shutil.rmtree(Pmodules_db_path)
+    filelist = [f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")]
+    for f in filelist:
+        os.remove(os.path.join(Pmodules_db_path, f))
 
-    # Recreating dir and writing new database
-    os.makedirs(Pmodules_db_path)
-    with open(Pmodules_db_path + current_sha, "w") as current_pmodule_file:
+    # Writing new database
+    with open(os.path.join(Pmodules_db_path, current_sha), "w") as current_pmodule_file:
         current_pmodule_file.write(current_pmodule_state)

pmodules_tools/pmodules_tools.py

@@ -1,7 +1,6 @@
 import argparse
 import sys
 
-from datetime import date
 from db_diff.check import db_check_diff
 from deps_status.check import deps_status_check, subprocess_cmd