diff --git a/doc/deleted_modules.adoc b/doc/deleted_modules.adoc new file mode 100644 index 00000000..d52d51d8 --- /dev/null +++ b/doc/deleted_modules.adoc @@ -0,0 +1 @@ +No modules have been removed! diff --git a/doc/index.adoc b/doc/index.adoc index bb7eaae3..fb3681d9 100644 --- a/doc/index.adoc +++ b/doc/index.adoc @@ -25,4 +25,4 @@ include::changed_modules.adoc[] === Removed Modules -include::removed_modules.adoc[] +include::deleted_modules.adoc[] diff --git a/pmodules_tools/db_diff/check.py b/pmodules_tools/db_diff/check.py index e56705bf..385ad978 100644 --- a/pmodules_tools/db_diff/check.py +++ b/pmodules_tools/db_diff/check.py @@ -1,15 +1,33 @@ import difflib import hashlib +import json import os import re import shutil -from datetime import date +from datetime import date, datetime, timedelta from pytablewriter import AsciiDocTableWriter -def asciidoc_dump(list, name, steps): +def asciidoc_json_dump(list, name, steps, Pmodules_db_path): matrix = [] + matrix_db = [] + + # Check the existing database + db_file = os.path.join(Pmodules_db_path, name + "_modules.json") + if os.path.exists(db_file): + with open(db_file, "r") as f: + matrix_db = json.load(f) + temp_matrix_db = matrix_db.copy() + for elem in matrix_db: + if ( + datetime.strptime(elem[-1], "%Y-%m-%d") + timedelta(days=30) + < datetime.now() + ): + temp_matrix_db.remove(elem) + matrix_db = temp_matrix_db + + # Update the database for new changes if len(list) > 1: list_range = len(list) - steps + 1 for index in range(0, list_range, steps): @@ -26,12 +44,28 @@ def asciidoc_dump(list, name, steps): str(date.today()), ] matrix.append(matrix_row) - writer = AsciiDocTableWriter( - table_name=name.capitalize() + " modules", - headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"], - value_matrix=matrix, - ) - writer.dump(os.path.join("doc", name + "_modules.adoc")) + + # Update database file + matrix += matrix_db + with open(db_file, "w") as f: + json.dump(matrix, f) + + # Dump asciidoc file + writer = AsciiDocTableWriter( + table_name=name.capitalize() + " modules", + headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"], + value_matrix=matrix, + ) + writer.dump(os.path.join("doc", name + "_modules.adoc")) + + +def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha): + if (len(old_pmodule_state)) != 0: + old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest() + if current_sha != old_sha: + return True + else: + return False def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states): @@ -41,11 +75,12 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states): # Retrieve old state for comparison try: - old_pmodule_file = sorted( - [Pmodules_db_path + f for f in os.listdir(Pmodules_db_path)], - key=os.path.getctime, - )[0] - old_pmodule_state = open(old_pmodule_file, "r").read() + old_pmodule_file = [ + f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json") + ][0] + old_pmodule_state = open( + os.path.join(Pmodules_db_path, old_pmodule_file), "r" + ).read() except: print( "There is no old Pmodule database available on path " @@ -59,7 +94,7 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states): current_pmodule_state, old_pmodule_state, current_sha ): print_pmodules_differences( - current_pmodule_state, old_pmodule_state, Pmodules_states + current_pmodule_state, old_pmodule_state, Pmodules_states, Pmodules_db_path ) # There is no database available or there are differences with the old pmodule state, writing current state. @@ -69,23 +104,16 @@ def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states): write_current_state(current_pmodule_state, current_sha, Pmodules_db_path) -def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha): - if (len(old_pmodule_state)) != 0: - old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest() - if current_sha != old_sha: - return True - else: - return False - - -def print_to_asciidoc(new_module_list, deleted_module_list, changed_module_list): - asciidoc_dump(new_module_list, "new", 1) - asciidoc_dump(deleted_module_list, "deleted", 1) - asciidoc_dump(changed_module_list, "changed", 2) +def print_to_asciidoc( + new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path +): + asciidoc_json_dump(new_module_list, "new", 1, Pmodules_db_path) + asciidoc_json_dump(deleted_module_list, "deleted", 1, Pmodules_db_path) + asciidoc_json_dump(changed_module_list, "changed", 2, Pmodules_db_path) def print_pmodules_differences( - current_pmodule_state, old_pmodule_state, Pmodules_states + current_pmodule_state, old_pmodule_state, Pmodules_states, Pmodules_db_path ): # Make sure the whitespaces in between the module descriptions are always one space only. current_db = set(" ".join(i.split()) for i in current_pmodule_state.splitlines()) @@ -125,7 +153,9 @@ def print_pmodules_differences( changed_module_list, ) - print_to_asciidoc(new_module_list, deleted_module_list, changed_module_list) + print_to_asciidoc( + new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path + ) def set_changed_module_list( @@ -155,19 +185,12 @@ def set_changed_module_list( deleted_module_list[deleted_module_index] = "" -def standard_print(file, module_list, string): - file.write("\n# " + string + " \n") - for diff in module_list: - if diff != "": - file.write(diff + "
") - - def write_current_state(current_pmodule_state, current_sha, Pmodules_db_path): # Emptying Pmodules database first - if os.path.exists(Pmodules_db_path): - shutil.rmtree(Pmodules_db_path) + filelist = [f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")] + for f in filelist: + os.remove(os.path.join(Pmodules_db_path, f)) - # Recreating dir and writing new database - os.makedirs(Pmodules_db_path) - with open(Pmodules_db_path + current_sha, "w") as current_pmodule_file: + # Writing new database + with open(os.path.join(Pmodules_db_path, current_sha), "w") as current_pmodule_file: current_pmodule_file.write(current_pmodule_state) diff --git a/pmodules_tools/pmodules_tools.py b/pmodules_tools/pmodules_tools.py index ef31cba4..1990204f 100644 --- a/pmodules_tools/pmodules_tools.py +++ b/pmodules_tools/pmodules_tools.py @@ -1,7 +1,6 @@ import argparse import sys -from datetime import date from db_diff.check import db_check_diff from deps_status.check import deps_status_check, subprocess_cmd