import difflib import hashlib import json import os import re import shutil from datetime import date, datetime, timedelta from operator import itemgetter, attrgetter from pytablewriter import AsciiDocTableWriter def asciidoc_json_dump(list, name, steps, Pmodules_db_path): matrix = [] matrix_db = [] temp_matrix_db = [] # Check the existing database db_file = os.path.join(Pmodules_db_path, name + "_modules.json") if os.path.exists(db_file): # No need to append the old database if we check the whole database if name != "all": with open(db_file, "r") as f: matrix_db = json.load(f) temp_matrix_db = matrix_db.copy() for elem in matrix_db: if ( datetime.strptime(elem[-1], "%Y-%m-%d") + timedelta(days=30) < datetime.now() ): temp_matrix_db.remove(elem) matrix_db = temp_matrix_db # Update the database for new changes if len(list) != 0: list_range = len(list) - steps + 1 for index in range(0, list_range, steps): if len(list[index]) != 0: matrix_row = [list[index].split()[0]] if steps == 2: matrix_row += [ list[index + 1].split()[1] + "->" + list[index].split()[1] ] else: matrix_row += [list[index].split()[1]] matrix_row += [ list[index].split()[2], " ".join(list[index].split()[3:]), str(date.today()), ] matrix.append(matrix_row) # Update database file matrix += matrix_db with open(db_file, "w") as f: json.dump(matrix, f) # Dump asciidoc file writer = AsciiDocTableWriter( table_name=name.capitalize() + " modules", headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"], value_matrix=matrix, ) writer.dump(os.path.join("doc", name + "_modules.adoc")) def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha): if (len(old_pmodule_state)) != 0: old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest() if current_sha != old_sha: return True else: return False def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states): current_sha = hashlib.sha256(current_pmodule_state.encode()).hexdigest() no_current_db = False # Retrieve old state for comparison try: old_pmodule_file = [ f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json") ][0] old_pmodule_state = open( os.path.join(Pmodules_db_path, old_pmodule_file), "r" ).read() except: print( "There is no old Pmodule database available on path " + Pmodules_db_path + "... Writing current one" ) no_current_db = True # Make sure the whitespaces in between the module descriptions are always one space only. # There is a database if not no_current_db: current_db = set( " ".join(i.split()) for i in current_pmodule_state.splitlines() ) old_db = set(" ".join(i.split()) for i in old_pmodule_state.splitlines()) all_module_list = list(current_db) all_module_list_sorted = sorted([x for x in all_module_list if " stable" in x]) all_module_list_sorted += sorted( [x for x in all_module_list if "unstable" in x] ) all_module_list_sorted += sorted( [x for x in all_module_list if "deprecated" in x] ) all_module_list = all_module_list_sorted # We have to check the differences between the pmodule states. if compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha): new_module_list = list(current_db - old_db) deleted_module_list = list(old_db - current_db) # An element cannot be deleted and new at the same time new_module_list = [ elem for elem in new_module_list if elem not in deleted_module_list ] deleted_module_list = [ elem for elem in deleted_module_list if elem not in new_module_list ] print_pmodules_differences( all_module_list, new_module_list, deleted_module_list, Pmodules_states, Pmodules_db_path, ) else: print_to_asciidoc(all_module_list, [], [], [], Pmodules_db_path) # There is no database available or there are differences with the old pmodule state, writing current state. if no_current_db or compare_states_sha( current_pmodule_state, old_pmodule_state, current_sha ): write_current_state(current_pmodule_state, current_sha, Pmodules_db_path) def print_to_asciidoc( all_module_list, new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path, ): asciidoc_json_dump(all_module_list, "all", 1, Pmodules_db_path) asciidoc_json_dump(new_module_list, "new", 1, Pmodules_db_path) asciidoc_json_dump(deleted_module_list, "deleted", 1, Pmodules_db_path) asciidoc_json_dump(changed_module_list, "changed", 2, Pmodules_db_path) def print_pmodules_differences( all_module_list, new_module_list, deleted_module_list, Pmodules_states, Pmodules_db_path, ): changed_module_list = [] # Replace the state of the modules with * for Regex comparison new_module_list_no_state = [] deleted_module_list_no_state = [] for state in Pmodules_states: new_module_list_no_state = [ diff.replace(state, " .*") for diff in new_module_list ] deleted_module_list_no_state = [ diff.replace(state, " .*") for diff in deleted_module_list ] # Check if the state of the module is the only thing that changed # Append changed_module_list if yes if len(deleted_module_list_no_state) >= len(new_module_list_no_state): set_changed_module_list( new_module_list_no_state, deleted_module_list_no_state, new_module_list, deleted_module_list, changed_module_list, ) else: set_changed_module_list( deleted_module_list_no_state, new_module_list_no_state, new_module_list, deleted_module_list, changed_module_list, ) print_to_asciidoc( all_module_list, new_module_list, deleted_module_list, changed_module_list, Pmodules_db_path, ) def set_changed_module_list( small_module_list_no_state, big_module_list_no_state, new_module_list, deleted_module_list, changed_module_list, ): for diff in small_module_list_no_state: if diff in big_module_list_no_state: new_module_index = [ idx for idx, string in enumerate(new_module_list) if re.search(diff, string) ][0] deleted_module_index = [ idx for idx, string in enumerate(deleted_module_list) if re.search(diff, string) ][0] changed_module_list += [ new_module_list[new_module_index], deleted_module_list[deleted_module_index], ] new_module_list[new_module_index] = "" deleted_module_list[deleted_module_index] = "" def write_current_state(current_pmodule_state, current_sha, Pmodules_db_path): # Emptying Pmodules database first filelist = [f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")] for f in filelist: os.remove(os.path.join(Pmodules_db_path, f)) # Writing new database with open(os.path.join(Pmodules_db_path, current_sha), "w") as current_pmodule_file: current_pmodule_file.write(current_pmodule_state)