228 lines
7.7 KiB
Python
228 lines
7.7 KiB
Python
import difflib
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
|
|
from datetime import date, datetime, timedelta
|
|
from operator import itemgetter, attrgetter
|
|
from pytablewriter import AsciiDocTableWriter
|
|
|
|
|
|
def asciidoc_json_dump(list, name, steps, Pmodules_db_path):
|
|
matrix = []
|
|
matrix_db = []
|
|
temp_matrix_db = []
|
|
|
|
# Check the existing database
|
|
db_file = os.path.join(Pmodules_db_path, name + "_modules.json")
|
|
if os.path.exists(db_file):
|
|
# No need to append the old database if we check the whole database
|
|
if name != "all":
|
|
with open(db_file, "r") as f:
|
|
matrix_db = json.load(f)
|
|
temp_matrix_db = matrix_db.copy()
|
|
for elem in matrix_db:
|
|
if (
|
|
datetime.strptime(elem[-1], "%Y-%m-%d") + timedelta(days=30)
|
|
< datetime.now()
|
|
):
|
|
temp_matrix_db.remove(elem)
|
|
matrix_db = temp_matrix_db
|
|
|
|
# Update the database for new changes
|
|
if len(list) != 0:
|
|
list_range = len(list) - steps + 1
|
|
for index in range(0, list_range, steps):
|
|
if len(list[index]) != 0:
|
|
matrix_row = [list[index].split()[0]]
|
|
if steps == 2:
|
|
matrix_row += [
|
|
list[index + 1].split()[1] + "->" + list[index].split()[1]
|
|
]
|
|
else:
|
|
matrix_row += [list[index].split()[1]]
|
|
matrix_row += [
|
|
list[index].split()[2],
|
|
" ".join(list[index].split()[3:]),
|
|
str(date.today()),
|
|
]
|
|
matrix.append(matrix_row)
|
|
|
|
# Update database file
|
|
matrix += matrix_db
|
|
with open(db_file, "w") as f:
|
|
json.dump(matrix, f)
|
|
|
|
# Dump asciidoc file
|
|
writer = AsciiDocTableWriter(
|
|
table_name=name.capitalize() + " modules",
|
|
headers=["Name/Version", "Release Stage", "Group", "Deps", "Change Date"],
|
|
value_matrix=matrix,
|
|
)
|
|
writer.dump(os.path.join("doc", name + "_modules.adoc"))
|
|
|
|
|
|
def compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
|
|
if (len(old_pmodule_state)) != 0:
|
|
old_sha = hashlib.sha256(old_pmodule_state.encode()).hexdigest()
|
|
if current_sha != old_sha:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def db_check_diff(current_pmodule_state, Pmodules_db_path, Pmodules_states):
|
|
current_sha = hashlib.sha256(current_pmodule_state.encode()).hexdigest()
|
|
|
|
no_current_db = False
|
|
|
|
# Retrieve old state for comparison
|
|
try:
|
|
old_pmodule_file = [
|
|
f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")
|
|
][0]
|
|
old_pmodule_state = open(
|
|
os.path.join(Pmodules_db_path, old_pmodule_file), "r"
|
|
).read()
|
|
except:
|
|
print(
|
|
"There is no old Pmodule database available on path "
|
|
+ Pmodules_db_path
|
|
+ "... Writing current one"
|
|
)
|
|
no_current_db = True
|
|
|
|
# Make sure the whitespaces in between the module descriptions are always one space only.
|
|
# There is a database
|
|
if not no_current_db:
|
|
current_db = set(
|
|
" ".join(i.split()) for i in current_pmodule_state.splitlines()
|
|
)
|
|
old_db = set(" ".join(i.split()) for i in old_pmodule_state.splitlines())
|
|
all_module_list = list(current_db)
|
|
all_module_list_sorted = sorted([x for x in all_module_list if 'stable' in x])
|
|
all_module_list_sorted += sorted([x for x in all_module_list if 'unstable' in x])
|
|
all_module_list_sorted += sorted([x for x in all_module_list if 'deprecated' in x])
|
|
all_module_list = all_module_list_sorted
|
|
|
|
# We have to check the differences between the pmodule states.
|
|
if compare_states_sha(current_pmodule_state, old_pmodule_state, current_sha):
|
|
new_module_list = list(current_db - old_db)
|
|
deleted_module_list = list(old_db - current_db)
|
|
print_pmodules_differences(
|
|
all_module_list,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
Pmodules_states,
|
|
Pmodules_db_path,
|
|
)
|
|
else:
|
|
print_to_asciidoc(all_module_list, [], [], [], Pmodules_db_path)
|
|
|
|
# There is no database available or there are differences with the old pmodule state, writing current state.
|
|
if no_current_db or compare_states_sha(
|
|
current_pmodule_state, old_pmodule_state, current_sha
|
|
):
|
|
write_current_state(current_pmodule_state, current_sha, Pmodules_db_path)
|
|
|
|
|
|
def print_to_asciidoc(
|
|
all_module_list,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
changed_module_list,
|
|
Pmodules_db_path,
|
|
):
|
|
asciidoc_json_dump(all_module_list, "all", 1, Pmodules_db_path)
|
|
asciidoc_json_dump(new_module_list, "new", 1, Pmodules_db_path)
|
|
asciidoc_json_dump(deleted_module_list, "deleted", 1, Pmodules_db_path)
|
|
asciidoc_json_dump(changed_module_list, "changed", 2, Pmodules_db_path)
|
|
|
|
|
|
def print_pmodules_differences(
|
|
all_module_list,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
Pmodules_states,
|
|
Pmodules_db_path,
|
|
):
|
|
changed_module_list = []
|
|
|
|
# Replace the state of the modules with * for Regex comparison
|
|
new_module_list_no_state = new_module_list.copy()
|
|
deleted_module_list_no_state = deleted_module_list.copy()
|
|
for state in Pmodules_states:
|
|
new_module_list_no_state = [
|
|
diff.replace(state, ".*") for diff in new_module_list_no_state
|
|
]
|
|
deleted_module_list_no_state = [
|
|
diff.replace(state, ".*") for diff in deleted_module_list_no_state
|
|
]
|
|
|
|
# Check if the state of the module is the only thing that changed
|
|
# Append changed_module_list if yes
|
|
if len(deleted_module_list_no_state) >= len(new_module_list_no_state):
|
|
set_changed_module_list(
|
|
new_module_list_no_state,
|
|
deleted_module_list_no_state,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
changed_module_list,
|
|
)
|
|
else:
|
|
set_changed_module_list(
|
|
deleted_module_list_no_state,
|
|
new_module_list_no_state,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
changed_module_list,
|
|
)
|
|
|
|
print_to_asciidoc(
|
|
all_module_list,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
changed_module_list,
|
|
Pmodules_db_path,
|
|
)
|
|
|
|
|
|
def set_changed_module_list(
|
|
small_module_list_no_state,
|
|
big_module_list_no_state,
|
|
new_module_list,
|
|
deleted_module_list,
|
|
changed_module_list,
|
|
):
|
|
for diff in small_module_list_no_state:
|
|
if diff in big_module_list_no_state:
|
|
new_module_index = [
|
|
idx
|
|
for idx, string in enumerate(new_module_list)
|
|
if re.search(diff, string)
|
|
][0]
|
|
deleted_module_index = [
|
|
idx
|
|
for idx, string in enumerate(deleted_module_list)
|
|
if re.search(diff, string)
|
|
][0]
|
|
changed_module_list += [
|
|
new_module_list[new_module_index],
|
|
deleted_module_list[deleted_module_index],
|
|
]
|
|
new_module_list[new_module_index] = ""
|
|
deleted_module_list[deleted_module_index] = ""
|
|
|
|
|
|
def write_current_state(current_pmodule_state, current_sha, Pmodules_db_path):
|
|
# Emptying Pmodules database first
|
|
filelist = [f for f in os.listdir(Pmodules_db_path) if not f.endswith(".json")]
|
|
for f in filelist:
|
|
os.remove(os.path.join(Pmodules_db_path, f))
|
|
|
|
# Writing new database
|
|
with open(os.path.join(Pmodules_db_path, current_sha), "w") as current_pmodule_file:
|
|
current_pmodule_file.write(current_pmodule_state)
|