5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Included new delete attribute and restart review features.

Browse Source
This commit is contained in:
florez_j
2024-04-26 13:08:27 +02:00
parent 14ae29bf3c
commit 4d91e59279
1 changed files with 36 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/metadata_review_lib.py
View File

@ -54,9 +54,10 @@ def checkout_review_branch(branch_name):
current_branch_command = ['git','branch','--show-current'] current_branch_command = ['git','branch','--show-current']
status_command = ['git','status'] status_command = ['git','status']
add_command = lambda add_list: ['git','add'] + add_list add_command = lambda add_list: ['git','add'] + add_list
rm_command = lambda add_list: ['git','add'] + add_list
commit_command = lambda message: ['git','commit','-m', message] commit_command = lambda message: ['git','commit','-m', message]
def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs, restart = False):
""" """
First: Initialize review branch with review folder with a copy of yaml representation of First: Initialize review branch with review folder with a copy of yaml representation of
@ -92,14 +93,19 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
# Check if review file already exists and then check if it is still untracked # Check if review file already exists and then check if it is still untracked
review_yaml_file_path = os.path.join("review/",filename+YAML_EXT) review_yaml_file_path = os.path.join("review/",filename+YAML_EXT)
if not os.path.exists(review_yaml_file_path):
review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review')
review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path) review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path)
review_status_yaml_file_path = os.path.join(review_yaml_file_path_tail+"-review_status"+".txt")
if not os.path.exists(review_yaml_file_path) or restart:
review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review')
if restart:
print('metadata review has been reinitialized. The review files will reflect the current state of the hdf5 files metadata')
#if not os.path.exists(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt")): #if not os.path.exists(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt")):
with open(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt"),'w') as f:
with open(review_status_yaml_file_path,'w') as f:
f.write('under review') f.write('under review')
# Stage untracked review files and commit them to local repository # Stage untracked review files and commit them to local repository
@ -108,7 +114,13 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
for line in status.stdout.splitlines(): for line in status.stdout.splitlines():
#tmp = line.decode("utf-8") #tmp = line.decode("utf-8")
#modified_files.append(tmp.split()[1]) #modified_files.append(tmp.split()[1])
if 'review/' in line and not 'modified' in line: # untracked files if 'review/' in line:
if not 'modified' in line: # untracked filesand
untracked_files.append(line.strip())
else:
untracked_files.append(line.strip().split()[1])
if 'output_files/'+filename+YAML_EXT in line and not 'modified' in line:
untracked_files.append(line.strip()) untracked_files.append(line.strip())
if untracked_files: if untracked_files:
@ -118,8 +130,8 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
for line in commit_output.stdout.splitlines(): for line in commit_output.stdout.splitlines():
print(line.decode('utf-8')) print(line.decode('utf-8'))
else: #else:
print('This action will not have any effect because metadata review process has been already initialized.') # print('This action will not have any effect because metadata review process has been already initialized.')
@ -139,7 +151,7 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
#print("Add and commit"+"\n") #print("Add and commit"+"\n")
return review_yaml_file_path return review_yaml_file_path, review_status_yaml_file_path
@ -204,14 +216,16 @@ def parse_attribute(attr_value):
dtype.append((key,f'S{max_length}')) dtype.append((key,f'S{max_length}'))
values_list.append(attr_value[key]) values_list.append(attr_value[key])
if len(values_list)>2: if len(values_list)>1:
new_attr_value = np.array([tuple(values_list)],dtype=dtype) new_attr_value = np.array([tuple(values_list)],dtype=dtype)
elif values_list:
new_attr_value = values_list[0]
else: else:
new_attr_value = attr_value['value'] new_attr_value = 'missing'
return new_attr_value return new_attr_value
def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, reviewer_attrs = {}, hdf5_upload : bool = False): def third_update_hdf5_file_with_review(input_hdf5_file, yaml_review_file, reviewer_attrs = {}, hdf5_upload : bool = False):
"""Third""" """Third"""
# compare review file with current yalm file and then based on the changes open hdf5 file and access only # compare review file with current yalm file and then based on the changes open hdf5 file and access only
@ -237,7 +251,7 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review
print('additions',count_additions, 'deletions', count_delections) print('additions',count_additions, 'deletions', count_delections)
with open(yalm_review_file,'r') as stream: with open(yaml_review_file,'r') as stream:
try: try:
yaml_dict = yaml.load(stream, Loader=yaml.FullLoader) yaml_dict = yaml.load(stream, Loader=yaml.FullLoader)
except yaml.YAMLError as exc: except yaml.YAMLError as exc:
@ -277,9 +291,11 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review
# yaml indicates a renaming of the attribute. # yaml indicates a renaming of the attribute.
if not (new_attr_name == attr_name): if not (new_attr_name == attr_name):
hdf5_obj.attrs.__delitem__(attr_name) hdf5_obj.attrs.__delitem__(attr_name)
#else: #else:
# hdf5_obj.attrs[attr_name] = attr_value # hdf5_obj.attrs[attr_name] = attr_value
else: # attribute inclusion elif not attr_value.get('delete',False): # if true inclusion, else don't take any action
#hdf5_obj.attrs.__delitem__(attr_name): # attribute inclusion
#if isinstance(attr_value,dict): #if isinstance(attr_value,dict):
# Retreive new attribute's name and value # Retreive new attribute's name and value
# new_attr_name = attr_value.get('rename_as',attr_name) # if 'rename_as' is a key in attr_value returns the value, otherwise it return the existing value # new_attr_name = attr_value.get('rename_as',attr_name) # if 'rename_as' is a key in attr_value returns the value, otherwise it return the existing value
@ -287,9 +303,11 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review
hdf5_obj.attrs[attr_name] = parse_attribute(attr_value) hdf5_obj.attrs[attr_name] = parse_attribute(attr_value)
#else: #else:
# hdf5_obj.attrs[attr_name] = attr_value # hdf5_obj.attrs[attr_name] = attr_value
print(input_hdf5_file + ' was successfully updated\n')
# Recreate/or update yaml representation of updated input_hdf5_file. # Recreate/or update yaml representation of updated input_hdf5_file.
output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(input_hdf5_file) output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(input_hdf5_file)
print(output_yml_filename_path + ' was successfully regenerated from the updated version of ', input_hdf5_file)
status_command = ['git','status'] status_command = ['git','status']
@ -317,17 +335,10 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review
#status = subprocess.run(status_command,capture_output=True,check=True) #status = subprocess.run(status_command,capture_output=True,check=True)
message = 'Updated hdf5 file with yaml review file.' message = 'Updated hdf5 file with yaml review file.'
commit_output = subprocess.run(commit_command(message),capture_output=True,check=True) commit_output = subprocess.run(commit_command(message),capture_output=True,check=True)
print(commit_output.stdout)
else: else:
print("There were no found h5 and yaml files, needing to be saved. This action will not have effect on the review process' commit history.") print("There were no found h5 and yaml files, needing to be saved. This action will not have effect on the review process' commit history.")
#with open('review/review_status.txt','r+') as f:
# f.write('hdf5 file updated w/ metadata review')
#try:
# yaml_dict = yaml.load(stream, Loader=yaml.FullLoader)
#except yaml.YAMLError as exc:
# print(exc)
def last_submit_metadata_review(reviewer_attrs): def last_submit_metadata_review(reviewer_attrs):
@ -336,7 +347,7 @@ def last_submit_metadata_review(reviewer_attrs):
initials =reviewer_attrs['initials'] initials =reviewer_attrs['initials']
repository = 'origin' repository = 'origin'
branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials]) branch_name = '_'.join(['review',initials])
push_command = lambda repository,refspec: ['git','push',repository,refspec] push_command = lambda repository,refspec: ['git','push',repository,refspec]
current_branch_command = ['git','branch','--show-current'] current_branch_command = ['git','branch','--show-current']