diff --git a/src/metadata_review_lib.py b/src/metadata_review_lib.py index 4be7c0b..d959cb9 100644 --- a/src/metadata_review_lib.py +++ b/src/metadata_review_lib.py @@ -54,9 +54,10 @@ def checkout_review_branch(branch_name): current_branch_command = ['git','branch','--show-current'] status_command = ['git','status'] add_command = lambda add_list: ['git','add'] + add_list +rm_command = lambda add_list: ['git','add'] + add_list commit_command = lambda message: ['git','commit','-m', message] -def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): +def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs, restart = False): """ First: Initialize review branch with review folder with a copy of yaml representation of @@ -92,14 +93,19 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): # Check if review file already exists and then check if it is still untracked review_yaml_file_path = os.path.join("review/",filename+YAML_EXT) + review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path) + review_status_yaml_file_path = os.path.join(review_yaml_file_path_tail+"-review_status"+".txt") - if not os.path.exists(review_yaml_file_path): + if not os.path.exists(review_yaml_file_path) or restart: review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review') + if restart: + print('metadata review has been reinitialized. The review files will reflect the current state of the hdf5 files metadata') - review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path) + #if not os.path.exists(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt")): - with open(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt"),'w') as f: + + with open(review_status_yaml_file_path,'w') as f: f.write('under review') # Stage untracked review files and commit them to local repository @@ -108,7 +114,13 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): for line in status.stdout.splitlines(): #tmp = line.decode("utf-8") #modified_files.append(tmp.split()[1]) - if 'review/' in line and not 'modified' in line: # untracked files + if 'review/' in line: + if not 'modified' in line: # untracked filesand + untracked_files.append(line.strip()) + else: + untracked_files.append(line.strip().split()[1]) + + if 'output_files/'+filename+YAML_EXT in line and not 'modified' in line: untracked_files.append(line.strip()) if untracked_files: @@ -118,8 +130,8 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): for line in commit_output.stdout.splitlines(): print(line.decode('utf-8')) - else: - print('This action will not have any effect because metadata review process has been already initialized.') + #else: + # print('This action will not have any effect because metadata review process has been already initialized.') @@ -139,7 +151,7 @@ def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs): #print("Add and commit"+"\n") - return review_yaml_file_path + return review_yaml_file_path, review_status_yaml_file_path @@ -204,14 +216,16 @@ def parse_attribute(attr_value): dtype.append((key,f'S{max_length}')) values_list.append(attr_value[key]) - if len(values_list)>2: + if len(values_list)>1: new_attr_value = np.array([tuple(values_list)],dtype=dtype) - else: - new_attr_value = attr_value['value'] + elif values_list: + new_attr_value = values_list[0] + else: + new_attr_value = 'missing' return new_attr_value -def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, reviewer_attrs = {}, hdf5_upload : bool = False): +def third_update_hdf5_file_with_review(input_hdf5_file, yaml_review_file, reviewer_attrs = {}, hdf5_upload : bool = False): """Third""" # compare review file with current yalm file and then based on the changes open hdf5 file and access only @@ -237,7 +251,7 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review print('additions',count_additions, 'deletions', count_delections) - with open(yalm_review_file,'r') as stream: + with open(yaml_review_file,'r') as stream: try: yaml_dict = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: @@ -277,9 +291,11 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review # yaml indicates a renaming of the attribute. if not (new_attr_name == attr_name): hdf5_obj.attrs.__delitem__(attr_name) + #else: # hdf5_obj.attrs[attr_name] = attr_value - else: # attribute inclusion + elif not attr_value.get('delete',False): # if true inclusion, else don't take any action + #hdf5_obj.attrs.__delitem__(attr_name): # attribute inclusion #if isinstance(attr_value,dict): # Retreive new attribute's name and value # new_attr_name = attr_value.get('rename_as',attr_name) # if 'rename_as' is a key in attr_value returns the value, otherwise it return the existing value @@ -287,9 +303,11 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review hdf5_obj.attrs[attr_name] = parse_attribute(attr_value) #else: # hdf5_obj.attrs[attr_name] = attr_value + print(input_hdf5_file + ' was successfully updated\n') - # Recreate/or update yaml representation of updated input_hdf5_file. - output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(input_hdf5_file) + # Recreate/or update yaml representation of updated input_hdf5_file. + output_yml_filename_path = hdf5_vis.take_yml_snapshot_of_hdf5_file(input_hdf5_file) + print(output_yml_filename_path + ' was successfully regenerated from the updated version of ', input_hdf5_file) status_command = ['git','status'] @@ -317,17 +335,10 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, review #status = subprocess.run(status_command,capture_output=True,check=True) message = 'Updated hdf5 file with yaml review file.' commit_output = subprocess.run(commit_command(message),capture_output=True,check=True) + print(commit_output.stdout) else: print("There were no found h5 and yaml files, needing to be saved. This action will not have effect on the review process' commit history.") - #with open('review/review_status.txt','r+') as f: - # f.write('hdf5 file updated w/ metadata review') - - #try: - # yaml_dict = yaml.load(stream, Loader=yaml.FullLoader) - #except yaml.YAMLError as exc: - # print(exc) - def last_submit_metadata_review(reviewer_attrs): @@ -336,7 +347,7 @@ def last_submit_metadata_review(reviewer_attrs): initials =reviewer_attrs['initials'] repository = 'origin' - branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials]) + branch_name = '_'.join(['review',initials]) push_command = lambda repository,refspec: ['git','push',repository,refspec] current_branch_command = ['git','branch','--show-current']