5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generalized workflow functions to consider reviewer attributes such as initials and type e.g., data-owner and metadata-reviewer.

Browse Source
This commit is contained in:
florez_j
2024-03-28 16:11:01 +01:00
parent 37fd603943
commit 06429e6def
1 changed files with 35 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
src/metadata_review_lib.py
View File

@ -35,9 +35,9 @@ def get_review_status(filename_path):
workflow_steps.append(line) workflow_steps.append(line)
return workflow_steps[-1] return workflow_steps[-1]
def checkout_review_branch(repo_obj,initials): def checkout_review_branch(repo_obj,branch_name):
# Create a new branch # Create a new branch
branch_name = 'metadata-review-by-'+initials #branch_name = 'metadata-review-by-'+initials
head_commit = repo_obj.head.peel()# Get the commit hash associated with HEAD head_commit = repo_obj.head.peel()# Get the commit hash associated with HEAD
if not branch_name in repo_obj.branches: if not branch_name in repo_obj.branches:
@ -46,45 +46,45 @@ def checkout_review_branch(repo_obj,initials):
branch = repo_obj.branches[branch_name] branch = repo_obj.branches[branch_name]
repo_obj.checkout(branch) repo_obj.checkout(branch)
def first_initialize_metadata_review(filename_path,initials): def first_initialize_metadata_review(hdf5_file_path, reviewer_attrs):
""" """
Initialize metadata review by creating review folder with a copy of yaml representation of Initialize review branch with review folder with a copy of yaml representation of
hdf5 file under review and by creating a txt file with the state of the review process, e.g., under review. hdf5 file under review and by creating a txt file with the state of the review process, e.g., under review.
""" """
filename_path_tail, filename_path_head = os.path.split(filename_path) initials = reviewer_attrs['initials']
branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials])
hdf5_file_path_tail, filename_path_head = os.path.split(hdf5_file_path)
filename, ext = os.path.splitext(filename_path_head) filename, ext = os.path.splitext(filename_path_head)
# Check file_path points to h5 file # Check file_path points to h5 file
if not 'h5' in ext: if not 'h5' in ext:
raise ValueError("filename_path needs to point to a suitable h5 file.") raise ValueError("filename_path needs to point to an h5 file.")
# Verify if yaml snapshot of input h5 file exists # Verify if yaml snapshot of input h5 file exists
if not os.path.exists(os.path.join(filename_path_tail,filename+YAML_EXT)): if not os.path.exists(os.path.join(hdf5_file_path_tail,filename+YAML_EXT)):
raise ValueError("metadata review cannot be initialized. The associated .yaml file under review was not found. Run take_yml_snapshot_of_hdf5_file(filename_path) ") raise ValueError("metadata review cannot be initialized. The associated .yaml file under review was not found. Run take_yml_snapshot_of_hdf5_file(filename_path) ")
# Initialize metadata review workflow # Initialize metadata review workflow
print("Create branch metadata-review-by-"+initials+"\n") print("Create branch metadata-review-by-"+initials+"\n")
# Check if review file already exists and then check if it is still untracked # Check if review file already exists and then check if it is still untracked
if not os.path.exists(os.path.join("review/",filename+YAML_EXT)): review_yaml_file_path = os.path.join("review/",filename+YAML_EXT)
review_filename_path = utils.make_file_copy(os.path.join(filename_path_tail,filename+YAML_EXT), 'review') if not os.path.exists(review_yaml_file_path):
else: review_yaml_file_path = utils.make_file_copy(os.path.join(hdf5_file_path_tail,filename+YAML_EXT), 'review')
review_filename_path = os.path.join("review/",filename+YAML_EXT)
#else: #else:
# raise Warning("the file " + os.path.join("review/",filename+YAML_EXT)+ " already exists. Delete this file to reinitialize the metadata review process.") # raise Warning("the file " + os.path.join("review/",filename+YAML_EXT)+ " already exists. Delete this file to reinitialize the metadata review process.")
review_filename_tail, ext = os.path.splitext(review_filename_path) review_yaml_file_path_tail, ext = os.path.splitext(review_yaml_file_path)
with open(os.path.join(review_filename_tail+"-review_status"+".txt"),'w') as f: with open(os.path.join(review_yaml_file_path_tail+"-review_status"+".txt"),'w') as f:
f.write('under review') f.write('under review')
checkout_review_branch(repo_obj,initials) checkout_review_branch(repo_obj, branch_name)
status_dict = repo_obj.status() status_dict = repo_obj.status()
for filepath, file_status in status_dict.items(): for filepath, file_status in status_dict.items():
@ -101,9 +101,11 @@ def first_initialize_metadata_review(filename_path,initials):
#print("Add and commit"+"\n") #print("Add and commit"+"\n")
return review_yaml_file_path
def second_submit_metadata_review(filename_path, initials):
def second_submit_metadata_review(filename_path, reviewer_attrs):
""" """
Once you're done reviewing the yaml representation of hdf5 file in review folder. Once you're done reviewing the yaml representation of hdf5 file in review folder.
Change the review status to complete and save (add and commit) modified .yalm and .txt files in the project by Change the review status to complete and save (add and commit) modified .yalm and .txt files in the project by
@ -114,6 +116,8 @@ def second_submit_metadata_review(filename_path, initials):
# 2. change review status in txt to complete # 2. change review status in txt to complete
# 3. git add review/ and git commit -m "Submitted metadata review" # 3. git add review/ and git commit -m "Submitted metadata review"
initials = reviewer_attrs['initials']
branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials])
if any([status in get_review_status(filename_path) for status in ['under review','submitted']]): if any([status in get_review_status(filename_path) for status in ['under review','submitted']]):
filename_path_tail, filename_path_head = os.path.split(filename_path) filename_path_tail, filename_path_head = os.path.split(filename_path)
@ -122,16 +126,8 @@ def second_submit_metadata_review(filename_path, initials):
with open(os.path.join("review/",filename+"-review_status"+TXT_EXT),'a') as f: with open(os.path.join("review/",filename+"-review_status"+TXT_EXT),'a') as f:
f.write('\nsubmitted') f.write('\nsubmitted')
# Create a new branch # TODO: replace with subprocess + git
branch_name = 'metadata-review-by-'+initials checkout_review_branch(repo_obj, branch_name)
head_commit = repo_obj.head.peel()# Get the commit hash associated with HEAD
if not branch_name in repo_obj.branches:
branch = repo_obj.create_branch(branch_name, head_commit)
else:
branch = repo_obj.branches[branch_name]
repo_obj.checkout(branch)
status_dict = repo_obj.status() status_dict = repo_obj.status()
for filepath, file_status in status_dict.items(): for filepath, file_status in status_dict.items():
@ -148,13 +144,16 @@ def second_submit_metadata_review(filename_path, initials):
def third_complete_metadata_review(initials): def third_complete_metadata_review(reviewer_attrs):
initials = reviewer_attrs['initials']
push_command = lambda repository,refspec: ['git','push',repository,refspec] push_command = lambda repository,refspec: ['git','push',repository,refspec]
list_branches_command = ['git','branch','--list'] list_branches_command = ['git','branch','--list']
repository = 'origin' repository = 'origin'
branch_name = 'metadata-review-by-'+initials # refspec branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials])
branches = subprocess.run(list_branches_command,capture_output=True,text=True,check=True) branches = subprocess.run(list_branches_command,capture_output=True,text=True,check=True)
if not branch_name in branches.stdout: if not branch_name in branches.stdout:
@ -177,7 +176,8 @@ def third_complete_metadata_review(initials):
return result.returncode return result.returncode
def third_update_hdf5_file_with_review(input_hdf5_file, yaml_file): def third_update_hdf5_file_with_review(input_hdf5_file, yalm_review_file, reviewer_attrs = {}):
# compare review file with current yalm file and then based on the changes open hdf5 file and access only # compare review file with current yalm file and then based on the changes open hdf5 file and access only
# groups that changed :). the below approach is suboptimal # groups that changed :). the below approach is suboptimal
@ -202,7 +202,7 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yaml_file):
print('additions',count_additions, 'deletions', count_delections) print('additions',count_additions, 'deletions', count_delections)
with open(yaml_file,'r') as stream: with open(yalm_review_file,'r') as stream:
try: try:
yaml_dict = yaml.load(stream, Loader=yaml.FullLoader) yaml_dict = yaml.load(stream, Loader=yaml.FullLoader)
except yaml.YAMLError as exc: except yaml.YAMLError as exc:
@ -282,10 +282,12 @@ def third_update_hdf5_file_with_review(input_hdf5_file, yaml_file):
# print(exc) # print(exc)
def fourth_complete_metadata_review(initials): def fourth_complete_metadata_review(reviewer_attrs):
initials =reviewer_attrs['initials']
repository = 'origin' repository = 'origin'
branch_name = 'data-owner-review-by-'+initials branch_name = '-'.join([reviewer_attrs['type'],'review','by',initials])
current_branch_command = ['git','branch','--show-current'] current_branch_command = ['git','branch','--show-current']
list_branches_command = ['git','branch','--list'] list_branches_command = ['git','branch','--list']