diff --git a/pipelines/data_integration.py b/pipelines/data_integration.py index d76f532..af0c2cc 100644 --- a/pipelines/data_integration.py +++ b/pipelines/data_integration.py @@ -41,7 +41,7 @@ def load_config_and_setup_logging(yaml_config_file_path, log_dir): # Set up logging - date = utils.created_at().replace(":", "-") + date = utils.created_at("%Y_%m").replace(":", "-") utils.setup_logging(log_dir, f"integrate_data_sources_{date}.log") # Load YAML configuration file diff --git a/utils/g5505_utils.py b/utils/g5505_utils.py index 36c5d93..2343181 100644 --- a/utils/g5505_utils.py +++ b/utils/g5505_utils.py @@ -100,13 +100,13 @@ def make_file_copy(source_file_path, output_folder_name : str = 'tmp_files'): return tmp_file_path -def created_at(): +def created_at(datetime_format = '%Y-%m-%d %H:%M:%S'): now = datetime.datetime.now() # Populate now object with time zone information obtained from the local system now_tz_aware = now.astimezone() tz = now_tz_aware.strftime('%z') # Replace colons in the time part of the timestamp with hyphens to make it file name friendly - created_at = now_tz_aware.strftime('%Y-%m-%d %H:%M:%S') #+ '_UTC-OFST_' + tz + created_at = now_tz_aware.strftime(datetime_format) #+ '_UTC-OFST_' + tz return created_at def convert_dataframe_to_np_structured_array(df: pd.DataFrame): @@ -240,7 +240,7 @@ def copy_directory_with_contraints(input_dir_path, output_dir_path, select_file_keywords = select_file_keywords or [] allowed_file_extensions = allowed_file_extensions or [] - date = created_at().replace(":", "-") + date = created_at('%Y_%m').replace(":", "-") log_dir='logs/' setup_logging(log_dir, f"copy_directory_with_contraints_{date}.log")