Modified logger setup to create monthly logs

pipelines/data_integration.py

@@ -41,7 +41,7 @@ def load_config_and_setup_logging(yaml_config_file_path, log_dir):
 
 
     # Set up logging
-    date = utils.created_at().replace(":", "-")
+    date = utils.created_at("%Y_%m").replace(":", "-")
     utils.setup_logging(log_dir, f"integrate_data_sources_{date}.log")
 
     # Load YAML configuration file

utils/g5505_utils.py

@@ -100,13 +100,13 @@ def make_file_copy(source_file_path, output_folder_name : str = 'tmp_files'):
 
     return tmp_file_path
 
-def created_at():
+def created_at(datetime_format = '%Y-%m-%d %H:%M:%S'):
     now = datetime.datetime.now()
     # Populate now object with time zone information obtained from the local system
     now_tz_aware = now.astimezone()
     tz = now_tz_aware.strftime('%z')
     # Replace colons in the time part of the timestamp with hyphens to make it file name friendly
-    created_at = now_tz_aware.strftime('%Y-%m-%d %H:%M:%S') #+ '_UTC-OFST_' + tz
+    created_at = now_tz_aware.strftime(datetime_format) #+ '_UTC-OFST_' + tz
     return created_at
 
 def convert_dataframe_to_np_structured_array(df: pd.DataFrame):
@@ -240,7 +240,7 @@ def copy_directory_with_contraints(input_dir_path, output_dir_path,
     select_file_keywords = select_file_keywords or []
     allowed_file_extensions = allowed_file_extensions or []
 
-    date = created_at().replace(":", "-")
+    date = created_at('%Y_%m').replace(":", "-")
     log_dir='logs/'
     setup_logging(log_dir, f"copy_directory_with_contraints_{date}.log")