Update to pipelines/data_integration.py. Added feature to use environment variable MOUNT_DRIVE defined in .env file.

pipelines/data_integration.py

@@ -75,12 +75,28 @@ def load_config_and_setup_logging(yaml_config_file_path, log_dir):
         except yaml.YAMLError as exc:
             logging.error("Error loading YAML file: %s", exc)
             raise ValueError(f"Failed to load YAML file: {exc}")
-    
+        
     # Check if required keys are present
     missing_keys = [key for key in required_keys if key not in config_dict]
     if missing_keys:
         raise KeyError(f"Missing required keys in YAML configuration: {missing_keys}")
     
+    # Look for all placeholders like ${VAR_NAME}
+    input_dir = config_dict['input_file_directory']
+    placeholders = re.findall(r'\$\{([^}^{]+)\}', input_dir)
+
+    success = utils.load_env_from_root()
+    print(f'Success : {success}')
+
+    for var in placeholders:
+        env_value = os.environ.get(var)
+        if env_value is None:
+            raise ValueError(f"Environment variable '{var}' is not set but used in the config.")
+        input_dir = input_dir.replace(f"${{{var}}}", env_value)
+
+    config_dict['input_file_directory'] = input_dir
+    
+   
     # Check the instrument_datafolder required type and ensure the list is of at least length one.    
     if isinstance(config_dict['instrument_datafolder'], list) and not len(config_dict['instrument_datafolder'])>=1:
         raise ValueError('Invalid value for key "instrument_datafolder". Expected a list of strings with at least one item.'