diff --git a/src/data_integration_lib.py b/src/data_integration_lib.py index b8c3042..876eb1d 100644 --- a/src/data_integration_lib.py +++ b/src/data_integration_lib.py @@ -1,22 +1,35 @@ import os import src.hdf5_lib as hdf5_lib +import src.g5505_utils as utils import yaml + +import logging from datetime import datetime -def integrate_data_sources(yaml_config_file_path): +def integrate_data_sources(yaml_config_file_path, log_dir='logs/'): - """ returns an hdf5 file for each experimental campaign specified by the input configuration file. - + """ Integrates data sources specified by the input configuration file into HDF5 files. + + Parameters: + yaml_config_file_path (str): Path to the YAML configuration file. + log_dir (str): Directory to save the log file. + + Returns: + str: Path to the created HDF5 file. """ - # TODO: add and commit changes to yaml_config_file_path + + date = utils.created_at() + utils.setup_logging(log_dir, f"integrate_data_sources_{date}.log") + with open(yaml_config_file_path,'r') as stream: try: config_dict = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: - print(exc) + logging.error("Error loading YAML file: %s", exc) + raise output_filename = lambda name, datetime, initials: '_'.join([name,datetime,initials])+'.h5' exp_campaign_name = config_dict['experiment_name']