diff --git a/config/complete_example.yaml b/config/complete_example.yaml index d1ea7eb..99964f4 100644 --- a/config/complete_example.yaml +++ b/config/complete_example.yaml @@ -51,7 +51,7 @@ cluster: # SLURM-specific settings walltime: "4-00:00:00" # Job time limit (days-hours:minutes:seconds) - partition: general # SLURM partition: general, bigmem, gpu, etc. + partition: general # SLURM partition log_dir: ./slurm_out # SLURM log directory @@ -65,7 +65,7 @@ chunking: # ============================================================================ -# BLACK CARBON PROPERTIES - Physical and optical parameters +# BLACK CARBON PROPERTIES # ============================================================================ bc: # Density settings diff --git a/config/conversion_config_template.yaml b/config/conversion_config_template.yaml index 9bf7b0f..f9ccbac 100644 --- a/config/conversion_config_template.yaml +++ b/config/conversion_config_template.yaml @@ -5,7 +5,7 @@ # # USAGE: # 1. Update the paths and parameters below for your dataset -# 2. Run: python scripts/sp2xr_csv2parquet.py --config my_conversion_config.yaml +# 2. Run: python scripts/sp2xr_csv2parquet.py --config conversion_config_template.yaml # # NOTES: # - For local execution, the script auto-detects available CPU cores and memory @@ -21,7 +21,7 @@ source_directory: data/SP2XR_orig_files target_directory: data/pbp_files_parquet_2 # Path to your data schema config file (generated by sp2xr_generate_config.py) -schema_config: config/config_schema_with_mapping.yaml +schema_config: config/new_data_schema_with_mapping.yaml # Pattern to filter which files to process diff --git a/config/my_data_schema.yaml b/config/my_data_schema.yaml index 5acd583..16fb5fc 100644 --- a/config/my_data_schema.yaml +++ b/config/my_data_schema.yaml @@ -9,16 +9,16 @@ pbp_schema: Time (sec): float Packet Time Stamp: float - Flag: int - Dropped Records: int - Record Count: int - Record Size: int + Flag: float + Dropped Records: float + Record Count: float + Record Size: float Particle Time Stamp: float - Particle Flags: int + Particle Flags: float Scatter relPeak: float - Scatter Transit Time: int - Scatter Peak Time: int - Scatter FWHM: int + Scatter Transit Time: float + Scatter Peak Time: float + Scatter FWHM: float Scatter Size (nm): float Incand relPeak: float Incand Transit Time: float @@ -26,27 +26,27 @@ pbp_schema: Incand FWHM: float Incand Delay: float Incand Mass (fg): float - Reserved: int + Reserved: float hk_schema: Time Stamp: string Time (sec): float Time Stamp (UTC sec): float Elapsed Time: float - Error Code: int + Error Code: float Packet Time Stamp: float Laser TEC Temp (C): float - Crystal TEC Temp (C): int + Crystal TEC Temp (C): float Inlet Air Temp (C): float Computer Heatsink Temp (C): float Laser Heatsink Temp (C): float Outlet Air Temp (C): float YAG Output Monitor (V): float Cavity Pressure (hPa): float - Laser Driver Power Monitor (uA): int + Laser Driver Power Monitor (uA): float Laser Driver Current Limit Monitor (A): float Laser Driver Current Monitor (A): float Laser TEC Sense: float - Laser Over Temp (On/Off): int + Laser Over Temp (On/Off): float +5V Laser Rail (V): float ' +5V Rail (V)': float +12V Rail (V): float @@ -56,117 +56,117 @@ hk_schema: 12V Iso Rail (V): float 5V Iso Rail (V): float 3.3V Iso Rail (V): float - Spare 22: int - Spare 23: int - 408 Board Spare 0: int - 408 Board Spare 1: int - 408 Board Spare 2: int - 408 Board Spare 3: int - 408 Board Spare 4: int + Spare 22: float + Spare 23: float + 408 Board Spare 0: float + 408 Board Spare 1: float + 408 Board Spare 2: float + 408 Board Spare 3: float + 408 Board Spare 4: float Purge Flow Monitor (sccm): float System Input Voltage (V): float Board Temperature (C): float - 408 Board Spare 8: int - 408 Board Spare 9: int - 408 Board Spare 10: int - 408 Board Spare 11: int - 408 Board Spare 12: int - 408 Board Spare 13: int - 408 Board Spare 14: int - 408 Board Spare 15: int - Sheath Flow Controller Read (vccm): int - Sheath Flow Controller Read (sccm): int + 408 Board Spare 8: float + 408 Board Spare 9: float + 408 Board Spare 10: float + 408 Board Spare 11: float + 408 Board Spare 12: float + 408 Board Spare 13: float + 408 Board Spare 14: float + 408 Board Spare 15: float + Sheath Flow Controller Read (vccm): float + Sheath Flow Controller Read (sccm): float Sheath Flow Controller Pressure (psia): float Sheath Flow Controller Temperature (C): float Sample Flow Controller Read (vccm): float Sample Flow Controller Read (sccm): float Sample Flow Controller Pressure (psia): float Sample Flow Controller Temperature (C): float - Fan 1 (RPM): int - Fan 2 (RPM): int - Laser Fan (RPM): int - Spare tach: int - Threshold Crossing Events: int - Dual Qualified Scatter and Incand Particles: int - Qualified Scatter Only Particles: int - Qualified Incand Only Particles: int - Disqualified Due to Scatter Saturation: int - Disqualified Due to Scatter Transit Time Min: int - Disqualified Due to Scatter Transit Time Max: int - Disqualified Due to Scatter FWHM Min: int - Disqualified Due to Scatter FWHM Max: int - Scatter Inter Part Period Min Violation: int - Disqualified Due to Incand Saturation: int - Disqualified Due to Incand Transit Time Min: int - Disqualified Due to Incand Transit Time Max: int - Disqualified Due to Incand FWHM Min: int - Disqualified Due to Incand FWHM Max: int - Incand Inter Part Period Min Violation: int - Baseline Sizer Lo: int - Baseline Sizer Hi: int - Baseline Incand Lo: int - Baseline Incand Hi: int - Bandwidth Sizer Hi: int - Bandwidth Sizer Lo: int - Bandwidth Incand Lo: int - Bandwidth Incand Hi: int - ABD-0408 HK ADCs min: int - ABD-0436 HK ADCs min: int - ABD-0408 HK ADCs max: int - ABD-0436 HK ADCs max: int + Fan 1 (RPM): float + Fan 2 (RPM): float + Laser Fan (RPM): float + Spare tach: float + Threshold Crossing Events: float + Dual Qualified Scatter and Incand Particles: float + Qualified Scatter Only Particles: float + Qualified Incand Only Particles: float + Disqualified Due to Scatter Saturation: float + Disqualified Due to Scatter Transit Time Min: float + Disqualified Due to Scatter Transit Time Max: float + Disqualified Due to Scatter FWHM Min: float + Disqualified Due to Scatter FWHM Max: float + Scatter Inter Part Period Min Violation: float + Disqualified Due to Incand Saturation: float + Disqualified Due to Incand Transit Time Min: float + Disqualified Due to Incand Transit Time Max: float + Disqualified Due to Incand FWHM Min: float + Disqualified Due to Incand FWHM Max: float + Incand Inter Part Period Min Violation: float + Baseline Sizer Lo: float + Baseline Sizer Hi: float + Baseline Incand Lo: float + Baseline Incand Hi: float + Bandwidth Sizer Hi: float + Bandwidth Sizer Lo: float + Bandwidth Incand Lo: float + Bandwidth Incand Hi: float + ABD-0408 HK ADCs min: float + ABD-0436 HK ADCs min: float + ABD-0408 HK ADCs max: float + ABD-0436 HK ADCs max: float Incand Particle Conc (cts/ccm): float Scattering Particle Conc (cts/ccm): float Incand Mass Conc (fg/sccm): float Scattering Mass Conc (fg/sccm): float - Sheath Flow Set Point: int - Sample Flow Set Point: int - Laser Temp Set Point: int + Sheath Flow Set Point: float + Sample Flow Set Point: float + Laser Temp Set Point: float Laser Current Set Point: float - Spare 4 Set Point: int - Spare 5 Set Point: int + Spare 4 Set Point: float + Spare 5 Set Point: float PMT HV Set Point: float Particle Density (g/ccm): float PbP Packet Time: float - Scatter Bin 1: int - Scatter Bin 2: int - Scatter Bin 3: int - Scatter Bin 4: int - Scatter Bin 5: int - Scatter Bin 6: int - Scatter Bin 7: int - Scatter Bin 8: int - Scatter Bin 9: int - Scatter Bin 10: int - Scatter Bin 11: int - Scatter Bin 12: int - Scatter Bin 13: int - Scatter Bin 14: int - Scatter Bin 15: int - Scatter Bin 16: int - Scatter Bin 17: int - Scatter Bin 18: int - Scatter Bin 19: int - Scatter Bin 20: int - Incand Bin 1: int - Incand Bin 2: int - Incand Bin 3: int - Incand Bin 4: int - Incand Bin 5: int - Incand Bin 6: int - Incand Bin 7: int - Incand Bin 8: int - Incand Bin 9: int - Incand Bin 10: int - Incand Bin 11: int - Incand Bin 12: int - Incand Bin 13: int - Incand Bin 14: int - Incand Bin 15: int - Incand Bin 16: int - Incand Bin 17: int - Incand Bin 18: int - Incand Bin 19: int - Incand Bin 20: int + Scatter Bin 1: float + Scatter Bin 2: float + Scatter Bin 3: float + Scatter Bin 4: float + Scatter Bin 5: float + Scatter Bin 6: float + Scatter Bin 7: float + Scatter Bin 8: float + Scatter Bin 9: float + Scatter Bin 10: float + Scatter Bin 11: float + Scatter Bin 12: float + Scatter Bin 13: float + Scatter Bin 14: float + Scatter Bin 15: float + Scatter Bin 16: float + Scatter Bin 17: float + Scatter Bin 18: float + Scatter Bin 19: float + Scatter Bin 20: float + Incand Bin 1: float + Incand Bin 2: float + Incand Bin 3: float + Incand Bin 4: float + Incand Bin 5: float + Incand Bin 6: float + Incand Bin 7: float + Incand Bin 8: float + Incand Bin 9: float + Incand Bin 10: float + Incand Bin 11: float + Incand Bin 12: float + Incand Bin 13: float + Incand Bin 14: float + Incand Bin 15: float + Incand Bin 16: float + Incand Bin 17: float + Incand Bin 18: float + Incand Bin 19: float + Incand Bin 20: float pbp_canonical_schema: Time (sec): float Packet Time Stamp: float diff --git a/scripts/sp2xr_generate_config.py b/scripts/sp2xr_generate_config.py index fd50354..bbb0f21 100644 --- a/scripts/sp2xr_generate_config.py +++ b/scripts/sp2xr_generate_config.py @@ -152,7 +152,7 @@ def get_canonical_schemas() -> dict[str, dict[str, str]]: def generate_basic_config( pbp_file: Path, hk_file: Path, - schema_output: str = "config_schema.yaml", + schema_output: str = "new_data_schema.yaml", ini_file: str = None, instrument_output: str = None, ) -> None: @@ -185,9 +185,7 @@ def generate_basic_config( if instrument_output: instrument_path = Path(instrument_output) else: - instrument_path = ( - schema_path.parent / f"{schema_path.stem}_instrument_settings.yaml" - ) + instrument_path = schema_path.parent / "new_instrument_settings.yaml" try: from sp2xr.helpers import export_xr_ini_to_yaml_with_source @@ -212,7 +210,7 @@ def generate_basic_config( def generate_mapping_config( pbp_file: Path, hk_file: Path, - schema_output: str = "config_schema_with_mapping.yaml", + schema_output: str = "new_data_schema_with_mapping.yaml", ini_file: str = None, instrument_output: str = None, ) -> None: @@ -291,9 +289,7 @@ def generate_mapping_config( if instrument_output: instrument_path = Path(instrument_output) else: - instrument_path = ( - schema_path.parent / f"{schema_path.stem}_instrument_settings.yaml" - ) + instrument_path = schema_path.parent / "new_instrument_settings.yaml" try: from sp2xr.helpers import export_xr_ini_to_yaml_with_source @@ -346,8 +342,8 @@ Examples: parser.add_argument( "--schema-output", "-s", - default="config/config_schema.yaml", - help="Output filename for data schema config (default: config/config_schema.yaml)", + default="config/new_data_schema.yaml", + help="Output filename for data schema config (default: config/new_data_schema.yaml)", ) parser.add_argument( @@ -431,8 +427,8 @@ def main(): # Generate configuration if args.mapping: schema_file = ( - "config/config_schema_with_mapping.yaml" - if args.schema_output == "config/config_schema.yaml" + "config/new_data_schema_with_mapping.yaml" + if args.schema_output == "config/new_data_schema.yaml" else args.schema_output ) generate_mapping_config(