From 06dd589bc784a8f3ebe54ec1cbb8332dbe6b3315 Mon Sep 17 00:00:00 2001 From: Barbara Bertozzi Date: Fri, 25 Jul 2025 15:27:01 +0200 Subject: [PATCH] refactor: rename and update schema config generator to support YAML output --- meta_files/config.yaml | 314 ++++++++++----------- meta_files/{read.py => generate_config.py} | 15 +- 2 files changed, 170 insertions(+), 159 deletions(-) rename meta_files/{read.py => generate_config.py} (66%) diff --git a/meta_files/config.yaml b/meta_files/config.yaml index 146e78c..701e851 100644 --- a/meta_files/config.yaml +++ b/meta_files/config.yaml @@ -1,159 +1,159 @@ pbp_schema: - Time (sec): float64 - Packet Time Stamp: float64 - Flag: float64 - Dropped Records: float64 - Record Count: float64 - Record Size: float64 - Particle Time Stamp: float64 - Particle Flags: float64 - Scatter relPeak: float64 - Scatter Transit Time: float64 - Scatter Peak Time: float64 - Scatter FWHM: float64 - Scatter Size (nm): float64 - Incand relPeak: float64 - Incand Transit Time: float64 - Incand Peak Time: float64 - Incand FWHM: float64 - Incand Delay: float64 - Incand Mass (fg): float64 - Reserved: float64 + Time (sec): float + Packet Time Stamp: float + Flag: float + Dropped Records: float + Record Count: float + Record Size: float + Particle Time Stamp: float + Particle Flags: float + Scatter relPeak: float + Scatter Transit Time: float + Scatter Peak Time: float + Scatter FWHM: float + Scatter Size (nm): float + Incand relPeak: float + Incand Transit Time: float + Incand Peak Time: float + Incand FWHM: float + Incand Delay: float + Incand Mass (fg): float + Reserved: float hk_schema: - Time Stamp: datetime64[us] - Time (sec): float64 - Time Stamp (UTC sec): float64 - Elapsed Time: float64 - Error Code: float64 - Packet Time Stamp: float64 - Laser TEC Temp (C): float64 - Crystal TEC Temp (C): float64 - Inlet Air Temp (C): float64 - Computer Heatsink Temp (C): float64 - Laser Heatsink Temp (C): float64 - Outlet Air Temp (C): float64 - YAG Output Monitor (V): float64 - Cavity Pressure (hPa): float64 - Laser Driver Power Monitor (uA): float64 - Laser Driver Current Limit Monitor (A): float64 - Laser Driver Current Monitor (A): float64 - Laser TEC Sense: float64 - Laser Over Temp (On/Off): float64 - +5V Laser Rail (V): float64 - ' +5V Rail (V)': float64 - +12V Rail (V): float64 - High Voltage (V): float64 - Battery Temp (C): float64 - UPS Output (V): float64 - 12V Iso Rail (V): float64 - 5V Iso Rail (V): float64 - 3.3V Iso Rail (V): float64 - Spare 22: float64 - Spare 23: float64 - 408 Board Spare 0: float64 - 408 Board Spare 1: float64 - 408 Board Spare 2: float64 - 408 Board Spare 3: float64 - 408 Board Spare 4: float64 - Purge Flow Monitor (sccm): float64 - System Input Voltage (V): float64 - Board Temperature (C): float64 - 408 Board Spare 8: float64 - 408 Board Spare 9: float64 - 408 Board Spare 10: float64 - 408 Board Spare 11: float64 - 408 Board Spare 12: float64 - 408 Board Spare 13: float64 - 408 Board Spare 14: float64 - 408 Board Spare 15: float64 - Sheath Flow Controller Read (vccm): float64 - Sheath Flow Controller Read (sccm): float64 - Sheath Flow Controller Pressure (psia): float64 - Sheath Flow Controller Temperature (C): float64 - Sample Flow Controller Read (vccm): float64 - Sample Flow Controller Read (sccm): float64 - Sample Flow Controller Pressure (psia): float64 - Sample Flow Controller Temperature (C): float64 - Fan 1 (RPM): float64 - Fan 2 (RPM): float64 - Laser Fan (RPM): float64 - Spare tach: float64 - Threshold Crossing Events: float64 - Dual Qualified Scatter and Incand Particles: float64 - Qualified Scatter Only Particles: float64 - Qualified Incand Only Particles: float64 - Disqualified Due to Scatter Saturation: float64 - Disqualified Due to Scatter Transit Time Min: float64 - Disqualified Due to Scatter Transit Time Max: float64 - Disqualified Due to Scatter FWHM Min: float64 - Disqualified Due to Scatter FWHM Max: float64 - Scatter Inter Part Period Min Violation: float64 - Disqualified Due to Incand Saturation: float64 - Disqualified Due to Incand Transit Time Min: float64 - Disqualified Due to Incand Transit Time Max: float64 - Disqualified Due to Incand FWHM Min: float64 - Disqualified Due to Incand FWHM Max: float64 - Incand Inter Part Period Min Violation: float64 - Baseline Sizer Lo: float64 - Baseline Sizer Hi: float64 - Baseline Incand Lo: float64 - Baseline Incand Hi: float64 - Bandwidth Sizer Hi: float64 - Bandwidth Sizer Lo: float64 - Bandwidth Incand Lo: float64 - Bandwidth Incand Hi: float64 - ABD-0408 HK ADCs min: float64 - ABD-0436 HK ADCs min: float64 - ABD-0408 HK ADCs max: float64 - ABD-0436 HK ADCs max: float64 - Incand Particle Conc (cts/ccm): float64 - Scattering Particle Conc (cts/ccm): float64 - Incand Mass Conc (fg/sccm): float64 - Scattering Mass Conc (fg/sccm): float64 - Sheath Flow Set Point: float64 - Sample Flow Set Point: float64 - Laser Temp Set Point: float64 - Laser Current Set Point: float64 - Spare 4 Set Point: float64 - Spare 5 Set Point: float64 - PMT HV Set Point: float64 - Particle Density (g/ccm): float64 - PbP Packet Time: float64 - Scatter Bin 1: float64 - Scatter Bin 2: float64 - Scatter Bin 3: float64 - Scatter Bin 4: float64 - Scatter Bin 5: float64 - Scatter Bin 6: float64 - Scatter Bin 7: float64 - Scatter Bin 8: float64 - Scatter Bin 9: float64 - Scatter Bin 10: float64 - Scatter Bin 11: float64 - Scatter Bin 12: float64 - Scatter Bin 13: float64 - Scatter Bin 14: float64 - Scatter Bin 15: float64 - Scatter Bin 16: float64 - Scatter Bin 17: float64 - Scatter Bin 18: float64 - Scatter Bin 19: float64 - Incand Bin 1: float64 - Incand Bin 2: float64 - Incand Bin 3: float64 - Incand Bin 4: float64 - Incand Bin 5: float64 - Incand Bin 6: float64 - Incand Bin 7: float64 - Incand Bin 8: float64 - Incand Bin 9: float64 - Incand Bin 10: float64 - Incand Bin 11: float64 - Incand Bin 12: float64 - Incand Bin 13: float64 - Incand Bin 14: float64 - Incand Bin 15: float64 - Incand Bin 16: float64 - Incand Bin 17: float64 - Incand Bin 18: float64 - Incand Bin 19: float64 + Time Stamp: datetime + Time (sec): float + Time Stamp (UTC sec): float + Elapsed Time: float + Error Code: float + Packet Time Stamp: float + Laser TEC Temp (C): float + Crystal TEC Temp (C): float + Inlet Air Temp (C): float + Computer Heatsink Temp (C): float + Laser Heatsink Temp (C): float + Outlet Air Temp (C): float + YAG Output Monitor (V): float + Cavity Pressure (hPa): float + Laser Driver Power Monitor (uA): float + Laser Driver Current Limit Monitor (A): float + Laser Driver Current Monitor (A): float + Laser TEC Sense: float + Laser Over Temp (On/Off): float + +5V Laser Rail (V): float + ' +5V Rail (V)': float + +12V Rail (V): float + High Voltage (V): float + Battery Temp (C): float + UPS Output (V): float + 12V Iso Rail (V): float + 5V Iso Rail (V): float + 3.3V Iso Rail (V): float + Spare 22: float + Spare 23: float + 408 Board Spare 0: float + 408 Board Spare 1: float + 408 Board Spare 2: float + 408 Board Spare 3: float + 408 Board Spare 4: float + Purge Flow Monitor (sccm): float + System Input Voltage (V): float + Board Temperature (C): float + 408 Board Spare 8: float + 408 Board Spare 9: float + 408 Board Spare 10: float + 408 Board Spare 11: float + 408 Board Spare 12: float + 408 Board Spare 13: float + 408 Board Spare 14: float + 408 Board Spare 15: float + Sheath Flow Controller Read (vccm): float + Sheath Flow Controller Read (sccm): float + Sheath Flow Controller Pressure (psia): float + Sheath Flow Controller Temperature (C): float + Sample Flow Controller Read (vccm): float + Sample Flow Controller Read (sccm): float + Sample Flow Controller Pressure (psia): float + Sample Flow Controller Temperature (C): float + Fan 1 (RPM): float + Fan 2 (RPM): float + Laser Fan (RPM): float + Spare tach: float + Threshold Crossing Events: float + Dual Qualified Scatter and Incand Particles: float + Qualified Scatter Only Particles: float + Qualified Incand Only Particles: float + Disqualified Due to Scatter Saturation: float + Disqualified Due to Scatter Transit Time Min: float + Disqualified Due to Scatter Transit Time Max: float + Disqualified Due to Scatter FWHM Min: float + Disqualified Due to Scatter FWHM Max: float + Scatter Inter Part Period Min Violation: float + Disqualified Due to Incand Saturation: float + Disqualified Due to Incand Transit Time Min: float + Disqualified Due to Incand Transit Time Max: float + Disqualified Due to Incand FWHM Min: float + Disqualified Due to Incand FWHM Max: float + Incand Inter Part Period Min Violation: float + Baseline Sizer Lo: float + Baseline Sizer Hi: float + Baseline Incand Lo: float + Baseline Incand Hi: float + Bandwidth Sizer Hi: float + Bandwidth Sizer Lo: float + Bandwidth Incand Lo: float + Bandwidth Incand Hi: float + ABD-0408 HK ADCs min: float + ABD-0436 HK ADCs min: float + ABD-0408 HK ADCs max: float + ABD-0436 HK ADCs max: float + Incand Particle Conc (cts/ccm): float + Scattering Particle Conc (cts/ccm): float + Incand Mass Conc (fg/sccm): float + Scattering Mass Conc (fg/sccm): float + Sheath Flow Set Point: float + Sample Flow Set Point: float + Laser Temp Set Point: float + Laser Current Set Point: float + Spare 4 Set Point: float + Spare 5 Set Point: float + PMT HV Set Point: float + Particle Density (g/ccm): float + PbP Packet Time: float + Scatter Bin 1: float + Scatter Bin 2: float + Scatter Bin 3: float + Scatter Bin 4: float + Scatter Bin 5: float + Scatter Bin 6: float + Scatter Bin 7: float + Scatter Bin 8: float + Scatter Bin 9: float + Scatter Bin 10: float + Scatter Bin 11: float + Scatter Bin 12: float + Scatter Bin 13: float + Scatter Bin 14: float + Scatter Bin 15: float + Scatter Bin 16: float + Scatter Bin 17: float + Scatter Bin 18: float + Scatter Bin 19: float + Incand Bin 1: float + Incand Bin 2: float + Incand Bin 3: float + Incand Bin 4: float + Incand Bin 5: float + Incand Bin 6: float + Incand Bin 7: float + Incand Bin 8: float + Incand Bin 9: float + Incand Bin 10: float + Incand Bin 11: float + Incand Bin 12: float + Incand Bin 13: float + Incand Bin 14: float + Incand Bin 15: float + Incand Bin 16: float + Incand Bin 17: float + Incand Bin 18: float + Incand Bin 19: float diff --git a/meta_files/read.py b/meta_files/generate_config.py similarity index 66% rename from meta_files/read.py rename to meta_files/generate_config.py index 88748be..f9d52cf 100644 --- a/meta_files/read.py +++ b/meta_files/generate_config.py @@ -3,6 +3,17 @@ import yaml import os +def infer_general_dtype(dtype): + if pd.api.types.is_integer_dtype(dtype): + return "int" + elif pd.api.types.is_float_dtype(dtype): + return "float" + elif pd.api.types.is_datetime64_any_dtype(dtype): + return "datetime" + else: + return "string" + + def load_schema(input_file): ext = os.path.splitext(input_file)[1].lower() @@ -13,7 +24,7 @@ def load_schema(input_file): else: raise ValueError(f"Unsupported file format: {ext}") - schema = {col: str(dtype) for col, dtype in df.dtypes.items()} + schema = {col: infer_general_dtype(dtype) for col, dtype in df.dtypes.items()} return schema @@ -29,5 +40,5 @@ def generate_combined_config(pbp_file, hk_file, output_file="config.yaml"): print(f"Unified config saved to: {output_file}") -# Example usage: +# Example usage generate_combined_config("pbp_meta.parquet", "hk_meta.parquet")