From ecd08d4c0266ad3351846c8d0a634fd1ec4e65e5 Mon Sep 17 00:00:00 2001 From: Barbara Bertozzi Date: Fri, 25 Jul 2025 14:51:48 +0200 Subject: [PATCH] chore(meta): update config.yaml with correct column types --- meta_files/config.yaml | 314 +++++++++++++++++++-------------------- meta_files/hk_meta.csv | 1 - meta_files/hk_meta.yaml | 277 ---------------------------------- meta_files/pbp_meta.csv | 1 - meta_files/pbp_meta.yaml | 43 ------ meta_files/read.py | 15 +- 6 files changed, 159 insertions(+), 492 deletions(-) delete mode 100644 meta_files/hk_meta.csv delete mode 100644 meta_files/hk_meta.yaml delete mode 100644 meta_files/pbp_meta.csv delete mode 100644 meta_files/pbp_meta.yaml diff --git a/meta_files/config.yaml b/meta_files/config.yaml index 9bbecdc..146e78c 100644 --- a/meta_files/config.yaml +++ b/meta_files/config.yaml @@ -1,159 +1,159 @@ pbp_schema: - Time (sec): string - Packet Time Stamp: string - Flag: string - Dropped Records: string - Record Count: string - Record Size: string - Particle Time Stamp: string - Particle Flags: string - Scatter relPeak: string - Scatter Transit Time: string - Scatter Peak Time: string - Scatter FWHM: string - Scatter Size (nm): string - Incand relPeak: string - Incand Transit Time: string - Incand Peak Time: string - Incand FWHM: string - Incand Delay: string - Incand Mass (fg): string - Reserved: string + Time (sec): float64 + Packet Time Stamp: float64 + Flag: float64 + Dropped Records: float64 + Record Count: float64 + Record Size: float64 + Particle Time Stamp: float64 + Particle Flags: float64 + Scatter relPeak: float64 + Scatter Transit Time: float64 + Scatter Peak Time: float64 + Scatter FWHM: float64 + Scatter Size (nm): float64 + Incand relPeak: float64 + Incand Transit Time: float64 + Incand Peak Time: float64 + Incand FWHM: float64 + Incand Delay: float64 + Incand Mass (fg): float64 + Reserved: float64 hk_schema: - Time Stamp: string - Time (sec): string - Time Stamp (UTC sec): string - Elapsed Time: string - Error Code: string - Packet Time Stamp: string - Laser TEC Temp (C): string - Crystal TEC Temp (C): string - Inlet Air Temp (C): string - Computer Heatsink Temp (C): string - Laser Heatsink Temp (C): string - Outlet Air Temp (C): string - YAG Output Monitor (V): string - Cavity Pressure (hPa): string - Laser Driver Power Monitor (uA): string - Laser Driver Current Limit Monitor (A): string - Laser Driver Current Monitor (A): string - Laser TEC Sense: string - Laser Over Temp (On/Off): string - +5V Laser Rail (V): string - ' +5V Rail (V)': string - +12V Rail (V): string - High Voltage (V): string - Battery Temp (C): string - UPS Output (V): string - 12V Iso Rail (V): string - 5V Iso Rail (V): string - 3.3V Iso Rail (V): string - Spare 22: string - Spare 23: string - 408 Board Spare 0: string - 408 Board Spare 1: string - 408 Board Spare 2: string - 408 Board Spare 3: string - 408 Board Spare 4: string - Purge Flow Monitor (sccm): string - System Input Voltage (V): string - Board Temperature (C): string - 408 Board Spare 8: string - 408 Board Spare 9: string - 408 Board Spare 10: string - 408 Board Spare 11: string - 408 Board Spare 12: string - 408 Board Spare 13: string - 408 Board Spare 14: string - 408 Board Spare 15: string - Sheath Flow Controller Read (vccm): string - Sheath Flow Controller Read (sccm): string - Sheath Flow Controller Pressure (psia): string - Sheath Flow Controller Temperature (C): string - Sample Flow Controller Read (vccm): string - Sample Flow Controller Read (sccm): string - Sample Flow Controller Pressure (psia): string - Sample Flow Controller Temperature (C): string - Fan 1 (RPM): string - Fan 2 (RPM): string - Laser Fan (RPM): string - Spare tach: string - Threshold Crossing Events: string - Dual Qualified Scatter and Incand Particles: string - Qualified Scatter Only Particles: string - Qualified Incand Only Particles: string - Disqualified Due to Scatter Saturation: string - Disqualified Due to Scatter Transit Time Min: string - Disqualified Due to Scatter Transit Time Max: string - Disqualified Due to Scatter FWHM Min: string - Disqualified Due to Scatter FWHM Max: string - Scatter Inter Part Period Min Violation: string - Disqualified Due to Incand Saturation: string - Disqualified Due to Incand Transit Time Min: string - Disqualified Due to Incand Transit Time Max: string - Disqualified Due to Incand FWHM Min: string - Disqualified Due to Incand FWHM Max: string - Incand Inter Part Period Min Violation: string - Baseline Sizer Lo: string - Baseline Sizer Hi: string - Baseline Incand Lo: string - Baseline Incand Hi: string - Bandwidth Sizer Hi: string - Bandwidth Sizer Lo: string - Bandwidth Incand Lo: string - Bandwidth Incand Hi: string - ABD-0408 HK ADCs min: string - ABD-0436 HK ADCs min: string - ABD-0408 HK ADCs max: string - ABD-0436 HK ADCs max: string - Incand Particle Conc (cts/ccm): string - Scattering Particle Conc (cts/ccm): string - Incand Mass Conc (fg/sccm): string - Scattering Mass Conc (fg/sccm): string - Sheath Flow Set Point: string - Sample Flow Set Point: string - Laser Temp Set Point: string - Laser Current Set Point: string - Spare 4 Set Point: string - Spare 5 Set Point: string - PMT HV Set Point: string - Particle Density (g/ccm): string - PbP Packet Time: string - Scatter Bin 1: string - Scatter Bin 2: string - Scatter Bin 3: string - Scatter Bin 4: string - Scatter Bin 5: string - Scatter Bin 6: string - Scatter Bin 7: string - Scatter Bin 8: string - Scatter Bin 9: string - Scatter Bin 10: string - Scatter Bin 11: string - Scatter Bin 12: string - Scatter Bin 13: string - Scatter Bin 14: string - Scatter Bin 15: string - Scatter Bin 16: string - Scatter Bin 17: string - Scatter Bin 18: string - Scatter Bin 19: string - Incand Bin 1: string - Incand Bin 2: string - Incand Bin 3: string - Incand Bin 4: string - Incand Bin 5: string - Incand Bin 6: string - Incand Bin 7: string - Incand Bin 8: string - Incand Bin 9: string - Incand Bin 10: string - Incand Bin 11: string - Incand Bin 12: string - Incand Bin 13: string - Incand Bin 14: string - Incand Bin 15: string - Incand Bin 16: string - Incand Bin 17: string - Incand Bin 18: string - Incand Bin 19: string + Time Stamp: datetime64[us] + Time (sec): float64 + Time Stamp (UTC sec): float64 + Elapsed Time: float64 + Error Code: float64 + Packet Time Stamp: float64 + Laser TEC Temp (C): float64 + Crystal TEC Temp (C): float64 + Inlet Air Temp (C): float64 + Computer Heatsink Temp (C): float64 + Laser Heatsink Temp (C): float64 + Outlet Air Temp (C): float64 + YAG Output Monitor (V): float64 + Cavity Pressure (hPa): float64 + Laser Driver Power Monitor (uA): float64 + Laser Driver Current Limit Monitor (A): float64 + Laser Driver Current Monitor (A): float64 + Laser TEC Sense: float64 + Laser Over Temp (On/Off): float64 + +5V Laser Rail (V): float64 + ' +5V Rail (V)': float64 + +12V Rail (V): float64 + High Voltage (V): float64 + Battery Temp (C): float64 + UPS Output (V): float64 + 12V Iso Rail (V): float64 + 5V Iso Rail (V): float64 + 3.3V Iso Rail (V): float64 + Spare 22: float64 + Spare 23: float64 + 408 Board Spare 0: float64 + 408 Board Spare 1: float64 + 408 Board Spare 2: float64 + 408 Board Spare 3: float64 + 408 Board Spare 4: float64 + Purge Flow Monitor (sccm): float64 + System Input Voltage (V): float64 + Board Temperature (C): float64 + 408 Board Spare 8: float64 + 408 Board Spare 9: float64 + 408 Board Spare 10: float64 + 408 Board Spare 11: float64 + 408 Board Spare 12: float64 + 408 Board Spare 13: float64 + 408 Board Spare 14: float64 + 408 Board Spare 15: float64 + Sheath Flow Controller Read (vccm): float64 + Sheath Flow Controller Read (sccm): float64 + Sheath Flow Controller Pressure (psia): float64 + Sheath Flow Controller Temperature (C): float64 + Sample Flow Controller Read (vccm): float64 + Sample Flow Controller Read (sccm): float64 + Sample Flow Controller Pressure (psia): float64 + Sample Flow Controller Temperature (C): float64 + Fan 1 (RPM): float64 + Fan 2 (RPM): float64 + Laser Fan (RPM): float64 + Spare tach: float64 + Threshold Crossing Events: float64 + Dual Qualified Scatter and Incand Particles: float64 + Qualified Scatter Only Particles: float64 + Qualified Incand Only Particles: float64 + Disqualified Due to Scatter Saturation: float64 + Disqualified Due to Scatter Transit Time Min: float64 + Disqualified Due to Scatter Transit Time Max: float64 + Disqualified Due to Scatter FWHM Min: float64 + Disqualified Due to Scatter FWHM Max: float64 + Scatter Inter Part Period Min Violation: float64 + Disqualified Due to Incand Saturation: float64 + Disqualified Due to Incand Transit Time Min: float64 + Disqualified Due to Incand Transit Time Max: float64 + Disqualified Due to Incand FWHM Min: float64 + Disqualified Due to Incand FWHM Max: float64 + Incand Inter Part Period Min Violation: float64 + Baseline Sizer Lo: float64 + Baseline Sizer Hi: float64 + Baseline Incand Lo: float64 + Baseline Incand Hi: float64 + Bandwidth Sizer Hi: float64 + Bandwidth Sizer Lo: float64 + Bandwidth Incand Lo: float64 + Bandwidth Incand Hi: float64 + ABD-0408 HK ADCs min: float64 + ABD-0436 HK ADCs min: float64 + ABD-0408 HK ADCs max: float64 + ABD-0436 HK ADCs max: float64 + Incand Particle Conc (cts/ccm): float64 + Scattering Particle Conc (cts/ccm): float64 + Incand Mass Conc (fg/sccm): float64 + Scattering Mass Conc (fg/sccm): float64 + Sheath Flow Set Point: float64 + Sample Flow Set Point: float64 + Laser Temp Set Point: float64 + Laser Current Set Point: float64 + Spare 4 Set Point: float64 + Spare 5 Set Point: float64 + PMT HV Set Point: float64 + Particle Density (g/ccm): float64 + PbP Packet Time: float64 + Scatter Bin 1: float64 + Scatter Bin 2: float64 + Scatter Bin 3: float64 + Scatter Bin 4: float64 + Scatter Bin 5: float64 + Scatter Bin 6: float64 + Scatter Bin 7: float64 + Scatter Bin 8: float64 + Scatter Bin 9: float64 + Scatter Bin 10: float64 + Scatter Bin 11: float64 + Scatter Bin 12: float64 + Scatter Bin 13: float64 + Scatter Bin 14: float64 + Scatter Bin 15: float64 + Scatter Bin 16: float64 + Scatter Bin 17: float64 + Scatter Bin 18: float64 + Scatter Bin 19: float64 + Incand Bin 1: float64 + Incand Bin 2: float64 + Incand Bin 3: float64 + Incand Bin 4: float64 + Incand Bin 5: float64 + Incand Bin 6: float64 + Incand Bin 7: float64 + Incand Bin 8: float64 + Incand Bin 9: float64 + Incand Bin 10: float64 + Incand Bin 11: float64 + Incand Bin 12: float64 + Incand Bin 13: float64 + Incand Bin 14: float64 + Incand Bin 15: float64 + Incand Bin 16: float64 + Incand Bin 17: float64 + Incand Bin 18: float64 + Incand Bin 19: float64 diff --git a/meta_files/hk_meta.csv b/meta_files/hk_meta.csv deleted file mode 100644 index 19ee3e1..0000000 --- a/meta_files/hk_meta.csv +++ /dev/null @@ -1 +0,0 @@ -Time Stamp,Time (sec),Time Stamp (UTC sec),Elapsed Time,Error Code,Packet Time Stamp,Laser TEC Temp (C),Crystal TEC Temp (C),Inlet Air Temp (C),Computer Heatsink Temp (C),Laser Heatsink Temp (C),Outlet Air Temp (C),YAG Output Monitor (V),Cavity Pressure (hPa),Laser Driver Power Monitor (uA),Laser Driver Current Limit Monitor (A),Laser Driver Current Monitor (A),Laser TEC Sense,Laser Over Temp (On/Off),+5V Laser Rail (V), +5V Rail (V),+12V Rail (V),High Voltage (V),Battery Temp (C),UPS Output (V),12V Iso Rail (V),5V Iso Rail (V),3.3V Iso Rail (V),Spare 22,Spare 23,408 Board Spare 0,408 Board Spare 1,408 Board Spare 2,408 Board Spare 3,408 Board Spare 4,Purge Flow Monitor (sccm),System Input Voltage (V),Board Temperature (C),408 Board Spare 8,408 Board Spare 9,408 Board Spare 10,408 Board Spare 11,408 Board Spare 12,408 Board Spare 13,408 Board Spare 14,408 Board Spare 15,Sheath Flow Controller Read (vccm),Sheath Flow Controller Read (sccm),Sheath Flow Controller Pressure (psia),Sheath Flow Controller Temperature (C),Sample Flow Controller Read (vccm),Sample Flow Controller Read (sccm),Sample Flow Controller Pressure (psia),Sample Flow Controller Temperature (C),Fan 1 (RPM),Fan 2 (RPM),Laser Fan (RPM),Spare tach,Threshold Crossing Events,Dual Qualified Scatter and Incand Particles,Qualified Scatter Only Particles,Qualified Incand Only Particles,Disqualified Due to Scatter Saturation,Disqualified Due to Scatter Transit Time Min,Disqualified Due to Scatter Transit Time Max,Disqualified Due to Scatter FWHM Min,Disqualified Due to Scatter FWHM Max,Scatter Inter Part Period Min Violation,Disqualified Due to Incand Saturation,Disqualified Due to Incand Transit Time Min,Disqualified Due to Incand Transit Time Max,Disqualified Due to Incand FWHM Min,Disqualified Due to Incand FWHM Max,Incand Inter Part Period Min Violation,Baseline Sizer Lo,Baseline Sizer Hi,Baseline Incand Lo,Baseline Incand Hi,Bandwidth Sizer Hi,Bandwidth Sizer Lo,Bandwidth Incand Lo,Bandwidth Incand Hi,ABD-0408 HK ADCs min,ABD-0436 HK ADCs min,ABD-0408 HK ADCs max,ABD-0436 HK ADCs max,Incand Particle Conc (cts/ccm),Scattering Particle Conc (cts/ccm),Incand Mass Conc (fg/sccm),Scattering Mass Conc (fg/sccm),Sheath Flow Set Point,Sample Flow Set Point,Laser Temp Set Point,Laser Current Set Point,Spare 4 Set Point,Spare 5 Set Point,PMT HV Set Point,Particle Density (g/ccm),PbP Packet Time,Scatter Bin 1,Scatter Bin 2,Scatter Bin 3,Scatter Bin 4,Scatter Bin 5,Scatter Bin 6,Scatter Bin 7,Scatter Bin 8,Scatter Bin 9,Scatter Bin 10,Scatter Bin 11,Scatter Bin 12,Scatter Bin 13,Scatter Bin 14,Scatter Bin 15,Scatter Bin 16,Scatter Bin 17,Scatter Bin 18,Scatter Bin 19,Incand Bin 1,Incand Bin 2,Incand Bin 3,Incand Bin 4,Incand Bin 5,Incand Bin 6,Incand Bin 7,Incand Bin 8,Incand Bin 9,Incand Bin 10,Incand Bin 11,Incand Bin 12,Incand Bin 13,Incand Bin 14,Incand Bin 15,Incand Bin 16,Incand Bin 17,Incand Bin 18,Incand Bin 19 diff --git a/meta_files/hk_meta.yaml b/meta_files/hk_meta.yaml deleted file mode 100644 index 34ce115..0000000 --- a/meta_files/hk_meta.yaml +++ /dev/null @@ -1,277 +0,0 @@ -file_type: csv -name: hk_meta -columns: -- name: Time Stamp - type: string -- name: Time (sec) - type: string -- name: Time Stamp (UTC sec) - type: string -- name: Elapsed Time - type: string -- name: Error Code - type: string -- name: Packet Time Stamp - type: string -- name: Laser TEC Temp (C) - type: string -- name: Crystal TEC Temp (C) - type: string -- name: Inlet Air Temp (C) - type: string -- name: Computer Heatsink Temp (C) - type: string -- name: Laser Heatsink Temp (C) - type: string -- name: Outlet Air Temp (C) - type: string -- name: YAG Output Monitor (V) - type: string -- name: Cavity Pressure (hPa) - type: string -- name: Laser Driver Power Monitor (uA) - type: string -- name: Laser Driver Current Limit Monitor (A) - type: string -- name: Laser Driver Current Monitor (A) - type: string -- name: Laser TEC Sense - type: string -- name: Laser Over Temp (On/Off) - type: string -- name: +5V Laser Rail (V) - type: string -- name: ' +5V Rail (V)' - type: string -- name: +12V Rail (V) - type: string -- name: High Voltage (V) - type: string -- name: Battery Temp (C) - type: string -- name: UPS Output (V) - type: string -- name: 12V Iso Rail (V) - type: string -- name: 5V Iso Rail (V) - type: string -- name: 3.3V Iso Rail (V) - type: string -- name: Spare 22 - type: string -- name: Spare 23 - type: string -- name: 408 Board Spare 0 - type: string -- name: 408 Board Spare 1 - type: string -- name: 408 Board Spare 2 - type: string -- name: 408 Board Spare 3 - type: string -- name: 408 Board Spare 4 - type: string -- name: Purge Flow Monitor (sccm) - type: string -- name: System Input Voltage (V) - type: string -- name: Board Temperature (C) - type: string -- name: 408 Board Spare 8 - type: string -- name: 408 Board Spare 9 - type: string -- name: 408 Board Spare 10 - type: string -- name: 408 Board Spare 11 - type: string -- name: 408 Board Spare 12 - type: string -- name: 408 Board Spare 13 - type: string -- name: 408 Board Spare 14 - type: string -- name: 408 Board Spare 15 - type: string -- name: Sheath Flow Controller Read (vccm) - type: string -- name: Sheath Flow Controller Read (sccm) - type: string -- name: Sheath Flow Controller Pressure (psia) - type: string -- name: Sheath Flow Controller Temperature (C) - type: string -- name: Sample Flow Controller Read (vccm) - type: string -- name: Sample Flow Controller Read (sccm) - type: string -- name: Sample Flow Controller Pressure (psia) - type: string -- name: Sample Flow Controller Temperature (C) - type: string -- name: Fan 1 (RPM) - type: string -- name: Fan 2 (RPM) - type: string -- name: Laser Fan (RPM) - type: string -- name: Spare tach - type: string -- name: Threshold Crossing Events - type: string -- name: Dual Qualified Scatter and Incand Particles - type: string -- name: Qualified Scatter Only Particles - type: string -- name: Qualified Incand Only Particles - type: string -- name: Disqualified Due to Scatter Saturation - type: string -- name: Disqualified Due to Scatter Transit Time Min - type: string -- name: Disqualified Due to Scatter Transit Time Max - type: string -- name: Disqualified Due to Scatter FWHM Min - type: string -- name: Disqualified Due to Scatter FWHM Max - type: string -- name: Scatter Inter Part Period Min Violation - type: string -- name: Disqualified Due to Incand Saturation - type: string -- name: Disqualified Due to Incand Transit Time Min - type: string -- name: Disqualified Due to Incand Transit Time Max - type: string -- name: Disqualified Due to Incand FWHM Min - type: string -- name: Disqualified Due to Incand FWHM Max - type: string -- name: Incand Inter Part Period Min Violation - type: string -- name: Baseline Sizer Lo - type: string -- name: Baseline Sizer Hi - type: string -- name: Baseline Incand Lo - type: string -- name: Baseline Incand Hi - type: string -- name: Bandwidth Sizer Hi - type: string -- name: Bandwidth Sizer Lo - type: string -- name: Bandwidth Incand Lo - type: string -- name: Bandwidth Incand Hi - type: string -- name: ABD-0408 HK ADCs min - type: string -- name: ABD-0436 HK ADCs min - type: string -- name: ABD-0408 HK ADCs max - type: string -- name: ABD-0436 HK ADCs max - type: string -- name: Incand Particle Conc (cts/ccm) - type: string -- name: Scattering Particle Conc (cts/ccm) - type: string -- name: Incand Mass Conc (fg/sccm) - type: string -- name: Scattering Mass Conc (fg/sccm) - type: string -- name: Sheath Flow Set Point - type: string -- name: Sample Flow Set Point - type: string -- name: Laser Temp Set Point - type: string -- name: Laser Current Set Point - type: string -- name: Spare 4 Set Point - type: string -- name: Spare 5 Set Point - type: string -- name: PMT HV Set Point - type: string -- name: Particle Density (g/ccm) - type: string -- name: PbP Packet Time - type: string -- name: Scatter Bin 1 - type: string -- name: Scatter Bin 2 - type: string -- name: Scatter Bin 3 - type: string -- name: Scatter Bin 4 - type: string -- name: Scatter Bin 5 - type: string -- name: Scatter Bin 6 - type: string -- name: Scatter Bin 7 - type: string -- name: Scatter Bin 8 - type: string -- name: Scatter Bin 9 - type: string -- name: Scatter Bin 10 - type: string -- name: Scatter Bin 11 - type: string -- name: Scatter Bin 12 - type: string -- name: Scatter Bin 13 - type: string -- name: Scatter Bin 14 - type: string -- name: Scatter Bin 15 - type: string -- name: Scatter Bin 16 - type: string -- name: Scatter Bin 17 - type: string -- name: Scatter Bin 18 - type: string -- name: Scatter Bin 19 - type: string -- name: Incand Bin 1 - type: string -- name: Incand Bin 2 - type: string -- name: Incand Bin 3 - type: string -- name: Incand Bin 4 - type: string -- name: Incand Bin 5 - type: string -- name: Incand Bin 6 - type: string -- name: Incand Bin 7 - type: string -- name: Incand Bin 8 - type: string -- name: Incand Bin 9 - type: string -- name: Incand Bin 10 - type: string -- name: Incand Bin 11 - type: string -- name: Incand Bin 12 - type: string -- name: Incand Bin 13 - type: string -- name: Incand Bin 14 - type: string -- name: Incand Bin 15 - type: string -- name: Incand Bin 16 - type: string -- name: Incand Bin 17 - type: string -- name: Incand Bin 18 - type: string -- name: Incand Bin 19 - type: string diff --git a/meta_files/pbp_meta.csv b/meta_files/pbp_meta.csv deleted file mode 100644 index dbcb564..0000000 --- a/meta_files/pbp_meta.csv +++ /dev/null @@ -1 +0,0 @@ -Time (sec),Packet Time Stamp,Flag,Dropped Records,Record Count,Record Size,Particle Time Stamp,Particle Flags,Scatter relPeak,Scatter Transit Time,Scatter Peak Time,Scatter FWHM,Scatter Size (nm),Incand relPeak,Incand Transit Time,Incand Peak Time,Incand FWHM,Incand Delay,Incand Mass (fg),Reserved diff --git a/meta_files/pbp_meta.yaml b/meta_files/pbp_meta.yaml deleted file mode 100644 index 3b28f9b..0000000 --- a/meta_files/pbp_meta.yaml +++ /dev/null @@ -1,43 +0,0 @@ -file_type: csv -name: pbp_meta -columns: -- name: Time (sec) - type: string -- name: Packet Time Stamp - type: string -- name: Flag - type: string -- name: Dropped Records - type: string -- name: Record Count - type: string -- name: Record Size - type: string -- name: Particle Time Stamp - type: string -- name: Particle Flags - type: string -- name: Scatter relPeak - type: string -- name: Scatter Transit Time - type: string -- name: Scatter Peak Time - type: string -- name: Scatter FWHM - type: string -- name: Scatter Size (nm) - type: string -- name: Incand relPeak - type: string -- name: Incand Transit Time - type: string -- name: Incand Peak Time - type: string -- name: Incand FWHM - type: string -- name: Incand Delay - type: string -- name: Incand Mass (fg) - type: string -- name: Reserved - type: string diff --git a/meta_files/read.py b/meta_files/read.py index 66ae8b6..88748be 100644 --- a/meta_files/read.py +++ b/meta_files/read.py @@ -3,17 +3,6 @@ import yaml import os -def infer_dtype(dtype): - if pd.api.types.is_integer_dtype(dtype): - return "int" - elif pd.api.types.is_float_dtype(dtype): - return "float" - elif pd.api.types.is_datetime64_any_dtype(dtype): - return "datetime" - else: - return "string" - - def load_schema(input_file): ext = os.path.splitext(input_file)[1].lower() @@ -24,7 +13,7 @@ def load_schema(input_file): else: raise ValueError(f"Unsupported file format: {ext}") - schema = {col: infer_dtype(df[col].dtype) for col in df.columns} + schema = {col: str(dtype) for col, dtype in df.dtypes.items()} return schema @@ -41,4 +30,4 @@ def generate_combined_config(pbp_file, hk_file, output_file="config.yaml"): # Example usage: -generate_combined_config("pbp_meta.csv", "hk_meta.csv") +generate_combined_config("pbp_meta.parquet", "hk_meta.parquet")