refactor: rename and update schema config generator to support YAML output
This commit is contained in:
@@ -1,159 +1,159 @@
|
||||
pbp_schema:
|
||||
Time (sec): float64
|
||||
Packet Time Stamp: float64
|
||||
Flag: float64
|
||||
Dropped Records: float64
|
||||
Record Count: float64
|
||||
Record Size: float64
|
||||
Particle Time Stamp: float64
|
||||
Particle Flags: float64
|
||||
Scatter relPeak: float64
|
||||
Scatter Transit Time: float64
|
||||
Scatter Peak Time: float64
|
||||
Scatter FWHM: float64
|
||||
Scatter Size (nm): float64
|
||||
Incand relPeak: float64
|
||||
Incand Transit Time: float64
|
||||
Incand Peak Time: float64
|
||||
Incand FWHM: float64
|
||||
Incand Delay: float64
|
||||
Incand Mass (fg): float64
|
||||
Reserved: float64
|
||||
Time (sec): float
|
||||
Packet Time Stamp: float
|
||||
Flag: float
|
||||
Dropped Records: float
|
||||
Record Count: float
|
||||
Record Size: float
|
||||
Particle Time Stamp: float
|
||||
Particle Flags: float
|
||||
Scatter relPeak: float
|
||||
Scatter Transit Time: float
|
||||
Scatter Peak Time: float
|
||||
Scatter FWHM: float
|
||||
Scatter Size (nm): float
|
||||
Incand relPeak: float
|
||||
Incand Transit Time: float
|
||||
Incand Peak Time: float
|
||||
Incand FWHM: float
|
||||
Incand Delay: float
|
||||
Incand Mass (fg): float
|
||||
Reserved: float
|
||||
hk_schema:
|
||||
Time Stamp: datetime64[us]
|
||||
Time (sec): float64
|
||||
Time Stamp (UTC sec): float64
|
||||
Elapsed Time: float64
|
||||
Error Code: float64
|
||||
Packet Time Stamp: float64
|
||||
Laser TEC Temp (C): float64
|
||||
Crystal TEC Temp (C): float64
|
||||
Inlet Air Temp (C): float64
|
||||
Computer Heatsink Temp (C): float64
|
||||
Laser Heatsink Temp (C): float64
|
||||
Outlet Air Temp (C): float64
|
||||
YAG Output Monitor (V): float64
|
||||
Cavity Pressure (hPa): float64
|
||||
Laser Driver Power Monitor (uA): float64
|
||||
Laser Driver Current Limit Monitor (A): float64
|
||||
Laser Driver Current Monitor (A): float64
|
||||
Laser TEC Sense: float64
|
||||
Laser Over Temp (On/Off): float64
|
||||
+5V Laser Rail (V): float64
|
||||
' +5V Rail (V)': float64
|
||||
+12V Rail (V): float64
|
||||
High Voltage (V): float64
|
||||
Battery Temp (C): float64
|
||||
UPS Output (V): float64
|
||||
12V Iso Rail (V): float64
|
||||
5V Iso Rail (V): float64
|
||||
3.3V Iso Rail (V): float64
|
||||
Spare 22: float64
|
||||
Spare 23: float64
|
||||
408 Board Spare 0: float64
|
||||
408 Board Spare 1: float64
|
||||
408 Board Spare 2: float64
|
||||
408 Board Spare 3: float64
|
||||
408 Board Spare 4: float64
|
||||
Purge Flow Monitor (sccm): float64
|
||||
System Input Voltage (V): float64
|
||||
Board Temperature (C): float64
|
||||
408 Board Spare 8: float64
|
||||
408 Board Spare 9: float64
|
||||
408 Board Spare 10: float64
|
||||
408 Board Spare 11: float64
|
||||
408 Board Spare 12: float64
|
||||
408 Board Spare 13: float64
|
||||
408 Board Spare 14: float64
|
||||
408 Board Spare 15: float64
|
||||
Sheath Flow Controller Read (vccm): float64
|
||||
Sheath Flow Controller Read (sccm): float64
|
||||
Sheath Flow Controller Pressure (psia): float64
|
||||
Sheath Flow Controller Temperature (C): float64
|
||||
Sample Flow Controller Read (vccm): float64
|
||||
Sample Flow Controller Read (sccm): float64
|
||||
Sample Flow Controller Pressure (psia): float64
|
||||
Sample Flow Controller Temperature (C): float64
|
||||
Fan 1 (RPM): float64
|
||||
Fan 2 (RPM): float64
|
||||
Laser Fan (RPM): float64
|
||||
Spare tach: float64
|
||||
Threshold Crossing Events: float64
|
||||
Dual Qualified Scatter and Incand Particles: float64
|
||||
Qualified Scatter Only Particles: float64
|
||||
Qualified Incand Only Particles: float64
|
||||
Disqualified Due to Scatter Saturation: float64
|
||||
Disqualified Due to Scatter Transit Time Min: float64
|
||||
Disqualified Due to Scatter Transit Time Max: float64
|
||||
Disqualified Due to Scatter FWHM Min: float64
|
||||
Disqualified Due to Scatter FWHM Max: float64
|
||||
Scatter Inter Part Period Min Violation: float64
|
||||
Disqualified Due to Incand Saturation: float64
|
||||
Disqualified Due to Incand Transit Time Min: float64
|
||||
Disqualified Due to Incand Transit Time Max: float64
|
||||
Disqualified Due to Incand FWHM Min: float64
|
||||
Disqualified Due to Incand FWHM Max: float64
|
||||
Incand Inter Part Period Min Violation: float64
|
||||
Baseline Sizer Lo: float64
|
||||
Baseline Sizer Hi: float64
|
||||
Baseline Incand Lo: float64
|
||||
Baseline Incand Hi: float64
|
||||
Bandwidth Sizer Hi: float64
|
||||
Bandwidth Sizer Lo: float64
|
||||
Bandwidth Incand Lo: float64
|
||||
Bandwidth Incand Hi: float64
|
||||
ABD-0408 HK ADCs min: float64
|
||||
ABD-0436 HK ADCs min: float64
|
||||
ABD-0408 HK ADCs max: float64
|
||||
ABD-0436 HK ADCs max: float64
|
||||
Incand Particle Conc (cts/ccm): float64
|
||||
Scattering Particle Conc (cts/ccm): float64
|
||||
Incand Mass Conc (fg/sccm): float64
|
||||
Scattering Mass Conc (fg/sccm): float64
|
||||
Sheath Flow Set Point: float64
|
||||
Sample Flow Set Point: float64
|
||||
Laser Temp Set Point: float64
|
||||
Laser Current Set Point: float64
|
||||
Spare 4 Set Point: float64
|
||||
Spare 5 Set Point: float64
|
||||
PMT HV Set Point: float64
|
||||
Particle Density (g/ccm): float64
|
||||
PbP Packet Time: float64
|
||||
Scatter Bin 1: float64
|
||||
Scatter Bin 2: float64
|
||||
Scatter Bin 3: float64
|
||||
Scatter Bin 4: float64
|
||||
Scatter Bin 5: float64
|
||||
Scatter Bin 6: float64
|
||||
Scatter Bin 7: float64
|
||||
Scatter Bin 8: float64
|
||||
Scatter Bin 9: float64
|
||||
Scatter Bin 10: float64
|
||||
Scatter Bin 11: float64
|
||||
Scatter Bin 12: float64
|
||||
Scatter Bin 13: float64
|
||||
Scatter Bin 14: float64
|
||||
Scatter Bin 15: float64
|
||||
Scatter Bin 16: float64
|
||||
Scatter Bin 17: float64
|
||||
Scatter Bin 18: float64
|
||||
Scatter Bin 19: float64
|
||||
Incand Bin 1: float64
|
||||
Incand Bin 2: float64
|
||||
Incand Bin 3: float64
|
||||
Incand Bin 4: float64
|
||||
Incand Bin 5: float64
|
||||
Incand Bin 6: float64
|
||||
Incand Bin 7: float64
|
||||
Incand Bin 8: float64
|
||||
Incand Bin 9: float64
|
||||
Incand Bin 10: float64
|
||||
Incand Bin 11: float64
|
||||
Incand Bin 12: float64
|
||||
Incand Bin 13: float64
|
||||
Incand Bin 14: float64
|
||||
Incand Bin 15: float64
|
||||
Incand Bin 16: float64
|
||||
Incand Bin 17: float64
|
||||
Incand Bin 18: float64
|
||||
Incand Bin 19: float64
|
||||
Time Stamp: datetime
|
||||
Time (sec): float
|
||||
Time Stamp (UTC sec): float
|
||||
Elapsed Time: float
|
||||
Error Code: float
|
||||
Packet Time Stamp: float
|
||||
Laser TEC Temp (C): float
|
||||
Crystal TEC Temp (C): float
|
||||
Inlet Air Temp (C): float
|
||||
Computer Heatsink Temp (C): float
|
||||
Laser Heatsink Temp (C): float
|
||||
Outlet Air Temp (C): float
|
||||
YAG Output Monitor (V): float
|
||||
Cavity Pressure (hPa): float
|
||||
Laser Driver Power Monitor (uA): float
|
||||
Laser Driver Current Limit Monitor (A): float
|
||||
Laser Driver Current Monitor (A): float
|
||||
Laser TEC Sense: float
|
||||
Laser Over Temp (On/Off): float
|
||||
+5V Laser Rail (V): float
|
||||
' +5V Rail (V)': float
|
||||
+12V Rail (V): float
|
||||
High Voltage (V): float
|
||||
Battery Temp (C): float
|
||||
UPS Output (V): float
|
||||
12V Iso Rail (V): float
|
||||
5V Iso Rail (V): float
|
||||
3.3V Iso Rail (V): float
|
||||
Spare 22: float
|
||||
Spare 23: float
|
||||
408 Board Spare 0: float
|
||||
408 Board Spare 1: float
|
||||
408 Board Spare 2: float
|
||||
408 Board Spare 3: float
|
||||
408 Board Spare 4: float
|
||||
Purge Flow Monitor (sccm): float
|
||||
System Input Voltage (V): float
|
||||
Board Temperature (C): float
|
||||
408 Board Spare 8: float
|
||||
408 Board Spare 9: float
|
||||
408 Board Spare 10: float
|
||||
408 Board Spare 11: float
|
||||
408 Board Spare 12: float
|
||||
408 Board Spare 13: float
|
||||
408 Board Spare 14: float
|
||||
408 Board Spare 15: float
|
||||
Sheath Flow Controller Read (vccm): float
|
||||
Sheath Flow Controller Read (sccm): float
|
||||
Sheath Flow Controller Pressure (psia): float
|
||||
Sheath Flow Controller Temperature (C): float
|
||||
Sample Flow Controller Read (vccm): float
|
||||
Sample Flow Controller Read (sccm): float
|
||||
Sample Flow Controller Pressure (psia): float
|
||||
Sample Flow Controller Temperature (C): float
|
||||
Fan 1 (RPM): float
|
||||
Fan 2 (RPM): float
|
||||
Laser Fan (RPM): float
|
||||
Spare tach: float
|
||||
Threshold Crossing Events: float
|
||||
Dual Qualified Scatter and Incand Particles: float
|
||||
Qualified Scatter Only Particles: float
|
||||
Qualified Incand Only Particles: float
|
||||
Disqualified Due to Scatter Saturation: float
|
||||
Disqualified Due to Scatter Transit Time Min: float
|
||||
Disqualified Due to Scatter Transit Time Max: float
|
||||
Disqualified Due to Scatter FWHM Min: float
|
||||
Disqualified Due to Scatter FWHM Max: float
|
||||
Scatter Inter Part Period Min Violation: float
|
||||
Disqualified Due to Incand Saturation: float
|
||||
Disqualified Due to Incand Transit Time Min: float
|
||||
Disqualified Due to Incand Transit Time Max: float
|
||||
Disqualified Due to Incand FWHM Min: float
|
||||
Disqualified Due to Incand FWHM Max: float
|
||||
Incand Inter Part Period Min Violation: float
|
||||
Baseline Sizer Lo: float
|
||||
Baseline Sizer Hi: float
|
||||
Baseline Incand Lo: float
|
||||
Baseline Incand Hi: float
|
||||
Bandwidth Sizer Hi: float
|
||||
Bandwidth Sizer Lo: float
|
||||
Bandwidth Incand Lo: float
|
||||
Bandwidth Incand Hi: float
|
||||
ABD-0408 HK ADCs min: float
|
||||
ABD-0436 HK ADCs min: float
|
||||
ABD-0408 HK ADCs max: float
|
||||
ABD-0436 HK ADCs max: float
|
||||
Incand Particle Conc (cts/ccm): float
|
||||
Scattering Particle Conc (cts/ccm): float
|
||||
Incand Mass Conc (fg/sccm): float
|
||||
Scattering Mass Conc (fg/sccm): float
|
||||
Sheath Flow Set Point: float
|
||||
Sample Flow Set Point: float
|
||||
Laser Temp Set Point: float
|
||||
Laser Current Set Point: float
|
||||
Spare 4 Set Point: float
|
||||
Spare 5 Set Point: float
|
||||
PMT HV Set Point: float
|
||||
Particle Density (g/ccm): float
|
||||
PbP Packet Time: float
|
||||
Scatter Bin 1: float
|
||||
Scatter Bin 2: float
|
||||
Scatter Bin 3: float
|
||||
Scatter Bin 4: float
|
||||
Scatter Bin 5: float
|
||||
Scatter Bin 6: float
|
||||
Scatter Bin 7: float
|
||||
Scatter Bin 8: float
|
||||
Scatter Bin 9: float
|
||||
Scatter Bin 10: float
|
||||
Scatter Bin 11: float
|
||||
Scatter Bin 12: float
|
||||
Scatter Bin 13: float
|
||||
Scatter Bin 14: float
|
||||
Scatter Bin 15: float
|
||||
Scatter Bin 16: float
|
||||
Scatter Bin 17: float
|
||||
Scatter Bin 18: float
|
||||
Scatter Bin 19: float
|
||||
Incand Bin 1: float
|
||||
Incand Bin 2: float
|
||||
Incand Bin 3: float
|
||||
Incand Bin 4: float
|
||||
Incand Bin 5: float
|
||||
Incand Bin 6: float
|
||||
Incand Bin 7: float
|
||||
Incand Bin 8: float
|
||||
Incand Bin 9: float
|
||||
Incand Bin 10: float
|
||||
Incand Bin 11: float
|
||||
Incand Bin 12: float
|
||||
Incand Bin 13: float
|
||||
Incand Bin 14: float
|
||||
Incand Bin 15: float
|
||||
Incand Bin 16: float
|
||||
Incand Bin 17: float
|
||||
Incand Bin 18: float
|
||||
Incand Bin 19: float
|
||||
|
||||
@@ -3,6 +3,17 @@ import yaml
|
||||
import os
|
||||
|
||||
|
||||
def infer_general_dtype(dtype):
|
||||
if pd.api.types.is_integer_dtype(dtype):
|
||||
return "int"
|
||||
elif pd.api.types.is_float_dtype(dtype):
|
||||
return "float"
|
||||
elif pd.api.types.is_datetime64_any_dtype(dtype):
|
||||
return "datetime"
|
||||
else:
|
||||
return "string"
|
||||
|
||||
|
||||
def load_schema(input_file):
|
||||
ext = os.path.splitext(input_file)[1].lower()
|
||||
|
||||
@@ -13,7 +24,7 @@ def load_schema(input_file):
|
||||
else:
|
||||
raise ValueError(f"Unsupported file format: {ext}")
|
||||
|
||||
schema = {col: str(dtype) for col, dtype in df.dtypes.items()}
|
||||
schema = {col: infer_general_dtype(dtype) for col, dtype in df.dtypes.items()}
|
||||
return schema
|
||||
|
||||
|
||||
@@ -29,5 +40,5 @@ def generate_combined_config(pbp_file, hk_file, output_file="config.yaml"):
|
||||
print(f"Unified config saved to: {output_file}")
|
||||
|
||||
|
||||
# Example usage:
|
||||
# Example usage
|
||||
generate_combined_config("pbp_meta.parquet", "hk_meta.parquet")
|
||||
Reference in New Issue
Block a user