Update config files

config/complete_example.yaml

@@ -51,7 +51,7 @@ cluster:
 
   # SLURM-specific settings
   walltime: "4-00:00:00"  # Job time limit (days-hours:minutes:seconds)
-  partition: general      # SLURM partition: general, bigmem, gpu, etc.
+  partition: general      # SLURM partition
   log_dir: ./slurm_out    # SLURM log directory
 
 
@@ -65,7 +65,7 @@ chunking:
 
 
 # ============================================================================
-# BLACK CARBON PROPERTIES - Physical and optical parameters
+# BLACK CARBON PROPERTIES
 # ============================================================================
 bc:
   # Density settings

config/conversion_config_template.yaml

@@ -5,7 +5,7 @@
 #
 # USAGE:
 #   1. Update the paths and parameters below for your dataset
-#   2. Run: python scripts/sp2xr_csv2parquet.py --config my_conversion_config.yaml
+#   2. Run: python scripts/sp2xr_csv2parquet.py --config conversion_config_template.yaml
 #
 # NOTES:
 # - For local execution, the script auto-detects available CPU cores and memory
@@ -21,7 +21,7 @@ source_directory: data/SP2XR_orig_files
 target_directory: data/pbp_files_parquet_2
 
 # Path to your data schema config file (generated by sp2xr_generate_config.py)
-schema_config: config/config_schema_with_mapping.yaml
+schema_config: config/new_data_schema_with_mapping.yaml
 
 
 # Pattern to filter which files to process

config/my_data_schema.yaml

@@ -9,16 +9,16 @@
 pbp_schema:
   Time (sec): float
   Packet Time Stamp: float
-  Flag: int
+  Flag: float
-  Dropped Records: int
+  Dropped Records: float
-  Record Count: int
+  Record Count: float
-  Record Size: int
+  Record Size: float
   Particle Time Stamp: float
-  Particle Flags: int
+  Particle Flags: float
   Scatter relPeak: float
-  Scatter Transit Time: int
+  Scatter Transit Time: float
-  Scatter Peak Time: int
+  Scatter Peak Time: float
-  Scatter FWHM: int
+  Scatter FWHM: float
   Scatter Size (nm): float
   Incand relPeak: float
   Incand Transit Time: float
@@ -26,27 +26,27 @@ pbp_schema:
   Incand FWHM: float
   Incand Delay: float
   Incand Mass (fg): float
-  Reserved: int
+  Reserved: float
 hk_schema:
   Time Stamp: string
   Time (sec): float
   Time Stamp (UTC sec): float
   Elapsed Time: float
-  Error Code: int
+  Error Code: float
   Packet Time Stamp: float
   Laser TEC Temp (C): float
-  Crystal TEC Temp (C): int
+  Crystal TEC Temp (C): float
   Inlet Air Temp (C): float
   Computer Heatsink Temp (C): float
   Laser Heatsink Temp (C): float
   Outlet Air Temp (C): float
   YAG Output Monitor (V): float
   Cavity Pressure (hPa): float
-  Laser Driver Power Monitor (uA): int
+  Laser Driver Power Monitor (uA): float
   Laser Driver Current Limit Monitor (A): float
   Laser Driver Current Monitor (A): float
   Laser TEC Sense: float
-  Laser Over Temp (On/Off): int
+  Laser Over Temp (On/Off): float
   +5V Laser Rail (V): float
   ' +5V Rail (V)': float
   +12V Rail (V): float
@@ -56,117 +56,117 @@ hk_schema:
   12V Iso Rail (V): float
   5V Iso Rail (V): float
   3.3V Iso Rail (V): float
-  Spare 22: int
+  Spare 22: float
-  Spare 23: int
+  Spare 23: float
-  408 Board Spare 0: int
+  408 Board Spare 0: float
-  408 Board Spare 1: int
+  408 Board Spare 1: float
-  408 Board Spare 2: int
+  408 Board Spare 2: float
-  408 Board Spare 3: int
+  408 Board Spare 3: float
-  408 Board Spare 4: int
+  408 Board Spare 4: float
   Purge Flow Monitor (sccm): float
   System Input Voltage (V): float
   Board Temperature (C): float
-  408 Board Spare 8: int
+  408 Board Spare 8: float
-  408 Board Spare 9: int
+  408 Board Spare 9: float
-  408 Board Spare 10: int
+  408 Board Spare 10: float
-  408 Board Spare 11: int
+  408 Board Spare 11: float
-  408 Board Spare 12: int
+  408 Board Spare 12: float
-  408 Board Spare 13: int
+  408 Board Spare 13: float
-  408 Board Spare 14: int
+  408 Board Spare 14: float
-  408 Board Spare 15: int
+  408 Board Spare 15: float
-  Sheath Flow Controller Read (vccm): int
+  Sheath Flow Controller Read (vccm): float
-  Sheath Flow Controller Read (sccm): int
+  Sheath Flow Controller Read (sccm): float
   Sheath Flow Controller Pressure (psia): float
   Sheath Flow Controller Temperature (C): float
   Sample Flow Controller Read (vccm): float
   Sample Flow Controller Read (sccm): float
   Sample Flow Controller Pressure (psia): float
   Sample Flow Controller Temperature (C): float
-  Fan 1 (RPM): int
+  Fan 1 (RPM): float
-  Fan 2 (RPM): int
+  Fan 2 (RPM): float
-  Laser Fan (RPM): int
+  Laser Fan (RPM): float
-  Spare tach: int
+  Spare tach: float
-  Threshold Crossing Events: int
+  Threshold Crossing Events: float
-  Dual Qualified Scatter and Incand Particles: int
+  Dual Qualified Scatter and Incand Particles: float
-  Qualified Scatter Only Particles: int
+  Qualified Scatter Only Particles: float
-  Qualified Incand Only Particles: int
+  Qualified Incand Only Particles: float
-  Disqualified Due to Scatter Saturation: int
+  Disqualified Due to Scatter Saturation: float
-  Disqualified Due to Scatter Transit Time Min: int
+  Disqualified Due to Scatter Transit Time Min: float
-  Disqualified Due to Scatter Transit Time Max: int
+  Disqualified Due to Scatter Transit Time Max: float
-  Disqualified Due to Scatter FWHM Min: int
+  Disqualified Due to Scatter FWHM Min: float
-  Disqualified Due to Scatter FWHM Max: int
+  Disqualified Due to Scatter FWHM Max: float
-  Scatter Inter Part Period Min Violation: int
+  Scatter Inter Part Period Min Violation: float
-  Disqualified Due to Incand Saturation: int
+  Disqualified Due to Incand Saturation: float
-  Disqualified Due to Incand Transit Time Min: int
+  Disqualified Due to Incand Transit Time Min: float
-  Disqualified Due to Incand Transit Time Max: int
+  Disqualified Due to Incand Transit Time Max: float
-  Disqualified Due to Incand FWHM Min: int
+  Disqualified Due to Incand FWHM Min: float
-  Disqualified Due to Incand FWHM Max: int
+  Disqualified Due to Incand FWHM Max: float
-  Incand Inter Part Period Min Violation: int
+  Incand Inter Part Period Min Violation: float
-  Baseline Sizer Lo: int
+  Baseline Sizer Lo: float
-  Baseline Sizer Hi: int
+  Baseline Sizer Hi: float
-  Baseline Incand Lo: int
+  Baseline Incand Lo: float
-  Baseline Incand Hi: int
+  Baseline Incand Hi: float
-  Bandwidth Sizer Hi: int
+  Bandwidth Sizer Hi: float
-  Bandwidth Sizer Lo: int
+  Bandwidth Sizer Lo: float
-  Bandwidth Incand Lo: int
+  Bandwidth Incand Lo: float
-  Bandwidth Incand Hi: int
+  Bandwidth Incand Hi: float
-  ABD-0408 HK ADCs min: int
+  ABD-0408 HK ADCs min: float
-  ABD-0436 HK ADCs min: int
+  ABD-0436 HK ADCs min: float
-  ABD-0408 HK ADCs max: int
+  ABD-0408 HK ADCs max: float
-  ABD-0436 HK ADCs max: int
+  ABD-0436 HK ADCs max: float
   Incand Particle Conc (cts/ccm): float
   Scattering Particle Conc (cts/ccm): float
   Incand Mass Conc (fg/sccm): float
   Scattering Mass Conc (fg/sccm): float
-  Sheath Flow Set Point: int
+  Sheath Flow Set Point: float
-  Sample Flow Set Point: int
+  Sample Flow Set Point: float
-  Laser Temp Set Point: int
+  Laser Temp Set Point: float
   Laser Current Set Point: float
-  Spare 4 Set Point: int
+  Spare 4 Set Point: float
-  Spare 5 Set Point: int
+  Spare 5 Set Point: float
   PMT HV Set Point: float
   Particle Density (g/ccm): float
   PbP Packet Time: float
-  Scatter Bin 1: int
+  Scatter Bin 1: float
-  Scatter Bin 2: int
+  Scatter Bin 2: float
-  Scatter Bin 3: int
+  Scatter Bin 3: float
-  Scatter Bin 4: int
+  Scatter Bin 4: float
-  Scatter Bin 5: int
+  Scatter Bin 5: float
-  Scatter Bin 6: int
+  Scatter Bin 6: float
-  Scatter Bin 7: int
+  Scatter Bin 7: float
-  Scatter Bin 8: int
+  Scatter Bin 8: float
-  Scatter Bin 9: int
+  Scatter Bin 9: float
-  Scatter Bin 10: int
+  Scatter Bin 10: float
-  Scatter Bin 11: int
+  Scatter Bin 11: float
-  Scatter Bin 12: int
+  Scatter Bin 12: float
-  Scatter Bin 13: int
+  Scatter Bin 13: float
-  Scatter Bin 14: int
+  Scatter Bin 14: float
-  Scatter Bin 15: int
+  Scatter Bin 15: float
-  Scatter Bin 16: int
+  Scatter Bin 16: float
-  Scatter Bin 17: int
+  Scatter Bin 17: float
-  Scatter Bin 18: int
+  Scatter Bin 18: float
-  Scatter Bin 19: int
+  Scatter Bin 19: float
-  Scatter Bin 20: int
+  Scatter Bin 20: float
-  Incand Bin 1: int
+  Incand Bin 1: float
-  Incand Bin 2: int
+  Incand Bin 2: float
-  Incand Bin 3: int
+  Incand Bin 3: float
-  Incand Bin 4: int
+  Incand Bin 4: float
-  Incand Bin 5: int
+  Incand Bin 5: float
-  Incand Bin 6: int
+  Incand Bin 6: float
-  Incand Bin 7: int
+  Incand Bin 7: float
-  Incand Bin 8: int
+  Incand Bin 8: float
-  Incand Bin 9: int
+  Incand Bin 9: float
-  Incand Bin 10: int
+  Incand Bin 10: float
-  Incand Bin 11: int
+  Incand Bin 11: float
-  Incand Bin 12: int
+  Incand Bin 12: float
-  Incand Bin 13: int
+  Incand Bin 13: float
-  Incand Bin 14: int
+  Incand Bin 14: float
-  Incand Bin 15: int
+  Incand Bin 15: float
-  Incand Bin 16: int
+  Incand Bin 16: float
-  Incand Bin 17: int
+  Incand Bin 17: float
-  Incand Bin 18: int
+  Incand Bin 18: float
-  Incand Bin 19: int
+  Incand Bin 19: float
-  Incand Bin 20: int
+  Incand Bin 20: float
 pbp_canonical_schema:
   Time (sec): float
   Packet Time Stamp: float

scripts/sp2xr_generate_config.py

@@ -152,7 +152,7 @@ def get_canonical_schemas() -> dict[str, dict[str, str]]:
 def generate_basic_config(
     pbp_file: Path,
     hk_file: Path,
-    schema_output: str = "config_schema.yaml",
+    schema_output: str = "new_data_schema.yaml",
     ini_file: str = None,
     instrument_output: str = None,
 ) -> None:
@@ -185,9 +185,7 @@ def generate_basic_config(
         if instrument_output:
             instrument_path = Path(instrument_output)
         else:
-            instrument_path = (
+            instrument_path = schema_path.parent / "new_instrument_settings.yaml"
-                schema_path.parent / f"{schema_path.stem}_instrument_settings.yaml"
-            )
 
         try:
             from sp2xr.helpers import export_xr_ini_to_yaml_with_source
@@ -212,7 +210,7 @@ def generate_basic_config(
 def generate_mapping_config(
     pbp_file: Path,
     hk_file: Path,
-    schema_output: str = "config_schema_with_mapping.yaml",
+    schema_output: str = "new_data_schema_with_mapping.yaml",
     ini_file: str = None,
     instrument_output: str = None,
 ) -> None:
@@ -291,9 +289,7 @@ def generate_mapping_config(
         if instrument_output:
             instrument_path = Path(instrument_output)
         else:
-            instrument_path = (
+            instrument_path = schema_path.parent / "new_instrument_settings.yaml"
-                schema_path.parent / f"{schema_path.stem}_instrument_settings.yaml"
-            )
 
         try:
             from sp2xr.helpers import export_xr_ini_to_yaml_with_source
@@ -346,8 +342,8 @@ Examples:
     parser.add_argument(
         "--schema-output",
         "-s",
-        default="config/config_schema.yaml",
+        default="config/new_data_schema.yaml",
-        help="Output filename for data schema config (default: config/config_schema.yaml)",
+        help="Output filename for data schema config (default: config/new_data_schema.yaml)",
     )
 
     parser.add_argument(
@@ -431,8 +427,8 @@ def main():
         # Generate configuration
         if args.mapping:
             schema_file = (
-                "config/config_schema_with_mapping.yaml"
+                "config/new_data_schema_with_mapping.yaml"
-                if args.schema_output == "config/config_schema.yaml"
+                if args.schema_output == "config/new_data_schema.yaml"
                 else args.schema_output
             )
             generate_mapping_config(