diff --git a/scripts/sp2xr_pipeline.py b/scripts/sp2xr_pipeline.py index bbbb37a..186820e 100644 --- a/scripts/sp2xr_pipeline.py +++ b/scripts/sp2xr_pipeline.py @@ -389,6 +389,9 @@ def main(): merged_ddf.to_parquet( f"{run_config['output']}/hists_{run_config['dt']}s", partition_on=["date"], + engine="pyarrow", + write_index=True, + write_metadata_file=True, append=True, schema="infer", )