fix: fix parquet saving of distributions (specify engine, write metadata, ...)
This commit is contained in:
@@ -389,6 +389,9 @@ def main():
|
||||
merged_ddf.to_parquet(
|
||||
f"{run_config['output']}/hists_{run_config['dt']}s",
|
||||
partition_on=["date"],
|
||||
engine="pyarrow",
|
||||
write_index=True,
|
||||
write_metadata_file=True,
|
||||
append=True,
|
||||
schema="infer",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user