feat: add the possibility to decide the saving partition schema between date or date/hour

This commit is contained in:
2025-09-09 17:09:38 +02:00
parent 29e2351341
commit f437b1c5fe
4 changed files with 18 additions and 4 deletions

View File

@@ -201,7 +201,7 @@ def main():
write_future = ddf_pbp_with_flow.to_parquet(
path=f"{run_config['output']}/pbp_calibrated",
partition_on=["date", "hour"],
partition_on=run_config["saving_schema"],
engine="pyarrow",
write_index=True,
write_metadata_file=True,
@@ -256,7 +256,7 @@ def main():
conc_future = ddf_conc.to_parquet(
f"{run_config['output']}/conc_{run_config['dt']}s",
partition_on=["date", "hour"],
partition_on=run_config["saving_schema"],
engine="pyarrow",
write_index=True,
write_metadata_file=True,