feat: add the possibility to decide the saving partition schema between date or date/hour
This commit is contained in:
@@ -201,7 +201,7 @@ def main():
|
||||
|
||||
write_future = ddf_pbp_with_flow.to_parquet(
|
||||
path=f"{run_config['output']}/pbp_calibrated",
|
||||
partition_on=["date", "hour"],
|
||||
partition_on=run_config["saving_schema"],
|
||||
engine="pyarrow",
|
||||
write_index=True,
|
||||
write_metadata_file=True,
|
||||
@@ -256,7 +256,7 @@ def main():
|
||||
|
||||
conc_future = ddf_conc.to_parquet(
|
||||
f"{run_config['output']}/conc_{run_config['dt']}s",
|
||||
partition_on=["date", "hour"],
|
||||
partition_on=run_config["saving_schema"],
|
||||
engine="pyarrow",
|
||||
write_index=True,
|
||||
write_metadata_file=True,
|
||||
|
||||
Reference in New Issue
Block a user