APOG_public/SP2XR
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: user can now decide frequency of repartition of dask dataframes after being loaded (both hk and pbp)

Browse Source
This commit is contained in:
bertoz_b
2025-09-09 16:03:42 +02:00
parent e946d4ff94
commit 29e2351341
3 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/sp2xr_pipeline.py
View File

@@ -135,7 +135,7 @@ def main():
ddf_hk = ddf_hk.reset_index().set_index( # 'calculated_time' becomes a column
"calculated_time", sorted=False, shuffle="tasks"
) # Dask now infers divisions
ddf_hk = ddf_hk.repartition(freq="1h")
ddf_hk = ddf_hk.repartition(freq=run_config["repartition"])
meta = pd.DataFrame(
{
"Sample Flow Controller Read (sccm)": pd.Series(
@@ -182,7 +182,7 @@ def main():
ddf_raw = ddf_raw.reset_index().set_index( # 'calculated_time' becomes a column
"calculated_time", sorted=False, shuffle="tasks"
) # Dask now infers divisions
ddf_raw = ddf_raw.repartition(freq="1h")
ddf_raw = ddf_raw.repartition(freq=run_config["repartition"])
ddf_cal = calibrate_single_particle(ddf_raw, instr_config, run_config)
dask_objects.append(ddf_cal)