Chore: cleanup old code
This commit is contained in:
@@ -304,180 +304,6 @@ def main():
|
||||
client,
|
||||
)
|
||||
|
||||
"""if run_config["do_BC_hist"]:
|
||||
print("Computing BC distributions...")
|
||||
# --- Mass histogram
|
||||
BC_hist_configs = [
|
||||
{"flag_col": None, "flag_value": None},
|
||||
{"flag_col": "cnts_thin", "flag_value": 1},
|
||||
{"flag_col": "cnts_thin_noScatt", "flag_value": 1},
|
||||
{"flag_col": "cnts_thick", "flag_value": 1},
|
||||
{"flag_col": "cnts_thick_sat", "flag_value": 1},
|
||||
{"flag_col": "cnts_thin_sat", "flag_value": 1},
|
||||
{"flag_col": "cnts_ntl_sat", "flag_value": 1},
|
||||
{"flag_col": "cnts_ntl", "flag_value": 1},
|
||||
{
|
||||
"flag_col": "cnts_extreme_positive_timelag",
|
||||
"flag_value": 1,
|
||||
},
|
||||
{
|
||||
"flag_col": "cnts_thin_low_inc_scatt_ratio",
|
||||
"flag_value": 1,
|
||||
},
|
||||
{"flag_col": "cnts_thin_total", "flag_value": 1},
|
||||
{"flag_col": "cnts_thick_total", "flag_value": 1},
|
||||
{"flag_col": "cnts_unclassified", "flag_value": 1},
|
||||
]
|
||||
|
||||
results = []
|
||||
|
||||
for cfg_hist in BC_hist_configs[:2]:
|
||||
meta_hist = (
|
||||
make_hist_meta(
|
||||
bin_ctrs=inc_mass_bin_ctrs,
|
||||
kind="mass",
|
||||
flag_col=cfg_hist["flag_col"],
|
||||
rho_eff=run_config["rho_eff"],
|
||||
BC_type=run_config["BC_type"],
|
||||
)
|
||||
.astype(DEFAULT_FLOAT, copy=False)
|
||||
.convert_dtypes(dtype_backend="pyarrow")
|
||||
)
|
||||
ddf_out = ddf_pbp_with_flow.map_partitions(
|
||||
process_hist_and_dist_partition,
|
||||
col="BC mass within range",
|
||||
flag_col=cfg_hist["flag_col"],
|
||||
flag_value=cfg_hist["flag_value"],
|
||||
bin_lims=inc_mass_bin_lims,
|
||||
bin_ctrs=inc_mass_bin_ctrs,
|
||||
dt=run_config["dt"],
|
||||
calculate_conc=True,
|
||||
flow=None,
|
||||
rho_eff=run_config["rho_eff"],
|
||||
BC_type=run_config["BC_type"],
|
||||
#t=1,
|
||||
meta=meta_hist,
|
||||
).map_partitions(cast_and_arrow, meta=meta_hist)
|
||||
results.append(ddf_out)
|
||||
|
||||
# --- Scattering histogram
|
||||
if run_config["do_scatt_hist"]:
|
||||
print("Computing scattering distribution...")
|
||||
meta_hist = (
|
||||
make_hist_meta(
|
||||
bin_ctrs=scatt_bin_ctrs,
|
||||
kind="scatt",
|
||||
flag_col=None,
|
||||
rho_eff=None,
|
||||
BC_type=None,
|
||||
)
|
||||
.astype(DEFAULT_FLOAT, copy=False)
|
||||
.convert_dtypes(dtype_backend="pyarrow")
|
||||
)
|
||||
ddf_scatt = ddf_pbp_with_flow.map_partitions(
|
||||
process_hist_and_dist_partition,
|
||||
col="Opt diam scatt only",
|
||||
flag_col=None,
|
||||
flag_value=None,
|
||||
bin_lims=scatt_bin_lims,
|
||||
bin_ctrs=scatt_bin_ctrs,
|
||||
dt=run_config["dt"],
|
||||
calculate_conc=True,
|
||||
flow=None,
|
||||
rho_eff=None,
|
||||
BC_type=None,
|
||||
#t=1,
|
||||
meta=meta_hist,
|
||||
).map_partitions(cast_and_arrow, meta=meta_hist)
|
||||
results.append(ddf_scatt)
|
||||
|
||||
# --- Timelag histogram
|
||||
if run_config["do_timelag_hist"]:
|
||||
print("Computing time delay distribution...")
|
||||
mass_bins = (
|
||||
ddf_pbp_with_flow[["BC mass bin"]]
|
||||
.compute()
|
||||
.astype("Int64")
|
||||
.drop_duplicates()
|
||||
.dropna()
|
||||
)
|
||||
|
||||
for idx, mass_bin in enumerate(mass_bins[:1]):
|
||||
ddf_bin = ddf_pbp_with_flow[
|
||||
ddf_pbp_with_flow["BC mass bin"] == mass_bin
|
||||
]
|
||||
|
||||
name_prefix = f"dNdlogDmev_{inc_mass_bin_ctrs[idx]:.2f}_timelag"
|
||||
|
||||
meta_hist = make_hist_meta(
|
||||
bin_ctrs=timelag_bin_ctrs,
|
||||
kind="timelag",
|
||||
flag_col="cnts_particles_for_tl_dist",
|
||||
name_prefix=name_prefix,
|
||||
rho_eff=None,
|
||||
BC_type=None,
|
||||
)
|
||||
|
||||
tl_ddf = ddf_bin.map_partitions(
|
||||
process_hist_and_dist_partition,
|
||||
col="time_lag",
|
||||
flag_col="cnts_particles_for_tl_dist",
|
||||
flag_value=1,
|
||||
bin_lims=timelag_bins_lims,
|
||||
bin_ctrs=timelag_bin_ctrs,
|
||||
dt=run_config["dt"],
|
||||
calculate_conc=True,
|
||||
flow=None,
|
||||
rho_eff=None,
|
||||
BC_type=None,
|
||||
#t=1,
|
||||
name_prefix=name_prefix,
|
||||
meta=meta_hist,
|
||||
)
|
||||
|
||||
#
|
||||
tl_ddf = tl_ddf.map_partitions(cast_and_arrow, meta=meta_hist)
|
||||
|
||||
results.append(tl_ddf)
|
||||
# --- Merge all hists
|
||||
|
||||
merged_ddf = dd.concat(results, axis=1, interleave_partitions=True)
|
||||
|
||||
idx_target = "datetime64[ns]"
|
||||
merged_ddf = merged_ddf.map_partitions(
|
||||
lambda pdf: pdf.set_index(pdf.index.astype(idx_target, copy=False)),
|
||||
meta=merged_ddf._meta,
|
||||
)
|
||||
|
||||
index_as_dt = dd.to_datetime(merged_ddf.index.to_series())
|
||||
merged_ddf["date"] = index_as_dt.map_partitions(
|
||||
lambda s: s.dt.normalize(), meta=("date", "datetime64[ns]")
|
||||
)
|
||||
|
||||
# --- Save hists to parquet
|
||||
|
||||
delete_partition_if_exists(
|
||||
output_path=f"{run_config['output']}/hists_{run_config['dt']}s",
|
||||
partition_values={
|
||||
"date": chunk_start.strftime("%Y-%m-%d"),
|
||||
"hour": chunk_start.hour,
|
||||
},
|
||||
)
|
||||
merged_ddf.to_parquet(
|
||||
f"{run_config['output']}/hists_{run_config['dt']}s",
|
||||
partition_on=["date"],
|
||||
engine="pyarrow",
|
||||
write_index=True,
|
||||
write_metadata_file=True,
|
||||
append=True,
|
||||
schema="infer",
|
||||
)
|
||||
|
||||
client.cancel([ddf_pbp_with_flow, ddf_hk,
|
||||
ddf_hk_dt, ddf_pbp_dt, ddf_pbp_hk_dt])
|
||||
del ddf_pbp_with_flow
|
||||
client.run(gc.collect) # workers
|
||||
gc.collect() # client"""
|
||||
finally:
|
||||
# Comprehensive cleanup
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user