{ "cells": [ { "cell_type": "code", "execution_count": 74, "id": "ea414770-9f9a-42f8-a7d8-25ac6983d76c", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import dask.dataframe as dd\n", "\n", "import numpy as np\n", "import yaml\n", "import matplotlib.pyplot as plt\n", "from sp2xr.toolkit_legacy import read_xr_ini_file, bin_lims_to_ctrs, process_hist_and_dist\n", "from sp2xr.calibration import polynomial, powerlaw" ] }, { "cell_type": "markdown", "id": "c80d37dc-f526-426c-8083-a78e791821f9", "metadata": {}, "source": [ "# Old processing" ] }, { "cell_type": "code", "execution_count": 27, "id": "5dd8ee52-9032-4f3b-8327-a0561ac3576e", "metadata": {}, "outputs": [], "source": [ "new_config_run = yaml.safe_load(open('tests/run_config.yaml'))" ] }, { "cell_type": "code", "execution_count": 28, "id": "8d7cc57e-5b88-4388-9303-7777bad73e7f", "metadata": {}, "outputs": [], "source": [ "dir_path_pbp = '/data/user/bertoz_b/merlin6data/SP2XR_code/tests/data/pbp_files_test/date=2019-05-07'\n", "dir_path_hk = '/data/user/bertoz_b/merlin6data/SP2XR_code/tests/data/hk_files_test/date=2019-05-07'\n", "\n", "config_path = '/data/user/bertoz_b/merlin6data/SP2XR_code/meta_files/20190417210227 Calibration 20181005.ini'" ] }, { "cell_type": "code", "execution_count": 29, "id": "0225b191-7066-4840-8acb-ecac239d66d0", "metadata": {}, "outputs": [], "source": [ "dt=1\n", "rho_eff=1800\n", "BC_type=\"constant_effective_density\"\n", "inc_calib_curve=new_config_run['calibration']['incandescence']['curve_type']\n", "inc_calib_params=new_config_run['calibration']['incandescence']['parameters']\n", "scatt_calib_curve=new_config_run['calibration']['scattering']['curve_type']\n", "scatt_calib_params=new_config_run['calibration']['scattering']['parameters']\n", "config_file_dir=config_path\n", "minM=0.3\n", "maxM=400\n", "n_incbins=50\n", "minOptD=100\n", "maxOptD=500\n", "n_scattbins=20\n", "minTL=-10\n", "maxTL=400\n", "n_timelag=100" ] }, { "cell_type": "code", "execution_count": 38, "id": "e01462f6-3f2f-44f5-967a-6e63fd2df3c1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'polynomial'" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inc_calib_curve" ] }, { "cell_type": "code", "execution_count": 39, "id": "c29a6acb-581b-415f-8f41-e5a7447654e9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "polynomial curve applied\n" ] } ], "source": [ "read_dir_pbp = dir_path_pbp\n", "read_dir_hk = dir_path_hk\n", "ddf_pbp = pd.read_parquet(\n", " read_dir_pbp\n", ") # , calculate_divisions=True)#.repartition(freq='1h')\n", "ddf_hk = pd.read_parquet(\n", " read_dir_hk\n", ") # , calculate_divisions=True)#.repartition(freq='1h')\n", "\n", "ini_params = read_xr_ini_file(config_path)\n", "ini_params[\"IncSatPoint\"] = (\n", " 1.7e9 # 2e9 # Guess for now, need to see where to obtain this from. 2e9 is probably better guess\n", ")\n", "ini_params[\"ScattSatPoint\"] = (\n", " 1.7e9 # Guess for now, need to see where to obtain this from. 1.85e9 is probably better\n", ")\n", "ini_params[\"DelayTimeThresh\"] = 200 # Also a guess for now, to be updated\n", "\n", "inc_mass_bin_lims = np.logspace(np.log10(minM), np.log10(maxM), n_incbins)\n", "inc_mass_bin_lims_for_pd_cut = np.logspace(\n", " np.log10(minM), np.log10(maxM + 0.01), n_incbins\n", ")\n", "inc_mass_bin_ctrs = bin_lims_to_ctrs(inc_mass_bin_lims)\n", "\n", "dt_str = f\"{dt}s\"\n", "\n", "# ***** ***** *****\n", "# Apply calibration\n", "# ***** ***** *****\n", "\n", "if inc_calib_curve:\n", " if inc_calib_curve == \"polynomial\":\n", " print('polynomial curve applied')\n", " ddf_pbp[\"BC mass\"] = ddf_pbp[\"Incand relPeak\"].apply(\n", " lambda x: polynomial(x, *inc_calib_params)\n", " ) # , meta=('BC mass', 'float64'))\n", " # minM_timelag = polynomial(5e6, *inc_calib_params)\n", " elif inc_calib_curve == \"powerlaw\":\n", " ddf_pbp[\"BC mass\"] = ddf_pbp[\"Incand relPeak\"].apply(\n", " lambda x: powerlaw(x, inc_calib_params)\n", " ) # , meta=('BC mass', 'float64'))\n", " # minM_timelag = powerlaw(5e6, inc_calib_params)\n", "else:\n", " ddf_pbp[\"BC mass\"] = ddf_pbp[\"Incand Mass (fg)\"]\n", " # minM_timelag = ddf_pbp.loc[\n", " # (ddf_pbp[\"Incand relPeak\"] <= 5.05e6)\n", " # & (ddf_pbp[\"Incand relPeak\"] >= 4.95e6),\n", " # \"Incand Mass (fg)\",\n", " # ].mean() # this could create problems if there are no incandescence signals in the selected range\n", "\n", "ddf_pbp.loc[ddf_pbp[\"Incand relPeak\"] == 0, \"BC mass\"] = np.nan\n", "\n", "ddf_pbp[\"BC mass bin\"], mass_bins = pd.cut(\n", " ddf_pbp[\"BC mass\"],\n", " bins=inc_mass_bin_lims_for_pd_cut,\n", " right=False,\n", " include_lowest=True,\n", " retbins=True,\n", ")\n", "\n", "if scatt_calib_curve:\n", " if scatt_calib_curve == \"polynomial\":\n", " ddf_pbp[\"Opt diam\"] = ddf_pbp[\"Scatter relPeak\"].apply(\n", " lambda x: polynomial(x, *scatt_calib_params)\n", " ) # , meta=('Opt diam', 'float64'))\n", " elif scatt_calib_curve == \"powerlaw\":\n", " ddf_pbp[\"Opt diam\"] = ddf_pbp[\"Scatter relPeak\"].apply(\n", " lambda x: powerlaw(x, *scatt_calib_params)\n", " ) # , meta=('Opt diam', 'float64'))\n", "else:\n", " ddf_pbp[\"Opt diam\"] = ddf_pbp[\"Scatter Size (nm)\"]\n", "\n", "ddf_pbp.loc[ddf_pbp[\"Scatter relPeak\"] == 0, \"Opt diam\"] = np.nan\n", "\n", "ddf_pbp[\"time_lag_new\"] = ddf_pbp[\"Incand Peak Time\"] - ddf_pbp[\"Scatter Peak Time\"]" ] }, { "cell_type": "code", "execution_count": 48, "id": "6bb7dc88-4fa7-4f4e-a86a-1b5fd4eba1bf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/data/user/bertoz_b/merlin6data/SP2XR_code/.venv/lib/python3.9/site-packages/pandas/core/arraylike.py:399: RuntimeWarning: divide by zero encountered in log10\n", " result = getattr(ufunc, method)(*inputs, **kwargs)\n" ] } ], "source": [ "# ***** ***** ***** *****\n", "# Set flags\n", "# ***** ***** ***** *****\n", "flag_inc_transit_time = (\n", " ddf_pbp[\"Incand Transit Time\"] >= ini_params[\"IncTransitMin\"]\n", ") & (ddf_pbp[\"Incand Transit Time\"] <= ini_params[\"IncTransitMax\"])\n", "flag_inc_fwhm = (ddf_pbp[\"Incand FWHM\"] >= ini_params[\"IncFWHMMin\"]) & (\n", " ddf_pbp[\"Incand FWHM\"] <= ini_params[\"IncFWHMMax\"]\n", ")\n", "# flag_inc_not_sat = ddf_pbp[\"Incand relPeak\"] < ini_params[\"IncSatPoint\"]\n", "\n", "flag_scatt_transit_time = (\n", " ddf_pbp[\"Scatter Transit Time\"] >= ini_params[\"ScattTransitMin\"]\n", ") & (ddf_pbp[\"Scatter Transit Time\"] <= ini_params[\"ScattTransitMax\"])\n", "flag_scatt_fwhm = (ddf_pbp[\"Scatter FWHM\"] >= ini_params[\"ScattFWHMMin\"]) & (\n", " ddf_pbp[\"Scatter FWHM\"] <= ini_params[\"ScattFWHMMax\"]\n", ")\n", "flag_scatt_not_sat = ddf_pbp[\"Scatter relPeak\"] < ini_params[\"ScattSatPoint\"]\n", "\n", "flag_inc = flag_inc_transit_time & flag_inc_fwhm\n", "flag_inc_in_range = (\n", " flag_inc & (ddf_pbp[\"BC mass\"] >= minM) & (ddf_pbp[\"BC mass\"] <= maxM)\n", ")\n", "# flag_inc_in_range_tl_analysis = (\n", "# flag_inc & (ddf_pbp[\"BC mass\"] >= minM_timelag) & (ddf_pbp[\"BC mass\"] <= maxM)\n", "# )\n", "\n", "flag_scatt = flag_scatt_transit_time & flag_scatt_fwhm\n", "flag_scatt_in_range = (\n", " flag_scatt & (ddf_pbp[\"Opt diam\"] >= minOptD) & (ddf_pbp[\"Opt diam\"] <= maxOptD)\n", ")\n", "\n", "flag_negative_timelag = ddf_pbp[\"time_lag_new\"] < -10\n", "flag_extreme_positive_timelag = ddf_pbp[\"time_lag_new\"] >= 400\n", "flag_timelag_0_50 = (ddf_pbp[\"time_lag_new\"] < 50) & (\n", " ddf_pbp[\"time_lag_new\"] >= -10\n", ")\n", "flag_timelag_greater_50 = (ddf_pbp[\"time_lag_new\"] >= 50) & (\n", " ddf_pbp[\"time_lag_new\"] < 400\n", ")\n", "\n", "ddf_pbp[\"ratio_inc_scatt\"] = np.log10(ddf_pbp[\"Incand relPeak\"]) / np.log10(\n", " ddf_pbp[\"Scatter relPeak\"]\n", ")\n", "flag_low_ratio_inc_scatt = ddf_pbp[\"ratio_inc_scatt\"] < 1.1\n", "\n", "# ***** ***** ***** ***** ***** ***** *****\n", "# Create column for purely scattering particle analysis\n", "# ***** ***** ***** ***** ***** ***** *****\n", "ddf_pbp[\"Opt diam scatt only\"] = ddf_pbp[\"Opt diam\"].where(\n", " flag_scatt & ~flag_inc, np.nan\n", ") # ddf_pbp['Opt diam'].mask(ddf_pbp['BC mass'].notnull(), np.nan)\n", "\n", "# ***** ***** *****\n", "# Set flag for thin/thick\n", "# ***** ***** *****\n", "\n", "ddf_pbp[\"cnts_thin\"] = 0\n", "ddf_pbp[\"cnts_thin_noScatt\"] = 0\n", "ddf_pbp[\"cnts_thick\"] = 0\n", "ddf_pbp[\"cnts_thick_sat\"] = 0\n", "ddf_pbp[\"cnts_thin_sat\"] = 0\n", "ddf_pbp[\"cnts_ntl_sat\"] = 0\n", "ddf_pbp[\"cnts_ntl\"] = 0\n", "ddf_pbp[\"cnts_extreme_positive_timelag\"] = 0\n", "ddf_pbp[\"cnts_thin_low_inc_scatt_ratio\"] = 0\n", "ddf_pbp[\"cnts_particles_for_tl_dist\"] = (\n", " 0 # this flag is used for the calculation of time lag distributions below (it includes the particles classfified as \"thin\" or \"thick\")\n", ")\n", "\n", "ddf_pbp.loc[~flag_scatt & flag_inc_in_range, \"cnts_thin_noScatt\"] = 1\n", "ddf_pbp.loc[\n", " flag_scatt\n", " & flag_scatt_not_sat\n", " & flag_inc_in_range\n", " & flag_timelag_0_50\n", " & ~flag_low_ratio_inc_scatt,\n", " \"cnts_thin\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt\n", " & flag_scatt_not_sat\n", " & flag_inc_in_range\n", " & flag_timelag_0_50\n", " & flag_low_ratio_inc_scatt,\n", " \"cnts_thin_low_inc_scatt_ratio\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt\n", " & flag_scatt_not_sat\n", " & flag_inc_in_range\n", " & flag_timelag_greater_50\n", " & ~flag_extreme_positive_timelag,\n", " \"cnts_thick\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt\n", " & ~flag_scatt_not_sat\n", " & flag_inc_in_range\n", " & flag_timelag_greater_50\n", " & ~flag_extreme_positive_timelag,\n", " \"cnts_thick_sat\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt & ~flag_scatt_not_sat & flag_inc_in_range & flag_timelag_0_50,\n", " \"cnts_thin_sat\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt & ~flag_scatt_not_sat & flag_inc_in_range & flag_negative_timelag,\n", " \"cnts_ntl_sat\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt & flag_scatt_not_sat & flag_inc_in_range & flag_negative_timelag,\n", " \"cnts_ntl\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt & flag_inc_in_range & flag_extreme_positive_timelag,\n", " \"cnts_extreme_positive_timelag\",\n", "] = 1\n", "ddf_pbp.loc[\n", " flag_scatt\n", " & flag_scatt_not_sat\n", " & flag_inc_in_range\n", " & (\n", " (flag_timelag_0_50 & ~flag_low_ratio_inc_scatt)\n", " | (flag_timelag_greater_50 & ~flag_extreme_positive_timelag)\n", " ),\n", " \"cnts_particles_for_tl_dist\",\n", "] = 1\n", "\n", "ddf_pbp[\"cnts_thin_total\"] = ddf_pbp[\"cnts_thin\"] + ddf_pbp[\"cnts_thin_noScatt\"]\n", "ddf_pbp[\"cnts_thick_total\"] = (\n", " ddf_pbp[\"cnts_thick\"]\n", " + ddf_pbp[\"cnts_thick_sat\"]\n", " + ddf_pbp[\"cnts_ntl_sat\"]\n", " + ddf_pbp[\"cnts_ntl\"]\n", " + ddf_pbp[\"cnts_thin_sat\"]\n", ")\n", "ddf_pbp[\"cnts_unclassified\"] = (\n", " ddf_pbp[\"cnts_extreme_positive_timelag\"]\n", " + ddf_pbp[\"cnts_thin_low_inc_scatt_ratio\"]\n", ")\n", "\n", "ddf_pbp[\"temporary_col\"] = (\n", " 1 # this is to keep track of time stamps that are not originally in the file but are added when i do the resampling\n", ")\n", "\n", "# Create two new columns where the particles out of range are changed to np.nan (!! do not change them to 0, otherwise below where we resample and count, the 0s would be counted being a not null value!!)\n", "ddf_pbp[\"BC mass within range\"] = ddf_pbp[\"BC mass\"].where(\n", " flag_inc_in_range, np.nan\n", ") # ddf_pbp['BC mass'].mask(mask_BC_within_range)\n", "ddf_pbp[\"Opt diam scatt only within range\"] = ddf_pbp[\"Opt diam scatt only\"].where(\n", " flag_scatt_in_range & ~flag_inc, np.nan\n", ")\n", "\n", "ddf_pbp_1s = (\n", " ddf_pbp[\n", " [\n", " \"Dropped Records\",\n", " \"Incand Mass (fg)\", # This is the original column from the file, the calibration applied is the one uploaded in the software\n", " \"BC mass\", # If calibration parameters are provided, this is the mass calculated form those parameters. If calib param are not provided this is equal to 'Incand Mass (fg)'\n", " \"BC mass within range\", # In this column only the masses within the range specified by the user are considered\n", " \"cnts_thin\",\n", " \"cnts_thin_noScatt\",\n", " \"cnts_thick\",\n", " \"cnts_thick_sat\",\n", " \"cnts_thin_sat\",\n", " \"cnts_ntl_sat\",\n", " \"cnts_ntl\",\n", " \"cnts_extreme_positive_timelag\",\n", " \"cnts_thin_low_inc_scatt_ratio\",\n", " \"cnts_thin_total\",\n", " \"cnts_thick_total\",\n", " \"cnts_unclassified\",\n", " ]\n", " ]\n", " .resample(dt_str)\n", " .sum()\n", ")\n", "\n", "ddf_pbp_1s[\n", " [\n", " \"BC numb from file\", # number of BC particles counted from the column 'Incand Mass (fg)'\n", " \"BC numb\", # number of BC particles counted from the column 'BC mass'\n", " \"BC numb within range\", # number of BC particles counted from the column 'BC mass within range'\n", " \"scatter numb from file\", # number of scattering particles counted from the column 'Scatter Size (nm)'\n", " \"Scatt numb\", # number of purely scattering particles counted from the column 'Opt diam'\n", " \"Scatt numb within range\", # number of purely scattering particles counted from the column 'Opt diam within range'\n", " \"original_idx\",\n", " ]\n", "] = (\n", " ddf_pbp[\n", " [\n", " \"Incand Mass (fg)\",\n", " \"BC mass\",\n", " \"BC mass within range\",\n", " \"Scatter Size (nm)\",\n", " \"Opt diam scatt only\",\n", " \"Opt diam scatt only within range\",\n", " \"temporary_col\",\n", " ]\n", " ]\n", " .resample(dt_str)\n", " .count()\n", ")\n", "\n", "ddf_pbp_1s[[\"Secs_2GB_mean\"]] = ddf_pbp[[\"Secs_2GB\"]].resample(dt_str).mean()\n", "\n", "# ddf_pbp_1s = ddf_pbp_1s.map_partitions(add_date_column)\n", "ddf_pbp_1s[\"date\"] = ddf_pbp_1s.index.normalize()\n", "ddf_pbp_1s[\"hour\"] = ddf_pbp_1s.index.hour\n", "\n", "ddf_pbp_1s = ddf_pbp_1s[ddf_pbp_1s[\"original_idx\"] != 0]\n", "ddf_pbp_1s = ddf_pbp_1s.drop(\"original_idx\", axis=1) # .persist()\n", "\n", "# **** **** **** **** **** **** ****\n", "# Create a 1s ddf for the hk file\n", "# **** **** **** **** **** **** ****\n", "\n", "ddf_hk[\"temporary_col\"] = 1\n", "\n", "ddf_hk_1s = (\n", " ddf_hk[\n", " [\"Sample Flow Controller Read (sccm)\", \"Sample Flow Controller Read (vccm)\"]\n", " ]\n", " .resample(dt_str)\n", " .mean()\n", ") # do i need a fill na here?\n", "ddf_hk_1s[[\"original_idx\"]] = ddf_hk[[\"temporary_col\"]].resample(dt_str).count()\n", "\n", "ddf_hk_1s = ddf_hk_1s[ddf_hk_1s[\"original_idx\"] != 0]\n", "ddf_hk_1s = ddf_hk_1s.drop(\"original_idx\", axis=1) # .persist()\n", "\n", "# **** **** **** **** **** **** ****\n", "# Create a 1s join ddf for the pbp_1s and hk_1s data\n", "# **** **** **** **** **** **** ****\n", "\n", "ddf_pbp_hk = pd.merge(\n", " ddf_pbp_1s, ddf_hk_1s, how=\"left\", left_index=True, right_index=True\n", ")\n", "# ddf_pbp_hk = ddf_pbp_hk.map_partitions(calculate_concentrations, t=dt)\n", "ddf_pbp_hk[\"BC_massConc_std\"] = (\n", " ddf_pbp_hk[\"BC mass\"]\n", " * 1e-9\n", " / (ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60) * 1e-6)\n", ")\n", "ddf_pbp_hk[\"BC_massConc_vol\"] = (\n", " ddf_pbp_hk[\"BC mass\"]\n", " * 1e-9\n", " / (ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60) * 1e-6)\n", ")\n", "\n", "ddf_pbp_hk[\"BC_numConc_std\"] = ddf_pbp_hk[\"BC numb\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60)\n", ")\n", "ddf_pbp_hk[\"BC_numConc_vol\"] = ddf_pbp_hk[\"BC numb\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60)\n", ")\n", "\n", "ddf_pbp_hk[\"BC_massConc_within_range_std\"] = (\n", " ddf_pbp_hk[\"BC mass within range\"]\n", " * 1e-9\n", " / (ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60) * 1e-6)\n", ")\n", "ddf_pbp_hk[\"BC_massConc_within_range_vol\"] = (\n", " ddf_pbp_hk[\"BC mass within range\"]\n", " * 1e-9\n", " / (ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60) * 1e-6)\n", ")\n", "\n", "ddf_pbp_hk[\"BC_numConc_within_range_std\"] = ddf_pbp_hk[\"BC numb within range\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60)\n", ")\n", "ddf_pbp_hk[\"BC_numConc_within_range_vol\"] = ddf_pbp_hk[\"BC numb within range\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60)\n", ")\n", "\n", "ddf_pbp_hk[\"S_numConc_std\"] = ddf_pbp_hk[\"Scatt numb\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60)\n", ")\n", "ddf_pbp_hk[\"S_numConc_vol\"] = ddf_pbp_hk[\"Scatt numb\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60)\n", ")\n", "\n", "ddf_pbp_hk[\"S_numConc_within_range_std\"] = ddf_pbp_hk[\"Scatt numb within range\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (sccm)\"] * (dt / 60)\n", ")\n", "ddf_pbp_hk[\"S_numConc_within_range_vol\"] = ddf_pbp_hk[\"Scatt numb within range\"] / (\n", " ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"] * (dt / 60)\n", ")" ] }, { "cell_type": "code", "execution_count": 75, "id": "807231f0-9925-4f41-8dec-e17ffa62dd93", "metadata": {}, "outputs": [], "source": [ "# Calculate histograms of different classifications/flags:\n", "ddf_pbp[\"temporary_col\"] = 1\n", "\n", "dNdlogDmev, dMdlogDmev = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " None,\n", " None,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thin, dMdlogDmev_thin = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thin\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thin_noScatt, dMdlogDmev_thin_noScatt = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thin_noScatt\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thick, dMdlogDmev_thick = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thick\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thick_sat, dMdlogDmev_thick_sat = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thick_sat\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thin_sat, dMdlogDmev_thin_sat = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thin_sat\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_ntl_sat, dMdlogDmev_ntl_sat = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_ntl_sat\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_ntl, dMdlogDmev_ntl = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_ntl\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_extreme_positive_timelag, dMdlogDmev_extreme_positive_timelag = (\n", " process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_extreme_positive_timelag\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", " )\n", ")\n", "dNdlogDmev_thin_low_inc_scatt_ratio, dMdlogDmev_thin_low_inc_scatt_ratio = (\n", " process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thin_low_inc_scatt_ratio\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", " )\n", ")\n", "dNdlogDmev_thin_total, dMdlogDmev_thin_total = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thin_total\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_thick_total, dMdlogDmev_thick_total = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_thick_total\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "dNdlogDmev_unclassified, dMdlogDmev_unclassified = process_hist_and_dist(\n", " ddf_pbp,\n", " \"BC mass within range\",\n", " \"cnts_unclassified\",\n", " 1,\n", " inc_mass_bin_lims,\n", " inc_mass_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=rho_eff,\n", " BC_type=BC_type,\n", " t=dt,\n", ")\n", "\n", "scatt_bin_lims = np.logspace(np.log10(minOptD), np.log10(maxOptD), n_scattbins)\n", "scatt_bin_ctrs = bin_lims_to_ctrs(scatt_bin_lims)\n", "\n", "dNdlogDsc, _ = process_hist_and_dist(\n", " ddf_pbp,\n", " \"Opt diam scatt only\",\n", " None,\n", " None,\n", " scatt_bin_lims,\n", " scatt_bin_ctrs,\n", " dt_str,\n", " flow=ddf_pbp_hk[\"Sample Flow Controller Read (vccm)\"],\n", " rho_eff=None,\n", " BC_type=None,\n", " t=dt,\n", ")\n", "\n", "timelag_bins_lims = np.linspace(minTL, maxTL, n_timelag)\n", "timelag_bin_ctrs = bin_lims_to_ctrs(timelag_bins_lims)" ] }, { "cell_type": "code", "execution_count": 83, "id": "c51a8764-cf93-4150-baac-ec76e77064ad", "metadata": {}, "outputs": [], "source": [ "final_df = pd.concat(\n", " [\n", " ddf_pbp_hk,\n", " dNdlogDmev,\n", " dMdlogDmev,\n", " dNdlogDmev_thin,\n", " dMdlogDmev_thin,\n", " dNdlogDmev_thin_noScatt,\n", " dMdlogDmev_thin_noScatt,\n", " dNdlogDmev_thick,\n", " dMdlogDmev_thick,\n", " dNdlogDmev_thick_sat,\n", " dMdlogDmev_thick_sat,\n", " dNdlogDmev_thin_sat,\n", " dMdlogDmev_thin_sat,\n", " dNdlogDmev_ntl_sat,\n", " dMdlogDmev_ntl_sat,\n", " dNdlogDmev_ntl,\n", " dMdlogDmev_ntl,\n", " dNdlogDmev_extreme_positive_timelag,\n", " dMdlogDmev_extreme_positive_timelag,\n", " dNdlogDmev_thin_low_inc_scatt_ratio,\n", " dMdlogDmev_thin_low_inc_scatt_ratio,\n", " dNdlogDmev_thin_total,\n", " dMdlogDmev_thin_total,\n", " dNdlogDmev_thick_total,\n", " dMdlogDmev_thick_total,\n", " dNdlogDmev_unclassified,\n", " dMdlogDmev_unclassified,\n", " dNdlogDsc,\n", " ],\n", " axis=1,\n", " )\n" ] }, { "cell_type": "code", "execution_count": 65, "id": "3fd14901-a2b5-4e81-801c-1c2593c9c627", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['Dropped Records', 'Incand Mass (fg)', 'BC mass',\n", " 'BC mass within range', 'cnts_thin', 'cnts_thin_noScatt', 'cnts_thick',\n", " 'cnts_thick_sat', 'cnts_thin_sat', 'cnts_ntl_sat', 'cnts_ntl',\n", " 'cnts_extreme_positive_timelag', 'cnts_thin_low_inc_scatt_ratio',\n", " 'cnts_thin_total', 'cnts_thick_total', 'cnts_unclassified',\n", " 'BC numb from file', 'BC numb', 'BC numb within range',\n", " 'scatter numb from file', 'Scatt numb', 'Scatt numb within range',\n", " 'Secs_2GB_mean', 'date', 'hour', 'Sample Flow Controller Read (sccm)',\n", " 'Sample Flow Controller Read (vccm)', 'BC_massConc_std',\n", " 'BC_massConc_vol', 'BC_numConc_std', 'BC_numConc_vol',\n", " 'BC_massConc_within_range_std', 'BC_massConc_within_range_vol',\n", " 'BC_numConc_within_range_std', 'BC_numConc_within_range_vol',\n", " 'S_numConc_std', 'S_numConc_vol', 'S_numConc_within_range_std',\n", " 'S_numConc_within_range_vol'],\n", " dtype='object')" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ddf_pbp_hk.columns" ] }, { "cell_type": "markdown", "id": "59d7f603-6a83-402b-98c1-e8e279a19167", "metadata": {}, "source": [ "# New code" ] }, { "cell_type": "code", "execution_count": 8, "id": "ac398f9f-e0a9-4798-b6d9-99d61ab4bb62", "metadata": {}, "outputs": [], "source": [ "from sp2xr.helpers import load_yaml_cfg" ] }, { "cell_type": "code", "execution_count": 6, "id": "859a5acc-8038-4cf3-bc5d-256a2539042e", "metadata": { "jupyter": { "source_hidden": true } }, "outputs": [], "source": [ "def apply_sets(cfg, sets):\n", " def coerce(v):\n", " if v.lower() in (\"true\", \"false\"):\n", " return v.lower() == \"true\"\n", " try:\n", " if \".\" in v:\n", " return float(v)\n", " return int(v)\n", " except (ValueError, TypeError):\n", " return v\n", "\n", " for item in sets:\n", " key, _, val = item.partition(\"=\")\n", " if not _:\n", " raise ValueError(f\"--set needs key=value, got: {item}\")\n", " parts = key.split(\".\")\n", " cur = cfg\n", " for p in parts[:-1]:\n", " cur = cur.setdefault(p, {})\n", " cur[parts[-1]] = coerce(val)\n", " return cfg\n", "\n", "\n", "def get(cfg, dotted, default=None):\n", " cur = cfg\n", " for part in dotted.split(\".\"):\n", " if not isinstance(cur, dict) or part not in cur:\n", " return default\n", " cur = cur[part]\n", " return cur\n", "\n", "\n", "def choose(cli_val, cfg, dotted, default=None):\n", " return cli_val if cli_val is not None else get(cfg, dotted, default)" ] }, { "cell_type": "code", "execution_count": 36, "id": "4d65613e-50a5-4a12-a4b3-06bf1f1852d6", "metadata": {}, "outputs": [], "source": [ "base = load_yaml_cfg(\"tests/run_config.yaml\")\n", "a = get(base, \"calibration.incandescence.curve_type\", None)" ] }, { "cell_type": "code", "execution_count": 48, "id": "7701756a-ed84-4ee6-b2e1-64a98c8d646b", "metadata": {}, "outputs": [], "source": [ "pbp_raw = dd.read_parquet('/data/user/bertoz_b/merlin6data/SP2XR_code/tests/data/pbp_files_test/date=2019-05-07')" ] }, { "cell_type": "code", "execution_count": 114, "id": "707f20e0-1f79-4d68-b632-f887afb7994a", "metadata": {}, "outputs": [], "source": [ "pbp_calib = pd.read_parquet('/data/user/bertoz_b/merlin6data/SP2XR_code/tests/data3/pbp_calibrated/date=2019-05-07 00:00:00')" ] }, { "cell_type": "code", "execution_count": 115, "id": "ed06bd1c-2dae-46b8-b08f-c0507fa00674", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | Time (sec) | \n", "Packet Time Stamp | \n", "Flag | \n", "Dropped Records | \n", "Record Count | \n", "Record Size | \n", "Particle Time Stamp | \n", "Particle Flags | \n", "Scatter relPeak | \n", "Scatter Transit Time | \n", "... | \n", "cnts_particles_for_tl_dist | \n", "cnts_thin_total | \n", "cnts_thick_total | \n", "cnts_unclassified | \n", "BC mass within range | \n", "Opt diam scatt only within range | \n", "temporary_col | \n", "Sample Flow Controller Read (vccm) | \n", "BC mass bin | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-07 00:00:00 | \n", "2.246402e+06 | \n", "2246430.75 | \n", "0.0 | \n", "0.0 | \n", "47.0 | \n", "32.0 | \n", "2246429.75 | \n", "1.0 | \n", "8814524.0 | \n", "1157.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "255.719130 | \n", "1 | \n", "33.400002 | \n", "NaN | \n", "0 | \n", "
| 2019-05-07 00:00:00 | \n", "2.246402e+06 | \n", "2246430.75 | \n", "0.0 | \n", "0.0 | \n", "47.0 | \n", "32.0 | \n", "2246429.75 | \n", "1.0 | \n", "58048.0 | \n", "310.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "108.520037 | \n", "1 | \n", "33.400002 | \n", "NaN | \n", "0 | \n", "
| 2019-05-07 00:00:00 | \n", "2.246402e+06 | \n", "2246430.75 | \n", "0.0 | \n", "0.0 | \n", "47.0 | \n", "32.0 | \n", "2246429.75 | \n", "1.0 | \n", "20283240.0 | \n", "1332.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "294.642287 | \n", "1 | \n", "33.400002 | \n", "NaN | \n", "0 | \n", "
| 2019-05-07 00:00:00 | \n", "2.246402e+06 | \n", "2246430.75 | \n", "0.0 | \n", "0.0 | \n", "47.0 | \n", "32.0 | \n", "2246429.75 | \n", "1.0 | \n", "17956730.0 | \n", "1407.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "288.604383 | \n", "1 | \n", "33.400002 | \n", "NaN | \n", "0 | \n", "
| 2019-05-07 00:00:00 | \n", "2.246402e+06 | \n", "2246430.75 | \n", "0.0 | \n", "0.0 | \n", "47.0 | \n", "32.0 | \n", "2246429.75 | \n", "1.0 | \n", "1770382.0 | \n", "914.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "194.579235 | \n", "1 | \n", "33.400002 | \n", "NaN | \n", "0 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 2019-05-07 09:59:59 | \n", "2.282400e+06 | \n", "2282429.75 | \n", "0.0 | \n", "0.0 | \n", "48.0 | \n", "32.0 | \n", "2282429.50 | \n", "1.0 | \n", "19365740.0 | \n", "1357.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "292.333517 | \n", "1 | \n", "33.299999 | \n", "NaN | \n", "9 | \n", "
| 2019-05-07 09:59:59 | \n", "2.282400e+06 | \n", "2282429.75 | \n", "0.0 | \n", "0.0 | \n", "48.0 | \n", "32.0 | \n", "2282429.50 | \n", "1.0 | \n", "1585856.0 | \n", "874.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "190.963786 | \n", "1 | \n", "33.299999 | \n", "NaN | \n", "9 | \n", "
| 2019-05-07 09:59:59 | \n", "2.282400e+06 | \n", "2282429.75 | \n", "0.0 | \n", "0.0 | \n", "48.0 | \n", "32.0 | \n", "2282429.50 | \n", "1.0 | \n", "1547770.0 | \n", "877.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "190.174346 | \n", "1 | \n", "33.299999 | \n", "NaN | \n", "9 | \n", "
| 2019-05-07 09:59:59 | \n", "2.282400e+06 | \n", "2282429.75 | \n", "0.0 | \n", "0.0 | \n", "48.0 | \n", "32.0 | \n", "2282429.50 | \n", "1.0 | \n", "663630.0 | \n", "774.0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "NaN | \n", "164.603875 | \n", "1 | \n", "33.299999 | \n", "NaN | \n", "9 | \n", "
| 2019-05-07 09:59:59 | \n", "2.282400e+06 | \n", "2282429.75 | \n", "0.0 | \n", "0.0 | \n", "48.0 | \n", "32.0 | \n", "2282429.75 | \n", "3.0 | \n", "314938.0 | \n", "373.0 | \n", "... | \n", "1 | \n", "1 | \n", "0 | \n", "0 | \n", "2.679357 | \n", "NaN | \n", "1 | \n", "33.299999 | \n", "14.0 | \n", "9 | \n", "
3432203 rows × 65 columns
\n", "| \n", " | dNdlogDmev_all_700.33 | \n", "dNdlogDmev_all_735.46 | \n", "dNdlogDmev_all_772.36 | \n", "dNdlogDmev_all_811.10 | \n", "dNdlogDmev_all_851.79 | \n", "dNdlogDmev_all_894.53 | \n", "dNdlogDmev_all_939.40 | \n", "dNdlogDmev_all_986.53 | \n", "dNdlogDmev_all_1036.02 | \n", "dNdlogDmev_all_1087.99 | \n", "... | \n", "dNdlogDmev_0.32_timelag_360.66 | \n", "dNdlogDmev_0.32_timelag_364.80 | \n", "dNdlogDmev_0.32_timelag_368.94 | \n", "dNdlogDmev_0.32_timelag_373.08 | \n", "dNdlogDmev_0.32_timelag_377.22 | \n", "dNdlogDmev_0.32_timelag_381.36 | \n", "dNdlogDmev_0.32_timelag_385.51 | \n", "dNdlogDmev_0.32_timelag_389.65 | \n", "dNdlogDmev_0.32_timelag_393.79 | \n", "dNdlogDmev_0.32_timelag_397.93 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-07 00:00:00 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "169.009294 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 00:00:01 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 00:00:02 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "84.75842 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 00:00:03 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "84.75842 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 00:00:04 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "84.758420 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "84.75842 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 2019-05-07 23:59:55 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:56 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "85.013713 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:57 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:58 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "85.013713 | \n", "0.0 | \n", "85.013713 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:59 | \n", "82.932638 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
82568 rows × 1392 columns
\n", "| \n", " | dNdlogDmev_all_700.33 | \n", "dNdlogDmev_all_735.46 | \n", "dNdlogDmev_all_772.36 | \n", "dNdlogDmev_all_811.10 | \n", "dNdlogDmev_all_851.79 | \n", "dNdlogDmev_all_894.53 | \n", "dNdlogDmev_all_939.40 | \n", "dNdlogDmev_all_986.53 | \n", "dNdlogDmev_all_1036.02 | \n", "dNdlogDmev_all_1087.99 | \n", "... | \n", "dNdlogDmev_0.32_timelag_360.66 | \n", "dNdlogDmev_0.32_timelag_364.80 | \n", "dNdlogDmev_0.32_timelag_368.94 | \n", "dNdlogDmev_0.32_timelag_373.08 | \n", "dNdlogDmev_0.32_timelag_377.22 | \n", "dNdlogDmev_0.32_timelag_381.36 | \n", "dNdlogDmev_0.32_timelag_385.51 | \n", "dNdlogDmev_0.32_timelag_389.65 | \n", "dNdlogDmev_0.32_timelag_393.79 | \n", "dNdlogDmev_0.32_timelag_397.93 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
0 rows × 1392 columns
\n", "| \n", " | dNdlogDmev_all_70.03 | \n", "dNdlogDmev_all_73.55 | \n", "dNdlogDmev_all_77.24 | \n", "dNdlogDmev_all_81.11 | \n", "dNdlogDmev_all_85.18 | \n", "dNdlogDmev_all_89.45 | \n", "dNdlogDmev_all_93.94 | \n", "dNdlogDmev_all_98.65 | \n", "dNdlogDmev_all_103.60 | \n", "dNdlogDmev_all_108.80 | \n", "... | \n", "dNdlogDmev_all_472.46 | \n", "dNdlogDmev_all_496.16 | \n", "dNdlogDmev_all_521.06 | \n", "dNdlogDmev_all_547.19 | \n", "dNdlogDmev_all_574.65 | \n", "dNdlogDmev_all_603.47 | \n", "dNdlogDmev_all_633.75 | \n", "dNdlogDmev_all_665.54 | \n", "dNdlogDmev_all_698.93 | \n", "dNdlogDmev_all_733.99 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-07 00:00:00 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "169.009294 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| 2019-05-07 00:00:01 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 00:00:02 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "84.75842 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| 2019-05-07 00:00:03 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "84.75842 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| 2019-05-07 00:00:04 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "84.758420 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "84.75842 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 2019-05-07 23:59:55 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:56 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "85.013713 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| 2019-05-07 23:59:57 | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "... | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "NaN | \n", "
| 2019-05-07 23:59:58 | \n", "0.000000 | \n", "0.0 | \n", "0.0 | \n", "85.013713 | \n", "0.0 | \n", "85.013713 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
| 2019-05-07 23:59:59 | \n", "82.932638 | \n", "0.0 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.0 | \n", "0.000000 | \n", "0.00000 | \n", "0.00000 | \n", "... | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "0.0 | \n", "
82568 rows × 49 columns
\n", "