{ "cells": [ { "cell_type": "markdown", "id": "83bf48ba", "metadata": {}, "source": [ "# Data Analysis for CWFT Experiments Runfile" ] }, { "cell_type": "markdown", "id": "4d16bf54", "metadata": {}, "source": [ "Atmospheric Chemistry Lab Course 2026 Edition" ] }, { "cell_type": "markdown", "id": "9be00b84", "metadata": {}, "source": [ "Contact nicole.leemann@psi.ch if you encounter any bugs." ] }, { "cell_type": "markdown", "id": "4a452fa8", "metadata": {}, "source": [ "## Import" ] }, { "cell_type": "code", "execution_count": null, "id": "15c13492", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import plotly.express as px\n", "import plotly.graph_objects as go\n", "from plotly.subplots import make_subplots\n", "from pathlib import Path\n", "from babel.dates import format_date\n", "from scipy.optimize import curve_fit\n", "import re\n", "import ast" ] }, { "cell_type": "code", "execution_count": null, "id": "0154d14b", "metadata": {}, "outputs": [], "source": [ "import ACLC26_data_analysis_module as dam" ] }, { "cell_type": "markdown", "id": "b17b4725", "metadata": {}, "source": [ "## Initialize Experiment Class" ] }, { "cell_type": "code", "execution_count": null, "id": "592ecf99", "metadata": {}, "outputs": [], "source": [ "# Input\n", "\n", "from_summary = False # set to true to load parameters from a summary file instead of manual input\n", "\n", "savename = \"20251202_FeCit_CA_Nitrate_1_10_10_RH80\" # name that will be used for saving the processed data and data analysis summary of the experiment\n", "#savename = \"20251204_FeCit_CA_Nitrate_1_10_10_RH50\"\n", "#savename = \"20251203_CA_Nitrate_1_1_RH80\"\n", "savefolder = Path(\"C:/Users/leeman_n/OneDrive - Paul Scherrer Institut/Documents/Teaching/2026_AtmosChemLabCourse/ACLC26_DataAnalysis/ACLC26_Output\") / savename # name of the folder to save the outputs and summary of the experiment\n", "\n", "color_dict = { # colors that will be used for plotting\n", " \"T360U_CO2\": [\"#807F80\"], # CO2\n", " \"T200_NOx\": [\"#F08080\", \"#84B8E9\", \"#C5A1F5\"], # NO, NO2, NOx \n", " \"Ecophysics_NOx_899\": [\"#CC2626\",\"#0B60AF\", \"#5533B4\"], # NO, NO2, NOx\n", " \"ICAD_HONO\": [\"#db2fa8\", \"#7ddfc9\"], # HONO, NO2\n", " \"ICAD_NO2\": [\"#6cdf72\"] # NO2\n", " }\n", "\n", "if from_summary:\n", " FeCit_N = dam.init_from_summary(savefolder, savename, color_dict) # initialize from summary\n", "\n", "else: # manual input\n", " exp_name = \"FeCit/CA/Nitrate 1:10:10 RH 80% (2.12.2026)\" # for plot title (or comparisons)\n", "\n", " #instrument_list = [\"T360U_CO2\", \"T200_NOx\", \"Ecophysics_NOx_899\", \"ICAD_HONO\", \"ICAD_NO2\", \"Humidity_Sensors\"]\n", " instrument_list = [\"T360U_CO2\", \"Ecophysics_NOx_899\", \"ICAD_HONO\", \"ICAD_NO2\", \"Humidity_Sensors\"] # list of all instruments used\n", " \n", " setup_dict = {\n", " \"p_room\": 979, # ambient pressure [mbar]\n", " \"T_room\": 23, # room temperature [°C]\n", " \"T_tube\": 18, # CWFT temperature [°C]\n", " \"T_hum\": 15, # humidifier temperature [°C]\n", " \"flow_N2\": 1300, # MFC set flow for N2\n", " \"flow_O2\": 300, # MFC set flow for O2\n", " \"dilution\": 3000, # MFC set flow for N2 dilution\n", " \"dil_BTOF\": 1000, # MFC set flow for BTOF dilution\n", " \"tube_length\": 32.6, # tube length [cm]\n", " \"tube_diam\": 1.64, # tube inner diameter [cm]\n", " \"vol_coat\": 1000 # volume of coating solution [muL]\n", " }\n", " exp_start = \"2025-12-02 09:00:00\" # YYYY-MM-DD hh:mm:ss (for locating the correct data) \n", " time_tube_in = \"2025-12-02 14:58:19\" # switch from bypass to tube flow \n", " time_lights_on = \"2025-12-02 15:18:25\" # irradiation period\n", " time_lights_off = \"2025-12-02 15:53:37\" \n", " time_lights_on_2 = None # YYYY-MM-DD hh:mm:ss or None # second irradiation period (optional)\n", " time_lights_off_2 = None\n", " time_tube_out = \"2025-12-02 16:09:08\" # switch from tube to bypass flow \n", " exp_end = \"2025-12-02 18:00:00\" # for start and end: choose big enough interval that also includes the timeshifted data \n", "\n", " FeCit_N = dam.experiment(exp_name, instrument_list, setup_dict, exp_start, exp_end, time_tube_in, time_lights_on, time_lights_off, time_tube_out, color_dict, savefolder,\n", " time_lights_on_2, time_lights_off_2) # initialize\n" ] }, { "cell_type": "code", "execution_count": null, "id": "bf8e0b86", "metadata": {}, "outputs": [], "source": [ "# set up folders for saving processed data and plots\n", "savefolder.mkdir(parents=True, exist_ok=True) # create folder if it doesn't exist already (parent folder)\n", "\n", "savefolder_data = Path(savefolder) / \"Processed_Data\" # csv files of processed data for each instrument\n", "savefolder_data.mkdir(parents=True, exist_ok=True)\n", "savefolder_plots = Path(savefolder) / \"Plots\" # all saved plots\n", "savefolder_plots.mkdir(parents=True, exist_ok=True)\n" ] }, { "cell_type": "markdown", "id": "73434a57", "metadata": {}, "source": [ "## Load raw data" ] }, { "cell_type": "code", "execution_count": null, "id": "92749af5", "metadata": {}, "outputs": [], "source": [ "if \"T360U_CO2\" in FeCit_N.instruments:\n", " FeCit_N.get_rawdata_T360U_CO2() \n", " display(FeCit_N.rawdata_co2.head())\n", " print(\"Size: \", FeCit_N.rawdata_co2.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "60744a02", "metadata": {}, "outputs": [], "source": [ "# if \"T200_NOx\" in FeCit_N.instruments:\n", "# FeCit_N.get_rawdata_T200_NOx()\n", "# display(FeCit_N.rawdata_nox.head())\n", "# print(\"Size: \", FeCit_N.rawdata_nox.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "19002631", "metadata": {}, "outputs": [], "source": [ "if \"Humidity_Sensors\" in FeCit_N.instruments:\n", " FeCit_N.get_rawdata_RH()\n", " display(FeCit_N.rawdata_rh.head())\n", " print(\"Size: \", FeCit_N.rawdata_rh.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "f717e5d5", "metadata": {}, "outputs": [], "source": [ "if \"ICAD_HONO\" in FeCit_N.instruments:\n", " FeCit_N.get_rawdata_ICAD_HONO()\n", " display(FeCit_N.rawdata_icad_hono.head())\n", " print(\"Size: \", FeCit_N.rawdata_icad_hono.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "85c731f4", "metadata": {}, "outputs": [], "source": [ "if \"ICAD_NO2\" in FeCit_N.instruments:\n", " FeCit_N.get_rawdata_ICAD_NO2()\n", " display(FeCit_N.rawdata_icad_no2.head())\n", " print(\"Size: \", FeCit_N.rawdata_icad_no2.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "51a2c112", "metadata": {}, "outputs": [], "source": [ "if \"Ecophysics_NOx_899\" in FeCit_N.instruments:\n", " FeCit_N.get_rawdata_NOx_eco()\n", " display(FeCit_N.rawdata_nox_eco.head())\n", " print(\"Size: \", FeCit_N.rawdata_nox_eco.shape)" ] }, { "cell_type": "markdown", "id": "07320072", "metadata": {}, "source": [ "## Preview Plot" ] }, { "cell_type": "markdown", "id": "3f3e99d5", "metadata": {}, "source": [ "For checking and adjusting alignment between instruments" ] }, { "cell_type": "code", "execution_count": null, "id": "e8b2954c", "metadata": {}, "outputs": [], "source": [ "# copy the output to the cell below for easier adjustment ;)\n", "# set pd.Timestamp(\"None\") to None!\n", "\n", "print(f\"FeCit_N.time_tube_in = pd.Timestamp(\\\"{FeCit_N.time_tube_in}\\\")\")\n", "print(f\"FeCit_N.time_lights_on = pd.Timestamp(\\\"{FeCit_N.time_lights_on}\\\")\")\n", "print(f\"FeCit_N.time_lights_off = pd.Timestamp(\\\"{FeCit_N.time_lights_off}\\\")\")\n", "print(f\"FeCit_N.time_lights_on_2 = pd.Timestamp(\\\"{FeCit_N.time_lights_on_2}\\\")\")\n", "print(f\"FeCit_N.time_lights_off_2 = pd.Timestamp(\\\"{FeCit_N.time_lights_off_2}\\\")\")\n", "print(f\"FeCit_N.time_tube_out = pd.Timestamp(\\\"{FeCit_N.time_tube_out}\\\")\")" ] }, { "cell_type": "code", "execution_count": null, "id": "152175f8", "metadata": {}, "outputs": [], "source": [ "# from_summary = False # uncomment this to make adjustments to the parameters from the summary\n", "\n", "if from_summary==False: # only overwrite if not loading from summary\n", "\n", " # set timeshift of instruments (align times to CO2 analyzer)\n", " FeCit_N.timeshift_nox = pd.Timedelta(minutes=0, seconds=0)\n", " FeCit_N.timeshift_nox_eco = pd.Timedelta(minutes=56, seconds=34)\n", " FeCit_N.timeshift_icad_hono = pd.Timedelta(minutes=61, seconds=50)\n", " FeCit_N.timeshift_icad_no2 = pd.Timedelta(minutes=60, seconds=0)\n", " FeCit_N.timeshift_rh = pd.Timedelta(minutes=0, seconds=0)\n", "\n", " # for demo\n", " # FeCit_N.timeshift_nox_eco = pd.Timedelta(minutes=60, seconds=0)\n", " # FeCit_N.timeshift_icad_hono = pd.Timedelta(minutes=60, seconds=0)\n", " # FeCit_N.timeshift_icad_no2 = pd.Timedelta(minutes=60, seconds=0)\n", "\n", " # set irradiation time periods & tube in / out times\n", " FeCit_N.time_tube_in = pd.Timestamp(\"2025-12-02 14:58:19\")\n", " FeCit_N.time_lights_on = pd.Timestamp(\"2025-12-02 15:18:25\")\n", " FeCit_N.time_lights_off = pd.Timestamp(\"2025-12-02 15:53:37\")\n", " FeCit_N.time_lights_on_2 = None\n", " FeCit_N.time_lights_off_2 = None\n", " FeCit_N.time_tube_out = pd.Timestamp(\"2025-12-02 16:09:08\")\n", "\n", " # FeCit_N.time_tube_in = pd.Timestamp(\"2025-12-04 11:47:25\")\n", " # FeCit_N.time_lights_on = pd.Timestamp(\"2025-12-04 12:00:27\")\n", " # FeCit_N.time_lights_off = pd.Timestamp(\"2025-12-04 12:35:41\")\n", " # FeCit_N.time_lights_on_2 = None\n", " # FeCit_N.time_lights_off_2 = None\n", " # FeCit_N.time_tube_out = pd.Timestamp(\"2025-12-04 13:35:26\")\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "44ff59d5", "metadata": {}, "outputs": [], "source": [ "preview = FeCit_N.preview_plot(FeCit_N.instruments) # clean up data and make preview plot\n", "#preview.update_xaxes(range=(pd.Timestamp(\"2025-12-04 10:30:00\"), pd.Timestamp(\"2025-12-04 14:30:00\"))) # set x-axis range manually!\n", "preview.update_yaxes(range=(0,5), secondary_y=False) # y-axis range (left)\n", "preview.show()\n", "#preview.write_html(f\"{savefolder_plots}/{savename}_Preview_Plot.html\")" ] }, { "cell_type": "code", "execution_count": null, "id": "9f65a0b4", "metadata": {}, "outputs": [], "source": [ "if \"Humidity_Sensors\" in FeCit_N.instruments:\n", " rhtdp_plot = FeCit_N.RHTDP_plot()\n", " #rhtdp_plot.update_xaxes(range=(pd.Timestamp(\"2025-11-07 13:00:00\"), pd.Timestamp(\"2025-11-07 17:00:00\")))\n", " #rhtdp_plot.update_yaxes(title_text=\"RH [%]\", range=[10, 35], row=1, col=1)\n", " \n", " #rhtdp_plot.show() # uncomment to show the plot" ] }, { "cell_type": "markdown", "id": "45b39e79", "metadata": {}, "source": [ "## Background Fitting" ] }, { "cell_type": "code", "execution_count": null, "id": "ea43f3ef", "metadata": {}, "outputs": [], "source": [ "# add column with numeric timeaxis to all datasets\n", "\n", "if \"T360U_CO2\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_co2, ts=pd.Timedelta(minutes=0, seconds=0))\n", "if \"T200_NOx\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_nox, ts=FeCit_N.timeshift_nox)\n", "if \"Ecophysics_NOx_899\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_nox_eco, ts=FeCit_N.timeshift_nox_eco)\n", "if \"ICAD_HONO\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_icad_hono, ts=FeCit_N.timeshift_icad_hono, ts_sign=\"+\")\n", "if \"ICAD_NO2\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_icad_no2, ts=FeCit_N.timeshift_icad_no2, ts_sign=\"+\")\n", "if \"Humidity_Sensors\" in FeCit_N.instruments:\n", " FeCit_N.get_num_timeaxis(FeCit_N.predata_rh, ts=FeCit_N.timeshift_rh)\n", "FeCit_N.predata_icad_hono.head() # preview\n", "FeCit_N.predata_co2.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "3cae3223", "metadata": {}, "outputs": [], "source": [ "# CO2\n", "if \"T360U_CO2\" in FeCit_N.instruments:\n", "\n", " df_data = FeCit_N.predata_co2.copy()\n", " if from_summary==False:\n", " FeCit_N.bkgint_co2 = ['2025-12-02 12:15:00', '2025-12-02 13:30:00'] # manual input interval for bkg fit\n", "\n", " pred_start, pred_end = pd.Timestamp(FeCit_N.bkgint_co2[0]), pd.Timestamp(FeCit_N.bkgint_co2[1]) # if load from summary\n", "\n", " # fit the background with linear function and return plot & fit parameters\n", " bkgfit_plot, popt, pcov, residuals = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw CO2 [ppm]\", mode=\"lin\")\n", " bkgfit_plot.update_layout(height=500, width=1500) # figure size\n", " bkgfit_plot.update_yaxes(range=(2,4))\n", " #FeCit_N.add_timemarkers(bkgfit_plot) # add yellow rectangle to mark irradiation time\n", " bkgfit_plot.show()\n", " #bkgfit_plot.write_html(f\"{savefolder_plots}/{savename}_Background_T360U_CO2.html\") # save interactive plot\n", " bkgfit_plot.write_image(f\"{savefolder_plots}/{savename}_Background_T360U_CO2.png\") # save static plot\n", "\n", " FeCit_N.bkgcorr_co2 = [popt] # save the parameters of the background fit\n", "\n", " print(popt)\n", "\n", " # residuals_plot = px.scatter(x=df_data[\"Time since Irradiation [min]\"],y=residuals, title=\"residuals\")\n", " # residuals_plot.update_layout(height=500, width=1500)\n", " # residuals_plot.show()\n", "\n", " # y_true = df_data[\"raw CO2 [ppm]\"].to_numpy()\n", " # ss_res = (residuals**2).sum()\n", " # ss_tot = ((y_true - y_true.mean())**2).sum()\n", "\n", " # r2 = 1 - ss_res / ss_tot\n", " # print(r2)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f3b108f1", "metadata": {}, "outputs": [], "source": [ "# NOx eco\n", "if \"Ecophysics_NOx_899\" in FeCit_N.instruments:\n", "\n", " df_data = FeCit_N.predata_nox_eco.copy()\n", " if from_summary==False:\n", " FeCit_N.bkgint_nox_eco = ['2025-12-02 13:00:00', '2025-12-02 15:15:00'] # manual input\n", " #FeCit_N.bkgint_nox_eco = ['2025-12-04 10:40:00', '2025-12-04 12:00:00']\n", "\n", " pred_start, pred_end = pd.Timestamp(FeCit_N.bkgint_nox_eco[0]), pd.Timestamp(FeCit_N.bkgint_nox_eco[1]) # if load from summary\n", "\n", " # individual plots\n", " bkgfit_plot_poly, popt_poly, pcov_poly, resi_poly = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"poly\")\n", " #bkgfit_plot_poly.show()\n", " #bkgfit_plot_logdec, popt_logdec, pcov_logdec, resi_logdec = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"log_decay\")\n", " #bkgfit_plot_logdec.show()\n", " bkgfit_plot1, popt1, pcov1, resi1 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO [ppb]\", mode=\"lin\")\n", " #bkgfit_plot1.show()\n", " bkgfit_plot2, popt2, pcov2, resi2 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"lin\")\n", " #bkgfit_plot2.show()\n", " bkgfit_plot3, popt3, pcov3, resi3 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NOx [ppb]\", mode=\"lin\")\n", " #bkgfit_plot3.show()\n", "\n", " # combined plot\n", " fig = make_subplots(\n", " rows=2,\n", " cols=2,\n", " subplot_titles=[\"NO\", \"NO2 lin\", \"NOx\", \"NO2 log decay\"],\n", " horizontal_spacing = 0.03,\n", " vertical_spacing = 0.1\n", " )\n", "\n", " for trace in bkgfit_plot1.data:\n", " fig.add_trace(trace, row=1, col=1)\n", "\n", " for trace in bkgfit_plot2.data:\n", " fig.add_trace(trace, row=1, col=2)\n", "\n", " for trace in bkgfit_plot3.data:\n", " fig.add_trace(trace, row=2, col=1)\n", "\n", " #for trace in bkgfit_plot_logdec.data:\n", " # fig.add_trace(trace, row=2, col=2)\n", "\n", " fig.update_layout(\n", " showlegend=True,\n", " title_text=\"Ecophysics NOx 899\"\n", " )\n", " fig.update_layout(height=750, width=1500)\n", " #FeCit_N.add_timemarkers(fig)\n", " fig.show()\n", " #fig.write_html(f\"{savefolder_plots}/{savename}_Background_Ecophysics_NOx_899.html\")\n", " fig.write_image(f\"{savefolder_plots}/{savename}_Background_Ecophysics_NOx_899.png\") # save static plot\n", "\n", " FeCit_N.bkgcorr_nox_eco = [popt1, popt2, popt3] # save fit parameters\n", " print(FeCit_N.bkgcorr_nox_eco)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "bada713d", "metadata": {}, "outputs": [], "source": [ "# NOx\n", "if \"T200_NOx\" in FeCit_N.instruments:\n", "\n", " df_data = FeCit_N.predata_nox.copy()\n", " if from_summary==False:\n", " FeCit_N.bkgint_nox = ['2025-12-02 13:00:00', '2025-12-02 15:15:00'] # manual input\n", "\n", " pred_start, pred_end = pd.Timestamp(FeCit_N.bkgint_nox[0]), pd.Timestamp(FeCit_N.bkgint_nox[1]) # if load from summary\n", " \n", " bkgfit_plot1, popt1, pcov1, resi1 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO [ppb]\", mode=\"lin\")\n", " #bkgfit_plot1.show()\n", " bkgfit_plot2, popt2, pcov2, resi2 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"lin\")\n", " #bkgfit_plot2.show()\n", " bkgfit_plot3, popt3, pcov3, resi3 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NOx [ppb]\", mode=\"lin\")\n", " #bkgfit_plot3.show()\n", "\n", " fig = make_subplots(\n", " rows=2,\n", " cols=2,\n", " subplot_titles=[\"NO\", \"NO2\", \"NOx\", \" \"],\n", " horizontal_spacing = 0.03,\n", " vertical_spacing = 0.1\n", " )\n", "\n", " for trace in bkgfit_plot1.data:\n", " fig.add_trace(trace, row=1, col=1)\n", "\n", " for trace in bkgfit_plot2.data:\n", " fig.add_trace(trace, row=1, col=2)\n", "\n", " for trace in bkgfit_plot3.data:\n", " fig.add_trace(trace, row=2, col=1)\n", "\n", " fig.update_layout(\n", " #height=800,\n", " showlegend=True,\n", " title_text=\"T200 NOx\"\n", " )\n", " #FeCit_N.add_timemarkers(fig)\n", " fig.update_layout(height=750, width=1500)\n", " fig.show()\n", " #fig.write_html(f\"{savefolder_plots}/{savename}_Background_T200_NOx.html\")\n", " fig.write_image(f\"{savefolder_plots}/{savename}_Background_T200_NOx.png\") # save static plot\n", "\n", " FeCit_N.bkgcorr_nox = [popt1, popt2, popt3] # save fit parameters\n", " print(FeCit_N.bkgcorr_nox)" ] }, { "cell_type": "code", "execution_count": null, "id": "68fd32d5", "metadata": {}, "outputs": [], "source": [ "# ICAD HONO\n", "if \"ICAD_HONO\" in FeCit_N.instruments:\n", "\n", " df_data = FeCit_N.predata_icad_hono.copy()\n", " if from_summary==False:\n", " FeCit_N.bkgint_icad_hono = ['2025-12-02 11:00:00', '2025-12-02 15:00:00'] # manual input\n", "\n", " pred_start, pred_end = pd.Timestamp(FeCit_N.bkgint_icad_hono[0]), pd.Timestamp(FeCit_N.bkgint_icad_hono[1]) # if load from summary\n", " \n", " bkgfit_plot1, popt1, pcov1, resi1 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw HONO [ppb]\", mode=\"lin\")\n", " #bkgfit_plot1.show()\n", " bkgfit_plot2, popt2, pcov2, resi2 = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"lin\")\n", " #bkgfit_plot2.show()\n", "\n", " fig = make_subplots(\n", " rows=1,\n", " cols=2,\n", " subplot_titles=[\"HONO\", \"NO2\"],\n", " horizontal_spacing = 0.03,\n", " vertical_spacing = 0.1\n", " )\n", "\n", " for trace in bkgfit_plot1.data:\n", " fig.add_trace(trace, row=1, col=1)\n", "\n", " for trace in bkgfit_plot2.data:\n", " fig.add_trace(trace, row=1, col=2)\n", "\n", " #FeCit_N.add_timemarkers(fig)\n", " fig.update_layout(height=450, width=1500, title_text=\"ICAD HONO\")\n", " fig.show()\n", " #fig.write_html(f\"{savefolder_plots}/{savename}_Background_ICAD_HONO.html\")\n", " fig.write_image(f\"{savefolder_plots}/{savename}_Background_ICAD_HONO.png\") # save static plot\n", "\n", " FeCit_N.bkgcorr_icad_hono = [popt1, popt2] # save fit parameters\n", " print(FeCit_N.bkgcorr_icad_hono)" ] }, { "cell_type": "code", "execution_count": null, "id": "b9597687", "metadata": {}, "outputs": [], "source": [ "# ICAD NO2\n", "if \"ICAD_NO2\" in FeCit_N.instruments:\n", " \n", " df_data = FeCit_N.predata_icad_no2.copy()\n", " if from_summary==False:\n", " FeCit_N.bkgint_icad_no2 = ['2025-12-02 12:00:00', '2025-12-02 15:00:00'] # manual input\n", "\n", " pred_start, pred_end = pd.Timestamp(FeCit_N.bkgint_icad_no2[0]), pd.Timestamp(FeCit_N.bkgint_icad_no2[1]) # if load from summary\n", " \n", " bkgfit_plot, popt, pcov, residuals = FeCit_N.get_bkg_fit(df_data, pred_start, pred_end, param=\"raw NO2 [ppb]\", mode=\"lin\")\n", " bkgfit_plot.update_layout(height=500, width=1500)\n", " #FeCit_N.add_timemarkers(bkgfit_plot)\n", " bkgfit_plot.show()\n", " #bkgfit_plot.write_html(f\"{savefolder_plots}/{savename}_Background_ICAD_NO2.html\")\n", " bkgfit_plot.write_image(f\"{savefolder_plots}/{savename}_Background_ICAD_NO2.png\") # save static plot\n", "\n", " FeCit_N.bkgcorr_icad_no2 = [popt] # save fit parameters\n", " print(FeCit_N.bkgcorr_icad_no2)" ] }, { "cell_type": "markdown", "id": "1630159c", "metadata": {}, "source": [ "## Final Processing" ] }, { "cell_type": "markdown", "id": "bb2ad088", "metadata": {}, "source": [ "Finish processing the instrument data (correct for timeshifts, background and dilution) and save the cleaned dataframes in a csv file (one per instrument)" ] }, { "cell_type": "code", "execution_count": null, "id": "f2b9eb53", "metadata": {}, "outputs": [], "source": [ "FeCit_N.process_data(savefolder, savename)\n", "FeCit_N.data_co2.head() # preview" ] }, { "cell_type": "markdown", "id": "0577973c", "metadata": {}, "source": [ "## Summary" ] }, { "cell_type": "code", "execution_count": null, "id": "534940ef", "metadata": {}, "outputs": [], "source": [ "savefile = f\"{savefolder}/{savename}_DA_Summary.txt\" # save summary of data analysis in this file\n", "FeCit_N.get_summary(savefile)" ] }, { "cell_type": "markdown", "id": "3cdfb3ec", "metadata": {}, "source": [ "## Plots" ] }, { "cell_type": "markdown", "id": "e2b41f4a", "metadata": {}, "source": [ "### One Instrument per Figure" ] }, { "cell_type": "code", "execution_count": null, "id": "e6f6c555", "metadata": {}, "outputs": [], "source": [ "# CO2\n", "if \"T360U_CO2\" in FeCit_N.instruments:\n", "\n", " #data = pd.read_csv(f\"{savefolder}/Processed_Data/{savename}_Data_T360U_CO2.csv\", sep=';') # get data from file\n", " data = FeCit_N.data_co2.copy() # get data from experiment\n", "\n", " fig_co2 = FeCit_N.plot_one_instrument(instr=\"T360U_CO2\", data=data, plot_raw = False)\n", " fig_co2.update_xaxes(range=(-30, 60))\n", " fig_co2.update_yaxes(range=(-2,35))\n", " #fig_co2.update_yaxes(range=(-0.05,0.5))\n", " fig_co2.update_layout(height=500, width=1500)\n", " fig_co2.update_layout( \n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_co2.show()\n", " #fig_co2.write_html(f\"{savefolder_plots}/{savename}_Plot_T360U_CO2.html\")\n", " fig_co2.write_image(f\"{savefolder_plots}/{savename}_Plot_T360U_CO2.png\") # save static plot" ] }, { "cell_type": "code", "execution_count": null, "id": "f297db4c", "metadata": {}, "outputs": [], "source": [ "# NOx\n", "if \"T200_NOx\" in FeCit_N.instruments:\n", "\n", " #data = pd.read_csv(f\"{savefolder}/Processed_Data/{savename}_Data_T200_NOx.csv\", sep=';') # get data from file\n", " data = FeCit_N.data_nox.copy() # get data from experiment\n", "\n", " fig_nox = FeCit_N.plot_one_instrument(instr=\"T200_NOx\", data=data, plot_raw = False)\n", " fig_nox.update_xaxes(range=(-30, 60))\n", " #fig_nox.update_yaxes(range=(-10,40))\n", " fig_nox.update_layout(height=500, width=1500)\n", " fig_nox.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_nox.show()\n", " #fig_nox.write_html(f\"{savefolder_plots}/{savename}_Plot_T200_NOx.html\")\n", " fig_nox.write_image(f\"{savefolder_plots}/{savename}_Plot_T200_NOx.png\") # save static plot" ] }, { "cell_type": "code", "execution_count": null, "id": "774f3ba0", "metadata": {}, "outputs": [], "source": [ "# NOx eco\n", "if \"Ecophysics_NOx_899\" in FeCit_N.instruments:\n", "\n", " #data = pd.read_csv(f\"{savefolder}/Processed_Data/{savename}_Data_Ecophysics_NOx_899.csv\", sep=';') # get data from file\n", " data = FeCit_N.data_nox_eco.copy() # get data from experiment\n", "\n", " fig_nox_eco = FeCit_N.plot_one_instrument(instr=\"Ecophysics_NOx_899\", data=data, plot_raw = False)\n", " fig_nox_eco.update_xaxes(range=(-30, 60))\n", " #fig_nox_eco.update_yaxes(range=(-5,80))\n", " fig_nox_eco.update_layout(height=500, width=1500)\n", " fig_nox_eco.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_nox_eco.show()\n", " #fig_nox_eco.write_html(f\"{savefolder_plots}/{savename}_Plot_Ecophysics_NOx_899.html\")\n", " fig_nox_eco.write_image(f\"{savefolder_plots}/{savename}_Plot_Ecophysics_NOx_899.png\") # save static plot" ] }, { "cell_type": "code", "execution_count": null, "id": "4138edbb", "metadata": {}, "outputs": [], "source": [ "# ICAD HONO\n", "if \"ICAD_HONO\" in FeCit_N.instruments:\n", "\n", " inst = \"ICAD_HONO\"\n", " #data = pd.read_csv(f\"{savefolder}/Processed_Data/{savename}_Data_ICAD_HONO.csv\", sep=';') # get data from file\n", " data = FeCit_N.data_icad_hono.copy() # get data from experiment\n", "\n", " fig_hono = FeCit_N.plot_one_instrument(instr=\"ICAD_HONO\", data=data, plot_raw = False)\n", " fig_hono.update_xaxes(range=(-30, 60))\n", " #fig_hono.update_yaxes(range=(-2,5))\n", " fig_hono.update_layout(height=500, width=1500)\n", " fig_hono.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_hono.show()\n", " #fig_hono.write_html(f\"{savefolder_plots}/{savename}_Plot_ICAD_HONO.html\")\n", " fig_hono.write_image(f\"{savefolder_plots}/{savename}_Plot_ICAD_HONO.png\") # save static plot" ] }, { "cell_type": "code", "execution_count": null, "id": "5a92726a", "metadata": {}, "outputs": [], "source": [ "# ICAD NO2\n", "if \"ICAD_NO2\" in FeCit_N.instruments:\n", "\n", " #data = pd.read_csv(f\"{savefolder}/Processed_Data/{savename}_Data_ICAD_NO2.csv\", sep=';') # get data from file\n", " data = FeCit_N.data_icad_no2.copy() # get data from experiment\n", "\n", " fig_no2 = FeCit_N.plot_one_instrument(instr=\"ICAD_NO2\", data=None, plot_raw = False)\n", " fig_no2.update_xaxes(range=(-30, 60))\n", " fig_no2.update_yaxes(range=(-1,8))\n", " fig_no2.update_layout(height=500, width=1500)\n", " fig_no2.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_no2.show()\n", " #fig_no2.write_html(f\"{savefolder_plots}/{savename}_Plot_ICAD_NO2.html\")\n", " fig_no2.write_image(f\"{savefolder_plots}/{savename}_Plot_ICAD_NO2.png\") # save static plot" ] }, { "cell_type": "code", "execution_count": null, "id": "ff36304f", "metadata": {}, "outputs": [], "source": [ "if \"Humidity_Sensors\" in FeCit_N.instruments:\n", "\n", " rh_plot = FeCit_N.RHTDP_plot(num_timeaxis=True) # make RH-temperature-dewpoint plot\n", " FeCit_N.add_timemarkers(rh_plot) # add yellow rectangle to mark irradiation period\n", " rh_plot.update_layout(height=500, width=1500, showlegend=True)\n", " rh_plot.update_xaxes(range=(-30, 60))\n", "\n", " #rh_plot.update_yaxes(range=(20, 30), row=1)\n", " #rh_plot.update_yaxes(range=(-0, 10), row=3)\n", " rh_plot.update_yaxes(range=(10, 30), row=1)\n", " rh_plot.update_yaxes(range=(-5, 5), row=3)\n", "\n", " rh_plot.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", "\n", " # dummy legend for irradiation\n", " rh_plot.add_trace(\n", " go.Scatter(\n", " x=[None],\n", " y=[None],\n", " mode=\"markers\",\n", " marker=dict(\n", " size=12,\n", " symbol=\"square\",\n", " color=\"gold\",\n", " opacity=0.3\n", " ),\n", " name=\"Irradiation\",\n", " showlegend=True,\n", " ),\n", " row=1, col=1\n", " )\n", " rh_plot.show()\n", " #rh_plot.write_html(f\"{savefolder_plots}/{savename}_Plot_Humidity_Sensors.html\")\n", " rh_plot.write_image(f\"{savefolder_plots}/{savename}_Plot_Humidity_Sensors.png\") # save static plot" ] }, { "cell_type": "markdown", "id": "7043e2b1", "metadata": {}, "source": [ "### Multiple Instruments per Figure" ] }, { "cell_type": "markdown", "id": "975678b0", "metadata": {}, "source": [ "Note: Do not plot CO2 here as it will use the wrong yaxis unit label. Use plot_multiple_2axes to plot CO2 with other traces (below)." ] }, { "cell_type": "code", "execution_count": null, "id": "fe45e82b", "metadata": {}, "outputs": [], "source": [ "instrs = [\"ICAD_HONO\", \"ICAD_NO2\"] # instruments to plot\n", "title = \"ICADs\"\n", "\n", "if set(instrs).issubset(set(FeCit_N.instruments)): # if chosen instruments were all actually used\n", "\n", " fig_icad = FeCit_N.plot_multiple(title, instrs)\n", " fig_icad.update_xaxes(range=(-30, 60))\n", " fig_icad.update_yaxes(range=(-5,45))\n", " fig_icad.update_layout(height=500, width=1500)\n", " fig_icad.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig_icad.show()\n", " #fig_icad.write_html(f\"{savefolder_plots}/{savename}_Plot_ICADs.html\")\n", " #fig_icad.write_image(f\"{savefolder_plots}/{savename}_Plot_ICADs.png\") # save static plot\n", "else:\n", " print(\"One or more instruments in instrs are not in instrument_list. Skip plot.\")" ] }, { "cell_type": "markdown", "id": "24bf11de", "metadata": {}, "source": [ "Multiple Instruments Figures with 2 y-axes:" ] }, { "cell_type": "code", "execution_count": null, "id": "7a20aa3c", "metadata": {}, "outputs": [], "source": [ "instrs1 = [\"T360U_CO2\"] # instruments to plot on left axis (never mix CO2 with other instruments because of different units)\n", "instrs2 = FeCit_N.instruments[1:-1] # everything except the first (CO2)\n", "title = \"All parameters\"\n", "\n", "if set(instrs1).issubset(set(FeCit_N.instruments)) and set(instrs2).issubset(set(FeCit_N.instruments)):\n", " fig = FeCit_N.plot_multiple_2axes(title, instrs1, instrs2)\n", " fig.update_xaxes(range=(-30, 60))\n", "\n", " ymin, ymax = -5, 35 # y-axis range \n", " #ymin, ymax = -0.5, 1\n", " fig.update_yaxes(range=(ymin,ymax), secondary_y=False)\n", " fig.update_yaxes(range=(ymin*10,ymax*10), secondary_y=True) # right axis scaled based on left axis\n", "\n", " fig.update_layout(height=600, width=1800)\n", " fig.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig.show()\n", " fig.write_html(f\"{savefolder_plots}/{savename}_Plot_All.html\") # save interactive plot\n", " fig.write_image(f\"{savefolder_plots}/{savename}_Plot_All.png\") # save static plot\n", "else:\n", " print(\"One or more instruments in instrs1 or instrs2 are not in instrument_list. Skip plot.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "d5c913e0", "metadata": {}, "outputs": [], "source": [ "instrs1 = [\"Ecophysics_NOx_899\"] # instruments to plot on left axis\n", "instrs2 = [\"ICAD_HONO\", \"ICAD_NO2\"] # instruments to plot on right axis\n", "title = \"NOx analyser vs ICADs\"\n", "\n", "if set(instrs1).issubset(set(FeCit_N.instruments)) and set(instrs2).issubset(set(FeCit_N.instruments)):\n", " fig = FeCit_N.plot_multiple_2axes(title, instrs1, instrs2)\n", " fig.update_xaxes(range=(-30, 60))\n", " #fig.update_yaxes(range=(-10,40))\n", " fig.update_layout(height=500, width=1500)\n", " fig.update_layout(\n", " margin=dict(l=70, r=210, t=70, b=70) # control margins to keep the plot area consistent between plots\n", " )\n", " fig.show()\n", "else:\n", " print(\"One or more instruments in instrs1 or instrs2 are not in instrument_list. Skip plot.\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8e61aea5", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "venv310", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.19" } }, "nbformat": 4, "nbformat_minor": 5 }