{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "d85e9883", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import matplotlib.dates as mdates" ] }, { "cell_type": "markdown", "id": "d7eb024f", "metadata": {}, "source": [ "### Unprocessed parquet files" ] }, { "cell_type": "code", "execution_count": 2, "id": "a1df7aa6", "metadata": {}, "outputs": [], "source": [ "selection = [(\"date\", \">=\", \"2019-05-09\")]\n", "hk_files = pd.read_parquet('data/hk_files_parquet', filters=selection).sort_index()\n", "pbp_files = pd.read_parquet('data/pbp_files_parquet', filters=selection).sort_index()" ] }, { "cell_type": "code", "execution_count": 3, "id": "130d8bbc", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | Time Stamp | \n", "Time (sec) | \n", "Time Stamp (UTC sec) | \n", "Elapsed Time | \n", "Error Code | \n", "Packet Time Stamp | \n", "Laser TEC Temp (C) | \n", "Crystal TEC Temp (C) | \n", "Inlet Air Temp (C) | \n", "Computer Heatsink Temp (C) | \n", "... | \n", "t0 | \n", "file | \n", "folder_name | \n", "delta_sec | \n", "file_datetime | \n", "date_floored | \n", "floor_time | \n", "Secs_2GB | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00.992400 | \n", "2019-05-09 00:00:01.090 | \n", "2.419201e+06 | \n", "3.640205e+09 | \n", "2394625.789 | \n", "0 | \n", "2394657.25 | \n", "29.230175 | \n", "30352 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:00 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:01.992400 | \n", "2019-05-09 00:00:02.090 | \n", "2.419202e+06 | \n", "3.640205e+09 | \n", "2394626.789 | \n", "0 | \n", "2394658.25 | \n", "29.203979 | \n", "30352 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23863.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:01 | \n", "2419202 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:02.992400 | \n", "2019-05-09 00:00:03.020 | \n", "2.419203e+06 | \n", "3.640205e+09 | \n", "2394627.716 | \n", "0 | \n", "2394659.25 | \n", "29.230175 | \n", "30352 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23864.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:02 | \n", "2419203 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:03.992400 | \n", "2019-05-09 00:00:04.050 | \n", "2.419204e+06 | \n", "3.640205e+09 | \n", "2394628.747 | \n", "0 | \n", "2394660.25 | \n", "29.230175 | \n", "30352 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23865.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:03 | \n", "2419204 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:04.992400 | \n", "2019-05-09 00:00:05.070 | \n", "2.419205e+06 | \n", "3.640205e+09 | \n", "2394629.760 | \n", "0 | \n", "2394661.25 | \n", "29.230175 | \n", "30352 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23866.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:04 | \n", "2419205 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 151 columns
\n", "| \n", " | Time (sec) | \n", "Packet Time Stamp | \n", "Flag | \n", "Dropped Records | \n", "Record Count | \n", "Record Size | \n", "Particle Time Stamp | \n", "Particle Flags | \n", "Scatter relPeak | \n", "Scatter Transit Time | \n", "... | \n", "t0 | \n", "file | \n", "folder_name | \n", "delta_sec | \n", "file_datetime | \n", "date_floored | \n", "floor_time | \n", "Secs_2GB | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419232.25 | \n", "1 | \n", "2015056.0 | \n", "904.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "2107910.0 | \n", "920.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "10092470.0 | \n", "1282.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "873502.0 | \n", "822.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "36306660.0 | \n", "1582.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 32 columns
\n", "| \n", " | Dropped Records | \n", "Incand Mass (fg) | \n", "BC mass | \n", "BC mass within range | \n", "cnts_thin | \n", "cnts_thin_noScatt | \n", "cnts_thick | \n", "cnts_thick_sat | \n", "cnts_thin_sat | \n", "cnts_ntl_sat | \n", "... | \n", "BC_massConc_within_range_std | \n", "BC_massConc_within_range_vol | \n", "BC_numConc_within_range_std | \n", "BC_numConc_within_range_vol | \n", "S_numConc_std | \n", "S_numConc_vol | \n", "S_numConc_within_range_std | \n", "S_numConc_within_range_vol | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:01:00 | \n", "0.0 | \n", "16.855231 | \n", "174.442428 | \n", "159.740085 | \n", "22 | \n", "6 | \n", "52 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.005323 | \n", "0.004794 | \n", "2.732726 | \n", "2.460738 | \n", "125.238836 | \n", "112.773832 | \n", "124.872251 | \n", "112.443733 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:02:00 | \n", "0.0 | \n", "30.337629 | \n", "192.119622 | \n", "177.837469 | \n", "29 | \n", "1 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.005929 | \n", "0.005339 | \n", "2.200244 | \n", "1.981288 | \n", "120.646739 | \n", "108.640617 | \n", "120.613402 | \n", "108.610597 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:03:00 | \n", "0.0 | \n", "30.950136 | \n", "198.343122 | \n", "188.532185 | \n", "24 | \n", "5 | \n", "55 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.006284 | \n", "0.00566 | \n", "2.799689 | \n", "2.522018 | \n", "117.786913 | \n", "106.104884 | \n", "117.586935 | \n", "105.92474 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:04:00 | \n", "0.0 | \n", "44.184496 | \n", "220.411625 | \n", "206.865198 | \n", "17 | \n", "7 | \n", "65 | \n", "0 | \n", "0 | \n", "1 | \n", "... | \n", "0.006898 | \n", "0.006212 | \n", "3.034345 | \n", "2.732733 | \n", "118.772924 | \n", "106.966968 | \n", "118.506169 | \n", "106.726727 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:05:00 | \n", "0.0 | \n", "50.056468 | \n", "230.294057 | \n", "217.496214 | \n", "27 | \n", "2 | \n", "58 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.007251 | \n", "0.00653 | \n", "2.900645 | \n", "2.61209 | \n", "119.993332 | \n", "108.056445 | \n", "119.793287 | \n", "107.876301 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 39 columns
\n", "| \n", " | dNdlogDmev_all_70.03 | \n", "dNdlogDmev_all_73.55 | \n", "dNdlogDmev_all_77.24 | \n", "dNdlogDmev_all_81.11 | \n", "dNdlogDmev_all_85.18 | \n", "dNdlogDmev_all_89.45 | \n", "dNdlogDmev_all_93.94 | \n", "dNdlogDmev_all_98.65 | \n", "dNdlogDmev_all_103.60 | \n", "dNdlogDmev_all_108.80 | \n", "... | \n", "dNdlogDsc_all_223.81 | \n", "dNdlogDsc_all_243.59 | \n", "dNdlogDsc_all_265.12 | \n", "dNdlogDsc_all_288.56 | \n", "dNdlogDsc_all_314.07 | \n", "dNdlogDsc_all_341.83 | \n", "dNdlogDsc_all_372.05 | \n", "dNdlogDsc_all_404.94 | \n", "dNdlogDsc_all_440.74 | \n", "dNdlogDsc_all_479.70 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00 | \n", "5.509758 | \n", "7.060021 | \n", "7.060021 | \n", "7.060021 | \n", "4.236013 | \n", "11.296034 | \n", "5.648017 | \n", "9.88403 | \n", "12.708039 | \n", "4.236013 | \n", "... | \n", "247.228126 | \n", "213.774815 | \n", "179.50557 | \n", "142.788522 | \n", "77.513769 | \n", "64.458818 | \n", "31.005508 | \n", "20.39836 | \n", "10.607147 | \n", "1.701024 | \n", "
| 2019-05-09 00:01:00 | \n", "11.01676 | \n", "8.469907 | \n", "7.058256 | \n", "5.646605 | \n", "4.234953 | \n", "14.116512 | \n", "4.234953 | \n", "5.646605 | \n", "7.058256 | \n", "4.234953 | \n", "... | \n", "252.876411 | \n", "235.746073 | \n", "200.669668 | \n", "132.964048 | \n", "75.862923 | \n", "53.838204 | \n", "28.550562 | \n", "16.314607 | \n", "9.788764 | \n", "0.850299 | \n", "
| 2019-05-09 00:02:00 | \n", "8.265464 | \n", "4.236437 | \n", "4.236437 | \n", "1.412146 | \n", "1.412146 | \n", "2.824291 | \n", "5.648582 | \n", "4.236437 | \n", "5.648582 | \n", "2.824291 | \n", "... | \n", "238.276687 | \n", "172.995403 | \n", "167.283291 | \n", "137.090697 | \n", "80.785589 | \n", "41.616819 | \n", "28.560562 | \n", "16.320321 | \n", "13.056257 | \n", "4.252986 | \n", "
| 2019-05-09 00:03:00 | \n", "2.755568 | \n", "5.64943 | \n", "11.29886 | \n", "7.061788 | \n", "7.061788 | \n", "11.29886 | \n", "5.64943 | \n", "7.061788 | \n", "1.412358 | \n", "9.886503 | \n", "... | \n", "215.460578 | \n", "194.240976 | \n", "171.389096 | \n", "115.075536 | \n", "87.326825 | \n", "53.865144 | \n", "37.542373 | \n", "17.13891 | \n", "6.529108 | \n", "6.805799 | \n", "
| 2019-05-09 00:04:00 | \n", "12.402539 | \n", "8.475842 | \n", "8.475842 | \n", "12.713763 | \n", "7.063202 | \n", "8.475842 | \n", "5.650561 | \n", "11.301123 | \n", "1.41264 | \n", "2.825281 | \n", "... | \n", "235.09496 | \n", "191.830957 | \n", "184.48424 | \n", "131.424613 | \n", "93.874724 | \n", "53.059626 | \n", "29.38687 | \n", "12.244529 | \n", "5.714114 | \n", "2.552685 | \n", "
5 rows × 1293 columns
\n", "