{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "d85e9883", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import matplotlib.dates as mdates" ] }, { "cell_type": "markdown", "id": "d7eb024f", "metadata": {}, "source": [ "### Unprocessed parquet files" ] }, { "cell_type": "code", "execution_count": 6, "id": "a1df7aa6", "metadata": {}, "outputs": [], "source": [ "selection = [(\"date\", \">=\", \"2019-05-09\")]\n", "hk_files = pd.read_parquet('data/hk_files_parquet_2', filters=selection).sort_index()\n", "pbp_files = pd.read_parquet('data/pbp_files_parquet_2', filters=selection).sort_index()" ] }, { "cell_type": "code", "execution_count": 7, "id": "130d8bbc", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | Time Stamp | \n", "Time (sec) | \n", "Time Stamp (UTC sec) | \n", "Elapsed Time | \n", "Error Code | \n", "Packet Time Stamp | \n", "Laser TEC Temp (C) | \n", "Crystal TEC Temp (C) | \n", "Inlet Air Temp (C) | \n", "Computer Heatsink Temp (C) | \n", "... | \n", "t0 | \n", "file | \n", "folder_name | \n", "delta_sec | \n", "file_datetime | \n", "date_floored | \n", "floor_time | \n", "Secs_2GB | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00.992400 | \n", "2019-05-09 00:00:01.090 | \n", "2.419201e+06 | \n", "3.640205e+09 | \n", "2394625.789 | \n", "0.0 | \n", "2394657.25 | \n", "29.230175 | \n", "30352.0 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:00 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:01.992400 | \n", "2019-05-09 00:00:02.090 | \n", "2.419202e+06 | \n", "3.640205e+09 | \n", "2394626.789 | \n", "0.0 | \n", "2394658.25 | \n", "29.203979 | \n", "30352.0 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23863.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:01 | \n", "2419202 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:02.992400 | \n", "2019-05-09 00:00:03.020 | \n", "2.419203e+06 | \n", "3.640205e+09 | \n", "2394627.716 | \n", "0.0 | \n", "2394659.25 | \n", "29.230175 | \n", "30352.0 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23864.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:02 | \n", "2419203 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:03.992400 | \n", "2019-05-09 00:00:04.050 | \n", "2.419204e+06 | \n", "3.640205e+09 | \n", "2394628.747 | \n", "0.0 | \n", "2394660.25 | \n", "29.230175 | \n", "30352.0 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23865.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:03 | \n", "2419204 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:04.992400 | \n", "2019-05-09 00:00:05.070 | \n", "2.419205e+06 | \n", "3.640205e+09 | \n", "2394629.760 | \n", "0.0 | \n", "2394661.25 | \n", "29.230175 | \n", "30352.0 | \n", "34.956111 | \n", "28.543153 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0002 | \n", "20190508172218 | \n", "23866.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 00:00:04 | \n", "2419205 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 151 columns
\n", "| \n", " | Time (sec) | \n", "Packet Time Stamp | \n", "Flag | \n", "Dropped Records | \n", "Record Count | \n", "Record Size | \n", "Particle Time Stamp | \n", "Particle Flags | \n", "Scatter relPeak | \n", "Scatter Transit Time | \n", "... | \n", "t0 | \n", "file | \n", "folder_name | \n", "delta_sec | \n", "file_datetime | \n", "date_floored | \n", "floor_time | \n", "Secs_2GB | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419232.25 | \n", "1 | \n", "2015056.0 | \n", "904.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "2107910.0 | \n", "920.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "10092470.0 | \n", "1282.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "873502.0 | \n", "822.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:00:00.992400 | \n", "2.419202e+06 | \n", "2419233.25 | \n", "0 | \n", "0 | \n", "76 | \n", "32 | \n", "2419233.00 | \n", "1 | \n", "36306660.0 | \n", "1582.0 | \n", "... | \n", "2019-05-08 17:22:18.400 | \n", "20190508172218_x0004 | \n", "20190508172218 | \n", "23862.5924 | \n", "2019-05-08 17:22:18 | \n", "2019-05-09 | \n", "2019-05-09 | \n", "2419201 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 32 columns
\n", "| \n", " | Dropped Records | \n", "Incand Mass (fg) | \n", "BC mass | \n", "BC mass within range | \n", "cnts_thin | \n", "cnts_thin_noScatt | \n", "cnts_thick | \n", "cnts_thick_sat | \n", "cnts_thin_sat | \n", "cnts_ntl_sat | \n", "... | \n", "BC_massConc_within_range_std | \n", "BC_massConc_within_range_vol | \n", "BC_numConc_within_range_std | \n", "BC_numConc_within_range_vol | \n", "S_numConc_std | \n", "S_numConc_vol | \n", "S_numConc_within_range_std | \n", "S_numConc_within_range_vol | \n", "date | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:01:00 | \n", "0.0 | \n", "16.855231 | \n", "152.94891 | \n", "138.284824 | \n", "22 | \n", "6 | \n", "52 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.004608 | \n", "0.00415 | \n", "2.732726 | \n", "2.460738 | \n", "125.238836 | \n", "112.773832 | \n", "124.872251 | \n", "112.443733 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:02:00 | \n", "0.0 | \n", "30.337629 | \n", "159.173818 | \n", "144.926793 | \n", "29 | \n", "1 | \n", "34 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.004831 | \n", "0.004351 | \n", "2.200244 | \n", "1.981288 | \n", "120.646739 | \n", "108.640617 | \n", "120.613402 | \n", "108.610597 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:03:00 | \n", "0.0 | \n", "30.950136 | \n", "160.031576 | \n", "150.240547 | \n", "24 | \n", "5 | \n", "55 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.005007 | \n", "0.004511 | \n", "2.799689 | \n", "2.522018 | \n", "117.786913 | \n", "106.104884 | \n", "117.586935 | \n", "105.92474 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:04:00 | \n", "0.0 | \n", "44.184496 | \n", "168.535785 | \n", "155.026056 | \n", "17 | \n", "7 | \n", "65 | \n", "0 | \n", "0 | \n", "1 | \n", "... | \n", "0.005169 | \n", "0.004655 | \n", "3.034345 | \n", "2.732733 | \n", "118.772924 | \n", "106.966968 | \n", "118.506169 | \n", "106.726727 | \n", "2019-05-09 | \n", "0 | \n", "
| 2019-05-09 00:05:00 | \n", "0.0 | \n", "50.056468 | \n", "161.55546 | \n", "148.487292 | \n", "27 | \n", "2 | \n", "57 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0.004951 | \n", "0.004458 | \n", "2.867304 | \n", "2.582066 | \n", "119.993332 | \n", "108.056445 | \n", "119.793287 | \n", "107.876301 | \n", "2019-05-09 | \n", "0 | \n", "
5 rows × 39 columns
\n", "| \n", " | dNdlogDmev_all_70.03 | \n", "dNdlogDmev_all_73.55 | \n", "dNdlogDmev_all_77.24 | \n", "dNdlogDmev_all_81.11 | \n", "dNdlogDmev_all_85.18 | \n", "dNdlogDmev_all_89.45 | \n", "dNdlogDmev_all_93.94 | \n", "dNdlogDmev_all_98.65 | \n", "dNdlogDmev_all_103.60 | \n", "dNdlogDmev_all_108.80 | \n", "... | \n", "dNdlogDsc_all_223.81 | \n", "dNdlogDsc_all_243.59 | \n", "dNdlogDsc_all_265.12 | \n", "dNdlogDsc_all_288.56 | \n", "dNdlogDsc_all_314.07 | \n", "dNdlogDsc_all_341.83 | \n", "dNdlogDsc_all_372.05 | \n", "dNdlogDsc_all_404.94 | \n", "dNdlogDsc_all_440.74 | \n", "dNdlogDsc_all_479.70 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| calculated_time | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 2019-05-09 00:00:00 | \n", "5.509758 | \n", "8.472026 | \n", "5.648017 | \n", "7.060021 | \n", "4.236013 | \n", "11.296034 | \n", "9.88403 | \n", "8.472026 | \n", "11.296034 | \n", "8.472026 | \n", "... | \n", "250.491864 | \n", "210.511078 | \n", "180.321504 | \n", "142.788522 | \n", "77.513769 | \n", "62.82695 | \n", "31.005508 | \n", "20.39836 | \n", "10.607147 | \n", "1.701024 | \n", "
| 2019-05-09 00:01:00 | \n", "11.01676 | \n", "9.881558 | \n", "5.646605 | \n", "5.646605 | \n", "7.058256 | \n", "11.293209 | \n", "4.234953 | \n", "7.058256 | \n", "7.058256 | \n", "2.823302 | \n", "... | \n", "255.323602 | \n", "234.114612 | \n", "199.853937 | \n", "132.964048 | \n", "75.047193 | \n", "53.838204 | \n", "28.550562 | \n", "15.498877 | \n", "9.788764 | \n", "0.850299 | \n", "
| 2019-05-09 00:02:00 | \n", "8.265464 | \n", "5.648582 | \n", "2.824291 | \n", "1.412146 | \n", "1.412146 | \n", "2.824291 | \n", "8.472873 | \n", "2.824291 | \n", "5.648582 | \n", "1.412146 | \n", "... | \n", "235.828639 | \n", "176.259467 | \n", "167.283291 | \n", "137.090697 | \n", "77.521525 | \n", "44.880883 | \n", "25.296498 | \n", "16.320321 | \n", "13.056257 | \n", "4.252986 | \n", "
| 2019-05-09 00:03:00 | \n", "5.511136 | \n", "2.824715 | \n", "11.29886 | \n", "8.474145 | \n", "5.64943 | \n", "12.711218 | \n", "5.64943 | \n", "7.061788 | \n", "4.237073 | \n", "5.64943 | \n", "... | \n", "213.012162 | \n", "191.79256 | \n", "171.389096 | \n", "118.34009 | \n", "84.87841 | \n", "54.681283 | \n", "35.910096 | \n", "17.13891 | \n", "6.529108 | \n", "6.805799 | \n", "
| 2019-05-09 00:04:00 | \n", "12.402539 | \n", "8.475842 | \n", "8.475842 | \n", "12.713763 | \n", "7.063202 | \n", "11.301123 | \n", "4.237921 | \n", "9.888482 | \n", "1.41264 | \n", "5.650561 | \n", "... | \n", "235.911262 | \n", "190.198353 | \n", "185.300541 | \n", "131.424613 | \n", "93.058422 | \n", "52.243324 | \n", "29.38687 | \n", "11.428227 | \n", "5.714114 | \n", "2.552685 | \n", "
5 rows × 1293 columns
\n", "