5505-public/dima
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
Files
83cec97e8363f7a24f5522ecdb55ff1c5b6b6af4
dima/docs/build/html/_modules/src/g5505_file_reader.html
T
florez_j 32bba4239a Synch with remote repo
2025-02-03 10:31:48 +01:00

448 lines
51 KiB
HTML
Raw Blame History

<!DOCTYPE html>
<html class="writer-html5" lang="en" data-content_root="../../">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>src.g5505_file_reader &mdash; DIMA 1.0.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=19f00094" />
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
<script src="../../_static/jquery.js?v=5d32c60e"></script>
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
<script src="../../_static/documentation_options.js?v=8d563738"></script>
<script src="../../_static/doctools.js?v=9a2dae69"></script>
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
<script src="../../_static/js/theme.js"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../index.html" class="icon icon-home">
DIMA
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html">HDF5 data operations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.hdf5_lib">Data integration with HDF5</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.metadata_review_lib">Data annotation and review</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.hdf5_vis">Data visualization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.g5505_file_reader">Utilities</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">DIMA</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="Page navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
<li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
<li class="breadcrumb-item active">src.g5505_file_reader</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for src.g5505_file_reader</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">import</span> <span class="nn">collections</span>
<span class="kn">from</span> <span class="nn">igor2.binarywave</span> <span class="kn">import</span> <span class="n">load</span> <span class="k">as</span> <span class="n">loadibw</span>
<span class="kn">import</span> <span class="nn">src.g5505_utils</span> <span class="k">as</span> <span class="nn">utils</span>
<span class="c1">#import src.metadata_review_lib as metadata</span>
<span class="c1">#from src.metadata_review_lib import parse_attribute</span>
<span class="kn">import</span> <span class="nn">yaml</span>
<span class="kn">import</span> <span class="nn">h5py</span>
<span class="n">ROOT_DIR</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">curdir</span><span class="p">)</span>
<div class="viewcode-block" id="read_xps_ibw_file_as_dict">
<a class="viewcode-back" href="../../modules/src.html#src.g5505_file_reader.read_xps_ibw_file_as_dict">[docs]</a>
<span class="k">def</span> <span class="nf">read_xps_ibw_file_as_dict</span><span class="p">(</span><span class="n">filename</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Reads IBW files from the Multiphase Chemistry Group, which contain XPS spectra and acquisition settings,</span>
<span class="sd"> and formats the data into a dictionary with the structure {datasets: list of datasets}. Each dataset in the</span>
<span class="sd"> list has the following structure:</span>
<span class="sd"> {</span>
<span class="sd"> &#39;name&#39;: &#39;name&#39;,</span>
<span class="sd"> &#39;data&#39;: data_array,</span>
<span class="sd"> &#39;data_units&#39;: &#39;units&#39;,</span>
<span class="sd"> &#39;shape&#39;: data_shape,</span>
<span class="sd"> &#39;dtype&#39;: data_type</span>
<span class="sd"> }</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> filename : str</span>
<span class="sd"> The IBW filename from the Multiphase Chemistry Group beamline.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> file_dict : dict</span>
<span class="sd"> A dictionary containing the datasets from the IBW file. </span>
<span class="sd"> Raises</span>
<span class="sd"> ------</span>
<span class="sd"> ValueError</span>
<span class="sd"> If the input IBW file is not a valid IBW file.</span>
<span class="sd"> </span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">file_obj</span> <span class="o">=</span> <span class="n">loadibw</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
<span class="n">required_keys</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;wData&#39;</span><span class="p">,</span><span class="s1">&#39;data_units&#39;</span><span class="p">,</span><span class="s1">&#39;dimension_units&#39;</span><span class="p">,</span><span class="s1">&#39;note&#39;</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">sum</span><span class="p">([</span><span class="n">item</span> <span class="ow">in</span> <span class="n">required_keys</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">file_obj</span><span class="p">[</span><span class="s1">&#39;wave&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">keys</span><span class="p">()])</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">required_keys</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;This is not a valid xps ibw file. It does not satisfy minimum adimissibility criteria.&#39;</span><span class="p">)</span>
<span class="n">file_dict</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">path_tail</span><span class="p">,</span> <span class="n">path_head</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
<span class="c1"># Group name and attributes</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">path_head</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;attributes_dict&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1"># Convert notes of bytes class to string class and split string into a list of elements separated by &#39;\r&#39;. </span>
<span class="n">notes_list</span> <span class="o">=</span> <span class="n">file_obj</span><span class="p">[</span><span class="s1">&#39;wave&#39;</span><span class="p">][</span><span class="s1">&#39;note&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\r</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">exclude_list</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;Excitation Energy&#39;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">notes_list</span><span class="p">:</span>
<span class="k">if</span> <span class="s1">&#39;=&#39;</span> <span class="ow">in</span> <span class="n">item</span><span class="p">:</span>
<span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">item</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;=&#39;</span><span class="p">))</span>
<span class="c1"># TODO: check if value can be converted into a numeric type. Now all values are string type</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">exclude_list</span><span class="p">:</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;attributes_dict&#39;</span><span class="p">][</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
<span class="c1"># TODO: talk to Thorsten to see if there is an easier way to access the below attributes</span>
<span class="n">dimension_labels</span> <span class="o">=</span> <span class="n">file_obj</span><span class="p">[</span><span class="s1">&#39;wave&#39;</span><span class="p">][</span><span class="s1">&#39;dimension_units&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;]&#39;</span><span class="p">)</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;attributes_dict&#39;</span><span class="p">][</span><span class="s1">&#39;dimension_units&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="o">+</span><span class="s1">&#39;]&#39;</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">dimension_labels</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="nb">len</span><span class="p">(</span><span class="n">dimension_labels</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span>
<span class="c1"># Datasets and their attributes</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;datasets&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">dataset</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;spectrum&#39;</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_obj</span><span class="p">[</span><span class="s1">&#39;wave&#39;</span><span class="p">][</span><span class="s1">&#39;wData&#39;</span><span class="p">]</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data_units&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">file_obj</span><span class="p">[</span><span class="s1">&#39;wave&#39;</span><span class="p">][</span><span class="s1">&#39;data_units&#39;</span><span class="p">]</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;shape&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;dtype&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">])</span>
<span class="c1"># TODO: include energy axis dataset</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;datasets&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
<span class="k">return</span> <span class="n">file_dict</span></div>
<div class="viewcode-block" id="copy_file_in_group">
<a class="viewcode-back" href="../../modules/src.html#src.g5505_file_reader.copy_file_in_group">[docs]</a>
<span class="k">def</span> <span class="nf">copy_file_in_group</span><span class="p">(</span><span class="n">source_file_path</span><span class="p">,</span> <span class="n">dest_file_obj</span> <span class="p">:</span> <span class="n">h5py</span><span class="o">.</span><span class="n">File</span><span class="p">,</span> <span class="n">dest_group_name</span><span class="p">,</span> <span class="n">work_with_copy</span> <span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
<span class="c1"># Create copy of original file to avoid possible file corruption and work with it.</span>
<span class="k">if</span> <span class="n">work_with_copy</span><span class="p">:</span>
<span class="n">tmp_file_path</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">make_file_copy</span><span class="p">(</span><span class="n">source_file_path</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">tmp_file_path</span> <span class="o">=</span> <span class="n">source_file_path</span>
<span class="c1"># Open backup h5 file and copy complet filesystem directory onto a group in h5file</span>
<span class="k">with</span> <span class="n">h5py</span><span class="o">.</span><span class="n">File</span><span class="p">(</span><span class="n">tmp_file_path</span><span class="p">,</span><span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">src_file</span><span class="p">:</span>
<span class="n">dest_file_obj</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">source</span><span class="o">=</span> <span class="n">src_file</span><span class="p">[</span><span class="s1">&#39;/&#39;</span><span class="p">],</span> <span class="n">dest</span><span class="o">=</span> <span class="n">dest_group_name</span><span class="p">)</span>
<span class="k">if</span> <span class="s1">&#39;tmp_files&#39;</span> <span class="ow">in</span> <span class="n">tmp_file_path</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">tmp_file_path</span><span class="p">)</span></div>
<div class="viewcode-block" id="read_txt_files_as_dict">
<a class="viewcode-back" href="../../modules/src.html#src.g5505_file_reader.read_txt_files_as_dict">[docs]</a>
<span class="k">def</span> <span class="nf">read_txt_files_as_dict</span><span class="p">(</span><span class="n">filename</span> <span class="p">:</span> <span class="nb">str</span> <span class="p">,</span> <span class="n">work_with_copy</span> <span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span> <span class="p">):</span>
<span class="c1"># Get the directory of the current module</span>
<span class="n">module_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span>
<span class="c1"># Construct the relative file path</span>
<span class="n">instrument_configs_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">module_dir</span><span class="p">,</span> <span class="s1">&#39;instruments&#39;</span><span class="p">,</span> <span class="s1">&#39;text_data_sources.yaml&#39;</span><span class="p">)</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">instrument_configs_path</span><span class="p">,</span><span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">stream</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">config_dict</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">stream</span><span class="p">,</span> <span class="n">Loader</span><span class="o">=</span><span class="n">yaml</span><span class="o">.</span><span class="n">FullLoader</span><span class="p">)</span>
<span class="k">except</span> <span class="n">yaml</span><span class="o">.</span><span class="n">YAMLError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
<span class="c1"># Verify if file can be read by available intrument configurations.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">key</span> <span class="ow">in</span> <span class="n">filename</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">sep</span><span class="p">,</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">config_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">()):</span>
<span class="k">return</span> <span class="p">{}</span>
<span class="c1">#TODO: this may be prone to error if assumed folder structure is non compliant </span>
<span class="n">file_encoding</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="s1">&#39;default&#39;</span><span class="p">][</span><span class="s1">&#39;file_encoding&#39;</span><span class="p">]</span> <span class="c1">#&#39;utf-8&#39;</span>
<span class="n">separator</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="s1">&#39;default&#39;</span><span class="p">][</span><span class="s1">&#39;separator&#39;</span><span class="p">]</span>
<span class="n">table_header</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="s1">&#39;default&#39;</span><span class="p">][</span><span class="s1">&#39;table_header&#39;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">config_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span><span class="n">os</span><span class="o">.</span><span class="n">sep</span><span class="p">)</span> <span class="ow">in</span> <span class="n">filename</span><span class="p">:</span>
<span class="n">file_encoding</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;file_encoding&#39;</span><span class="p">,</span><span class="n">file_encoding</span><span class="p">)</span>
<span class="n">separator</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;separator&#39;</span><span class="p">,</span><span class="n">separator</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">t&#39;</span><span class="p">,</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">)</span>
<span class="n">table_header</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;table_header&#39;</span><span class="p">,</span><span class="n">table_header</span><span class="p">)</span>
<span class="n">timestamp_variables</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;timestamp&#39;</span><span class="p">,[])</span>
<span class="n">datetime_format</span> <span class="o">=</span> <span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;datetime_format&#39;</span><span class="p">,[])</span>
<span class="n">description_dict</span> <span class="o">=</span> <span class="p">{}</span>
<span class="c1">#link_to_description = config_dict[key].get(&#39;link_to_description&#39;,[]).replace(&#39;/&#39;,os.sep)</span>
<span class="n">link_to_description</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">module_dir</span><span class="p">,</span><span class="n">config_dict</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;link_to_description&#39;</span><span class="p">,[])</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">,</span><span class="n">os</span><span class="o">.</span><span class="n">sep</span><span class="p">))</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">link_to_description</span><span class="p">,</span><span class="s1">&#39;r&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">stream</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">description_dict</span> <span class="o">=</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">stream</span><span class="p">,</span> <span class="n">Loader</span><span class="o">=</span><span class="n">yaml</span><span class="o">.</span><span class="n">FullLoader</span><span class="p">)</span>
<span class="k">except</span> <span class="n">yaml</span><span class="o">.</span><span class="n">YAMLError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
<span class="k">break</span>
<span class="c1">#if &#39;None&#39; in table_header:</span>
<span class="c1"># return {}</span>
<span class="c1"># Read header as a dictionary and detect where data table starts</span>
<span class="n">header_dict</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">data_start</span> <span class="o">=</span> <span class="kc">False</span>
<span class="c1"># Work with copy of the file for safety</span>
<span class="k">if</span> <span class="n">work_with_copy</span><span class="p">:</span>
<span class="n">tmp_filename</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">make_file_copy</span><span class="p">(</span><span class="n">source_file_path</span><span class="o">=</span><span class="n">filename</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">tmp_filename</span> <span class="o">=</span> <span class="n">filename</span>
<span class="c1">#with open(tmp_filename,&#39;rb&#39;,encoding=file_encoding,errors=&#39;ignore&#39;) as f:</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">tmp_filename</span><span class="p">,</span><span class="s1">&#39;rb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">table_preamble</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">line_number</span><span class="p">,</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
<span class="k">if</span> <span class="n">table_header</span> <span class="ow">in</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">file_encoding</span><span class="p">):</span>
<span class="n">list_of_substrings</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">file_encoding</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">separator</span><span class="p">)</span>
<span class="c1"># Count occurrences of each substring</span>
<span class="n">substring_counts</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">Counter</span><span class="p">(</span><span class="n">list_of_substrings</span><span class="p">)</span>
<span class="n">data_start</span> <span class="o">=</span> <span class="kc">True</span>
<span class="c1"># Generate column names with appended index only for repeated substrings</span>
<span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">name</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">if</span> <span class="n">substring_counts</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="k">else</span> <span class="n">name</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">list_of_substrings</span><span class="p">)]</span>
<span class="c1">#column_names = [str(i)+&#39;_&#39;+name.strip() for i, name in enumerate(list_of_substrings)]</span>
<span class="c1">#column_names = []</span>
<span class="c1">#for i, name in enumerate(list_of_substrings):</span>
<span class="c1"># column_names.append(str(i)+&#39;_&#39;+name) </span>
<span class="c1">#print(line_number, len(column_names ),&#39;\n&#39;)</span>
<span class="k">break</span>
<span class="c1"># Subdivide line into words, and join them by single space. </span>
<span class="c1"># I asumme this can produce a cleaner line that contains no weird separator characters \t \r or extra spaces and so on.</span>
<span class="n">list_of_substrings</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">file_encoding</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
<span class="c1"># TODO: ideally we should use a multilinear string but the yalm parser is not recognizing \n as special character</span>
<span class="c1">#line = &#39; &#39;.join(list_of_substrings+[&#39;\n&#39;])</span>
<span class="c1">#line = &#39; &#39;.join(list_of_substrings) </span>
<span class="n">table_preamble</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">item</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">list_of_substrings</span><span class="p">]))</span><span class="c1"># += new_line </span>
<span class="c1"># Represent string values as fixed length strings in the HDF5 file, which need</span>
<span class="c1"># to be decoded as string when we read them. It provides better control than variable strings,</span>
<span class="c1"># at the expense of flexibility.</span>
<span class="c1"># https://docs.h5py.org/en/stable/strings.html</span>
<span class="k">if</span> <span class="n">table_preamble</span><span class="p">:</span>
<span class="n">header_dict</span><span class="p">[</span><span class="s2">&quot;table_preamble&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">convert_string_to_bytes</span><span class="p">(</span><span class="n">table_preamble</span><span class="p">)</span>
<span class="c1"># TODO: it does not work with separator as none :(. fix for RGA</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">tmp_filename</span><span class="p">,</span>
<span class="n">delimiter</span> <span class="o">=</span> <span class="n">separator</span><span class="p">,</span>
<span class="n">header</span><span class="o">=</span><span class="n">line_number</span><span class="p">,</span>
<span class="c1">#encoding=&#39;latin-1&#39;,</span>
<span class="n">encoding</span> <span class="o">=</span> <span class="n">file_encoding</span><span class="p">,</span>
<span class="n">names</span><span class="o">=</span><span class="n">column_names</span><span class="p">,</span>
<span class="n">skip_blank_lines</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">df_numerical_attrs</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select_dtypes</span><span class="p">(</span><span class="n">include</span> <span class="o">=</span><span class="s1">&#39;number&#39;</span><span class="p">)</span>
<span class="n">df_categorical_attrs</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">select_dtypes</span><span class="p">(</span><span class="n">exclude</span><span class="o">=</span><span class="s1">&#39;number&#39;</span><span class="p">)</span>
<span class="n">numerical_variables</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">df_numerical_attrs</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
<span class="c1"># Consolidate into single timestamp column the separate columns &#39;date&#39; &#39;time&#39; specified in text_data_source.yaml</span>
<span class="k">if</span> <span class="n">timestamp_variables</span><span class="p">:</span>
<span class="c1">#df_categorical_attrs[&#39;timestamps&#39;] = [&#39; &#39;.join(df_categorical_attrs.loc[i,timestamp_variables].to_numpy()) for i in df.index]</span>
<span class="c1">#df_categorical_attrs[&#39;timestamps&#39;] = [ df_categorical_attrs.loc[i,&#39;0_Date&#39;]+&#39; &#39;+df_categorical_attrs.loc[i,&#39;1_Time&#39;] for i in df.index]</span>
<span class="c1">#df_categorical_attrs[&#39;timestamps&#39;] = df_categorical_attrs[timestamp_variables].astype(str).agg(&#39; &#39;.join, axis=1)</span>
<span class="n">timestamps_name</span> <span class="o">=</span> <span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">timestamp_variables</span><span class="p">)</span>
<span class="n">df_categorical_attrs</span><span class="p">[</span> <span class="n">timestamps_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamp_variables</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">str</span><span class="p">)</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="n">valid_indices</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">datetime_format</span><span class="p">:</span>
<span class="n">df_categorical_attrs</span><span class="p">[</span> <span class="n">timestamps_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">(</span><span class="n">df_categorical_attrs</span><span class="p">[</span> <span class="n">timestamps_name</span><span class="p">],</span><span class="nb">format</span><span class="o">=</span><span class="n">datetime_format</span><span class="p">,</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;coerce&#39;</span><span class="p">)</span>
<span class="n">valid_indices</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">])</span><span class="o">.</span><span class="n">index</span>
<span class="n">df_categorical_attrs</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">valid_indices</span><span class="p">,:]</span>
<span class="n">df_numerical_attrs</span> <span class="o">=</span> <span class="n">df_numerical_attrs</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">valid_indices</span><span class="p">,:]</span>
<span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span><span class="o">.</span><span class="n">dt</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">config_dict</span><span class="p">[</span><span class="s1">&#39;default&#39;</span><span class="p">][</span><span class="s1">&#39;desired_format&#39;</span><span class="p">])</span>
<span class="n">startdate</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span><span class="o">.</span><span class="n">min</span><span class="p">()</span>
<span class="n">enddate</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span><span class="o">.</span><span class="n">max</span><span class="p">()</span>
<span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="p">[</span><span class="n">timestamps_name</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">str</span><span class="p">)</span>
<span class="c1">#header_dict.update({&#39;stastrrtdate&#39;:startdate,&#39;enddate&#39;:enddate})</span>
<span class="n">header_dict</span><span class="p">[</span><span class="s1">&#39;startdate&#39;</span><span class="p">]</span><span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">startdate</span><span class="p">)</span>
<span class="n">header_dict</span><span class="p">[</span><span class="s1">&#39;enddate&#39;</span><span class="p">]</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">enddate</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">timestamp_variables</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">df_categorical_attrs</span> <span class="o">=</span> <span class="n">df_categorical_attrs</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">columns</span> <span class="o">=</span> <span class="n">timestamp_variables</span><span class="p">)</span>
<span class="c1">#df_categorical_attrs.reindex(drop=True)</span>
<span class="c1">#df_numerical_attrs.reindex(drop=True)</span>
<span class="n">categorical_variables</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">df_categorical_attrs</span><span class="o">.</span><span class="n">columns</span><span class="p">]</span>
<span class="c1">####</span>
<span class="c1">#elif &#39;RGA&#39; in filename:</span>
<span class="c1"># df_categorical_attrs = df_categorical_attrs.rename(columns={&#39;0_Time(s)&#39; : &#39;timestamps&#39;})</span>
<span class="c1">###</span>
<span class="n">file_dict</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">path_tail</span><span class="p">,</span> <span class="n">path_head</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">tmp_filename</span><span class="p">)</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">path_head</span>
<span class="c1"># TODO: review this header dictionary, it may not be the best way to represent header data</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;attributes_dict&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">header_dict</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;datasets&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1">####</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">((</span><span class="n">df_categorical_attrs</span><span class="p">,</span><span class="n">df_numerical_attrs</span><span class="p">),</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="c1">#if numerical_variables:</span>
<span class="n">dataset</span> <span class="o">=</span> <span class="p">{}</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;data_table&#39;</span><span class="c1">#_numerical_variables&#39;</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">dataframe_to_np_structured_array</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="c1">#df_numerical_attrs.to_numpy()</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;shape&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;dtype&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;data&#39;</span><span class="p">])</span>
<span class="c1">#dataset[&#39;data_units&#39;] = file_obj[&#39;wave&#39;][&#39;data_units&#39;] </span>
<span class="c1"># </span>
<span class="c1"># Create attribute descriptions based on description_dict</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;attributes&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">for</span> <span class="n">column_name</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
<span class="n">column_attr_dict</span> <span class="o">=</span> <span class="n">description_dict</span><span class="p">[</span><span class="s1">&#39;table_header&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">column_name</span><span class="p">,</span>
<span class="p">{</span><span class="s1">&#39;note&#39;</span><span class="p">:</span><span class="s1">&#39;there was no description available. Review instrument files.&#39;</span><span class="p">})</span>
<span class="n">dataset</span><span class="p">[</span><span class="s1">&#39;attributes&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">column_name</span><span class="p">:</span> <span class="n">utils</span><span class="o">.</span><span class="n">parse_attribute</span><span class="p">(</span><span class="n">column_attr_dict</span><span class="p">)})</span>
<span class="c1">#try:</span>
<span class="c1"># dataset[&#39;attributes&#39;] = description_dict[&#39;table_header&#39;].copy()</span>
<span class="c1"># for key in description_dict[&#39;table_header&#39;].keys():</span>
<span class="c1"># if not key in numerical_variables:</span>
<span class="c1"># dataset[&#39;attributes&#39;].pop(key) # delete key</span>
<span class="c1"># else:</span>
<span class="c1"># dataset[&#39;attributes&#39;][key] = utils.parse_attribute(dataset[&#39;attributes&#39;][key])</span>
<span class="c1"># if timestamps_name in categorical_variables:</span>
<span class="c1"># dataset[&#39;attributes&#39;][timestamps_name] = utils.parse_attribute({&#39;unit&#39;:&#39;YYYY-MM-DD HH:MM:SS.ffffff&#39;})</span>
<span class="c1">#except ValueError as err:</span>
<span class="c1"># print(err)</span>
<span class="n">file_dict</span><span class="p">[</span><span class="s1">&#39;datasets&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
<span class="c1">#if categorical_variables:</span>
<span class="c1"># dataset = {}</span>
<span class="c1"># dataset[&#39;name&#39;] = &#39;table_categorical_variables&#39;</span>
<span class="c1"># dataset[&#39;data&#39;] = dataframe_to_np_structured_array(df_categorical_attrs) #df_categorical_attrs.loc[:,categorical_variables].to_numpy()</span>
<span class="c1"># dataset[&#39;shape&#39;] = dataset[&#39;data&#39;].shape</span>
<span class="c1"># dataset[&#39;dtype&#39;] = type(dataset[&#39;data&#39;])</span>
<span class="c1"># if timestamps_name in categorical_variables:</span>
<span class="c1"># dataset[&#39;attributes&#39;] = {timestamps_name: utils.parse_attribute({&#39;unit&#39;:&#39;YYYY-MM-DD HH:MM:SS.ffffff&#39;})}</span>
<span class="c1"># file_dict[&#39;datasets&#39;].append(dataset) </span>
<span class="k">except</span><span class="p">:</span>
<span class="k">return</span> <span class="p">{}</span>
<span class="k">return</span> <span class="n">file_dict</span></div>
<div class="viewcode-block" id="main">
<a class="viewcode-back" href="../../modules/src.html#src.g5505_file_reader.main">[docs]</a>
<span class="k">def</span> <span class="nf">main</span><span class="p">():</span>
<span class="n">inputfile_dir</span> <span class="o">=</span> <span class="s1">&#39;</span><span class="se">\\\\</span><span class="s1">fs101</span><span class="se">\\</span><span class="s1">5505</span><span class="se">\\</span><span class="s1">People</span><span class="se">\\</span><span class="s1">Juan</span><span class="se">\\</span><span class="s1">TypicalBeamTime&#39;</span>
<span class="n">file_dict</span> <span class="o">=</span> <span class="n">read_xps_ibw_file_as_dict</span><span class="p">(</span><span class="n">inputfile_dir</span><span class="o">+</span><span class="s1">&#39;</span><span class="se">\\</span><span class="s1">SES</span><span class="se">\\</span><span class="s1">0069069_N1s_495eV.ibw&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">file_dict</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="nb">print</span><span class="p">(</span><span class="n">key</span><span class="p">,</span><span class="n">file_dict</span><span class="p">[</span><span class="n">key</span><span class="p">])</span></div>
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
<span class="n">main</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="s1">&#39;:)&#39;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>&#169; Copyright 2024, JFFO.</p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script>
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink