Add draft of dima documentation
This commit is contained in:
109
docs/build/html/modules/instruments.html
vendored
Normal file
109
docs/build/html/modules/instruments.html
vendored
Normal file
@ -0,0 +1,109 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title><no title> — DIMA 1.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
||||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav">
|
||||
<div class="wy-grid-for-nav">
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||
<div class="wy-side-scroll">
|
||||
<div class="wy-side-nav-search" >
|
||||
|
||||
|
||||
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
DIMA
|
||||
</a>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
<input type="hidden" name="check_keywords" value="yes" />
|
||||
<input type="hidden" name="area" value="default" />
|
||||
</form>
|
||||
</div>
|
||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html">HDF5 data operations</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#data-integration-with-hdf5">Data integration with HDF5</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-pipelines.data_integration">Pipelines and workflows</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_vis">Data visualization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-utils.g5505_utils">Utilities</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="utils.html">Utilities</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../index.html">DIMA</a>
|
||||
</nav>
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="Page navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||||
<li class="breadcrumb-item active"><no title></li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
<a href="../_sources/modules/instruments.rst.txt" rel="nofollow"> View page source</a>
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||
<div itemprop="articleBody">
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2024, JFFO.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
|
||||
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
jQuery(function () {
|
||||
SphinxRtdTheme.Navigation.enable(true);
|
||||
});
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
111
docs/build/html/modules/notebooks.html
vendored
Normal file
111
docs/build/html/modules/notebooks.html
vendored
Normal file
@ -0,0 +1,111 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Notebooks — DIMA 1.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
||||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav">
|
||||
<div class="wy-grid-for-nav">
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||
<div class="wy-side-scroll">
|
||||
<div class="wy-side-nav-search" >
|
||||
|
||||
|
||||
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
DIMA
|
||||
</a>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
<input type="hidden" name="check_keywords" value="yes" />
|
||||
<input type="hidden" name="area" value="default" />
|
||||
</form>
|
||||
</div>
|
||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||
<ul>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html">HDF5 data operations</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_writer">Data integration with HDF5</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_vis">Data visualization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="pipelines.html">Pipelines and workflows</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="utils.html">Utilities</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../index.html">DIMA</a>
|
||||
</nav>
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="Page navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||||
<li class="breadcrumb-item active">Notebooks</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
<a href="../_sources/modules/notebooks.rst.txt" rel="nofollow"> View page source</a>
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="module-notebooks">
|
||||
<span id="notebooks"></span><h1>Notebooks<a class="headerlink" href="#module-notebooks" title="Link to this heading"></a></h1>
|
||||
</section>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2024, JFFO.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
|
||||
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
jQuery(function () {
|
||||
SphinxRtdTheme.Navigation.enable(true);
|
||||
});
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
194
docs/build/html/modules/pipelines.html
vendored
Normal file
194
docs/build/html/modules/pipelines.html
vendored
Normal file
@ -0,0 +1,194 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Pipelines and workflows — DIMA 1.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
||||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
<link rel="next" title="Utilities" href="utils.html" />
|
||||
<link rel="prev" title="HDF5 data operations" href="src.html" />
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav">
|
||||
<div class="wy-grid-for-nav">
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||
<div class="wy-side-scroll">
|
||||
<div class="wy-side-nav-search" >
|
||||
|
||||
|
||||
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
DIMA
|
||||
</a>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
<input type="hidden" name="check_keywords" value="yes" />
|
||||
<input type="hidden" name="area" value="default" />
|
||||
</form>
|
||||
</div>
|
||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html">HDF5 data operations</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_writer">Data integration with HDF5</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_vis">Data visualization</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Pipelines and workflows</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.copy_subtree_and_create_hdf5"><code class="docutils literal notranslate"><span class="pre">copy_subtree_and_create_hdf5()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.load_config_and_setup_logging"><code class="docutils literal notranslate"><span class="pre">load_config_and_setup_logging()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.run_pipeline"><code class="docutils literal notranslate"><span class="pre">run_pipeline()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.count"><code class="docutils literal notranslate"><span class="pre">count()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.load_yaml"><code class="docutils literal notranslate"><span class="pre">load_yaml()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.update_hdf5_file_with_review"><code class="docutils literal notranslate"><span class="pre">update_hdf5_file_with_review()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.validate_yaml_dict"><code class="docutils literal notranslate"><span class="pre">validate_yaml_dict()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="utils.html">Utilities</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../index.html">DIMA</a>
|
||||
</nav>
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="Page navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||||
<li class="breadcrumb-item active">Pipelines and workflows</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
<a href="../_sources/modules/pipelines.rst.txt" rel="nofollow"> View page source</a>
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="module-pipelines.data_integration">
|
||||
<span id="pipelines-and-workflows"></span><h1>Pipelines and workflows<a class="headerlink" href="#module-pipelines.data_integration" title="Link to this heading"></a></h1>
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.data_integration.copy_subtree_and_create_hdf5">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">copy_subtree_and_create_hdf5</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dst</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_dir_keywords</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_file_keywords</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">allowed_file_extensions</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">root_metadata_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#copy_subtree_and_create_hdf5"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.copy_subtree_and_create_hdf5" title="Link to this definition"></a></dt>
|
||||
<dd><p>Helper function to copy directory with constraints and create HDF5.</p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.data_integration.load_config_and_setup_logging">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">load_config_and_setup_logging</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">yaml_config_file_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">log_dir</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#load_config_and_setup_logging"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.load_config_and_setup_logging" title="Link to this definition"></a></dt>
|
||||
<dd><p>Load YAML configuration file, set up logging, and validate required keys and datetime_steps.</p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.data_integration.run_pipeline">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">run_pipeline</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path_to_config_yamlFile</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">log_dir</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'logs/'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#run_pipeline"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.run_pipeline" title="Link to this definition"></a></dt>
|
||||
<dd><p>Integrates data sources specified by the input configuration file into HDF5 files.</p>
|
||||
<dl class="simple">
|
||||
<dt>Parameters:</dt><dd><p>yaml_config_file_path (str): Path to the YAML configuration file.
|
||||
log_dir (str): Directory to save the log file.</p>
|
||||
</dd>
|
||||
<dt>Returns:</dt><dd><p>list: List of Paths to the created HDF5 file(s).</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function" id="module-pipelines.metadata_revision">
|
||||
<dt class="sig sig-object py" id="pipelines.metadata_revision.count">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hdf5_obj</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">yml_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#count"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.count" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.metadata_revision.load_yaml">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">load_yaml</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">review_yaml_file</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#load_yaml"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.load_yaml" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.metadata_revision.update_hdf5_file_with_review">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">update_hdf5_file_with_review</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_hdf5_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">review_yaml_file</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#update_hdf5_file_with_review"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.update_hdf5_file_with_review" title="Link to this definition"></a></dt>
|
||||
<dd><p>Updates, appends, or deletes metadata attributes in an HDF5 file based on a provided YAML dictionary.</p>
|
||||
<section id="parameters">
|
||||
<h2>Parameters:<a class="headerlink" href="#parameters" title="Link to this heading"></a></h2>
|
||||
<dl>
|
||||
<dt>input_hdf5_file<span class="classifier">str</span></dt><dd><p>Path to the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>yaml_dict<span class="classifier">dict</span></dt><dd><p>Dictionary specifying objects and their attributes with operations. Example format:
|
||||
{</p>
|
||||
<blockquote>
|
||||
<div><dl>
|
||||
<dt>“object_name”: { “attributes”<span class="classifier">“attr_name”: { “value”: attr_value,</span></dt><dd><blockquote>
|
||||
<div><blockquote>
|
||||
<div><p>“delete”: true | false</p>
|
||||
</div></blockquote>
|
||||
<p>}</p>
|
||||
</div></blockquote>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</div></blockquote>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="pipelines.metadata_revision.validate_yaml_dict">
|
||||
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">validate_yaml_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_hdf5_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">yaml_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#validate_yaml_dict"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.validate_yaml_dict" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
</section>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||||
<a href="src.html" class="btn btn-neutral float-left" title="HDF5 data operations" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||||
<a href="utils.html" class="btn btn-neutral float-right" title="Utilities" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||||
</div>
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2024, JFFO.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
|
||||
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
jQuery(function () {
|
||||
SphinxRtdTheme.Navigation.enable(true);
|
||||
});
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
472
docs/build/html/modules/src.html
vendored
Normal file
472
docs/build/html/modules/src.html
vendored
Normal file
@ -0,0 +1,472 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>HDF5 Data Operations — DIMA 1.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
||||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
<link rel="next" title="Pipelines and workflows" href="pipelines.html" />
|
||||
<link rel="prev" title="Welcome to DIMA’s documentation!" href="../index.html" />
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav">
|
||||
<div class="wy-grid-for-nav">
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||
<div class="wy-side-scroll">
|
||||
<div class="wy-side-nav-search" >
|
||||
|
||||
|
||||
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
DIMA
|
||||
</a>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
<input type="hidden" name="check_keywords" value="yes" />
|
||||
<input type="hidden" name="area" value="default" />
|
||||
</form>
|
||||
</div>
|
||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">HDF5 Data Operations</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager</span></code></a><ul>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.append_dataset"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.append_dataset()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.append_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.append_metadata()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.delete_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.delete_metadata()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.extract_and_load_dataset_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.extract_and_load_dataset_metadata()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.extract_dataset_as_dataframe"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.extract_dataset_as_dataframe()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.get_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.get_metadata()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.load_file_obj"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.load_file_obj()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.reformat_datetime_column"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.reformat_datetime_column()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.rename_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.rename_metadata()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.unload_file_obj"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.unload_file_obj()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.update_file"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.update_file()</span></code></a></li>
|
||||
<li class="toctree-l3"><a class="reference internal" href="#src.hdf5_ops.HDF5DataOpsManager.update_metadata"><code class="docutils literal notranslate"><span class="pre">HDF5DataOpsManager.update_metadata()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_ops.get_groups_at_a_level"><code class="docutils literal notranslate"><span class="pre">get_groups_at_a_level()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_ops.get_parent_child_relationships"><code class="docutils literal notranslate"><span class="pre">get_parent_child_relationships()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_ops.read_mtable_as_dataframe"><code class="docutils literal notranslate"><span class="pre">read_mtable_as_dataframe()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_ops.serialize_metadata"><code class="docutils literal notranslate"><span class="pre">serialize_metadata()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="#module-src.hdf5_writer">HDF5 Writer</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_writer.create_hdf5_file_from_dataframe"><code class="docutils literal notranslate"><span class="pre">create_hdf5_file_from_dataframe()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_writer.create_hdf5_file_from_filesystem_path"><code class="docutils literal notranslate"><span class="pre">create_hdf5_file_from_filesystem_path()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_writer.save_processed_dataframe_to_hdf5"><code class="docutils literal notranslate"><span class="pre">save_processed_dataframe_to_hdf5()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="#module-src.hdf5_vis">Data Visualization</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#src.hdf5_vis.display_group_hierarchy_on_a_treemap"><code class="docutils literal notranslate"><span class="pre">display_group_hierarchy_on_a_treemap()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="pipelines.html">Pipelines and workflows</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="utils.html">Data Structure Conversion</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="notebooks.html">Notebooks</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../index.html">DIMA</a>
|
||||
</nav>
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="Page navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||||
<li class="breadcrumb-item active">HDF5 Data Operations</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
<a href="../_sources/modules/src.rst.txt" rel="nofollow"> View page source</a>
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="module-src.hdf5_ops">
|
||||
<span id="hdf5-data-operations"></span><h1>HDF5 Data Operations<a class="headerlink" href="#module-src.hdf5_ops" title="Link to this heading"></a></h1>
|
||||
<dl class="py class">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager">
|
||||
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">src.hdf5_ops.</span></span><span class="sig-name descname"><span class="pre">HDF5DataOpsManager</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">mode</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'r+'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager" title="Link to this definition"></a></dt>
|
||||
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
|
||||
<p>A class to handle HDF5 fundamental middle level file operations to power data updates, metadata revision, and data analysis
|
||||
with hdf5 files encoding multi-instrument experimental campaign data.</p>
|
||||
<section id="parameters">
|
||||
<h2>Parameters:<a class="headerlink" href="#parameters" title="Link to this heading"></a></h2>
|
||||
<blockquote>
|
||||
<div><dl class="simple">
|
||||
<dt>path_to_file<span class="classifier">str</span></dt><dd><p>path/to/hdf5file.</p>
|
||||
</dd>
|
||||
<dt>mode<span class="classifier">str</span></dt><dd><p>‘r’ or ‘r+’ read or read/write mode only when file exists</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</div></blockquote>
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.append_dataset">
|
||||
<span class="sig-name descname"><span class="pre">append_dataset</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dataset_dict</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">group_name</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.append_dataset"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.append_dataset" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.append_metadata">
|
||||
<span class="sig-name descname"><span class="pre">append_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">annotation_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.append_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.append_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Appends metadata attributes to the specified object (obj_name) based on the provided annotation_dict.</p>
|
||||
<p>This method ensures that the provided metadata attributes do not overwrite any existing ones. If an attribute already exists,
|
||||
a ValueError is raised. The function supports storing scalar values (int, float, str) and compound values such as dictionaries
|
||||
that are converted into NumPy structured arrays before being added to the metadata.</p>
|
||||
<section id="id1">
|
||||
<h3>Parameters:<a class="headerlink" href="#id1" title="Link to this heading"></a></h3>
|
||||
<dl>
|
||||
<dt>obj_name: str</dt><dd><p>Path to the target object (dataset or group) within the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>annotation_dict: dict</dt><dd><dl>
|
||||
<dt>A dictionary where the keys represent new attribute names (strings), and the values can be:</dt><dd><ul class="simple">
|
||||
<li><p>Scalars: int, float, or str.</p></li>
|
||||
<li><p>Compound values (dictionaries) for more complex metadata, which are converted to NumPy structured arrays.</p></li>
|
||||
</ul>
|
||||
<p>Example of a compound value:</p>
|
||||
<dl>
|
||||
<dt>annotation_dict = {</dt><dd><dl class="simple">
|
||||
<dt>“relative_humidity”: {</dt><dd><p>“value”: 65,
|
||||
“units”: “percentage”,
|
||||
“range”: “[0,100]”,
|
||||
“definition”: “amount of water vapor present …”</p>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.delete_metadata">
|
||||
<span class="sig-name descname"><span class="pre">delete_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">annotation_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.delete_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.delete_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Deletes metadata attributes of the specified object (obj_name) based on the provided annotation_dict.</p>
|
||||
<section id="id2">
|
||||
<h3>Parameters:<a class="headerlink" href="#id2" title="Link to this heading"></a></h3>
|
||||
<dl class="simple">
|
||||
<dt>obj_name: str</dt><dd><p>Path to the target object (dataset or group) within the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>annotation_dict: dict</dt><dd><p>Dictionary where keys represent attribute names, and values should be dictionaries containing
|
||||
{“delete”: True} to mark them for deletion.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="example">
|
||||
<h3>Example:<a class="headerlink" href="#example" title="Link to this heading"></a></h3>
|
||||
<p>annotation_dict = {“attr_to_be_deleted”: {“delete”: True}}</p>
|
||||
</section>
|
||||
<section id="behavior">
|
||||
<h3>Behavior:<a class="headerlink" href="#behavior" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p>Deletes the specified attributes from the object’s metadata if marked for deletion.</p></li>
|
||||
<li><p>Issues a warning if the attribute is not found or not marked for deletion.</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.extract_and_load_dataset_metadata">
|
||||
<span class="sig-name descname"><span class="pre">extract_and_load_dataset_metadata</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.extract_and_load_dataset_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.extract_and_load_dataset_metadata" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.extract_dataset_as_dataframe">
|
||||
<span class="sig-name descname"><span class="pre">extract_dataset_as_dataframe</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dataset_name</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.extract_dataset_as_dataframe"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.extract_dataset_as_dataframe" title="Link to this definition"></a></dt>
|
||||
<dd><p>returns a copy of the dataset content in the form of dataframe when possible or numpy array</p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.get_metadata">
|
||||
<span class="sig-name descname"><span class="pre">get_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_path</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.get_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.get_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Get file attributes from object at path = obj_path. For example,
|
||||
obj_path = ‘/’ will get root level attributes or metadata.</p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.load_file_obj">
|
||||
<span class="sig-name descname"><span class="pre">load_file_obj</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.load_file_obj"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.load_file_obj" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.reformat_datetime_column">
|
||||
<span class="sig-name descname"><span class="pre">reformat_datetime_column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">dataset_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">column_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">src_format</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">desired_format</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'%Y-%m-%d</span> <span class="pre">%H:%M:%S.%f'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.reformat_datetime_column"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.reformat_datetime_column" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.rename_metadata">
|
||||
<span class="sig-name descname"><span class="pre">rename_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">renaming_map</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.rename_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.rename_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Renames metadata attributes of the specified object (obj_name) based on the provided renaming_map.</p>
|
||||
<section id="id3">
|
||||
<h3>Parameters:<a class="headerlink" href="#id3" title="Link to this heading"></a></h3>
|
||||
<dl>
|
||||
<dt>obj_name: str</dt><dd><p>Path to the target object (dataset or group) within the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>renaming_map: dict</dt><dd><p>A dictionary where keys are current attribute names (strings), and values are the new attribute names (strings or byte strings) to rename to.</p>
|
||||
<dl class="simple">
|
||||
<dt>renaming_map = {</dt><dd><p>“old_attr_name”: “new_attr_name”,
|
||||
“old_attr_2”: “new_attr_2”</p>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.unload_file_obj">
|
||||
<span class="sig-name descname"><span class="pre">unload_file_obj</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.unload_file_obj"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.unload_file_obj" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.update_file">
|
||||
<span class="sig-name descname"><span class="pre">update_file</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path_to_append_dir</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.update_file"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.update_file" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py method">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.HDF5DataOpsManager.update_metadata">
|
||||
<span class="sig-name descname"><span class="pre">update_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj_name</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">annotation_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#HDF5DataOpsManager.update_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.HDF5DataOpsManager.update_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Updates the value of existing metadata attributes of the specified object (obj_name) based on the provided annotation_dict.</p>
|
||||
<p>The function disregards non-existing attributes and suggests to use the append_metadata() method to include those in the metadata.</p>
|
||||
<section id="id4">
|
||||
<h3>Parameters:<a class="headerlink" href="#id4" title="Link to this heading"></a></h3>
|
||||
<dl>
|
||||
<dt>obj_name<span class="classifier">str</span></dt><dd><p>Path to the target object (dataset or group) within the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>annotation_dict: dict</dt><dd><dl>
|
||||
<dt>A dictionary where the keys represent existing attribute names (strings), and the values can be:</dt><dd><ul class="simple">
|
||||
<li><p>Scalars: int, float, or str.</p></li>
|
||||
<li><p>Compound values (dictionaries) for more complex metadata, which are converted to NumPy structured arrays.</p></li>
|
||||
</ul>
|
||||
<p>Example of a compound value:</p>
|
||||
<dl>
|
||||
<dt>annotation_dict = {</dt><dd><dl class="simple">
|
||||
<dt>“relative_humidity”: {</dt><dd><p>“value”: 65,
|
||||
“units”: “percentage”,
|
||||
“range”: “[0,100]”,
|
||||
“definition”: “amount of water vapor present …”</p>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.get_groups_at_a_level">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_ops.</span></span><span class="sig-name descname"><span class="pre">get_groups_at_a_level</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">File</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">level</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#get_groups_at_a_level"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.get_groups_at_a_level" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.get_parent_child_relationships">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_ops.</span></span><span class="sig-name descname"><span class="pre">get_parent_child_relationships</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">File</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#get_parent_child_relationships"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.get_parent_child_relationships" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.read_mtable_as_dataframe">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_ops.</span></span><span class="sig-name descname"><span class="pre">read_mtable_as_dataframe</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filename</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_ops.html#read_mtable_as_dataframe"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.read_mtable_as_dataframe" title="Link to this definition"></a></dt>
|
||||
<dd><p>Reconstruct a MATLAB Table encoded in a .h5 file as a Pandas DataFrame.</p>
|
||||
<p>This function reads a .h5 file containing a MATLAB Table and reconstructs it as a Pandas DataFrame.
|
||||
The input .h5 file contains one group per row of the MATLAB Table. Each group stores the table’s
|
||||
dataset-like variables as Datasets, while categorical and numerical variables are represented as
|
||||
attributes of the respective group.</p>
|
||||
<p>To ensure homogeneity of data columns, the DataFrame is constructed column-wise.</p>
|
||||
<section id="id5">
|
||||
<h2>Parameters<a class="headerlink" href="#id5" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>filename<span class="classifier">str</span></dt><dd><p>The name of the .h5 file. This may include the file’s location and path information.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="returns">
|
||||
<h2>Returns<a class="headerlink" href="#returns" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>pd.DataFrame</dt><dd><p>The MATLAB Table reconstructed as a Pandas DataFrame.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_ops.serialize_metadata">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_ops.</span></span><span class="sig-name descname"><span class="pre">serialize_metadata</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_filename_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">folder_depth</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">4</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_format</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'yaml'</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">str</span></span></span><a class="reference internal" href="../_modules/src/hdf5_ops.html#serialize_metadata"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_ops.serialize_metadata" title="Link to this definition"></a></dt>
|
||||
<dd><p>Serialize metadata from an HDF5 file into YAML or JSON format.</p>
|
||||
<section id="id6">
|
||||
<h2>Parameters<a class="headerlink" href="#id6" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>input_filename_path<span class="classifier">str</span></dt><dd><p>The path to the input HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>folder_depth<span class="classifier">int, optional</span></dt><dd><p>The folder depth to control how much of the HDF5 file hierarchy is traversed (default is 4).</p>
|
||||
</dd>
|
||||
<dt>output_format<span class="classifier">str, optional</span></dt><dd><p>The format to serialize the output, either ‘yaml’ or ‘json’ (default is ‘yaml’).</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="id7">
|
||||
<h2>Returns<a class="headerlink" href="#id7" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>str</dt><dd><p>The output file path where the serialized metadata is stored (either .yaml or .json).</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
</section>
|
||||
<section id="module-src.hdf5_writer">
|
||||
<span id="hdf5-writer"></span><h1>HDF5 Writer<a class="headerlink" href="#module-src.hdf5_writer" title="Link to this heading"></a></h1>
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_writer.create_hdf5_file_from_dataframe">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_writer.</span></span><span class="sig-name descname"><span class="pre">create_hdf5_file_from_dataframe</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">ofilename</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">input_data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">group_by_funcs</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">approach</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">extract_attrs_func</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_writer.html#create_hdf5_file_from_dataframe"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_writer.create_hdf5_file_from_dataframe" title="Link to this definition"></a></dt>
|
||||
<dd><p>Creates an HDF5 file with hierarchical groups based on the specified grouping functions or columns.</p>
|
||||
<section id="id8">
|
||||
<h2>Parameters:<a class="headerlink" href="#id8" title="Link to this heading"></a></h2>
|
||||
<blockquote>
|
||||
<div><p>ofilename (str): Path for the output HDF5 file.
|
||||
input_data (pd.DataFrame or str): Input data as a DataFrame or a valid file system path.
|
||||
group_by_funcs (list): List of callables or column names to define hierarchical grouping.
|
||||
approach (str): Specifies the approach (‘top-down’ or ‘bottom-up’) for creating the HDF5 file.
|
||||
extract_attrs_func (callable, optional): Function to extract additional attributes for HDF5 groups.</p>
|
||||
</div></blockquote>
|
||||
</section>
|
||||
<section id="id9">
|
||||
<h2>Returns:<a class="headerlink" href="#id9" title="Link to this heading"></a></h2>
|
||||
<blockquote>
|
||||
<div><p>None</p>
|
||||
</div></blockquote>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_writer.create_hdf5_file_from_filesystem_path">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_writer.</span></span><span class="sig-name descname"><span class="pre">create_hdf5_file_from_filesystem_path</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path_to_input_directory</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">path_to_filenames_dict</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_dir_keywords</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">[]</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">root_metadata_dict</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">{}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">mode</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'w'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_writer.html#create_hdf5_file_from_filesystem_path"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_writer.create_hdf5_file_from_filesystem_path" title="Link to this definition"></a></dt>
|
||||
<dd><p>Creates an .h5 file with name “output_filename” that preserves the directory tree (or folder structure)
|
||||
of a given filesystem path.</p>
|
||||
<p>The data integration capabilities are limited by our file reader, which can only access data from a list of
|
||||
admissible file formats. These, however, can be extended. Directories are groups in the resulting HDF5 file.
|
||||
Files are formatted as composite objects consisting of a group, file, and attributes.</p>
|
||||
<section id="id10">
|
||||
<h2>Parameters<a class="headerlink" href="#id10" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>output_filename<span class="classifier">str</span></dt><dd><p>Name of the output HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>path_to_input_directory<span class="classifier">str</span></dt><dd><p>Path to root directory, specified with forward slashes, e.g., path/to/root.</p>
|
||||
</dd>
|
||||
<dt>path_to_filenames_dict<span class="classifier">dict, optional</span></dt><dd><p>A pre-processed dictionary where keys are directory paths on the input directory’s tree and values are lists of files.
|
||||
If provided, ‘input_file_system_path’ is ignored.</p>
|
||||
</dd>
|
||||
<dt>select_dir_keywords<span class="classifier">list</span></dt><dd><dl class="simple">
|
||||
<dt>List of string elements to consider or select only directory paths that contain</dt><dd><p>a word in ‘select_dir_keywords’. When empty, all directory paths are considered
|
||||
to be included in the HDF5 file group hierarchy.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd>
|
||||
<dt>root_metadata_dict<span class="classifier">dict</span></dt><dd><p>Metadata to include at the root level of the HDF5 file.</p>
|
||||
</dd>
|
||||
<dt>mode<span class="classifier">str</span></dt><dd><p>‘w’ create File, truncate if it exists, or ‘r+’ read/write, File must exists. By default, mode = “w”.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="id11">
|
||||
<h2>Returns<a class="headerlink" href="#id11" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>output_filename<span class="classifier">str</span></dt><dd><p>Path to the created HDF5 file.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_writer.save_processed_dataframe_to_hdf5">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_writer.</span></span><span class="sig-name descname"><span class="pre">save_processed_dataframe_to_hdf5</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">annotator</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_writer.html#save_processed_dataframe_to_hdf5"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_writer.save_processed_dataframe_to_hdf5" title="Link to this definition"></a></dt>
|
||||
<dd><p>Save processed dataframe columns with annotations to an HDF5 file.</p>
|
||||
<dl class="simple">
|
||||
<dt>Parameters:</dt><dd><p>df (pd.DataFrame): DataFrame containing processed time series.
|
||||
annotator (): Annotator object with get_metadata method.
|
||||
output_filename (str): Path to the source HDF5 file.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd></dl>
|
||||
|
||||
</section>
|
||||
<section id="module-src.hdf5_vis">
|
||||
<span id="data-visualization"></span><h1>Data Visualization<a class="headerlink" href="#module-src.hdf5_vis" title="Link to this heading"></a></h1>
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="src.hdf5_vis.display_group_hierarchy_on_a_treemap">
|
||||
<span class="sig-prename descclassname"><span class="pre">src.hdf5_vis.</span></span><span class="sig-name descname"><span class="pre">display_group_hierarchy_on_a_treemap</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filename</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/src/hdf5_vis.html#display_group_hierarchy_on_a_treemap"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#src.hdf5_vis.display_group_hierarchy_on_a_treemap" title="Link to this definition"></a></dt>
|
||||
<dd><p>filename (str): hdf5 file’s filename</p>
|
||||
</dd></dl>
|
||||
|
||||
</section>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||||
<a href="../index.html" class="btn btn-neutral float-left" title="Welcome to DIMA’s documentation!" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||||
<a href="pipelines.html" class="btn btn-neutral float-right" title="Pipelines and workflows" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||||
</div>
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2024, JFFO.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
|
||||
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
jQuery(function () {
|
||||
SphinxRtdTheme.Navigation.enable(true);
|
||||
});
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
304
docs/build/html/modules/utils.html
vendored
Normal file
304
docs/build/html/modules/utils.html
vendored
Normal file
@ -0,0 +1,304 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Data Structure Conversion — DIMA 1.0.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
||||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
<link rel="next" title="Notebooks" href="notebooks.html" />
|
||||
<link rel="prev" title="Pipelines and workflows" href="pipelines.html" />
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav">
|
||||
<div class="wy-grid-for-nav">
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||||
<div class="wy-side-scroll">
|
||||
<div class="wy-side-nav-search" >
|
||||
|
||||
|
||||
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
DIMA
|
||||
</a>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
<input type="hidden" name="check_keywords" value="yes" />
|
||||
<input type="hidden" name="area" value="default" />
|
||||
</form>
|
||||
</div>
|
||||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
||||
<ul class="current">
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html">HDF5 Data Operations</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_writer">HDF5 Writer</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_vis">Data Visualization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="pipelines.html">Pipelines and workflows</a></li>
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Data Structure Conversion</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.augment_with_filenumber"><code class="docutils literal notranslate"><span class="pre">augment_with_filenumber()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.augment_with_filetype"><code class="docutils literal notranslate"><span class="pre">augment_with_filetype()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.convert_attrdict_to_np_structured_array"><code class="docutils literal notranslate"><span class="pre">convert_attrdict_to_np_structured_array()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.convert_dataframe_to_np_structured_array"><code class="docutils literal notranslate"><span class="pre">convert_dataframe_to_np_structured_array()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.convert_string_to_bytes"><code class="docutils literal notranslate"><span class="pre">convert_string_to_bytes()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.copy_directory_with_contraints"><code class="docutils literal notranslate"><span class="pre">copy_directory_with_contraints()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.created_at"><code class="docutils literal notranslate"><span class="pre">created_at()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.group_by_df_column"><code class="docutils literal notranslate"><span class="pre">group_by_df_column()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.infer_units"><code class="docutils literal notranslate"><span class="pre">infer_units()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.is_callable_list"><code class="docutils literal notranslate"><span class="pre">is_callable_list()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.is_str_list"><code class="docutils literal notranslate"><span class="pre">is_str_list()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.is_structured_array"><code class="docutils literal notranslate"><span class="pre">is_structured_array()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.make_file_copy"><code class="docutils literal notranslate"><span class="pre">make_file_copy()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.progressBar"><code class="docutils literal notranslate"><span class="pre">progressBar()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.sanitize_dataframe"><code class="docutils literal notranslate"><span class="pre">sanitize_dataframe()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.setup_logging"><code class="docutils literal notranslate"><span class="pre">setup_logging()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.split_sample_col_into_sample_and_data_quality_cols"><code class="docutils literal notranslate"><span class="pre">split_sample_col_into_sample_and_data_quality_cols()</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#utils.g5505_utils.to_serializable_dtype"><code class="docutils literal notranslate"><span class="pre">to_serializable_dtype()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="notebooks.html">Notebooks</a></li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../index.html">DIMA</a>
|
||||
</nav>
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="Page navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||||
<li class="breadcrumb-item active">Data Structure Conversion</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
<a href="../_sources/modules/utils.rst.txt" rel="nofollow"> View page source</a>
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="module-utils.g5505_utils">
|
||||
<span id="data-structure-conversion"></span><h1>Data Structure Conversion<a class="headerlink" href="#module-utils.g5505_utils" title="Link to this heading"></a></h1>
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.augment_with_filenumber">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">augment_with_filenumber</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#augment_with_filenumber"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.augment_with_filenumber" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.augment_with_filetype">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">augment_with_filetype</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#augment_with_filetype"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.augment_with_filetype" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.convert_attrdict_to_np_structured_array">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">convert_attrdict_to_np_structured_array</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">attr_value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#convert_attrdict_to_np_structured_array"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.convert_attrdict_to_np_structured_array" title="Link to this definition"></a></dt>
|
||||
<dd><p>Converts a dictionary of attributes into a numpy structured array for HDF5
|
||||
compound type compatibility.</p>
|
||||
<p>Each dictionary key is mapped to a field in the structured array, with the
|
||||
data type (S) determined by the longest string representation of the values.
|
||||
If the dictionary is empty, the function returns ‘missing’.</p>
|
||||
<section id="parameters">
|
||||
<h2>Parameters<a class="headerlink" href="#parameters" title="Link to this heading"></a></h2>
|
||||
<dl>
|
||||
<dt>attr_value<span class="classifier">dict</span></dt><dd><p>Dictionary containing the attributes to be converted. Example:
|
||||
attr_value = {</p>
|
||||
<blockquote>
|
||||
<div><p>‘name’: ‘Temperature’,
|
||||
‘unit’: ‘Celsius’,
|
||||
‘value’: 23.5,
|
||||
‘timestamp’: ‘2023-09-26 10:00’</p>
|
||||
</div></blockquote>
|
||||
<p>}</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="returns">
|
||||
<h2>Returns<a class="headerlink" href="#returns" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>new_attr_value<span class="classifier">ndarray or str</span></dt><dd><p>Numpy structured array with UTF-8 encoded fields. Returns ‘missing’ if
|
||||
the input dictionary is empty.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.convert_dataframe_to_np_structured_array">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">convert_dataframe_to_np_structured_array</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">DataFrame</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#convert_dataframe_to_np_structured_array"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.convert_dataframe_to_np_structured_array" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.convert_string_to_bytes">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">convert_string_to_bytes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_list</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#convert_string_to_bytes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.convert_string_to_bytes" title="Link to this definition"></a></dt>
|
||||
<dd><p>Convert a list of strings into a numpy array with utf8-type entries.</p>
|
||||
<section id="id1">
|
||||
<h2>Parameters<a class="headerlink" href="#id1" title="Link to this heading"></a></h2>
|
||||
<p>input_list (list) : list of string objects</p>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h2>Returns<a class="headerlink" href="#id2" title="Link to this heading"></a></h2>
|
||||
<p>input_array_bytes (ndarray): array of ut8-type entries.</p>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.copy_directory_with_contraints">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">copy_directory_with_contraints</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_dir_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_dir_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_dir_keywords</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_file_keywords</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">allowed_file_extensions</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dry_run</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#copy_directory_with_contraints"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.copy_directory_with_contraints" title="Link to this definition"></a></dt>
|
||||
<dd><p>Copies files from input_dir_path to output_dir_path based on specified constraints.</p>
|
||||
<section id="id3">
|
||||
<h2>Parameters<a class="headerlink" href="#id3" title="Link to this heading"></a></h2>
|
||||
<blockquote>
|
||||
<div><p>input_dir_path (str): Path to the input directory.
|
||||
output_dir_path (str): Path to the output directory.
|
||||
select_dir_keywords (list): optional, List of keywords for selecting directories.
|
||||
select_file_keywords (list): optional, List of keywords for selecting files.
|
||||
allowed_file_extensions (list): optional, List of allowed file extensions.</p>
|
||||
</div></blockquote>
|
||||
</section>
|
||||
<section id="id4">
|
||||
<h2>Returns<a class="headerlink" href="#id4" title="Link to this heading"></a></h2>
|
||||
<blockquote>
|
||||
<div><p>path_to_files_dict (dict): dictionary mapping directory paths to lists of copied file names satisfying the constraints.</p>
|
||||
</div></blockquote>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.created_at">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">created_at</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">datetime_format</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'%Y-%m-%d</span> <span class="pre">%H:%M:%S'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#created_at"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.created_at" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.group_by_df_column">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">group_by_df_column</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">column_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#group_by_df_column"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.group_by_df_column" title="Link to this definition"></a></dt>
|
||||
<dd><p>df (pandas.DataFrame):
|
||||
column_name (str): column_name of df by which grouping operation will take place.</p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.infer_units">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">infer_units</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">column_name</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#infer_units"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.infer_units" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.is_callable_list">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">is_callable_list</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#is_callable_list"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.is_callable_list" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.is_str_list">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">is_str_list</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#is_str_list"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.is_str_list" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.is_structured_array">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">is_structured_array</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">attr_val</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#is_structured_array"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.is_structured_array" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.make_file_copy">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">make_file_copy</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">source_file_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_folder_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'tmp_files'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#make_file_copy"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.make_file_copy" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.progressBar">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">progressBar</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">count_value</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">total</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">suffix</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">''</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#progressBar"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.progressBar" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.sanitize_dataframe">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">sanitize_dataframe</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">df</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">DataFrame</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">→</span> <span class="sig-return-typehint"><span class="pre">DataFrame</span></span></span><a class="reference internal" href="../_modules/utils/g5505_utils.html#sanitize_dataframe"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.sanitize_dataframe" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.setup_logging">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">setup_logging</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">log_dir</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">log_filename</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#setup_logging"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.setup_logging" title="Link to this definition"></a></dt>
|
||||
<dd><p>Sets up logging to a specified directory and file.</p>
|
||||
<dl class="simple">
|
||||
<dt>Parameters:</dt><dd><p>log_dir (str): Directory to save the log file.
|
||||
log_filename (str): Name of the log file.</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.split_sample_col_into_sample_and_data_quality_cols">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">split_sample_col_into_sample_and_data_quality_cols</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">DataFrame</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#split_sample_col_into_sample_and_data_quality_cols"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.split_sample_col_into_sample_and_data_quality_cols" title="Link to this definition"></a></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="py function">
|
||||
<dt class="sig sig-object py" id="utils.g5505_utils.to_serializable_dtype">
|
||||
<span class="sig-prename descclassname"><span class="pre">utils.g5505_utils.</span></span><span class="sig-name descname"><span class="pre">to_serializable_dtype</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">value</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/utils/g5505_utils.html#to_serializable_dtype"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#utils.g5505_utils.to_serializable_dtype" title="Link to this definition"></a></dt>
|
||||
<dd><p>Transform value’s dtype into YAML/JSON compatible dtype</p>
|
||||
<section id="id5">
|
||||
<h2>Parameters<a class="headerlink" href="#id5" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>value<span class="classifier">_type_</span></dt><dd><p>_description_</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
<section id="id6">
|
||||
<h2>Returns<a class="headerlink" href="#id6" title="Link to this heading"></a></h2>
|
||||
<dl class="simple">
|
||||
<dt>_type_</dt><dd><p>_description_</p>
|
||||
</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</dd></dl>
|
||||
|
||||
</section>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||||
<a href="pipelines.html" class="btn btn-neutral float-left" title="Pipelines and workflows" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||||
<a href="notebooks.html" class="btn btn-neutral float-right" title="Notebooks" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||||
</div>
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2024, JFFO.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
|
||||
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
jQuery(function () {
|
||||
SphinxRtdTheme.Navigation.enable(true);
|
||||
});
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user