194 lines
14 KiB
HTML
194 lines
14 KiB
HTML
<!DOCTYPE html>
|
|
<html class="writer-html5" lang="en" data-content_root="../">
|
|
<head>
|
|
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>Pipelines and workflows — DIMA 1.0.0 documentation</title>
|
|
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
|
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=19f00094" />
|
|
|
|
|
|
<!--[if lt IE 9]>
|
|
<script src="../_static/js/html5shiv.min.js"></script>
|
|
<![endif]-->
|
|
|
|
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
|
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
|
<script src="../_static/documentation_options.js?v=8d563738"></script>
|
|
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
|
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
|
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
|
<script src="../_static/js/theme.js"></script>
|
|
<link rel="index" title="Index" href="../genindex.html" />
|
|
<link rel="search" title="Search" href="../search.html" />
|
|
<link rel="next" title="Utilities" href="utils.html" />
|
|
<link rel="prev" title="HDF5 data operations" href="src.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
<div class="wy-grid-for-nav">
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home">
|
|
DIMA
|
|
</a>
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
|
<ul class="current">
|
|
<li class="toctree-l1"><a class="reference internal" href="src.html">HDF5 data operations</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_writer">Data integration with HDF5</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="src.html#module-src.hdf5_vis">Data visualization</a></li>
|
|
<li class="toctree-l1 current"><a class="current reference internal" href="#">Pipelines and workflows</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.copy_subtree_and_create_hdf5"><code class="docutils literal notranslate"><span class="pre">copy_subtree_and_create_hdf5()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.load_config_and_setup_logging"><code class="docutils literal notranslate"><span class="pre">load_config_and_setup_logging()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.data_integration.run_pipeline"><code class="docutils literal notranslate"><span class="pre">run_pipeline()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.count"><code class="docutils literal notranslate"><span class="pre">count()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.load_yaml"><code class="docutils literal notranslate"><span class="pre">load_yaml()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.update_hdf5_file_with_review"><code class="docutils literal notranslate"><span class="pre">update_hdf5_file_with_review()</span></code></a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#pipelines.metadata_revision.validate_yaml_dict"><code class="docutils literal notranslate"><span class="pre">validate_yaml_dict()</span></code></a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l1"><a class="reference internal" href="utils.html">Utilities</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">DIMA</a>
|
|
</nav>
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="Page navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
|
<li class="breadcrumb-item active">Pipelines and workflows</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
<a href="../_sources/modules/pipelines.rst.txt" rel="nofollow"> View page source</a>
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<section id="module-pipelines.data_integration">
|
|
<span id="pipelines-and-workflows"></span><h1>Pipelines and workflows<a class="headerlink" href="#module-pipelines.data_integration" title="Link to this heading"></a></h1>
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.data_integration.copy_subtree_and_create_hdf5">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">copy_subtree_and_create_hdf5</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dst</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_dir_keywords</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">select_file_keywords</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">allowed_file_extensions</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">root_metadata_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#copy_subtree_and_create_hdf5"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.copy_subtree_and_create_hdf5" title="Link to this definition"></a></dt>
|
|
<dd><p>Helper function to copy directory with constraints and create HDF5.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.data_integration.load_config_and_setup_logging">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">load_config_and_setup_logging</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">yaml_config_file_path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">log_dir</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#load_config_and_setup_logging"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.load_config_and_setup_logging" title="Link to this definition"></a></dt>
|
|
<dd><p>Load YAML configuration file, set up logging, and validate required keys and datetime_steps.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.data_integration.run_pipeline">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.data_integration.</span></span><span class="sig-name descname"><span class="pre">run_pipeline</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path_to_config_yamlFile</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">log_dir</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'logs/'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/data_integration.html#run_pipeline"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.data_integration.run_pipeline" title="Link to this definition"></a></dt>
|
|
<dd><p>Integrates data sources specified by the input configuration file into HDF5 files.</p>
|
|
<dl class="simple">
|
|
<dt>Parameters:</dt><dd><p>yaml_config_file_path (str): Path to the YAML configuration file.
|
|
log_dir (str): Directory to save the log file.</p>
|
|
</dd>
|
|
<dt>Returns:</dt><dd><p>list: List of Paths to the created HDF5 file(s).</p>
|
|
</dd>
|
|
</dl>
|
|
</dd></dl>
|
|
|
|
<dl class="py function" id="module-pipelines.metadata_revision">
|
|
<dt class="sig sig-object py" id="pipelines.metadata_revision.count">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">count</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hdf5_obj</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">yml_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#count"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.count" title="Link to this definition"></a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.metadata_revision.load_yaml">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">load_yaml</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">review_yaml_file</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#load_yaml"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.load_yaml" title="Link to this definition"></a></dt>
|
|
<dd></dd></dl>
|
|
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.metadata_revision.update_hdf5_file_with_review">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">update_hdf5_file_with_review</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_hdf5_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">review_yaml_file</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#update_hdf5_file_with_review"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.update_hdf5_file_with_review" title="Link to this definition"></a></dt>
|
|
<dd><p>Updates, appends, or deletes metadata attributes in an HDF5 file based on a provided YAML dictionary.</p>
|
|
<section id="parameters">
|
|
<h2>Parameters:<a class="headerlink" href="#parameters" title="Link to this heading"></a></h2>
|
|
<dl>
|
|
<dt>input_hdf5_file<span class="classifier">str</span></dt><dd><p>Path to the HDF5 file.</p>
|
|
</dd>
|
|
<dt>yaml_dict<span class="classifier">dict</span></dt><dd><p>Dictionary specifying objects and their attributes with operations. Example format:
|
|
{</p>
|
|
<blockquote>
|
|
<div><dl>
|
|
<dt>“object_name”: { “attributes”<span class="classifier">“attr_name”: { “value”: attr_value,</span></dt><dd><blockquote>
|
|
<div><blockquote>
|
|
<div><p>“delete”: true | false</p>
|
|
</div></blockquote>
|
|
<p>}</p>
|
|
</div></blockquote>
|
|
<p>}</p>
|
|
</dd>
|
|
</dl>
|
|
</div></blockquote>
|
|
<p>}</p>
|
|
</dd>
|
|
</dl>
|
|
</section>
|
|
</dd></dl>
|
|
|
|
<dl class="py function">
|
|
<dt class="sig sig-object py" id="pipelines.metadata_revision.validate_yaml_dict">
|
|
<span class="sig-prename descclassname"><span class="pre">pipelines.metadata_revision.</span></span><span class="sig-name descname"><span class="pre">validate_yaml_dict</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_hdf5_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">yaml_dict</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pipelines/metadata_revision.html#validate_yaml_dict"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pipelines.metadata_revision.validate_yaml_dict" title="Link to this definition"></a></dt>
|
|
<dd></dd></dl>
|
|
|
|
</section>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
|
<a href="src.html" class="btn btn-neutral float-left" title="HDF5 data operations" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
|
<a href="utils.html" class="btn btn-neutral float-right" title="Utilities" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
|
</div>
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>© Copyright 2024, JFFO.</p>
|
|
</div>
|
|
|
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</div>
|
|
<script>
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
</body>
|
|
</html> |