299 lines
32 KiB
HTML
299 lines
32 KiB
HTML
<!DOCTYPE html>
|
|
<html class="writer-html5" lang="en" data-content_root="../../">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>pipelines.metadata_revision — DIMA 1.0.0 documentation</title>
|
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
|
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=19f00094" />
|
|
|
|
|
|
<!--[if lt IE 9]>
|
|
<script src="../../_static/js/html5shiv.min.js"></script>
|
|
<![endif]-->
|
|
|
|
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
|
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
|
<script src="../../_static/documentation_options.js?v=8d563738"></script>
|
|
<script src="../../_static/doctools.js?v=9a2dae69"></script>
|
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
|
<script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
|
|
<script src="../../_static/js/theme.js"></script>
|
|
<link rel="index" title="Index" href="../../genindex.html" />
|
|
<link rel="search" title="Search" href="../../search.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
<div class="wy-grid-for-nav">
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../../index.html" class="icon icon-home">
|
|
DIMA
|
|
</a>
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
|
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html">HDF5 data operations</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.hdf5_writer">Data integration with HDF5</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules/src.html#module-src.hdf5_vis">Data visualization</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules/pipelines.html">Pipelines and workflows</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules/utils.html">Utilities</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../../index.html">DIMA</a>
|
|
</nav>
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="Page navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
|
<li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
|
|
<li class="breadcrumb-item active">pipelines.metadata_revision</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for pipelines.metadata_revision</h1><div class="highlight"><pre>
|
|
<span></span><span class="kn">import</span> <span class="nn">sys</span>
|
|
<span class="kn">import</span> <span class="nn">os</span>
|
|
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="n">thisFilePath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"Error: __file__ is not available. Ensure the script is being run from a file."</span><span class="p">)</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"[Notice] Path to DIMA package may not be resolved properly."</span><span class="p">)</span>
|
|
<span class="n">thisFilePath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span> <span class="c1"># Use current directory or specify a default</span>
|
|
|
|
<span class="n">dimaPath</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">normpath</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">thisFilePath</span><span class="p">,</span> <span class="s2">".."</span><span class="p">,</span><span class="s1">'..'</span><span class="p">))</span> <span class="c1"># Move up to project root</span>
|
|
|
|
<span class="k">if</span> <span class="n">dimaPath</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">:</span> <span class="c1"># Avoid duplicate entries</span>
|
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">dimaPath</span><span class="p">)</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">h5py</span>
|
|
<span class="kn">import</span> <span class="nn">yaml</span>
|
|
<span class="kn">import</span> <span class="nn">src.hdf5_ops</span> <span class="k">as</span> <span class="nn">hdf5_ops</span>
|
|
|
|
|
|
<div class="viewcode-block" id="load_yaml">
|
|
<a class="viewcode-back" href="../../modules/pipelines.html#pipelines.metadata_revision.load_yaml">[docs]</a>
|
|
<span class="k">def</span> <span class="nf">load_yaml</span><span class="p">(</span><span class="n">review_yaml_file</span><span class="p">):</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">review_yaml_file</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span> <span class="k">as</span> <span class="n">stream</span><span class="p">:</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">yaml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">stream</span><span class="p">,</span> <span class="n">Loader</span><span class="o">=</span><span class="n">yaml</span><span class="o">.</span><span class="n">FullLoader</span><span class="p">)</span>
|
|
<span class="k">except</span> <span class="n">yaml</span><span class="o">.</span><span class="n">YAMLError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="kc">None</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="validate_yaml_dict">
|
|
<a class="viewcode-back" href="../../modules/pipelines.html#pipelines.metadata_revision.validate_yaml_dict">[docs]</a>
|
|
<span class="k">def</span> <span class="nf">validate_yaml_dict</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">,</span> <span class="n">yaml_dict</span><span class="p">):</span>
|
|
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
|
|
<span class="n">notes</span> <span class="o">=</span> <span class="p">[]</span>
|
|
|
|
<span class="k">with</span> <span class="n">h5py</span><span class="o">.</span><span class="n">File</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">)</span> <span class="k">as</span> <span class="n">hdf5_file</span><span class="p">:</span>
|
|
<span class="c1"># 1. Check for valid object names</span>
|
|
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">yaml_dict</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">hdf5_file</span><span class="p">:</span>
|
|
<span class="n">error_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error: </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> is not a valid object's name in the HDF5 file."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
|
|
<span class="c1"># 2. Confirm metadata dict for each object is a dictionary</span>
|
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">meta_dict</span> <span class="ow">in</span> <span class="n">yaml_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">meta_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="n">error_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error: Metadata for </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> should be a dictionary."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="s1">'attributes'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">meta_dict</span><span class="p">:</span>
|
|
<span class="n">warning_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Warning: No 'attributes' in metadata dict for </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">warning_msg</span><span class="p">)</span>
|
|
<span class="n">notes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">warning_msg</span><span class="p">)</span>
|
|
|
|
<span class="c1"># 3. Verify update, append, and delete operations are well specified </span>
|
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">meta_dict</span> <span class="ow">in</span> <span class="n">yaml_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="n">attributes</span> <span class="o">=</span> <span class="n">meta_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"attributes"</span><span class="p">,</span> <span class="p">{})</span>
|
|
|
|
<span class="k">for</span> <span class="n">attr_name</span><span class="p">,</span> <span class="n">attr_value</span> <span class="ow">in</span> <span class="n">attributes</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="c1"># Ensure the object exists before accessing attributes</span>
|
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">hdf5_file</span><span class="p">:</span>
|
|
<span class="n">hdf5_obj_attrs</span> <span class="o">=</span> <span class="n">hdf5_file</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">attrs</span> <span class="c1"># Access object-specific attributes</span>
|
|
|
|
<span class="k">if</span> <span class="n">attr_name</span> <span class="ow">in</span> <span class="n">hdf5_obj_attrs</span><span class="p">:</span>
|
|
<span class="c1"># Attribute exists: it can be updated or deleted</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">attr_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">"delete"</span> <span class="ow">in</span> <span class="n">attr_value</span><span class="p">:</span>
|
|
<span class="n">note_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Note: '</span><span class="si">{</span><span class="n">attr_name</span><span class="si">}</span><span class="s2">' in </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> may be deleted if 'delete' is set as true."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="n">notes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">note_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Note: '</span><span class="si">{</span><span class="n">attr_name</span><span class="si">}</span><span class="s2">' in </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> will be updated."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="n">notes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># Attribute does not exist: it can be appended or flagged as an invalid delete</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">attr_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">"delete"</span> <span class="ow">in</span> <span class="n">attr_value</span><span class="p">:</span>
|
|
<span class="n">error_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error: Cannot delete non-existent attribute '</span><span class="si">{</span><span class="n">attr_name</span><span class="si">}</span><span class="s2">' in </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">note_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Note: '</span><span class="si">{</span><span class="n">attr_name</span><span class="si">}</span><span class="s2">' in </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> will be appended."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="n">notes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">note_msg</span><span class="p">)</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">error_msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Error: '</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">' is not a valid object in the HDF5 file."</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
<span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">errors</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">,</span> <span class="n">errors</span><span class="p">,</span> <span class="n">notes</span></div>
|
|
|
|
|
|
|
|
<div class="viewcode-block" id="update_hdf5_file_with_review">
|
|
<a class="viewcode-back" href="../../modules/pipelines.html#pipelines.metadata_revision.update_hdf5_file_with_review">[docs]</a>
|
|
<span class="k">def</span> <span class="nf">update_hdf5_file_with_review</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">,</span> <span class="n">review_yaml_file</span><span class="p">):</span>
|
|
<span class="w"> </span>
|
|
<span class="w"> </span><span class="sd">"""</span>
|
|
<span class="sd"> Updates, appends, or deletes metadata attributes in an HDF5 file based on a provided YAML dictionary.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> Parameters:</span>
|
|
<span class="sd"> -----------</span>
|
|
<span class="sd"> input_hdf5_file : str</span>
|
|
<span class="sd"> Path to the HDF5 file.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> yaml_dict : dict</span>
|
|
<span class="sd"> Dictionary specifying objects and their attributes with operations. Example format:</span>
|
|
<span class="sd"> {</span>
|
|
<span class="sd"> "object_name": { "attributes" : "attr_name": { "value": attr_value,</span>
|
|
<span class="sd"> "delete": true | false</span>
|
|
<span class="sd"> }</span>
|
|
<span class="sd"> }</span>
|
|
<span class="sd"> }</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">yaml_dict</span> <span class="o">=</span> <span class="n">load_yaml</span><span class="p">(</span><span class="n">review_yaml_file</span><span class="p">)</span>
|
|
|
|
<span class="n">success</span><span class="p">,</span> <span class="n">errors</span><span class="p">,</span> <span class="n">notes</span> <span class="o">=</span> <span class="n">validate_yaml_dict</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">,</span><span class="n">yaml_dict</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">success</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Review yaml file </span><span class="si">{</span><span class="n">review_yaml_file</span><span class="si">}</span><span class="s2"> is invalid. Validation errors: </span><span class="si">{</span><span class="n">errors</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Initialize HDF5 operations manager</span>
|
|
<span class="n">DataOpsAPI</span> <span class="o">=</span> <span class="n">hdf5_ops</span><span class="o">.</span><span class="n">HDF5DataOpsManager</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">)</span>
|
|
<span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">load_file_obj</span><span class="p">()</span>
|
|
|
|
<span class="c1"># Iterate over each object in the YAML dictionary</span>
|
|
<span class="k">for</span> <span class="n">obj_name</span><span class="p">,</span> <span class="n">attr_dict</span> <span class="ow">in</span> <span class="n">yaml_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="c1"># Prepare dictionaries for append, update, and delete actions</span>
|
|
<span class="n">append_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">update_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
|
<span class="n">delete_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
|
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">obj_name</span> <span class="ow">in</span> <span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">file_obj</span><span class="p">:</span>
|
|
<span class="k">continue</span> <span class="c1"># Skip if the object does not exist</span>
|
|
|
|
<span class="c1"># Iterate over each attribute in the current object</span>
|
|
<span class="k">for</span> <span class="n">attr_name</span><span class="p">,</span> <span class="n">attr_props</span> <span class="ow">in</span> <span class="n">attr_dict</span><span class="p">[</span><span class="s1">'attributes'</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">attr_props</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
|
<span class="c1">#attr_props = {'value': attr_props}</span>
|
|
<span class="c1"># Check if the attribute exists (for updating)</span>
|
|
<span class="k">if</span> <span class="n">attr_name</span> <span class="ow">in</span> <span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">file_obj</span><span class="p">[</span><span class="n">obj_name</span><span class="p">]</span><span class="o">.</span><span class="n">attrs</span><span class="p">:</span>
|
|
<span class="n">update_dict</span><span class="p">[</span><span class="n">attr_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">attr_props</span>
|
|
<span class="c1"># Otherwise, it's a new attribute to append</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="n">append_dict</span><span class="p">[</span><span class="n">attr_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">attr_props</span>
|
|
<span class="k">else</span><span class="p">:</span>
|
|
<span class="c1"># Check if the attribute is marked for deletion</span>
|
|
<span class="k">if</span> <span class="n">attr_props</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'delete'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
|
|
<span class="n">delete_dict</span><span class="p">[</span><span class="n">attr_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">attr_props</span>
|
|
|
|
<span class="c1"># Perform a single pass for all three operations</span>
|
|
<span class="k">if</span> <span class="n">append_dict</span><span class="p">:</span>
|
|
<span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">append_metadata</span><span class="p">(</span><span class="n">obj_name</span><span class="p">,</span> <span class="n">append_dict</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">update_dict</span><span class="p">:</span>
|
|
<span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">update_metadata</span><span class="p">(</span><span class="n">obj_name</span><span class="p">,</span> <span class="n">update_dict</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">delete_dict</span><span class="p">:</span>
|
|
<span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">delete_metadata</span><span class="p">(</span><span class="n">obj_name</span><span class="p">,</span> <span class="n">delete_dict</span><span class="p">)</span>
|
|
|
|
<span class="c1"># Close hdf5 file</span>
|
|
<span class="n">DataOpsAPI</span><span class="o">.</span><span class="n">unload_file_obj</span><span class="p">()</span>
|
|
<span class="c1"># Regenerate yaml snapshot of updated HDF5 file</span>
|
|
<span class="n">output_yml_filename_path</span> <span class="o">=</span> <span class="n">hdf5_ops</span><span class="o">.</span><span class="n">serialize_metadata</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">)</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">output_yml_filename_path</span><span class="si">}</span><span class="s1"> was successfully regenerated from the updated version of</span><span class="si">{</span><span class="n">input_hdf5_file</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="count">
|
|
<a class="viewcode-back" href="../../modules/pipelines.html#pipelines.metadata_revision.count">[docs]</a>
|
|
<span class="k">def</span> <span class="nf">count</span><span class="p">(</span><span class="n">hdf5_obj</span><span class="p">,</span><span class="n">yml_dict</span><span class="p">):</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="n">hdf5_obj</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hdf5_obj</span><span class="p">,</span><span class="n">h5py</span><span class="o">.</span><span class="n">Group</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">hdf5_obj</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">))</span> <span class="o"><=</span> <span class="mi">4</span><span class="p">:</span>
|
|
<span class="n">obj_review</span> <span class="o">=</span> <span class="n">yml_dict</span><span class="p">[</span><span class="n">hdf5_obj</span><span class="o">.</span><span class="n">name</span><span class="p">]</span>
|
|
<span class="n">additions</span> <span class="o">=</span> <span class="p">[</span><span class="ow">not</span> <span class="p">(</span><span class="n">item</span> <span class="ow">in</span> <span class="n">hdf5_obj</span><span class="o">.</span><span class="n">attrs</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">obj_review</span><span class="p">[</span><span class="s1">'attributes'</span><span class="p">]</span><span class="o">.</span><span class="n">keys</span><span class="p">()]</span>
|
|
<span class="n">count_additions</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">additions</span><span class="p">)</span>
|
|
<span class="n">deletions</span> <span class="o">=</span> <span class="p">[</span><span class="ow">not</span> <span class="p">(</span><span class="n">item</span> <span class="ow">in</span> <span class="n">obj_review</span><span class="p">[</span><span class="s1">'attributes'</span><span class="p">]</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">hdf5_obj</span><span class="o">.</span><span class="n">attrs</span><span class="o">.</span><span class="n">keys</span><span class="p">()]</span>
|
|
<span class="n">count_delections</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">deletions</span><span class="p">)</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="s1">'additions'</span><span class="p">,</span><span class="n">count_additions</span><span class="p">,</span> <span class="s1">'deletions'</span><span class="p">,</span> <span class="n">count_delections</span><span class="p">)</span></div>
|
|
|
|
|
|
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span>
|
|
|
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">argv</span><span class="p">)</span> <span class="o"><</span> <span class="mi">4</span><span class="p">:</span>
|
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"Usage: python metadata_revision.py update <path/to/target_file.hdf5> <path/to/metadata_review_file.yaml>"</span><span class="p">)</span>
|
|
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">argv</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'update'</span><span class="p">:</span>
|
|
<span class="n">input_hdf5_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">argv</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
|
|
<span class="n">review_yaml_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">argv</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span>
|
|
<span class="n">update_hdf5_file_with_review</span><span class="p">(</span><span class="n">input_hdf5_file</span><span class="p">,</span> <span class="n">review_yaml_file</span><span class="p">)</span>
|
|
<span class="c1">#run(sys.argv[2])</span>
|
|
</pre></div>
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>© Copyright 2024, JFFO.</p>
|
|
</div>
|
|
|
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</div>
|
|
<script>
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
</body>
|
|
</html> |