diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle index 00f1ec7..c56cc41 100644 Binary files a/docs/build/doctrees/environment.pickle and b/docs/build/doctrees/environment.pickle differ diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree index 8bde9db..057047d 100644 Binary files a/docs/build/doctrees/index.doctree and b/docs/build/doctrees/index.doctree differ diff --git a/docs/build/doctrees/modules/src.doctree b/docs/build/doctrees/modules/src.doctree index ba7a7f3..f4a56a0 100644 Binary files a/docs/build/doctrees/modules/src.doctree and b/docs/build/doctrees/modules/src.doctree differ diff --git a/docs/build/doctrees/modules/vis.doctree b/docs/build/doctrees/modules/vis.doctree new file mode 100644 index 0000000..dfb1cdd Binary files /dev/null and b/docs/build/doctrees/modules/vis.doctree differ diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html index 21717aa..e125ff0 100644 --- a/docs/build/html/_modules/index.html +++ b/docs/build/html/_modules/index.html @@ -46,8 +46,8 @@ @@ -79,9 +79,9 @@ diff --git a/docs/build/html/_modules/visualization/hdf5_vis.html b/docs/build/html/_modules/visualization/hdf5_vis.html new file mode 100644 index 0000000..9155ff6 --- /dev/null +++ b/docs/build/html/_modules/visualization/hdf5_vis.html @@ -0,0 +1,182 @@ + + + + + + visualization.hdf5_vis — DIMA 1.0.0 documentation + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+
+
+ +
+
+
+
+ +

Source code for visualization.hdf5_vis

+import sys
+import os
+root_dir = os.path.abspath(os.curdir)
+sys.path.append(root_dir)
+
+import h5py
+import yaml
+
+import numpy as np
+import pandas as pd
+
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+import plotly.express as px
+#import plotly.io as pio
+from src.hdf5_ops import get_parent_child_relationships
+
+ 
+
+
+[docs] +def display_group_hierarchy_on_a_treemap(filename: str): + + """ + filename (str): hdf5 file's filename""" + + with h5py.File(filename,'r') as file: + nodes, parents, values = get_parent_child_relationships(file) + + metadata_list = [] + metadata_dict={} + for key in file.attrs.keys(): + #if 'metadata' in key: + if isinstance(file.attrs[key], str): # Check if the attribute is a string + metadata_key = key[key.find('_') + 1:] + metadata_value = file.attrs[key] + metadata_dict[metadata_key] = metadata_value + metadata_list.append(f'{metadata_key}: {metadata_value}') + + #metadata_dict[key[key.find('_')+1::]]= file.attrs[key] + #metadata_list.append(key[key.find('_')+1::]+':'+file.attrs[key]) + + metadata = '<br>'.join(['<br>'] + metadata_list) + + customdata_series = pd.Series(nodes) + customdata_series[0] = metadata + + fig = make_subplots(1, 1, specs=[[{"type": "domain"}]],) + fig.add_trace(go.Treemap( + labels=nodes, #formating_df['formated_names'][nodes], + parents=parents,#formating_df['formated_names'][parents], + values=values, + branchvalues='remainder', + customdata= customdata_series, + #marker=dict( + # colors=df_all_trees['color'], + # colorscale='RdBu', + # cmid=average_score), + #hovertemplate='<b>%{label} </b> <br> Number of files: %{value}<br> Success rate: %{color:.2f}', + hovertemplate='<b>%{label} </b> <br> Count: %{value} <br> Path: %{customdata}', + name='', + root_color="lightgrey" + )) + fig.update_layout(width = 800, height= 600, margin = dict(t=50, l=25, r=25, b=25)) + fig.show() + file_name, file_ext = os.path.splitext(filename) + fig.write_html(file_name + ".html")
+ + + #pio.write_image(fig,file_name + ".png",width=800,height=600,format='png') + +# +
+ +
+
+ +
+
+
+
+ + + + \ No newline at end of file diff --git a/docs/build/html/_sources/index.rst.txt b/docs/build/html/_sources/index.rst.txt index f8489d2..aadb26a 100644 --- a/docs/build/html/_sources/index.rst.txt +++ b/docs/build/html/_sources/index.rst.txt @@ -14,6 +14,8 @@ Welcome to DIMA's documentation! modules/pipelines + modules/vis + modules/utils modules/notebooks diff --git a/docs/build/html/_sources/modules/src.rst.txt b/docs/build/html/_sources/modules/src.rst.txt index e9c4fc2..ecc8e69 100644 --- a/docs/build/html/_sources/modules/src.rst.txt +++ b/docs/build/html/_sources/modules/src.rst.txt @@ -14,14 +14,5 @@ HDF5 Writer :undoc-members: :show-inheritance: -Data Visualization -================== - -.. automodule:: src.hdf5_vis - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/build/html/_sources/modules/vis.rst.txt b/docs/build/html/_sources/modules/vis.rst.txt new file mode 100644 index 0000000..5a25e0f --- /dev/null +++ b/docs/build/html/_sources/modules/vis.rst.txt @@ -0,0 +1,7 @@ +Data Visualization +================== + +.. automodule:: visualization.hdf5_vis + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/build/html/genindex.html b/docs/build/html/genindex.html index 14c5ef7..3afeba2 100644 --- a/docs/build/html/genindex.html +++ b/docs/build/html/genindex.html @@ -46,8 +46,8 @@ @@ -146,7 +146,7 @@ @@ -233,12 +233,12 @@
  • pipelines.metadata_revision
  • src.hdf5_ops -
  • -
  • src.hdf5_vis
  • src.hdf5_writer
  • utils.g5505_utils +
  • +
  • visualization.hdf5_vis
  • @@ -307,23 +307,16 @@
  • serialize_metadata() (in module src.hdf5_ops)
  • setup_logging() (in module utils.g5505_utils) -
  • -
  • split_sample_col_into_sample_and_data_quality_cols() (in module utils.g5505_utils)
  • + diff --git a/docs/build/html/index.html b/docs/build/html/index.html index cb8b2e4..5f1c624 100644 --- a/docs/build/html/index.html +++ b/docs/build/html/index.html @@ -48,8 +48,8 @@ @@ -97,10 +97,6 @@
  • save_processed_dataframe_to_hdf5()
  • -
  • Data Visualization -
  • Pipelines and workflows
  • +
  • Data Visualization +
  • Data Structure Conversion
  • -
  • Data Visualization -
  • Pipelines and workflows
  • +
  • Data Visualization
  • Data Structure Conversion
  • Notebooks
  • @@ -427,15 +424,6 @@ output_filename (str): Path to the source HDF5 file.

    - -
    -

    Data Visualization

    -
    -
    -src.hdf5_vis.display_group_hierarchy_on_a_treemap(filename: str)[source]
    -

    filename (str): hdf5 file’s filename

    -
    -
    diff --git a/docs/build/html/modules/vis.html b/docs/build/html/modules/vis.html new file mode 100644 index 0000000..99b4687 --- /dev/null +++ b/docs/build/html/modules/vis.html @@ -0,0 +1,127 @@ + + + + + + + Data Visualization — DIMA 1.0.0 documentation + + + + + + + + + + + + + + + + + + + + +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    + +
    +

    Data Visualization

    +
    +
    +visualization.hdf5_vis.display_group_hierarchy_on_a_treemap(filename: str)[source]
    +

    filename (str): hdf5 file’s filename

    +
    + +
    + + +
    +
    + +
    +
    +
    +
    + + + + \ No newline at end of file diff --git a/docs/build/html/objects.inv b/docs/build/html/objects.inv index d118b39..88583ef 100644 Binary files a/docs/build/html/objects.inv and b/docs/build/html/objects.inv differ diff --git a/docs/build/html/py-modindex.html b/docs/build/html/py-modindex.html index 9209cea..7683a8a 100644 --- a/docs/build/html/py-modindex.html +++ b/docs/build/html/py-modindex.html @@ -49,8 +49,8 @@ @@ -85,7 +85,8 @@ n | p | s | - u + u | + v @@ -130,11 +131,6 @@ - - - + + + + + + + +
        src.hdf5_ops
        - src.hdf5_vis -
        @@ -154,6 +150,20 @@     utils.g5505_utils
     
    + v
    + visualization +
        + visualization.hdf5_vis +
    diff --git a/docs/build/html/search.html b/docs/build/html/search.html index f78149d..58f776c 100644 --- a/docs/build/html/search.html +++ b/docs/build/html/search.html @@ -49,8 +49,8 @@ diff --git a/docs/build/html/searchindex.js b/docs/build/html/searchindex.js index 998b53a..7b9b7e1 100644 --- a/docs/build/html/searchindex.js +++ b/docs/build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"Behavior:": [[4, "behavior"]], "Contents:": [[0, null]], "Data Structure Conversion": [[5, "module-utils.g5505_utils"]], "Data Visualization": [[4, "module-src.hdf5_vis"]], "Example:": [[4, "example"]], "HDF5 Data Operations": [[4, "module-src.hdf5_ops"]], "HDF5 Writer": [[4, "module-src.hdf5_writer"]], "Indices and tables": [[0, "indices-and-tables"]], "Notebooks": [[2, "module-notebooks"]], "Parameters": [[4, "id5"], [4, "id6"], [4, "id10"], [5, "parameters"], [5, "id1"], [5, "id3"], [5, "id5"]], "Parameters:": [[3, "parameters"], [4, "parameters"], [4, "id1"], [4, "id2"], [4, "id3"], [4, "id4"], [4, "id8"]], "Pipelines and workflows": [[3, "module-pipelines.data_integration"]], "Returns": [[4, "returns"], [4, "id7"], [4, "id11"], [5, "returns"], [5, "id2"], [5, "id4"], [5, "id6"]], "Returns:": [[4, "id9"]], "Welcome to DIMA\u2019s documentation!": [[0, "welcome-to-dima-s-documentation"]]}, "docnames": ["index", "modules/instruments", "modules/notebooks", "modules/pipelines", "modules/src", "modules/utils"], "envversion": {"nbsphinx": 4, "sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["index.rst", "modules/instruments.rst", "modules/notebooks.rst", "modules/pipelines.rst", "modules/src.rst", "modules/utils.rst"], "indexentries": {"append_dataset() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.append_dataset", false]], "append_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.append_metadata", false]], "augment_with_filenumber() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.augment_with_filenumber", false]], "augment_with_filetype() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.augment_with_filetype", false]], "convert_attrdict_to_np_structured_array() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.convert_attrdict_to_np_structured_array", false]], "convert_dataframe_to_np_structured_array() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.convert_dataframe_to_np_structured_array", false]], "convert_string_to_bytes() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.convert_string_to_bytes", false]], "copy_directory_with_contraints() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.copy_directory_with_contraints", false]], "create_hdf5_file_from_dataframe() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.create_hdf5_file_from_dataframe", false]], "create_hdf5_file_from_filesystem_path() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.create_hdf5_file_from_filesystem_path", false]], "created_at() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.created_at", false]], "delete_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.delete_metadata", false]], "display_group_hierarchy_on_a_treemap() (in module src.hdf5_vis)": [[4, "src.hdf5_vis.display_group_hierarchy_on_a_treemap", false]], "extract_and_load_dataset_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.extract_and_load_dataset_metadata", false]], "extract_dataset_as_dataframe() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.extract_dataset_as_dataframe", false]], "get_groups_at_a_level() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.get_groups_at_a_level", false]], "get_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.get_metadata", false]], "get_parent_child_relationships() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.get_parent_child_relationships", false]], "group_by_df_column() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.group_by_df_column", false]], "hdf5dataopsmanager (class in src.hdf5_ops)": [[4, "src.hdf5_ops.HDF5DataOpsManager", false]], "infer_units() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.infer_units", false]], "is_callable_list() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.is_callable_list", false]], "is_str_list() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.is_str_list", false]], "is_structured_array() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.is_structured_array", false]], "load_file_obj() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.load_file_obj", false]], "make_file_copy() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.make_file_copy", false]], "module": [[4, "module-src.hdf5_ops", false], [4, "module-src.hdf5_vis", false], [4, "module-src.hdf5_writer", false], [5, "module-utils.g5505_utils", false]], "progressbar() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.progressBar", false]], "read_mtable_as_dataframe() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.read_mtable_as_dataframe", false]], "reformat_datetime_column() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.reformat_datetime_column", false]], "rename_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.rename_metadata", false]], "sanitize_dataframe() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.sanitize_dataframe", false]], "save_processed_dataframe_to_hdf5() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.save_processed_dataframe_to_hdf5", false]], "serialize_metadata() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.serialize_metadata", false]], "setup_logging() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.setup_logging", false]], "split_sample_col_into_sample_and_data_quality_cols() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.split_sample_col_into_sample_and_data_quality_cols", false]], "src.hdf5_ops": [[4, "module-src.hdf5_ops", false]], "src.hdf5_vis": [[4, "module-src.hdf5_vis", false]], "src.hdf5_writer": [[4, "module-src.hdf5_writer", false]], "to_serializable_dtype() (in module utils.g5505_utils)": [[5, "utils.g5505_utils.to_serializable_dtype", false]], "unload_file_obj() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.unload_file_obj", false]], "update_file() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.update_file", false]], "update_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.update_metadata", false]], "utils.g5505_utils": [[5, "module-utils.g5505_utils", false]]}, "objects": {"": [[2, 0, 0, "-", "notebooks"]], "pipelines": [[3, 0, 0, "-", "data_integration"], [3, 0, 0, "-", "metadata_revision"]], "pipelines.data_integration": [[3, 1, 1, "", "copy_subtree_and_create_hdf5"], [3, 1, 1, "", "load_config_and_setup_logging"], [3, 1, 1, "", "run_pipeline"]], "pipelines.metadata_revision": [[3, 1, 1, "", "count"], [3, 1, 1, "", "load_yaml"], [3, 1, 1, "", "update_hdf5_file_with_review"], [3, 1, 1, "", "validate_yaml_dict"]], "src": [[4, 0, 0, "-", "hdf5_ops"], [4, 0, 0, "-", "hdf5_vis"], [4, 0, 0, "-", "hdf5_writer"]], "src.hdf5_ops": [[4, 2, 1, "", "HDF5DataOpsManager"], [4, 1, 1, "", "get_groups_at_a_level"], [4, 1, 1, "", "get_parent_child_relationships"], [4, 1, 1, "", "read_mtable_as_dataframe"], [4, 1, 1, "", "serialize_metadata"]], "src.hdf5_ops.HDF5DataOpsManager": [[4, 3, 1, "", "append_dataset"], [4, 3, 1, "", "append_metadata"], [4, 3, 1, "", "delete_metadata"], [4, 3, 1, "", "extract_and_load_dataset_metadata"], [4, 3, 1, "", "extract_dataset_as_dataframe"], [4, 3, 1, "", "get_metadata"], [4, 3, 1, "", "load_file_obj"], [4, 3, 1, "", "reformat_datetime_column"], [4, 3, 1, "", "rename_metadata"], [4, 3, 1, "", "unload_file_obj"], [4, 3, 1, "", "update_file"], [4, 3, 1, "", "update_metadata"]], "src.hdf5_vis": [[4, 1, 1, "", "display_group_hierarchy_on_a_treemap"]], "src.hdf5_writer": [[4, 1, 1, "", "create_hdf5_file_from_dataframe"], [4, 1, 1, "", "create_hdf5_file_from_filesystem_path"], [4, 1, 1, "", "save_processed_dataframe_to_hdf5"]], "utils": [[5, 0, 0, "-", "g5505_utils"]], "utils.g5505_utils": [[5, 1, 1, "", "augment_with_filenumber"], [5, 1, 1, "", "augment_with_filetype"], [5, 1, 1, "", "convert_attrdict_to_np_structured_array"], [5, 1, 1, "", "convert_dataframe_to_np_structured_array"], [5, 1, 1, "", "convert_string_to_bytes"], [5, 1, 1, "", "copy_directory_with_contraints"], [5, 1, 1, "", "created_at"], [5, 1, 1, "", "group_by_df_column"], [5, 1, 1, "", "infer_units"], [5, 1, 1, "", "is_callable_list"], [5, 1, 1, "", "is_str_list"], [5, 1, 1, "", "is_structured_array"], [5, 1, 1, "", "make_file_copy"], [5, 1, 1, "", "progressBar"], [5, 1, 1, "", "sanitize_dataframe"], [5, 1, 1, "", "setup_logging"], [5, 1, 1, "", "split_sample_col_into_sample_and_data_quality_cols"], [5, 1, 1, "", "to_serializable_dtype"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "method", "Python method"]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:method"}, "terms": {"": [3, 4, 5], "0": 4, "00": 5, "09": 5, "10": 5, "100": 4, "2023": 5, "23": 5, "26": 5, "4": 4, "5": 5, "65": 4, "8": 5, "A": 4, "By": 4, "For": 4, "If": [4, 5], "The": 4, "These": 4, "To": 4, "_description_": 5, "_type_": 5, "access": 4, "ad": 4, "add": [], "add_dataset_info": [], "add_environment_info": [], "add_instrument_info": [], "add_project_info": [], "add_sample_info": [], "addit": 4, "admiss": 4, "all": 4, "allow": 5, "allowed_file_extens": [3, 5], "alreadi": 4, "amount": 4, "an": [3, 4], "analysi": 4, "ani": 4, "annot": 4, "annotation_dict": 4, "append": [3, 4], "append_dataset": 4, "append_metadata": 4, "approach": 4, "ar": 4, "arrai": [4, 5], "attr_nam": 3, "attr_to_be_delet": 4, "attr_val": 5, "attr_valu": [3, 5], "attribut": [3, 4, 5], "augment": [], "augment_with_filenumb": [0, 5], "augment_with_filetyp": [0, 5], "base": [3, 4, 5], "befor": 4, "being": 4, "bottom": 4, "branch": [], "byte": 4, "callabl": 4, "campaign": 4, "can": 4, "capabl": 4, "categor": 4, "celsiu": 5, "chang": [], "class": 4, "clear_metadata": [], "close_fil": [], "column": 4, "column_nam": [4, 5], "commit": [], "compat": 5, "complet": [], "complex": 4, "composit": 4, "compound": [4, 5], "configur": 3, "consid": 4, "consist": 4, "constraint": [3, 5], "construct": 4, "contain": [4, 5], "content": 4, "control": 4, "convers": 0, "convert": [4, 5], "convert_attrdict_to_np_structured_arrai": [0, 5], "convert_dataframe_to_np_structured_arrai": [0, 5], "convert_string_to_byt": [0, 5], "copi": [3, 4, 5], "copy_directory_with_contraint": [0, 5], "copy_subtree_and_create_hdf5": [0, 3], "count": [0, 3], "count_valu": 5, "creat": [3, 4], "create_hdf5_file_from_datafram": [0, 4], "create_hdf5_file_from_filesystem_path": [0, 4], "created_at": [0, 5], "current": 4, "d": [4, 5], "data": [0, 3], "data_integr": 3, "datafram": [4, 5], "dataset": 4, "dataset_dict": 4, "dataset_nam": 4, "dataset_path": [], "datetim": [], "datetime_format": 5, "datetime_step": 3, "default": 4, "defin": 4, "definit": 4, "delet": [3, 4], "delete_metadata": 4, "depth": 4, "desired_format": 4, "determin": 5, "df": [4, 5], "di_pipelin": [], "dict": [3, 4, 5], "dictionari": [3, 4, 5], "dima_pipelin": [], "directori": [3, 4, 5], "display_group_hierarchy_on_a_treemap": [0, 4], "disregard": 4, "do": 4, "done": [], "down": 4, "dry_run": 5, "dst": 3, "dstdatafolderpath": [], "dtype": 5, "e": 4, "each": [4, 5], "either": 4, "element": 4, "empti": [4, 5], "encod": [4, 5], "ensur": 4, "entri": 5, "exampl": [3, 5], "exist": 4, "experiment": 4, "extend": 4, "extens": 5, "extract": 4, "extract_and_load_dataset_metadata": 4, "extract_attrs_func": 4, "extract_dataset_as_datafram": 4, "f": 4, "fals": [3, 5], "field": 5, "file": [3, 4, 5], "file_path": 4, "filenam": 4, "filename_path": [], "filesystem": 4, "first": [], "first_initialize_metadata_review": [], "float": 4, "folder": 4, "folder_depth": 4, "form": 4, "format": [3, 4], "forward": 4, "found": 4, "fourth": [], "from": [4, 5], "function": [3, 4, 5], "fundament": 4, "g": 4, "g5505_util": 5, "gener": [], "generate_datetime_dict": [], "get": 4, "get_groups_at_a_level": [0, 4], "get_metadata": 4, "get_parent_child_relationship": [0, 4], "get_review_statu": [], "given": 4, "group": [4, 5], "group_by_df_column": [0, 5], "group_by_func": 4, "group_nam": 4, "h": [4, 5], "h5": 4, "handl": 4, "hdf5": [0, 3, 5], "hdf5_file_path": [], "hdf5_lib": [], "hdf5_obj": 3, "hdf5_op": 4, "hdf5_upload": [], "hdf5_vi": 4, "hdf5_writer": 4, "hdf5dataopsmanag": [0, 4], "hdf5file": 4, "helper": 3, "hierarch": 4, "hierarchi": 4, "homogen": 4, "how": 4, "howev": 4, "i": [4, 5], "ignor": 4, "includ": 4, "index": 0, "infer_unit": [0, 5], "inform": 4, "initi": [], "input": [3, 4, 5], "input_array_byt": 5, "input_data": [4, 5], "input_dir_path": 5, "input_file_system_path": 4, "input_filename_path": 4, "input_hdf5_fil": 3, "input_list": 5, "instrument": 4, "int": 4, "integr": [3, 4], "integrate_data_sourc": [], "is_callable_list": [0, 5], "is_str_list": [0, 5], "is_structured_arrai": [0, 5], "issu": 4, "json": [4, 5], "kei": [3, 4, 5], "key_or_dict": [], "keyword": 5, "last_submit_metadata_review": [], "level": 4, "like": 4, "limit": 4, "list": [3, 4, 5], "load": 3, "load_config_and_setup_log": [0, 3], "load_dataset_metadata": [], "load_file_obj": 4, "load_yaml": [0, 3], "locat": 4, "log": [3, 5], "log_dir": [3, 5], "log_filenam": 5, "longest": 5, "m": [4, 5], "mai": 4, "main": [], "make_file_copi": [0, 5], "map": 5, "mark": 4, "matlab": 4, "metadata": [3, 4], "metadata_revis": 3, "metadataharvest": [], "method": 4, "middl": 4, "miss": 5, "mode": 4, "modifi": [], "modul": 0, "more": 4, "much": 4, "multi": 4, "must": 4, "name": [4, 5], "ndarrai": 5, "new": 4, "new_attr_2": 4, "new_attr_nam": 4, "new_attr_valu": 5, "non": 4, "none": [4, 5], "notebook": 0, "numer": 4, "numpi": [4, 5], "obj_nam": 4, "obj_path": 4, "object": [3, 4, 5], "object_nam": 3, "ofilenam": 4, "old_attr_2": 4, "old_attr_nam": 4, "onc": [], "one": 4, "ones": 4, "onli": 4, "open_fil": [], "oper": [0, 3, 5], "option": [4, 5], "our": 4, "output": [4, 5], "output_dir_path": 5, "output_filenam": 4, "output_folder_nam": 5, "output_format": 4, "overwrit": 4, "page": 0, "panda": [4, 5], "parent_fil": [], "path": [3, 4, 5], "path_to_append_dir": 4, "path_to_config_yamlfil": 3, "path_to_fil": 4, "path_to_filenames_dict": 4, "path_to_files_dict": 5, "path_to_input_dir": [], "path_to_input_directori": 4, "path_to_rawdata_fold": [], "pd": 4, "per": 4, "percentag": 4, "pipelin": 0, "place": 5, "possibl": 4, "power": 4, "pre": 4, "present": 4, "preserv": 4, "print_metadata": [], "process": 4, "progressbar": [0, 5], "project": [], "provid": [3, 4], "r": 4, "rais": 4, "rang": 4, "re": [], "read": 4, "read_dataset_as_datafram": [], "read_dataset_from_hdf5fil": [], "read_mtable_as_datafram": [0, 4], "reader": 4, "reconstruct": 4, "reformat_datetime_column": 4, "relative_humid": 4, "renam": 4, "rename_metadata": 4, "renaming_map": 4, "repres": 4, "represent": 5, "requir": 3, "respect": 4, "restart": [], "result": 4, "return": 3, "review": [], "review_yaml_fil": 3, "review_yaml_file_path": [], "reviewer_attr": [], "revis": 4, "root": 4, "root_metadata_dict": [3, 4], "row": 4, "run": [], "run_pipelin": [0, 3], "sanitize_datafram": [0, 5], "satisfi": 5, "save": [3, 4, 5], "save_processed_dataframe_to_hdf5": [0, 4], "scalar": 4, "search": 0, "second": [], "second_save_metadata_review": [], "select": [4, 5], "select_dir_keyword": [3, 4, 5], "select_file_keyword": [3, 5], "seri": 4, "serial": 4, "serialize_metadata": [0, 4], "set": [3, 5], "setup_log": [0, 5], "should": 4, "slash": 4, "sourc": [3, 4, 5], "source_file_path": 5, "specifi": [3, 4, 5], "split_sample_col_into_sample_and_data_quality_col": [0, 5], "src": [3, 4], "src_format": 4, "srcdatafolderpath": [], "state": [], "statu": [], "step": [], "store": 4, "str": [3, 4, 5], "string": [4, 5], "structur": [0, 4], "suffix": 5, "suggest": 4, "support": 4, "system": 4, "tabl": 4, "take": 5, "target": 4, "temperatur": 5, "them": 4, "thi": 4, "third_update_hdf5_file_with_review": [], "those": 4, "time": 4, "timestamp": 5, "tmp_file": 5, "to_serializable_dtyp": [0, 5], "top": 4, "total": 5, "transform": 5, "travers": 4, "tree": 4, "true": [3, 4], "truncat": 4, "txt": [], "type": 5, "under": [], "unit": [4, 5], "unload_file_obj": 4, "up": [3, 4, 5], "updat": [3, 4], "update_fil": 4, "update_hdf5_file_with_review": [0, 3], "update_metadata": 4, "us": 4, "ut8": 5, "utf": 5, "utf8": 5, "util": 5, "valid": [3, 4], "validate_yaml_dict": [0, 3], "valu": [3, 4, 5], "valueerror": 4, "vapor": 4, "variabl": 4, "visual": 0, "w": 4, "warn": 4, "water": 4, "when": 4, "where": 4, "which": [4, 5], "while": 4, "wise": 4, "within": 4, "word": 4, "workflow": 0, "write": 4, "writer": 0, "x": 5, "y": [4, 5], "yalm": [], "yaml": [3, 4, 5], "yaml_config_file_path": 3, "yaml_dict": 3, "yaml_review_fil": [], "yml_dict": 3, "you": []}, "titles": ["Welcome to DIMA\u2019s documentation!", "<no title>", "Notebooks", "Pipelines and workflows", "HDF5 Data Operations", "Data Structure Conversion"], "titleterms": {"": 0, "behavior": 4, "content": 0, "convers": 5, "data": [4, 5], "dima": 0, "document": 0, "exampl": 4, "hdf5": 4, "indic": 0, "integr": [], "notebook": 2, "oper": 4, "paramet": [3, 4, 5], "pipelin": 3, "return": [4, 5], "structur": 5, "tabl": 0, "tutori": [], "util": [], "visual": 4, "welcom": 0, "workflow": 3, "writer": 4}}) \ No newline at end of file +Search.setIndex({"alltitles": {"Behavior:": [[4, "behavior"]], "Contents:": [[0, null]], "Data Structure Conversion": [[5, "module-utils.g5505_utils"]], "Data Visualization": [[6, "module-visualization.hdf5_vis"]], "Example:": [[4, "example"]], "HDF5 Data Operations": [[4, "module-src.hdf5_ops"]], "HDF5 Writer": [[4, "module-src.hdf5_writer"]], "Indices and tables": [[0, "indices-and-tables"]], "Notebooks": [[2, "module-notebooks"]], "Parameters": [[4, "id5"], [4, "id6"], [4, "id10"], [5, "parameters"], [5, "id1"], [5, "id3"], [5, "id5"]], "Parameters:": [[3, "parameters"], [4, "parameters"], [4, "id1"], [4, "id2"], [4, "id3"], [4, "id4"], [4, "id8"]], "Pipelines and workflows": [[3, "module-pipelines.data_integration"]], "Returns": [[4, "returns"], [4, "id7"], [4, "id11"], [5, "returns"], [5, "id2"], [5, "id4"], [5, "id6"]], "Returns:": [[4, "id9"]], "Welcome to DIMA\u2019s documentation!": [[0, "welcome-to-dima-s-documentation"]]}, "docnames": ["index", "modules/instruments", "modules/notebooks", "modules/pipelines", "modules/src", "modules/utils", "modules/vis"], "envversion": {"nbsphinx": 4, "sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["index.rst", "modules/instruments.rst", "modules/notebooks.rst", "modules/pipelines.rst", "modules/src.rst", "modules/utils.rst", "modules/vis.rst"], "indexentries": {"append_dataset() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.append_dataset", false]], "append_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.append_metadata", false]], "create_hdf5_file_from_dataframe() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.create_hdf5_file_from_dataframe", false]], "create_hdf5_file_from_filesystem_path() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.create_hdf5_file_from_filesystem_path", false]], "delete_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.delete_metadata", false]], "extract_and_load_dataset_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.extract_and_load_dataset_metadata", false]], "extract_dataset_as_dataframe() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.extract_dataset_as_dataframe", false]], "get_groups_at_a_level() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.get_groups_at_a_level", false]], "get_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.get_metadata", false]], "get_parent_child_relationships() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.get_parent_child_relationships", false]], "hdf5dataopsmanager (class in src.hdf5_ops)": [[4, "src.hdf5_ops.HDF5DataOpsManager", false]], "load_file_obj() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.load_file_obj", false]], "module": [[4, "module-src.hdf5_ops", false], [4, "module-src.hdf5_writer", false]], "read_mtable_as_dataframe() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.read_mtable_as_dataframe", false]], "reformat_datetime_column() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.reformat_datetime_column", false]], "rename_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.rename_metadata", false]], "save_processed_dataframe_to_hdf5() (in module src.hdf5_writer)": [[4, "src.hdf5_writer.save_processed_dataframe_to_hdf5", false]], "serialize_metadata() (in module src.hdf5_ops)": [[4, "src.hdf5_ops.serialize_metadata", false]], "src.hdf5_ops": [[4, "module-src.hdf5_ops", false]], "src.hdf5_writer": [[4, "module-src.hdf5_writer", false]], "unload_file_obj() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.unload_file_obj", false]], "update_file() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.update_file", false]], "update_metadata() (src.hdf5_ops.hdf5dataopsmanager method)": [[4, "src.hdf5_ops.HDF5DataOpsManager.update_metadata", false]]}, "objects": {"": [[2, 0, 0, "-", "notebooks"]], "pipelines": [[3, 0, 0, "-", "data_integration"], [3, 0, 0, "-", "metadata_revision"]], "pipelines.data_integration": [[3, 1, 1, "", "copy_subtree_and_create_hdf5"], [3, 1, 1, "", "load_config_and_setup_logging"], [3, 1, 1, "", "run_pipeline"]], "pipelines.metadata_revision": [[3, 1, 1, "", "count"], [3, 1, 1, "", "load_yaml"], [3, 1, 1, "", "update_hdf5_file_with_review"], [3, 1, 1, "", "validate_yaml_dict"]], "src": [[4, 0, 0, "-", "hdf5_ops"], [4, 0, 0, "-", "hdf5_writer"]], "src.hdf5_ops": [[4, 2, 1, "", "HDF5DataOpsManager"], [4, 1, 1, "", "get_groups_at_a_level"], [4, 1, 1, "", "get_parent_child_relationships"], [4, 1, 1, "", "read_mtable_as_dataframe"], [4, 1, 1, "", "serialize_metadata"]], "src.hdf5_ops.HDF5DataOpsManager": [[4, 3, 1, "", "append_dataset"], [4, 3, 1, "", "append_metadata"], [4, 3, 1, "", "delete_metadata"], [4, 3, 1, "", "extract_and_load_dataset_metadata"], [4, 3, 1, "", "extract_dataset_as_dataframe"], [4, 3, 1, "", "get_metadata"], [4, 3, 1, "", "load_file_obj"], [4, 3, 1, "", "reformat_datetime_column"], [4, 3, 1, "", "rename_metadata"], [4, 3, 1, "", "unload_file_obj"], [4, 3, 1, "", "update_file"], [4, 3, 1, "", "update_metadata"]], "src.hdf5_writer": [[4, 1, 1, "", "create_hdf5_file_from_dataframe"], [4, 1, 1, "", "create_hdf5_file_from_filesystem_path"], [4, 1, 1, "", "save_processed_dataframe_to_hdf5"]], "utils": [[5, 0, 0, "-", "g5505_utils"]], "utils.g5505_utils": [[5, 1, 1, "", "augment_with_filenumber"], [5, 1, 1, "", "augment_with_filetype"], [5, 1, 1, "", "convert_attrdict_to_np_structured_array"], [5, 1, 1, "", "convert_dataframe_to_np_structured_array"], [5, 1, 1, "", "convert_string_to_bytes"], [5, 1, 1, "", "copy_directory_with_contraints"], [5, 1, 1, "", "created_at"], [5, 1, 1, "", "group_by_df_column"], [5, 1, 1, "", "infer_units"], [5, 1, 1, "", "is_callable_list"], [5, 1, 1, "", "is_str_list"], [5, 1, 1, "", "is_structured_array"], [5, 1, 1, "", "make_file_copy"], [5, 1, 1, "", "progressBar"], [5, 1, 1, "", "sanitize_dataframe"], [5, 1, 1, "", "setup_logging"], [5, 1, 1, "", "split_sample_col_into_sample_and_data_quality_cols"], [5, 1, 1, "", "to_serializable_dtype"]], "visualization": [[6, 0, 0, "-", "hdf5_vis"]], "visualization.hdf5_vis": [[6, 1, 1, "", "display_group_hierarchy_on_a_treemap"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "method", "Python method"]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:method"}, "terms": {"": [3, 4, 5, 6], "0": 4, "00": 5, "09": 5, "10": 5, "100": 4, "2023": 5, "23": 5, "26": 5, "4": 4, "5": 5, "65": 4, "8": 5, "A": 4, "By": 4, "For": 4, "If": [4, 5], "The": 4, "These": 4, "To": 4, "_description_": 5, "_type_": 5, "access": 4, "ad": 4, "add": [], "add_dataset_info": [], "add_environment_info": [], "add_instrument_info": [], "add_project_info": [], "add_sample_info": [], "addit": 4, "admiss": 4, "all": 4, "allow": 5, "allowed_file_extens": [3, 5], "alreadi": 4, "amount": 4, "an": [3, 4], "analysi": 4, "ani": 4, "annot": 4, "annotation_dict": 4, "append": [3, 4], "append_dataset": 4, "append_metadata": 4, "approach": 4, "ar": 4, "arrai": [4, 5], "attr_nam": 3, "attr_to_be_delet": 4, "attr_val": 5, "attr_valu": [3, 5], "attribut": [3, 4, 5], "augment": [], "augment_with_filenumb": [0, 5], "augment_with_filetyp": [0, 5], "base": [3, 4, 5], "befor": 4, "being": 4, "bottom": 4, "branch": [], "byte": 4, "callabl": 4, "campaign": 4, "can": 4, "capabl": 4, "categor": 4, "celsiu": 5, "chang": [], "class": 4, "clear_metadata": [], "close_fil": [], "column": 4, "column_nam": [4, 5], "commit": [], "compat": 5, "complet": [], "complex": 4, "composit": 4, "compound": [4, 5], "configur": 3, "consid": 4, "consist": 4, "constraint": [3, 5], "construct": 4, "contain": [4, 5], "content": 4, "control": 4, "convers": 0, "convert": [4, 5], "convert_attrdict_to_np_structured_arrai": [0, 5], "convert_dataframe_to_np_structured_arrai": [0, 5], "convert_string_to_byt": [0, 5], "copi": [3, 4, 5], "copy_directory_with_contraint": [0, 5], "copy_subtree_and_create_hdf5": [0, 3], "count": [0, 3], "count_valu": 5, "creat": [3, 4], "create_hdf5_file_from_datafram": [0, 4], "create_hdf5_file_from_filesystem_path": [0, 4], "created_at": [0, 5], "current": 4, "d": [4, 5], "data": [0, 3], "data_integr": 3, "datafram": [4, 5], "dataset": 4, "dataset_dict": 4, "dataset_nam": 4, "dataset_path": [], "datetim": [], "datetime_format": 5, "datetime_step": 3, "default": 4, "defin": 4, "definit": 4, "delet": [3, 4], "delete_metadata": 4, "depth": 4, "desired_format": 4, "determin": 5, "df": [4, 5], "di_pipelin": [], "dict": [3, 4, 5], "dictionari": [3, 4, 5], "dima_pipelin": [], "directori": [3, 4, 5], "display_group_hierarchy_on_a_treemap": [0, 6], "disregard": 4, "do": 4, "done": [], "down": 4, "dry_run": 5, "dst": 3, "dstdatafolderpath": [], "dtype": 5, "e": 4, "each": [4, 5], "either": 4, "element": 4, "empti": [4, 5], "encod": [4, 5], "ensur": 4, "entri": 5, "exampl": [3, 5], "exist": 4, "experiment": 4, "extend": 4, "extens": 5, "extract": 4, "extract_and_load_dataset_metadata": 4, "extract_attrs_func": 4, "extract_dataset_as_datafram": 4, "f": 4, "fals": [3, 5], "field": 5, "file": [3, 4, 5, 6], "file_path": 4, "filenam": [4, 6], "filename_path": [], "filesystem": 4, "first": [], "first_initialize_metadata_review": [], "float": 4, "folder": 4, "folder_depth": 4, "form": 4, "format": [3, 4], "forward": 4, "found": 4, "fourth": [], "from": [4, 5], "function": [3, 4, 5], "fundament": 4, "g": 4, "g5505_util": 5, "gener": [], "generate_datetime_dict": [], "get": 4, "get_groups_at_a_level": [0, 4], "get_metadata": 4, "get_parent_child_relationship": [0, 4], "get_review_statu": [], "given": 4, "group": [4, 5], "group_by_df_column": [0, 5], "group_by_func": 4, "group_nam": 4, "h": [4, 5], "h5": 4, "handl": 4, "hdf5": [0, 3, 5, 6], "hdf5_file_path": [], "hdf5_lib": [], "hdf5_obj": 3, "hdf5_op": 4, "hdf5_upload": [], "hdf5_vi": 6, "hdf5_writer": 4, "hdf5dataopsmanag": [0, 4], "hdf5file": 4, "helper": 3, "hierarch": 4, "hierarchi": 4, "homogen": 4, "how": 4, "howev": 4, "i": [4, 5], "ignor": 4, "includ": 4, "index": 0, "infer_unit": [0, 5], "inform": 4, "initi": [], "input": [3, 4, 5], "input_array_byt": 5, "input_data": [4, 5], "input_dir_path": 5, "input_file_system_path": 4, "input_filename_path": 4, "input_hdf5_fil": 3, "input_list": 5, "instrument": 4, "int": 4, "integr": [3, 4], "integrate_data_sourc": [], "is_callable_list": [0, 5], "is_str_list": [0, 5], "is_structured_arrai": [0, 5], "issu": 4, "json": [4, 5], "kei": [3, 4, 5], "key_or_dict": [], "keyword": 5, "last_submit_metadata_review": [], "level": 4, "like": 4, "limit": 4, "list": [3, 4, 5], "load": 3, "load_config_and_setup_log": [0, 3], "load_dataset_metadata": [], "load_file_obj": 4, "load_yaml": [0, 3], "locat": 4, "log": [3, 5], "log_dir": [3, 5], "log_filenam": 5, "longest": 5, "m": [4, 5], "mai": 4, "main": [], "make_file_copi": [0, 5], "map": 5, "mark": 4, "matlab": 4, "metadata": [3, 4], "metadata_revis": 3, "metadataharvest": [], "method": 4, "middl": 4, "miss": 5, "mode": 4, "modifi": [], "modul": 0, "more": 4, "much": 4, "multi": 4, "must": 4, "name": [4, 5], "ndarrai": 5, "new": 4, "new_attr_2": 4, "new_attr_nam": 4, "new_attr_valu": 5, "non": 4, "none": [4, 5], "notebook": 0, "numer": 4, "numpi": [4, 5], "obj_nam": 4, "obj_path": 4, "object": [3, 4, 5], "object_nam": 3, "ofilenam": 4, "old_attr_2": 4, "old_attr_nam": 4, "onc": [], "one": 4, "ones": 4, "onli": 4, "open_fil": [], "oper": [0, 3, 5], "option": [4, 5], "our": 4, "output": [4, 5], "output_dir_path": 5, "output_filenam": 4, "output_folder_nam": 5, "output_format": 4, "overwrit": 4, "page": 0, "panda": [4, 5], "parent_fil": [], "path": [3, 4, 5], "path_to_append_dir": 4, "path_to_config_yamlfil": 3, "path_to_fil": 4, "path_to_filenames_dict": 4, "path_to_files_dict": 5, "path_to_input_dir": [], "path_to_input_directori": 4, "path_to_rawdata_fold": [], "pd": 4, "per": 4, "percentag": 4, "pipelin": 0, "place": 5, "possibl": 4, "power": 4, "pre": 4, "present": 4, "preserv": 4, "print_metadata": [], "process": 4, "progressbar": [0, 5], "project": [], "provid": [3, 4], "r": 4, "rais": 4, "rang": 4, "re": [], "read": 4, "read_dataset_as_datafram": [], "read_dataset_from_hdf5fil": [], "read_mtable_as_datafram": [0, 4], "reader": 4, "reconstruct": 4, "reformat_datetime_column": 4, "relative_humid": 4, "renam": 4, "rename_metadata": 4, "renaming_map": 4, "repres": 4, "represent": 5, "requir": 3, "respect": 4, "restart": [], "result": 4, "return": 3, "review": [], "review_yaml_fil": 3, "review_yaml_file_path": [], "reviewer_attr": [], "revis": 4, "root": 4, "root_metadata_dict": [3, 4], "row": 4, "run": [], "run_pipelin": [0, 3], "sanitize_datafram": [0, 5], "satisfi": 5, "save": [3, 4, 5], "save_processed_dataframe_to_hdf5": [0, 4], "scalar": 4, "search": 0, "second": [], "second_save_metadata_review": [], "select": [4, 5], "select_dir_keyword": [3, 4, 5], "select_file_keyword": [3, 5], "seri": 4, "serial": 4, "serialize_metadata": [0, 4], "set": [3, 5], "setup_log": [0, 5], "should": 4, "slash": 4, "sourc": [3, 4, 5, 6], "source_file_path": 5, "specifi": [3, 4, 5], "split_sample_col_into_sample_and_data_quality_col": [0, 5], "src": [3, 4], "src_format": 4, "srcdatafolderpath": [], "state": [], "statu": [], "step": [], "store": 4, "str": [3, 4, 5, 6], "string": [4, 5], "structur": [0, 4], "suffix": 5, "suggest": 4, "support": 4, "system": 4, "tabl": 4, "take": 5, "target": 4, "temperatur": 5, "them": 4, "thi": 4, "third_update_hdf5_file_with_review": [], "those": 4, "time": 4, "timestamp": 5, "tmp_file": 5, "to_serializable_dtyp": [0, 5], "top": 4, "total": 5, "transform": 5, "travers": 4, "tree": 4, "true": [3, 4], "truncat": 4, "txt": [], "type": 5, "under": [], "unit": [4, 5], "unload_file_obj": 4, "up": [3, 4, 5], "updat": [3, 4], "update_fil": 4, "update_hdf5_file_with_review": [0, 3], "update_metadata": 4, "us": 4, "ut8": 5, "utf": 5, "utf8": 5, "util": 5, "valid": [3, 4], "validate_yaml_dict": [0, 3], "valu": [3, 4, 5], "valueerror": 4, "vapor": 4, "variabl": 4, "visual": 0, "w": 4, "warn": 4, "water": 4, "when": 4, "where": 4, "which": [4, 5], "while": 4, "wise": 4, "within": 4, "word": 4, "workflow": 0, "write": 4, "writer": 0, "x": 5, "y": [4, 5], "yalm": [], "yaml": [3, 4, 5], "yaml_config_file_path": 3, "yaml_dict": 3, "yaml_review_fil": [], "yml_dict": 3, "you": []}, "titles": ["Welcome to DIMA\u2019s documentation!", "<no title>", "Notebooks", "Pipelines and workflows", "HDF5 Data Operations", "Data Structure Conversion", "Data Visualization"], "titleterms": {"": 0, "behavior": 4, "content": 0, "convers": 5, "data": [4, 5, 6], "dima": 0, "document": 0, "exampl": 4, "hdf5": 4, "indic": 0, "integr": [], "notebook": 2, "oper": 4, "paramet": [3, 4, 5], "pipelin": 3, "return": [4, 5], "structur": 5, "tabl": 0, "tutori": [], "util": [], "visual": 6, "welcom": 0, "workflow": 3, "writer": 4}}) \ No newline at end of file diff --git a/docs/software arquitecture diagram.svg b/docs/software arquitecture diagram.svg new file mode 100644 index 0000000..cbdf89c --- /dev/null +++ b/docs/software arquitecture diagram.svg @@ -0,0 +1,1288 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + ExperimentalCampaignProject(Internal) + + + ScienceProject 1(Internal) + ScienceProject 2(External) + + + + DIMAPackage(Public) + + + + + + + + + + has subproject + + + + information flow + + + + + has subproject + + + + information flow + + + + + + + + + + + + File StandarizationModuleinstruments/ + + Data StructureConversionutils/ + + + + + + + + HDF5 Writersrc/ + + + + + HDF5 Data Operationssrc/ + + + + + + Git Operationssrc/ + + + + + + + + + Metadata Rev.Pipelinepipelines/ + + + + Visualizationvisualization/ + + + File Standardization and Storage + + + Information Processing and Data Analysis + + + Data Management + + + + + Data IntegrationPipelinepipelines/ + ETL and Data Analysis Demosnotebooks/ + + diff --git a/docs/source/index.rst b/docs/source/index.rst index f8489d2..aadb26a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,8 @@ Welcome to DIMA's documentation! modules/pipelines + modules/vis + modules/utils modules/notebooks diff --git a/docs/source/modules/src.rst b/docs/source/modules/src.rst index e9c4fc2..ecc8e69 100644 --- a/docs/source/modules/src.rst +++ b/docs/source/modules/src.rst @@ -14,14 +14,5 @@ HDF5 Writer :undoc-members: :show-inheritance: -Data Visualization -================== - -.. automodule:: src.hdf5_vis - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/source/modules/vis.rst b/docs/source/modules/vis.rst new file mode 100644 index 0000000..5a25e0f --- /dev/null +++ b/docs/source/modules/vis.rst @@ -0,0 +1,7 @@ +Data Visualization +================== + +.. automodule:: visualization.hdf5_vis + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file