HPCE/gitea-pages
0
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Activity
Files
0630bafb43cc9b996f31220bdd9f09ea6b5f90e4
gitea-pages/merlin7/02-How-To-Use-Merlin/archive/index.html
Gitea Actions 0630bafb43 Deployed acde833 with MkDocs version: 1.6.1
2026-02-10 11:25:50 +00:00

5115 lines
105 KiB
HTML
Raw Blame History

<!DOCTYPE html><html lang="en" class="no-js"><head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="canonical" href="https://hpce.pages.psi.ch/merlin7/02-How-To-Use-Merlin/archive/">
<link rel="prev" href="../transfer-data/">
<link rel="next" href="../nomachine/">
<link rel="icon" href="../../../images/favicon.ico">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.0">
<title>Archive &amp; PSI Data Catalog - HPCE User Documentation</title>
<link rel="stylesheet" href="../../../assets/stylesheets/main.618322db.min.css">
<link rel="stylesheet" href="../../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&amp;display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../../../css/custom-nav.css">
<script>__md_scope=new URL("../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
<link href="../../../assets/stylesheets/glightbox.min.css" rel="stylesheet"><script src="../../../assets/javascripts/glightbox.min.js"></script><style id="glightbox-style">
html.glightbox-open { overflow: initial; height: 100%; }
.gslide-title { margin-top: 0px; user-select: text; }
.gslide-desc { color: #666; user-select: text; }
.gslide-image img { background: white; }
.gscrollbar-fixer { padding-right: 15px; }
.gdesc-inner { font-size: 0.75rem; }
body[data-md-color-scheme="slate"] .gdesc-inner { background: var(--md-default-bg-color); }
body[data-md-color-scheme="slate"] .gslide-title { color: var(--md-default-fg-color); }
body[data-md-color-scheme="slate"] .gslide-desc { color: var(--md-default-fg-color); }
</style></head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#archive-psi-data-catalog" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../../.." title="HPCE User Documentation" class="md-header__button md-logo" aria-label="HPCE User Documentation" data-md-component="logo">
<img src="../../../images/hpce_logo.png" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"></path></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
HPCE User Documentation
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Archive &amp; PSI Data Catalog
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3.9 12c0-1.71 1.39-3.1 3.1-3.1h4V7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h4v-1.9H7c-1.71 0-3.1-1.39-3.1-3.1M8 13h8v-2H8zm9-6h-4v1.9h4c1.71 0 3.1 1.39 3.1 3.1s-1.39 3.1-3.1 3.1h-4V17h4a5 5 0 0 0 5-5 5 5 0 0 0-5-5"></path></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3"></path></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to system preference" type="radio" name="__palette" id="__palette_2">
<label class="md-header__button md-icon" title="Switch to system preference" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5M7 15a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3"></path></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"></path></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"></path></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"></path></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"></path></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://gitea.psi.ch/hpce/gitea-pages" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"></path></svg>
</div>
<div class="md-source__repository">
User Documentation Repository
</div>
</a>
</div>
</nav>
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../../../news/" class="md-tabs__link">
News
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../../01-Quick-Start-Guide/introduction/" class="md-tabs__link">
Merlin 7
</a>
</li>
<li class="md-tabs__item">
<a href="../../../merlin6/introduction/" class="md-tabs__link">
Merlin 6
</a>
</li>
<li class="md-tabs__item">
<a href="../../../meg/introduction/" class="md-tabs__link">
MeG
</a>
</li>
<li class="md-tabs__item">
<a href="../../../cscs-userlab/introduction/" class="md-tabs__link">
PSI@CSCS
</a>
</li>
<li class="md-tabs__item">
<a href="../../../support/introduction/" class="md-tabs__link">
Support
</a>
</li>
</ul>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../.." title="HPCE User Documentation" class="md-nav__button md-logo" aria-label="HPCE User Documentation" data-md-component="logo">
<img src="../../../images/hpce_logo.png" alt="logo">
</a>
HPCE User Documentation
</label>
<div class="md-nav__source">
<a href="https://gitea.psi.ch/hpce/gitea-pages" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"></path></svg>
</div>
<div class="md-source__repository">
User Documentation Repository
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2">
<div class="md-nav__link md-nav__container">
<a href="../../../news/" class="md-nav__link ">
<span class="md-ellipsis">
News
</span>
</a>
<label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="0">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
News
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_2">
<label class="md-nav__link" for="__nav_2_2" id="__nav_2_2_label" tabindex="0">
<span class="md-ellipsis">
Archive
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_2">
<span class="md-nav__icon md-icon"></span>
Archive
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../news/archive/2026/" class="md-nav__link">
<span class="md-ellipsis">
2026
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../news/archive/2024/" class="md-nav__link">
<span class="md-ellipsis">
2024
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../news/archive/2019/" class="md-nav__link">
<span class="md-ellipsis">
2019
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
<span class="md-ellipsis">
Merlin 7
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Merlin 7
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1">
<label class="md-nav__link" for="__nav_3_1" id="__nav_3_1_label" tabindex="">
<span class="md-ellipsis">
Quick Start Guide
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_1">
<span class="md-nav__icon md-icon"></span>
Quick Start Guide
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/introduction/" class="md-nav__link">
<span class="md-ellipsis">
Introduction
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/code-of-conduct/" class="md-nav__link">
<span class="md-ellipsis">
Code Of Conduct
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/requesting-accounts/" class="md-nav__link">
<span class="md-ellipsis">
Requesting Merlin Accounts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/requesting-projects/" class="md-nav__link">
<span class="md-ellipsis">
Requesting a Merlin Project
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/accessing-interactive-nodes/" class="md-nav__link">
<span class="md-ellipsis">
Accessing Interactive Nodes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../01-Quick-Start-Guide/accessing-slurm/" class="md-nav__link">
<span class="md-ellipsis">
Accessing Slurm Cluster
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" checked>
<label class="md-nav__link" for="__nav_3_2" id="__nav_3_2_label" tabindex="">
<span class="md-ellipsis">
How To Use Merlin7
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_3_2">
<span class="md-nav__icon md-icon"></span>
How To Use Merlin7
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../connect-from-linux/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a Linux Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../connect-from-windows/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a Windows Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../connect-from-macos/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a MacOS Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../storage/" class="md-nav__link">
<span class="md-ellipsis">
Merlin7 Storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../transfer-data/" class="md-nav__link">
<span class="md-ellipsis">
Transferring Data
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Archive &amp; PSI Data Catalog
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Archive &amp; PSI Data Catalog
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#psi-data-catalog-as-a-psi-central-service" class="md-nav__link">
<span class="md-ellipsis">
PSI Data Catalog as a PSI Central Service
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#procedure" class="md-nav__link">
<span class="md-ellipsis">
Procedure
</span>
</a>
<nav class="md-nav" aria-label="Procedure">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#account-registration" class="md-nav__link">
<span class="md-ellipsis">
Account Registration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#documentation" class="md-nav__link">
<span class="md-ellipsis">
Documentation
</span>
</a>
<nav class="md-nav" aria-label="Documentation">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#loading-datacatalog-tools" class="md-nav__link">
<span class="md-ellipsis">
Loading datacatalog tools
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#finding-your-token" class="md-nav__link">
<span class="md-ellipsis">
Finding your token
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#ingestion" class="md-nav__link">
<span class="md-ellipsis">
Ingestion
</span>
</a>
<nav class="md-nav" aria-label="Ingestion">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#useful-commands" class="md-nav__link">
<span class="md-ellipsis">
Useful commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting-known-bugs" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting &amp; Known Bugs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sample-ingestion-output-datasetingestor-1111" class="md-nav__link">
<span class="md-ellipsis">
Sample ingestion output (datasetIngestor 1.1.11)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#publishing" class="md-nav__link">
<span class="md-ellipsis">
Publishing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#retrieving-data" class="md-nav__link">
<span class="md-ellipsis">
Retrieving data
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#further-information" class="md-nav__link">
<span class="md-ellipsis">
Further Information
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../nomachine/" class="md-nav__link">
<span class="md-ellipsis">
Remote Desktop Access to Merlin7
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../merlin-rmount/" class="md-nav__link">
<span class="md-ellipsis">
Using merlin_rmount
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../merlin_tools/" class="md-nav__link">
<span class="md-ellipsis">
Merlin7 Tools
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ssh-keys/" class="md-nav__link">
<span class="md-ellipsis">
Configuring SSH Keys in Merlin
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../kerberos/" class="md-nav__link">
<span class="md-ellipsis">
Kerberos and AFS authentication
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../software-repositories/" class="md-nav__link">
<span class="md-ellipsis">
Software repositories
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3">
<label class="md-nav__link" for="__nav_3_3" id="__nav_3_3_label" tabindex="">
<span class="md-ellipsis">
Slurm General Documentation
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_3">
<span class="md-nav__icon md-icon"></span>
Slurm General Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../03-Slurm-General-Documentation/merlin7-configuration/" class="md-nav__link">
<span class="md-ellipsis">
Slurm cluster 'merlin7'
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../03-Slurm-General-Documentation/slurm-configuration/" class="md-nav__link">
<span class="md-ellipsis">
Slurm merlin7 Configuration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../03-Slurm-General-Documentation/interactive-jobs/" class="md-nav__link">
<span class="md-ellipsis">
Running Interactive Jobs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../03-Slurm-General-Documentation/slurm-examples/" class="md-nav__link">
<span class="md-ellipsis">
Slurm Examples
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_4">
<label class="md-nav__link" for="__nav_3_4" id="__nav_3_4_label" tabindex="">
<span class="md-ellipsis">
Jupyterhub
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_4">
<span class="md-nav__icon md-icon"></span>
Jupyterhub
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../04-Jupyterhub/jupyterhub/" class="md-nav__link">
<span class="md-ellipsis">
Jupyterhub on Merlin7
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_5">
<label class="md-nav__link" for="__nav_3_5" id="__nav_3_5_label" tabindex="">
<span class="md-ellipsis">
Software Support
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_5">
<span class="md-nav__icon md-icon"></span>
Software Support
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../05-Software-Support/pmodules/" class="md-nav__link">
<span class="md-ellipsis">
PSI Modules
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/spack/" class="md-nav__link">
<span class="md-ellipsis">
Spack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/openmpi/" class="md-nav__link">
<span class="md-ellipsis">
OpenMPI Support
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/ansys/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/ansys-rsm/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS RSM (Remote Resolve Manager)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/cp2k/" class="md-nav__link">
<span class="md-ellipsis">
CP2k
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/quantum-espresso/" class="md-nav__link">
<span class="md-ellipsis">
Quantum Espresso
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/lammps/" class="md-nav__link">
<span class="md-ellipsis">
LAMMPS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/gromacs/" class="md-nav__link">
<span class="md-ellipsis">
GROMACS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/opal-x/" class="md-nav__link">
<span class="md-ellipsis">
OPAL-X
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/ippl/" class="md-nav__link">
<span class="md-ellipsis">
IPPL
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../05-Software-Support/cray-module.env/" class="md-nav__link">
<span class="md-ellipsis">
Cray Programming Environment
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4">
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-ellipsis">
Merlin 6
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Merlin 6
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_1">
<label class="md-nav__link" for="__nav_4_1" id="__nav_4_1_label" tabindex="0">
<span class="md-ellipsis">
Quick Start Guide
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_1">
<span class="md-nav__icon md-icon"></span>
Quick Start Guide
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../merlin6/introduction/" class="md-nav__link">
<span class="md-ellipsis">
About Merlin6
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/quick-start-guide/code-of-conduct/" class="md-nav__link">
<span class="md-ellipsis">
Code Of Conduct
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/quick-start-guide/requesting-accounts/" class="md-nav__link">
<span class="md-ellipsis">
Requesting Merlin Accounts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/quick-start-guide/requesting-projects/" class="md-nav__link">
<span class="md-ellipsis">
Requesting a Merlin Project
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/quick-start-guide/accessing-interactive-nodes/" class="md-nav__link">
<span class="md-ellipsis">
Accessing Interactive Nodes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/quick-start-guide/accessing-slurm/" class="md-nav__link">
<span class="md-ellipsis">
Accessing Slurm Cluster
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_2">
<label class="md-nav__link" for="__nav_4_2" id="__nav_4_2_label" tabindex="0">
<span class="md-ellipsis">
How To Use Merlin
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_2">
<span class="md-nav__icon md-icon"></span>
How To Use Merlin
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/connect-from-linux/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a Linux Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/connect-from-windows/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a Windows Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/connect-from-macos/" class="md-nav__link">
<span class="md-ellipsis">
Connecting from a MacOS Client
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/storage/" class="md-nav__link">
<span class="md-ellipsis">
Merlin6 Storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/transfer-data/" class="md-nav__link">
<span class="md-ellipsis">
Transferring Data
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/archive/" class="md-nav__link">
<span class="md-ellipsis">
Archive &amp; PSI Data Catalog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/ssh-keys/" class="md-nav__link">
<span class="md-ellipsis">
Configuring SSH Keys in Merlin
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/kerberos/" class="md-nav__link">
<span class="md-ellipsis">
Kerberos and AFS authentication
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/how-to-use-merlin/using-modules/" class="md-nav__link">
<span class="md-ellipsis">
Using PModules
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_3">
<label class="md-nav__link" for="__nav_4_3" id="__nav_4_3_label" tabindex="0">
<span class="md-ellipsis">
Slurm General Documentation
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_3">
<span class="md-nav__icon md-icon"></span>
Slurm General Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../merlin6/slurm-general-docs/slurm-basic-commands/" class="md-nav__link">
<span class="md-ellipsis">
Slurm Basic Commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/slurm-general-docs/running-jobs/" class="md-nav__link">
<span class="md-ellipsis">
Running Slurm Scripts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/slurm-general-docs/interactive-jobs/" class="md-nav__link">
<span class="md-ellipsis">
Running Interactive Jobs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/slurm-general-docs/slurm-examples/" class="md-nav__link">
<span class="md-ellipsis">
Slurm Examples
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/slurm-general-docs/monitoring/" class="md-nav__link">
<span class="md-ellipsis">
Monitoring
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_4">
<label class="md-nav__link" for="__nav_4_4" id="__nav_4_4_label" tabindex="0">
<span class="md-ellipsis">
GPU Cluster
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_4">
<span class="md-nav__icon md-icon"></span>
GPU Cluster
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../gmerlin6/cluster-introduction/" class="md-nav__link">
<span class="md-ellipsis">
Introduction
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../gmerlin6/hardware-and-software-description/" class="md-nav__link">
<span class="md-ellipsis">
Hardware And Software Description
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../gmerlin6/slurm-configuration/" class="md-nav__link">
<span class="md-ellipsis">
Slurm cluster 'gmerlin6'
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5">
<label class="md-nav__link" for="__nav_4_5" id="__nav_4_5_label" tabindex="0">
<span class="md-ellipsis">
Jupyterhub
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_5">
<span class="md-nav__icon md-icon"></span>
Jupyterhub
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupyterhub/" class="md-nav__link">
<span class="md-ellipsis">
Jupyterhub on Merlin
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupyter-examples/" class="md-nav__link">
<span class="md-ellipsis">
Jupyter examples on merlin6
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupytext/" class="md-nav__link">
<span class="md-ellipsis">
Jupytext - efficient editing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupyter-extensions/" class="md-nav__link">
<span class="md-ellipsis">
Jupyter Extensions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupyterlab/" class="md-nav__link">
<span class="md-ellipsis">
Jupyterlab User interface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/jupyterhub/jupyterhub-trouble/" class="md-nav__link">
<span class="md-ellipsis">
Jupyterhub Troubleshooting
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_6">
<label class="md-nav__link" for="__nav_4_6" id="__nav_4_6_label" tabindex="0">
<span class="md-ellipsis">
Software Support
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_6">
<span class="md-nav__icon md-icon"></span>
Software Support
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys-rsm/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS - RSM
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys-cfx/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS - CFX
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys-fluent/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS - Fluent
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys-mapdl/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS - MAPDL
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/ansys-hfss/" class="md-nav__link">
<span class="md-ellipsis">
ANSYS HFSS (ElectroMagnetics)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/gothic/" class="md-nav__link">
<span class="md-ellipsis">
GOTHIC
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/merlin-rmount/" class="md-nav__link">
<span class="md-ellipsis">
merlin_rmount
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/impi/" class="md-nav__link">
<span class="md-ellipsis">
Intel MPI Support
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/openmpi/" class="md-nav__link">
<span class="md-ellipsis">
OpenMPI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/paraview/" class="md-nav__link">
<span class="md-ellipsis">
ParaView
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../merlin6/software-support/python/" class="md-nav__link">
<span class="md-ellipsis">
Python
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5">
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
MeG
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
MeG
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../meg/introduction/" class="md-nav__link">
<span class="md-ellipsis">
The MEG local HPC cluster
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../meg/contact/" class="md-nav__link">
<span class="md-ellipsis">
Support
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../meg/migration-to-merlin7/" class="md-nav__link">
<span class="md-ellipsis">
Meg to Merlin7 Migration Guide
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6">
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
<span class="md-ellipsis">
PSI@CSCS
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
PSI@CSCS
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../cscs-userlab/introduction/" class="md-nav__link">
<span class="md-ellipsis">
PSI HPC@CSCS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../cscs-userlab/transfer-data/" class="md-nav__link">
<span class="md-ellipsis">
Transferring Data
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7">
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Support
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Support
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../support/introduction/" class="md-nav__link">
<span class="md-ellipsis">
Getting Support
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../support/faq/" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../support/known-problems/" class="md-nav__link">
<span class="md-ellipsis">
Known Problems
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../support/troubleshooting/" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#psi-data-catalog-as-a-psi-central-service" class="md-nav__link">
<span class="md-ellipsis">
PSI Data Catalog as a PSI Central Service
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#procedure" class="md-nav__link">
<span class="md-ellipsis">
Procedure
</span>
</a>
<nav class="md-nav" aria-label="Procedure">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#account-registration" class="md-nav__link">
<span class="md-ellipsis">
Account Registration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#documentation" class="md-nav__link">
<span class="md-ellipsis">
Documentation
</span>
</a>
<nav class="md-nav" aria-label="Documentation">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#loading-datacatalog-tools" class="md-nav__link">
<span class="md-ellipsis">
Loading datacatalog tools
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#finding-your-token" class="md-nav__link">
<span class="md-ellipsis">
Finding your token
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#ingestion" class="md-nav__link">
<span class="md-ellipsis">
Ingestion
</span>
</a>
<nav class="md-nav" aria-label="Ingestion">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#useful-commands" class="md-nav__link">
<span class="md-ellipsis">
Useful commands
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting-known-bugs" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting &amp; Known Bugs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sample-ingestion-output-datasetingestor-1111" class="md-nav__link">
<span class="md-ellipsis">
Sample ingestion output (datasetIngestor 1.1.11)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#publishing" class="md-nav__link">
<span class="md-ellipsis">
Publishing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#retrieving-data" class="md-nav__link">
<span class="md-ellipsis">
Retrieving data
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#further-information" class="md-nav__link">
<span class="md-ellipsis">
Further Information
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<a href="https://gitea.psi.ch/hpce/gitea-pages/_edit/master/docs/merlin7/02-How-To-Use-Merlin/archive.md" title="Edit this page" class="md-content__button md-icon" rel="edit">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"></path></svg>
</a>
<h1 id="archive-psi-data-catalog">Archive &amp; PSI Data Catalog<a class="headerlink" href="#archive-psi-data-catalog" title="Permanent link"></a></h1>
<h2 id="psi-data-catalog-as-a-psi-central-service">PSI Data Catalog as a PSI Central Service<a class="headerlink" href="#psi-data-catalog-as-a-psi-central-service" title="Permanent link"></a></h2>
<p>PSI provides access to the <strong><em>Data Catalog</em></strong> for <strong>long-term data storage and retrieval</strong>. Data is
stored on the <strong><em>PetaByte Archive</em></strong> at the <strong>Swiss National Supercomputing Centre (CSCS)</strong>.</p>
<p>The Data Catalog and Archive is suitable for:</p>
<ul>
<li>Raw data generated by PSI instruments</li>
<li>Derived data produced by processing some inputs</li>
<li>Data required to reproduce PSI research and publications</li>
</ul>
<p>The Data Catalog is part of PSI's effort to conform to the FAIR principles for data management.
In accordance with this policy, <strong><em>data will be publicly released under CC-BY-SA 4.0 after an
embargo period expires.</em></strong></p>
<p>The Merlin cluster is connected to the Data Catalog. Hence, users archive data stored in the
Merlin storage under the <code>/data</code> directories (currentlyi, <code>/data/user</code> and <code>/data/project</code>).
Archiving from other directories is also possible, however the process is much slower as data
can not be directly retrieved by the PSI archive central servers (<strong>central mode</strong>), and needs to
be indirectly copied to these (<strong>decentral mode</strong>).</p>
<p>Archiving can be done from any node accessible by the users (usually from the login nodes).</p>
<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>Archiving can be done in two different ways:</p>
<ul>
<li><strong>Central mode</strong>: Possible for the user and project data directories, is
the fastest way as it does not require remote copy (data is directly retreived
by central AIT servers from Merlin through <merlin-archive.psi.ch>).</merlin-archive.psi.ch></li>
<li><strong>Decentral mode</strong>: Possible for any directory, is the slowest way of
archiving as it requires to copy ('rsync') the data from Merlin to the
central AIT servers.</li>
</ul>
</div>
<h2 id="procedure">Procedure<a class="headerlink" href="#procedure" title="Permanent link"></a></h2>
<h3 id="overview">Overview<a class="headerlink" href="#overview" title="Permanent link"></a></h3>
<p>Below are the main steps for using the Data Catalog.</p>
<ul>
<li>Ingest the dataset into the Data Catalog. This makes the data known to the Data Catalog system at PSI:<ul>
<li>Prepare a metadata file describing the dataset</li>
<li>Run <strong><code>datasetIngestor</code></strong> script</li>
<li>If necessary, the script will copy the data to the PSI archive servers<ul>
<li>Usually this is necessary when archiving from directories other than <strong><code>/data/user</code></strong> or
<strong><code>/data/project</code></strong>. It would be also necessary when the Merlin export server (<strong><code>merlin-archive.psi.ch</code></strong>)
is down for any reason.</li>
</ul>
</li>
</ul>
</li>
<li>Archive the dataset:<ul>
<li>Visit <a href="https://discovery.psi.ch&gt;">&lt;https://discovery.psi.ch</a></li>
<li>Click <strong><code>Archive</code></strong> for the dataset</li>
<li>The system will now copy the data to the PetaByte Archive at CSCS</li>
</ul>
</li>
<li>Retrieve data from the catalog:<ul>
<li>Find the dataset on <a href="https://discovery.psi.ch&gt;">&lt;https://discovery.psi.ch</a> and click <strong><code>Retrieve</code></strong></li>
<li>Wait for the data to be copied to the PSI retrieval system</li>
<li>Run <strong><code>datasetRetriever</code></strong> script</li>
</ul>
</li>
</ul>
<p>Since large data sets may take a lot of time to transfer, some steps are designed to happen in the
background. The discovery website can be used to track the progress of each step.</p>
<h3 id="account-registration">Account Registration<a class="headerlink" href="#account-registration" title="Permanent link"></a></h3>
<p>Two types of account permit access to the Data Catalog. If your data was collected at a <strong><em>beamline</em></strong>, you may
have been assigned a <strong><code>p-group</code></strong> (e.g. <code>p12345</code>) for the experiment. Other users are assigned <strong><code>a-group</code></strong>
(e.g. <code>a-12345</code>).</p>
<p>Groups are usually assigned to a PI, and then individual user accounts are added to the group. This must be done
under user request through PSI Service Now. For existing <strong>a-groups</strong> and <strong>p-groups</strong>, you can follow the standard
central procedures. Alternatively, if you do not know how to do that, follow the Merlin7
<strong><a href="../../01-Quick-Start-Guide/requesting-accounts/">Requesting extra Unix groups</a></strong> procedure, or open
a <strong><a href="https://psi.service-now.com/psisp">PSI Service Now</a></strong> ticket.</p>
<h3 id="documentation">Documentation<a class="headerlink" href="#documentation" title="Permanent link"></a></h3>
<p>Accessing the Data Catalog is done through the <a href="https://melanie.gitpages.psi.ch/SciCatPages/">SciCat software</a>.
Documentation is here: <a href="https://scicatproject.github.io/documentation/Ingestor/ingestManual.html">ingestManual</a>.</p>
<h4 id="loading-datacatalog-tools">Loading datacatalog tools<a class="headerlink" href="#loading-datacatalog-tools" title="Permanent link"></a></h4>
<p>The latest datacatalog software is maintained in the PSI module system. To access it from the Merlin systems, run the following command:</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>module<span class="w"> </span>load<span class="w"> </span>datacatalog
</span></code></pre></div>
<p>It can be done from any host in the Merlin cluster accessible by users. Usually, login nodes will be the nodes used for archiving.</p>
<h3 id="finding-your-token">Finding your token<a class="headerlink" href="#finding-your-token" title="Permanent link"></a></h3>
<p>As of 2022-04-14 a secure token is required to interact with the data catalog. This is a long random string that replaces the previous user/password authentication (allowing access for non-PSI use cases). <strong>This string should be treated like a password and not shared.</strong></p>
<ol>
<li>Go to discovery.psi.ch</li>
<li>Click 'Sign in' in the top right corner. Click the 'Login with PSI account' and log in on the PSI login1. page.</li>
<li>You should be redirected to your user settings and see a 'User Information' section. If not, click on1. your username in the top right and choose 'Settings' from the menu.</li>
<li>Look for the field 'Catamel Token'. This should be a 64-character string. Click the icon to copy the1. token.</li>
</ol>
<p><a class="glightbox" data-type="image" data-width="auto" data-height="auto" href="../../../images/scicat_token.png" data-desc-position="bottom"><img alt="SciCat website" src="../../../images/scicat_token.png"></a></p>
<p>You will need to save this token for later steps. To avoid including it in all the commands, I suggest saving it to an environmental variable (Linux):</p>
<div class="language-text highlight"><span class="filename">Text Only</span><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>$ SCICAT_TOKEN=RqYMZcqpqMJqluplbNYXLeSyJISLXfnkwlfBKuvTSdnlpKkU
</span></code></pre></div>
<p>(Hint: prefix this line with a space to avoid saving the token to your bash history.)</p>
<p>Tokens expire after 2 weeks and will need to be fetched from the website again.</p>
<h3 id="ingestion">Ingestion<a class="headerlink" href="#ingestion" title="Permanent link"></a></h3>
<p>The first step to ingesting your data into the catalog is to prepare a file describing what data you have. This is called
<strong><code>metadata.json</code></strong>, and can be created with a text editor (e.g. <em><code>vim</code></em>). It can in principle be saved anywhere,
but keeping it with your archived data is recommended. For more information about the format, see the 'Bio metadata'
section below. An example follows:</p>
<div class="language-yaml highlight"><span class="filename">YAML</span><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="p p-Indicator">{</span>
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="w"> </span><span class="s">"principalInvestigator"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"albrecht.gessler@psi.ch"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w"> </span><span class="s">"creationLocation"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"/PSI/EMF/JEOL2200FS"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w"> </span><span class="s">"dataFormat"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"TIFF+LZW</span><span class="nv"> </span><span class="s">Image</span><span class="nv"> </span><span class="s">Stack"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w"> </span><span class="s">"sourceFolder"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"/gpfs/group/LBR/pXXX/myimages"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="w"> </span><span class="s">"owner"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"Wilhelm</span><span class="nv"> </span><span class="s">Tell"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w"> </span><span class="s">"ownerEmail"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"wilhelm.tell@psi.ch"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="w"> </span><span class="s">"type"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"raw"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="w"> </span><span class="s">"description"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"EM</span><span class="nv"> </span><span class="s">micrographs</span><span class="nv"> </span><span class="s">of</span><span class="nv"> </span><span class="s">amygdalin"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a><span class="w"> </span><span class="s">"ownerGroup"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"a-12345"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a><span class="w"> </span><span class="s">"scientificMetadata"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a><span class="w"> </span><span class="s">"description"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"EM</span><span class="nv"> </span><span class="s">micrographs</span><span class="nv"> </span><span class="s">of</span><span class="nv"> </span><span class="s">amygdalin"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-13"><a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a><span class="w"> </span><span class="s">"sample"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-14"><a id="__codelineno-2-14" name="__codelineno-2-14" href="#__codelineno-2-14"></a><span class="w"> </span><span class="s">"name"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"Amygdalin</span><span class="nv"> </span><span class="s">beta-glucosidase</span><span class="nv"> </span><span class="s">1"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-15"><a id="__codelineno-2-15" name="__codelineno-2-15" href="#__codelineno-2-15"></a><span class="w"> </span><span class="s">"uniprot"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"P29259"</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-16"><a id="__codelineno-2-16" name="__codelineno-2-16" href="#__codelineno-2-16"></a><span class="w"> </span><span class="s">"species"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"Apple"</span>
</span><span id="__span-2-17"><a id="__codelineno-2-17" name="__codelineno-2-17" href="#__codelineno-2-17"></a><span class="w"> </span><span class="p p-Indicator">},</span>
</span><span id="__span-2-18"><a id="__codelineno-2-18" name="__codelineno-2-18" href="#__codelineno-2-18"></a><span class="w"> </span><span class="s">"dataCollection"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-19"><a id="__codelineno-2-19" name="__codelineno-2-19" href="#__codelineno-2-19"></a><span class="w"> </span><span class="s">"date"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"2018-08-01"</span>
</span><span id="__span-2-20"><a id="__codelineno-2-20" name="__codelineno-2-20" href="#__codelineno-2-20"></a><span class="w"> </span><span class="p p-Indicator">},</span>
</span><span id="__span-2-21"><a id="__codelineno-2-21" name="__codelineno-2-21" href="#__codelineno-2-21"></a><span class="w"> </span><span class="s">"microscopeParameters"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-22"><a id="__codelineno-2-22" name="__codelineno-2-22" href="#__codelineno-2-22"></a><span class="w"> </span><span class="s">"pixel</span><span class="nv"> </span><span class="s">size"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-23"><a id="__codelineno-2-23" name="__codelineno-2-23" href="#__codelineno-2-23"></a><span class="w"> </span><span class="s">"v"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="nv">0.885</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-24"><a id="__codelineno-2-24" name="__codelineno-2-24" href="#__codelineno-2-24"></a><span class="w"> </span><span class="s">"u"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"A"</span>
</span><span id="__span-2-25"><a id="__codelineno-2-25" name="__codelineno-2-25" href="#__codelineno-2-25"></a><span class="w"> </span><span class="p p-Indicator">},</span>
</span><span id="__span-2-26"><a id="__codelineno-2-26" name="__codelineno-2-26" href="#__codelineno-2-26"></a><span class="w"> </span><span class="s">"voltage"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-27"><a id="__codelineno-2-27" name="__codelineno-2-27" href="#__codelineno-2-27"></a><span class="w"> </span><span class="s">"v"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="nv">200</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-28"><a id="__codelineno-2-28" name="__codelineno-2-28" href="#__codelineno-2-28"></a><span class="w"> </span><span class="s">"u"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"kV"</span>
</span><span id="__span-2-29"><a id="__codelineno-2-29" name="__codelineno-2-29" href="#__codelineno-2-29"></a><span class="w"> </span><span class="p p-Indicator">},</span>
</span><span id="__span-2-30"><a id="__codelineno-2-30" name="__codelineno-2-30" href="#__codelineno-2-30"></a><span class="w"> </span><span class="s">"dosePerFrame"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="p p-Indicator">{</span>
</span><span id="__span-2-31"><a id="__codelineno-2-31" name="__codelineno-2-31" href="#__codelineno-2-31"></a><span class="w"> </span><span class="s">"v"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="nv">1.277</span><span class="p p-Indicator">,</span>
</span><span id="__span-2-32"><a id="__codelineno-2-32" name="__codelineno-2-32" href="#__codelineno-2-32"></a><span class="w"> </span><span class="s">"u"</span><span class="p p-Indicator">:</span><span class="w"> </span><span class="s">"e/A2"</span>
</span><span id="__span-2-33"><a id="__codelineno-2-33" name="__codelineno-2-33" href="#__codelineno-2-33"></a><span class="w"> </span><span class="p p-Indicator">}</span>
</span><span id="__span-2-34"><a id="__codelineno-2-34" name="__codelineno-2-34" href="#__codelineno-2-34"></a><span class="w"> </span><span class="p p-Indicator">}</span>
</span><span id="__span-2-35"><a id="__codelineno-2-35" name="__codelineno-2-35" href="#__codelineno-2-35"></a><span class="w"> </span><span class="p p-Indicator">}</span>
</span><span id="__span-2-36"><a id="__codelineno-2-36" name="__codelineno-2-36" href="#__codelineno-2-36"></a><span class="p p-Indicator">}</span>
</span></code></pre></div>
<p>It is recommended to use the <a href="https://bliven_s.gitpages.psi.ch/SciCatEditor/">ScicatEditor</a> for creating metadata files. This is a browser-based tool specifically for ingesting PSI data. Using the tool avoids syntax errors and provides templates for common data sets and options. The finished JSON file can then be downloaded to merlin or copied into a text editor.</p>
<p>Another option is to use the SciCat graphical interface from NoMachine. This provides a graphical interface for selecting data to archive. This is particularly useful for data associated with a DUO experiment and p-group. Type <code>SciCat`` to get started after loading the</code>datacatalog`` module. The GUI also replaces the the command-line ingestion described below.</p>
<p>The following steps can be run from wherever you saved your <code>metadata.json</code>. First, perform a "dry-run" which will check the metadata for errors:</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>datasetIngestor<span class="w"> </span>--token<span class="w"> </span><span class="nv">$SCICAT_TOKEN</span><span class="w"> </span>metadata.json
</span></code></pre></div>
<p>It will ask for your PSI credentials and then print some info about the data to be ingested. If there are no errors, proceed to the real ingestion:</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>datasetIngestor<span class="w"> </span>--token<span class="w"> </span><span class="nv">$SCICAT_TOKEN</span><span class="w"> </span>--ingest<span class="w"> </span>--autoarchive<span class="w"> </span>metadata.json
</span></code></pre></div>
<p>You will be asked whether you want to copy the data to the central system:</p>
<ul>
<li>If you are on the Merlin cluster and you are archiving data from <code>/data/user</code> or <code>/data/project</code>, answer 'no' since the data catalog can</li>
</ul>
<p>directly read the data.
* If you are on a directory other than <code>/data/user</code> and ``/data/project, or you are on a desktop computer, answer 'yes'. Copying large datasets
to the PSI archive system may take quite a while (minutes to hours).</p>
<p>If there are no errors, your data has been accepted into the data catalog! From now on, no changes should be made to the ingested data.
This is important, since the next step is for the system to copy all the data to the CSCS Petabyte archive. Writing to tape is slow, so
this process may take several days, and it will fail if any modifications are detected.</p>
<p>If using the <code>--autoarchive</code> option as suggested above, your dataset should now be in the queue. Check the data catalog:
<a href="https://discovery.psi.ch&gt;">&lt;https://discovery.psi.ch</a>. Your job should have status 'WorkInProgress'. You will receive an email when the ingestion
is complete.</p>
<p>If you didn't use <code>--autoarchive</code>, you need to manually move the dataset into the archive queue. From <strong>discovery.psi.ch</strong>, navigate to the 'Archive'
tab. You should see the newly ingested dataset. Check the dataset and click <strong><code>Archive</code></strong>. You should see the status change from <strong><code>datasetCreated</code></strong> to
<strong><code>scheduleArchiveJob</code></strong>. This indicates that the data is in the process of being transferred to CSCS.</p>
<p>After a few days the dataset's status will change to <strong><code>datasetOnAchive</code></strong> indicating the data is stored. At this point it is safe to delete the data.</p>
<h4 id="useful-commands">Useful commands<a class="headerlink" href="#useful-commands" title="Permanent link"></a></h4>
<p>Running the datasetIngestor in dry mode (<strong>without</strong> <code>--ingest</code>) finds most errors. However, it is sometimes convenient to find potential errors
yourself with simple unix commands.</p>
<p>Find problematic filenames</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>find<span class="w"> </span>.<span class="w"> </span>-iregex<span class="w"> </span><span class="s1">'.*/[^/]*[^a-zA-Z0-9_ ./-][^/]*'</span><span class="o">=</span>
</span></code></pre></div>
<p>Find broken links</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>find<span class="w"> </span>-L<span class="w"> </span>.<span class="w"> </span>-type<span class="w"> </span>l
</span></code></pre></div>
<p>Find outside links</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>find<span class="w"> </span>.<span class="w"> </span>-type<span class="w"> </span>l<span class="w"> </span>-exec<span class="w"> </span>bash<span class="w"> </span>-c<span class="w"> </span><span class="s1">'realpath --relative-base "`pwd`" "$0" 2&gt;/dev/null |egrep "^[./]" |sed "s|^|$0 -&gt;|" '</span><span class="w"> </span><span class="s1">'{}'</span><span class="w"> </span><span class="s1">';'</span>
</span></code></pre></div>
<p>Delete certain files (use with caution)</p>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="c1"># Empty directories</span>
</span><span id="__span-8-2"><a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a>find<span class="w"> </span>.<span class="w"> </span>-type<span class="w"> </span>d<span class="w"> </span>-empty<span class="w"> </span>-delete
</span><span id="__span-8-3"><a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a><span class="c1"># Backup files</span>
</span><span id="__span-8-4"><a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a>find<span class="w"> </span>.<span class="w"> </span>-name<span class="w"> </span><span class="s1">'*~'</span><span class="w"> </span>-delete
</span><span id="__span-8-5"><a id="__codelineno-8-5" name="__codelineno-8-5" href="#__codelineno-8-5"></a>find<span class="w"> </span>.<span class="w"> </span>-name<span class="w"> </span><span class="s1">'*#autosave#'</span><span class="w"> </span>-delete
</span></code></pre></div>
<h4 id="troubleshooting-known-bugs">Troubleshooting &amp; Known Bugs<a class="headerlink" href="#troubleshooting-known-bugs" title="Permanent link"></a></h4>
<ul>
<li>The following message can be safely ignored:</li>
</ul>
<p></p><div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a>key_cert_check_authority:<span class="w"> </span>invalid<span class="w"> </span>certificate
</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a>Certificate<span class="w"> </span>invalid:<span class="w"> </span>name<span class="w"> </span>is<span class="w"> </span>not<span class="w"> </span>a<span class="w"> </span>listed<span class="w"> </span>principal
</span></code></pre></div>
It indicates that no kerberos token was provided for authentication. You can avoid the warning by first running kinit (PSI linux systems).<p></p>
<ul>
<li>For decentral ingestion cases, the copy step is indicated by a message <code>Running [/usr/bin/rsync -e ssh -avxz ...</code>. It is expected that this
step will take a long time and may appear to have hung. You can check what files have been successfully transfered using rsync:</li>
</ul>
<div class="language-bash highlight"><span class="filename">Bash</span><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a>rsync<span class="w"> </span>--list-only<span class="w"> </span>user_n@pb-archive.psi.ch:archive/UID/PATH/
</span></code></pre></div>
<p>where UID is the dataset ID (12345678-1234-1234-1234-123456789012) and PATH is the absolute path to your data. Note that rsync creates directories first and that the transfer order is not alphabetical in some cases, but it should be possible to see whether any data has transferred.</p>
<ul>
<li>
<p>There is currently a limit on the number of files per dataset (technically, the limit is from the total length of all file paths). It is recommended to break up datasets into 300'000 files or less.</p>
<ul>
<li>If it is not possible or desirable to split data between multiple datasets, an alternate work-around is to package files into a tarball. For datasets which are already compressed, omit the -z option for a considerable speedup:</li>
</ul>
<div class="language-text highlight"><span class="filename">Text Only</span><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a>tar -f [output].tar [srcdir]
</span></code></pre></div>
<p>Uncompressed data can be compressed on the cluster using the following command:</p>
<div class="language-text highlight"><span class="filename">Text Only</span><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a>sbatch /data/software/Slurm/Utilities/Parallel_TarGz.batch -s [srcdir] -t [output].tar -n
</span></code></pre></div>
<p>Run /data/software/Slurm/Utilities/Parallel_TarGz.batch -h for more details and options.</p>
</li>
</ul>
<h4 id="sample-ingestion-output-datasetingestor-1111">Sample ingestion output (datasetIngestor 1.1.11)<a class="headerlink" href="#sample-ingestion-output-datasetingestor-1111" title="Permanent link"></a></h4>
<details>
<summary>[Show Example]: Sample ingestion output (datasetIngestor 1.1.11)</summary>
<pre class="terminal code highlight js-syntax-highlight plaintext" lang="plaintext" markdown="false">/data/project/bio/myproject/archive $ datasetIngestor -copy -autoarchive -allowexistingsource -ingest metadata.json
2019/11/06 11:04:43 Latest version: 1.1.11
2019/11/06 11:04:43 Your version of this program is up-to-date
2019/11/06 11:04:43 You are about to add a dataset to the === production === data catalog environment...
2019/11/06 11:04:43 Your username:
user_n
2019/11/06 11:04:48 Your password:
2019/11/06 11:04:52 User authenticated: XXX
2019/11/06 11:04:52 User is member in following a or p groups: XXX
2019/11/06 11:04:52 OwnerGroup information a-XXX verified successfully.
2019/11/06 11:04:52 contactEmail field added: XXX
2019/11/06 11:04:52 Scanning files in dataset /data/project/bio/myproject/archive
2019/11/06 11:04:52 No explicit filelistingPath defined - full folder /data/project/bio/myproject/archive is used.
2019/11/06 11:04:52 Source Folder: /data/project/bio/myproject/archive at /data/project/bio/myproject/archive
2019/11/06 11:04:57 The dataset contains 100000 files with a total size of 50000000000 bytes.
2019/11/06 11:04:57 creationTime field added: 2019-07-29 18:47:08 +0200 CEST
2019/11/06 11:04:57 endTime field added: 2019-11-06 10:52:17.256033 +0100 CET
2019/11/06 11:04:57 license field added: CC BY-SA 4.0
2019/11/06 11:04:57 isPublished field added: false
2019/11/06 11:04:57 classification field added: IN=medium,AV=low,CO=low
2019/11/06 11:04:57 Updated metadata object:
{
"accessGroups": [
"XXX"
],
"classification": "IN=medium,AV=low,CO=low",
"contactEmail": "XXX",
"creationLocation": "XXX",
"creationTime": "2019-07-29T18:47:08+02:00",
"dataFormat": "XXX",
"description": "XXX",
"endTime": "2019-11-06T10:52:17.256033+01:00",
"isPublished": false,
"license": "CC BY-SA 4.0",
"owner": "XXX",
"ownerEmail": "XXX",
"ownerGroup": "a-XXX",
"principalInvestigator": "XXX",
"scientificMetadata": {
...
},
"sourceFolder": "/data/project/bio/myproject/archive",
"type": "raw"
}
2019/11/06 11:04:57 Running [/usr/bin/ssh -l user_n pb-archive.psi.ch test -d /data/project/bio/myproject/archive].
key_cert_check_authority: invalid certificate
Certificate invalid: name is not a listed principal
user_n@pb-archive.psi.ch's password:
2019/11/06 11:05:04 The source folder /data/project/bio/myproject/archive is not centrally available (decentral use case).
The data must first be copied to a rsync cache server.
2019/11/06 11:05:04 Do you want to continue (Y/n)?
Y
2019/11/06 11:05:09 Created dataset with id 12.345.67890/12345678-1234-1234-1234-123456789012
2019/11/06 11:05:09 The dataset contains 108057 files.
2019/11/06 11:05:10 Created file block 0 from file 0 to 1000 with total size of 413229990 bytes
2019/11/06 11:05:10 Created file block 1 from file 1000 to 2000 with total size of 416024000 bytes
2019/11/06 11:05:10 Created file block 2 from file 2000 to 3000 with total size of 416024000 bytes
2019/11/06 11:05:10 Created file block 3 from file 3000 to 4000 with total size of 416024000 bytes
...
2019/11/06 11:05:26 Created file block 105 from file 105000 to 106000 with total size of 416024000 bytes
2019/11/06 11:05:27 Created file block 106 from file 106000 to 107000 with total size of 416024000 bytes
2019/11/06 11:05:27 Created file block 107 from file 107000 to 108000 with total size of 850195143 bytes
2019/11/06 11:05:27 Created file block 108 from file 108000 to 108057 with total size of 151904903 bytes
2019/11/06 11:05:27 short dataset id: 0a9fe316-c9e7-4cc5-8856-e1346dd31e31
2019/11/06 11:05:27 Running [/usr/bin/rsync -e ssh -avxz /data/project/bio/myproject/archive/ user_n@pb-archive.psi.ch:archive
/0a9fe316-c9e7-4cc5-8856-e1346dd31e31/data/project/bio/myproject/archive].
key_cert_check_authority: invalid certificate
Certificate invalid: name is not a listed principal
user_n@pb-archive.psi.ch's password:
Permission denied, please try again.
user_n@pb-archive.psi.ch's password:
/usr/libexec/test_acl.sh: line 30: /tmp/tmpacl.txt: Permission denied
/usr/libexec/test_acl.sh: line 30: /tmp/tmpacl.txt: Permission denied
/usr/libexec/test_acl.sh: line 30: /tmp/tmpacl.txt: Permission denied
/usr/libexec/test_acl.sh: line 30: /tmp/tmpacl.txt: Permission denied
/usr/libexec/test_acl.sh: line 30: /tmp/tmpacl.txt: Permission denied
...
2019/11/06 12:05:08 Successfully updated {"pid":"12.345.67890/12345678-1234-1234-1234-123456789012",...}
2019/11/06 12:05:08 Submitting Archive Job for the ingested datasets.
2019/11/06 12:05:08 Job response Status: okay
2019/11/06 12:05:08 A confirmation email will be sent to XXX
12.345.67890/12345678-1234-1234-1234-123456789012
</pre>
</details>
<h3 id="publishing">Publishing<a class="headerlink" href="#publishing" title="Permanent link"></a></h3>
<p>After datasets are are ingested they can be assigned a public DOI. This can be included in publications and will make the datasets on <a href="http://doi.psi.ch">http://doi.psi.ch</a>.</p>
<p>For instructions on this, please read the <a href="https://scicatproject.github.io/documentation/Ingestor/ingestManual.html#sec-8">'Publish' section in the ingest manual</a>.</p>
<h3 id="retrieving-data">Retrieving data<a class="headerlink" href="#retrieving-data" title="Permanent link"></a></h3>
<p>Retrieving data from the archive is also initiated through the Data Catalog. Please read the <a href="https://scicatproject.github.io/documentation/Ingestor/ingestManual.html#sec-6">'Retrieve' section in the ingest manual</a>.</p>
<h2 id="further-information">Further Information<a class="headerlink" href="#further-information" title="Permanent link"></a></h2>
<ul>
<li><a href="https://discovery.psi.ch">PSI Data Catalog</a></li>
<li><a href="https://scicatproject.github.io/documentation/Ingestor/ingestManual.html">Full Documentation</a></li>
<li><a href="https://doi.psi.ch">Published Datasets (doi.psi.ch)</a></li>
<li>Data Catalog <a href="https://www.psi.ch/photon-science-data-services/data-catalog-and-archive">PSI page</a></li>
<li>Data catalog <a href="https://scicatproject.github.io/">SciCat Software</a></li>
<li><a href="https://www.nature.com/articles/sdata201618">FAIR</a> definition and <a href="http://www.snf.ch/en/theSNSF/research-policies/open_research_data/Pages/default.aspx#FAIR%20Data%20Principles%20for%20Research%20Data%20Management">SNF Research Policy</a></li>
<li><a href="https://www.cscs.ch/fileadmin/user_upload/contents_publications/annual_reports/AR2017_Online.pdf">Petabyte Archive at CSCS</a></li>
</ul>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
<div class="md-copyright__highlight">
Copyright © 2025 HPC and Emerging Technologies Group/CSD @ Paul Scherrer Institut
</div>
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../../..", "features": ["navigation.instant", "navigation.tabs", "navigation.tabs.sticky", "navigation.sections", "navigation.indexes", "toc.follow", "content.action.edit"], "search": "../../../assets/javascripts/workers/search.7a47a382.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../../assets/javascripts/bundle.e71a0d61.min.js"></script>
<script id="init-glightbox">const lightbox = GLightbox({"touchNavigation": true, "loop": false, "zoomable": true, "draggable": true, "openEffect": "zoom", "closeEffect": "zoom", "slideEffect": "slide"});
document$.subscribe(()=>{ lightbox.reload(); });
</script></body></html>
Reference in New Issue View Git Blame Copy Permalink