diff --git a/.gitignore b/.gitignore index e05415b..95eb8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,2 @@ -_site/ -.sass-cache/ -.jekyll-metadata -_pdf -.DS_Store -.idea -vendor/ -.bundle/ -.ruby-gemset -.ruby-version +# mkdocs +.cache/ diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico index 0d505be..107a8fe 100644 Binary files a/docs/images/favicon.ico and b/docs/images/favicon.ico differ diff --git a/docs/images/NoMachine/screen_nx1.png b/docs/images/nomachine/screen_nx1.png similarity index 100% rename from docs/images/NoMachine/screen_nx1.png rename to docs/images/nomachine/screen_nx1.png diff --git a/docs/images/NoMachine/screen_nx10.png b/docs/images/nomachine/screen_nx10.png similarity index 100% rename from docs/images/NoMachine/screen_nx10.png rename to docs/images/nomachine/screen_nx10.png diff --git a/docs/images/nomachine/screen_nx11.png b/docs/images/nomachine/screen_nx11.png new file mode 100644 index 0000000..905caf3 Binary files /dev/null and b/docs/images/nomachine/screen_nx11.png differ diff --git a/docs/images/NoMachine/screen_nx2.png b/docs/images/nomachine/screen_nx2.png similarity index 100% rename from docs/images/NoMachine/screen_nx2.png rename to docs/images/nomachine/screen_nx2.png diff --git a/docs/images/NoMachine/screen_nx3.png b/docs/images/nomachine/screen_nx3.png similarity index 100% rename from docs/images/NoMachine/screen_nx3.png rename to docs/images/nomachine/screen_nx3.png diff --git a/docs/images/NoMachine/screen_nx4.png b/docs/images/nomachine/screen_nx4.png similarity index 100% rename from docs/images/NoMachine/screen_nx4.png rename to docs/images/nomachine/screen_nx4.png diff --git a/docs/images/NoMachine/screen_nx5.png b/docs/images/nomachine/screen_nx5.png similarity index 100% rename from docs/images/NoMachine/screen_nx5.png rename to docs/images/nomachine/screen_nx5.png diff --git a/docs/images/NoMachine/screen_nx6.png b/docs/images/nomachine/screen_nx6.png similarity index 100% rename from docs/images/NoMachine/screen_nx6.png rename to docs/images/nomachine/screen_nx6.png diff --git a/docs/images/NoMachine/screen_nx7.png b/docs/images/nomachine/screen_nx7.png similarity index 100% rename from docs/images/NoMachine/screen_nx7.png rename to docs/images/nomachine/screen_nx7.png diff --git a/docs/images/NoMachine/screen_nx8.png b/docs/images/nomachine/screen_nx8.png similarity index 100% rename from docs/images/NoMachine/screen_nx8.png rename to docs/images/nomachine/screen_nx8.png diff --git a/docs/images/NoMachine/screen_nx9.png b/docs/images/nomachine/screen_nx9.png similarity index 100% rename from docs/images/NoMachine/screen_nx9.png rename to docs/images/nomachine/screen_nx9.png diff --git a/docs/images/NoMachine/screen_nx_address.png b/docs/images/nomachine/screen_nx_address.png similarity index 100% rename from docs/images/NoMachine/screen_nx_address.png rename to docs/images/nomachine/screen_nx_address.png diff --git a/docs/images/NoMachine/screen_nx_auth.png b/docs/images/nomachine/screen_nx_auth.png similarity index 100% rename from docs/images/NoMachine/screen_nx_auth.png rename to docs/images/nomachine/screen_nx_auth.png diff --git a/docs/images/NoMachine/screen_nx_configuration.png b/docs/images/nomachine/screen_nx_configuration.png similarity index 100% rename from docs/images/NoMachine/screen_nx_configuration.png rename to docs/images/nomachine/screen_nx_configuration.png diff --git a/docs/images/nomachine/screen_nx_connect.png b/docs/images/nomachine/screen_nx_connect.png new file mode 100644 index 0000000..89b1875 Binary files /dev/null and b/docs/images/nomachine/screen_nx_connect.png differ diff --git a/docs/images/nomachine/screen_nx_existingsession.png b/docs/images/nomachine/screen_nx_existingsession.png new file mode 100644 index 0000000..b5ad86e Binary files /dev/null and b/docs/images/nomachine/screen_nx_existingsession.png differ diff --git a/docs/images/nomachine/screen_nx_newsession.png b/docs/images/nomachine/screen_nx_newsession.png new file mode 100644 index 0000000..cf7dce1 Binary files /dev/null and b/docs/images/nomachine/screen_nx_newsession.png differ diff --git a/docs/images/NoMachine/screen_nx_single_session.png b/docs/images/nomachine/screen_nx_single_session.png similarity index 100% rename from docs/images/NoMachine/screen_nx_single_session.png rename to docs/images/nomachine/screen_nx_single_session.png diff --git a/docs/merlin6/02-How-To-Use-Merlin/archive.md b/docs/merlin6/how-to-use-merlin/archive.md similarity index 77% rename from docs/merlin6/02-How-To-Use-Merlin/archive.md rename to docs/merlin6/how-to-use-merlin/archive.md index bf35541..c1f9728 100644 --- a/docs/merlin6/02-How-To-Use-Merlin/archive.md +++ b/docs/merlin6/how-to-use-merlin/archive.md @@ -1,12 +1,4 @@ ---- -title: Archive & PSI Data Catalog -#tags: -keywords: linux, archive, data catalog, archiving, lts, tape, long term storage, ingestion, datacatalog -last_updated: 31 January 2020 -summary: "This document describes how to use the PSI Data Catalog for archiving Merlin6 data." -sidebar: merlin6_sidebar -permalink: /merlin6/archive.html ---- +# Archive & PSI Data Catalog ## PSI Data Catalog as a PSI Central Service @@ -19,28 +11,28 @@ The Data Catalog and Archive is suitable for: * Derived data produced by processing some inputs * Data required to reproduce PSI research and publications -The Data Catalog is part of PSI's effort to conform to the FAIR principles for data management. -In accordance with this policy, ***data will be publicly released under CC-BY-SA 4.0 after an +The Data Catalog is part of PSI's effort to conform to the FAIR principles for data management. +In accordance with this policy, ***data will be publicly released under CC-BY-SA 4.0 after an embargo period expires.*** -The Merlin cluster is connected to the Data Catalog. Hence, users archive data stored in the +The Merlin cluster is connected to the Data Catalog. Hence, users archive data stored in the Merlin storage under the ``/data`` directories (currentlyi, ``/data/user`` and ``/data/project``). Archiving from other directories is also possible, however the process is much slower as data -can not be directly retrieved by the PSI archive central servers (**central mode**), and needs to +can not be directly retrieved by the PSI archive central servers (**central mode**), and needs to be indirectly copied to these (**decentral mode**). Archiving can be done from any node accessible by the users (usually from the login nodes). -{{site.data.alerts.tip}} Archiving can be done in two different ways: -
-'Central mode': Possible for the user and project data directories, is the -fastest way as it does not require remote copy (data is directly retreived by central AIT servers from Merlin -through 'merlin-archive.psi.ch'). -
-
-'Decentral mode': Possible for any directory, is the slowest way of archiving as it requires -to copy ('rsync') the data from Merlin to the central AIT servers. -{{site.data.alerts.end}} +!!! tip + + Archiving can be done in two different ways: + + **'Central mode':** Possible for the user and project data directories, is the + fastest way as it does not require remote copy (data is directly retreived by central AIT servers from Merlin + through 'merlin-archive.psi.ch'). + + **'Decentral mode':** Possible for any directory, is the slowest way of archiving as it requires + to copy ('rsync') the data from Merlin to the central AIT servers. ## Procedure @@ -49,33 +41,35 @@ to copy ('rsync') the data from Merlin to the central AIT servers. Below are the main steps for using the Data Catalog. * Ingest the dataset into the Data Catalog. This makes the data known to the Data Catalog system at PSI: - * Prepare a metadata file describing the dataset - * Run **``datasetIngestor``** script - * If necessary, the script will copy the data to the PSI archive servers - * Usually this is necessary when archiving from directories other than **``/data/user``** or - **``/data/project``**. It would be also necessary when the Merlin export server (**``merlin-archive.psi.ch``**) + * Prepare a metadata file describing the dataset + * Run **`datasetIngestor`** script + * If necessary, the script will copy the data to the PSI archive servers + * Usually this is necessary when archiving from directories other than **`/data/user`** or + **`/data/project`**. It would be also necessary when the Merlin export server (**`merlin-archive.psi.ch`**) is down for any reason. * Archive the dataset: - * Visit [https://discovery.psi.ch](https://discovery.psi.ch) - * Click **``Archive``** for the dataset - * The system will now copy the data to the PetaByte Archive at CSCS + * Visit [https://discovery.psi.ch](https://discovery.psi.ch) + * Click **`Archive`** for the dataset + * The system will now copy the data to the PetaByte Archive at CSCS * Retrieve data from the catalog: - * Find the dataset on [https://discovery.psi.ch](https://discovery.psi.ch) and click **``Retrieve``** - * Wait for the data to be copied to the PSI retrieval system - * Run **``datasetRetriever``** script + * Find the dataset on [https://discovery.psi.ch](https://discovery.psi.ch) and click **`Retrieve`** + * Wait for the data to be copied to the PSI retrieval system + * Run **`datasetRetriever`** script -Since large data sets may take a lot of time to transfer, some steps are designed to happen in the -background. The discovery website can be used to track the progress of each step. +Since large data sets may take a lot of time to transfer, some steps are +designed to happen in the background. The discovery website can be used to +track the progress of each step. -### Account Registration +### Account Registration -Two types of account permit access to the Data Catalog. If your data was collected at a ***beamline***, you may -have been assigned a **``p-group``** (e.g. ``p12345``) for the experiment. Other users are assigned **``a-group``** -(e.g. ``a-12345``). +Two types of account permit access to the Data Catalog. If your data was +collected at a ***beamline***, you may have been assigned a **`p-group`** +(e.g. `p12345`) for the experiment. Other users are assigned **`a-group`** +(e.g. `a-12345`). Groups are usually assigned to a PI, and then individual user accounts are added to the group. This must be done under user request through PSI Service Now. For existing **a-groups** and **p-groups**, you can follow the standard -central procedures. Alternatively, if you do not know how to do that, follow the Merlin6 +central procedures. Alternatively, if you do not know how to do that, follow the Merlin6 **[Requesting extra Unix groups](/merlin6/request-account.html#requesting-extra-unix-groups)** procedure, or open a **[PSI Service Now](https://psi.service-now.com/psisp)** ticket. @@ -107,19 +101,19 @@ As of 2022-04-14 a secure token is required to interact with the data catalog. T You will need to save this token for later steps. To avoid including it in all the commands, I suggest saving it to an environmental variable (Linux): -``` -$ SCICAT_TOKEN=RqYMZcqpqMJqluplbNYXLeSyJISLXfnkwlfBKuvTSdnlpKkU +```bash +SCICAT_TOKEN=RqYMZcqpqMJqluplbNYXLeSyJISLXfnkwlfBKuvTSdnlpKkU ``` (Hint: prefix this line with a space to avoid saving the token to your bash history.) Tokens expire after 2 weeks and will need to be fetched from the website again. -### Ingestion +### Ingestion -The first step to ingesting your data into the catalog is to prepare a file describing what data you have. This is called -**``metadata.json``**, and can be created with a text editor (e.g. *``vim``*). It can in principle be saved anywhere, -but keeping it with your archived data is recommended. For more information about the format, see the 'Bio metadata' +The first step to ingesting your data into the catalog is to prepare a file describing what data you have. This is called +**`metadata.json`**, and can be created with a text editor (e.g. *`vim`*). It can in principle be saved anywhere, +but keeping it with your archived data is recommended. For more information about the format, see the 'Bio metadata' section below. An example follows: ```yaml @@ -163,9 +157,9 @@ section below. An example follows: It is recommended to use the [ScicatEditor](https://bliven_s.gitpages.psi.ch/SciCatEditor/) for creating metadata files. This is a browser-based tool specifically for ingesting PSI data. Using the tool avoids syntax errors and provides templates for common data sets and options. The finished JSON file can then be downloaded to merlin or copied into a text editor. -Another option is to use the SciCat graphical interface from NoMachine. This provides a graphical interface for selecting data to archive. This is particularly useful for data associated with a DUO experiment and p-group. Type `SciCat`` to get started after loading the `datacatalog`` module. The GUI also replaces the the command-line ingestion described below. +Another option is to use the SciCat graphical interface from NoMachine. This provides a graphical interface for selecting data to archive. This is particularly useful for data associated with a DUO experiment and p-group. Type `SciCat` to get started after loading the `datacatalog` module. The GUI also replaces the the command-line ingestion described below. -The following steps can be run from wherever you saved your ``metadata.json``. First, perform a "dry-run" which will check the metadata for errors: +The following steps can be run from wherever you saved your `metadata.json`. First, perform a "dry-run" which will check the metadata for errors: ```bash datasetIngestor --token $SCICAT_TOKEN metadata.json @@ -177,30 +171,30 @@ It will ask for your PSI credentials and then print some info about the data to datasetIngestor --token $SCICAT_TOKEN --ingest --autoarchive metadata.json ``` -You will be asked whether you want to copy the data to the central system: +You will be asked whether you want to copy the data to the central system: -* If you are on the Merlin cluster and you are archiving data from ``/data/user`` or ``/data/project``, answer 'no' since the data catalog can +* If you are on the Merlin cluster and you are archiving data from `/data/user` or `/data/project`, answer 'no' since the data catalog can directly read the data. -* If you are on a directory other than ``/data/user`` and ``/data/project, or you are on a desktop computer, answer 'yes'. Copying large datasets +* If you are on a directory other than `/data/user` and `/data/project`, or you are on a desktop computer, answer 'yes'. Copying large datasets to the PSI archive system may take quite a while (minutes to hours). -If there are no errors, your data has been accepted into the data catalog! From now on, no changes should be made to the ingested data. -This is important, since the next step is for the system to copy all the data to the CSCS Petabyte archive. Writing to tape is slow, so +If there are no errors, your data has been accepted into the data catalog! From now on, no changes should be made to the ingested data. +This is important, since the next step is for the system to copy all the data to the CSCS Petabyte archive. Writing to tape is slow, so this process may take several days, and it will fail if any modifications are detected. -If using the ``--autoarchive`` option as suggested above, your dataset should now be in the queue. Check the data catalog: -[https://discovery.psi.ch](https://discovery.psi.ch). Your job should have status 'WorkInProgress'. You will receive an email when the ingestion +If using the `--autoarchive` option as suggested above, your dataset should now be in the queue. Check the data catalog: +[https://discovery.psi.ch](https://discovery.psi.ch). Your job should have status 'WorkInProgress'. You will receive an email when the ingestion is complete. -If you didn't use ``--autoarchive``, you need to manually move the dataset into the archive queue. From **discovery.psi.ch**, navigate to the 'Archive' -tab. You should see the newly ingested dataset. Check the dataset and click **``Archive``**. You should see the status change from **``datasetCreated``** to -**``scheduleArchiveJob``**. This indicates that the data is in the process of being transferred to CSCS. +If you didn't use `--autoarchive`, you need to manually move the dataset into the archive queue. From **discovery.psi.ch**, navigate to the 'Archive' +tab. You should see the newly ingested dataset. Check the dataset and click **`Archive`**. You should see the status change from **`datasetCreated`** to +**`scheduleArchiveJob`**. This indicates that the data is in the process of being transferred to CSCS. -After a few days the dataset's status will change to **``datasetOnAchive``** indicating the data is stored. At this point it is safe to delete the data. +After a few days the dataset's status will change to **`datasetOnAchive`** indicating the data is stored. At this point it is safe to delete the data. #### Useful commands -Running the datasetIngestor in dry mode (**without** ``--ingest``) finds most errors. However, it is sometimes convenient to find potential errors +Running the datasetIngestor in dry mode (**without** `--ingest`) finds most errors. However, it is sometimes convenient to find potential errors yourself with simple unix commands. Find problematic filenames @@ -239,9 +233,10 @@ find . -name '*#autosave#' -delete key_cert_check_authority: invalid certificate Certificate invalid: name is not a listed principal ``` + It indicates that no kerberos token was provided for authentication. You can avoid the warning by first running kinit (PSI linux systems). - -* For decentral ingestion cases, the copy step is indicated by a message ``Running [/usr/bin/rsync -e ssh -avxz ...``. It is expected that this + +* For decentral ingestion cases, the copy step is indicated by a message `Running [/usr/bin/rsync -e ssh -avxz ...`. It is expected that this step will take a long time and may appear to have hung. You can check what files have been successfully transfered using rsync: ```bash @@ -253,22 +248,21 @@ step will take a long time and may appear to have hung. You can check what files * There is currently a limit on the number of files per dataset (technically, the limit is from the total length of all file paths). It is recommended to break up datasets into 300'000 files or less. * If it is not possible or desirable to split data between multiple datasets, an alternate work-around is to package files into a tarball. For datasets which are already compressed, omit the -z option for a considerable speedup: - ``` + ```bash tar -f [output].tar [srcdir] ``` Uncompressed data can be compressed on the cluster using the following command: - ``` + ```bash sbatch /data/software/Slurm/Utilities/Parallel_TarGz.batch -s [srcdir] -t [output].tar -n ``` Run /data/software/Slurm/Utilities/Parallel_TarGz.batch -h for more details and options. #### Sample ingestion output (datasetIngestor 1.1.11) -
-[Show Example]: Sample ingestion output (datasetIngestor 1.1.11) -
+
+```text
 /data/project/bio/myproject/archive $ datasetIngestor -copy -autoarchive -allowexistingsource -ingest metadata.json
 2019/11/06 11:04:43 Latest version: 1.1.11
 
@@ -355,12 +349,11 @@ user_n@pb-archive.psi.ch's password:
 2019/11/06 12:05:08 Job response Status: okay
 2019/11/06 12:05:08 A confirmation email will be sent to XXX
 12.345.67890/12345678-1234-1234-1234-123456789012
-
-
+``` ### Publishing -After datasets are are ingested they can be assigned a public DOI. This can be included in publications and will make the datasets on http://doi.psi.ch. +After datasets are are ingested they can be assigned a public DOI. This can be included in publications and will make the datasets on . For instructions on this, please read the ['Publish' section in the ingest manual](https://scicatproject.github.io/documentation/Ingestor/ingestManual.html#sec-8). diff --git a/docs/merlin6/02-How-To-Use-Merlin/connect-from-linux.md b/docs/merlin6/how-to-use-merlin/connect-from-linux.md similarity index 50% rename from docs/merlin6/02-How-To-Use-Merlin/connect-from-linux.md rename to docs/merlin6/how-to-use-merlin/connect-from-linux.md index ccbe949..a7713a2 100644 --- a/docs/merlin6/02-How-To-Use-Merlin/connect-from-linux.md +++ b/docs/merlin6/how-to-use-merlin/connect-from-linux.md @@ -1,16 +1,8 @@ ---- -title: Connecting from a Linux Client -#tags: -keywords: linux, connecting, client, configuration, SSH, X11 -last_updated: 07 September 2022 -summary: "This document describes a recommended setup for a Linux client." -sidebar: merlin6_sidebar -permalink: /merlin6/connect-from-linux.html ---- +# Connecting from a Linux Client ## SSH without X11 Forwarding -This is the standard method. Official X11 support is provided through [NoMachine](/merlin6/nomachine.html). +This is the standard method. Official X11 support is provided through [NoMachine](nomachine.md). For normal SSH sessions, use your SSH client as follows: ```bash @@ -22,19 +14,19 @@ ssh $username@merlin-l-002.psi.ch ## SSH with X11 Forwarding Official X11 Forwarding support is through NoMachine. Please follow the document -[{Job Submission -> Interactive Jobs}](/merlin6/interactive-jobs.html#Requirements) and -[{Accessing Merlin -> NoMachine}](/merlin6/nomachine.html) for more details. However, +[{Job Submission -> Interactive Jobs}](##/merlin6/interactive-jobs.html#Requirements) and +[{Accessing Merlin -> NoMachine}](nomachine.md) for more details. However, we provide a small recipe for enabling X11 Forwarding in Linux. -* For enabling client X11 forwarding, add the following to the start of ``~/.ssh/config`` -to implicitly add ``-X`` to all ssh connections: +* For enabling client X11 forwarding, add the following to the start of `~/.ssh/config` +to implicitly add `-X` to all ssh connections: ```bash ForwardAgent yes ForwardX11Trusted yes ``` -* Alternatively, you can add the option ``-Y`` to the ``ssh`` command. In example: +* Alternatively, you can add the option `-Y` to the `ssh` command. In example: ```bash ssh -X $username@merlin-l-01.psi.ch @@ -42,7 +34,7 @@ to implicitly add ``-X`` to all ssh connections: ssh -X $username@merlin-l-002.psi.ch ``` -* For testing that X11 forwarding works, just run ``xclock``. A X11 based clock should +* For testing that X11 forwarding works, just run `xclock`. A X11 based clock should popup in your client session: ```bash diff --git a/docs/merlin6/02-How-To-Use-Merlin/connect-from-macos.md b/docs/merlin6/how-to-use-merlin/connect-from-macos.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/connect-from-macos.md rename to docs/merlin6/how-to-use-merlin/connect-from-macos.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/connect-from-windows.md b/docs/merlin6/how-to-use-merlin/connect-from-windows.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/connect-from-windows.md rename to docs/merlin6/how-to-use-merlin/connect-from-windows.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/kerberos.md b/docs/merlin6/how-to-use-merlin/kerberos.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/kerberos.md rename to docs/merlin6/how-to-use-merlin/kerberos.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/merlin-rmount.md b/docs/merlin6/how-to-use-merlin/merlin-rmount.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/merlin-rmount.md rename to docs/merlin6/how-to-use-merlin/merlin-rmount.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/nomachine.md b/docs/merlin6/how-to-use-merlin/nomachine.md similarity index 76% rename from docs/merlin6/02-How-To-Use-Merlin/nomachine.md rename to docs/merlin6/how-to-use-merlin/nomachine.md index bec0d4f..1958ea6 100644 --- a/docs/merlin6/02-How-To-Use-Merlin/nomachine.md +++ b/docs/merlin6/how-to-use-merlin/nomachine.md @@ -1,22 +1,14 @@ ---- -title: Remote Desktop Access -#tags: -keywords: NX, nomachine, remote desktop access, login node, merlin-l-001, merlin-l-002, merlin-nx-01, merlin-nx-02, merlin-nx, rem-acc, vpn -last_updated: 07 September 2022 -#summary: "" -sidebar: merlin6_sidebar -permalink: /merlin6/nomachine.html ---- +# Remote Desktop Access -Users can login in Merlin through a Linux Remote Desktop Session. NoMachine -is a desktop virtualization tool. It is similar to VNC, Remote Desktop, etc. +Users can login in Merlin through a Linux Remote Desktop Session. NoMachine +is a desktop virtualization tool. It is similar to VNC, Remote Desktop, etc. It uses the NX protocol to enable a graphical login to remote servers. ## Installation NoMachine is available for PSI Windows computers in the Software Kiosk under the name **NX Client**. Please use the latest version (at least 6.0). For MacOS and -Linux, the NoMachine client can be downloaded from https://www.nomachine.com/. +Linux, the NoMachine client can be downloaded from . ## Accessing Merlin6 NoMachine from PSI @@ -24,21 +16,21 @@ The Merlin6 NoMachine service is hosted in the following machine: * **`merlin-nx.psi.ch`** -This is the **front-end** (hence, *the door*) to the NoMachine **back-end nodes**, +This is the **front-end** (hence, *the door*) to the NoMachine **back-end nodes**, which contain the NoMachine desktop service. The **back-end nodes** are the following: * `merlin-l-001.psi.ch` * `merlin-l-002.psi.ch` -Any access to the login node desktops must be done through **`merlin-nx.psi.ch`** -(or from **`rem-acc.psi.ch -> merlin-nx.psi.ch`** when connecting from outside PSI). +Any access to the login node desktops must be done through **`merlin-nx.psi.ch`** +(or from **`rem-acc.psi.ch -> merlin-nx.psi.ch`** when connecting from outside PSI). -The **front-end** service running on **`merlin-nx.psi.ch`** will load balance the sessions +The **front-end** service running on **`merlin-nx.psi.ch`** will load balance the sessions and login to any of the available nodes in the **back-end**. **Only 1 session per back-end** is possible. -Below are explained all the steps necessary for configuring the access to the +Below are explained all the steps necessary for configuring the access to the NoMachine service running on a login node. ### Creating a Merlin6 NoMachine connection @@ -47,16 +39,17 @@ NoMachine service running on a login node. Click the **Add** button to create a new connection to the **`merlin-nx.psi.ch` front-end**, and fill up the following fields: + * **Name**: Specify a custom name for the connection. Examples: `merlin-nx`, `merlin-nx.psi.ch`, `Merlin Desktop` * **Host**: Specify the hostname of the **front-end** service: **`merlin-nx.psi.ch`** * **Protocol**: specify the protocol that will be used for the connection. *Recommended* protocol: **`NX`** * **Port**: Specify the listening port of the **front-end**. It must be **`4000`**. -![Create New NoMachine Connection]({{ "/images/NoMachine/screen_nx_connect.png" }}) +![Create New NoMachine Connection](/images/nomachine/screen_nx_connect.png) #### Configuring NoMachine Authentication Method -Depending on the client version, it may ask for different authentication options. +Depending on the client version, it may ask for different authentication options. If it's required, choose your authentication method and **Continue** (**Password** or *Kerberos* are the recommended ones). You will be requested for the crendentials (username / password). **Do not add `PSICH\`** as a prefix for the username. @@ -64,24 +57,24 @@ You will be requested for the crendentials (username / password). **Do not add ` ### Opening NoMachine desktop sessions By default, when connecting to the **`merlin-nx.psi.ch` front-end** it will automatically open a new -session if none exists. +session if none exists. -If there are existing sessions, instead of opening a new desktop session, users can reconnect to an +If there are existing sessions, instead of opening a new desktop session, users can reconnect to an existing one by clicking to the proper icon (see image below). -![Open an existing Session]({{ "/images/NoMachine/screen_nx_existingsession.png" }}) +![Open an existing Session](/images/nomachine/screen_nx_existingsession.png) -Users can also create a second desktop session by selecting the **`New Desktop`** button (*red* rectangle in the +Users can also create a second desktop session by selecting the **`New Desktop`** button (*red* rectangle in the below image). This will create a second session on the second login node, as long as this node is up and running. -![Open a New Desktop]({{ "/images/NoMachine/screen_nx_newsession.png" }}) +![Open a New Desktop](/images/nomachine/screen_nx_newsession.png) ### NoMachine LightDM Session Example An example of the NoMachine session, which is based on [LightDM](https://github.com/canonical/lightdm) X Windows: -![NoMachine Session: LightDM Desktop]({{ "/images/NoMachine/screen_nx11.png" }}) +![NoMachine Session: LightDM Desktop](/images/nomachine/screen_nx11.png) ## Accessing Merlin6 NoMachine from outside PSI @@ -95,28 +88,20 @@ services. ### VPN access Remote access is also possible through VPN, however, you **must not use 'rem-acc.psi.ch'**, and you have to connect directly -to the Merlin6 NoMachine **`merlin-nx.psi.ch` front-end** as if you were inside PSI. For VPN access, you should request -it to the IT department by opening a PSI Service Now ticket: +to the Merlin6 NoMachine **`merlin-nx.psi.ch` front-end** as if you were inside PSI. For VPN access, you should request +it to the IT department by opening a PSI Service Now ticket: [VPN Access (PSI employees)](https://psi.service-now.com/psisp?id=psi_new_sc_cat_item&sys_id=beccc01b6f44a200d02a82eeae3ee440). - ## Advanced Display Settings -**Nomachine Display Settings** can be accessed and changed either when creating a new session or by clicking the very top right corner of a running session. +**Nomachine Display Settings** can be accessed and changed either when creating a new session or by clicking the very top right corner of a running session. ### Prevent Rescaling These settings prevent "bluriness" at the cost of some performance! (You might want to choose depending on performance) * Display > Resize remote display (forces 1:1 pixel sizes) -* Display > Change settings > Quality: Choose Medium-Best Quality +* Display > Change settings > Quality: Choose Medium-Best Quality * Display > Change settings > Modify advanced settings - * Check: Disable network-adaptive display quality (diables lossy compression) - * Check: Disable client side image post-processing - - - - - - - + * Check: Disable network-adaptive display quality (diables lossy compression) + * Check: Disable client side image post-processing diff --git a/docs/merlin6/02-How-To-Use-Merlin/ssh-keys.md b/docs/merlin6/how-to-use-merlin/ssh-keys.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/ssh-keys.md rename to docs/merlin6/how-to-use-merlin/ssh-keys.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/storage.md b/docs/merlin6/how-to-use-merlin/storage.md similarity index 62% rename from docs/merlin6/02-How-To-Use-Merlin/storage.md rename to docs/merlin6/how-to-use-merlin/storage.md index 36ac7b6..891538a 100644 --- a/docs/merlin6/02-How-To-Use-Merlin/storage.md +++ b/docs/merlin6/how-to-use-merlin/storage.md @@ -1,13 +1,4 @@ ---- -title: Merlin6 Storage -#tags: -keywords: storage, /data/user, /data/software, /data/project, /scratch, /shared-scratch, quota, export, user, project, scratch, data, shared-scratch, merlin_quotas -last_updated: 07 September 2022 -#summary: "" -sidebar: merlin6_sidebar -redirect_from: /merlin6/data-directories.html -permalink: /merlin6/storage.html ---- +# Merlin6 Storage ## Introduction @@ -16,12 +7,13 @@ This document describes the different directories of the Merlin6 cluster. ### User and project data * ***Users are responsible for backing up their own data***. Is recommended to backup the data on third party independent systems (i.e. LTS, Archive, AFS, SwitchDrive, Windows Shares, etc.). - * **`/psi/home`**, as this contains a small amount of data, is the only directory where we can provide daily snapshots for one week. This can be found in the following directory **`/psi/home/.snapshot/`** -* ***When a user leaves PSI, she or her supervisor/team are responsible to backup and move the data out from the cluster***: every few months, the storage space will be recycled for those old users who do not have an existing and valid PSI account. + * **`/psi/home`**, as this contains a small amount of data, is the only directory where we can provide daily snapshots for one week. This can be found in the following directory **`/psi/home/.snapshot/`** +* ***When a user leaves PSI, she or her supervisor/team are responsible to backup and move the data out from the cluster***: every few months, the storage space will be recycled for those old users who do not have an existing and valid PSI account. -{{site.data.alerts.warning}}When a user leaves PSI and his account has been removed, her storage space in Merlin may be recycled. -Hence, when a user leaves PSI, she, her supervisor or team must ensure that the data is backed up to an external storage -{{site.data.alerts.end}} +!!! warning + + When a user leaves PSI and his account has been removed, her storage space in Merlin may be recycled. + Hence, **when a user leaves PSI**, she, her supervisor or team **must ensure that the data is backed up to an external storage** ### Checking user quota @@ -41,17 +33,21 @@ Merlin6 offers the following directory classes for users: * ``/psi/home/``: Private user **home** directory * ``/data/user/``: Private user **data** directory * ``/data/project/general/``: Shared **Project** directory - * For BIO experiments, a dedicated ``/data/project/bio/$projectname`` exists. + * For BIO experiments, a dedicated ``/data/project/bio/$projectname`` exists. * ``/scratch``: Local *scratch* disk (only visible by the node running a job). * ``/shared-scratch``: Shared *scratch* disk (visible from all nodes). * ``/export``: Export directory for data transfer, visible from `ra-merlin-01.psi.ch`, `ra-merlin-02.psi.ch` and Merlin login nodes. - * Refer to **[Transferring Data](/merlin6/transfer-data.html)** for more information about the export area and data transfer service. + * Refer to **[Transferring Data](../how-to-use-merlin/transfer-data.md)** for more information about the export area and data transfer service. -{{site.data.alerts.tip}}In GPFS there is a concept called GraceTime. Filesystems have a block (amount of data) and file (number of files) quota. -This quota contains a soft and hard limits. Once the soft limit is reached, users can keep writing up to their hard limit quota during the grace period. -Once GraceTime or hard limit are reached, users will be unable to write and will need remove data below the soft limit (or ask for a quota increase -when this is possible, see below table). -{{site.data.alerts.end}} +!!! tip + + In GPFS there is a concept called **GraceTime**. Filesystems have a block + (amount of data) and file (number of files) quota. This quota contains a soft + and hard limits. Once the soft limit is reached, users can keep writing up to + their hard limit quota during the **grace period**. Once **GraceTime** or hard + limit are reached, users will be unable to write and will need remove data + below the soft limit (or ask for a quota increase when this is possible, see + below table). Properties of the directory classes: @@ -65,9 +61,13 @@ Properties of the directory classes: | /shared-scratch | USR [512GB:2TB] | USR [2M:2.5M] | 7d | Up to x2 when strongly justified. | Changeable when justified. | no | N/A | | /export | USR [10MB:20TB] | USR [512K:5M] | 10d | Soft can be temporary increased. | Changeable when justified. | no | N/A | -{{site.data.alerts.warning}}The use of scratch and export areas as an extension of the quota is forbidden. scratch and export areas must not contain final data. -
Auto cleanup policies in the scratch and export areas are applied. -{{site.data.alerts.end}} +!!! warning + + The use of **scratch** and **export** areas as an extension of the quota + _is forbidden_. **scratch** and **export** areas _must not contain_ final + data. + + **_Auto cleanup policies_** in the **scratch** and **export** areas are applied. ### User home directory @@ -92,15 +92,15 @@ quota -s #### Home directory policy -* Read **[Important: Code of Conduct](## Important: Code of Conduct)** for more information about Merlin6 policies. +* Read **[Important: Code of Conduct](../quick-start-guide/code-of-conduct.md)** for more information about Merlin6 policies. * Is **forbidden** to use the home directories for IO intensive tasks - * Use ``/scratch``, ``/shared-scratch``, ``/data/user`` or ``/data/project`` for this purpose. + * Use `/scratch`, `/shared-scratch`, `/data/user` or `/data/project` for this purpose. * Users can retrieve up to 1 week of their lost data thanks to the automatic **daily snapshots for 1 week**. Snapshots can be accessed at this path: - ```bash - /psi/home/.snapshop/$username - ``` +```bash +/psi/home/.snapshop/$username +``` ### User data directory @@ -119,9 +119,9 @@ mmlsquota -u --block-size auto merlin-user #### User data directory policy -* Read **[Important: Code of Conduct](## Important: Code of Conduct)** for more information about Merlin6 policies. +* Read **[Important: Code of Conduct](../quick-start-guide/code-of-conduct.md)** for more information about Merlin6 policies. * Is **forbidden** to use the data directories as ``scratch`` area during a job runtime. - * Use ``/scratch``, ``/shared-scratch`` for this purpose. + * Use ``/scratch``, ``/shared-scratch`` for this purpose. * No backup policy is applied for user data directories: users are responsible for backing up their data. ### Project data directory @@ -131,7 +131,7 @@ shared by all members of the project (the project's corresponding unix group). W project related storage spaces, since it allows users to coordinate. Also, project spaces have more flexible policies regarding extending the available storage space. -Experiments can request a project space as described in **[[Accessing Merlin -> Requesting a Project]](/merlin6/request-project.html)** +Experiments can request a project space as described in **[[Accessing Merlin -> Requesting a Project]](../quick-start-guide/requesting-projects.md)** Once created, the project data directory will be mounted in the login and computing nodes under the dirctory: @@ -147,13 +147,13 @@ mmlsquota -j $projectname --block-size auto -C merlin.psi.ch merlin-proj #### Project Directory policy -* Read **[Important: Code of Conduct](## Important: Code of Conduct)** for more information about Merlin6 policies. -* It is **forbidden** to use the data directories as ``scratch`` area during a job's runtime, i.e. for high throughput I/O for a job's temporary files. Please Use ``/scratch``, ``/shared-scratch`` for this purpose. +* Read **[Important: Code of Conduct](../quick-start-guide/code-of-conduct.md)** for more information about Merlin6 policies. +* It is **forbidden** to use the data directories as `scratch` area during a job's runtime, i.e. for high throughput I/O for a job's temporary files. Please Use `/scratch`, `/shared-scratch` for this purpose. * No backups: users are responsible for managing the backups of their data directories. ### Scratch directories -There are two different types of scratch storage: **local** (``/scratch``) and **shared** (``/shared-scratch``). +There are two different types of scratch storage: **local** (`/scratch`) and **shared** (`/shared-scratch`). **local** scratch should be used for all jobs that do not require the scratch files to be accessible from multiple nodes, which is trivially true for all jobs running on a single node. @@ -162,36 +162,34 @@ and all tasks need to do I/O on the same temporary files. **local** scratch in Merlin6 computing nodes provides a huge number of IOPS thanks to the NVMe technology. **Shared** scratch is implemented using a distributed parallel filesystem (GPFS) resulting in a higher latency, since it involves remote storage resources and more complex I/O coordination. -``/shared-scratch`` is only mounted in the *Merlin6* computing nodes (i.e. not on the login nodes), and its current size is 50TB. This can be increased in the future. +`/shared-scratch` is only mounted in the *Merlin6* computing nodes (i.e. not on the login nodes), and its current size is 50TB. This can be increased in the future. The properties of the available scratch storage spaces are given in the following table -| Cluster | Service | Scratch | Scratch Mountpoint | Shared Scratch | Shared Scratch Mountpoint | Comments | -| ------- | -------------- | ------------ | ------------------ | -------------- | ------------------------- | -------------------------------------- | -| merlin5 | computing node | 50GB / SAS | ``/scratch`` | ``N/A`` | ``N/A`` | ``merlin-c-[01-64]`` | -| merlin6 | login node | 100GB / SAS | ``/scratch`` | 50TB / GPFS | ``/shared-scratch`` | ``merlin-l-0[1,2]`` | -| merlin6 | computing node | 1.3TB / NVMe | ``/scratch`` | 50TB / GPFS | ``/shared-scratch`` | ``merlin-c-[001-024,101-124,201-224]`` | -| merlin6 | login node | 2.0TB / NVMe | ``/scratch`` | 50TB / GPFS | ``/shared-scratch`` | ``merlin-l-00[1,2]`` | +| Cluster | Service | Scratch | Scratch Mountpoint | Shared Scratch | Shared Scratch Mountpoint | Comments | +| ------- | -------------- | ------------ | ------------------ | -------------- | ------------------------- | ------------------------------------ | +| merlin5 | computing node | 50GB / SAS | `/scratch` | `N/A` | `N/A` | `merlin-c-[01-64]` | +| merlin6 | login node | 100GB / SAS | `/scratch` | 50TB / GPFS | `/shared-scratch` | `merlin-l-0[1,2]` | +| merlin6 | computing node | 1.3TB / NVMe | `/scratch` | 50TB / GPFS | `/shared-scratch` | `merlin-c-[001-024,101-124,201-224]` | +| merlin6 | login node | 2.0TB / NVMe | `/scratch` | 50TB / GPFS | `/shared-scratch` | `merlin-l-00[1,2]` | #### Scratch directories policy -* Read **[Important: Code of Conduct](## Important: Code of Conduct)** for more information about Merlin6 policies. +* Read **[Important: Code of Conduct](../quick-start-guide/code-of-conduct.md)** for more information about Merlin6 policies. * By default, *always* use **local** first and only use **shared** if your specific use case requires it. * Temporary files *must be deleted at the end of the job by the user*. - * Remaining files will be deleted by the system if detected. - * Files not accessed within 28 days will be automatically cleaned up by the system. - * If for some reason the scratch areas get full, admins have the rights to cleanup the oldest data. + * Remaining files will be deleted by the system if detected. + * Files not accessed within 28 days will be automatically cleaned up by the system. + * If for some reason the scratch areas get full, admins have the rights to cleanup the oldest data. ### Export directory Export directory is exclusively intended for transferring data from outside PSI to Merlin and viceversa. Is a temporary directoy with an auto-cleanup policy. -Please read **[Transferring Data](/merlin6/transfer-data.html)** for more information about it. +Please read **[Transferring Data](../how-to-use-merlin/transfer-data.md)** for more information about it. #### Export directory policy * Temporary files *must be deleted at the end of the job by the user*. - * Remaining files will be deleted by the system if detected. - * Files not accessed within 28 days will be automatically cleaned up by the system. - * If for some reason the export area gets full, admins have the rights to cleanup the oldest data - ---- + * Remaining files will be deleted by the system if detected. + * Files not accessed within 28 days will be automatically cleaned up by the system. + * If for some reason the export area gets full, admins have the rights to cleanup the oldest data diff --git a/docs/merlin6/02-How-To-Use-Merlin/transfer-data.md b/docs/merlin6/how-to-use-merlin/transfer-data.md similarity index 100% rename from docs/merlin6/02-How-To-Use-Merlin/transfer-data.md rename to docs/merlin6/how-to-use-merlin/transfer-data.md diff --git a/docs/merlin6/02-How-To-Use-Merlin/using-modules.md b/docs/merlin6/how-to-use-merlin/using-modules.md similarity index 99% rename from docs/merlin6/02-How-To-Use-Merlin/using-modules.md rename to docs/merlin6/how-to-use-merlin/using-modules.md index 8ee1dc8..9005fa1 100644 --- a/docs/merlin6/02-How-To-Use-Merlin/using-modules.md +++ b/docs/merlin6/how-to-use-merlin/using-modules.md @@ -1,5 +1,4 @@ --- -title: Using PModules #tags: keywords: Pmodules, software, stable, unstable, deprecated, overlay, overlays, release stage, module, package, packages, library, libraries last_updated: 07 September 2022 @@ -8,6 +7,8 @@ sidebar: merlin6_sidebar permalink: /merlin6/using-modules.html --- +# Using PModules + ## Environment Modules On top of the operating system stack we provide different software using the PSI developed PModule system. diff --git a/favicon-16x16.png b/favicon-16x16.png deleted file mode 100644 index 31c82aa..0000000 Binary files a/favicon-16x16.png and /dev/null differ diff --git a/favicon-32x32.png b/favicon-32x32.png deleted file mode 100644 index d77d1b9..0000000 Binary files a/favicon-32x32.png and /dev/null differ diff --git a/favicon.ico b/favicon.ico deleted file mode 100644 index 107a8fe..0000000 Binary files a/favicon.ico and /dev/null differ diff --git a/mkdocs.yml b/mkdocs.yml index c9d9c25..cc9ffd5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,6 +85,17 @@ nav: - merlin6/quick-start-guide/requesting-projects.md - merlin6/quick-start-guide/accessing-interactive-nodes.md - merlin6/quick-start-guide/accessing-slurm.md + - How To Use Merlin: + - merlin6/how-to-use-merlin/connect-from-linux.md + - merlin6/how-to-use-merlin/connect-from-windows.md + - merlin6/how-to-use-merlin/connect-from-macos.md + - merlin6/how-to-use-merlin/storage.md + - merlin6/how-to-use-merlin/transfer-data.md + - merlin6/how-to-use-merlin/archive.md + - merlin6/how-to-use-merlin/nomachine.md + - merlin6/how-to-use-merlin/ssh-keys.md + - merlin6/how-to-use-merlin/kerberos.md + - merlin6/how-to-use-merlin/using-modules.md - PSI@CSCS: - cscs-userlab/index.md - cscs-userlab/transfer-data.md