From 1cecf79865a297e8acdd7f1a99bf040ae93dd0f3 Mon Sep 17 00:00:00 2001
From: Bliven Spencer Edward <bliven_s@merlin-l-02.psi.ch>
Date: Wed, 5 Dec 2018 15:47:01 +0100
Subject: [PATCH] cryosparcm cluster example slurm

---
 cluster_info.json | 11 +++++++++++
 cluster_script.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 cluster_info.json
 create mode 100644 cluster_script.sh

diff --git a/cluster_info.json b/cluster_info.json
new file mode 100644
index 0000000..7f67c7c
--- /dev/null
+++ b/cluster_info.json
@@ -0,0 +1,11 @@
+{
+    "name" : "slurmcluster",
+    "worker_bin_path" : "/path/to/cryosparc2_worker/bin/cryosparcw",
+    "cache_path" : "/path/to/local/SSD/on/cluster/nodes"
+    "send_cmd_tpl" : "ssh loginnode {{ command }}",
+    "qsub_cmd_tpl" : "sbatch {{ script_path_abs }}",
+    "qstat_cmd_tpl" : "squeue -j {{ cluster_job_id }}",
+    "qdel_cmd_tpl" : "scancel {{ cluster_job_id }}",
+    "qinfo_cmd_tpl" : "sinfo",
+    "transfer_cmd_tpl" : "scp {{ src_path }} loginnode:{{ dest_path }}"
+}
diff --git a/cluster_script.sh b/cluster_script.sh
new file mode 100644
index 0000000..d574e74
--- /dev/null
+++ b/cluster_script.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#### cryoSPARC cluster submission script template for SLURM
+## Available variables:
+## {{ run_cmd }}            - the complete command string to run the job
+## {{ num_cpu }}            - the number of CPUs needed
+## {{ num_gpu }}            - the number of GPUs needed. 
+##                            Note: the code will use this many GPUs starting from dev id 0
+##                                  the cluster scheduler or this script have the responsibility
+##                                  of setting CUDA_VISIBLE_DEVICES so that the job code ends up
+##                                  using the correct cluster-allocated GPUs.
+## {{ ram_gb }}             - the amount of RAM needed in GB
+## {{ job_dir_abs }}        - absolute path to the job directory
+## {{ project_dir_abs }}    - absolute path to the project dir
+## {{ job_log_path_abs }}   - absolute path to the log file for the job
+## {{ worker_bin_path }}    - absolute path to the cryosparc worker command
+## {{ run_args }}           - arguments to be passed to cryosparcw run
+## {{ project_uid }}        - uid of the project
+## {{ job_uid }}            - uid of the job
+## {{ job_creator }}        - name of the user that created the job (may contain spaces)
+## {{ cryosparc_username }} - cryosparc username of the user that created the job (usually an email)
+##
+## What follows is a simple SLURM script:
+
+#SBATCH --job-name cryosparc_{{ project_uid }}_{{ job_uid }}
+#SBATCH -n {{ num_cpu }}
+#SBATCH --gres=gpu:{{ num_gpu }}
+#SBATCH -p gpu
+#SBATCH --mem={{ (ram_gb*1000)|int }}MB             
+#SBATCH -o {{ job_dir_abs }}
+#SBATCH -e {{ job_dir_abs }}
+
+available_devs=""
+for devidx in $(seq 0 15);
+do
+    if [[ -z $(nvidia-smi -i $devidx --query-compute-apps=pid --format=csv,noheader) ]] ; then
+        if [[ -z "$available_devs" ]] ; then
+            available_devs=$devidx
+        else
+            available_devs=$available_devs,$devidx
+        fi
+    fi
+done
+export CUDA_VISIBLE_DEVICES=$available_devs
+
+{{ run_cmd }}
+