lemke_h
8ad76f9b82
bpy-default.sh and bpy-dev.sh were saved 'pixi shell-hook' output from another host, with PIXI_EXE and PATH pointing into a personal home that doesn't exist on the consoles. bpy-env.sh now runs the hook at runtime (--as-is, ~0.2 s) for its shared-env fallback instead of sourcing bpy-default.sh; nothing else used either file. Scripts that call pixi now prefer the system /usr/bin/pixi and fall back to /sf/bernina/bin/pixi (e.g. on ra, which has no system pixi).
610 lines
26 KiB
Bash
Executable File
610 lines
26 KiB
Bash
Executable File
#!/bin/bash
|
|
# Install a copy of the shared bernina pixi python environment into an
|
|
# experiment's pgroup work folder (or any folder), and make it visible as
|
|
# a kernel in JupyterHub.
|
|
#
|
|
# Copies SOURCE_DIR's pixi.toml/pixi.lock straight into the target folder
|
|
# (no python/ subfolder) and runs `pixi install --frozen` there, i.e.
|
|
# exactly what is pinned in the lock -- nothing gets re-solved. Once the
|
|
# pgroup's copy has local changes (packages added with -a/-A or `pixi
|
|
# add`), re-runs keep them; only -u resets to the shared files.
|
|
# Only one environment is installed: the one SOURCE_DIR's bpy-default
|
|
# link points to, unless -e picks another.
|
|
#
|
|
# Target folder:
|
|
# (no argument) the current folder
|
|
# pNNNNN /sf/bernina/data/pNNNNN/work
|
|
# <exp name> the pgroup behind /sf/bernina/exp/<exp name> (same
|
|
# lookup as name2pgroups; exact match wins, otherwise the
|
|
# first substring match), then its work folder
|
|
# <path> any folder given as a path (must contain a "/")
|
|
# pgroup work folders live on /das/work, which is only mounted on ra --
|
|
# run this there.
|
|
#
|
|
# Package cache: pixi hardlinks packages from its cache into the env when
|
|
# both are on the same filesystem, so by default the cache goes into
|
|
# <target>/pixi_cache (also keeps multi-GB downloads out of $HOME). An
|
|
# already set PIXI_CACHE_DIR or RATTLER_CACHE_DIR pointing at an existing,
|
|
# writable folder is respected instead.
|
|
#
|
|
# JupyterHub (nb_conda_kernels) lists every env found in ~/.condarc's
|
|
# envs_dirs as "Python [conda env:<name>]". <name> is the env folder's
|
|
# basename -- unless its parent folder is literally called "envs", in
|
|
# which case the grandparent's name is prepended (that's where the hub's
|
|
# "bm-bernina310" comes from, and why .pixi/envs/bpy312 would show up as
|
|
# ".pixi-bpy312"). So the env gets a symlink
|
|
# <target>/conda_envs/bernina-<exp>-<env> -> ../.pixi/envs/<env>
|
|
# (plus bernina-<exp>-default for the default env). conda_envs --
|
|
# deliberately NOT named "envs" -- is what each user who wants the kernels
|
|
# adds to envs_dirs in their own ~/.condarc, once (-J; the install prints
|
|
# that command ready to pass on).
|
|
# Symlinks are not resolved for naming (cf. bpy-default in the hub).
|
|
# On ra, ~ is /das/home/$USER, which is the .condarc the hub reads.
|
|
set -euo pipefail
|
|
|
|
SOURCE_DIR="/sf/bernina/applications/python"
|
|
PIXI_BIN="/usr/bin/pixi" # console (system) pixi first
|
|
[ -x "$PIXI_BIN" ] || PIXI_BIN="/sf/bernina/bin/pixi" # fallback, e.g. on ra (RHEL8)
|
|
EXP_DIR="/sf/bernina/exp"
|
|
DATA_DIR="/sf/bernina/data"
|
|
LINK_DIR_NAME="conda_envs" # must not be "envs", see above
|
|
CACHE_DIR_NAME="pixi_cache"
|
|
SELF_PATH="/sf/bernina/bin/create-python-install" # full path: bin is not on everyone's PATH on ra
|
|
|
|
ENV_NAME=""
|
|
PREFIX=""
|
|
JOIN_ONLY=0
|
|
ASSUME_YES=0
|
|
DRY_RUN=0
|
|
RESET=0
|
|
PERMS_ONLY=0
|
|
ADD_CONDA=()
|
|
ADD_PYPI=()
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") [-e env] [-a pkg] [-A pkg] [-u] [-p prefix] [-y] [-n] [pgroup | exp-name | path]
|
|
$(basename "$0") -J [pgroup | exp-name | path] show its kernels in your JupyterHub
|
|
$(basename "$0") -P [pgroup | exp-name | path] fix permissions of your files
|
|
$(basename "$0") -m short manual
|
|
$(basename "$0") -h this help
|
|
|
|
Install a copy of the shared bernina python environment
|
|
($SOURCE_DIR) into an experiment's pgroup work folder,
|
|
and make it show up as a kernel in JupyterHub.
|
|
|
|
Run this on ra: pgroup work folders (/das/work/...) are only mounted
|
|
there, and JupyterHub reads ra's ~/.condarc.
|
|
|
|
WHERE IT INSTALLS
|
|
(no argument) the current folder
|
|
pNNNNN /sf/bernina/data/pNNNNN/work
|
|
exp-name the pgroup behind /sf/bernina/exp/<exp-name> (as in
|
|
name2pgroups). A part of the name is enough, e.g.
|
|
"blanco" for 26i_blanco; an exact name wins, otherwise
|
|
the first match alphabetically is used (others listed).
|
|
path any folder, given as a path (must contain a "/")
|
|
|
|
On the first run, pixi.toml and pixi.lock are copied directly into that
|
|
folder (no subfolder). 'pixi install --frozen' then installs exactly
|
|
what the lock pins -- nothing is re-solved or updated. The env ends up
|
|
in <folder>/.pixi/envs/<env>.
|
|
|
|
Re-runs never throw away the pgroup's own changes: if the folder's
|
|
pixi.toml/pixi.lock differ from the shared ones (packages added with
|
|
-a/-A or 'pixi add'), they are kept and installed as they are. Only -u
|
|
resets them to the shared files (the old ones are kept as
|
|
pixi.toml.<date> / pixi.lock.<date>).
|
|
|
|
WHICH ENVIRONMENT
|
|
One env only. Default: the bernina default, i.e. what
|
|
$SOURCE_DIR/.pixi/envs/bpy-default points to
|
|
(currently: ${DEFAULT_ENV_HINT:-?}). Pick another with -e, e.g. -e bpy313.
|
|
Available: ${ENVS_HINT:-see [environments] in pixi.toml}
|
|
|
|
PACKAGE CACHE
|
|
If PIXI_CACHE_DIR or RATTLER_CACHE_DIR is set to an existing, writable
|
|
folder, that cache is used. Otherwise: <folder>/$CACHE_DIR_NAME
|
|
(same filesystem as the env, so packages are hardlinked, not copied, and
|
|
the multi-GB download stays out of \$HOME). That one is shared by the
|
|
whole pgroup. For plain pixi commands later, point pixi at it yourself:
|
|
export PIXI_CACHE_DIR=<folder>/$CACHE_DIR_NAME
|
|
|
|
SHARING WITHIN THE PGROUP
|
|
.pixi/, $LINK_DIR_NAME/ and $CACHE_DIR_NAME/ are made group-writable (setgid
|
|
folders + a default ACL, group rw files), so every pgroup member can use
|
|
the cache, update the env or install into it. Only the owner of a file can
|
|
change its permissions: each member who runs the script fixes their own.
|
|
Use umask 002 for plain pixi commands (and don't run two installs into
|
|
the same env at once). -P fixes up your own files without installing
|
|
anything, e.g. after plain pixi commands.
|
|
|
|
ADDING PACKAGES
|
|
-a pkg (conda-forge) / -A pkg (PyPI), repeatable, version specs allowed:
|
|
$(basename "$0") -a scikit-image -A "somepkg>=1.2" 26i_blanco
|
|
This does 'pixi add' into the pgroup's pixi.toml (recorded in its
|
|
pixi.lock, so it survives re-installs), installs, and fixes permissions
|
|
-- with umask 002 and the pgroup cache set. The package goes into the
|
|
part of pixi.toml that every env shares, so all envs' locks are
|
|
re-solved (versions only change where they must); only the selected env
|
|
is installed. A quick 'pip install' in the env also works, but isn't
|
|
recorded and can disappear on the next install.
|
|
|
|
JUPYTERHUB
|
|
The hub lists every conda env it finds in the folders named under
|
|
envs_dirs in ~/.condarc, as "Python [conda env:<folder name>]". So the
|
|
script creates named symlinks
|
|
<folder>/$LINK_DIR_NAME/bernina-<exp>-<env> -> ../.pixi/envs/<env>
|
|
<folder>/$LINK_DIR_NAME/bernina-<exp>-default (only for the default env)
|
|
E.g. for 26i_blanco the hub shows:
|
|
Python [conda env:bernina-26i-blanco-bpy312]
|
|
Python [conda env:bernina-26i-blanco-default]
|
|
The name is only the symlink's name: rename/add a link in
|
|
$LINK_DIR_NAME/ to rename it, no reinstall needed. (The folder must not
|
|
be called "envs" -- the hub would then prepend the parent folder's name.)
|
|
To see them, everyone (you too) adds <folder>/$LINK_DIR_NAME to envs_dirs
|
|
in their own ~/.condarc once -- on ra, since that's the ~/.condarc the
|
|
hub reads:
|
|
$(basename "$0") -J 26i_blanco
|
|
Nothing else is installed or changed by -J. Every install prints this
|
|
command for its pgroup, ready to pass on to collaborators.
|
|
|
|
OPTIONS
|
|
-e env install this environment instead of the default
|
|
-p prefix jupyter name prefix instead of bernina-<exp name>
|
|
(underscores in exp names become dashes)
|
|
-a pkg add a conda-forge package (repeatable), see ADDING PACKAGES
|
|
-A pkg add a PyPI package (repeatable)
|
|
-u reset the folder's pixi.toml/pixi.lock to the shared ones
|
|
(removes locally added packages; backups are kept)
|
|
-P only make your own files in the install group-writable
|
|
-J only add the folder's kernels to your ~/.condarc (JupyterHub)
|
|
-y don't ask for confirmation
|
|
-n dry run: show what would be done, change nothing
|
|
-m short manual (markdown, rendered with glow if available)
|
|
-h this help
|
|
|
|
EXAMPLES
|
|
$(basename "$0") -n 26i_blanco # see what would happen
|
|
$(basename "$0") 26i_blanco # bpy312 into p23620's work folder
|
|
$(basename "$0") -e bpy313 p23620 # a different env
|
|
$(basename "$0") -a scikit-image 26i_blanco # add a package
|
|
$(basename "$0") -u 26i_blanco # back to the shared env
|
|
cd /sf/bernina/data/p23620/work && $(basename "$0")
|
|
|
|
AFTERWARDS
|
|
Activate in a shell:
|
|
eval "\$($PIXI_BIN shell-hook -e <env> --manifest-path <folder>/pixi.toml)"
|
|
Re-running it is always safe (joining colleagues do exactly that); -u
|
|
brings the copy back in line with the shared pixi.lock.
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
die() { echo "error: $*" >&2; exit 1; }
|
|
run() { if [ "$DRY_RUN" -eq 1 ]; then echo " [dry-run] $*"; else "$@"; fi; }
|
|
bad_usage() { echo "error: $*" >&2; echo "see: $(basename "$0") -h" >&2; exit 1; }
|
|
|
|
# for the help text / manual
|
|
DEFAULT_ENV_HINT=$(basename "$(readlink "$SOURCE_DIR/.pixi/envs/bpy-default" 2>/dev/null)" 2>/dev/null || true)
|
|
ENVS_HINT=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); printf "%s ", $0}' "$SOURCE_DIR/pixi.toml" 2>/dev/null || true)
|
|
|
|
# brief manual as markdown; rendered with glow on a terminal (-m)
|
|
manual() {
|
|
local md
|
|
md=$(cat <<'EOF'
|
|
# create-python-install
|
|
|
|
**The bernina python environment in your pgroup, and in JupyterHub.**
|
|
|
|
## What it does
|
|
|
|
1. Copies the shared `pixi.toml` + `pixi.lock` from
|
|
`/sf/bernina/applications/python` into the pgroup's **work** folder.
|
|
2. Installs **one** environment there, exactly as pinned in the shared lock
|
|
(default: **@DEFAULT_ENV@**, the bernina default; others with `-e`).
|
|
3. Makes it show up in **JupyterHub** as `bernina-<exp>-<env>`.
|
|
|
|
## Quick start (on ra)
|
|
|
|
```bash
|
|
create-python-install -n 26i_blanco # preview, changes nothing
|
|
create-python-install 26i_blanco # install
|
|
create-python-install -J 26i_blanco # kernels into your JupyterHub
|
|
```
|
|
|
|
Then reload JupyterHub and pick the kernel
|
|
**Python [conda env:bernina-26i-blanco-@DEFAULT_ENV@]**.
|
|
|
|
## Where it installs
|
|
|
|
| you give | installs into |
|
|
|-------------------|----------------------------------------------|
|
|
| nothing | the current folder |
|
|
| `p23620` | `/sf/bernina/data/p23620/work` |
|
|
| `26i_blanco` | that experiment's pgroup work folder |
|
|
| `blanco` | same (part of the exp name is enough) |
|
|
| `/some/path` | that folder |
|
|
|
|
## What ends up in the work folder
|
|
|
|
```
|
|
pixi.toml, pixi.lock copies of the shared ones
|
|
.pixi/envs/<env>/ the environment itself
|
|
conda_envs/ named links JupyterHub picks up
|
|
bernina-26i-blanco-bpy312 -> ../.pixi/envs/bpy312
|
|
bernina-26i-blanco-default -> ../.pixi/envs/bpy312
|
|
pixi_cache/ downloaded packages (unless you set your own)
|
|
```
|
|
|
|
Nothing in anyone's home. Whoever wants the kernels (you too) runs `-J` once:
|
|
it adds one line under `envs_dirs` in their `~/.condarc`, pointing at `conda_envs/`.
|
|
|
|
## Good to know
|
|
|
|
- **Run it on ra**: `/das/work` (the pgroup work folders) is only mounted there,
|
|
and JupyterHub reads ra's `~/.condarc`.
|
|
- **Kernel name** = the link's name in `conda_envs/`. Rename the link to
|
|
rename the kernel; no reinstall needed.
|
|
- **Cache**: your `PIXI_CACHE_DIR` is used if set, otherwise `pixi_cache/`
|
|
in the work folder (keeps GBs out of your home), shared by the pgroup.
|
|
- **Colleagues**: env, links and cache are group-writable, so any pgroup
|
|
member can update or install into it. To see the kernels in *their*
|
|
JupyterHub they run `create-python-install -J 26i_blanco` once (every
|
|
install prints this command for its pgroup).
|
|
- **Add a package**: `-a scikit-image` (conda-forge) or `-A somepkg` (PyPI).
|
|
Recorded in the pgroup's own `pixi.toml`/`pixi.lock`, so re-runs keep it.
|
|
- **Back to the shared env** (drops added packages, keeps backups): `-u`.
|
|
- **Only fix permissions** of your files (e.g. after plain `pixi` commands): `-P`.
|
|
- **Remove**: delete `.pixi/ conda_envs/ pixi_cache/ pixi.toml pixi.lock` and the
|
|
`conda_envs` line in `~/.condarc`.
|
|
|
|
Options and details: `create-python-install -h`
|
|
EOF
|
|
)
|
|
md="${md//@DEFAULT_ENV@/${DEFAULT_ENV_HINT:-bpy312}}"
|
|
local glow
|
|
glow=$(command -v glow || true)
|
|
[ -n "$glow" ] || { [ -x /sf/daq/bin/glow ] && glow=/sf/daq/bin/glow; }
|
|
if [ -t 1 ] && [ -n "$glow" ]; then
|
|
printf '%s\n' "$md" | "$glow" -
|
|
else
|
|
printf '%s\n' "$md"
|
|
fi
|
|
exit 0
|
|
}
|
|
|
|
while getopts "e:p:a:A:uPJynmh" opt; do
|
|
case "$opt" in
|
|
e) ENV_NAME="$OPTARG" ;;
|
|
p) PREFIX="$OPTARG" ;;
|
|
a) ADD_CONDA+=("$OPTARG") ;;
|
|
A) ADD_PYPI+=("$OPTARG") ;;
|
|
u) RESET=1 ;;
|
|
P) PERMS_ONLY=1 ;;
|
|
J) JOIN_ONLY=1 ;;
|
|
y) ASSUME_YES=1 ;;
|
|
n) DRY_RUN=1 ;;
|
|
m) manual ;;
|
|
h) usage ;;
|
|
*) bad_usage "unknown option" ;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
[ $# -le 1 ] || bad_usage "only one target (pgroup, exp name or path) allowed"
|
|
ARG="${1:-}"
|
|
|
|
command -v "$PIXI_BIN" >/dev/null 2>&1 || PIXI_BIN="pixi"
|
|
command -v "$PIXI_BIN" >/dev/null 2>&1 || die "pixi not found"
|
|
[ -r "$SOURCE_DIR/pixi.toml" ] && [ -r "$SOURCE_DIR/pixi.lock" ] \
|
|
|| die "can't read pixi.toml/pixi.lock in $SOURCE_DIR"
|
|
|
|
# exp name -> exp link (same matching as name2pgroups: substring of the
|
|
# link name; an exact name wins, otherwise the first one alphabetically)
|
|
exp_link_for_name() {
|
|
local name="$1" l matches=()
|
|
if [ -L "$EXP_DIR/$name" ]; then echo "$EXP_DIR/$name"; return; fi
|
|
for l in "$EXP_DIR"/*"$name"*; do
|
|
[ -L "$l" ] && matches+=("$l")
|
|
done
|
|
[ ${#matches[@]} -gt 0 ] || return 1
|
|
if [ ${#matches[@]} -gt 1 ]; then
|
|
echo "note: '$name' matches several experiments, using the first:" >&2
|
|
for l in "${matches[@]}"; do
|
|
echo " $(basename "$l") -> $(basename "$(readlink -f "$l")")" >&2
|
|
done
|
|
fi
|
|
echo "${matches[0]}"
|
|
}
|
|
|
|
# pgroup -> exp name (like pgroup2name; first alphabetically if several)
|
|
exp_name_for_pgroup() {
|
|
local pgroup="$1" l
|
|
for l in "$EXP_DIR"/*; do
|
|
[ -L "$l" ] && [ "$(basename "$(readlink -f "$l")")" = "$pgroup" ] \
|
|
&& { basename "$l"; return; }
|
|
done
|
|
return 1
|
|
}
|
|
|
|
EXP_NAME=""
|
|
PGROUP=""
|
|
if [ -z "$ARG" ]; then
|
|
TARGET="$PWD"
|
|
elif [[ "$ARG" =~ ^p[0-9]{5}$ ]]; then
|
|
PGROUP="$ARG"
|
|
TARGET="$DATA_DIR/$PGROUP/work"
|
|
elif [[ "$ARG" == */* ]]; then
|
|
TARGET="$ARG"
|
|
else
|
|
link=$(exp_link_for_name "$ARG") || die "no experiment matching '$ARG' in $EXP_DIR"
|
|
EXP_NAME=$(basename "$link")
|
|
PGROUP=$(basename "$(readlink -f "$link")")
|
|
TARGET="$DATA_DIR/$PGROUP/work"
|
|
fi
|
|
|
|
# resolve work -> /das/work/pNN/pNNNNN: env paths get baked into the
|
|
# installed files, so use the real location, not the symlink
|
|
TARGET_REAL=$(readlink -f "$TARGET" 2>/dev/null || true)
|
|
if [ -z "$TARGET_REAL" ] || [ ! -d "$TARGET_REAL" ]; then
|
|
die "target folder $TARGET doesn't exist${TARGET_REAL:+ (-> $TARGET_REAL)}; pgroup work folders (/das/work) are only mounted on ra"
|
|
fi
|
|
TARGET="$TARGET_REAL"
|
|
[ "$TARGET" != "$(readlink -f "$SOURCE_DIR")" ] || die "target is the shared source project itself"
|
|
[ -w "$TARGET" ] || die "no write permission in $TARGET"
|
|
|
|
if [ -z "$PGROUP" ] && [[ "$TARGET" =~ /(p[0-9]{5})(/|$) ]]; then
|
|
PGROUP="${BASH_REMATCH[1]}"
|
|
fi
|
|
if [ -z "$EXP_NAME" ] && [ -n "$PGROUP" ]; then
|
|
EXP_NAME=$(exp_name_for_pgroup "$PGROUP" || true)
|
|
fi
|
|
if [ -z "$PREFIX" ]; then
|
|
label="${EXP_NAME:-${PGROUP:-$(basename "$TARGET")}}"
|
|
PREFIX="bernina-${label//_/-}"
|
|
fi
|
|
|
|
# Which pixi.toml/pixi.lock to install from:
|
|
# fresh - none in the folder yet: copy the shared ones
|
|
# shared - identical to the shared ones: nothing to copy
|
|
# local - they differ (packages added in this pgroup): keep them
|
|
# reset - they differ, but -u: back up and copy the shared ones
|
|
if [ ! -e "$TARGET/pixi.toml" ] && [ ! -e "$TARGET/pixi.lock" ]; then
|
|
SETUP=fresh
|
|
elif cmp -s "$SOURCE_DIR/pixi.toml" "$TARGET/pixi.toml" \
|
|
&& cmp -s "$SOURCE_DIR/pixi.lock" "$TARGET/pixi.lock"; then
|
|
SETUP=shared
|
|
elif [ "$RESET" -eq 1 ]; then
|
|
SETUP=reset
|
|
else
|
|
SETUP=local
|
|
[ -f "$TARGET/pixi.toml" ] && [ -f "$TARGET/pixi.lock" ] \
|
|
|| die "$TARGET has only one of pixi.toml/pixi.lock; -u resets both to the shared ones"
|
|
fi
|
|
MANIFEST_SRC="$SOURCE_DIR/pixi.toml"
|
|
[ "$SETUP" = local ] && MANIFEST_SRC="$TARGET/pixi.toml"
|
|
|
|
DEFAULT_ENV=$(basename "$(readlink "$SOURCE_DIR/.pixi/envs/bpy-default" 2>/dev/null)" 2>/dev/null || true)
|
|
[ -n "$DEFAULT_ENV" ] || die "can't resolve $SOURCE_DIR/.pixi/envs/bpy-default"
|
|
ENV_NAME="${ENV_NAME:-$DEFAULT_ENV}"
|
|
[ "$ENV_NAME" != "default" ] \
|
|
|| die "pixi's own 'default' env isn't meant for this; the bernina default is $DEFAULT_ENV (used when -e is omitted)"
|
|
defined_envs=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); print}' "$MANIFEST_SRC")
|
|
grep -qxF -- "$ENV_NAME" <<<"$defined_envs" \
|
|
|| die "no environment '$ENV_NAME' in $MANIFEST_SRC (have: $(echo $defined_envs))"
|
|
|
|
# package cache: keep a sensible user setting, otherwise the target folder
|
|
cache_ok() { [ -n "$1" ] && [ -d "$1" ] && [ -w "$1" ]; }
|
|
if cache_ok "${PIXI_CACHE_DIR:-}"; then
|
|
CACHE_NOTE="(your PIXI_CACHE_DIR)"
|
|
elif cache_ok "${RATTLER_CACHE_DIR:-}"; then
|
|
export PIXI_CACHE_DIR="$RATTLER_CACHE_DIR"
|
|
CACHE_NOTE="(your RATTLER_CACHE_DIR)"
|
|
else
|
|
export PIXI_CACHE_DIR="$TARGET/$CACHE_DIR_NAME"
|
|
CACHE_NOTE=""
|
|
fi
|
|
|
|
LINK_DIR="$TARGET/$LINK_DIR_NAME"
|
|
# only the pgroup's own cache is shared; a personal PIXI_CACHE_DIR is left alone
|
|
SHARED=("$TARGET/.pixi" "$LINK_DIR")
|
|
[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || SHARED+=("$PIXI_CACHE_DIR")
|
|
LINKS=("$PREFIX-$ENV_NAME")
|
|
[ "$ENV_NAME" = "$DEFAULT_ENV" ] && LINKS+=("$PREFIX-default")
|
|
|
|
if [ "$PERMS_ONLY" -eq 0 ] && [ "$JOIN_ONLY" -eq 0 ]; then
|
|
cat <<EOF
|
|
Source: $SOURCE_DIR
|
|
Target: $TARGET${EXP_NAME:+ ($EXP_NAME${PGROUP:+, $PGROUP})}
|
|
Environment: $ENV_NAME$([ "$ENV_NAME" = "$DEFAULT_ENV" ] && echo " (bernina default)")
|
|
Pixi cache: $PIXI_CACHE_DIR $CACHE_NOTE
|
|
pixi.toml: $(case "$SETUP" in
|
|
fresh) echo "copy of the shared one (first install)" ;;
|
|
shared) echo "same as the shared one" ;;
|
|
local) echo "the pgroup's own (differs from the shared one), kept -- -u resets" ;;
|
|
reset) echo "RESET to the shared one: locally added packages are removed (backups kept)" ;;
|
|
esac)
|
|
EOF
|
|
[ ${#ADD_CONDA[@]} -eq 0 ] || echo "Add (conda): ${ADD_CONDA[*]}"
|
|
[ ${#ADD_PYPI[@]} -eq 0 ] || echo "Add (PyPI): ${ADD_PYPI[*]}"
|
|
cat <<EOF
|
|
Jupyter: $(for l in "${LINKS[@]}"; do printf '"Python [conda env:%s]" ' "$l"; done)
|
|
via $LINK_DIR (each user: -J once)
|
|
EOF
|
|
|
|
if [ "$DRY_RUN" -eq 0 ] && [ "$ASSUME_YES" -eq 0 ] && [ -t 0 ]; then
|
|
read -r -p "Proceed? [y/N] " answer
|
|
[[ "$answer" =~ ^[Yy] ]] || { echo "aborted"; exit 1; }
|
|
fi
|
|
fi # PERMS_ONLY / JOIN_ONLY
|
|
|
|
# group members of the pgroup should be able to use and update this too
|
|
umask 002
|
|
|
|
# Share with the pgroup: setgid + group rwx on folders, group rw on files
|
|
# (rw, not just r: other members hardlink cache files into their installs
|
|
# and take lock files in the cache, and the kernel only allows hardlinking
|
|
# someone else's file with read+write access to it). A default ACL on the
|
|
# folders keeps folders created later group-writable too, even by a plain
|
|
# `pixi` run with umask 022. chmod/setfacl only work on what this user
|
|
# owns, so every member who runs the script fixes their own files.
|
|
HAVE_SETFACL=0
|
|
command -v setfacl >/dev/null 2>&1 && HAVE_SETFACL=1
|
|
share_dir() { # just this folder, not recursive
|
|
[ -d "$1" ] && [ -O "$1" ] || return 0
|
|
chmod g+rwxs "$1" 2>/dev/null || true
|
|
[ "$HAVE_SETFACL" -eq 0 ] || setfacl -m d:g::rwx,d:m::rwx "$1" 2>/dev/null || true
|
|
}
|
|
share_tree() { # everything below, owned by this user
|
|
local d="$1"
|
|
[ -d "$d" ] || return 0
|
|
find "$d" -user "$USER" -type d ! -perm -g+rwxs -exec chmod g+rwxs {} + 2>/dev/null || true
|
|
find "$d" -user "$USER" -type f ! -perm -g+rw -exec chmod g+rw {} + 2>/dev/null || true
|
|
[ "$HAVE_SETFACL" -eq 0 ] \
|
|
|| find "$d" -user "$USER" -type d -exec setfacl -m d:g::rwx,d:m::rwx {} + 2>/dev/null || true
|
|
}
|
|
|
|
# append (not prepend: the first envs_dirs entry is where `conda create -n`
|
|
# puts new envs) LINK_DIR to ~/.condarc; written with a redirect rather
|
|
# than a rename, so a ~/.condarc symlink stays a symlink
|
|
add_to_condarc() {
|
|
local rc="$HOME/.condarc" dir="$1" new
|
|
if [ -f "$rc" ]; then
|
|
if awk -v d="$dir" '{sub(/[[:space:]]+$/,"")} $0 ~ /^[[:space:]]*-[[:space:]]/ {sub(/^[[:space:]]*-[[:space:]]*/,""); if ($0==d || $0==d"/") f=1} END{exit !f}' "$rc"; then
|
|
echo "==> $dir already in $rc"
|
|
return
|
|
fi
|
|
grep -qE '^envs_dirs:[[:space:]]*[^[:space:]#]' "$rc" \
|
|
&& { echo "warning: envs_dirs in $rc isn't a block list, add $dir by hand" >&2; return; }
|
|
new=$(awk -v d="$dir" '
|
|
/^envs_dirs:[[:space:]]*$/ { print; inblk = 1; next }
|
|
inblk && /^[[:space:]]*-/ { print; next }
|
|
inblk { print " - " d; inblk = 0; done = 1 }
|
|
{ print }
|
|
END { if (inblk) { print " - " d; done = 1 }
|
|
if (!done) { print "envs_dirs:"; print " - " d } }' "$rc")
|
|
else
|
|
new=$(printf 'envs_dirs:\n - %s' "$dir")
|
|
fi
|
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
echo " [dry-run] $rc would become:"; sed 's/^/ /' <<<"$new"
|
|
else
|
|
printf '%s\n' "$new" > "$rc"
|
|
echo "==> added $dir to $rc"
|
|
fi
|
|
}
|
|
|
|
report_others() {
|
|
local others
|
|
others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ')
|
|
[ -z "$others" ] || echo "note: some files of $others aren't group-writable yet;" \
|
|
"they can fix that with: $(basename "$0") -P $TARGET"
|
|
}
|
|
|
|
if [ "$PERMS_ONLY" -eq 1 ]; then
|
|
[ -d "$TARGET/.pixi" ] || die "no pixi install in $TARGET"
|
|
[ "$DRY_RUN" -eq 0 ] || { echo " [dry-run] make your files group-writable in: ${SHARED[*]}"; exit 0; }
|
|
echo "==> making your files in $TARGET group-writable for $(stat -c %G "$TARGET")"
|
|
for d in "${SHARED[@]}"; do share_tree "$d"; done
|
|
report_others
|
|
exit 0
|
|
fi
|
|
|
|
# what collaborators (and you) run once to get the kernels in JupyterHub
|
|
JOIN_ARG="${EXP_NAME:-${PGROUP:-$TARGET}}"
|
|
JOIN_CMD="$SELF_PATH -J $JOIN_ARG"
|
|
|
|
if [ "$JOIN_ONLY" -eq 1 ]; then
|
|
kernels=$(ls "$LINK_DIR" 2>/dev/null || true)
|
|
[ -n "$kernels" ] || die "no kernels in $LINK_DIR yet -- install first"
|
|
case "$HOME" in
|
|
/das/home/*) ;;
|
|
*) echo "note: JupyterHub reads the ~/.condarc on ra (/das/home/$USER);" \
|
|
"this is $(hostname -s) with HOME=$HOME" >&2 ;;
|
|
esac
|
|
add_to_condarc "$LINK_DIR"
|
|
echo "JupyterHub kernel(s) (reload the hub page):"
|
|
for l in $kernels; do echo " Python [conda env:$l]"; done
|
|
exit 0
|
|
fi
|
|
|
|
# an active bpy-env.sh / pixi shell exports these and they would point pixi
|
|
# at the shared project -- always address the target explicitly instead
|
|
unset PIXI_PROJECT_MANIFEST PIXI_PROJECT_ROOT PIXI_IN_SHELL
|
|
|
|
stamp=$(date +%Y%m%d-%H%M%S)
|
|
case "$SETUP" in
|
|
fresh|reset)
|
|
for f in pixi.toml pixi.lock; do
|
|
if [ -f "$TARGET/$f" ] && ! cmp -s "$SOURCE_DIR/$f" "$TARGET/$f"; then
|
|
echo "==> keeping the pgroup's $f as $f.$stamp"
|
|
run cp -p "$TARGET/$f" "$TARGET/$f.$stamp"
|
|
fi
|
|
run cp "$SOURCE_DIR/$f" "$TARGET/$f"
|
|
done ;;
|
|
local)
|
|
echo "==> using the pgroup's own pixi.toml/pixi.lock (differ from the shared ones; -u resets)" ;;
|
|
esac
|
|
# (fails harmlessly if another member owns them -- then they're theirs to share)
|
|
[ "$DRY_RUN" -eq 1 ] || chmod 664 "$TARGET/pixi.toml" "$TARGET/pixi.lock" 2>/dev/null || true
|
|
|
|
# never let a global detached-environments setting move the envs away
|
|
# from .pixi/envs, the jupyter links below rely on that location
|
|
if [ ! -f "$TARGET/.pixi/config.toml" ]; then
|
|
run mkdir -p "$TARGET/.pixi"
|
|
if [ "$DRY_RUN" -eq 0 ]; then
|
|
echo "detached-environments = false" > "$TARGET/.pixi/config.toml"
|
|
else
|
|
echo " [dry-run] write $TARGET/.pixi/config.toml"
|
|
fi
|
|
fi
|
|
|
|
[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || run mkdir -p "$PIXI_CACHE_DIR"
|
|
run mkdir -p "$TARGET/.pixi/envs" "$LINK_DIR"
|
|
if [ "$DRY_RUN" -eq 0 ]; then
|
|
# before installing, so everything pixi creates below inherits it
|
|
for d in "${SHARED[@]}" "$TARGET/.pixi/envs"; do share_dir "$d"; done
|
|
else
|
|
echo " [dry-run] make group-writable: ${SHARED[*]}"
|
|
fi
|
|
|
|
# -a/-A: record the packages in pixi.toml/pixi.lock (re-solves the lock),
|
|
# the install below then puts them into the env
|
|
if [ ${#ADD_CONDA[@]} -gt 0 ]; then
|
|
echo "==> adding ${ADD_CONDA[*]}"
|
|
run "$PIXI_BIN" add --no-install --manifest-path "$TARGET/pixi.toml" "${ADD_CONDA[@]}"
|
|
fi
|
|
if [ ${#ADD_PYPI[@]} -gt 0 ]; then
|
|
echo "==> adding (PyPI) ${ADD_PYPI[*]}"
|
|
run "$PIXI_BIN" add --no-install --pypi --manifest-path "$TARGET/pixi.toml" "${ADD_PYPI[@]}"
|
|
fi
|
|
|
|
echo "==> installing $ENV_NAME"
|
|
run "$PIXI_BIN" install --frozen -e "$ENV_NAME" --manifest-path "$TARGET/pixi.toml"
|
|
|
|
for l in "${LINKS[@]}"; do
|
|
run ln -sfn "../.pixi/envs/$ENV_NAME" "$LINK_DIR/$l"
|
|
done
|
|
|
|
if [ "$DRY_RUN" -eq 0 ]; then
|
|
echo "==> making it group-writable for $(stat -c %G "$TARGET")"
|
|
for d in "${SHARED[@]}"; do share_tree "$d"; done
|
|
report_others
|
|
fi
|
|
|
|
[ "$DRY_RUN" -eq 0 ] || { echo; echo "Dry run, nothing changed."; exit 0; }
|
|
|
|
cat <<EOF
|
|
|
|
Done. $ENV_NAME installed in $TARGET/.pixi/envs/$ENV_NAME
|
|
|
|
Activate in a shell:
|
|
eval "\$($PIXI_BIN shell-hook -e $ENV_NAME --manifest-path $TARGET/pixi.toml)"
|
|
JupyterHub kernel(s):
|
|
$(for l in "${LINKS[@]}"; do echo " Python [conda env:$l]"; done)
|
|
|
|
To see them in JupyterHub, you and every collaborator run this once, on ra:
|
|
$JOIN_CMD
|
|
(it only adds this line under envs_dirs in ~/.condarc: - $LINK_DIR)
|
|
EOF
|