From 8efa32dc1c3d96144ad34aa433c9469e870318b0 Mon Sep 17 00:00:00 2001 From: lemke_h Date: Fri, 25 Sep 2026 16:16:24 +0200 Subject: [PATCH] create-python-install: renamed from pixi_pgroup_install; -J for JupyterHub - Renamed to create-python-install. - No longer edits ~/.condarc during the install. Each install ends by printing the command for you and collaborators to run once on ra: create-python-install -J , which only adds the folder's conda_envs to envs_dirs in ~/.condarc and lists the kernels. Replaces -C. --- cheatsheet.md | 2 +- pixi_pgroup_install => create-python-install | 143 +++++++++++-------- list_bin_commands | 2 +- 3 files changed, 84 insertions(+), 63 deletions(-) rename pixi_pgroup_install => create-python-install (90%) diff --git a/cheatsheet.md b/cheatsheet.md index 5b0dd0e..2a817f6 100644 --- a/cheatsheet.md +++ b/cheatsheet.md @@ -207,12 +207,12 @@ Every script in `/sf/bernina/bin`, grouped by topic. Generated by `list_bin_comm | `bpy-dev.sh` | Set PATH for the bpy312-dev pixi environment (source this) | | `bpy-env.sh` | Activate the Bernina pixi Python env -- local per-host mirror if in sync, else shared (source this) | | `bpy-select` | Interactively pick/switch the active bpy pixi environment | +| `create-python-install` | Install the bernina pixi env into a pgroup work folder and show it in JupyterHub | | `micromamba` | micromamba package manager | | `mm_bernina.sh` | Micromamba shell-init hook for bernina (source this) | | `mm_bernina_env` | Activate a named micromamba environment | | `pixi` | pixi package manager | | `pixi_local_sync` | Keep the local per-host pixi env mirror in sync (runs in background at login) | -| `pixi_pgroup_install` | Install the bernina pixi env into a pgroup work folder and show it in JupyterHub | | `pixi_rebuild_local` | Rebuild the bernina pixi env in an isolated local sandbox, for testing | ### Jupyter & notebooks diff --git a/pixi_pgroup_install b/create-python-install similarity index 90% rename from pixi_pgroup_install rename to create-python-install index 045e68c..90f5fe1 100755 --- a/pixi_pgroup_install +++ b/create-python-install @@ -34,8 +34,10 @@ # "bm-bernina310" comes from, and why .pixi/envs/bpy312 would show up as # ".pixi-bpy312"). So the env gets a symlink # /conda_envs/bernina-- -> ../.pixi/envs/ -# (plus bernina--default for the default env), and conda_envs -- -# deliberately NOT named "envs" -- is appended to ~/.condarc's envs_dirs. +# (plus bernina--default for the default env). conda_envs -- +# deliberately NOT named "envs" -- is what each user who wants the kernels +# adds to envs_dirs in their own ~/.condarc, once (-J; the install prints +# that command ready to pass on). # Symlinks are not resolved for naming (cf. bpy-default in the hub). # On ra, ~ is /das/home/$USER, which is the .condarc the hub reads. set -euo pipefail @@ -46,10 +48,11 @@ EXP_DIR="/sf/bernina/exp" DATA_DIR="/sf/bernina/data" LINK_DIR_NAME="conda_envs" # must not be "envs", see above CACHE_DIR_NAME="pixi_cache" +SELF_PATH="/sf/bernina/bin/create-python-install" # full path: bin is not on everyone's PATH on ra ENV_NAME="" PREFIX="" -UPDATE_CONDARC=1 +JOIN_ONLY=0 ASSUME_YES=0 DRY_RUN=0 RESET=0 @@ -59,8 +62,9 @@ ADD_PYPI=() usage() { cat </$LINK_DIR_NAME/bernina-- -> ../.pixi/envs/ /$LINK_DIR_NAME/bernina--default (only for the default env) - and appends /$LINK_DIR_NAME to envs_dirs in ~/.condarc. E.g. for 26i_blanco the hub shows: Python [conda env:bernina-26i-blanco-bpy312] Python [conda env:bernina-26i-blanco-default] The name is only the symlink's name: rename/add a link in $LINK_DIR_NAME/ to rename it, no reinstall needed. (The folder must not be called "envs" -- the hub would then prepend the parent folder's name.) - Other users see the kernels once they run this script themselves (or add - the $LINK_DIR_NAME line to their own ~/.condarc). + To see them, everyone (you too) adds /$LINK_DIR_NAME to envs_dirs + in their own ~/.condarc once -- on ra, since that's the ~/.condarc the + hub reads: + $(basename "$0") -J 26i_blanco + Nothing else is installed or changed by -J. Every install prints this + command for its pgroup, ready to pass on to collaborators. OPTIONS -e env install this environment instead of the default @@ -150,7 +157,7 @@ OPTIONS -u reset the folder's pixi.toml/pixi.lock to the shared ones (removes locally added packages; backups are kept) -P only make your own files in the install group-writable - -C don't touch ~/.condarc, just print the line to add + -J only add the folder's kernels to your ~/.condarc (JupyterHub) -y don't ask for confirmation -n dry run: show what would be done, change nothing -m short manual (markdown, rendered with glow if available) @@ -185,7 +192,7 @@ ENVS_HINT=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:] manual() { local md md=$(cat <<'EOF' -# pixi_pgroup_install +# create-python-install **The bernina python environment in your pgroup, and in JupyterHub.** @@ -200,8 +207,9 @@ manual() { ## Quick start (on ra) ```bash -pixi_pgroup_install -n 26i_blanco # preview, changes nothing -pixi_pgroup_install 26i_blanco # install +create-python-install -n 26i_blanco # preview, changes nothing +create-python-install 26i_blanco # install +create-python-install -J 26i_blanco # kernels into your JupyterHub ``` Then reload JupyterHub and pick the kernel @@ -228,7 +236,8 @@ conda_envs/ named links JupyterHub picks up pixi_cache/ downloaded packages (unless you set your own) ``` -Plus one line in your `~/.condarc` (under `envs_dirs`) pointing at `conda_envs/`. +Nothing in anyone's home. Whoever wants the kernels (you too) runs `-J` once: +it adds one line under `envs_dirs` in their `~/.condarc`, pointing at `conda_envs/`. ## Good to know @@ -240,8 +249,8 @@ Plus one line in your `~/.condarc` (under `envs_dirs`) pointing at `conda_envs/` in the work folder (keeps GBs out of your home), shared by the pgroup. - **Colleagues**: env, links and cache are group-writable, so any pgroup member can update or install into it. To see the kernels in *their* - JupyterHub they run the script once themselves (fast, already installed; - this also makes *their* files group-writable). + JupyterHub they run `create-python-install -J 26i_blanco` once (every + install prints this command for its pgroup). - **Add a package**: `-a scikit-image` (conda-forge) or `-A somepkg` (PyPI). Recorded in the pgroup's own `pixi.toml`/`pixi.lock`, so re-runs keep it. - **Back to the shared env** (drops added packages, keeps backups): `-u`. @@ -249,7 +258,7 @@ Plus one line in your `~/.condarc` (under `envs_dirs`) pointing at `conda_envs/` - **Remove**: delete `.pixi/ conda_envs/ pixi_cache/ pixi.toml pixi.lock` and the `conda_envs` line in `~/.condarc`. -Options and details: `pixi_pgroup_install -h` +Options and details: `create-python-install -h` EOF ) md="${md//@DEFAULT_ENV@/${DEFAULT_ENV_HINT:-bpy312}}" @@ -264,7 +273,7 @@ EOF exit 0 } -while getopts "e:p:a:A:uPCynmh" opt; do +while getopts "e:p:a:A:uPJynmh" opt; do case "$opt" in e) ENV_NAME="$OPTARG" ;; p) PREFIX="$OPTARG" ;; @@ -272,7 +281,7 @@ while getopts "e:p:a:A:uPCynmh" opt; do A) ADD_PYPI+=("$OPTARG") ;; u) RESET=1 ;; P) PERMS_ONLY=1 ;; - C) UPDATE_CONDARC=0 ;; + J) JOIN_ONLY=1 ;; y) ASSUME_YES=1 ;; n) DRY_RUN=1 ;; m) manual ;; @@ -402,7 +411,7 @@ SHARED=("$TARGET/.pixi" "$LINK_DIR") LINKS=("$PREFIX-$ENV_NAME") [ "$ENV_NAME" = "$DEFAULT_ENV" ] && LINKS+=("$PREFIX-default") -if [ "$PERMS_ONLY" -eq 0 ]; then +if [ "$PERMS_ONLY" -eq 0 ] && [ "$JOIN_ONLY" -eq 0 ]; then cat </dev/null || true } +# append (not prepend: the first envs_dirs entry is where `conda create -n` +# puts new envs) LINK_DIR to ~/.condarc; written with a redirect rather +# than a rename, so a ~/.condarc symlink stays a symlink +add_to_condarc() { + local rc="$HOME/.condarc" dir="$1" new + if [ -f "$rc" ]; then + if awk -v d="$dir" '{sub(/[[:space:]]+$/,"")} $0 ~ /^[[:space:]]*-[[:space:]]/ {sub(/^[[:space:]]*-[[:space:]]*/,""); if ($0==d || $0==d"/") f=1} END{exit !f}' "$rc"; then + echo "==> $dir already in $rc" + return + fi + grep -qE '^envs_dirs:[[:space:]]*[^[:space:]#]' "$rc" \ + && { echo "warning: envs_dirs in $rc isn't a block list, add $dir by hand" >&2; return; } + new=$(awk -v d="$dir" ' + /^envs_dirs:[[:space:]]*$/ { print; inblk = 1; next } + inblk && /^[[:space:]]*-/ { print; next } + inblk { print " - " d; inblk = 0; done = 1 } + { print } + END { if (inblk) { print " - " d; done = 1 } + if (!done) { print "envs_dirs:"; print " - " d } }' "$rc") + else + new=$(printf 'envs_dirs:\n - %s' "$dir") + fi + if [ "$DRY_RUN" -eq 1 ]; then + echo " [dry-run] $rc would become:"; sed 's/^/ /' <<<"$new" + else + printf '%s\n' "$new" > "$rc" + echo "==> added $dir to $rc" + fi +} + report_others() { local others others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ') @@ -470,6 +509,24 @@ if [ "$PERMS_ONLY" -eq 1 ]; then exit 0 fi +# what collaborators (and you) run once to get the kernels in JupyterHub +JOIN_ARG="${EXP_NAME:-${PGROUP:-$TARGET}}" +JOIN_CMD="$SELF_PATH -J $JOIN_ARG" + +if [ "$JOIN_ONLY" -eq 1 ]; then + kernels=$(ls "$LINK_DIR" 2>/dev/null || true) + [ -n "$kernels" ] || die "no kernels in $LINK_DIR yet -- install first" + case "$HOME" in + /das/home/*) ;; + *) echo "note: JupyterHub reads the ~/.condarc on ra (/das/home/$USER);" \ + "this is $(hostname -s) with HOME=$HOME" >&2 ;; + esac + add_to_condarc "$LINK_DIR" + echo "JupyterHub kernel(s) (reload the hub page):" + for l in $kernels; do echo " Python [conda env:$l]"; done + exit 0 +fi + # an active bpy-env.sh / pixi shell exports these and they would point pixi # at the shared project -- always address the target explicitly instead unset PIXI_PROJECT_MANIFEST PIXI_PROJECT_ROOT PIXI_IN_SHELL @@ -534,40 +591,6 @@ if [ "$DRY_RUN" -eq 0 ]; then report_others fi -# append (not prepend: the first envs_dirs entry is where `conda create -n` -# puts new envs) LINK_DIR to ~/.condarc; written with a redirect rather -# than a rename, so a ~/.condarc symlink stays a symlink -add_to_condarc() { - local rc="$HOME/.condarc" dir="$1" new - if [ -f "$rc" ]; then - if awk -v d="$dir" '{sub(/[[:space:]]+$/,"")} $0 ~ /^[[:space:]]*-[[:space:]]/ {sub(/^[[:space:]]*-[[:space:]]*/,""); if ($0==d || $0==d"/") f=1} END{exit !f}' "$rc"; then - echo "==> $dir already in $rc" - return - fi - grep -qE '^envs_dirs:[[:space:]]*[^[:space:]#]' "$rc" \ - && { echo "warning: envs_dirs in $rc isn't a block list, add $dir by hand" >&2; return; } - new=$(awk -v d="$dir" ' - /^envs_dirs:[[:space:]]*$/ { print; inblk = 1; next } - inblk && /^[[:space:]]*-/ { print; next } - inblk { print " - " d; inblk = 0; done = 1 } - { print } - END { if (inblk) { print " - " d; done = 1 } - if (!done) { print "envs_dirs:"; print " - " d } }' "$rc") - else - new=$(printf 'envs_dirs:\n - %s' "$dir") - fi - if [ "$DRY_RUN" -eq 1 ]; then - echo " [dry-run] $rc would become:"; sed 's/^/ /' <<<"$new" - else - printf '%s\n' "$new" > "$rc" - echo "==> added $dir to $rc" - fi -} - -if [ "$UPDATE_CONDARC" -eq 1 ]; then - add_to_condarc "$LINK_DIR" -fi - [ "$DRY_RUN" -eq 0 ] || { echo; echo "Dry run, nothing changed."; exit 0; } cat <