From 0eea66bec1acdded6c95493e7957cc79758917f1 Mon Sep 17 00:00:00 2001 From: lemke_h Date: Fri, 25 Sep 2026 16:06:09 +0200 Subject: [PATCH] pixi_pgroup_install: keep pgroup changes, add -a/-A, -u, -P - Re-runs keep the pgroup's own pixi.toml/pixi.lock when they differ from the shared ones (e.g. packages added), so a colleague running it to get the JupyterHub kernels no longer resets the env. -u resets to the shared files explicitly (backups kept). - -a pkg / -A pkg: add conda-forge / PyPI packages (pixi add, install, group permissions, pgroup cache). - -P: only make your own files in the install group-writable. --- pixi_pgroup_install | 163 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 130 insertions(+), 33 deletions(-) diff --git a/pixi_pgroup_install b/pixi_pgroup_install index 7bdb555..045e68c 100755 --- a/pixi_pgroup_install +++ b/pixi_pgroup_install @@ -5,7 +5,9 @@ # # Copies SOURCE_DIR's pixi.toml/pixi.lock straight into the target folder # (no python/ subfolder) and runs `pixi install --frozen` there, i.e. -# exactly what is pinned in the shared lock -- nothing gets re-solved. +# exactly what is pinned in the lock -- nothing gets re-solved. Once the +# pgroup's copy has local changes (packages added with -a/-A or `pixi +# add`), re-runs keep them; only -u resets to the shared files. # Only one environment is installed: the one SOURCE_DIR's bpy-default # link points to, unless -e picks another. # @@ -50,10 +52,15 @@ PREFIX="" UPDATE_CONDARC=1 ASSUME_YES=0 DRY_RUN=0 +RESET=0 +PERMS_ONLY=0 +ADD_CONDA=() +ADD_PYPI=() usage() { cat < / pixi.lock.. Then 'pixi install --frozen' installs - exactly what the shared lock pins -- nothing is re-solved or updated. - The env ends up in /.pixi/envs/. + On the first run, pixi.toml and pixi.lock are copied directly into that + folder (no subfolder). 'pixi install --frozen' then installs exactly + what the lock pins -- nothing is re-solved or updated. The env ends up + in /.pixi/envs/. + + Re-runs never throw away the pgroup's own changes: if the folder's + pixi.toml/pixi.lock differ from the shared ones (packages added with + -a/-A or 'pixi add'), they are kept and installed as they are. Only -u + resets them to the shared files (the old ones are kept as + pixi.toml. / pixi.lock.). WHICH ENVIRONMENT One env only. Default: the bernina default, i.e. what @@ -99,8 +111,19 @@ SHARING WITHIN THE PGROUP the cache, update the env or install into it. Only the owner of a file can change its permissions: each member who runs the script fixes their own. Use umask 002 for plain pixi commands (and don't run two installs into - the same env at once). A 'pixi add' changes the local pixi.toml/pixi.lock; - re-running this script resets them to the shared ones (keeping backups). + the same env at once). -P fixes up your own files without installing + anything, e.g. after plain pixi commands. + +ADDING PACKAGES + -a pkg (conda-forge) / -A pkg (PyPI), repeatable, version specs allowed: + $(basename "$0") -a scikit-image -A "somepkg>=1.2" 26i_blanco + This does 'pixi add' into the pgroup's pixi.toml (recorded in its + pixi.lock, so it survives re-installs), installs, and fixes permissions + -- with umask 002 and the pgroup cache set. The package goes into the + part of pixi.toml that every env shares, so all envs' locks are + re-solved (versions only change where they must); only the selected env + is installed. A quick 'pip install' in the env also works, but isn't + recorded and can disappear on the next install. JUPYTERHUB The hub lists every conda env it finds in the folders named under @@ -122,6 +145,11 @@ OPTIONS -e env install this environment instead of the default -p prefix jupyter name prefix instead of bernina- (underscores in exp names become dashes) + -a pkg add a conda-forge package (repeatable), see ADDING PACKAGES + -A pkg add a PyPI package (repeatable) + -u reset the folder's pixi.toml/pixi.lock to the shared ones + (removes locally added packages; backups are kept) + -P only make your own files in the install group-writable -C don't touch ~/.condarc, just print the line to add -y don't ask for confirmation -n dry run: show what would be done, change nothing @@ -132,13 +160,15 @@ EXAMPLES $(basename "$0") -n 26i_blanco # see what would happen $(basename "$0") 26i_blanco # bpy312 into p23620's work folder $(basename "$0") -e bpy313 p23620 # a different env + $(basename "$0") -a scikit-image 26i_blanco # add a package + $(basename "$0") -u 26i_blanco # back to the shared env cd /sf/bernina/data/p23620/work && $(basename "$0") AFTERWARDS Activate in a shell: eval "\$($PIXI_BIN shell-hook -e --manifest-path /pixi.toml)" - Re-run the script any time to bring the copy in line with the shared - pixi.lock again. + Re-running it is always safe (joining colleagues do exactly that); -u + brings the copy back in line with the shared pixi.lock. EOF exit 0 } @@ -212,7 +242,10 @@ Plus one line in your `~/.condarc` (under `envs_dirs`) pointing at `conda_envs/` member can update or install into it. To see the kernels in *their* JupyterHub they run the script once themselves (fast, already installed; this also makes *their* files group-writable). -- **Update** to the current shared env: just run it again. +- **Add a package**: `-a scikit-image` (conda-forge) or `-A somepkg` (PyPI). + Recorded in the pgroup's own `pixi.toml`/`pixi.lock`, so re-runs keep it. +- **Back to the shared env** (drops added packages, keeps backups): `-u`. +- **Only fix permissions** of your files (e.g. after plain `pixi` commands): `-P`. - **Remove**: delete `.pixi/ conda_envs/ pixi_cache/ pixi.toml pixi.lock` and the `conda_envs` line in `~/.condarc`. @@ -231,10 +264,14 @@ EOF exit 0 } -while getopts "e:p:Cynmh" opt; do +while getopts "e:p:a:A:uPCynmh" opt; do case "$opt" in e) ENV_NAME="$OPTARG" ;; p) PREFIX="$OPTARG" ;; + a) ADD_CONDA+=("$OPTARG") ;; + A) ADD_PYPI+=("$OPTARG") ;; + u) RESET=1 ;; + P) PERMS_ONLY=1 ;; C) UPDATE_CONDARC=0 ;; y) ASSUME_YES=1 ;; n) DRY_RUN=1 ;; @@ -317,14 +354,34 @@ if [ -z "$PREFIX" ]; then PREFIX="bernina-${label//_/-}" fi +# Which pixi.toml/pixi.lock to install from: +# fresh - none in the folder yet: copy the shared ones +# shared - identical to the shared ones: nothing to copy +# local - they differ (packages added in this pgroup): keep them +# reset - they differ, but -u: back up and copy the shared ones +if [ ! -e "$TARGET/pixi.toml" ] && [ ! -e "$TARGET/pixi.lock" ]; then + SETUP=fresh +elif cmp -s "$SOURCE_DIR/pixi.toml" "$TARGET/pixi.toml" \ + && cmp -s "$SOURCE_DIR/pixi.lock" "$TARGET/pixi.lock"; then + SETUP=shared +elif [ "$RESET" -eq 1 ]; then + SETUP=reset +else + SETUP=local + [ -f "$TARGET/pixi.toml" ] && [ -f "$TARGET/pixi.lock" ] \ + || die "$TARGET has only one of pixi.toml/pixi.lock; -u resets both to the shared ones" +fi +MANIFEST_SRC="$SOURCE_DIR/pixi.toml" +[ "$SETUP" = local ] && MANIFEST_SRC="$TARGET/pixi.toml" + DEFAULT_ENV=$(basename "$(readlink "$SOURCE_DIR/.pixi/envs/bpy-default" 2>/dev/null)" 2>/dev/null || true) [ -n "$DEFAULT_ENV" ] || die "can't resolve $SOURCE_DIR/.pixi/envs/bpy-default" ENV_NAME="${ENV_NAME:-$DEFAULT_ENV}" [ "$ENV_NAME" != "default" ] \ || die "pixi's own 'default' env isn't meant for this; the bernina default is $DEFAULT_ENV (used when -e is omitted)" -defined_envs=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); print}' "$SOURCE_DIR/pixi.toml") +defined_envs=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); print}' "$MANIFEST_SRC") grep -qxF -- "$ENV_NAME" <<<"$defined_envs" \ - || die "no environment '$ENV_NAME' in $SOURCE_DIR/pixi.toml (have: $(echo $defined_envs))" + || die "no environment '$ENV_NAME' in $MANIFEST_SRC (have: $(echo $defined_envs))" # package cache: keep a sensible user setting, otherwise the target folder cache_ok() { [ -n "$1" ] && [ -d "$1" ] && [ -w "$1" ]; } @@ -339,14 +396,28 @@ else fi LINK_DIR="$TARGET/$LINK_DIR_NAME" +# only the pgroup's own cache is shared; a personal PIXI_CACHE_DIR is left alone +SHARED=("$TARGET/.pixi" "$LINK_DIR") +[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || SHARED+=("$PIXI_CACHE_DIR") LINKS=("$PREFIX-$ENV_NAME") [ "$ENV_NAME" = "$DEFAULT_ENV" ] && LINKS+=("$PREFIX-default") +if [ "$PERMS_ONLY" -eq 0 ]; then cat </dev/null || true } +report_others() { + local others + others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ') + [ -z "$others" ] || echo "note: some files of $others aren't group-writable yet;" \ + "they can fix that with: $(basename "$0") -P $TARGET" +} + +if [ "$PERMS_ONLY" -eq 1 ]; then + [ -d "$TARGET/.pixi" ] || die "no pixi install in $TARGET" + [ "$DRY_RUN" -eq 0 ] || { echo " [dry-run] make your files group-writable in: ${SHARED[*]}"; exit 0; } + echo "==> making your files in $TARGET group-writable for $(stat -c %G "$TARGET")" + for d in "${SHARED[@]}"; do share_tree "$d"; done + report_others + exit 0 +fi + # an active bpy-env.sh / pixi shell exports these and they would point pixi # at the shared project -- always address the target explicitly instead unset PIXI_PROJECT_MANIFEST PIXI_PROJECT_ROOT PIXI_IN_SHELL stamp=$(date +%Y%m%d-%H%M%S) -for f in pixi.toml pixi.lock; do - if [ -f "$TARGET/$f" ] && ! cmp -s "$SOURCE_DIR/$f" "$TARGET/$f"; then - echo "==> $TARGET/$f differs from the shared one, keeping it as $f.$stamp" - run cp -p "$TARGET/$f" "$TARGET/$f.$stamp" - fi - run cp "$SOURCE_DIR/$f" "$TARGET/$f" - # (fails harmlessly if another member owns it -- then it's theirs to share) - [ "$DRY_RUN" -eq 1 ] || chmod 664 "$TARGET/$f" 2>/dev/null || true -done +case "$SETUP" in + fresh|reset) + for f in pixi.toml pixi.lock; do + if [ -f "$TARGET/$f" ] && ! cmp -s "$SOURCE_DIR/$f" "$TARGET/$f"; then + echo "==> keeping the pgroup's $f as $f.$stamp" + run cp -p "$TARGET/$f" "$TARGET/$f.$stamp" + fi + run cp "$SOURCE_DIR/$f" "$TARGET/$f" + done ;; + local) + echo "==> using the pgroup's own pixi.toml/pixi.lock (differ from the shared ones; -u resets)" ;; +esac +# (fails harmlessly if another member owns them -- then they're theirs to share) +[ "$DRY_RUN" -eq 1 ] || chmod 664 "$TARGET/pixi.toml" "$TARGET/pixi.lock" 2>/dev/null || true # never let a global detached-environments setting move the envs away # from .pixi/envs, the jupyter links below rely on that location @@ -408,12 +501,7 @@ if [ ! -f "$TARGET/.pixi/config.toml" ]; then fi fi -# only the pgroup's own cache is shared; a personal PIXI_CACHE_DIR is left alone -SHARED=("$TARGET/.pixi" "$LINK_DIR") -if [ "$PIXI_CACHE_DIR" = "$TARGET/$CACHE_DIR_NAME" ]; then - run mkdir -p "$PIXI_CACHE_DIR" - SHARED+=("$PIXI_CACHE_DIR") -fi +[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || run mkdir -p "$PIXI_CACHE_DIR" run mkdir -p "$TARGET/.pixi/envs" "$LINK_DIR" if [ "$DRY_RUN" -eq 0 ]; then # before installing, so everything pixi creates below inherits it @@ -422,6 +510,17 @@ else echo " [dry-run] make group-writable: ${SHARED[*]}" fi +# -a/-A: record the packages in pixi.toml/pixi.lock (re-solves the lock), +# the install below then puts them into the env +if [ ${#ADD_CONDA[@]} -gt 0 ]; then + echo "==> adding ${ADD_CONDA[*]}" + run "$PIXI_BIN" add --no-install --manifest-path "$TARGET/pixi.toml" "${ADD_CONDA[@]}" +fi +if [ ${#ADD_PYPI[@]} -gt 0 ]; then + echo "==> adding (PyPI) ${ADD_PYPI[*]}" + run "$PIXI_BIN" add --no-install --pypi --manifest-path "$TARGET/pixi.toml" "${ADD_PYPI[@]}" +fi + echo "==> installing $ENV_NAME" run "$PIXI_BIN" install --frozen -e "$ENV_NAME" --manifest-path "$TARGET/pixi.toml" @@ -432,9 +531,7 @@ done if [ "$DRY_RUN" -eq 0 ]; then echo "==> making it group-writable for $(stat -c %G "$TARGET")" for d in "${SHARED[@]}"; do share_tree "$d"; done - others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ') - [ -z "$others" ] || echo "note: some files of $others aren't group-writable yet;" \ - "running this script once as them fixes that" + report_others fi # append (not prepend: the first envs_dirs entry is where `conda create -n`