pixi_pgroup_install: keep pgroup changes, add -a/-A, -u, -P
- Re-runs keep the pgroup's own pixi.toml/pixi.lock when they differ from the shared ones (e.g. packages added), so a colleague running it to get the JupyterHub kernels no longer resets the env. -u resets to the shared files explicitly (backups kept). - -a pkg / -A pkg: add conda-forge / PyPI packages (pixi add, install, group permissions, pgroup cache). - -P: only make your own files in the install group-writable.
This commit is contained in:
lemke_h committed
1 parent
76b985975a
commit
0eea66bec1
1 file changed
+130
-33
+130
-33
@@ -5,7 +5,9 @@
|
||||
#
|
||||
# Copies SOURCE_DIR's pixi.toml/pixi.lock straight into the target folder
|
||||
# (no python/ subfolder) and runs `pixi install --frozen` there, i.e.
|
||||
# exactly what is pinned in the shared lock -- nothing gets re-solved.
|
||||
# exactly what is pinned in the lock -- nothing gets re-solved. Once the
|
||||
# pgroup's copy has local changes (packages added with -a/-A or `pixi
|
||||
# add`), re-runs keep them; only -u resets to the shared files.
|
||||
# Only one environment is installed: the one SOURCE_DIR's bpy-default
|
||||
# link points to, unless -e picks another.
|
||||
#
|
||||
@@ -50,10 +52,15 @@ PREFIX=""
|
||||
UPDATE_CONDARC=1
|
||||
ASSUME_YES=0
|
||||
DRY_RUN=0
|
||||
RESET=0
|
||||
PERMS_ONLY=0
|
||||
ADD_CONDA=()
|
||||
ADD_PYPI=()
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") [-e env] [-p prefix] [-C] [-y] [-n] [pgroup | exp-name | path]
|
||||
Usage: $(basename "$0") [-e env] [-a pkg] [-A pkg] [-u] [-p prefix] [-C] [-y] [-n] [pgroup | exp-name | path]
|
||||
$(basename "$0") -P [pgroup | exp-name | path]
|
||||
$(basename "$0") -m short manual
|
||||
$(basename "$0") -h this help
|
||||
|
||||
@@ -73,11 +80,16 @@ WHERE IT INSTALLS
|
||||
the first match alphabetically is used (others listed).
|
||||
path any folder, given as a path (must contain a "/")
|
||||
|
||||
pixi.toml and pixi.lock are copied directly into that folder (no
|
||||
subfolder); an existing, different pixi.toml/pixi.lock there is kept as
|
||||
pixi.toml.<date> / pixi.lock.<date>. Then 'pixi install --frozen' installs
|
||||
exactly what the shared lock pins -- nothing is re-solved or updated.
|
||||
The env ends up in <folder>/.pixi/envs/<env>.
|
||||
On the first run, pixi.toml and pixi.lock are copied directly into that
|
||||
folder (no subfolder). 'pixi install --frozen' then installs exactly
|
||||
what the lock pins -- nothing is re-solved or updated. The env ends up
|
||||
in <folder>/.pixi/envs/<env>.
|
||||
|
||||
Re-runs never throw away the pgroup's own changes: if the folder's
|
||||
pixi.toml/pixi.lock differ from the shared ones (packages added with
|
||||
-a/-A or 'pixi add'), they are kept and installed as they are. Only -u
|
||||
resets them to the shared files (the old ones are kept as
|
||||
pixi.toml.<date> / pixi.lock.<date>).
|
||||
|
||||
WHICH ENVIRONMENT
|
||||
One env only. Default: the bernina default, i.e. what
|
||||
@@ -99,8 +111,19 @@ SHARING WITHIN THE PGROUP
|
||||
the cache, update the env or install into it. Only the owner of a file can
|
||||
change its permissions: each member who runs the script fixes their own.
|
||||
Use umask 002 for plain pixi commands (and don't run two installs into
|
||||
the same env at once). A 'pixi add' changes the local pixi.toml/pixi.lock;
|
||||
re-running this script resets them to the shared ones (keeping backups).
|
||||
the same env at once). -P fixes up your own files without installing
|
||||
anything, e.g. after plain pixi commands.
|
||||
|
||||
ADDING PACKAGES
|
||||
-a pkg (conda-forge) / -A pkg (PyPI), repeatable, version specs allowed:
|
||||
$(basename "$0") -a scikit-image -A "somepkg>=1.2" 26i_blanco
|
||||
This does 'pixi add' into the pgroup's pixi.toml (recorded in its
|
||||
pixi.lock, so it survives re-installs), installs, and fixes permissions
|
||||
-- with umask 002 and the pgroup cache set. The package goes into the
|
||||
part of pixi.toml that every env shares, so all envs' locks are
|
||||
re-solved (versions only change where they must); only the selected env
|
||||
is installed. A quick 'pip install' in the env also works, but isn't
|
||||
recorded and can disappear on the next install.
|
||||
|
||||
JUPYTERHUB
|
||||
The hub lists every conda env it finds in the folders named under
|
||||
@@ -122,6 +145,11 @@ OPTIONS
|
||||
-e env install this environment instead of the default
|
||||
-p prefix jupyter name prefix instead of bernina-<exp name>
|
||||
(underscores in exp names become dashes)
|
||||
-a pkg add a conda-forge package (repeatable), see ADDING PACKAGES
|
||||
-A pkg add a PyPI package (repeatable)
|
||||
-u reset the folder's pixi.toml/pixi.lock to the shared ones
|
||||
(removes locally added packages; backups are kept)
|
||||
-P only make your own files in the install group-writable
|
||||
-C don't touch ~/.condarc, just print the line to add
|
||||
-y don't ask for confirmation
|
||||
-n dry run: show what would be done, change nothing
|
||||
@@ -132,13 +160,15 @@ EXAMPLES
|
||||
$(basename "$0") -n 26i_blanco # see what would happen
|
||||
$(basename "$0") 26i_blanco # bpy312 into p23620's work folder
|
||||
$(basename "$0") -e bpy313 p23620 # a different env
|
||||
$(basename "$0") -a scikit-image 26i_blanco # add a package
|
||||
$(basename "$0") -u 26i_blanco # back to the shared env
|
||||
cd /sf/bernina/data/p23620/work && $(basename "$0")
|
||||
|
||||
AFTERWARDS
|
||||
Activate in a shell:
|
||||
eval "\$($PIXI_BIN shell-hook -e <env> --manifest-path <folder>/pixi.toml)"
|
||||
Re-run the script any time to bring the copy in line with the shared
|
||||
pixi.lock again.
|
||||
Re-running it is always safe (joining colleagues do exactly that); -u
|
||||
brings the copy back in line with the shared pixi.lock.
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
@@ -212,7 +242,10 @@ Plus one line in your `~/.condarc` (under `envs_dirs`) pointing at `conda_envs/`
|
||||
member can update or install into it. To see the kernels in *their*
|
||||
JupyterHub they run the script once themselves (fast, already installed;
|
||||
this also makes *their* files group-writable).
|
||||
- **Update** to the current shared env: just run it again.
|
||||
- **Add a package**: `-a scikit-image` (conda-forge) or `-A somepkg` (PyPI).
|
||||
Recorded in the pgroup's own `pixi.toml`/`pixi.lock`, so re-runs keep it.
|
||||
- **Back to the shared env** (drops added packages, keeps backups): `-u`.
|
||||
- **Only fix permissions** of your files (e.g. after plain `pixi` commands): `-P`.
|
||||
- **Remove**: delete `.pixi/ conda_envs/ pixi_cache/ pixi.toml pixi.lock` and the
|
||||
`conda_envs` line in `~/.condarc`.
|
||||
|
||||
@@ -231,10 +264,14 @@ EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
while getopts "e:p:Cynmh" opt; do
|
||||
while getopts "e:p:a:A:uPCynmh" opt; do
|
||||
case "$opt" in
|
||||
e) ENV_NAME="$OPTARG" ;;
|
||||
p) PREFIX="$OPTARG" ;;
|
||||
a) ADD_CONDA+=("$OPTARG") ;;
|
||||
A) ADD_PYPI+=("$OPTARG") ;;
|
||||
u) RESET=1 ;;
|
||||
P) PERMS_ONLY=1 ;;
|
||||
C) UPDATE_CONDARC=0 ;;
|
||||
y) ASSUME_YES=1 ;;
|
||||
n) DRY_RUN=1 ;;
|
||||
@@ -317,14 +354,34 @@ if [ -z "$PREFIX" ]; then
|
||||
PREFIX="bernina-${label//_/-}"
|
||||
fi
|
||||
|
||||
# Which pixi.toml/pixi.lock to install from:
|
||||
# fresh - none in the folder yet: copy the shared ones
|
||||
# shared - identical to the shared ones: nothing to copy
|
||||
# local - they differ (packages added in this pgroup): keep them
|
||||
# reset - they differ, but -u: back up and copy the shared ones
|
||||
if [ ! -e "$TARGET/pixi.toml" ] && [ ! -e "$TARGET/pixi.lock" ]; then
|
||||
SETUP=fresh
|
||||
elif cmp -s "$SOURCE_DIR/pixi.toml" "$TARGET/pixi.toml" \
|
||||
&& cmp -s "$SOURCE_DIR/pixi.lock" "$TARGET/pixi.lock"; then
|
||||
SETUP=shared
|
||||
elif [ "$RESET" -eq 1 ]; then
|
||||
SETUP=reset
|
||||
else
|
||||
SETUP=local
|
||||
[ -f "$TARGET/pixi.toml" ] && [ -f "$TARGET/pixi.lock" ] \
|
||||
|| die "$TARGET has only one of pixi.toml/pixi.lock; -u resets both to the shared ones"
|
||||
fi
|
||||
MANIFEST_SRC="$SOURCE_DIR/pixi.toml"
|
||||
[ "$SETUP" = local ] && MANIFEST_SRC="$TARGET/pixi.toml"
|
||||
|
||||
DEFAULT_ENV=$(basename "$(readlink "$SOURCE_DIR/.pixi/envs/bpy-default" 2>/dev/null)" 2>/dev/null || true)
|
||||
[ -n "$DEFAULT_ENV" ] || die "can't resolve $SOURCE_DIR/.pixi/envs/bpy-default"
|
||||
ENV_NAME="${ENV_NAME:-$DEFAULT_ENV}"
|
||||
[ "$ENV_NAME" != "default" ] \
|
||||
|| die "pixi's own 'default' env isn't meant for this; the bernina default is $DEFAULT_ENV (used when -e is omitted)"
|
||||
defined_envs=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); print}' "$SOURCE_DIR/pixi.toml")
|
||||
defined_envs=$(awk '/^\[/{s=($0=="[environments]")} s && /^[A-Za-z0-9_-]+[[:space:]]*=/{sub(/[[:space:]]*=.*/,""); print}' "$MANIFEST_SRC")
|
||||
grep -qxF -- "$ENV_NAME" <<<"$defined_envs" \
|
||||
|| die "no environment '$ENV_NAME' in $SOURCE_DIR/pixi.toml (have: $(echo $defined_envs))"
|
||||
|| die "no environment '$ENV_NAME' in $MANIFEST_SRC (have: $(echo $defined_envs))"
|
||||
|
||||
# package cache: keep a sensible user setting, otherwise the target folder
|
||||
cache_ok() { [ -n "$1" ] && [ -d "$1" ] && [ -w "$1" ]; }
|
||||
@@ -339,14 +396,28 @@ else
|
||||
fi
|
||||
|
||||
LINK_DIR="$TARGET/$LINK_DIR_NAME"
|
||||
# only the pgroup's own cache is shared; a personal PIXI_CACHE_DIR is left alone
|
||||
SHARED=("$TARGET/.pixi" "$LINK_DIR")
|
||||
[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || SHARED+=("$PIXI_CACHE_DIR")
|
||||
LINKS=("$PREFIX-$ENV_NAME")
|
||||
[ "$ENV_NAME" = "$DEFAULT_ENV" ] && LINKS+=("$PREFIX-default")
|
||||
|
||||
if [ "$PERMS_ONLY" -eq 0 ]; then
|
||||
cat <<EOF
|
||||
Source: $SOURCE_DIR
|
||||
Target: $TARGET${EXP_NAME:+ ($EXP_NAME${PGROUP:+, $PGROUP})}
|
||||
Environment: $ENV_NAME$([ "$ENV_NAME" = "$DEFAULT_ENV" ] && echo " (bernina default)")
|
||||
Pixi cache: $PIXI_CACHE_DIR $CACHE_NOTE
|
||||
pixi.toml: $(case "$SETUP" in
|
||||
fresh) echo "copy of the shared one (first install)" ;;
|
||||
shared) echo "same as the shared one" ;;
|
||||
local) echo "the pgroup's own (differs from the shared one), kept -- -u resets" ;;
|
||||
reset) echo "RESET to the shared one: locally added packages are removed (backups kept)" ;;
|
||||
esac)
|
||||
EOF
|
||||
[ ${#ADD_CONDA[@]} -eq 0 ] || echo "Add (conda): ${ADD_CONDA[*]}"
|
||||
[ ${#ADD_PYPI[@]} -eq 0 ] || echo "Add (PyPI): ${ADD_PYPI[*]}"
|
||||
cat <<EOF
|
||||
Jupyter: $(for l in "${LINKS[@]}"; do printf '"Python [conda env:%s]" ' "$l"; done)
|
||||
via $LINK_DIR$([ "$UPDATE_CONDARC" -eq 1 ] && echo " (added to ~/.condarc)")
|
||||
EOF
|
||||
@@ -355,6 +426,7 @@ if [ "$DRY_RUN" -eq 0 ] && [ "$ASSUME_YES" -eq 0 ] && [ -t 0 ]; then
|
||||
read -r -p "Proceed? [y/N] " answer
|
||||
[[ "$answer" =~ ^[Yy] ]] || { echo "aborted"; exit 1; }
|
||||
fi
|
||||
fi # PERMS_ONLY
|
||||
|
||||
# group members of the pgroup should be able to use and update this too
|
||||
umask 002
|
||||
@@ -382,20 +454,41 @@ share_tree() { # everything below, owned by this user
|
||||
|| find "$d" -user "$USER" -type d -exec setfacl -m d:g::rwx,d:m::rwx {} + 2>/dev/null || true
|
||||
}
|
||||
|
||||
report_others() {
|
||||
local others
|
||||
others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ')
|
||||
[ -z "$others" ] || echo "note: some files of $others aren't group-writable yet;" \
|
||||
"they can fix that with: $(basename "$0") -P $TARGET"
|
||||
}
|
||||
|
||||
if [ "$PERMS_ONLY" -eq 1 ]; then
|
||||
[ -d "$TARGET/.pixi" ] || die "no pixi install in $TARGET"
|
||||
[ "$DRY_RUN" -eq 0 ] || { echo " [dry-run] make your files group-writable in: ${SHARED[*]}"; exit 0; }
|
||||
echo "==> making your files in $TARGET group-writable for $(stat -c %G "$TARGET")"
|
||||
for d in "${SHARED[@]}"; do share_tree "$d"; done
|
||||
report_others
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# an active bpy-env.sh / pixi shell exports these and they would point pixi
|
||||
# at the shared project -- always address the target explicitly instead
|
||||
unset PIXI_PROJECT_MANIFEST PIXI_PROJECT_ROOT PIXI_IN_SHELL
|
||||
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
for f in pixi.toml pixi.lock; do
|
||||
if [ -f "$TARGET/$f" ] && ! cmp -s "$SOURCE_DIR/$f" "$TARGET/$f"; then
|
||||
echo "==> $TARGET/$f differs from the shared one, keeping it as $f.$stamp"
|
||||
run cp -p "$TARGET/$f" "$TARGET/$f.$stamp"
|
||||
fi
|
||||
run cp "$SOURCE_DIR/$f" "$TARGET/$f"
|
||||
# (fails harmlessly if another member owns it -- then it's theirs to share)
|
||||
[ "$DRY_RUN" -eq 1 ] || chmod 664 "$TARGET/$f" 2>/dev/null || true
|
||||
done
|
||||
case "$SETUP" in
|
||||
fresh|reset)
|
||||
for f in pixi.toml pixi.lock; do
|
||||
if [ -f "$TARGET/$f" ] && ! cmp -s "$SOURCE_DIR/$f" "$TARGET/$f"; then
|
||||
echo "==> keeping the pgroup's $f as $f.$stamp"
|
||||
run cp -p "$TARGET/$f" "$TARGET/$f.$stamp"
|
||||
fi
|
||||
run cp "$SOURCE_DIR/$f" "$TARGET/$f"
|
||||
done ;;
|
||||
local)
|
||||
echo "==> using the pgroup's own pixi.toml/pixi.lock (differ from the shared ones; -u resets)" ;;
|
||||
esac
|
||||
# (fails harmlessly if another member owns them -- then they're theirs to share)
|
||||
[ "$DRY_RUN" -eq 1 ] || chmod 664 "$TARGET/pixi.toml" "$TARGET/pixi.lock" 2>/dev/null || true
|
||||
|
||||
# never let a global detached-environments setting move the envs away
|
||||
# from .pixi/envs, the jupyter links below rely on that location
|
||||
@@ -408,12 +501,7 @@ if [ ! -f "$TARGET/.pixi/config.toml" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# only the pgroup's own cache is shared; a personal PIXI_CACHE_DIR is left alone
|
||||
SHARED=("$TARGET/.pixi" "$LINK_DIR")
|
||||
if [ "$PIXI_CACHE_DIR" = "$TARGET/$CACHE_DIR_NAME" ]; then
|
||||
run mkdir -p "$PIXI_CACHE_DIR"
|
||||
SHARED+=("$PIXI_CACHE_DIR")
|
||||
fi
|
||||
[ "$PIXI_CACHE_DIR" != "$TARGET/$CACHE_DIR_NAME" ] || run mkdir -p "$PIXI_CACHE_DIR"
|
||||
run mkdir -p "$TARGET/.pixi/envs" "$LINK_DIR"
|
||||
if [ "$DRY_RUN" -eq 0 ]; then
|
||||
# before installing, so everything pixi creates below inherits it
|
||||
@@ -422,6 +510,17 @@ else
|
||||
echo " [dry-run] make group-writable: ${SHARED[*]}"
|
||||
fi
|
||||
|
||||
# -a/-A: record the packages in pixi.toml/pixi.lock (re-solves the lock),
|
||||
# the install below then puts them into the env
|
||||
if [ ${#ADD_CONDA[@]} -gt 0 ]; then
|
||||
echo "==> adding ${ADD_CONDA[*]}"
|
||||
run "$PIXI_BIN" add --no-install --manifest-path "$TARGET/pixi.toml" "${ADD_CONDA[@]}"
|
||||
fi
|
||||
if [ ${#ADD_PYPI[@]} -gt 0 ]; then
|
||||
echo "==> adding (PyPI) ${ADD_PYPI[*]}"
|
||||
run "$PIXI_BIN" add --no-install --pypi --manifest-path "$TARGET/pixi.toml" "${ADD_PYPI[@]}"
|
||||
fi
|
||||
|
||||
echo "==> installing $ENV_NAME"
|
||||
run "$PIXI_BIN" install --frozen -e "$ENV_NAME" --manifest-path "$TARGET/pixi.toml"
|
||||
|
||||
@@ -432,9 +531,7 @@ done
|
||||
if [ "$DRY_RUN" -eq 0 ]; then
|
||||
echo "==> making it group-writable for $(stat -c %G "$TARGET")"
|
||||
for d in "${SHARED[@]}"; do share_tree "$d"; done
|
||||
others=$(find "${SHARED[@]}" ! -user "$USER" -type f ! -perm -g+w -printf '%u\n' 2>/dev/null | sort -u | tr '\n' ' ')
|
||||
[ -z "$others" ] || echo "note: some files of $others aren't group-writable yet;" \
|
||||
"running this script once as them fixes that"
|
||||
report_others
|
||||
fi
|
||||
|
||||
# append (not prepend: the first envs_dirs entry is where `conda create -n`
|
||||
|
||||
Reference in new issue
Block a user