diff --git a/ccp4/README.md b/ccp4/README.md new file mode 100644 index 0000000..a110664 --- /dev/null +++ b/ccp4/README.md @@ -0,0 +1,89 @@ +# ccp4 + +CCP4 suite (binary distribution, bundled with SHELX and ARP/wARP) for +macromolecular crystallography. https://www.ccp4.ac.uk/ + +- Vendor tarball install: custom `prep` downloads the ~3.7 GB tarball with + `wget --continue` (resumes partial downloads; the builtin curl can't) and + verifies it with `gzip -t`; `install` untars it into `$PREFIX` and runs + `BINARY.setup`. +- Non-interactive: the CCP4 licence is pre-agreed via `.agree2ccp4v6` + before `BINARY.setup` runs, so the build never prompts. +- No `shasums:`: the download link is session-gated (`sid=` expires), so + the checksum cannot be pinned ahead of the first fetch. After the first + successful build, `sha256sum` the cached tarball and add it if wanted. +- Module version is the series (`9.0`); minor updates (9.0.xxx) come via + the CCP4 update manager inside the installation. + +## Build on Ra + +Build tmp and download cache MUST be on /das, not the defaults +(/var/tmp is too small, the tarball alone blows the $HOME quota — a +download dying with "Failure writing output to destination" means +exactly that). Pass both dirs explicitly on the command line so there +is no doubt where things land — **adapt the p-group path +(`p21/p21515`) to your own**: + +```bash +module use unstable && module load modbuild/2.1.3 +cd ccp4 +mkdir -p /das/work/p21/p21515/.cache/Pmodules +modbuild build \ + --tmpdir=/das/work/p21/p21515/.cache/Pmodules \ + --distdir=/das/work/p21/p21515/.cache/Pmodules \ + --clean-install -v --debug 9.0 +``` + +(`--clean-install` removes a previous broken install first; `-v +--debug` make the otherwise silent long steps visible.) + +Alternative: `~/.Pmodules/Pmodules.yaml` with top-level `tmp_dir:` and +`download_dir:` keys sets the same defaults per user — but in practice +`download_dir` did not reliably take effect there, so the explicit +flags above are the maintained route. + +If the download fails (expired sid), fetch a fresh link from +https://www.ccp4.ac.uk/download/ and update `files/config.yaml`, or drop a +hand-downloaded tarball into the distfiles dir under the `name:` given in +`files/config.yaml`. + +## Install is SLOW — not stuck + +The `install` step untars 3.7 GB into ~15-20 GB of small files on shared +/opt/psi storage: the tar alone runs 10-30 min with no output (gzip is +single-threaded, small-file creation on network storage is the +bottleneck). `BINARY.setup` afterwards is just as bad — it walks the +whole installation patching paths and then **byte-compiles the bundled +Python environment** ("compiling py-files"), thousands of tiny .pyc +writes on network storage; expect another 15-30+ min. Budget roughly an +hour for the whole install on Ra. Watch progress from a second shell: + +```bash +watch -n 30 'du -sh /opt/psi/MX/ccp4/9.0/' +``` + +Growing `du` = working. Only investigate if it's frozen for 10+ min. + +## Known failure modes + +- **"done" but broken install**: modbuild has no errexit in hooks — errors in + `pbuild::install` used to scroll by and the build still reported done. The + build script now `std::die`s on every critical step. +- **Interrupted download**: prep detects a truncated tarball with + `gzip -t` and resumes it with `wget --continue` — just re-run the + build command above. A download that dies with + `Failure writing output to destination` means quota/disk full at the + distfiles dir, not a network problem — check `--distdir` points to + /das before retrying. +- The tarball unpacks to `ccp4-9` (not `ccp4-9.0` as the CCP4 install doc + implies); the build globs `ccp4-*/` instead of assuming the name. + +## Verify + +```bash +module use MX unstable && module load ccp4/9.0 +which refmac5 ccp4i2 +echo $CCP4 # should point into /opt/psi/MX/ccp4/9.0/ccp4 (-> ccp4-9) +``` + +Bundled SHELX and ARP/wARP still need their own academic registrations. diff --git a/ccp4/build b/ccp4/build new file mode 100755 index 0000000..011b0c1 --- /dev/null +++ b/ccp4/build @@ -0,0 +1,57 @@ +#!/usr/bin/env modbuild + +# Custom prep instead of urls: in config.yaml — modbuild's builtin curl has +# no retry/resume, so an interrupted 3.7 GB download restarts from zero and +# leaves a partial file the next prep reuses silently. wget --continue +# resumes it, and gzip -t gates the build on an intact tarball. +# sid= in the URL is session-generated on ccp4.ac.uk and expires; on download +# failure get a fresh link from https://www.ccp4.ac.uk/download/ (package +# "CCP4 + SHELX + ARP/wARP", Linux x86_64), or pre-stage the tarball by hand +# in $PMODULES_DISTFILESDIR under the name below. +pbuild::prep() { + local -r tarball="${PMODULES_DISTFILESDIR}/ccp4-${V}-shelx-arp-x86_64.tar.gz" + local -r url='https://www.ccp4.ac.uk/download/download_file.php?os=linux&pkg=ccp4-shelx-arp-x86_64&sid=09b78d1369f605f517efa393f88d2bae713ad0b6' + if ! gzip -t "${tarball}" 2>/dev/null; then + mkdir -p "${PMODULES_DISTFILESDIR}" + wget --continue --tries=20 --waitretry=30 \ + --output-document="${tarball}" "${url}" || \ + std::die 42 "ccp4: download failed — quota/disk full, or expired sid (fetch a fresh link from https://www.ccp4.ac.uk/download/)" + gzip -t "${tarball}" || \ + std::die 42 "ccp4: tarball fails gzip -t after download — re-run to resume, or rm it to restart" + fi +} + +pbuild::configure() { + : +} + +pbuild::compile() { + : +} + +pbuild::install() { + cd "${PREFIX}" + # modbuild has no errexit in hooks — without the die-guards a failed step + # still ends in a "done" build with a broken PREFIX (happened 2026-08-18 + # with a truncated cached tarball). + # untar straight from the distfiles cache — no point copying 3.7 GB into + # SRC_DIR first (prep already verified it with gzip -t) + tar -xzf "${PMODULES_DISTFILESDIR}/ccp4-${V}-shelx-arp-x86_64.tar.gz" || \ + std::die 42 "ccp4: untar failed — rm the tarball from the distfiles dir and re-run" + # the 9-series tarball unpacks to ccp4-9 (NOT ccp4-9.0 as the install doc + # implies) — glob instead of guessing the name + local -a top=( ccp4-*/ ) + [[ -x "${top[0]}BINARY.setup" ]] || \ + std::die 42 "ccp4: ${top[0]}BINARY.setup not found — vendor layout changed, inspect ${PREFIX}" + # Pre-agree the CCP4 licence so BINARY.setup skips its interactive prompt + # (downloading from ccp4.ac.uk already required accepting it). The printf + # answers any prompt a newer BINARY.setup might still ask; printf instead + # of `yes` because modbuild runs with pipefail and yes dies by SIGPIPE. + touch "${PREFIX}/.agree2ccp4v6" "${HOME}/.agree2ccp4v6" + printf 'y\ny\ny\n' | "./${top[0]}BINARY.setup" || \ + std::die 42 "ccp4: BINARY.setup failed" + # version-independent symlink so the modulefile survives 9.x updates + ln -sfn "${top[0]%/}" ccp4 + [[ -r ccp4/bin/ccp4.setup-sh ]] || \ + std::die 42 "ccp4: ccp4/bin/ccp4.setup-sh missing after setup — the modulefile would be broken" +} diff --git a/ccp4/files/config.yaml b/ccp4/files/config.yaml new file mode 100644 index 0000000..0c88ce7 --- /dev/null +++ b/ccp4/files/config.yaml @@ -0,0 +1,14 @@ +--- +format: 1 +ccp4: + defaults: + group: MX + overlay: base + relstage: unstable + # no urls: — download lives in build's pbuild::prep (wget --continue + + # gzip -t), because the builtin curl has no retry/resume for the 3.7 GB + # session-gated tarball + versions: + 9.0: + config: + relstage: unstable diff --git a/ccp4/modulefile b/ccp4/modulefile new file mode 100644 index 0000000..4984cdc --- /dev/null +++ b/ccp4/modulefile @@ -0,0 +1,19 @@ +#%Module1.0 + +module-whatis "CCP4: software suite for macromolecular X-ray crystallography" +module-url "https://www.ccp4.ac.uk/" +module-license "CCP4 licence; bundled SHELX and ARP/wARP need their own academic registration" +module-maintainer "Jiaxin Duan " + +module-help " +CCP4 suite (binary distribution bundled with SHELX and ARP/wARP) for +macromolecular crystallography: data processing, phasing, model building +and refinement (refmac5, ccp4i2, coot, shelx, arp/warp, ...). + +Loading this module sources ccp4.setup-sh, which sets CCP4, CBIN, +CCP4_SCR and PATH. +" + +# build's install() leaves a version-independent 'ccp4' symlink next to +# ccp4-, so this line survives 9.0.xxx updates +puts stdout "source $PREFIX/ccp4/bin/ccp4.setup-sh;"