#!/usr/bin/env bash # Build (and optionally push) the jungfraujoch build-environment images, in parallel. # # Each docker//Dockerfile is a *self-contained build environment* (toolchain, static Qt, # static libdbus, static OpenSSL, Eigen, DIALS, XDS, Node, ...). It does NOT copy the project in -- # the source is mounted and built at run time (see build_in_rocky9.sh). # # The build context is docker/ rather than docker//, because every variant COPYs the # shared steps in docker/common/ (see common/README.md). That context is still only the Dockerfiles # and those scripts; the huge repo is never sent to the Docker daemon. # # Images are tagged gitea.psi.ch/leonarski_f/jfjoch_: (matches the CI runner images). # # Concurrency (JOBS): # JOBS controls how many image builds run at once (default 2). Each build internally runs # `make -j$(nproc)` for Qt/dbus, so J parallel builds ~= J*nproc compile threads and several GB of # RAM each at link time. JOBS=2 is a safe default that overlaps the (network-bound) base-image # pull / package-install / source-download phases across builds without thrashing the CPU. Bump to # 4 for full parallelism on a big machine (enough RAM + disk for 4 CUDA bases), or JOBS=1 for serial. # # Usage: # docker/build_images.sh # all four, JOBS=2 # docker/build_images.sh rocky9 rocky8 # subset # JOBS=4 docker/build_images.sh # all four at once # JOBS=1 docker/build_images.sh # serial # TAG=2610 docker/build_images.sh # override the tag (default 2609) # # The tag has to match what .gitea/workflows/build_and_test.yml asks the runner for, or CI pulls a # different image than the one just built. Every job there names its image explicitly, and all four # are on the same tag (2609). Build a single variant with, e.g.: # TAG=2609 docker/build_images.sh ubuntu2404 # # JOBS: static Qt needs ~15-20 GB of scratch per concurrent build on top of a 10-20 GB result, so # four at once will exhaust a 200 GB disk. JOBS=1 is the safe choice on a box that also runs CI. # PUSH=1 docker/build_images.sh # build (parallel) then push (serial; needs docker login) # # Parallel stdout would be an unreadable interleave, so each build streams to its own log file: # docker/build-logs/-.log (follow live with: tail -f docker/build-logs/*.log) set -euo pipefail REGISTRY="${REGISTRY:-gitea.psi.ch/leonarski_f}" TAG="${TAG:-2609}" PUSH="${PUSH:-0}" JOBS="${JOBS:-2}" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" logdir="$here/build-logs" mkdir -p "$logdir" all=(rocky8 rocky9 ubuntu2204 ubuntu2404) variants=("$@"); [ "${#variants[@]}" -eq 0 ] && variants=("${all[@]}") for v in "${variants[@]}"; do [ -f "$here/$v/Dockerfile" ] || { echo "!! no Dockerfile for variant '$v' ($here/$v/Dockerfile)"; exit 1; } done # One image build. Records OK/FAIL to a status file so the parent can summarise after wait; never # returns non-zero itself, so the job-pool's `wait -n` under `set -e` stays happy. build_one() { local v="$1" local img="$REGISTRY/jfjoch_$v:$TAG" local log="$logdir/${v}-${TAG}.log" if docker build --pull -t "$img" -f "$here/$v/Dockerfile" "$here" >"$log" 2>&1; then echo OK > "$logdir/${v}-${TAG}.status" else echo FAIL > "$logdir/${v}-${TAG}.status" fi } echo "==> building ${#variants[@]} image(s): ${variants[*]}" echo "==> JOBS=$JOBS TAG=$TAG REGISTRY=$REGISTRY" echo "==> logs in $logdir/ (follow: tail -f $logdir/*.log)" echo # Launch with a concurrency cap of JOBS. `wait -n` reaps exactly one finished build before the next # is started once we are at the cap; the trailing `wait` drains the rest. running=0 for v in "${variants[@]}"; do if [ "$running" -ge "$JOBS" ]; then wait -n || true; running=$((running - 1)); fi : > "$logdir/${v}-${TAG}.status" # clear any stale status from a previous run echo "==> [$v] started -> $REGISTRY/jfjoch_$v:$TAG" build_one "$v" & running=$((running + 1)) done wait echo echo "===================== build summary =====================" rc=0; ok_variants=() for v in "${variants[@]}"; do s="$(cat "$logdir/${v}-${TAG}.status" 2>/dev/null || echo '??')" printf " %-12s %s\n" "$v" "$s" if [ "$s" = OK ]; then ok_variants+=("$v"); else rc=1; fi done [ "$rc" -eq 0 ] || echo "!! one or more builds FAILED -- see the per-variant logs above" echo if [ "$PUSH" = "1" ]; then for v in "${ok_variants[@]}"; do img="$REGISTRY/jfjoch_$v:$TAG" echo "==> push $img" docker push "$img" done else echo "Built locally. To push the successful ones (after 'docker login gitea.psi.ch'):" for v in "${ok_variants[@]}"; do echo " docker push $REGISTRY/jfjoch_$v:$TAG"; done fi exit "$rc"