Jungfraujoch/docker/build_images.sh

#!/usr/bin/env bash
# Build (and optionally push) the jungfraujoch build-environment images, in parallel.
#
# Each docker/<variant>/Dockerfile is a *self-contained build environment* (toolchain, static Qt,
# static libdbus, static OpenSSL, Eigen, DIALS, XDS, Node, ...). It does NOT copy the project in --
# the source is mounted and built at run time (see build_in_rocky9.sh).
#
# The build context is docker/ rather than docker/<variant>/, because every variant COPYs the
# shared steps in docker/common/ (see common/README.md). That context is still only the Dockerfiles
# and those scripts; the huge repo is never sent to the Docker daemon.
#
# Images are tagged  gitea.psi.ch/leonarski_f/jfjoch_<variant>:<TAG>  (matches the CI runner images).
#
# Concurrency (JOBS):
#   JOBS controls how many image builds run at once (default 2). Each build internally runs
#   `make -j$(nproc)` for Qt/dbus, so J parallel builds ~= J*nproc compile threads and several GB of
#   RAM each at link time. JOBS=2 is a safe default that overlaps the (network-bound) base-image
#   pull / package-install / source-download phases across builds without thrashing the CPU. Bump to
#   4 for full parallelism on a big machine (enough RAM + disk for 4 CUDA bases), or JOBS=1 for serial.
#
# Usage:
#   docker/build_images.sh                     # all four, JOBS=2
#   docker/build_images.sh rocky9 rocky8       # subset
#   JOBS=4 docker/build_images.sh              # all four at once
#   JOBS=1 docker/build_images.sh              # serial
#   TAG=2610 docker/build_images.sh           # override the tag (default 2609)
#
# The tag has to match what .gitea/workflows/build_and_test.yml asks the runner for, or CI pulls a
# different image than the one just built. Every job there names its image explicitly, and all four
# are on the same tag (2609). Build a single variant with, e.g.:
#   TAG=2609 docker/build_images.sh ubuntu2404
#
# JOBS: static Qt needs ~15-20 GB of scratch per concurrent build on top of a 10-20 GB result, so
# four at once will exhaust a 200 GB disk. JOBS=1 is the safe choice on a box that also runs CI.
#   PUSH=1   docker/build_images.sh            # build (parallel) then push (serial; needs docker login)
#
# Parallel stdout would be an unreadable interleave, so each build streams to its own log file:
#   docker/build-logs/<variant>-<TAG>.log      (follow live with:  tail -f docker/build-logs/*.log)
set -euo pipefail

REGISTRY="${REGISTRY:-gitea.psi.ch/leonarski_f}"
TAG="${TAG:-2609}"
PUSH="${PUSH:-0}"
JOBS="${JOBS:-2}"

here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
logdir="$here/build-logs"
mkdir -p "$logdir"

all=(rocky8 rocky9 ubuntu2204 ubuntu2404)
variants=("$@"); [ "${#variants[@]}" -eq 0 ] && variants=("${all[@]}")

for v in "${variants[@]}"; do
    [ -f "$here/$v/Dockerfile" ] || { echo "!! no Dockerfile for variant '$v' ($here/$v/Dockerfile)"; exit 1; }
done

# One image build. Records OK/FAIL to a status file so the parent can summarise after wait; never
# returns non-zero itself, so the job-pool's `wait -n` under `set -e` stays happy.
build_one() {
    local v="$1"
    local img="$REGISTRY/jfjoch_$v:$TAG"
    local log="$logdir/${v}-${TAG}.log"
    if docker build --pull -t "$img" -f "$here/$v/Dockerfile" "$here" >"$log" 2>&1; then
        echo OK   > "$logdir/${v}-${TAG}.status"
    else
        echo FAIL > "$logdir/${v}-${TAG}.status"
    fi
}

echo "==> building ${#variants[@]} image(s): ${variants[*]}"
echo "==> JOBS=$JOBS  TAG=$TAG  REGISTRY=$REGISTRY"
echo "==> logs in $logdir/  (follow: tail -f $logdir/*.log)"
echo

# Launch with a concurrency cap of JOBS. `wait -n` reaps exactly one finished build before the next
# is started once we are at the cap; the trailing `wait` drains the rest.
running=0
for v in "${variants[@]}"; do
    if [ "$running" -ge "$JOBS" ]; then wait -n || true; running=$((running - 1)); fi
    : > "$logdir/${v}-${TAG}.status"          # clear any stale status from a previous run
    echo "==> [$v] started -> $REGISTRY/jfjoch_$v:$TAG"
    build_one "$v" &
    running=$((running + 1))
done
wait

echo
echo "===================== build summary ====================="
rc=0; ok_variants=()
for v in "${variants[@]}"; do
    s="$(cat "$logdir/${v}-${TAG}.status" 2>/dev/null || echo '??')"
    printf "  %-12s %s\n" "$v" "$s"
    if [ "$s" = OK ]; then ok_variants+=("$v"); else rc=1; fi
done
[ "$rc" -eq 0 ] || echo "!! one or more builds FAILED -- see the per-variant logs above"

echo
if [ "$PUSH" = "1" ]; then
    for v in "${ok_variants[@]}"; do
        img="$REGISTRY/jfjoch_$v:$TAG"
        echo "==> push $img"
        docker push "$img"
    done
else
    echo "Built locally. To push the successful ones (after 'docker login gitea.psi.ch'):"
    for v in "${ok_variants[@]}"; do echo "  docker push $REGISTRY/jfjoch_$v:$TAG"; done
fi

exit "$rc"