docker: add parallel image-build + in-container viewer-verify scripts

build_images.sh builds (and optionally pushes) the four build-environment images as gitea.psi.ch/leonarski_f/jfjoch_<variant>:<TAG>. Runs a JOBS-capped parallel job pool (default 2; each build internally runs make -j$(nproc), so a low cap avoids CPU/RAM thrash while still overlapping the network-bound base pull / install / download phases), streams each build to docker/build-logs/<variant>-<TAG>.log, prints an OK/FAIL summary, pushes only the ones that succeeded, and exits non-zero on any failure. Context is the tiny per-variant dir (no COPY in the Dockerfiles), so the repo is never sent to the daemon. build-logs/ is already covered by the root .gitignore build*/ rule. build_in_rocky9.sh builds the viewer (JFJOCH_VIEWER_ONLY) inside a chosen variant's image and ldd-checks that the dbus/systemd/glib/selinux tail is gone -- the quick verification for the static-libdbus + glib-off changes. The repo is mounted but the build tree lives in the container's /tmp, so nothing root-owned lands in the working copy. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 15:43:18 +02:00
parent 5baf2a2cac
commit 254bd82d52
2 changed files with 146 additions and 0 deletions
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Build (and optionally push) the jungfraujoch build-environment images, in parallel.
+#
+# Each docker/<variant>/Dockerfile is a *self-contained build environment* (toolchain, static Qt,
+# static libdbus, static OpenSSL, Eigen, DIALS, XDS, Node, ...). It does NOT copy the project in --
+# the source is mounted and built at run time (see build_in_rocky9.sh). So the build context is just
+# the per-variant directory; the huge repo is never sent to the Docker daemon.
+#
+# Images are tagged  gitea.psi.ch/leonarski_f/jfjoch_<variant>:<TAG>  (matches the CI runner images).
+#
+# Concurrency (JOBS):
+#   JOBS controls how many image builds run at once (default 2). Each build internally runs
+#   `make -j$(nproc)` for Qt/dbus, so J parallel builds ~= J*nproc compile threads and several GB of
+#   RAM each at link time. JOBS=2 is a safe default that overlaps the (network-bound) base-image
+#   pull / package-install / source-download phases across builds without thrashing the CPU. Bump to
+#   4 for full parallelism on a big machine (enough RAM + disk for 4 CUDA bases), or JOBS=1 for serial.
+#
+# Usage:
+#   docker/build_images.sh                     # all four, JOBS=2
+#   docker/build_images.sh rocky9 rocky8       # subset
+#   JOBS=4 docker/build_images.sh              # all four at once
+#   JOBS=1 docker/build_images.sh              # serial
+#   TAG=2607b docker/build_images.sh           # override the tag (default 2607b)
+#   PUSH=1   docker/build_images.sh            # build (parallel) then push (serial; needs docker login)
+#
+# Parallel stdout would be an unreadable interleave, so each build streams to its own log file:
+#   docker/build-logs/<variant>-<TAG>.log      (follow live with:  tail -f docker/build-logs/*.log)
+set -euo pipefail
+
+REGISTRY="${REGISTRY:-gitea.psi.ch/leonarski_f}"
+TAG="${TAG:-2607b}"
+PUSH="${PUSH:-0}"
+JOBS="${JOBS:-2}"
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+logdir="$here/build-logs"
+mkdir -p "$logdir"
+
+all=(rocky8 rocky9 ubuntu2204 ubuntu2404)
+variants=("$@"); [ "${#variants[@]}" -eq 0 ] && variants=("${all[@]}")
+
+for v in "${variants[@]}"; do
+    [ -f "$here/$v/Dockerfile" ] || { echo "!! no Dockerfile for variant '$v' ($here/$v/Dockerfile)"; exit 1; }
+done
+
+# One image build. Records OK/FAIL to a status file so the parent can summarise after wait; never
+# returns non-zero itself, so the job-pool's `wait -n` under `set -e` stays happy.
+build_one() {
+    local v="$1"
+    local img="$REGISTRY/jfjoch_$v:$TAG"
+    local log="$logdir/${v}-${TAG}.log"
+    if docker build --pull -t "$img" "$here/$v" >"$log" 2>&1; then
+        echo OK   > "$logdir/${v}-${TAG}.status"
+    else
+        echo FAIL > "$logdir/${v}-${TAG}.status"
+    fi
+}
+
+echo "==> building ${#variants[@]} image(s): ${variants[*]}"
+echo "==> JOBS=$JOBS  TAG=$TAG  REGISTRY=$REGISTRY"
+echo "==> logs in $logdir/  (follow: tail -f $logdir/*.log)"
+echo
+
+# Launch with a concurrency cap of JOBS. `wait -n` reaps exactly one finished build before the next
+# is started once we are at the cap; the trailing `wait` drains the rest.
+running=0
+for v in "${variants[@]}"; do
+    if [ "$running" -ge "$JOBS" ]; then wait -n || true; running=$((running - 1)); fi
+    : > "$logdir/${v}-${TAG}.status"          # clear any stale status from a previous run
+    echo "==> [$v] started -> $REGISTRY/jfjoch_$v:$TAG"
+    build_one "$v" &
+    running=$((running + 1))
+done
+wait
+
+echo
+echo "===================== build summary ====================="
+rc=0; ok_variants=()
+for v in "${variants[@]}"; do
+    s="$(cat "$logdir/${v}-${TAG}.status" 2>/dev/null || echo '??')"
+    printf "  %-12s %s\n" "$v" "$s"
+    if [ "$s" = OK ]; then ok_variants+=("$v"); else rc=1; fi
+done
+[ "$rc" -eq 0 ] || echo "!! one or more builds FAILED -- see the per-variant logs above"
+
+echo
+if [ "$PUSH" = "1" ]; then
+    for v in "${ok_variants[@]}"; do
+        img="$REGISTRY/jfjoch_$v:$TAG"
+        echo "==> push $img"
+        docker push "$img"
+    done
+else
+    echo "Built locally. To push the successful ones (after 'docker login gitea.psi.ch'):"
+    for v in "${ok_variants[@]}"; do echo "  docker push $REGISTRY/jfjoch_$v:$TAG"; done
+fi
+
+exit "$rc"
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Manually build the viewer inside one of the build-environment images and inspect its runtime .so
+# deps -- the quick way to confirm the static-libdbus + glib-off changes actually dropped the
+# dbus/systemd/glib/selinux tail. Defaults to the rocky9 image built by build_images.sh.
+#
+# The repo is mounted read-write at /workspace, but the CMake build tree lives in the container's
+# /tmp (thrown away with --rm), so nothing root-owned is left in your working copy. Building needs
+# network (FetchContent pulls curl/zstd/hdf5/...) but NOT a GPU.
+#
+# Usage:
+#   docker/build_in_rocky9.sh                 # build viewer in jfjoch_rocky9:$TAG, then ldd it
+#   docker/build_in_rocky9.sh rocky8          # use a different variant's image
+#   TAG=2607b docker/build_in_rocky9.sh
+set -euo pipefail
+
+REGISTRY="${REGISTRY:-gitea.psi.ch/leonarski_f}"
+TAG="${TAG:-2607b}"
+VARIANT="${1:-rocky9}"
+IMG="$REGISTRY/jfjoch_$VARIANT:$TAG"
+
+repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"   # repo root = docker/..
+
+echo "==> image $IMG"
+echo "==> repo  $repo  (mounted at /workspace; build tree in container /tmp)"
+
+docker run --rm \
+    -v "$repo":/workspace -w /workspace \
+    "$IMG" bash -lc '
+        set -eux
+        git config --global --add safe.directory /workspace || true
+        cmake -G Ninja -S /workspace -B /tmp/build \
+            -DJFJOCH_VIEWER_ONLY=ON -DJFJOCH_USE_CUDA=ON \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_CXX_FLAGS="-march=x86-64-v3" -DCMAKE_C_FLAGS="-march=x86-64-v3"
+        ninja -C /tmp/build -j"$(nproc)" jfjoch_viewer
+        bin=/tmp/build/viewer/jfjoch_viewer
+
+        echo "===================== ldd jfjoch_viewer ====================="
+        ldd "$bin" | sort
+        echo "===================== shed-lib check ========================"
+        # These should ALL be gone now (libpcre2-16 is Qt s own and legitimately stays).
+        shed="dbus|systemd|zstd|lz4|selinux|glib|gthread|libpcre\.|libpcre2-8|gcrypt|gpg-error"
+        if ldd "$bin" | grep -Ei "$shed"; then
+            echo ">>> UNEXPECTED: the above libs are still linked"
+        else
+            echo ">>> CLEAN: dbus / systemd / glib / selinux tail all gone"
+        fi
+    '