ci: add benchmark workflow

.github/scripts/aggregate_benchmarks.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Aggregate benchmark JSON files by taking the median across runner attempts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+from pathlib import Path
+
+from compare_benchmarks import Benchmark, extract_benchmarks
+
+
+def collect_benchmarks(paths: list[Path]) -> dict[str, list[Benchmark]]:
+    collected: dict[str, list[Benchmark]] = {}
+    for path in paths:
+        for name, benchmark in extract_benchmarks(path).items():
+            collected.setdefault(name, []).append(benchmark)
+    return collected
+
+
+def aggregate(collected: dict[str, list[Benchmark]]) -> dict[str, dict[str, object]]:
+    aggregated: dict[str, dict[str, object]] = {}
+    for name, benchmarks in sorted(collected.items()):
+        values = [benchmark.value for benchmark in benchmarks]
+        unit = next((benchmark.unit for benchmark in benchmarks if benchmark.unit), "")
+        metric = next((benchmark.metric for benchmark in benchmarks if benchmark.metric), "value")
+        aggregated[name] = {
+            "value": statistics.median(values),
+            "unit": unit,
+            "metric": f"median-of-attempt-{metric}",
+            "attempts": len(values),
+            "attempt_values": values,
+        }
+    return aggregated
+
+
+def main_from_paths(input_dir: Path, output: Path) -> int:
+    paths = sorted(input_dir.rglob("*.json"))
+    if not paths:
+        raise ValueError(f"No benchmark JSON files found in {input_dir}")
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(
+        json.dumps(aggregate(collect_benchmarks(paths)), indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-dir", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    args = parser.parse_args()
+    return main_from_paths(input_dir=args.input_dir, output=args.output)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/compare_benchmarks.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+"""Compare benchmark JSON files and write a GitHub Actions summary."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Benchmark:
+    name: str
+    value: float
+    unit: str
+    metric: str = "value"
+
+
+@dataclass(frozen=True)
+class Comparison:
+    name: str
+    baseline: float
+    current: float
+    delta_percent: float
+    unit: str
+    metric: str
+    regressed: bool
+
+
+def _read_json(path: Path) -> Any:
+    with path.open("r", encoding="utf-8") as stream:
+        return json.load(stream)
+
+
+def _as_float(value: Any) -> float | None:
+    try:
+        result = float(value)
+    except (TypeError, ValueError):
+        return None
+    if math.isfinite(result):
+        return result
+    return None
+
+
+def _extract_hyperfine(data: dict[str, Any]) -> dict[str, Benchmark]:
+    benchmarks: dict[str, Benchmark] = {}
+    for result in data.get("results", []):
+        if not isinstance(result, dict):
+            continue
+        name = str(result.get("command") or result.get("name") or "").strip()
+        metric = "median"
+        value = _as_float(result.get(metric))
+        if value is None:
+            metric = "mean"
+            value = _as_float(result.get(metric))
+        if name and value is not None:
+            benchmarks[name] = Benchmark(name=name, value=value, unit="s", metric=metric)
+    return benchmarks
+
+
+def _extract_pytest_benchmark(data: dict[str, Any]) -> dict[str, Benchmark]:
+    benchmarks: dict[str, Benchmark] = {}
+    for benchmark in data.get("benchmarks", []):
+        if not isinstance(benchmark, dict):
+            continue
+
+        name = str(benchmark.get("fullname") or benchmark.get("name") or "").strip()
+        stats = benchmark.get("stats", {})
+        value = None
+        metric = "median"
+        if isinstance(stats, dict):
+            value = _as_float(stats.get(metric))
+            if value is None:
+                metric = "mean"
+                value = _as_float(stats.get(metric))
+        if name and value is not None:
+            benchmarks[name] = Benchmark(name=name, value=value, unit="s", metric=metric)
+    return benchmarks
+
+
+def _extract_simple_mapping(data: dict[str, Any]) -> dict[str, Benchmark]:
+    benchmarks: dict[str, Benchmark] = {}
+
+    for name, raw_value in data.items():
+        if name in {"version", "context", "commit", "timestamp"}:
+            continue
+
+        value = _as_float(raw_value)
+        unit = ""
+        metric = "value"
+        if value is None and isinstance(raw_value, dict):
+            value = _as_float(raw_value.get("value"))
+            unit = str(raw_value.get("unit") or "")
+            metric = str(raw_value.get("metric") or "value")
+
+        if value is not None:
+            benchmarks[str(name)] = Benchmark(name=str(name), value=value, unit=unit, metric=metric)
+
+    return benchmarks
+
+
+def extract_benchmarks(path: Path) -> dict[str, Benchmark]:
+    data = _read_json(path)
+    if not isinstance(data, dict):
+        raise ValueError(f"{path} must contain a JSON object")
+
+    extractors = (_extract_hyperfine, _extract_pytest_benchmark, _extract_simple_mapping)
+    for extractor in extractors:
+        benchmarks = extractor(data)
+        if benchmarks:
+            return benchmarks
+
+    raise ValueError(f"No supported benchmark entries found in {path}")
+
+
+def compare_benchmarks(
+    baseline: dict[str, Benchmark],
+    current: dict[str, Benchmark],
+    threshold_percent: float,
+    higher_is_better: bool,
+) -> tuple[list[Comparison], list[str], list[str]]:
+    comparisons: list[Comparison] = []
+    missing_in_current: list[str] = []
+    new_in_current: list[str] = []
+
+    for name, baseline_benchmark in sorted(baseline.items()):
+        current_benchmark = current.get(name)
+        if current_benchmark is None:
+            missing_in_current.append(name)
+            continue
+
+        if baseline_benchmark.value == 0:
+            delta_percent = 0.0
+        else:
+            delta_percent = (
+                (current_benchmark.value - baseline_benchmark.value)
+                / abs(baseline_benchmark.value)
+                * 100
+            )
+
+        if higher_is_better:
+            regressed = delta_percent <= -threshold_percent
+        else:
+            regressed = delta_percent >= threshold_percent
+
+        comparisons.append(
+            Comparison(
+                name=name,
+                baseline=baseline_benchmark.value,
+                current=current_benchmark.value,
+                delta_percent=delta_percent,
+                unit=current_benchmark.unit or baseline_benchmark.unit,
+                metric=current_benchmark.metric,
+                regressed=regressed,
+            )
+        )
+
+    for name in sorted(set(current) - set(baseline)):
+        new_in_current.append(name)
+
+    return comparisons, missing_in_current, new_in_current
+
+
+def _format_value(value: float, unit: str) -> str:
+    suffix = f" {unit}" if unit else ""
+    return f"{value:.6g}{suffix}"
+
+
+def write_summary(
+    path: Path,
+    comparisons: list[Comparison],
+    missing_in_current: list[str],
+    new_in_current: list[str],
+    threshold_percent: float,
+    higher_is_better: bool,
+) -> None:
+    regressions = [comparison for comparison in comparisons if comparison.regressed]
+    direction = "higher is better" if higher_is_better else "lower is better"
+    sorted_comparisons = sorted(comparisons, key=lambda comparison: comparison.name)
+
+    lines = [
+        "<!-- bw-benchmark-comment -->",
+        "## Benchmark comparison",
+        "",
+        f"Threshold: {threshold_percent:g}% ({direction}).",
+    ]
+    lines.append("")
+
+    if regressions:
+        lines.extend(
+            [
+                f"{len(regressions)} benchmark(s) regressed beyond the configured threshold.",
+                "",
+                "| Benchmark | Baseline | Current | Change |",
+                "| --- | ---: | ---: | ---: |",
+            ]
+        )
+        for comparison in regressions:
+            lines.append(
+                "| "
+                f"{comparison.name} | "
+                f"{_format_value(comparison.baseline, comparison.unit)} | "
+                f"{_format_value(comparison.current, comparison.unit)} | "
+                f"{comparison.delta_percent:+.2f}% |"
+            )
+    else:
+        lines.append("No benchmark regression exceeded the configured threshold.")
+
+    if sorted_comparisons:
+        lines.extend(
+            [
+                "",
+                "<details>",
+                "<summary>All benchmark results</summary>",
+                "",
+                "| Benchmark | Baseline | Current | Change | Status |",
+                "| --- | ---: | ---: | ---: | --- |",
+            ]
+        )
+        for comparison in sorted_comparisons:
+            status = "regressed" if comparison.regressed else "ok"
+            lines.append(
+                "| "
+                f"{comparison.name} | "
+                f"{_format_value(comparison.baseline, comparison.unit)} | "
+                f"{_format_value(comparison.current, comparison.unit)} | "
+                f"{comparison.delta_percent:+.2f}% | "
+                f"{status} |"
+            )
+        lines.extend(["", "</details>"])
+
+    if missing_in_current:
+        lines.extend(["", "Missing benchmarks in the current run:"])
+        lines.extend(f"- `{name}`" for name in missing_in_current)
+
+    if new_in_current:
+        lines.extend(["", "New benchmarks in the current run:"])
+        lines.extend(f"- `{name}`" for name in new_in_current)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--baseline", required=True, type=Path)
+    parser.add_argument("--current", required=True, type=Path)
+    parser.add_argument("--summary", required=True, type=Path)
+    parser.add_argument("--threshold-percent", required=True, type=float)
+    parser.add_argument("--higher-is-better", action="store_true")
+    args = parser.parse_args()
+
+    baseline = extract_benchmarks(args.baseline)
+    current = extract_benchmarks(args.current)
+    comparisons, missing_in_current, new_in_current = compare_benchmarks(
+        baseline=baseline,
+        current=current,
+        threshold_percent=args.threshold_percent,
+        higher_is_better=args.higher_is_better,
+    )
+
+    write_summary(
+        path=args.summary,
+        comparisons=comparisons,
+        missing_in_current=missing_in_current,
+        new_in_current=new_in_current,
+        threshold_percent=args.threshold_percent,
+        higher_is_better=args.higher_is_better,
+    )
+
+    return 1 if any(comparison.regressed for comparison in comparisons) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/run_benchmarks.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p benchmark-results
+benchmark_json="${BENCHMARK_JSON:-benchmark-results/current.json}"
+benchmark_dir="${BENCHMARK_HYPERFINE_DIR:-tests/benchmarks/hyperfine}"
+
+shopt -s nullglob
+benchmark_scripts=("$benchmark_dir"/benchmark_*.sh)
+shopt -u nullglob
+
+if [ "${#benchmark_scripts[@]}" -eq 0 ]; then
+  echo "No hyperfine benchmark scripts matching benchmark_*.sh found in $benchmark_dir" >&2
+  exit 1
+fi
+
+echo "Benchmark Python: $(command -v python)"
+python -c 'import sys; print(sys.version)'
+
+commands=()
+names=()
+for benchmark_script in "${benchmark_scripts[@]}"; do
+  title="$(sed -n 's/^# BENCHMARK_TITLE:[[:space:]]*//p' "$benchmark_script" | head -n 1)"
+  if [ -z "$title" ]; then
+    title="$(basename "$benchmark_script" .sh)"
+  fi
+  echo "Preflight benchmark script: $benchmark_script"
+  bash "$benchmark_script"
+  names+=(--command-name "$title")
+  commands+=("bash $(printf "%q" "$benchmark_script")")
+done
+
+hyperfine \
+  --show-output \
+  --warmup 1 \
+  --runs 5 \
+  "${names[@]}" \
+  --export-json "$benchmark_json" \
+  "${commands[@]}"

.github/scripts/run_with_bec_servers.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Run a command with BEC e2e services available."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+import bec_lib
+from bec_ipython_client import BECIPythonClient
+from bec_lib.redis_connector import RedisConnector
+from bec_lib.service_config import ServiceConfig, ServiceConfigModel
+from redis import Redis
+
+
+def _wait_for_redis(host: str, port: int) -> None:
+    client = Redis(host=host, port=port)
+    deadline = time.monotonic() + 10
+    while time.monotonic() < deadline:
+        try:
+            if client.ping():
+                return
+        except Exception:
+            time.sleep(0.1)
+    raise RuntimeError(f"Redis did not start on {host}:{port}")
+
+
+def _start_redis(files_path: Path, host: str, port: int) -> subprocess.Popen:
+    redis_server = shutil.which("redis-server")
+    if redis_server is None:
+        raise RuntimeError("redis-server executable not found")
+
+    return subprocess.Popen(
+        [
+            redis_server,
+            "--bind",
+            host,
+            "--port",
+            str(port),
+            "--save",
+            "",
+            "--appendonly",
+            "no",
+            "--dir",
+            str(files_path),
+        ]
+    )
+
+
+def _write_configs(files_path: Path, host: str, port: int) -> Path:
+    test_config = files_path / "test_config.yaml"
+    services_config = files_path / "services_config.yaml"
+
+    bec_lib_path = Path(bec_lib.__file__).resolve().parent
+    shutil.copyfile(bec_lib_path / "tests" / "test_config.yaml", test_config)
+
+    service_config = ServiceConfigModel(
+        redis={"host": host, "port": port}, file_writer={"base_path": str(files_path)}
+    )
+    services_config.write_text(service_config.model_dump_json(indent=4), encoding="utf-8")
+    return services_config
+
+
+def _load_demo_config(services_config: Path) -> None:
+    bec = BECIPythonClient(ServiceConfig(services_config), RedisConnector, forced=True)
+    bec.start()
+    try:
+        bec.config.load_demo_config()
+    finally:
+        bec.shutdown()
+        bec._client._reset_singleton()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("command", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+
+    if args.command[:1] == ["--"]:
+        args.command = args.command[1:]
+    if not args.command:
+        raise ValueError("No command provided")
+
+    host = "127.0.0.1"
+    port = 6379
+
+    with tempfile.TemporaryDirectory(prefix="bec-benchmark-") as tmp:
+        files_path = Path(tmp)
+        services_config = _write_configs(files_path, host, port)
+        redis_process = _start_redis(files_path, host, port)
+        processes = None
+        service_handler = None
+        try:
+            _wait_for_redis(host, port)
+
+            from bec_server.bec_server_utils.service_handler import ServiceHandler
+
+            service_handler = ServiceHandler(
+                bec_path=files_path, config_path=services_config, interface="subprocess"
+            )
+            processes = service_handler.start()
+            _load_demo_config(services_config)
+
+            env = os.environ.copy()
+            env["BEC_BENCHMARK_CONFIG"] = str(services_config)
+            env["BEC_BENCHMARK_SERVER"] = f"{host}:{port}"
+            return subprocess.run(args.command, env=env, check=False).returncode
+        finally:
+            if service_handler is not None and processes is not None:
+                service_handler.stop(processes)
+            redis_process.terminate()
+            try:
+                redis_process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                redis_process.kill()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/benchmark.yml

@@ -0,0 +1,241 @@
+name: BW Benchmarks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - ci/benchmark_workflow
+  workflow_dispatch:
+    inputs:
+      benchmark_command:
+        description: "Command that writes the benchmark JSON file"
+        required: false
+        type: string
+      threshold_percent:
+        description: "Regression threshold in percent"
+        required: false
+        type: string
+      higher_is_better:
+        description: "Set to true for throughput-style benchmarks"
+        required: false
+        type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  BENCHMARK_JSON: benchmark-results/current.json
+  BENCHMARK_BASELINE_JSON: gh-pages-benchmark-data/benchmarks/latest.json
+  BENCHMARK_SUMMARY: benchmark-results/summary.md
+  BENCHMARK_COMMAND: ${{ inputs.benchmark_command || 'bash .github/scripts/run_benchmarks.sh' }}
+  BENCHMARK_THRESHOLD_PERCENT: ${{ inputs.threshold_percent || '20' }}
+  BENCHMARK_HIGHER_IS_BETTER: ${{ inputs.higher_is_better || false }}
+
+jobs:
+  check_pr_status:
+    uses: ./.github/workflows/check_pr.yml
+
+  benchmark_attempt:
+    needs: check_pr_status
+    if: needs.check_pr_status.outputs.branch-pr == ''
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        attempt: [1, 2, 3]
+
+    env:
+      BENCHMARK_JSON: benchmark-results/current-${{ matrix.attempt }}.json
+      BEC_CORE_BRANCH: main
+      OPHYD_DEVICES_BRANCH: main
+      PLUGIN_REPO_BRANCH: main
+      QTWEBENGINE_DISABLE_SANDBOX: 1
+      QT_QPA_PLATFORM: "offscreen"
+
+    steps:
+      - name: Checkout BEC Widgets
+        uses: actions/checkout@v4
+        with:
+          repository: bec-project/bec_widgets
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          auto-activate-base: true
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libgl1 libegl1 x11-utils libxkbcommon-x11-0 libdbus-1-3 xvfb
+          sudo apt-get -y install libnss3 libxdamage1 libasound2t64 libatomic1 libxcursor1
+          sudo apt-get -y install ttyd hyperfine redis-server
+
+      - name: Install full e2e environment
+        run: |
+          echo -e "\033[35;1m Using branch $BEC_CORE_BRANCH of BEC CORE \033[0;m";
+          git clone --branch "$BEC_CORE_BRANCH" https://github.com/bec-project/bec.git
+          echo -e "\033[35;1m Using branch $OPHYD_DEVICES_BRANCH of OPHYD_DEVICES \033[0;m";
+          git clone --branch "$OPHYD_DEVICES_BRANCH" https://github.com/bec-project/ophyd_devices.git
+          export OHPYD_DEVICES_PATH=$PWD/ophyd_devices
+          echo -e "\033[35;1m Using branch $PLUGIN_REPO_BRANCH of bec_testing_plugin \033[0;m";
+          git clone --branch "$PLUGIN_REPO_BRANCH" https://github.com/bec-project/bec_testing_plugin.git
+          cd ./bec
+          conda create -q -n test-environment python=3.11
+          conda activate test-environment
+          source ./bin/install_bec_dev.sh -t
+          cd ../
+          python -m pip install -e ./ophyd_devices -e .[dev,pyside6] -e ./bec_testing_plugin pytest-benchmark
+
+          mkdir -p "$(dirname "$BENCHMARK_JSON")"
+          python .github/scripts/run_with_bec_servers.py -- bash -lc "$BENCHMARK_COMMAND"
+          test -s "$BENCHMARK_JSON"
+
+      - name: Upload benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bw-benchmark-json-${{ matrix.attempt }}
+          path: ${{ env.BENCHMARK_JSON }}
+
+  benchmark:
+    needs: [check_pr_status, benchmark_attempt]
+    if: needs.check_pr_status.outputs.branch-pr == ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout BEC Widgets
+        uses: actions/checkout@v4
+        with:
+          repository: bec-project/bec_widgets
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Download benchmark attempts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: bw-benchmark-json-*
+          path: benchmark-results/attempts
+          merge-multiple: true
+
+      - name: Aggregate benchmark attempts
+        run: |
+          python .github/scripts/aggregate_benchmarks.py \
+            --input-dir benchmark-results/attempts \
+            --output "$BENCHMARK_JSON"
+
+      - name: Upload aggregate benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bw-benchmark-json
+          path: ${{ env.BENCHMARK_JSON }}
+
+      - name: Fetch gh-pages benchmark data
+        if: github.event_name == 'pull_request'
+        run: |
+          if git ls-remote --exit-code --heads origin gh-pages; then
+            git clone --depth=1 --branch gh-pages "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" gh-pages-benchmark-data
+          else
+            mkdir -p gh-pages-benchmark-data
+          fi
+
+      - name: Compare with latest gh-pages benchmark
+        id: compare
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        run: |
+          if [ ! -s "$BENCHMARK_BASELINE_JSON" ]; then
+            mkdir -p "$(dirname "$BENCHMARK_SUMMARY")"
+            {
+              echo "<!-- bw-benchmark-comment -->"
+              echo "## Benchmark comparison"
+              echo
+              echo "No benchmark baseline was found on gh-pages."
+            } > "$BENCHMARK_SUMMARY"
+            exit 0
+          fi
+
+          args=(
+            --baseline "$BENCHMARK_BASELINE_JSON"
+            --current "$BENCHMARK_JSON"
+            --summary "$BENCHMARK_SUMMARY"
+            --threshold-percent "$BENCHMARK_THRESHOLD_PERCENT"
+          )
+
+          if [ "$BENCHMARK_HIGHER_IS_BETTER" = "true" ]; then
+            args+=(--higher-is-better)
+          fi
+
+          set +e
+          python .github/scripts/compare_benchmarks.py "${args[@]}"
+          status=$?
+          set -e
+
+          if [ ! -s "$BENCHMARK_SUMMARY" ]; then
+            mkdir -p "$(dirname "$BENCHMARK_SUMMARY")"
+            {
+              echo "<!-- bw-benchmark-comment -->"
+              echo "## Benchmark comparison"
+              echo
+              echo "Benchmark comparison failed before writing a summary."
+            } > "$BENCHMARK_SUMMARY"
+          fi
+
+          exit "$status"
+
+      - name: Find existing benchmark PR comment
+        if: github.event_name == 'pull_request'
+        id: fc
+        uses: peter-evans/find-comment@v3
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+          body-includes: "<!-- bw-benchmark-comment -->"
+
+      - name: Create or update benchmark PR comment
+        if: github.event_name == 'pull_request'
+        uses: peter-evans/create-or-update-comment@v5
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-path: ${{ env.BENCHMARK_SUMMARY }}
+          edit-mode: replace
+
+      - name: Fail on benchmark regression
+        if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure'
+        run: exit 1
+
+      - name: Prepare gh-pages for publishing
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow')
+        run: |
+          if git ls-remote --exit-code --heads origin gh-pages; then
+            git fetch --depth=1 origin gh-pages
+            git worktree add gh-pages-benchmark-data FETCH_HEAD
+          else
+            git worktree add --detach gh-pages-benchmark-data
+            git -C gh-pages-benchmark-data checkout --orphan gh-pages
+            git -C gh-pages-benchmark-data rm -rf .
+          fi
+
+      - name: Publish benchmark data to gh-pages
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow')
+        working-directory: gh-pages-benchmark-data
+        run: |
+          mkdir -p benchmarks/history
+          cp "../$BENCHMARK_JSON" benchmarks/latest.json
+          cp "../$BENCHMARK_JSON" "benchmarks/history/${GITHUB_SHA}.json"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add benchmarks/latest.json "benchmarks/history/${GITHUB_SHA}.json"
+          git commit -m "Update BW benchmark data for ${GITHUB_SHA}" || exit 0
+          git push origin HEAD:gh-pages

tests/benchmarks/hyperfine/benchmark_import_bec_widgets.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# BENCHMARK_TITLE: Import bec_widgets
+set -euo pipefail
+
+python -c 'import bec_widgets; print(bec_widgets.__file__)'

tests/benchmarks/hyperfine/benchmark_launch_bec_with_companion.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# BENCHMARK_TITLE: BEC IPython client with companion app
+set -euo pipefail
+
+bec --nogui --post-startup-file tests/benchmarks/hyperfine/utils/exit_bec_startup.py

tests/benchmarks/hyperfine/benchmark_launch_bec_without_companion.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# BENCHMARK_TITLE: BEC IPython client without companion app
+set -euo pipefail
+
+bec --post-startup-file tests/benchmarks/hyperfine/utils/exit_bec_startup.py

tests/benchmarks/hyperfine/utils/exit_bec_startup.py

@@ -0,0 +1,5 @@
+import time
+
+_ip = get_ipython()
+_ip.confirm_exit = False
+_ip.ask_exit()