From d10252cfafeb51c10d92a1ad828f73bfeb50ba78 Mon Sep 17 00:00:00 2001 From: wakonig_k Date: Fri, 17 Apr 2026 10:07:22 +0200 Subject: [PATCH] ci: add benchmark workflow --- .github/scripts/aggregate_benchmarks.py | 60 ++++ .github/scripts/compare_benchmarks.py | 278 ++++++++++++++++++ .github/scripts/run_benchmarks.sh | 39 +++ .github/scripts/run_with_bec_servers.py | 124 ++++++++ .github/workflows/benchmark.yml | 241 +++++++++++++++ .../hyperfine/benchmark_import_bec_widgets.sh | 5 + .../benchmark_launch_bec_with_companion.sh | 5 + .../benchmark_launch_bec_without_companion.sh | 5 + .../hyperfine/utils/exit_bec_startup.py | 5 + 9 files changed, 762 insertions(+) create mode 100644 .github/scripts/aggregate_benchmarks.py create mode 100644 .github/scripts/compare_benchmarks.py create mode 100644 .github/scripts/run_benchmarks.sh create mode 100644 .github/scripts/run_with_bec_servers.py create mode 100644 .github/workflows/benchmark.yml create mode 100755 tests/benchmarks/hyperfine/benchmark_import_bec_widgets.sh create mode 100755 tests/benchmarks/hyperfine/benchmark_launch_bec_with_companion.sh create mode 100755 tests/benchmarks/hyperfine/benchmark_launch_bec_without_companion.sh create mode 100644 tests/benchmarks/hyperfine/utils/exit_bec_startup.py diff --git a/.github/scripts/aggregate_benchmarks.py b/.github/scripts/aggregate_benchmarks.py new file mode 100644 index 00000000..9baa581a --- /dev/null +++ b/.github/scripts/aggregate_benchmarks.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Aggregate benchmark JSON files by taking the median across runner attempts.""" + +from __future__ import annotations + +import argparse +import json +import statistics +from pathlib import Path + +from compare_benchmarks import Benchmark, extract_benchmarks + + +def collect_benchmarks(paths: list[Path]) -> dict[str, list[Benchmark]]: + collected: dict[str, list[Benchmark]] = {} + for path in paths: + for name, benchmark in extract_benchmarks(path).items(): + collected.setdefault(name, []).append(benchmark) + return collected + + +def aggregate(collected: dict[str, list[Benchmark]]) -> dict[str, dict[str, object]]: + aggregated: dict[str, dict[str, object]] = {} + for name, benchmarks in sorted(collected.items()): + values = [benchmark.value for benchmark in benchmarks] + unit = next((benchmark.unit for benchmark in benchmarks if benchmark.unit), "") + metric = next((benchmark.metric for benchmark in benchmarks if benchmark.metric), "value") + aggregated[name] = { + "value": statistics.median(values), + "unit": unit, + "metric": f"median-of-attempt-{metric}", + "attempts": len(values), + "attempt_values": values, + } + return aggregated + + +def main_from_paths(input_dir: Path, output: Path) -> int: + paths = sorted(input_dir.rglob("*.json")) + if not paths: + raise ValueError(f"No benchmark JSON files found in {input_dir}") + + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(aggregate(collect_benchmarks(paths)), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input-dir", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + return main_from_paths(input_dir=args.input_dir, output=args.output) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/compare_benchmarks.py b/.github/scripts/compare_benchmarks.py new file mode 100644 index 00000000..dc431b8b --- /dev/null +++ b/.github/scripts/compare_benchmarks.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Compare benchmark JSON files and write a GitHub Actions summary.""" + +from __future__ import annotations + +import argparse +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class Benchmark: + name: str + value: float + unit: str + metric: str = "value" + + +@dataclass(frozen=True) +class Comparison: + name: str + baseline: float + current: float + delta_percent: float + unit: str + metric: str + regressed: bool + + +def _read_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as stream: + return json.load(stream) + + +def _as_float(value: Any) -> float | None: + try: + result = float(value) + except (TypeError, ValueError): + return None + if math.isfinite(result): + return result + return None + + +def _extract_hyperfine(data: dict[str, Any]) -> dict[str, Benchmark]: + benchmarks: dict[str, Benchmark] = {} + for result in data.get("results", []): + if not isinstance(result, dict): + continue + name = str(result.get("command") or result.get("name") or "").strip() + metric = "median" + value = _as_float(result.get(metric)) + if value is None: + metric = "mean" + value = _as_float(result.get(metric)) + if name and value is not None: + benchmarks[name] = Benchmark(name=name, value=value, unit="s", metric=metric) + return benchmarks + + +def _extract_pytest_benchmark(data: dict[str, Any]) -> dict[str, Benchmark]: + benchmarks: dict[str, Benchmark] = {} + for benchmark in data.get("benchmarks", []): + if not isinstance(benchmark, dict): + continue + + name = str(benchmark.get("fullname") or benchmark.get("name") or "").strip() + stats = benchmark.get("stats", {}) + value = None + metric = "median" + if isinstance(stats, dict): + value = _as_float(stats.get(metric)) + if value is None: + metric = "mean" + value = _as_float(stats.get(metric)) + if name and value is not None: + benchmarks[name] = Benchmark(name=name, value=value, unit="s", metric=metric) + return benchmarks + + +def _extract_simple_mapping(data: dict[str, Any]) -> dict[str, Benchmark]: + benchmarks: dict[str, Benchmark] = {} + + for name, raw_value in data.items(): + if name in {"version", "context", "commit", "timestamp"}: + continue + + value = _as_float(raw_value) + unit = "" + metric = "value" + if value is None and isinstance(raw_value, dict): + value = _as_float(raw_value.get("value")) + unit = str(raw_value.get("unit") or "") + metric = str(raw_value.get("metric") or "value") + + if value is not None: + benchmarks[str(name)] = Benchmark(name=str(name), value=value, unit=unit, metric=metric) + + return benchmarks + + +def extract_benchmarks(path: Path) -> dict[str, Benchmark]: + data = _read_json(path) + if not isinstance(data, dict): + raise ValueError(f"{path} must contain a JSON object") + + extractors = (_extract_hyperfine, _extract_pytest_benchmark, _extract_simple_mapping) + for extractor in extractors: + benchmarks = extractor(data) + if benchmarks: + return benchmarks + + raise ValueError(f"No supported benchmark entries found in {path}") + + +def compare_benchmarks( + baseline: dict[str, Benchmark], + current: dict[str, Benchmark], + threshold_percent: float, + higher_is_better: bool, +) -> tuple[list[Comparison], list[str], list[str]]: + comparisons: list[Comparison] = [] + missing_in_current: list[str] = [] + new_in_current: list[str] = [] + + for name, baseline_benchmark in sorted(baseline.items()): + current_benchmark = current.get(name) + if current_benchmark is None: + missing_in_current.append(name) + continue + + if baseline_benchmark.value == 0: + delta_percent = 0.0 + else: + delta_percent = ( + (current_benchmark.value - baseline_benchmark.value) + / abs(baseline_benchmark.value) + * 100 + ) + + if higher_is_better: + regressed = delta_percent <= -threshold_percent + else: + regressed = delta_percent >= threshold_percent + + comparisons.append( + Comparison( + name=name, + baseline=baseline_benchmark.value, + current=current_benchmark.value, + delta_percent=delta_percent, + unit=current_benchmark.unit or baseline_benchmark.unit, + metric=current_benchmark.metric, + regressed=regressed, + ) + ) + + for name in sorted(set(current) - set(baseline)): + new_in_current.append(name) + + return comparisons, missing_in_current, new_in_current + + +def _format_value(value: float, unit: str) -> str: + suffix = f" {unit}" if unit else "" + return f"{value:.6g}{suffix}" + + +def write_summary( + path: Path, + comparisons: list[Comparison], + missing_in_current: list[str], + new_in_current: list[str], + threshold_percent: float, + higher_is_better: bool, +) -> None: + regressions = [comparison for comparison in comparisons if comparison.regressed] + direction = "higher is better" if higher_is_better else "lower is better" + sorted_comparisons = sorted(comparisons, key=lambda comparison: comparison.name) + + lines = [ + "", + "## Benchmark comparison", + "", + f"Threshold: {threshold_percent:g}% ({direction}).", + ] + lines.append("") + + if regressions: + lines.extend( + [ + f"{len(regressions)} benchmark(s) regressed beyond the configured threshold.", + "", + "| Benchmark | Baseline | Current | Change |", + "| --- | ---: | ---: | ---: |", + ] + ) + for comparison in regressions: + lines.append( + "| " + f"{comparison.name} | " + f"{_format_value(comparison.baseline, comparison.unit)} | " + f"{_format_value(comparison.current, comparison.unit)} | " + f"{comparison.delta_percent:+.2f}% |" + ) + else: + lines.append("No benchmark regression exceeded the configured threshold.") + + if sorted_comparisons: + lines.extend( + [ + "", + "
", + "All benchmark results", + "", + "| Benchmark | Baseline | Current | Change | Status |", + "| --- | ---: | ---: | ---: | --- |", + ] + ) + for comparison in sorted_comparisons: + status = "regressed" if comparison.regressed else "ok" + lines.append( + "| " + f"{comparison.name} | " + f"{_format_value(comparison.baseline, comparison.unit)} | " + f"{_format_value(comparison.current, comparison.unit)} | " + f"{comparison.delta_percent:+.2f}% | " + f"{status} |" + ) + lines.extend(["", "
"]) + + if missing_in_current: + lines.extend(["", "Missing benchmarks in the current run:"]) + lines.extend(f"- `{name}`" for name in missing_in_current) + + if new_in_current: + lines.extend(["", "New benchmarks in the current run:"]) + lines.extend(f"- `{name}`" for name in new_in_current) + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--current", required=True, type=Path) + parser.add_argument("--summary", required=True, type=Path) + parser.add_argument("--threshold-percent", required=True, type=float) + parser.add_argument("--higher-is-better", action="store_true") + args = parser.parse_args() + + baseline = extract_benchmarks(args.baseline) + current = extract_benchmarks(args.current) + comparisons, missing_in_current, new_in_current = compare_benchmarks( + baseline=baseline, + current=current, + threshold_percent=args.threshold_percent, + higher_is_better=args.higher_is_better, + ) + + write_summary( + path=args.summary, + comparisons=comparisons, + missing_in_current=missing_in_current, + new_in_current=new_in_current, + threshold_percent=args.threshold_percent, + higher_is_better=args.higher_is_better, + ) + + return 1 if any(comparison.regressed for comparison in comparisons) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/run_benchmarks.sh b/.github/scripts/run_benchmarks.sh new file mode 100644 index 00000000..63484b50 --- /dev/null +++ b/.github/scripts/run_benchmarks.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p benchmark-results +benchmark_json="${BENCHMARK_JSON:-benchmark-results/current.json}" +benchmark_dir="${BENCHMARK_HYPERFINE_DIR:-tests/benchmarks/hyperfine}" + +shopt -s nullglob +benchmark_scripts=("$benchmark_dir"/benchmark_*.sh) +shopt -u nullglob + +if [ "${#benchmark_scripts[@]}" -eq 0 ]; then + echo "No hyperfine benchmark scripts matching benchmark_*.sh found in $benchmark_dir" >&2 + exit 1 +fi + +echo "Benchmark Python: $(command -v python)" +python -c 'import sys; print(sys.version)' + +commands=() +names=() +for benchmark_script in "${benchmark_scripts[@]}"; do + title="$(sed -n 's/^# BENCHMARK_TITLE:[[:space:]]*//p' "$benchmark_script" | head -n 1)" + if [ -z "$title" ]; then + title="$(basename "$benchmark_script" .sh)" + fi + echo "Preflight benchmark script: $benchmark_script" + bash "$benchmark_script" + names+=(--command-name "$title") + commands+=("bash $(printf "%q" "$benchmark_script")") +done + +hyperfine \ + --show-output \ + --warmup 1 \ + --runs 5 \ + "${names[@]}" \ + --export-json "$benchmark_json" \ + "${commands[@]}" diff --git a/.github/scripts/run_with_bec_servers.py b/.github/scripts/run_with_bec_servers.py new file mode 100644 index 00000000..64bfb678 --- /dev/null +++ b/.github/scripts/run_with_bec_servers.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Run a command with BEC e2e services available.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import tempfile +import time +from pathlib import Path + +import bec_lib +from bec_ipython_client import BECIPythonClient +from bec_lib.redis_connector import RedisConnector +from bec_lib.service_config import ServiceConfig, ServiceConfigModel +from redis import Redis + + +def _wait_for_redis(host: str, port: int) -> None: + client = Redis(host=host, port=port) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + if client.ping(): + return + except Exception: + time.sleep(0.1) + raise RuntimeError(f"Redis did not start on {host}:{port}") + + +def _start_redis(files_path: Path, host: str, port: int) -> subprocess.Popen: + redis_server = shutil.which("redis-server") + if redis_server is None: + raise RuntimeError("redis-server executable not found") + + return subprocess.Popen( + [ + redis_server, + "--bind", + host, + "--port", + str(port), + "--save", + "", + "--appendonly", + "no", + "--dir", + str(files_path), + ] + ) + + +def _write_configs(files_path: Path, host: str, port: int) -> Path: + test_config = files_path / "test_config.yaml" + services_config = files_path / "services_config.yaml" + + bec_lib_path = Path(bec_lib.__file__).resolve().parent + shutil.copyfile(bec_lib_path / "tests" / "test_config.yaml", test_config) + + service_config = ServiceConfigModel( + redis={"host": host, "port": port}, file_writer={"base_path": str(files_path)} + ) + services_config.write_text(service_config.model_dump_json(indent=4), encoding="utf-8") + return services_config + + +def _load_demo_config(services_config: Path) -> None: + bec = BECIPythonClient(ServiceConfig(services_config), RedisConnector, forced=True) + bec.start() + try: + bec.config.load_demo_config() + finally: + bec.shutdown() + bec._client._reset_singleton() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + + if args.command[:1] == ["--"]: + args.command = args.command[1:] + if not args.command: + raise ValueError("No command provided") + + host = "127.0.0.1" + port = 6379 + + with tempfile.TemporaryDirectory(prefix="bec-benchmark-") as tmp: + files_path = Path(tmp) + services_config = _write_configs(files_path, host, port) + redis_process = _start_redis(files_path, host, port) + processes = None + service_handler = None + try: + _wait_for_redis(host, port) + + from bec_server.bec_server_utils.service_handler import ServiceHandler + + service_handler = ServiceHandler( + bec_path=files_path, config_path=services_config, interface="subprocess" + ) + processes = service_handler.start() + _load_demo_config(services_config) + + env = os.environ.copy() + env["BEC_BENCHMARK_CONFIG"] = str(services_config) + env["BEC_BENCHMARK_SERVER"] = f"{host}:{port}" + return subprocess.run(args.command, env=env, check=False).returncode + finally: + if service_handler is not None and processes is not None: + service_handler.stop(processes) + redis_process.terminate() + try: + redis_process.wait(timeout=10) + except subprocess.TimeoutExpired: + redis_process.kill() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..fe84522e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,241 @@ +name: BW Benchmarks + +on: + pull_request: + push: + branches: + - main + - ci/benchmark_workflow + workflow_dispatch: + inputs: + benchmark_command: + description: "Command that writes the benchmark JSON file" + required: false + type: string + threshold_percent: + description: "Regression threshold in percent" + required: false + type: string + higher_is_better: + description: "Set to true for throughput-style benchmarks" + required: false + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +env: + BENCHMARK_JSON: benchmark-results/current.json + BENCHMARK_BASELINE_JSON: gh-pages-benchmark-data/benchmarks/latest.json + BENCHMARK_SUMMARY: benchmark-results/summary.md + BENCHMARK_COMMAND: ${{ inputs.benchmark_command || 'bash .github/scripts/run_benchmarks.sh' }} + BENCHMARK_THRESHOLD_PERCENT: ${{ inputs.threshold_percent || '20' }} + BENCHMARK_HIGHER_IS_BETTER: ${{ inputs.higher_is_better || false }} + +jobs: + check_pr_status: + uses: ./.github/workflows/check_pr.yml + + benchmark_attempt: + needs: check_pr_status + if: needs.check_pr_status.outputs.branch-pr == '' + runs-on: ubuntu-latest + continue-on-error: true + defaults: + run: + shell: bash -el {0} + strategy: + fail-fast: false + matrix: + attempt: [1, 2, 3] + + env: + BENCHMARK_JSON: benchmark-results/current-${{ matrix.attempt }}.json + BEC_CORE_BRANCH: main + OPHYD_DEVICES_BRANCH: main + PLUGIN_REPO_BRANCH: main + QTWEBENGINE_DISABLE_SANDBOX: 1 + QT_QPA_PLATFORM: "offscreen" + + steps: + - name: Checkout BEC Widgets + uses: actions/checkout@v4 + with: + repository: bec-project/bec_widgets + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: true + python-version: "3.11" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libgl1 libegl1 x11-utils libxkbcommon-x11-0 libdbus-1-3 xvfb + sudo apt-get -y install libnss3 libxdamage1 libasound2t64 libatomic1 libxcursor1 + sudo apt-get -y install ttyd hyperfine redis-server + + - name: Install full e2e environment + run: | + echo -e "\033[35;1m Using branch $BEC_CORE_BRANCH of BEC CORE \033[0;m"; + git clone --branch "$BEC_CORE_BRANCH" https://github.com/bec-project/bec.git + echo -e "\033[35;1m Using branch $OPHYD_DEVICES_BRANCH of OPHYD_DEVICES \033[0;m"; + git clone --branch "$OPHYD_DEVICES_BRANCH" https://github.com/bec-project/ophyd_devices.git + export OHPYD_DEVICES_PATH=$PWD/ophyd_devices + echo -e "\033[35;1m Using branch $PLUGIN_REPO_BRANCH of bec_testing_plugin \033[0;m"; + git clone --branch "$PLUGIN_REPO_BRANCH" https://github.com/bec-project/bec_testing_plugin.git + cd ./bec + conda create -q -n test-environment python=3.11 + conda activate test-environment + source ./bin/install_bec_dev.sh -t + cd ../ + python -m pip install -e ./ophyd_devices -e .[dev,pyside6] -e ./bec_testing_plugin pytest-benchmark + + mkdir -p "$(dirname "$BENCHMARK_JSON")" + python .github/scripts/run_with_bec_servers.py -- bash -lc "$BENCHMARK_COMMAND" + test -s "$BENCHMARK_JSON" + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bw-benchmark-json-${{ matrix.attempt }} + path: ${{ env.BENCHMARK_JSON }} + + benchmark: + needs: [check_pr_status, benchmark_attempt] + if: needs.check_pr_status.outputs.branch-pr == '' + runs-on: ubuntu-latest + + steps: + - name: Checkout BEC Widgets + uses: actions/checkout@v4 + with: + repository: bec-project/bec_widgets + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Download benchmark attempts + uses: actions/download-artifact@v4 + with: + pattern: bw-benchmark-json-* + path: benchmark-results/attempts + merge-multiple: true + + - name: Aggregate benchmark attempts + run: | + python .github/scripts/aggregate_benchmarks.py \ + --input-dir benchmark-results/attempts \ + --output "$BENCHMARK_JSON" + + - name: Upload aggregate benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bw-benchmark-json + path: ${{ env.BENCHMARK_JSON }} + + - name: Fetch gh-pages benchmark data + if: github.event_name == 'pull_request' + run: | + if git ls-remote --exit-code --heads origin gh-pages; then + git clone --depth=1 --branch gh-pages "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" gh-pages-benchmark-data + else + mkdir -p gh-pages-benchmark-data + fi + + - name: Compare with latest gh-pages benchmark + id: compare + if: github.event_name == 'pull_request' + continue-on-error: true + run: | + if [ ! -s "$BENCHMARK_BASELINE_JSON" ]; then + mkdir -p "$(dirname "$BENCHMARK_SUMMARY")" + { + echo "" + echo "## Benchmark comparison" + echo + echo "No benchmark baseline was found on gh-pages." + } > "$BENCHMARK_SUMMARY" + exit 0 + fi + + args=( + --baseline "$BENCHMARK_BASELINE_JSON" + --current "$BENCHMARK_JSON" + --summary "$BENCHMARK_SUMMARY" + --threshold-percent "$BENCHMARK_THRESHOLD_PERCENT" + ) + + if [ "$BENCHMARK_HIGHER_IS_BETTER" = "true" ]; then + args+=(--higher-is-better) + fi + + set +e + python .github/scripts/compare_benchmarks.py "${args[@]}" + status=$? + set -e + + if [ ! -s "$BENCHMARK_SUMMARY" ]; then + mkdir -p "$(dirname "$BENCHMARK_SUMMARY")" + { + echo "" + echo "## Benchmark comparison" + echo + echo "Benchmark comparison failed before writing a summary." + } > "$BENCHMARK_SUMMARY" + fi + + exit "$status" + + - name: Find existing benchmark PR comment + if: github.event_name == 'pull_request' + id: fc + uses: peter-evans/find-comment@v3 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: github-actions[bot] + body-includes: "" + + - name: Create or update benchmark PR comment + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v5 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.fc.outputs.comment-id }} + body-path: ${{ env.BENCHMARK_SUMMARY }} + edit-mode: replace + + - name: Fail on benchmark regression + if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure' + run: exit 1 + + - name: Prepare gh-pages for publishing + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow') + run: | + if git ls-remote --exit-code --heads origin gh-pages; then + git fetch --depth=1 origin gh-pages + git worktree add gh-pages-benchmark-data FETCH_HEAD + else + git worktree add --detach gh-pages-benchmark-data + git -C gh-pages-benchmark-data checkout --orphan gh-pages + git -C gh-pages-benchmark-data rm -rf . + fi + + - name: Publish benchmark data to gh-pages + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow') + working-directory: gh-pages-benchmark-data + run: | + mkdir -p benchmarks/history + cp "../$BENCHMARK_JSON" benchmarks/latest.json + cp "../$BENCHMARK_JSON" "benchmarks/history/${GITHUB_SHA}.json" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add benchmarks/latest.json "benchmarks/history/${GITHUB_SHA}.json" + git commit -m "Update BW benchmark data for ${GITHUB_SHA}" || exit 0 + git push origin HEAD:gh-pages diff --git a/tests/benchmarks/hyperfine/benchmark_import_bec_widgets.sh b/tests/benchmarks/hyperfine/benchmark_import_bec_widgets.sh new file mode 100755 index 00000000..4cf9fc0f --- /dev/null +++ b/tests/benchmarks/hyperfine/benchmark_import_bec_widgets.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# BENCHMARK_TITLE: Import bec_widgets +set -euo pipefail + +python -c 'import bec_widgets; print(bec_widgets.__file__)' diff --git a/tests/benchmarks/hyperfine/benchmark_launch_bec_with_companion.sh b/tests/benchmarks/hyperfine/benchmark_launch_bec_with_companion.sh new file mode 100755 index 00000000..e8ea0a49 --- /dev/null +++ b/tests/benchmarks/hyperfine/benchmark_launch_bec_with_companion.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# BENCHMARK_TITLE: BEC IPython client with companion app +set -euo pipefail + +bec --nogui --post-startup-file tests/benchmarks/hyperfine/utils/exit_bec_startup.py diff --git a/tests/benchmarks/hyperfine/benchmark_launch_bec_without_companion.sh b/tests/benchmarks/hyperfine/benchmark_launch_bec_without_companion.sh new file mode 100755 index 00000000..94a08fa4 --- /dev/null +++ b/tests/benchmarks/hyperfine/benchmark_launch_bec_without_companion.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# BENCHMARK_TITLE: BEC IPython client without companion app +set -euo pipefail + +bec --post-startup-file tests/benchmarks/hyperfine/utils/exit_bec_startup.py diff --git a/tests/benchmarks/hyperfine/utils/exit_bec_startup.py b/tests/benchmarks/hyperfine/utils/exit_bec_startup.py new file mode 100644 index 00000000..d4ed74ae --- /dev/null +++ b/tests/benchmarks/hyperfine/utils/exit_bec_startup.py @@ -0,0 +1,5 @@ +import time + +_ip = get_ipython() +_ip.confirm_exit = False +_ip.ask_exit()