ci: add benchmark workflow

.github/workflows/benchmark.yml

@@ -0,0 +1,241 @@
+name: BW Benchmarks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - ci/benchmark_workflow
+  workflow_dispatch:
+    inputs:
+      benchmark_command:
+        description: "Command that writes the benchmark JSON file"
+        required: false
+        type: string
+      threshold_percent:
+        description: "Regression threshold in percent"
+        required: false
+        type: string
+      higher_is_better:
+        description: "Set to true for throughput-style benchmarks"
+        required: false
+        type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  BENCHMARK_JSON: benchmark-results/current.json
+  BENCHMARK_BASELINE_JSON: gh-pages-benchmark-data/benchmarks/latest.json
+  BENCHMARK_SUMMARY: benchmark-results/summary.md
+  BENCHMARK_COMMAND: ${{ inputs.benchmark_command || 'bash .github/scripts/run_benchmarks.sh' }}
+  BENCHMARK_THRESHOLD_PERCENT: ${{ inputs.threshold_percent || '20' }}
+  BENCHMARK_HIGHER_IS_BETTER: ${{ inputs.higher_is_better || false }}
+
+jobs:
+  check_pr_status:
+    uses: ./.github/workflows/check_pr.yml
+
+  benchmark_attempt:
+    needs: check_pr_status
+    if: needs.check_pr_status.outputs.branch-pr == ''
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        attempt: [1, 2, 3]
+
+    env:
+      BENCHMARK_JSON: benchmark-results/current-${{ matrix.attempt }}.json
+      BEC_CORE_BRANCH: main
+      OPHYD_DEVICES_BRANCH: main
+      PLUGIN_REPO_BRANCH: main
+      QTWEBENGINE_DISABLE_SANDBOX: 1
+      QT_QPA_PLATFORM: "offscreen"
+
+    steps:
+      - name: Checkout BEC Widgets
+        uses: actions/checkout@v4
+        with:
+          repository: bec-project/bec_widgets
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          auto-activate-base: true
+          python-version: "3.11"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libgl1 libegl1 x11-utils libxkbcommon-x11-0 libdbus-1-3 xvfb
+          sudo apt-get -y install libnss3 libxdamage1 libasound2t64 libatomic1 libxcursor1
+          sudo apt-get -y install ttyd hyperfine redis-server
+
+      - name: Install full e2e environment
+        run: |
+          echo -e "\033[35;1m Using branch $BEC_CORE_BRANCH of BEC CORE \033[0;m";
+          git clone --branch "$BEC_CORE_BRANCH" https://github.com/bec-project/bec.git
+          echo -e "\033[35;1m Using branch $OPHYD_DEVICES_BRANCH of OPHYD_DEVICES \033[0;m";
+          git clone --branch "$OPHYD_DEVICES_BRANCH" https://github.com/bec-project/ophyd_devices.git
+          export OHPYD_DEVICES_PATH=$PWD/ophyd_devices
+          echo -e "\033[35;1m Using branch $PLUGIN_REPO_BRANCH of bec_testing_plugin \033[0;m";
+          git clone --branch "$PLUGIN_REPO_BRANCH" https://github.com/bec-project/bec_testing_plugin.git
+          cd ./bec
+          conda create -q -n test-environment python=3.11
+          conda activate test-environment
+          source ./bin/install_bec_dev.sh -t
+          cd ../
+          python -m pip install -e ./ophyd_devices -e .[dev,pyside6] -e ./bec_testing_plugin pytest-benchmark
+
+          mkdir -p "$(dirname "$BENCHMARK_JSON")"
+          python .github/scripts/run_with_bec_servers.py -- bash -lc "$BENCHMARK_COMMAND"
+          test -s "$BENCHMARK_JSON"
+
+      - name: Upload benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bw-benchmark-json-${{ matrix.attempt }}
+          path: ${{ env.BENCHMARK_JSON }}
+
+  benchmark:
+    needs: [check_pr_status, benchmark_attempt]
+    if: needs.check_pr_status.outputs.branch-pr == ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout BEC Widgets
+        uses: actions/checkout@v4
+        with:
+          repository: bec-project/bec_widgets
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Download benchmark attempts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: bw-benchmark-json-*
+          path: benchmark-results/attempts
+          merge-multiple: true
+
+      - name: Aggregate benchmark attempts
+        run: |
+          python .github/scripts/aggregate_benchmarks.py \
+            --input-dir benchmark-results/attempts \
+            --output "$BENCHMARK_JSON"
+
+      - name: Upload aggregate benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bw-benchmark-json
+          path: ${{ env.BENCHMARK_JSON }}
+
+      - name: Fetch gh-pages benchmark data
+        if: github.event_name == 'pull_request'
+        run: |
+          if git ls-remote --exit-code --heads origin gh-pages; then
+            git clone --depth=1 --branch gh-pages "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" gh-pages-benchmark-data
+          else
+            mkdir -p gh-pages-benchmark-data
+          fi
+
+      - name: Compare with latest gh-pages benchmark
+        id: compare
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        run: |
+          if [ ! -s "$BENCHMARK_BASELINE_JSON" ]; then
+            mkdir -p "$(dirname "$BENCHMARK_SUMMARY")"
+            {
+              echo "<!-- bw-benchmark-comment -->"
+              echo "## Benchmark comparison"
+              echo
+              echo "No benchmark baseline was found on gh-pages."
+            } > "$BENCHMARK_SUMMARY"
+            exit 0
+          fi
+
+          args=(
+            --baseline "$BENCHMARK_BASELINE_JSON"
+            --current "$BENCHMARK_JSON"
+            --summary "$BENCHMARK_SUMMARY"
+            --threshold-percent "$BENCHMARK_THRESHOLD_PERCENT"
+          )
+
+          if [ "$BENCHMARK_HIGHER_IS_BETTER" = "true" ]; then
+            args+=(--higher-is-better)
+          fi
+
+          set +e
+          python .github/scripts/compare_benchmarks.py "${args[@]}"
+          status=$?
+          set -e
+
+          if [ ! -s "$BENCHMARK_SUMMARY" ]; then
+            mkdir -p "$(dirname "$BENCHMARK_SUMMARY")"
+            {
+              echo "<!-- bw-benchmark-comment -->"
+              echo "## Benchmark comparison"
+              echo
+              echo "Benchmark comparison failed before writing a summary."
+            } > "$BENCHMARK_SUMMARY"
+          fi
+
+          exit "$status"
+
+      - name: Find existing benchmark PR comment
+        if: github.event_name == 'pull_request'
+        id: fc
+        uses: peter-evans/find-comment@v3
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+          body-includes: "<!-- bw-benchmark-comment -->"
+
+      - name: Create or update benchmark PR comment
+        if: github.event_name == 'pull_request'
+        uses: peter-evans/create-or-update-comment@v5
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-path: ${{ env.BENCHMARK_SUMMARY }}
+          edit-mode: replace
+
+      - name: Fail on benchmark regression
+        if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure'
+        run: exit 1
+
+      - name: Prepare gh-pages for publishing
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow')
+        run: |
+          if git ls-remote --exit-code --heads origin gh-pages; then
+            git fetch --depth=1 origin gh-pages
+            git worktree add gh-pages-benchmark-data FETCH_HEAD
+          else
+            git worktree add --detach gh-pages-benchmark-data
+            git -C gh-pages-benchmark-data checkout --orphan gh-pages
+            git -C gh-pages-benchmark-data rm -rf .
+          fi
+
+      - name: Publish benchmark data to gh-pages
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/ci/benchmark_workflow')
+        working-directory: gh-pages-benchmark-data
+        run: |
+          mkdir -p benchmarks/history
+          cp "../$BENCHMARK_JSON" benchmarks/latest.json
+          cp "../$BENCHMARK_JSON" "benchmarks/history/${GITHUB_SHA}.json"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add benchmarks/latest.json "benchmarks/history/${GITHUB_SHA}.json"
+          git commit -m "Update BW benchmark data for ${GITHUB_SHA}" || exit 0
+          git push origin HEAD:gh-pages