ci: add knowledge-graph build workflow

Builds the graphify knowledge graph after each semantic-release and attaches it
to that release as knowledge-graph-v<version>.tar.gz, so every developer can fetch
the map for a given version. The map carries its own identity — released package
versions, tag and commit — in build_meta.json and embedded in graph.json, which is
what lets a consumer tell a current map from a stale one. The graph is never
committed; graphify-out/ is gitignored.

.github/scripts/graphify/graph_rebuild.py

@@ -0,0 +1,235 @@
+"""Build the graphify knowledge graph for this repository (CI-friendly, headless).
+
+Usage: python graph_rebuild.py <repo_root> <spec.json>
+
+The spec lists code roots with their scan roots (paths relative to <repo_root>):
+    {"code_roots": [{"files": "bec_lib/bec_lib", "root": "bec_lib"}, ...]}
+Scan roots control node-id relativization so that import ids resolve (e.g. the
+``bec_lib.endpoints`` import matches the ``bec_lib_endpoints`` file stem). Roots are
+processed in order; each file is claimed by the first root that matches, so package
+roots must come before the repo-wide catch-all.
+
+AST extraction is deterministic and needs no LLM. The semantic layer (docs, diagrams)
+is merged from graphify-out/cache/semantic when present (e.g. restored from a CI
+cache); otherwise the build is code-only and a note is printed.
+
+The build writes ``graphify-out/build_meta.json``, which identifies *which* state of the
+repository the graph describes. ``version`` — the released package version the graph was
+built from — is the key consumers compare against their installed package to decide
+whether their local copy is stale. ``requires`` records the BEC-family constraints in
+force at build time, so a graph built for one bec_lib range can be spotted as mismatched
+against a very different installed one. GITHUB_* environment variables fill in the
+release identity in CI; locally those fields stay null.
+"""
+
+import json
+import os
+import sys
+import tomllib
+from datetime import datetime, timezone
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _dist_version
+from pathlib import Path
+
+# Dependencies whose version the graph is meaningfully coupled to: a graph of this
+# repository describes code that calls into these, so a large drift in them dates it.
+BEC_FAMILY = ("bec_lib", "bec_ipython_client", "bec_widgets", "ophyd_devices", "bec_qthemes")
+
+
+def _projects(repo: Path, spec: dict) -> list[dict]:
+    """Load the [project] table of every pyproject.toml this repository releases.
+
+    ``version_from`` is one path or a list of them, relative to the repo root. A
+    monorepo lists each distribution it ships (bec releases four from one tree); a
+    single-package repo can leave it out and get the root pyproject.toml.
+    """
+    rels = spec.get("version_from", "pyproject.toml")
+    rels = [rels] if isinstance(rels, str) else rels
+    projects = []
+    for rel in rels:
+        path = repo / rel
+        if not path.is_file():
+            print(f"  note: {rel} not found - it contributes no version to build_meta")
+            continue
+        project = tomllib.loads(path.read_text(encoding="utf-8")).get("project", {})
+        if project.get("name"):
+            projects.append(project)
+    if not projects:
+        print("  note: no pyproject.toml found - build_meta version will be null")
+    return projects
+
+
+def _build_identity(repo: Path, spec: dict, graph) -> dict:
+    """Assemble the identity block that consumers use for staleness checks."""
+    projects = _projects(repo, spec)
+    # every distribution built from this tree, so a monorepo stamps all of them rather
+    # than silently speaking for one; packages released together share a version
+    packages = {p["name"]: p.get("version") for p in projects}
+    # only constraints on BEC-family packages released from *other* repositories; these
+    # are floors ("bec_lib~=3.134"), not the version that was resolved at build time
+    requires = {
+        name: req
+        for p in projects
+        for req in p.get("dependencies", [])
+        for name in BEC_FAMILY
+        if req.replace("-", "_").startswith(name) and name not in packages
+    }
+    try:
+        graphify_version = _dist_version("graphifyy")
+    except PackageNotFoundError:
+        graphify_version = None
+
+    tag = os.environ.get("GRAPH_RELEASE_TAG") or None
+    return {
+        "built_at": datetime.now(timezone.utc).isoformat(),
+        "package": projects[0]["name"] if projects else None,
+        # the staleness key; co-released packages share it, so the first is the repo's
+        "version": projects[0].get("version") if projects else None,
+        "packages": packages,
+        "tag": tag,
+        "release": bool(tag),
+        # the exact revision the graph describes - the precise check when a checkout
+        # sits ahead of the last release and versions alone cannot separate them
+        "commit": os.environ.get("GITHUB_SHA") or None,
+        "repository": os.environ.get("GITHUB_REPOSITORY") or None,
+        "graphify_version": graphify_version,
+        "requires": requires,
+        "nodes": graph.number_of_nodes(),
+        "edges": graph.number_of_edges(),
+    }
+
+
+def main() -> None:
+    repo = Path(sys.argv[1]).resolve()
+    spec = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8"))
+    os.chdir(repo)
+
+    from graphify.analyze import god_nodes, suggest_questions, surprising_connections
+    from graphify.build import build_from_json
+    from graphify.cache import check_semantic_cache
+    from graphify.cluster import cluster, score_all
+    from graphify.detect import detect, save_manifest
+    from graphify.diagnostics import diagnose_extraction
+    from graphify.export import to_json
+    from graphify.extract import extract
+    from graphify.report import generate
+
+    out = Path("graphify-out")
+    out.mkdir(exist_ok=True)
+
+    detection = detect(repo)
+    # drop any stray literal-~ directories (unexpanded-tilde artifacts)
+    for cat in list(detection.get("files", {})):
+        detection["files"][cat] = [f for f in detection["files"][cat] if "/~/" not in f]
+
+    all_nodes, all_edges, all_hyper = [], [], []
+    seen_ids: set[str] = set()
+    claimed: set[str] = set()
+    detected_code = [f for f in detection["files"].get("code", [])]
+    for code_root in spec["code_roots"]:
+        files_dir = (repo / code_root["files"]).resolve()
+        croot = (repo / code_root["root"]).resolve()
+        prefix = str(files_dir) + "/"
+        files = [Path(f) for f in detected_code if f.startswith(prefix) and f not in claimed]
+        claimed.update(str(f) for f in files)
+        if not files:
+            print(f"  {code_root['files']}: no files, skipped")
+            continue
+        res = extract(files, cache_root=croot)
+        for node in res["nodes"]:
+            if node["id"] not in seen_ids:
+                seen_ids.add(node["id"])
+                all_nodes.append(node)
+        all_edges += res["edges"]
+        print(
+            f"  {code_root['files']} (root={code_root['root']}): "
+            f"{len(res['nodes'])} nodes, {len(res['edges'])} edges"
+        )
+
+    doc_files = [
+        f for cat in ("document", "paper", "image") for f in detection["files"].get(cat, [])
+    ]
+    c_nodes, c_edges, c_hyper, uncached = check_semantic_cache(doc_files, root=str(repo))
+    for node in c_nodes:
+        if node["id"] not in seen_ids:
+            seen_ids.add(node["id"])
+            all_nodes.append(node)
+    all_edges += c_edges
+    all_hyper += c_hyper
+    if uncached:
+        print(
+            f"  note: {len(uncached)}/{len(doc_files)} docs have no cached semantic layer "
+            "(code-only build for those files; refresh the semantic layer locally)"
+        )
+
+    extraction = {
+        "nodes": all_nodes,
+        "edges": all_edges,
+        "hyperedges": all_hyper,
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }
+    print(f"extraction: {len(all_nodes)} nodes, {len(all_edges)} edges")
+
+    graph = build_from_json(extraction, root=str(repo), directed=True)
+    if graph.number_of_nodes() == 0:
+        raise SystemExit("ERROR: extraction produced an empty graph")
+    communities = cluster(graph)
+    cohesion = score_all(graph, communities)
+    gods = god_nodes(graph)
+    surprises = surprising_connections(graph, communities)
+    # keep any labels from a previous (cached) run; new communities get placeholders
+    labels_path = out / ".graphify_labels.json"
+    prior = json.loads(labels_path.read_text(encoding="utf-8")) if labels_path.exists() else {}
+    labels = {cid: prior.get(str(cid), f"Community {cid}") for cid in communities}
+    questions = suggest_questions(graph, communities, labels)
+
+    (out / "graph.json").unlink(missing_ok=True)
+    if not to_json(graph, communities, "graphify-out/graph.json"):
+        raise SystemExit("ERROR: graph.json was not written")
+    report = generate(
+        graph,
+        communities,
+        cohesion,
+        labels,
+        gods,
+        surprises,
+        detection,
+        {"input": 0, "output": 0},
+        str(repo),
+        suggested_questions=questions,
+    )
+    (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8")
+    labels_path.write_text(
+        json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8"
+    )
+    print(
+        f"graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges, "
+        f"{len(communities)} communities (directed)"
+    )
+
+    summary = diagnose_extraction(extraction, directed=True, root=str(repo))
+    print(
+        "health:",
+        {
+            k: summary.get(k, 0)
+            for k in ("dangling_endpoint_edges", "missing_endpoint_edges", "self_loop_edges")
+        },
+    )
+
+    save_manifest(detection.get("all_files") or detection["files"], root=str(repo))
+    meta = _build_identity(repo, spec, graph)
+    (out / "build_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+    # Embed the same identity inside the map. build_meta.json alone is not enough: a
+    # graph.json routinely gets copied around on its own, and an unlabelled map that
+    # cannot say which release it describes cannot be trusted for anything.
+    graph_doc = json.loads((out / "graph.json").read_text(encoding="utf-8"))
+    graph_doc["build_meta"] = meta
+    (out / "graph.json").write_text(json.dumps(graph_doc), encoding="utf-8")
+
+    print(f"meta: {meta['package']} {meta['version']} (tag={meta['tag']})")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/graphify/graph_spec.json

@@ -0,0 +1,3 @@
+{"code_roots": [
+  {"files": ".", "root": "."}
+]}

.github/workflows/knowledge-graph.yml

@@ -0,0 +1,123 @@
+# Builds the graphify knowledge graph and publishes it with the release it describes.
+#
+# Triggered after "Continuous Delivery" (semantic-release) completes: releases happen
+# exactly when releasable code (feat/fix/perf) lands on the default branch, so the graph
+# tracks actual code changes rather than every push. A release-tag gate skips CD runs
+# that released nothing — and, because it only proceeds when HEAD carries a v* tag, it
+# also guarantees the graph describes exactly the commit that was released. Note:
+# chaining via workflow_run is deliberate — tags pushed by a workflow's own GITHUB_TOKEN
+# do not fire `on: push: tags` workflows.
+#
+# Two publication paths:
+#   * release runs attach knowledge-graph.tar.gz to the GitHub Release for that tag, so
+#     every developer can fetch the map for a given version, unauthenticated and
+#     permanently, at a URL derived from the version alone;
+#   * all runs also upload a job artifact, which is what makes PR and workflow_dispatch
+#     dry-runs inspectable.
+#
+# The graph is never committed (graphify-out/ is gitignored). actions/cache carries the
+# AST cache between runs, so rebuilds are incremental. CI builds are AST-only
+# (deterministic, no LLM key); the semantic docs layer and community labels ride along
+# via the cache when a locally refreshed one was uploaded.
+
+name: knowledge-graph
+
+on:
+  workflow_run:
+    workflows: ["Continuous Delivery"]
+    types: [completed]
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  build-graph:
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    # write is needed only to attach the tarball to an existing release; the workflow
+    # never pushes commits, tags or branches.
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+
+      - name: Release gate
+        id: gate
+        run: |
+          TAG="$(git tag --points-at HEAD | grep '^v' | head -n 1 || true)"
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          if [ "${{ github.event_name }}" != "workflow_run" ]; then
+            # dry-run contexts: build, but there is no release to attach to
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          elif [ -n "$TAG" ]; then
+            echo "Released $TAG - building and attaching the graph."
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "CD run made no release - skipping graph build."
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - uses: actions/setup-python@v5
+        if: steps.gate.outputs.run == 'true'
+        with:
+          python-version: "3.12"
+
+      - name: Restore graph cache
+        if: steps.gate.outputs.run == 'true'
+        uses: actions/cache@v4
+        with:
+          path: graphify-out
+          key: graphify-${{ runner.os }}-${{ github.sha }}
+          restore-keys: |
+            graphify-${{ runner.os }}-
+
+      - name: Install graphify
+        if: steps.gate.outputs.run == 'true'
+        # pinned so CI node ids stay identical to locally built graphs;
+        # bump deliberately together with local installs
+        run: pip install --quiet 'graphifyy==0.9.5'
+
+      - name: Build knowledge graph
+        if: steps.gate.outputs.run == 'true'
+        env:
+          PYTHONHASHSEED: "0" # deterministic clustering
+          # identity written into build_meta.json; GITHUB_SHA is overridden because on
+          # workflow_run it names the triggering run's head, not what we checked out
+          GITHUB_SHA: ${{ steps.gate.outputs.sha }}
+          GRAPH_RELEASE_TAG: ${{ steps.gate.outputs.tag }}
+        run: python .github/scripts/graphify/graph_rebuild.py . .github/scripts/graphify/graph_spec.json
+
+      - name: Package graph
+        id: package
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          # name the asset after the version it describes, so a downloaded file is
+          # identifiable on disk without unpacking it first
+          VERSION=$(python -c "import json; print(json.load(open('graphify-out/build_meta.json'))['version'])")
+          NAME="knowledge-graph-v${VERSION}.tar.gz"
+          tar -czf "$NAME" -C graphify-out graph.json GRAPH_REPORT.md build_meta.json
+          echo "name=$NAME" >> "$GITHUB_OUTPUT"
+          echo "Packaged $NAME ($(du -h "$NAME" | cut -f1)):"
+          cat graphify-out/build_meta.json
+
+      - name: Attach graph to release
+        # only release runs have a tag to attach to; dry-runs stop at the artifact below
+        if: steps.gate.outputs.run == 'true' && steps.gate.outputs.tag != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload "${{ steps.gate.outputs.tag }}" "${{ steps.package.outputs.name }}" --clobber
+
+      - name: Publish graph artifact
+        if: steps.gate.outputs.run == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: knowledge-graph
+          path: ${{ steps.package.outputs.name }}
+          retention-days: 90

.gitignore

@@ -180,3 +180,6 @@ cython_debug/
 #.idea/
 #
 tombi.toml
+
+# graphify knowledge-graph outputs (built in CI as artifacts, or locally via hooks)
+graphify-out/