diff --git a/.github/scripts/graphify/graph_rebuild.py b/.github/scripts/graphify/graph_rebuild.py new file mode 100644 index 00000000..9098675b --- /dev/null +++ b/.github/scripts/graphify/graph_rebuild.py @@ -0,0 +1,235 @@ +"""Build the graphify knowledge graph for this repository (CI-friendly, headless). + +Usage: python graph_rebuild.py + +The spec lists code roots with their scan roots (paths relative to ): + {"code_roots": [{"files": "bec_lib/bec_lib", "root": "bec_lib"}, ...]} +Scan roots control node-id relativization so that import ids resolve (e.g. the +``bec_lib.endpoints`` import matches the ``bec_lib_endpoints`` file stem). Roots are +processed in order; each file is claimed by the first root that matches, so package +roots must come before the repo-wide catch-all. + +AST extraction is deterministic and needs no LLM. The semantic layer (docs, diagrams) +is merged from graphify-out/cache/semantic when present (e.g. restored from a CI +cache); otherwise the build is code-only and a note is printed. + +The build writes ``graphify-out/build_meta.json``, which identifies *which* state of the +repository the graph describes. ``version`` — the released package version the graph was +built from — is the key consumers compare against their installed package to decide +whether their local copy is stale. ``requires`` records the BEC-family constraints in +force at build time, so a graph built for one bec_lib range can be spotted as mismatched +against a very different installed one. GITHUB_* environment variables fill in the +release identity in CI; locally those fields stay null. +""" + +import json +import os +import sys +import tomllib +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _dist_version +from pathlib import Path + +# Dependencies whose version the graph is meaningfully coupled to: a graph of this +# repository describes code that calls into these, so a large drift in them dates it. +BEC_FAMILY = ("bec_lib", "bec_ipython_client", "bec_widgets", "ophyd_devices", "bec_qthemes") + + +def _projects(repo: Path, spec: dict) -> list[dict]: + """Load the [project] table of every pyproject.toml this repository releases. + + ``version_from`` is one path or a list of them, relative to the repo root. A + monorepo lists each distribution it ships (bec releases four from one tree); a + single-package repo can leave it out and get the root pyproject.toml. + """ + rels = spec.get("version_from", "pyproject.toml") + rels = [rels] if isinstance(rels, str) else rels + projects = [] + for rel in rels: + path = repo / rel + if not path.is_file(): + print(f" note: {rel} not found - it contributes no version to build_meta") + continue + project = tomllib.loads(path.read_text(encoding="utf-8")).get("project", {}) + if project.get("name"): + projects.append(project) + if not projects: + print(" note: no pyproject.toml found - build_meta version will be null") + return projects + + +def _build_identity(repo: Path, spec: dict, graph) -> dict: + """Assemble the identity block that consumers use for staleness checks.""" + projects = _projects(repo, spec) + # every distribution built from this tree, so a monorepo stamps all of them rather + # than silently speaking for one; packages released together share a version + packages = {p["name"]: p.get("version") for p in projects} + # only constraints on BEC-family packages released from *other* repositories; these + # are floors ("bec_lib~=3.134"), not the version that was resolved at build time + requires = { + name: req + for p in projects + for req in p.get("dependencies", []) + for name in BEC_FAMILY + if req.replace("-", "_").startswith(name) and name not in packages + } + try: + graphify_version = _dist_version("graphifyy") + except PackageNotFoundError: + graphify_version = None + + tag = os.environ.get("GRAPH_RELEASE_TAG") or None + return { + "built_at": datetime.now(timezone.utc).isoformat(), + "package": projects[0]["name"] if projects else None, + # the staleness key; co-released packages share it, so the first is the repo's + "version": projects[0].get("version") if projects else None, + "packages": packages, + "tag": tag, + "release": bool(tag), + # the exact revision the graph describes - the precise check when a checkout + # sits ahead of the last release and versions alone cannot separate them + "commit": os.environ.get("GITHUB_SHA") or None, + "repository": os.environ.get("GITHUB_REPOSITORY") or None, + "graphify_version": graphify_version, + "requires": requires, + "nodes": graph.number_of_nodes(), + "edges": graph.number_of_edges(), + } + + +def main() -> None: + repo = Path(sys.argv[1]).resolve() + spec = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + os.chdir(repo) + + from graphify.analyze import god_nodes, suggest_questions, surprising_connections + from graphify.build import build_from_json + from graphify.cache import check_semantic_cache + from graphify.cluster import cluster, score_all + from graphify.detect import detect, save_manifest + from graphify.diagnostics import diagnose_extraction + from graphify.export import to_json + from graphify.extract import extract + from graphify.report import generate + + out = Path("graphify-out") + out.mkdir(exist_ok=True) + + detection = detect(repo) + # drop any stray literal-~ directories (unexpanded-tilde artifacts) + for cat in list(detection.get("files", {})): + detection["files"][cat] = [f for f in detection["files"][cat] if "/~/" not in f] + + all_nodes, all_edges, all_hyper = [], [], [] + seen_ids: set[str] = set() + claimed: set[str] = set() + detected_code = [f for f in detection["files"].get("code", [])] + for code_root in spec["code_roots"]: + files_dir = (repo / code_root["files"]).resolve() + croot = (repo / code_root["root"]).resolve() + prefix = str(files_dir) + "/" + files = [Path(f) for f in detected_code if f.startswith(prefix) and f not in claimed] + claimed.update(str(f) for f in files) + if not files: + print(f" {code_root['files']}: no files, skipped") + continue + res = extract(files, cache_root=croot) + for node in res["nodes"]: + if node["id"] not in seen_ids: + seen_ids.add(node["id"]) + all_nodes.append(node) + all_edges += res["edges"] + print( + f" {code_root['files']} (root={code_root['root']}): " + f"{len(res['nodes'])} nodes, {len(res['edges'])} edges" + ) + + doc_files = [ + f for cat in ("document", "paper", "image") for f in detection["files"].get(cat, []) + ] + c_nodes, c_edges, c_hyper, uncached = check_semantic_cache(doc_files, root=str(repo)) + for node in c_nodes: + if node["id"] not in seen_ids: + seen_ids.add(node["id"]) + all_nodes.append(node) + all_edges += c_edges + all_hyper += c_hyper + if uncached: + print( + f" note: {len(uncached)}/{len(doc_files)} docs have no cached semantic layer " + "(code-only build for those files; refresh the semantic layer locally)" + ) + + extraction = { + "nodes": all_nodes, + "edges": all_edges, + "hyperedges": all_hyper, + "input_tokens": 0, + "output_tokens": 0, + } + print(f"extraction: {len(all_nodes)} nodes, {len(all_edges)} edges") + + graph = build_from_json(extraction, root=str(repo), directed=True) + if graph.number_of_nodes() == 0: + raise SystemExit("ERROR: extraction produced an empty graph") + communities = cluster(graph) + cohesion = score_all(graph, communities) + gods = god_nodes(graph) + surprises = surprising_connections(graph, communities) + # keep any labels from a previous (cached) run; new communities get placeholders + labels_path = out / ".graphify_labels.json" + prior = json.loads(labels_path.read_text(encoding="utf-8")) if labels_path.exists() else {} + labels = {cid: prior.get(str(cid), f"Community {cid}") for cid in communities} + questions = suggest_questions(graph, communities, labels) + + (out / "graph.json").unlink(missing_ok=True) + if not to_json(graph, communities, "graphify-out/graph.json"): + raise SystemExit("ERROR: graph.json was not written") + report = generate( + graph, + communities, + cohesion, + labels, + gods, + surprises, + detection, + {"input": 0, "output": 0}, + str(repo), + suggested_questions=questions, + ) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") + labels_path.write_text( + json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8" + ) + print( + f"graph: {graph.number_of_nodes()} nodes, {graph.number_of_edges()} edges, " + f"{len(communities)} communities (directed)" + ) + + summary = diagnose_extraction(extraction, directed=True, root=str(repo)) + print( + "health:", + { + k: summary.get(k, 0) + for k in ("dangling_endpoint_edges", "missing_endpoint_edges", "self_loop_edges") + }, + ) + + save_manifest(detection.get("all_files") or detection["files"], root=str(repo)) + meta = _build_identity(repo, spec, graph) + (out / "build_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") + + # Embed the same identity inside the map. build_meta.json alone is not enough: a + # graph.json routinely gets copied around on its own, and an unlabelled map that + # cannot say which release it describes cannot be trusted for anything. + graph_doc = json.loads((out / "graph.json").read_text(encoding="utf-8")) + graph_doc["build_meta"] = meta + (out / "graph.json").write_text(json.dumps(graph_doc), encoding="utf-8") + + print(f"meta: {meta['package']} {meta['version']} (tag={meta['tag']})") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/graphify/graph_spec.json b/.github/scripts/graphify/graph_spec.json new file mode 100644 index 00000000..538c3aaf --- /dev/null +++ b/.github/scripts/graphify/graph_spec.json @@ -0,0 +1,3 @@ +{"code_roots": [ + {"files": ".", "root": "."} +]} diff --git a/.github/workflows/knowledge-graph.yml b/.github/workflows/knowledge-graph.yml new file mode 100644 index 00000000..6abe48cd --- /dev/null +++ b/.github/workflows/knowledge-graph.yml @@ -0,0 +1,123 @@ +# Builds the graphify knowledge graph and publishes it with the release it describes. +# +# Triggered after "Continuous Delivery" (semantic-release) completes: releases happen +# exactly when releasable code (feat/fix/perf) lands on the default branch, so the graph +# tracks actual code changes rather than every push. A release-tag gate skips CD runs +# that released nothing — and, because it only proceeds when HEAD carries a v* tag, it +# also guarantees the graph describes exactly the commit that was released. Note: +# chaining via workflow_run is deliberate — tags pushed by a workflow's own GITHUB_TOKEN +# do not fire `on: push: tags` workflows. +# +# Two publication paths: +# * release runs attach knowledge-graph.tar.gz to the GitHub Release for that tag, so +# every developer can fetch the map for a given version, unauthenticated and +# permanently, at a URL derived from the version alone; +# * all runs also upload a job artifact, which is what makes PR and workflow_dispatch +# dry-runs inspectable. +# +# The graph is never committed (graphify-out/ is gitignored). actions/cache carries the +# AST cache between runs, so rebuilds are incremental. CI builds are AST-only +# (deterministic, no LLM key); the semantic docs layer and community labels ride along +# via the cache when a locally refreshed one was uploaded. + +name: knowledge-graph + +on: + workflow_run: + workflows: ["Continuous Delivery"] + types: [completed] + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build-graph: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + + # write is needed only to attach the tarball to an existing release; the workflow + # never pushes commits, tags or branches. + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-tags: true + + - name: Release gate + id: gate + run: | + TAG="$(git tag --points-at HEAD | grep '^v' | head -n 1 || true)" + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + if [ "${{ github.event_name }}" != "workflow_run" ]; then + # dry-run contexts: build, but there is no release to attach to + echo "run=true" >> "$GITHUB_OUTPUT" + elif [ -n "$TAG" ]; then + echo "Released $TAG - building and attaching the graph." + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "CD run made no release - skipping graph build." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/setup-python@v5 + if: steps.gate.outputs.run == 'true' + with: + python-version: "3.12" + + - name: Restore graph cache + if: steps.gate.outputs.run == 'true' + uses: actions/cache@v4 + with: + path: graphify-out + key: graphify-${{ runner.os }}-${{ github.sha }} + restore-keys: | + graphify-${{ runner.os }}- + + - name: Install graphify + if: steps.gate.outputs.run == 'true' + # pinned so CI node ids stay identical to locally built graphs; + # bump deliberately together with local installs + run: pip install --quiet 'graphifyy==0.9.5' + + - name: Build knowledge graph + if: steps.gate.outputs.run == 'true' + env: + PYTHONHASHSEED: "0" # deterministic clustering + # identity written into build_meta.json; GITHUB_SHA is overridden because on + # workflow_run it names the triggering run's head, not what we checked out + GITHUB_SHA: ${{ steps.gate.outputs.sha }} + GRAPH_RELEASE_TAG: ${{ steps.gate.outputs.tag }} + run: python .github/scripts/graphify/graph_rebuild.py . .github/scripts/graphify/graph_spec.json + + - name: Package graph + id: package + if: steps.gate.outputs.run == 'true' + run: | + # name the asset after the version it describes, so a downloaded file is + # identifiable on disk without unpacking it first + VERSION=$(python -c "import json; print(json.load(open('graphify-out/build_meta.json'))['version'])") + NAME="knowledge-graph-v${VERSION}.tar.gz" + tar -czf "$NAME" -C graphify-out graph.json GRAPH_REPORT.md build_meta.json + echo "name=$NAME" >> "$GITHUB_OUTPUT" + echo "Packaged $NAME ($(du -h "$NAME" | cut -f1)):" + cat graphify-out/build_meta.json + + - name: Attach graph to release + # only release runs have a tag to attach to; dry-runs stop at the artifact below + if: steps.gate.outputs.run == 'true' && steps.gate.outputs.tag != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload "${{ steps.gate.outputs.tag }}" "${{ steps.package.outputs.name }}" --clobber + + - name: Publish graph artifact + if: steps.gate.outputs.run == 'true' + uses: actions/upload-artifact@v4 + with: + name: knowledge-graph + path: ${{ steps.package.outputs.name }} + retention-days: 90 diff --git a/.gitignore b/.gitignore index 1f5f9435..0e038153 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,6 @@ cython_debug/ #.idea/ # tombi.toml + +# graphify knowledge-graph outputs (built in CI as artifacts, or locally via hooks) +graphify-out/