Jungfraujoch/tools/battery/doc_check.py

#!/usr/bin/env python3
"""doc_check.py -- the battery manifests against the pages that document their datasets.

  open.json            <-> docs/EXTERNAL_TEST_DATA.md
  still_manifest.json  <-> docs/EXTERNAL_TEST_DATA_STILLS.md
  inhouse.json         <-> the inhouse row of the Arms table in tools/battery/README.md
  multisweep.json      <-> docs/EXTERNAL_TEST_DATA.md, "Multi-sweep battery rows"

Open arm: every manifest row is a row of the page's Datasets table (through the "Battery rows
whose id is not the PDB code" table where the id is not the PDB code) or of "Datasets with no PDB
entry"; every Datasets row is in the manifest or under "Datasets in the table that are not in the
battery"; space group, cell (to the page's 0.1 A / 0.1 deg) and d_min (to 0.01 A) agree; the
stated row count and the per-repository counts are the table's. Still battery: every set is in
the page's table with its CXIDB entry, PDB entry, space group and d_min, and every set the page
names is in the manifest. In-house: each set's specimen is named in the README. Multi-sweep: every
row's base row exists, and the page's table lists exactly the manifest's rows, with their base and
number of sweeps.

  python3 doc_check.py        (exit status 1 on any mismatch; stdlib only, no data needed)
"""
import collections
import json
import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
DOCS = os.path.join(HERE, "..", "..", "docs")
# in-house id prefix -> the name the README's Arms table gives the specimen
INHOUSE = {"lyso": "lysozyme", "lysoI": "lysozyme", "thau": "thaumatin", "insu": "insulin",
           "cytc": "cytochrome C", "myob": "myoglobin", "nothing": "no-crystal controls",
           "aspirin": "aspirin", "citricacid": "citric acid", "hepes": "HEPES", "yag": "YAG",
           "kdp": "KDP", "lcystine": "L-cystine", "lascorbate": "L-ascorbic acid"}

errors = []


def section(text, heading):
    """The body of a '## heading' section, up to the next '## '."""
    body = text.split("\n## " + heading + "\n", 1)[1]
    return body.split("\n## ", 1)[0]


def table(body, n=0):
    """Rows of the n-th Markdown table in body, as lists of stripped cells."""
    blocks = re.findall(r"(?:^\|.*\n?)+", body, re.M)
    rows = blocks[n].splitlines()[2:]
    return [[c.strip() for c in r.strip("|").split(" | ")] for r in rows]


def check_open():
    sets = {s["id"]: s for s in json.load(open(os.path.join(HERE, "open.json")))["sets"]}
    text = open(os.path.join(DOCS, "EXTERNAL_TEST_DATA.md")).read()
    rows = {re.match(r"\[(\w{4})\]", r[0]).group(1): r for r in table(section(text, "Datasets"))}
    renamed = {r[0].strip("`"): r[1] for r in table(section(text, "Battery rows whose id is not the PDB code"))}
    absent = {r[0] for r in table(section(text, "Datasets in the table that are not in the battery"))}
    no_pdb = {r[0].strip("`"): r for r in table(section(text, "Datasets with no PDB entry"))}
    small = {r[0].strip("`"): r for r in table(section(text, "Datasets with no PDB entry"), 1)}

    used = set()
    for sid, s in sets.items():
        if sid in no_pdb:
            ref, row = s.get("ref"), small.get(sid)
            if ref and row and (row[1].split("`")[1] != ref["sg"] or not same_cell(row[2].replace("/", " "), ref["cell"], 0.0005)):
                errors.append(f"open {sid}: space group or cell differ from the small-molecule table")
            continue
        pdb = renamed.get(sid, sid.upper())
        row = rows.get(pdb)
        if row is None:
            errors.append(f"open {sid}: not on EXTERNAL_TEST_DATA.md")
            continue
        used.add(pdb)
        ref = s["ref"]
        if row[4] != ref["sg"]:
            errors.append(f"open {sid}: space group {row[4]!r} on the page, {ref['sg']!r} in the manifest")
        if not same_cell(row[5], ref["cell"], 0.051):
            errors.append(f"open {sid}: cell {row[5]} on the page, {ref['cell']} in the manifest")
        if ref["dmin"] is not None and abs(float(row[3]) - ref["dmin"]) > 0.0051:
            errors.append(f"open {sid}: d_min {row[3]} on the page, {ref['dmin']} in the manifest")
    for pdb in sorted(set(rows) - used - absent):
        errors.append(f"open {pdb}: on the page but not in open.json, nor listed as not in the battery")
    for pdb in sorted(absent & used):
        errors.append(f"open {pdb}: listed as not in the battery, but open.json has it")

    m = re.search(r"The table holds (\d+) PDB entries", text)
    if not m or int(m.group(1)) != len(rows):
        errors.append(f"open: the page states {m and m.group(1)} PDB entries, its table has {len(rows)}")
    repos = collections.Counter(r[1].split(" [")[0] for r in rows.values())
    repos.update(r[1].split(" record")[0] for r in no_pdb.values())
    stated = dict((name, int(n)) for name, n in re.findall(
        r"([A-Za-z][\w ]*?) (\d+)[,.]", section(text, "The collection in numbers").split("- **Repository**")[1].split(":", 1)[1].split("\n- ")[0]))
    if stated != dict(repos):
        errors.append(f"open: repository counts on the page {stated}, the tables give {dict(repos)}")


def same_cell(text, cell, tol):
    vals = [float(v) for v in text.split()]
    return len(vals) == 6 and all(abs(a - b) <= tol for a, b in zip(vals, cell))


def check_still():
    sets = {s["id"]: s for s in json.load(open(os.path.join(HERE, "still_manifest.json")))["sets"]}
    text = open(os.path.join(DOCS, "EXTERNAL_TEST_DATA_STILLS.md")).read()
    seen = set()
    for row in table(section(text, "Datasets")):
        ids = re.findall(r"`(\w+)`", row[0])
        pdbs = re.findall(r"\[(\w{4})\]\(https://www.rcsb", row[4])
        cxidb = int(re.match(r"\[(\d+)\]", row[1]).group(1))
        sg, dmin = row[5].rsplit(", ", 1)
        for i, sid in enumerate(ids):
            seen.add(sid)
            s = sets.get(sid)
            if s is None:
                errors.append(f"still {sid}: on the page but not in still_manifest.json")
                continue
            pdb = pdbs[i] if len(pdbs) == len(ids) else pdbs[0]
            got = (s["cxidb"], s["pdb"], s["sg"], s["d_cut"])
            if got != (cxidb, pdb, sg, float(dmin)):
                errors.append(f"still {sid}: page {(cxidb, pdb, sg, float(dmin))}, manifest {got}")
    for sid in sorted(set(sets) - seen):
        errors.append(f"still {sid}: in still_manifest.json but not on the page")


def check_inhouse():
    sets = json.load(open(os.path.join(HERE, "inhouse.json")))["sets"]
    readme = open(os.path.join(HERE, "README.md")).read()
    arm = next(l for l in readme.splitlines() if l.startswith("| **inhouse** |"))
    for s in sets:
        name = INHOUSE.get(s["id"].split("_")[0])
        if name is None or name not in arm:
            errors.append(f"inhouse {s['id']}: its specimen is not named in README.md's Arms table")


def check_multisweep():
    sets = json.load(open(os.path.join(HERE, "multisweep.json")))["sets"]
    ids = {arm: {e["id"] for e in json.load(open(os.path.join(HERE, arm + ".json")))["sets"]}
           for arm in ("open", "inhouse")}
    page = open(os.path.join(DOCS, "EXTERNAL_TEST_DATA.md")).read()
    rows = {r[0].strip("`"): r for r in table(section(page, "Multi-sweep battery rows"))}
    for s in sets:
        if "base" in s:
            arm, _, base = s["base"].partition("/")
            if base not in ids.get(arm, ()):
                errors.append(f"multisweep {s['id']}: base row {s['base']} is not in {arm}.json")
            base_cell = s["base"]
        else:                              # its own reference, documented in the Datasets table
            base_cell = f"{s['data']} data, own reference"
        row = rows.get(s["id"])
        if row is None:
            errors.append(f"multisweep {s['id']}: not in the page's multi-sweep table")
        elif (row[1], row[2]) != (base_cell, str(len(s["inputs"]))):
            errors.append(f"multisweep {s['id']}: page {row[1]}, {row[2]} sweeps; manifest {base_cell}, "
                          f"{len(s['inputs'])} sweeps")
    for sid in sorted(set(rows) - {s["id"] for s in sets}):
        errors.append(f"multisweep {sid}: on the page but not in multisweep.json")


check_open()
check_still()
check_inhouse()
check_multisweep()
for e in errors:
    print(e)
print("doc_check: " + (f"{len(errors)} mismatch(es)" if errors else "manifests and docs agree"))
sys.exit(1 if errors else 0)