474 lines
13 KiB
Python
474 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Convert Foswiki-exported HTML pages to Markdown suitable for Gitea wiki.
|
||
|
||
Focus:
|
||
- keep only topic content (title/sections/paragraphs/lists)
|
||
- drop TOC and edit widgets ("EditChapterPlugin"/ecp*)
|
||
- rewrite internal links to .md
|
||
- add HTML comment provenance from span.patternRevInfo when present
|
||
|
||
Usage (single file):
|
||
python foswiki_html2gitea_md.py CameronKewish.html -o CameronKewish.md
|
||
|
||
Usage (directory):
|
||
python foswiki_html2gitea_md.py input_dir -o output_dir
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import re
|
||
from pathlib import Path
|
||
from bs4 import BeautifulSoup
|
||
from bs4.element import NavigableString, Tag
|
||
|
||
|
||
# -------- provenance --------
|
||
|
||
_REV_RE = re.compile(
|
||
r"Topic revision:\s*(r\d+)\s*-\s*([^,]+),\s*(.+)$",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
def extract_provenance(soup: BeautifulSoup) -> dict[str, str]:
|
||
"""
|
||
Extract Foswiki revision info from span.patternRevInfo if available.
|
||
Example text:
|
||
'Topic revision: r1 - 02 Feb 2010, kewish'
|
||
"""
|
||
rev = soup.select_one("span.patternRevInfo")
|
||
if not rev:
|
||
return {}
|
||
|
||
txt = rev.get_text(" ", strip=True)
|
||
m = _REV_RE.search(txt)
|
||
if not m:
|
||
return {"foswiki_revinfo_raw": txt}
|
||
|
||
return {
|
||
"foswiki_revision": m.group(1).strip(),
|
||
"foswiki_last_modified": m.group(2).strip(),
|
||
"foswiki_last_author": m.group(3).strip(),
|
||
}
|
||
|
||
|
||
# -------- link rewriting --------
|
||
|
||
|
||
def rewrite_href(href: str) -> str:
|
||
"""
|
||
Rewrite likely-internal Foswiki links to .md targets.
|
||
|
||
Handles:
|
||
- Foo.html -> Foo.md
|
||
- Foo%3Fcover=print%3B.html -> Foo.md
|
||
- .../viewauth/CSAXS/Foo -> Foo.md
|
||
- Foo.html#Anchor -> Foo.md#Anchor
|
||
Leaves external links unchanged.
|
||
"""
|
||
if not href:
|
||
return href
|
||
|
||
# External links untouched
|
||
if re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", href):
|
||
return href
|
||
|
||
# strip common Foswiki export artifacts like Foo%3Frestrictions=on.html
|
||
# Keep anchor part if present.
|
||
anchor = ""
|
||
if "#" in href:
|
||
href, anchor = href.split("#", 1)
|
||
anchor = "#" + anchor if anchor else ""
|
||
|
||
# decode only the simple "%3F..." pattern we see in exports
|
||
href = re.sub(r"%3F.*$", "", href, flags=re.IGNORECASE)
|
||
|
||
# If it ends in .html, map to .md
|
||
if href.lower().endswith(".html"):
|
||
base = Path(href).name
|
||
return base[:-5] + ".md" + anchor # replace .html
|
||
|
||
# Foswiki "view" style: .../viewauth/WEB/TOPIC (no extension)
|
||
m = re.search(r"/viewauth/[^/]+/([^/?#]+)$", href)
|
||
if m:
|
||
return m.group(1) + ".md" + anchor
|
||
m = re.search(r"/view/[^/]+/([^/?#]+)$", href)
|
||
if m:
|
||
return m.group(1) + ".md" + anchor
|
||
|
||
# otherwise keep relative href as-is (plus any anchor)
|
||
return href + anchor
|
||
|
||
|
||
# -------- invisible characters --------
|
||
|
||
|
||
def strip_invisible_unicode(s: str) -> str:
|
||
# Keep this list small and explicit
|
||
mapping = {
|
||
"\u00a0": " ", # NBSP
|
||
"\u200b": "", # zero-width space
|
||
"\u200c": "",
|
||
"\u200d": "",
|
||
"\u200e": "",
|
||
"\u200f": "",
|
||
"\ufeff": "", # BOM
|
||
# Windows-1252 punctuation bytes
|
||
"\x91": "‘",
|
||
"\x92": "’",
|
||
"\x93": "“",
|
||
"\x94": "”",
|
||
"\x96": "–",
|
||
"\x97": "—",
|
||
"\x85": "…",
|
||
}
|
||
for k, v in mapping.items():
|
||
s = s.replace(k, v)
|
||
return s
|
||
|
||
|
||
# -------- HTML -> Markdown rendering --------
|
||
|
||
|
||
def safe_classes(el: Tag) -> list[str]:
|
||
try:
|
||
return el.get("class") or []
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
def cleanup_topic_dom(topic: Tag) -> None:
|
||
# Remove TOC blocks
|
||
for el in list(topic.select(".foswikiToc, #foswikiTOC, #foswikiTOC *")):
|
||
try:
|
||
el.decompose()
|
||
except Exception:
|
||
pass
|
||
|
||
# Remove EditChapterPlugin controls WITHOUT unwrapping anything
|
||
for el in list(topic.select(".ecpEdit, img[src*='EditChapterPlugin']")):
|
||
try:
|
||
el.decompose()
|
||
except Exception:
|
||
pass
|
||
|
||
# Remove horizontal rules
|
||
for hr in list(topic.find_all("hr")):
|
||
try:
|
||
hr.decompose()
|
||
except Exception:
|
||
pass
|
||
|
||
# Remove empty paragraphs
|
||
for p in list(topic.find_all("p")):
|
||
if not isinstance(p, Tag):
|
||
continue
|
||
if not p.get_text(strip=True) and not p.find(["img", "a", "code", "tt"]):
|
||
try:
|
||
p.decompose()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def table_md(table: Tag) -> str:
|
||
# Extract rows
|
||
rows = []
|
||
for tr in table.find_all("tr", recursive=True):
|
||
cells = tr.find_all(["th", "td"], recursive=False)
|
||
if not cells:
|
||
# sometimes <tr><td> nested; fall back to recursive
|
||
cells = tr.find_all(["th", "td"])
|
||
row = [
|
||
re.sub(r"\s+", " ", "".join(inline_md(c) for c in cell.children).strip())
|
||
for cell in cells
|
||
]
|
||
if row:
|
||
rows.append((cells, row))
|
||
|
||
if not rows:
|
||
return ""
|
||
|
||
# Decide header
|
||
header = None
|
||
body_rows = []
|
||
|
||
# If any <th> in first row => header
|
||
first_cells, first_row = rows[0]
|
||
if any(c.name.lower() == "th" for c in first_cells):
|
||
header = first_row
|
||
body_rows = [r for _, r in rows[1:]]
|
||
else:
|
||
# Otherwise, treat first row as header (common in wikis)
|
||
header = first_row
|
||
body_rows = [r for _, r in rows[1:]]
|
||
|
||
ncols = max(len(header), *(len(r) for r in body_rows)) if body_rows else len(header)
|
||
|
||
def pad(r):
|
||
return r + [""] * (ncols - len(r))
|
||
|
||
header = pad(header)
|
||
body_rows = [pad(r) for r in body_rows]
|
||
|
||
# Escape pipes
|
||
def esc(cell: str) -> str:
|
||
return cell.replace("|", r"\|")
|
||
|
||
header = [esc(c) for c in header]
|
||
body_rows = [[esc(c) for c in r] for r in body_rows]
|
||
|
||
# Build markdown
|
||
out = []
|
||
out.append("| " + " | ".join(header) + " |")
|
||
out.append("| " + " | ".join(["---"] * ncols) + " |")
|
||
for r in body_rows:
|
||
out.append("| " + " | ".join(r) + " |")
|
||
|
||
return "\n" + "\n".join(out) + "\n"
|
||
|
||
|
||
def inline_md(node: Tag | NavigableString) -> str:
|
||
"""Convert inline HTML within a block element to Markdown."""
|
||
if isinstance(node, NavigableString):
|
||
return str(node)
|
||
|
||
if not isinstance(node, Tag):
|
||
return ""
|
||
|
||
name = node.name.lower()
|
||
|
||
if name == "code" or name == "tt":
|
||
txt = node.get_text()
|
||
txt = txt.replace("`", r"\`")
|
||
return f"`{txt}`"
|
||
|
||
if name in ("em", "i"):
|
||
return f"*{''.join(inline_md(c) for c in node.children).strip()}*"
|
||
|
||
if name in ("strong", "b"):
|
||
return f"**{''.join(inline_md(c) for c in node.children).strip()}**"
|
||
|
||
if name == "a":
|
||
href = rewrite_href(node.get("href", ""))
|
||
label = "".join(inline_md(c) for c in node.children).strip()
|
||
# skip empty labels (icons etc.)
|
||
if not label:
|
||
return ""
|
||
return f"[{label}]({href})"
|
||
|
||
if name == "br":
|
||
return "\n"
|
||
|
||
# default: recurse into children
|
||
return "".join(inline_md(c) for c in node.children)
|
||
|
||
|
||
def block_md(el: Tag) -> str:
|
||
"""Convert block-level elements to Markdown."""
|
||
name = el.name.lower()
|
||
|
||
if name in ("h1", "h2", "h3", "h4"):
|
||
level = int(name[1])
|
||
title = "".join(inline_md(c) for c in el.children).strip()
|
||
# remove redundant whitespace
|
||
title = re.sub(r"\s+", " ", title)
|
||
return f"\n{'#' * level} {title}\n"
|
||
|
||
if name == "p":
|
||
txt = "".join(inline_md(c) for c in el.children).strip()
|
||
txt = re.sub(r"\s+", " ", txt)
|
||
return f"\n{txt}\n" if txt else ""
|
||
|
||
if name in ("ul", "ol"):
|
||
out = []
|
||
ordered = name == "ol"
|
||
idx = 1
|
||
for li in el.find_all("li", recursive=False):
|
||
item = "".join(inline_md(c) for c in li.children).strip()
|
||
item = re.sub(r"\s+", " ", item)
|
||
if not item:
|
||
continue
|
||
prefix = f"{idx}." if ordered else "-"
|
||
out.append(f"{prefix} {item}")
|
||
idx += 1
|
||
return "\n" + "\n".join(out) + "\n" if out else ""
|
||
|
||
if name == "pre":
|
||
txt = el.get_text()
|
||
return "\n```text\n" + txt.rstrip("\n") + "\n```\n"
|
||
|
||
# Very conservative: ignore everything else for now (tables later, etc.)
|
||
return ""
|
||
|
||
|
||
def flush_paragraph(buf):
|
||
txt = " ".join(buf).strip()
|
||
txt = re.sub(r"\s+", " ", txt)
|
||
return f"\n{txt}\n" if txt else ""
|
||
|
||
|
||
def convert_one(html_path: Path) -> str:
|
||
soup = BeautifulSoup(html_path.read_text(encoding="utf-8", errors="ignore"), "lxml")
|
||
|
||
topic = soup.select_one("#patternMainContents div.foswikiTopic")
|
||
if not topic:
|
||
raise RuntimeError(
|
||
"Could not locate topic body: '#patternMainContents div.foswikiTopic'"
|
||
)
|
||
|
||
cleanup_topic_dom(topic)
|
||
|
||
prov = extract_provenance(soup)
|
||
|
||
parts = []
|
||
para_buf = []
|
||
parts.append("<!--")
|
||
parts.append(f"source: {html_path.name}")
|
||
for k in ("foswiki_revision", "foswiki_last_modified", "foswiki_last_author"):
|
||
if k in prov:
|
||
parts.append(f"{k}: {prov[k]}")
|
||
parts.append("-->\n")
|
||
|
||
# Walk only meaningful top-level blocks within the topic
|
||
|
||
for node in topic.children:
|
||
# Headings: flush paragraph, then emit heading
|
||
if isinstance(node, Tag) and node.name in ("h1", "h2", "h3", "h4"):
|
||
if para_buf:
|
||
parts.append(flush_paragraph(para_buf))
|
||
para_buf = []
|
||
|
||
level = int(node.name[1])
|
||
title = "".join(inline_md(c) for c in node.children).strip()
|
||
title = re.sub(r"\s+", " ", title)
|
||
parts.append(f"\n{'#' * level} {title}\n")
|
||
continue
|
||
|
||
# Lists: flush paragraph, then emit list
|
||
if isinstance(node, Tag) and node.name in ("ul", "ol"):
|
||
if para_buf:
|
||
parts.append(flush_paragraph(para_buf))
|
||
para_buf = []
|
||
|
||
parts.append(block_md(node))
|
||
continue
|
||
|
||
# Line breaks: treat as paragraph separators
|
||
if isinstance(node, Tag) and node.name == "br":
|
||
if para_buf:
|
||
parts.append(flush_paragraph(para_buf))
|
||
para_buf = []
|
||
continue
|
||
|
||
# tables
|
||
if isinstance(node, Tag) and node.name == "table":
|
||
if para_buf:
|
||
parts.append(flush_paragraph(para_buf))
|
||
para_buf = []
|
||
parts.append(table_md(node))
|
||
continue
|
||
|
||
# Plain text or inline markup
|
||
txt = inline_md(node).strip()
|
||
if txt:
|
||
para_buf.append(txt)
|
||
|
||
# flush trailing paragraph
|
||
if para_buf:
|
||
parts.append(flush_paragraph(para_buf))
|
||
|
||
md = "\n".join(parts)
|
||
|
||
# normalize excessive blank lines
|
||
md = "\n".join(parts)
|
||
|
||
# remove invisible characters
|
||
|
||
md = strip_invisible_unicode(md)
|
||
|
||
# fix small systematic artefacts (quotes, i.e. spacing, etc.)
|
||
md = postprocess_md(md)
|
||
|
||
# 1) remove embedded Foswiki footer remnants from body text
|
||
# Case A: "-- Main.Topic - 02 Feb 2010" (or any Web.Topic)
|
||
md = re.sub(
|
||
r"\s+--\s+[A-Za-z0-9_]+\.[A-Za-z0-9_]+\s*-\s*\d{2}\s+\w+\s+\d{4}\s*$",
|
||
"",
|
||
md,
|
||
flags=re.MULTILINE,
|
||
)
|
||
# Case B: "-- [Someone](link) - 24 Sep 2008" (often inline at end)
|
||
md = re.sub(
|
||
r"\s+--\s+\[[^\]]+\]\([^)]+\)\s*-\s*\d{2}\s+\w+\s+\d{4}\s*$",
|
||
"",
|
||
md,
|
||
flags=re.MULTILINE,
|
||
)
|
||
|
||
# 2) add standardized footer (your preferred style)
|
||
page = html_path.stem
|
||
date = prov.get("foswiki_last_modified", "")
|
||
author = prov.get("foswiki_last_author", "").strip()
|
||
|
||
label = author if author else page
|
||
|
||
if date:
|
||
md = md.rstrip() + f"\n\n---\n[{label}]({label}.md) - {date}\n"
|
||
else:
|
||
md = md.rstrip() + f"\n\n---\n[{label}]({label}.md)\n"
|
||
|
||
# 3) normalize excessive blank lines
|
||
md = re.sub(r"\n{3,}", "\n\n", md).strip() + "\n"
|
||
return md
|
||
|
||
|
||
def postprocess_md(md: str) -> str:
|
||
# fix patterns like: `in press' -> `in press`
|
||
md = re.sub(r"`([^`]+)'\b", r"`\1`", md)
|
||
|
||
# fix patterns like: `nanofish' -> `nanofish`
|
||
md = re.sub(r"`([^`]+)'\b", r"`\1`", md)
|
||
|
||
# tighten spacing around italic abbreviations: "*i.e.* ," -> "*i.e.*,"
|
||
md = re.sub(r"\*i\.e\.\*\s+,", "*i.e.*,", md)
|
||
|
||
return md
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("input", type=Path, help="HTML file or directory")
|
||
ap.add_argument(
|
||
"-o",
|
||
"--output",
|
||
type=Path,
|
||
required=True,
|
||
help="Output .md file or output directory",
|
||
)
|
||
args = ap.parse_args()
|
||
|
||
inp: Path = args.input
|
||
out: Path = args.output
|
||
|
||
if inp.is_file():
|
||
md = convert_one(inp)
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text(md, encoding="utf-8")
|
||
return
|
||
|
||
if inp.is_dir():
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
for html_path in sorted(inp.glob("*.html")):
|
||
md = convert_one(html_path)
|
||
md_path = out / (html_path.stem + ".md")
|
||
md_path.write_text(md, encoding="utf-8")
|
||
return
|
||
|
||
raise SystemExit("Input must be a file or directory.")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|