#!/usr/bin/env python3 """ linkcheck.py — the gate that runs before every deploy. There is no build step on this site, so nothing else catches a reference to a file that is not there. This walks every HTML page and resolves every local reference against the tree on disk. Exits non-zero if any of them is missing. It exists because the episodes page once referenced /episodes/ep03/ep03-540.mp4 for twenty-three minutes before the file existed. A Netlify deploy is a full snapshot: a page that references a file that was never staged is a broken page for everybody, immediately, with nothing in the owner's own view to show why. Usage: python3 tools/linkcheck.py [site-root] # default: the parent of tools/ Exit codes: 0 every local reference resolves 1 at least one does not """ import os import re import sys from urllib.parse import unquote # Attributes that can carry a URL on this site, including the data-* ones the # player and the connection-aware loaders read. # `content=` is deliberately absent: on this site it is almost always meta # prose or a viewport string, and the one case that is a path (og:image) is an # absolute URL, which is skipped anyway. Including it produced false failures # on "video.other" and "width=device-width, initial-scale=1.0". ATTRS = ( "src|href|srcset|poster|" "data-src|data-play|data-lo|data-hi|data-loop|data-poster" ) PAT = re.compile( r'(?:' + ATTRS + r')\s*=\s*["\']([^"\']+)["\']' r'|url\(\s*(["\']?)([^)"\']+)\2\s*\)', re.I, ) # content="..." is overwhelmingly meta prose, not a path. Only treat it as a # reference when it actually looks like one. SKIP_PREFIX = ("http://", "https://", "//", "mailto:", "tel:", "sms:", "data:", "javascript:", "#") def looks_like_path(u: str) -> bool: if not u or u.startswith(SKIP_PREFIX): return False if " " in u.strip(): # meta descriptions, viewport strings, etc. return False return "/" in u or "." in u def refs_in(text: str): for m in PAT.finditer(text): raw = m.group(1) or m.group(3) if not raw: continue # srcset is the only comma-separated form; everything else is one URL parts = raw.split(",") if m.re.pattern and "srcset" in (m.group(0)[:8].lower()) else [raw] for part in parts: u = unquote(part.strip().split()[0] if part.strip() else "") u = u.split("?")[0].split("#")[0] if looks_like_path(u): yield u def resolve(root: str, page_dir: str, u: str) -> str: p = u[1:] if u.startswith("/") else os.path.normpath(os.path.join(page_dir, u)) full = os.path.join(root, p) if u.endswith("/") or os.path.isdir(full): full = os.path.join(full, "index.html") return full def main() -> int: root = sys.argv[1] if len(sys.argv) > 1 else \ os.path.dirname(os.path.dirname(os.path.abspath(__file__))) root = os.path.abspath(root) pages, checked, missing = [], 0, [] for dirpath, dirnames, filenames in os.walk(root): dirnames[:] = [d for d in dirnames if d not in (".git", "node_modules", "__livecopy", "tools")] for f in filenames: if f.endswith(".html"): # /failures/repro/ holds SPECIMENS: files whose whole purpose is to # exhibit a defect. Their dead links and missing media are the # exhibits, not mistakes — this gate convicted all three the first # time it saw them, which is the correct behaviour and the wrong # verdict. A corpus of deliberate defects trips every gate you own, # and that is worth knowing before you cultivate one. rp = os.path.relpath(os.path.join(dirpath, f), root).replace(os.sep, "/") if rp.startswith("failures/repro/"): continue pages.append(os.path.join(dirpath, f)) for page in sorted(pages): rel_dir = os.path.dirname(os.path.relpath(page, root)) try: text = open(page, encoding="utf-8", errors="ignore").read() except OSError as e: missing.append((os.path.relpath(page, root), "", str(e))) continue for u in refs_in(text): checked += 1 full = resolve(root, rel_dir, u) # a pretty URL may be served as file.html if not os.path.exists(full) and not os.path.exists(full + ".html"): missing.append((os.path.relpath(page, root), u, full)) # ---- fragment gate --------------------------------------------------- # A href="/findings/#if-someone-calls" pointing at an id that does not # exist does not 404. The browser lands silently at the top of the page, # which renders identically to success. That is a failure mode the file # check above is structurally unable to see, so it gets its own check. src = {} for page in pages: rel = os.path.relpath(page, root).replace(os.sep, "/") try: src[rel] = open(page, encoding="utf-8", errors="ignore").read() except OSError: pass def ids_of(text): return set(re.findall(r'\bid="([^"]+)"', text)) | set(re.findall(r'\bname="([^"]+)"', text)) frags_checked = 0 for rel, text in src.items(): for href in re.findall(r'href="([^"]*#[^"]+)"', text): path, frag = href.split("#", 1) if not frag or frag == "top": continue if path.startswith(("http", "mailto", "tel", "sms", "//")): continue if path == "": target = rel else: t = os.path.normpath(os.path.join(os.path.dirname(rel), path)).replace(os.sep, "/") if t.endswith("/"): t += "index.html" elif not t.endswith(".html"): t = t.rstrip("/") + "/index.html" target = t.lstrip("./") frags_checked += 1 if target not in src: missing.append((rel, href, f"no such page: {target}")) elif frag not in ids_of(src[target]): missing.append((rel, href, f"no id=\"{frag}\" in {target}")) print(f"linkcheck: {len(pages)} pages, {checked} local references, {frags_checked} fragments") if missing: print(f"linkcheck: {len(missing)} MISSING\n") for page, u, full in missing: print(f" {page}") print(f" -> {u}") print(f" (looked for {full})") return 1 print("linkcheck: all references resolve") return 0 if __name__ == "__main__": sys.exit(main())