build.py

#!/usr/bin/env python3
"""content/ -> data/.  No dependencies, no network (except --validate).

    python3 build.py                 build everything into data/
    python3 build.py --check FILE...  validate content files, print "N clean" each
    python3 build.py --validate      run every starter and solution past all three judges
"""
from __future__ import annotations

import argparse
import ast
import concurrent.futures
import difflib
import json
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path

ROOT = Path(__file__).parent
CONTENT = ROOT / "content"
DATA = ROOT / "data"
CACHE = ROOT / ".cache"

# ---------------------------------------------------------------- the manifest

ACCENTS = ["gold", "denim", "ember", "moss", "teal", "plum", "clay"]

PHASES = [
    ("The machine", "Get to the object model before you write a loop."),
    ("Control and structure", "The shapes a program is built out of."),
    ("Data", "The four containers you will use for the rest of your life."),
    ("Iteration", "Python's crown jewel, three units deep."),
    ("Objects", "What a class actually is, all the way down to the descriptor."),
    ("Types", "Optional, gradual, and not enforced at runtime. All three matter."),
    ("Code as data", "Programs that read and rewrite programs."),
    ("Programs, not scripts", "The layer where a folder of files becomes software."),
    ("Concurrency and performance", "Doing more at once, and doing it faster."),
    ("The wider world", "The ecosystem, and the one skill that outlives this book."),
]

# (slug, phase, title, blurb)
TRACK = [
    ("00-toolchain", 0, "The toolchain",
     "What `python app.py` actually does between you pressing enter and the program starting: bytecode, `__pycache__`, and why Python has no compile step but definitely has a compiler."),
    ("01-names", 0, "Names and objects",
     "Python has no variables. `a = b` binds a second name to one object, and almost every surprising bug in this book begins by forgetting that."),
    ("02-mutability", 0, "Mutability and aliasing",
     "The list that changed when you were not looking. Aliasing, shallow copies, and the default argument that is created once and kept for the life of the process."),
    ("03-data-model", 0, "The data model",
     "Everything is an object and every operator is a method call. Duck typing stated precisely enough that you can predict what will break."),
    ("04-equality", 0, "Equality, hashing, truthiness",
     "The `__eq__` and `__hash__` contract, why a mutable dict key is a bug lying in wait, and what `if x:` actually calls."),
    ("05-expressions", 1, "Expressions and statements",
     "What counts as an expression, what the walrus is for, and the evaluation order you have been assuming without ever checking."),
    ("06-control-flow", 1, "Control flow",
     "`for`/`else`, the loop that finished versus the loop that broke, and `match`, the newest control structure in the language and the least used."),
    ("07-functions", 1, "Functions",
     "Positional-only, keyword-only, `*args`, `**kwargs`, and the single most important fact about defaults: when they are evaluated."),
    ("08-scope", 1, "Scope and closures",
     "LEGB, the closure that captured a variable rather than a value, and why every function you made in that loop returned the same answer."),
    ("09-exceptions", 1, "Exceptions",
     "A recoverable error is something you catch; a bug is a traceback. `raise ... from`, EAFP over LBYL, and the exception groups 3.11 added."),
    ("10-sequences", 2, "Sequences and slicing",
     "A slice is an object. Negative steps, slice assignment, and why `a[:]` copies but `a[:] = b` does not."),
    ("11-strings", 2, "Strings, bytes and encoding",
     "`str` is not `bytes`, and the difference will find you at a file boundary. Encodings, the format spec mini-language, and `!r`."),
    ("12-dicts", 2, "Dicts and sets",
     "Hashing, the insertion order that became a language guarantee in 3.7, views, and the shape `setdefault` and `defaultdict` exist to fill."),
    ("13-comprehensions", 2, "Comprehensions and collections",
     "A comprehension is a compiled construct with its own scope, not sugar for a loop. Plus the five `collections` types actually worth knowing."),
    ("14-sorting", 2, "Sorting and ordering",
     "Key functions, why stability matters more than you expect, and the two stdlib modules that replace half the sorting code you would write."),
    ("15-iterators", 3, "The iterator protocol",
     "`__iter__` and `__next__`, exhaustion, and the reason `for` works on things that are not sequences at all."),
    ("16-generators", 3, "Generators",
     "`yield` turns a function into a resumable object. Lazy pipelines, `send`, `throw`, and `yield from`."),
    ("17-itertools", 3, "itertools and functools",
     "Seventy building blocks that compose, `cache` and `singledispatch`, and an honest verdict on `map`, `filter` and `reduce`."),
    ("18-classes", 4, "Classes",
     "`__new__` versus `__init__`, what a method actually is, and the class attribute that everyone mutates by accident exactly once."),
    ("19-attributes", 4, "Attribute access",
     "`__dict__`, `__getattr__` versus `__getattribute__`, `__slots__`, and `property` as the first descriptor you ever meet."),
    ("20-descriptors", 4, "Descriptors",
     "The mechanism underneath `property`, methods, `classmethod` and `staticmethod`. One protocol explains all four, and this is the unit where the language clicks."),
    ("21-mro", 4, "Inheritance and the MRO",
     "C3 linearisation, what `super()` is really doing (not what you think), and the narrow case where multiple inheritance is the right call."),
    ("22-protocols", 4, "Dunder protocols",
     "Operator overloading, `__repr__` versus `__str__`, what `with` compiles to, and a container you can implement in four methods."),
    ("23-dataclasses", 4, "Modern data modelling",
     "`dataclass`, `NamedTuple` and `Enum`, and the precise point at which `attrs` and `pydantic` start earning their dependency."),
    ("24-typing", 5, "Type hints",
     "Annotations are runtime metadata, not enforcement. Generics, `TypeVar`, `Literal`, `TypedDict`, and `Protocol`: structural typing, finally."),
    ("25-typecheck", 5, "Type checking in practice",
     "mypy's strictness dials, the five errors you will actually hit, how to type a codebase that has none, and the point where types stop paying."),
    ("26-decorators", 6, "Decorators",
     "A decorator is a function that takes a function. `functools.wraps`, parameters, and why the ones in real libraries look nothing like the tutorial ones."),
    ("27-metaclasses", 6, "Metaclasses and __init_subclass__",
     "`type` is a callable that makes classes. When a metaclass is genuinely right (rarely) and why `__init_subclass__` usually beats reaching for one."),
    ("28-ast", 6, "Introspection and the AST",
     "`inspect`, `dis` and `ast`. Read your own bytecode, then write a transformer that rewrites source before it ever runs."),
    ("29-modules", 7, "Modules, packages, imports",
     "The import system in full: `sys.path`, packages, relative imports, and the circular import you can finally diagnose instead of shuffling."),
    ("30-packaging", 7, "Packaging and environments",
     "`pyproject.toml`, `uv`, and shipping a wheel. From a folder of scripts to something a stranger can install by name."),
    ("31-testing", 7, "Testing",
     "pytest for real: fixtures, `parametrize`, honest mocking, and property-based tests that find the input you would never have thought of."),
    ("32-tooling", 7, "Tooling and practice",
     "ruff, mypy and pre-commit as one pipeline, `logging` instead of `print`, and the handful of practices that survive contact with a team."),
    ("33-concurrency", 8, "Concurrency models",
     "The GIL, described accurately for once, including the free-threaded build. Threads, processes and async, and which one your problem actually needs."),
    ("34-async", 8, "async and await",
     "A coroutine does nothing until something awaits it. The event loop, `TaskGroup`, and why one blocking call stalls the entire program."),
    ("35-performance", 8, "Performance",
     "Profile first, always. `cProfile`, `timeit`, the algorithmic win versus the micro one, and where the C boundary changes every rule."),
    ("36-memory", 8, "Memory and the runtime",
     "Reference counting plus a cycle collector. Weak references, why `__del__` is a trap, and the reason `getsizeof` lies to you."),
    ("37-ecosystem", 9, "The ecosystem, mapped",
     "A map, not a tutorial: the stdlib modules you will really use and the dozen packages worth knowing, each with the case where it is the wrong answer."),
    ("38-tracebacks", 9, "Reading the traceback",
     "Every other unit teaches a topic. This one teaches reading the traceback, which is what makes the next error survivable, including errors this book never covers."),
]

# (slug, tier, domain, stages, minutes, title, blurb)
PROJECTS = [
    ("bloom-filter", "mini", "data", 4, 45, "A Bloom filter",
     "A set that answers \"definitely not\" or \"probably yes\" in constant space, built on a bit array and k hashes, with the false positive rate measured against the formula."),
    ("lru-cache", "mini", "systems", 4, 50, "An LRU cache",
     "A dict and a doubly linked list, O(1) on both ends, then raced against `functools.lru_cache` to find out what the standard library is doing differently."),
    ("retry-decorator", "mini", "tools", 4, 40, "A retry library",
     "Decorators with parameters, exponential backoff with jitter, and the exception-filtering predicate that turns a toy into something you would actually deploy."),
    ("regex-engine", "mini", "languages", 4, 60, "A regex engine",
     "Parse a pattern into a tree and walk it with a backtracking matcher, then find the pattern that makes your own engine take exponential time."),
    ("ngram", "core", "ai", 8, 90, "An n-gram language model",
     "Count the contexts, smooth the counts so unseen words do not get probability zero, sample from the distribution, and read the text it writes back at you."),
    ("micrograd", "core", "ai", 8, 120, "micrograd",
     "A `Value` that remembers how it was computed, a backward pass over the graph, an MLP built from it, and a loss curve that actually falls."),
    ("bpe-tokenizer", "core", "ai", 8, 90, "A BPE tokenizer",
     "The algorithm GPT and Llama really use: merge the commonest byte pair, over and over, until text is integers and the round trip is exact."),
    ("json-parser", "core", "languages", 8, 90, "A JSON parser",
     "A tokenizer and a recursive descent parser that reads a real document and, when it fails, names the exact character it failed on."),
    ("test-framework", "core", "tools", 8, 110, "A test framework",
     "Rebuild the useful third of pytest: collection, fixtures with teardown, and assertion rewriting through the `ast` module so a bare `assert` explains itself."),
    ("orm", "core", "web", 8, 110, "An ORM",
     "Descriptors for the fields, a metaclass for the table, and generated SQL. After this, Django and SQLAlchemy stop being magic and start being code."),
    ("async-crawler", "core", "web", 8, 100, "An async crawler",
     "`asyncio` under load: a worker pool, a rate limiter, `TaskGroup` for structured concurrency, and backpressure so the queue cannot eat your memory."),
    ("kv-store", "core", "systems", 8, 100, "A key-value store",
     "An append-only write-ahead log, an in-memory index, compaction, and a crash in the middle of a write that the store recovers from."),
    ("cli-to-pypi", "core", "tools", 8, 80, "A CLI, shipped",
     "From `__main__.py` to a wheel on an index: argument parsing, entry points, `pyproject.toml`, a test matrix, and a version someone else can install."),
    ("web-framework", "core", "web", 8, 110, "A web framework",
     "ASGI from the specification up: routing, middleware as a callable chain, request and response objects, and dependency injection in about forty lines."),
    ("build-a-gpt", "deep", "ai", 12, 240, "Build a GPT",
     "Tokenizer to embeddings to attention to a training loop to sampling. Consumes the BPE tokenizer and micrograd you already built, and ends with a model writing text."),
]

# The four verdict kinds. `silent` is the one Rust cannot have: every judge is
# happy and the code is still wrong.
VERDICTS = {"ruff", "mypy", "raises", "silent"}

# The tier and the domain are read straight out of PROJECTS and written into
# the manifest, where the frontend groups by them. An unrecognised value is not
# an error anywhere downstream: it is a heading nobody wrote and a filter
# nothing matches, and the only way to find it is to look at the rendered page.
TIERS = {"mini", "core", "deep"}
DOMAINS = {"data", "systems", "tools", "languages", "ai", "web"}

# The one description of the three judges. build.py runs them from here and the
# browser fetches this as data/judges.json, so what --validate calls clean and
# what a reader is told is clean cannot drift apart.
JUDGES = {
    "ruff": {
        "version": "0.16.5",
        "cdn": "https://cdn.jsdelivr.net/npm/@astral-sh/ruff-wasm-web@0.16.5/",
        "select": ["E", "F", "B", "SIM", "UP"],
        # line length is a formatting opinion, not a teaching signal
        "ignore": ["E501"],
        "lineLength": 88,
        # without this, ruff assumes an older Python and reports 3.11+ builtins
        # such as ExceptionGroup as undefined names
        "targetVersion": "py314",
    },
    "mypy": {
        "flags": ["--no-color-output", "--no-error-summary", "--hide-error-context"],
        # micropip does not resolve mypy's transitive dependencies in Pyodide
        "install": ["mypy_extensions", "pathspec", "tomli", "mypy"],
        # typing-extensions ships inside the Pyodide distribution
        "preload": ["micropip", "typing-extensions"],
    },
    "cpython": {
        "version": "3.14",
        "cdn": "https://cdn.jsdelivr.net/pyodide/v314.0.6/full/",
    },
}

# The four verdict kinds under the names the errors page groups them by, with the
# heading and blurb it renders. Kept here so the page cannot drift from the
# vocabulary the content is written in.
JUDGE_GROUP = {"ruff": "ruff", "mypy": "mypy", "raises": "runtime", "silent": "reading"}

# ---------------------------------------------------------------- parsing

FM = re.compile(r"\A---\n(.*?)\n---\n", re.S)
FENCE = re.compile(r"^~~~(\w+)\n(.*?)^~~~$", re.M | re.S)


def front_matter(text: str, path: Path) -> tuple[dict, str]:
    m = FM.match(text)
    if not m:
        die(path, "missing YAML front matter")
    meta: dict[str, object] = {}
    for line in m.group(1).splitlines():
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        if ":" not in line:
            die(path, f"front matter line is not key: value -> {line!r}")
        k, _, v = line.partition(":")
        v = v.strip()
        if v.startswith("[") and v.endswith("]"):
            meta[k.strip()] = [s.strip() for s in v[1:-1].split(",") if s.strip()]
        else:
            meta[k.strip()] = v

    body = text[m.end():]
    # Every parser comes through here, so the house-style and cross-reference
    # checks live here too rather than being pasted onto each one. Hanging them
    # off the parsers left the glossary and the projects silently exempt, and
    # made each parser re-derive the front matter's length to report a line.
    lines_before = text.count("\n", 0, m.end())
    _check_prose(path, body, lines_before)
    _check_cross_references(path, body)
    return meta, body


def die(path: Path, msg: str) -> None:
    print(f"{path}: {msg}", file=sys.stderr)
    sys.exit(1)


def slugify(s: str) -> str:
    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")


def strip_code(text: str, blank: bool = False) -> str:
    """The prose of a content file: fenced blocks, exercise blocks, inline code.

    `blank=True` replaces the code with spaces rather than removing it, so every
    newline survives and an offset into the result is still an offset into the
    original. That is what lets a style complaint name the line to open.
    """
    out = (lambda m: re.sub(r"[^\n]", " ", m.group(0))) if blank else ""
    text = re.sub(r"^```.*?^```", out, text, flags=re.M | re.S)
    text = re.sub(r"^~~~.*?^~~~", out, text, flags=re.M | re.S)
    return re.sub(r"`[^`]*`", out, text)


def word_count(s: str) -> int:
    # prose only: fenced code and inline code do not count toward the budget
    return len(strip_code(s).split())


# ---------------------------------------------------------------- units

# What the browser's markdown subset cannot draw. Anything here would reach the
# reader as raw source, so the build refuses it. test_frontend.mjs asserts the
# other half, that md() really cannot render each one, so the two lists fail
# together instead of drifting: this is the contract between build.py and md().
UNSUPPORTED_MARKDOWN = {
    "blockquote": r"^> ",
    "image": r"!\[",
    "heading deeper than ####": r"^#{5,} ",
    "setext heading": r"^=+$",
    "html block": r"^<\w+",
    "footnote": r"^\[\^",
}

NOTE_MIN, NOTE_MAX = 1400, 2600
EXERCISES_PER_UNIT = 8
DRILLS_PER_UNIT = 15


def parse_unit(path: Path) -> dict:
    meta, body = front_matter(path.read_text(), path)
    # Every other kind of file is keyed by its filename. A unit declares its
    # slug as well, so the two can disagree, and a copied front matter block
    # would then write one unit's JSON from another unit's file and report the
    # first as merely incomplete.
    if meta.get("slug") != path.stem:
        die(path, f"front matter says slug {meta.get('slug')!r}, "
                  f"but the file is called {path.stem!r}")
    words = word_count(body)
    if not (NOTE_MIN <= words <= NOTE_MAX):
        die(path, f"note is {words} words, must be {NOTE_MIN}-{NOTE_MAX}")

    # The browser renders these notes with a small hand-written markdown
    # subset. Anything it cannot draw would be shown to the reader as raw
    # source, so the build refuses it rather than letting it through.
    prose = strip_code(body)
    for label, pat in UNSUPPORTED_MARKDOWN.items():
        if re.search(pat, prose, re.M):
            die(path, f"note uses {label}, which the renderer does not support")

    sections = []
    for m in re.finditer(r"^## (.+)$", body, re.M):
        sections.append({"title": m.group(1).strip(), "id": slugify(m.group(1))})
    if len(sections) < 3:
        die(path, f"note has {len(sections)} `## ` sections, needs at least 3")

    return {
        "slug": meta["slug"],
        "title": meta["title"],
        "body": body.strip(),
        "sections": sections,
        "words": words,
    }


# ---------------------------------------------------------------- exercises

DIRECTIVE = re.compile(r"^@(expect|hint|diagnose|goal)[ \t]+(.+)$", re.M)


def parse_exercises(path: Path) -> list[dict]:
    meta, body = front_matter(path.read_text(), path)
    chunks = re.split(r"^## ", body, flags=re.M)[1:]
    if not chunks:
        die(path, "no exercises found (need `## ` headings)")

    out = []
    for i, chunk in enumerate(chunks, 1):
        title, _, rest = chunk.partition("\n")
        blocks = {m.group(1): m.group(2).rstrip() for m in FENCE.finditer(rest)}
        for need in ("starter", "tests", "solution"):
            if need not in blocks:
                die(path, f"exercise {i} ({title.strip()}) has no ~~~{need} block")

        expects, hints, diagnose = [], [], {}
        for kind, value in DIRECTIVE.findall(rest):
            if kind == "expect":
                if ":" in value:
                    judge, _, code = value.partition(":")
                else:
                    judge, code = value.strip(), ""
                judge = judge.strip()
                if judge not in VERDICTS:
                    die(path, f"exercise {i}: @expect {judge!r} not one of {sorted(VERDICTS)}")
                expects.append({"judge": judge, "code": code.strip()})
            elif kind == "hint":
                hints.append(value.strip())
            else:
                code, _, prose = value.partition(" ")
                if not prose.strip():
                    die(path, f"exercise {i}: @diagnose {code} has no prose")
                diagnose[code.strip()] = prose.strip()

        if not expects:
            die(path, f"exercise {i} ({title.strip()}) has no @expect")
        if not hints:
            die(path, f"exercise {i} ({title.strip()}) has no @hint")
        for e in expects:
            key = e["code"] or e["judge"]
            if key not in diagnose:
                die(path, f"exercise {i}: @expect {key} has no matching @diagnose")

        prompt = FENCE.sub("", DIRECTIVE.sub("", rest)).strip()
        if stray := re.search(r"^@(\w+)", prompt, re.M):
            die(path, f"exercise {i}: @{stray.group(1)} is not a directive")
        if len(prompt.split()) < 15:
            die(path, f"exercise {i} ({title.strip()}) prompt is too short to be a prompt")

        out.append({
            "n": i,
            "title": title.strip(),
            "prompt": prompt,
            "expects": expects,
            "hints": hints,
            "diagnose": diagnose,
            "starter": blocks["starter"],
            "tests": blocks["tests"],
            "solution": blocks["solution"],
        })

    if len(out) != EXERCISES_PER_UNIT:
        die(path, f"{len(out)} exercises, must be exactly {EXERCISES_PER_UNIT}")
    return out


# ---------------------------------------------------------------- drills

OPTION = re.compile(r"^- \(([ x])\) (.+)$", re.M)


def parse_drills(path: Path) -> list[dict]:
    meta, body = front_matter(path.read_text(), path)
    chunks = re.split(r"^## ", body, flags=re.M)[1:]
    out = []
    for i, chunk in enumerate(chunks, 1):
        question, _, rest = chunk.partition("\n")
        options = OPTION.findall(rest)
        if len(options) < 3:
            die(path, f"drill {i} has {len(options)} options, needs at least 3")
        correct = [n for n, (mark, _) in enumerate(options) if mark == "x"]
        if len(correct) != 1:
            die(path, f"drill {i} has {len(correct)} correct answers, needs exactly 1")
        why = re.search(r"^> (.+)$", rest, re.M)
        if not why:
            die(path, f"drill {i} has no `> ` explanation")
        out.append({
            "n": i,
            "q": question.strip(),
            "options": [text for _, text in options],
            "answer": correct[0],
            "why": why.group(1).strip(),
        })
    if len(out) != DRILLS_PER_UNIT:
        die(path, f"{len(out)} drills, must be exactly {DRILLS_PER_UNIT}")
    return out


# ---------------------------------------------------------------- projects

def parse_gloss(path: Path) -> list[dict]:
    meta, body = front_matter(path.read_text(), path)
    out = []
    for chunk in re.split(r"^## ", body, flags=re.M)[1:]:
        term, _, rest = chunk.partition("\n")
        see = re.findall(r"\[\[([\w-]+)\]\]", rest)
        text = re.sub(r"\[\[([\w-]+)\]\]", r"\1", rest).strip()
        if len(text.split()) < 8:
            die(path, f"glossary entry {term.strip()!r} is too short to be a definition")
        out.append({"term": term.strip(), "text": text, "see": see})
    return out


# A project stage is an exercise with the verdict question turned around. An
# exercise asks "what does this code already do"; a stage asks "make it do
# this", so there is no @expect and no @diagnose, and the contract is simpler:
# the starter must fail the stage's tests, the solution must pass them, and the
# solution must be clean under all three judges.
STAGE_BRIEF_MIN = 60

_PROJECT_STAGES = {slug: stages for slug, _, _, stages, *_ in PROJECTS}


def _mark_work(stages: list[dict], path: Path) -> None:
    """Record which lines of each starter are this stage's own work.

    A stage carries every earlier stage with it, so by stage twelve the starter
    is a thousand lines of which thirteen are the thing to write. Nobody should
    have to hunt for them, and nobody should have to annotate them either: the
    answer is the difference between this starter and the previous solution,
    which the file already contains.

    A real diff rather than set membership. Asking "does this line appear
    anywhere in the previous solution" gets the common case right and the
    important case exactly wrong: `raise NotImplementedError` appears in every
    starter, so the three of them a stage asks the reader to replace were
    marked as carried, which is the opposite of true. difflib aligns the two
    files and says which lines are actually new here, wherever they sit.
    """
    previous: list[str] = []
    for stage in stages:
        lines = stage["starter"].split("\n")
        work = []
        matcher = difflib.SequenceMatcher(None, previous, lines, autojunk=False)
        for tag, _, _, start, end in matcher.get_opcodes():
            if tag in ("insert", "replace"):
                work.extend(i + 1 for i in range(start, end) if lines[i].strip())
        stage["work"] = work
        try:
            stage["outline"] = _outline(lines, set(work))
        except SyntaxError as exc:
            die(path, f"stage {stage['n']} ({stage['title']}) starter does not "
                      f"parse: line {exc.lineno}, {exc.msg}")
        previous = stage["solution"].split("\n")


def _outline(lines: list[str], work: set[int]) -> list[dict]:
    """The top level shape of a file: what is in it, and where the work is.

    One entry per class or top level function, with the line it starts on and
    whether any of this stage's work falls inside it. It is what lets a reader
    see a thousand line file without scrolling through it.
    """
    kinds = {ast.ClassDef: "class", ast.FunctionDef: "def",
             ast.AsyncFunctionDef: "async def"}
    # Parsed rather than matched line by line. A `def` written at column zero
    # inside a docstring is a def to a regular expression and is not one to
    # Python, and this book's docstrings show example code constantly. The
    # parser is already here for the vocabulary gate.
    marks = [{"line": node.lineno, "kind": kinds[type(node)], "name": node.name}
             for node in ast.parse("\n".join(lines)).body
             if type(node) in kinds]
    for j, mark in enumerate(marks):
        end = marks[j + 1]["line"] if j + 1 < len(marks) else len(lines) + 1
        mark["mine"] = any(mark["line"] <= n < end for n in work)
        mark["lines"] = end - mark["line"]
    return marks


def parse_project(path: Path) -> dict:
    meta, body = front_matter(path.read_text(), path)
    slug = meta["slug"]
    chunks = re.split(r"^## ", body, flags=re.M)[1:]
    if not chunks:
        die(path, "no stages found (need `## ` headings)")

    stages = []
    for i, chunk in enumerate(chunks, 1):
        title, _, rest = chunk.partition("\n")
        title = title.strip()
        blocks = {m.group(1): m.group(2).rstrip() for m in FENCE.finditer(rest)}
        for need in ("starter", "tests", "solution"):
            if need not in blocks:
                die(path, f"stage {i} ({title}) has no ~~~{need} block")

        goals = [v.strip() for kind, v in DIRECTIVE.findall(rest) if kind == "goal"]
        if len(goals) != 1:
            die(path, f"stage {i} ({title}) needs exactly one @goal, found {len(goals)}")

        brief = FENCE.sub("", DIRECTIVE.sub("", rest)).strip()
        if stray := re.search(r"^@(\w+)", brief, re.M):
            die(path, f"stage {i}: @{stray.group(1)} is not a directive")
        if len(brief.split()) < STAGE_BRIEF_MIN:
            die(path, f"stage {i} ({title}) brief is {len(brief.split())} words, "
                      f"must be at least {STAGE_BRIEF_MIN}")

        stages.append({
            "n": i,
            "title": title,
            "brief": brief,
            "goal": goals[0],
            "starter": blocks["starter"],
            "tests": blocks["tests"],
            "solution": blocks["solution"],
        })

    _mark_work(stages, path)

    want = _PROJECT_STAGES.get(slug)
    if want is None:
        die(path, f"project slug {slug!r} is not in PROJECTS")
    if len(stages) != want:
        die(path, f"{len(stages)} stages, but PROJECTS declares {want}")
    return {"slug": slug, "stages": stages}



# ---------------------------------------------------------------- the vocabulary gate
#
# An exercise must be solvable with what the reader has already met. Relying on
# the author to remember the ordering does not survive contact with 39 units, so
# each unit declares what it introduces and the build refuses any exercise whose
# code uses something from further down the track.

# Assumed from the first page and therefore never gated: statements, calls,
# attribute access, f-strings, annotations and the four container literals. They
# are not listed, because a list of names no detector emits would imply a gate
# that does not exist. Only what appears below is enforced.

# slug -> what that unit's note teaches, and therefore what its exercises and
# every later unit's exercises may use. This lists only features the detector
# below can actually see: a name here that nothing detects gates nothing, which
# is worse than not listing it, so the build refuses one.
INTRODUCES = {
    "00-toolchain": {"assert", "compile", "eval", "math", "raise", "round"},
    "01-names": {"class", "del", "id"},
    "02-mutability": {"comprehension", "copy_module", "deepcopy", "slice", "sorted"},
    "03-data-model": {"callable", "getattr", "hash", "iter", "repr", "setattr", "sum"},
    "04-equality": {"all", "any", "frozenset", "set"},
    "05-expressions": {"conditional_expr", "next", "starargs", "walrus"},
    "06-control-flow": {"enumerate", "match", "zip"},
    "07-functions": {"lambda"},
    "08-scope": {"global", "nonlocal"},
    "09-exceptions": {"try"},
    "10-sequences": set(),
    "11-strings": set(),
    "12-dicts": {"counter", "defaultdict", "dict_comprehension", "setdefault"},
    "13-comprehensions": {"deque"},
    "14-sorting": {"bisect", "heapq"},
    "15-iterators": set(),
    "16-generators": {"yield"},
    "17-itertools": {"cache", "decorator", "filter", "functools", "itertools", "map", "partial", "reduce"},
    "18-classes": {"classmethod", "staticmethod"},
    "19-attributes": {"property"},
    "20-descriptors": set(),
    "21-mro": {"super"},
    "22-protocols": {"with"},
    "23-dataclasses": {"dataclass", "enum", "namedtuple"},
    "24-typing": {"optional"},
    "25-typecheck": set(),
    "26-decorators": {"wraps"},
    "27-metaclasses": set(),
    "28-ast": {"ast", "dis", "inspect"},
    "29-modules": set(),
    "30-packaging": set(),
    "31-testing": set(),
    "32-tooling": {"logging"},
    "33-concurrency": {"thread"},
    "34-async": {"async_def", "await"},
    "35-performance": {"timeit"},
    "36-memory": {"gc", "weakref"},
    "37-ecosystem": {"pathlib", "re", "subprocess"},
    "38-tracebacks": {"breakpoint", "pdb"},
}

_NODE_FEATURES = [
    ((ast.ListComp, ast.SetComp, ast.GeneratorExp), "comprehension"),
    ((ast.DictComp,), "dict_comprehension"),
    ((ast.Lambda,), "lambda"),
    ((ast.NamedExpr,), "walrus"),
    ((ast.Match,), "match"),
    ((ast.ClassDef,), "class"),
    ((ast.Yield, ast.YieldFrom), "yield"),
    ((ast.With, ast.AsyncWith), "with"),
    ((ast.Try,), "try"),
    ((ast.Slice,), "slice"),
    ((ast.Global,), "global"),
    ((ast.Nonlocal,), "nonlocal"),
    ((ast.AsyncFunctionDef,), "async_def"),
    ((ast.Await,), "await"),
    ((ast.IfExp,), "conditional_expr"),
    ((ast.Delete,), "del"),
    ((ast.Starred,), "starargs"),
    ((ast.Assert,), "assert"),
    ((ast.Raise,), "raise"),
]

# builtins and modules whose first legitimate appearance is a specific unit
# Emitted from syntax rather than from a name or a node type, so it needs saying
# here or the consistency check below cannot see it.
_SYNTHETIC_FEATURES = {"decorator"}

_NAME_FEATURES = {
    "map": "map", "filter": "filter", "reduce": "reduce", "zip": "zip",
    "enumerate": "enumerate", "sorted": "sorted", "sum": "sum", "any": "any",
    "all": "all", "next": "next", "iter": "iter", "id": "id", "hash": "hash",
    "set": "set", "frozenset": "frozenset", "setattr": "setattr",
    "getattr": "getattr", "repr": "repr", "callable": "callable",
    "property": "property", "classmethod": "classmethod",
    "staticmethod": "staticmethod", "super": "super", "compile": "compile",
    "eval": "eval", "round": "round", "breakpoint": "breakpoint",
    "dataclass": "dataclass", "defaultdict": "defaultdict",
    "namedtuple": "namedtuple", "NamedTuple": "namedtuple",
    "Counter": "counter", "deque": "deque",
    "setdefault": "setdefault", "wraps": "wraps", "partial": "partial",
    "deepcopy": "deepcopy",
}
# Reached only as an attribute (functools.cache) or a from-import, never as a
# bare name, so matching them as bare names only produced false positives on
# ordinary variables called `cache`.
_ATTR_FEATURES = {"cache": "cache", "lru_cache": "cache"}

# `from X import name` pairs where the module's own gate should not apply,
# because the module stands in for one construct and this name is a different
# one: `typing` is gated as unit 24's Optional, and NamedTuple is unit 23's own
# subject. The name still resolves through _NAME_FEATURES, so the feature is
# named in one place. Everything else adds both, because `from re import
# compile` really does use `re` as well as `compile`, and dropping the module
# gate there would let a unit-01 exercise import it.
_IMPORT_SKIPS_MODULE = {("typing", "NamedTuple")}

_MODULE_FEATURES = {
    "math": "math", "copy": "copy_module", "itertools": "itertools",
    "functools": "functools", "heapq": "heapq",
    "bisect": "bisect", "dataclasses": "dataclass", "enum": "enum",
    "ast": "ast", "dis": "dis", "inspect": "inspect", "logging": "logging",
    "weakref": "weakref", "gc": "gc", "pathlib": "pathlib", "re": "re",
    "subprocess": "subprocess", "threading": "thread", "asyncio": "async_def",
    "typing": "optional", "pdb": "pdb", "timeit": "timeit",
}


# INTRODUCES says what each unit unlocks; the tables above say what the detector
# can actually find. Nothing connected the two, so a feature named in INTRODUCES
# but never detected gated nothing, and a detected feature named in no unit gated
# everything forever. Both are now build failures.
DETECTABLE = ({name for _, name in _NODE_FEATURES} | _SYNTHETIC_FEATURES
              | set(_ATTR_FEATURES.values())
              | set(_NAME_FEATURES.values()) | set(_MODULE_FEATURES.values()))


# docs/AUTHORING.md lists the prose rules, and a rule nobody checks is a rule
# followed unevenly, which is unit 32's whole argument applied to this book's
# own text. Code is excluded: `!r` in an f-string is not an exclamation mark,
# and a dunder name is not a figurative underscore.
_PROSE_BANNED = {
    "an em or en dash": r"[\u2014\u2013]",
    "a curly quote": r"[\u2018\u2019\u201c\u201d]",
    "an exclamation mark": r"!",
    '"simply"': r"\bsimply\b",
    '"obviously"': r"\bobviously\b",
    '"just" as a minimiser': r"\bit is just\b|\bjust use\b|\bsimply just\b",
}


def _check_prose(path: Path, body: str, offset: int = 0) -> None:
    """The house style, enforced rather than remembered.

    `offset` is how many lines precede `body` in the file, so that a complaint
    names the line to open rather than a line number counted from the body.
    """
    prose = strip_code(body, blank=True)
    for label, pattern in _PROSE_BANNED.items():
        m = re.search(pattern, prose)
        if m:
            line = offset + prose[:m.start()].count("\n") + 1
            context = prose[max(0, m.start() - 40):m.end() + 40].replace("\n", " ").strip()
            die(path, f"prose contains {label} (near line {line}): ...{context}...")


# A unit that says "unit 21 explained" when unit 21 comes later is telling the
# reader to have read something they have not. There are several hundred of
# these references and nothing else checks them.
_LOOKS_BACK = re.compile(
    r"unit (\d{2})'?s? (?:own )?"
    r"(covered|said|introduced|made|described|established|explained|argued|"
    r"gave|met|answered|called|gets? you|gave you)\b")
_LOOKS_FORWARD = re.compile(r"unit (\d{2}) (?:is about|will|covers|gets to)\b")


_UNIT_NUMBERS = frozenset(u[0][:2] for u in TRACK)


def _check_cross_references(path: Path, body: str) -> None:
    """Every reference to another unit, checked against where this one sits.

    A file whose name does not start with a unit number, the glossary and the
    projects, still has its references checked for existing; only the "comes
    later" half needs to know where it is.
    """
    for m in re.finditer(r"[Uu]nits? (\d{2})(?: and (\d{2}))?", body):
        for group in m.groups():
            if group is not None and group not in _UNIT_NUMBERS:
                die(path, f"refers to unit {group}, which is not in the track")
    here = int(path.stem[:2]) if path.stem[:2].isdigit() else None
    if here is None:
        return
    for m in _LOOKS_BACK.finditer(body):
        if int(m.group(1)) > here:
            die(path, f"says unit {m.group(1)} {m.group(2)}, but that unit comes later")
    for m in _LOOKS_FORWARD.finditer(body):
        if int(m.group(1)) < here:
            die(path, f"points forward to unit {m.group(1)}, which the reader has already done")


def _check_tables() -> None:
    """The tables that have to agree with each other, checked once at startup.

    Every one of these is a rule somebody would otherwise have to remember
    while editing a tuple, which is the kind of rule that gets forgotten.
    """
    for slug, tier, domain, *_ in PROJECTS:
        if tier not in TIERS:
            raise SystemExit(f"project {slug}: tier {tier!r} is not one of {sorted(TIERS)}")
        if domain not in DOMAINS:
            raise SystemExit(
                f"project {slug}: domain {domain!r} is not one of {sorted(DOMAINS)}"
            )
    for slug, phase, *_ in TRACK:
        if not 0 <= phase < len(PHASES):
            raise SystemExit(
                f"unit {slug}: phase {phase} is not one of the {len(PHASES)} phases"
            )
    used = {phase for _, phase, *_ in TRACK}
    empty = sorted(set(range(len(PHASES))) - used)
    if empty:
        raise SystemExit(f"phases with no units in them: {[PHASES[i][0] for i in empty]}")


def _check_llms(counts: dict[str, object]) -> None:
    """llms.txt states numbers about the book. Check it still means them.

    It is the file somebody is handed when they want to rebuild this, so a
    stale number in it is worse than a stale number anywhere else: it is the
    one document read by a reader with no way to notice. Everything here is a
    phrase the file has to contain, and the build says what it should say.
    """
    path = ROOT / "llms.txt"
    if not path.is_file():
        return
    # Whitespace collapsed, and the blockquote markers with it, because the
    # file is wrapped for reading and "39 units and 15\n> projects" says the
    # same thing as "15 projects". A gate that trips on a line break is a gate
    # somebody turns off.
    text = re.sub(r"\s*\n>?\s*|\s+", " ", path.read_text(encoding="utf-8"))
    wrong = [f"{want!r}" for want in counts.values() if str(want) not in text]
    if wrong:
        raise SystemExit(
            "llms.txt no longer describes this book. It should say: "
            + ", ".join(wrong)
        )


def _check_feature_tables() -> None:
    introduced = {f for feats in INTRODUCES.values() for f in feats}
    undetectable = introduced - DETECTABLE
    ungated = DETECTABLE - introduced
    if undetectable:
        raise SystemExit(f"INTRODUCES names features no detector finds: {sorted(undetectable)}")
    if ungated:
        raise SystemExit(f"the detector finds features no unit introduces: {sorted(ungated)}")
    twice = [f for f in DETECTABLE
             if sum(1 for feats in INTRODUCES.values() if f in feats) > 1]
    if twice:
        raise SystemExit(f"features introduced by more than one unit: {sorted(twice)}")

    # TRACK and PROJECTS blurbs are prose too: they reach the reader through
    # manifest.json. Enforcing the style on content/ and remembering it here is
    # how the three em dashes this gate was written for got in.
    for row in list(TRACK) + list(PROJECTS):
        for field in row:
            if isinstance(field, str):
                _check_prose(Path("build.py"), field)

    # `from X import name` resolves per name, because one module can hold
    # constructs from different units: typing has both NamedTuple, which is
    # unit 23, and Optional, which is unit 24.
    for source, want in _IMPORT_CASES:
        got = features_used(source)
        if got != want:
            raise SystemExit(f"{source!r} detected {sorted(got)}, expected {sorted(want)}")


_IMPORT_CASES = [
    ("from typing import Optional", {"optional"}),
    ("from typing import NamedTuple", {"namedtuple"}),
    ("from typing import NamedTuple, Optional", {"namedtuple", "optional"}),
    ("from dataclasses import dataclass, field", {"dataclass"}),
    ("from itertools import chain", {"itertools"}),
    ("import typing", {"optional"}),
    # the module's own gate still applies when the name is not an override
    ("from re import compile", {"re", "compile"}),
    ("from subprocess import run", {"subprocess"}),
    ("from pathlib import Path", {"pathlib"}),
]


def features_used(source: str) -> set[str]:
    """Every gated construct appearing in a snippet."""
    try:
        tree = ast.parse(source)
    except SyntaxError:
        return set()          # a deliberately unparseable starter gates nothing

    found: set[str] = set()
    for node in ast.walk(tree):
        for types, name in _NODE_FEATURES:
            if isinstance(node, types):
                found.add(name)
        if isinstance(node, ast.Name) and node.id in _NAME_FEATURES:
            found.add(_NAME_FEATURES[node.id])
        if isinstance(node, ast.Attribute):
            if node.attr in _NAME_FEATURES:
                found.add(_NAME_FEATURES[node.attr])
            if node.attr in _ATTR_FEATURES:
                found.add(_ATTR_FEATURES[node.attr])
        if isinstance(node, ast.ImportFrom):
            for alias in node.names:
                if alias.name in _ATTR_FEATURES:
                    found.add(_ATTR_FEATURES[alias.name])
        if isinstance(node, ast.Import):
            for alias in node.names:
                if alias.name in _MODULE_FEATURES:
                    found.add(_MODULE_FEATURES[alias.name])
        if isinstance(node, ast.ImportFrom):
            for alias in node.names:
                if (node.module in _MODULE_FEATURES
                        and (node.module, alias.name) not in _IMPORT_SKIPS_MODULE):
                    found.add(_MODULE_FEATURES[node.module])
                if alias.name in _NAME_FEATURES:
                    found.add(_NAME_FEATURES[alias.name])
        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.decorator_list:
            found.add("decorator")
    return found


def available_by(slug: str) -> set[str]:
    """Everything a reader has met by the time they reach this unit's exercises."""
    order = [s for s, *_ in TRACK]
    if slug not in order:
        return set(BASELINE)
    upto = order[: order.index(slug) + 1]
    out: set[str] = set()
    for s in upto:
        out |= INTRODUCES.get(s, set())
    return out


def gate(path: Path) -> list[str]:
    """Complaints about an exercise file using constructs from further down the track."""
    _check_tables()
    _check_feature_tables()
    slug = path.stem
    allowed = available_by(slug)
    problems = []
    for ex in parse_exercises(path):
        for kind in ("starter", "solution"):   # the reader never writes the tests
            used = features_used(ex[kind])
            early = sorted(used - allowed)
            if early:
                problems.append(
                    f"{slug} #{ex['n']} {ex['title']}: {kind} uses {', '.join(early)} "
                    f"before the reader has met it")
    return problems


# ---------------------------------------------------------------- build

def build() -> int:
    DATA.mkdir(exist_ok=True)
    by_slug = {}
    track = []
    for i, (slug, phase, title, blurb) in enumerate(TRACK):
        entry = {
            "slug": slug, "n": i, "phase": phase, "title": title,
            "blurb": blurb, "accent": ACCENTS[i % len(ACCENTS)],
            "needs": TRACK[i - 1][0] if i else None,
            "hasNote": False, "hasEx": 0, "hasDrills": 0,
        }
        by_slug[slug] = entry
        track.append(entry)

    # Parse each file once. The JSON dump, the errors index and the search index
    # all want the same parsed result, and re-globbing meant every validation and
    # every regex ran two or three times per build.
    units = {p: parse_unit(p) for p in sorted(CONTENT.glob("units/*.md"))}
    exercises = {p: parse_exercises(p) for p in sorted(CONTENT.glob("ex/*.md"))}

    written = 0
    for path, unit in units.items():
        if unit["slug"] not in by_slug:
            die(path, f"slug {unit['slug']!r} is not in TRACK")
        by_slug[unit["slug"]]["hasNote"] = True
        (DATA / f"unit-{unit['slug']}.json").write_text(json.dumps(unit))
        written += 1

    # The gate belongs to the build, not to a shell script a contributor may
    # never run. The exercises are already parsed, so this costs nothing.
    _check_tables()
    _check_feature_tables()
    vocabulary = [p for path in exercises for p in gate(path)]
    for problem in vocabulary:
        print(f"VOCABULARY  {problem}", file=sys.stderr)

    for path, ex in exercises.items():
        slug = path.stem
        if slug not in by_slug:
            die(path, f"slug {slug!r} is not in TRACK")
        by_slug[slug]["hasEx"] = len(ex)
        # The book gives hints and never answers. Shipping the solutions to the
        # browser would put every one of them a single fetch away, so they stay
        # in content/ where --validate can still compile and run them.
        shipped = [{k: v for k, v in e.items() if k != "solution"} for e in ex]
        (DATA / f"ex-{slug}.json").write_text(json.dumps(shipped))
        written += 1

    for path in sorted(CONTENT.glob("drills/*.md")):
        slug = path.stem
        if slug not in by_slug:
            die(path, f"slug {slug!r} is not in TRACK")
        drills = parse_drills(path)
        by_slug[slug]["hasDrills"] = len(drills)
        (DATA / f"drills-{slug}.json").write_text(json.dumps(drills))
        written += 1

    projects = []
    for slug, tier, domain, stages, minutes, title, blurb in PROJECTS:
        projects.append({
            "slug": slug, "tier": tier, "tierLabel": tier.capitalize(), "domain": domain,
            "stages": stages, "minutes": minutes, "title": title, "blurb": blurb,
            "hasBody": False,
        })
    by_proj = {p["slug"]: p for p in projects}
    for path in sorted(CONTENT.glob("projects/*.md")):
        proj = parse_project(path)
        if proj["slug"] not in by_proj:
            die(path, f"project slug {proj['slug']!r} is not in PROJECTS")
        by_proj[proj["slug"]]["hasBody"] = True
        by_proj[proj["slug"]]["stageTitles"] = [s["title"] for s in proj["stages"]]
        # The reference implementation stays out of the browser, for the same
        # reason an exercise's solution does: hints, not answers.
        shipped = {**proj, "stages": [{k: v for k, v in s.items() if k != "solution"}
                                      for s in proj["stages"]]}
        (DATA / f"project-{proj['slug']}.json").write_text(json.dumps(shipped))
        written += 1

    # The errors index is derived from every @diagnose in the book, so it cannot
    # drift from the prose the workbench actually shows.
    errors: dict[str, dict] = {}
    for path, parsed in exercises.items():
        slug = path.stem
        for ex in parsed:
            # The judge travels with the @expect declaration. Guessing it back
            # from the shape of the code was wrong for B006, which looks like an
            # exception name, and would be wrong again for the next such code.
            declared = {e["code"] or e["judge"]: e["judge"] for e in ex["expects"]}