perf(delivery): ship verified Python bytecode caches

scripts/release/README.md

@@ -22,6 +22,15 @@ production dependencies and separately fingerprinted public runtimes. Backend
 source ships with those dependencies; uvx/pip and startup hooks are not runtime
 entry points.
 
+The pinned Python 3.13.16 builder compiles checked-hash bytecode from the copied
+stdlib, dependencies, backend and delivery sources. It never copies existing
+caches. Each read-only `__pycache__/*.cpython-313.pyc` file is included in the full
+bundle inventory, must have its source in that inventory, and has its interpreter
+header and source hash checked during byte verification. Other cache versions,
+timestamp or unchecked caches, orphan caches and caches inside judging images are
+refused. Services keep `PYTHONDONTWRITEBYTECODE=1`; they can read these verified
+caches without creating runtime files. Original source-only bundles remain valid.
+
 Build inputs contain explicit inventories and expected SHA-256 digests for public
 runtimes and the three image catalogs. Keep real input paths and receipts outside
 Git. The build receipt records source and manifest digests, file count and byte

scripts/release/build.py

@@ -9,7 +9,7 @@ import stat
 import subprocess
 from pathlib import Path
 
-from . import manifest, proxy_runtime
+from . import bytecode, manifest, proxy_runtime
 from .policy import DeliveryError, contract, digest, fields, read_json, relative_name, sha256
 
 ENTRYPOINTS = {
@@ -406,6 +406,7 @@ def build(repo: Path, inputs: Path, output: Path, work: Path) -> dict:
         proxy_runtime.copy_to_payload(
             Path(spec["proxy"]["root"]), spec["proxy"]["catalog_sha256"], payload / "proxy"
         )
+    bytecode.compile_payload(payload, "sd-" + source_sha[:12])
     for directory, directories, _ in os.walk(payload, topdown=False):
         for name in directories:
             (Path(directory) / name).chmod(0o555)

scripts/release/bytecode.py

@@ -0,0 +1,111 @@
+"""Build checked-hash caches from trusted source; never execute cache contents."""
+
+import importlib.util
+import os
+import re
+import stat
+import sys
+from pathlib import Path
+
+from .policy import BYTECODE_ROOTS, DeliveryError, bytecode_source, relative_name
+
+MAX_SOURCE_BYTES = 4 * 1024**2
+
+
+def _stamp(info: os.stat_result) -> tuple[int, ...]:
+    return (
+        info.st_dev,
+        info.st_ino,
+        info.st_mode,
+        info.st_nlink,
+        info.st_uid,
+        info.st_gid,
+        info.st_size,
+        info.st_mtime_ns,
+        info.st_ctime_ns,
+    )
+
+
+def source_bytes(path: Path) -> bytes:
+    descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK)
+    with os.fdopen(descriptor, "rb") as stream:
+        before = os.fstat(stream.fileno())
+        if (
+            not stat.S_ISREG(before.st_mode)
+            or before.st_nlink != 1
+            or stat.S_IMODE(before.st_mode) not in (0o444, 0o555)
+            or before.st_size > MAX_SOURCE_BYTES
+        ):
+            raise DeliveryError("bytecode source requires a bounded read-only ordinary file")
+        data = stream.read(MAX_SOURCE_BYTES + 1)
+        after = os.fstat(stream.fileno())
+    if (
+        len(data) != before.st_size
+        or _stamp(before) != _stamp(after)
+        or _stamp(after) != _stamp(path.lstat())
+    ):
+        raise DeliveryError("bytecode source changed during verification")
+    return data
+
+
+def verify_header(header: bytes, source: Path) -> None:
+    # Bundle SHA-256 verification authenticates every cache byte. This extra
+    # pairing binds the CPython header to the actual, separately hashed source.
+    # Never deserialize or execute an incoming marshal payload to inspect it.
+    if sys.version_info[:3] != (3, 13, 16):
+        raise DeliveryError("bytecode verification requires the pinned delivery interpreter")
+    expected = importlib.util.MAGIC_NUMBER + b"\x03\x00\x00\x00"
+    if len(header) != 16 or header[:8] != expected:
+        raise DeliveryError("only pinned checked-hash Python bytecode is supported")
+    if header[8:] != importlib.util.source_hash(source_bytes(source)):
+        raise DeliveryError("bytecode source hash differs")
+
+
+def compile_payload(payload: Path, release_id: str) -> int:
+    import py_compile
+
+    if (
+        sys.version_info[:3] != (3, 13, 16)
+        or sys.implementation.cache_tag != "cpython-313"
+        or not re.fullmatch(r"sd-[0-9a-f]{12}", release_id)
+    ):
+        raise DeliveryError("bytecode build requires the pinned delivery interpreter")
+    sources = []
+    for directory, directories, files in os.walk(payload, followlinks=False):
+        directories.sort()
+        if "__pycache__" in directories:
+            raise DeliveryError("inherited bytecode cache refused")
+        if any((Path(directory) / name).is_symlink() for name in directories):
+            raise DeliveryError("bytecode build directory link refused")
+        for name in sorted(files):
+            path = Path(directory) / name
+            relative = path.relative_to(payload).as_posix()
+            if relative.startswith(BYTECODE_ROOTS) and relative.endswith(".py"):
+                relative_name(relative)
+                sources.append((path, relative))
+                if len(sources) > 10000:
+                    raise DeliveryError("bytecode source count exceeds build bound")
+    for source, relative in sources:
+        before = _stamp(source.lstat())
+        source_bytes(source)
+        # Explicit cfile prevents an inherited PYTHONPYCACHEPREFIX from sending
+        # output outside this new bundle. Source filenames never expose Host paths.
+        cache = source.parent / "__pycache__" / (source.stem + ".cpython-313.pyc")
+        if bytecode_source(cache.relative_to(payload).as_posix()) != relative:
+            raise DeliveryError("bytecode source pairing differs")
+        if cache.exists() or cache.is_symlink():
+            raise DeliveryError("existing bytecode cache refused")
+        py_compile.compile(
+            str(source),
+            cfile=str(cache),
+            dfile=f"/opt/kf-onlinejudge/releases/{release_id}/payload/{relative}",
+            doraise=True,
+            optimize=0,
+            invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+        )
+        cache.chmod(0o444)
+        with cache.open("rb") as stream:
+            verify_header(stream.read(16), source)
+        if _stamp(source.lstat()) != before:
+            raise DeliveryError("source changed during bytecode compilation")
+    return len(sources)

scripts/release/manifest.py

@@ -21,6 +21,7 @@ from .policy import (
     PLATFORM,
     TOOLCHAINS,
     DeliveryError,
+    bytecode_source,
     canonical,
     digest,
     fields,
@@ -164,12 +165,17 @@ def _file_stamp(info: os.stat_result) -> tuple[int, ...]:
     )
 
 
-def _read_payload(row: tuple[Path, tuple[int, ...], dict]) -> dict:
-    path, expected, record = row
+def _read_payload(row: tuple[Path, tuple[int, ...], dict, Path | None]) -> dict:
+    path, expected, record, source = row
     descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
     with os.fdopen(descriptor, "rb") as stream:
         if _file_stamp(os.fstat(stream.fileno())) != expected:
             raise DeliveryError("payload file changed before byte verification")
+        if source is not None:
+            from .bytecode import verify_header
+
+            verify_header(stream.read(16), source)
+            stream.seek(0)
         checked = hashlib.file_digest(stream, "sha256").hexdigest()
         after = _file_stamp(os.fstat(stream.fileno()))
     if after != expected or _file_stamp(path.lstat()) != expected:
@@ -205,6 +211,7 @@ def inventory(root: Path, *, path_prefix: str = "") -> list[dict]:
                 raise DeliveryError("payload aggregate limit")
             relative = path.relative_to(root).as_posix()
             relative_name(path_prefix + relative)
+            source_name = bytecode_source(path_prefix + relative)
             entries.append(
                 (
                     path,
@@ -214,12 +221,15 @@ def inventory(root: Path, *, path_prefix: str = "") -> list[dict]:
                         "bytes": size,
                         "mode": "0555" if info.st_mode & 0o111 else "0444",
                     },
+                    root / source_name.removeprefix(path_prefix)
+                    if source_name is not None
+                    else None,
                 )
             )
     # The same finite four-call scheduler used for durable file copies also
     # bounds reads. It joins every dispatched reader before returning or raising.
     verified = file_copy.bounded_copies(entries, _read_payload)
-    for path, expected, _ in entries:
+    for path, expected, _, _ in entries:
         if _file_stamp(path.lstat()) != expected:
             raise DeliveryError("payload file changed before inventory completion")
     return sorted(verified, key=lambda x: x["path"])
@@ -339,6 +349,10 @@ def validate(data: dict) -> dict:
         or data["payload_bytes"] != total
     ):
         raise DeliveryError("bundle byte total")
+    for entry in files:
+        source = bytecode_source(entry["path"])
+        if source is not None and (source not in seen or entry["mode"] != "0444"):
+            raise DeliveryError("bytecode requires its inventoried source and read-only mode")
     required = {
         "bin/kf-sandbox",
         "bin/kf-worker",

scripts/release/policy.py

@@ -29,6 +29,24 @@ ROOTS = (
 FORBIDDEN = frozenset({".git", ".localplan", "docs", "__pycache__", ".env", ".ssh"})
 _UNSAFE_PARTS = FORBIDDEN | {".", ".."}
 _PAYLOAD_PATH = re.compile(r"[A-Za-z0-9_.+@-]{1,255}(?:/[A-Za-z0-9_.+@-]{1,255})*")
+BYTECODE_ROOTS = ("python/lib/python3.13/", "site/", "backend/src/", "delivery/")
+BYTECODE_TAG = ".cpython-313.pyc"
+
+
+def bytecode_source(value: str) -> str | None:
+    """A source-adjacent, unoptimized cache has exactly one fixed spelling."""
+    parent, _, leaf = value.rpartition("/")
+    directory, separator, cache = parent.rpartition("/")
+    if (
+        not separator
+        or cache != "__pycache__"
+        or not value.startswith(BYTECODE_ROOTS)
+        or value.split("/").count("__pycache__") != 1
+        or not leaf.endswith(BYTECODE_TAG)
+        or not leaf.removesuffix(BYTECODE_TAG)
+    ):
+        return None
+    return directory + "/" + leaf.removesuffix(BYTECODE_TAG) + ".py"
 
 
 class DeliveryError(ValueError):
@@ -63,9 +81,17 @@ def integer(value: Any, lower: int, upper: int, label: str) -> int:
 def _relative_name(value: str) -> str:
     if _PAYLOAD_PATH.fullmatch(value) is None:
         raise DeliveryError("noncanonical payload path")
-    if _UNSAFE_PARTS.intersection(value.split("/")):
+    parts = value.split("/")
+    cache_file = bytecode_source(value) is not None
+    cache_directory = (
+        value.startswith(BYTECODE_ROOTS)
+        and parts[-2:] == ["__pycache__", ".__directory__"]
+        and parts.count("__pycache__") == 1
+    )
+    unsafe = _UNSAFE_PARTS - {"__pycache__"} if cache_file or cache_directory else _UNSAFE_PARTS
+    if unsafe.intersection(parts):
         raise DeliveryError("private or unsafe payload path")
-    if not value.startswith(ROOTS) or value.endswith((".pyc", ".pyo")):
+    if not value.startswith(ROOTS) or (value.endswith((".pyc", ".pyo")) and not cache_file):
         raise DeliveryError("payload path outside delivery allowlist")
     return value