perf(delivery): ship verified Python bytecode caches

.github/workflows/engineering.yml

@@ -110,6 +110,7 @@ jobs:
             tests/test_delivery.py \
             tests/test_delivery_io.py \
             tests/test_delivery_validation_work.py \
+            tests/test_delivery_bytecode.py \
             tests/test_delivery_backup.py \
             tests/test_delivery_restore_io.py \
             tests/test_restore_io_contracts.py \

backend/tests/test_delivery_bytecode.py

@@ -0,0 +1,114 @@
+"""Ordinary owned source/cache checks; no service, isolation or recovery claims."""
+
+import importlib
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+bytecode = importlib.import_module("scripts.release.bytecode")
+manifest = importlib.import_module("scripts.release.manifest")
+policy = importlib.import_module("scripts.release.policy")
+
+
+@pytest.fixture
+def compiled(tmp_path):
+    source = tmp_path / "site/toy.py"
+    source.parent.mkdir()
+    source.write_text("VALUE = (17, 'ordinary trusted fixture')\n")
+    source.chmod(0o444)
+    assert bytecode.compile_payload(tmp_path, "sd-" + "b" * 12) == 1
+    cache = source.parent / "__pycache__/toy.cpython-313.pyc"
+    return tmp_path, source, cache
+
+
+def test_complete_cache_inventory_and_actual_import_without_source_compilation(compiled):
+    payload, source, cache = compiled
+    records = manifest.inventory(payload)
+    assert {r["path"] for r in records} == {
+        "site/toy.py",
+        "site/__pycache__/toy.cpython-313.pyc",
+    }
+    assert all(r["mode"] == "0444" for r in records)
+    # Only this tiny trusted fixture executes. An attempted source compilation
+    # fails, so success demonstrates CPython actually consumes the checked cache.
+    code = """import builtins,sys
+sys.path.insert(0,sys.argv[1])
+def refuse(*args,**kwargs): raise AssertionError('unexpected source compilation')
+builtins.compile=refuse
+import toy
+assert toy.VALUE==(17,'ordinary trusted fixture')
+"""
+    before = {p: bytecode._stamp(p.lstat()) for p in (source, cache)}
+    result = subprocess.run(
+        [sys.executable, "-I", "-B", "-c", code, str(source.parent)],
+        capture_output=True,
+        timeout=15,
+    )
+    assert result.returncode == 0, result.stderr
+    assert {p: bytecode._stamp(p.lstat()) for p in before} == before
+
+
+def test_source_change_cannot_reuse_cache_even_with_restored_mtime(compiled):
+    payload, source, _ = compiled
+    before = source.stat()
+    source.chmod(0o600)
+    source.write_text("VALUE = (18, 'ordinary trusted fixture')\n")
+    source.chmod(0o444)
+    os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
+    with pytest.raises(policy.DeliveryError, match="bytecode source hash differs"):
+        manifest.inventory(payload)
+
+
+@pytest.mark.parametrize("offset,value", [(0, 0), (4, 0), (4, 1), (4, 2)])
+def test_wrong_magic_timestamp_and_unchecked_hash_caches_are_refused(compiled, offset, value):
+    payload, _, cache = compiled
+    data = bytearray(cache.read_bytes())
+    data[offset] = value
+    cache.chmod(0o600)
+    cache.write_bytes(data)
+    cache.chmod(0o444)
+    with pytest.raises(policy.DeliveryError, match="only pinned checked-hash"):
+        manifest.inventory(payload)
+
+
+def test_builder_refuses_inherited_cache_and_does_not_write_to_external_prefix(
+    compiled, monkeypatch
+):
+    payload, _, cache = compiled
+    before = cache.read_bytes()
+    monkeypatch.setattr(sys, "pycache_prefix", str(payload / "outside"))
+    with pytest.raises(policy.DeliveryError, match="inherited bytecode cache"):
+        bytecode.compile_payload(payload, "sd-" + "b" * 12)
+    assert cache.read_bytes() == before and not (payload / "outside").exists()
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "site/toy.pyc",
+        "site/__pycache__/toy.cpython-314.pyc",
+        "site/__pycache__/toy.cpython-313.opt-1.pyc",
+        "site/__pycache__/toy.py",
+        "site/__pycache__/nested/__pycache__/toy.cpython-313.pyc",
+        "site/../__pycache__/toy.cpython-313.pyc",
+        "images/python313/__pycache__/toy.cpython-313.pyc",
+        "frontend/__pycache__/toy.cpython-313.pyc",
+    ],
+)
+def test_cache_allowlist_keeps_wrong_versions_domains_and_paths_closed(name):
+    with pytest.raises(policy.DeliveryError):
+        policy.relative_name(name)
+
+
+@pytest.mark.parametrize("prefix", policy.BYTECODE_ROOTS)
+def test_only_source_adjacent_pinned_cache_paths_are_allowed(prefix):
+    name = prefix + "package/__pycache__/toy.cpython-313.pyc"
+    assert policy.relative_name(name) == name
+    assert policy.bytecode_source(name) == prefix + "package/toy.py"
+    directory = prefix + "package/__pycache__/.__directory__"
+    assert policy.relative_name(directory) == directory

backend/tests/test_delivery_io.py

@@ -308,7 +308,27 @@ def test_payload_reader_refuses_identity_drift_model(ordinary_bundle, monkeypatc
     expected = list(manifest._file_stamp(info))
     expected[index] += 1
     with pytest.raises(policy.DeliveryError, match="changed before byte verification"):
-        manifest._read_payload((path, tuple(expected), {"path": "frontend/index.html"}))
+        manifest._read_payload((path, tuple(expected), {"path": "frontend/index.html"}, None))
+
+
+@pytest.mark.parametrize("source_present,mode", [(False, "0444"), (True, "0555")])
+def test_manifest_rejects_orphan_or_executable_bytecode(ordinary_bundle, source_present, mode):
+    bundle, _ = ordinary_bundle
+    data = json.loads((bundle / "manifest.json").read_bytes())
+    if source_present:
+        data["files"].append(
+            {"path": "site/toy.py", "bytes": 0, "mode": "0444", "sha256": "b" * 64}
+        )
+    data["files"].append(
+        {
+            "path": "site/__pycache__/toy.cpython-313.pyc",
+            "bytes": 0,
+            "mode": mode,
+            "sha256": "b" * 64,
+        }
+    )
+    with pytest.raises(policy.DeliveryError, match="bytecode requires its inventoried source"):
+        manifest.validate(data)
 
 
 def test_same_operation_reuses_bytes_but_not_mutable_metadata(ordinary_bundle, monkeypatch):

scripts/release/README.md

@@ -22,6 +22,15 @@ production dependencies and separately fingerprinted public runtimes. Backend
 source ships with those dependencies; uvx/pip and startup hooks are not runtime
 entry points.
 
+The pinned Python 3.13.16 builder compiles checked-hash bytecode from the copied
+stdlib, dependencies, backend and delivery sources. It never copies existing
+caches. Each read-only `__pycache__/*.cpython-313.pyc` file is included in the full
+bundle inventory, must have its source in that inventory, and has its interpreter
+header and source hash checked during byte verification. Other cache versions,
+timestamp or unchecked caches, orphan caches and caches inside judging images are
+refused. Services keep `PYTHONDONTWRITEBYTECODE=1`; they can read these verified
+caches without creating runtime files. Original source-only bundles remain valid.
+
 Build inputs contain explicit inventories and expected SHA-256 digests for public
 runtimes and the three image catalogs. Keep real input paths and receipts outside
 Git. The build receipt records source and manifest digests, file count and byte

scripts/release/build.py

@@ -9,7 +9,7 @@ import stat
 import subprocess
 from pathlib import Path
 
-from . import manifest, proxy_runtime
+from . import bytecode, manifest, proxy_runtime
 from .policy import DeliveryError, contract, digest, fields, read_json, relative_name, sha256
 
 ENTRYPOINTS = {
@@ -406,6 +406,7 @@ def build(repo: Path, inputs: Path, output: Path, work: Path) -> dict:
         proxy_runtime.copy_to_payload(
             Path(spec["proxy"]["root"]), spec["proxy"]["catalog_sha256"], payload / "proxy"
         )
+    bytecode.compile_payload(payload, "sd-" + source_sha[:12])
     for directory, directories, _ in os.walk(payload, topdown=False):
         for name in directories:
             (Path(directory) / name).chmod(0o555)

scripts/release/bytecode.py

@@ -0,0 +1,111 @@
+"""Build checked-hash caches from trusted source; never execute cache contents."""
+
+import importlib.util
+import os
+import re
+import stat
+import sys
+from pathlib import Path
+
+from .policy import BYTECODE_ROOTS, DeliveryError, bytecode_source, relative_name
+
+MAX_SOURCE_BYTES = 4 * 1024**2
+
+
+def _stamp(info: os.stat_result) -> tuple[int, ...]:
+    return (
+        info.st_dev,
+        info.st_ino,
+        info.st_mode,
+        info.st_nlink,
+        info.st_uid,
+        info.st_gid,
+        info.st_size,
+        info.st_mtime_ns,
+        info.st_ctime_ns,
+    )
+
+
+def source_bytes(path: Path) -> bytes:
+    descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK)
+    with os.fdopen(descriptor, "rb") as stream:
+        before = os.fstat(stream.fileno())
+        if (
+            not stat.S_ISREG(before.st_mode)
+            or before.st_nlink != 1
+            or stat.S_IMODE(before.st_mode) not in (0o444, 0o555)
+            or before.st_size > MAX_SOURCE_BYTES
+        ):
+            raise DeliveryError("bytecode source requires a bounded read-only ordinary file")
+        data = stream.read(MAX_SOURCE_BYTES + 1)
+        after = os.fstat(stream.fileno())
+    if (
+        len(data) != before.st_size
+        or _stamp(before) != _stamp(after)
+        or _stamp(after) != _stamp(path.lstat())
+    ):
+        raise DeliveryError("bytecode source changed during verification")
+    return data
+
+
+def verify_header(header: bytes, source: Path) -> None:
+    # Bundle SHA-256 verification authenticates every cache byte. This extra
+    # pairing binds the CPython header to the actual, separately hashed source.
+    # Never deserialize or execute an incoming marshal payload to inspect it.
+    if sys.version_info[:3] != (3, 13, 16):
+        raise DeliveryError("bytecode verification requires the pinned delivery interpreter")
+    expected = importlib.util.MAGIC_NUMBER + b"\x03\x00\x00\x00"
+    if len(header) != 16 or header[:8] != expected:
+        raise DeliveryError("only pinned checked-hash Python bytecode is supported")
+    if header[8:] != importlib.util.source_hash(source_bytes(source)):
+        raise DeliveryError("bytecode source hash differs")
+
+
+def compile_payload(payload: Path, release_id: str) -> int:
+    import py_compile
+
+    if (
+        sys.version_info[:3] != (3, 13, 16)
+        or sys.implementation.cache_tag != "cpython-313"
+        or not re.fullmatch(r"sd-[0-9a-f]{12}", release_id)
+    ):
+        raise DeliveryError("bytecode build requires the pinned delivery interpreter")
+    sources = []
+    for directory, directories, files in os.walk(payload, followlinks=False):
+        directories.sort()
+        if "__pycache__" in directories:
+            raise DeliveryError("inherited bytecode cache refused")
+        if any((Path(directory) / name).is_symlink() for name in directories):
+            raise DeliveryError("bytecode build directory link refused")
+        for name in sorted(files):
+            path = Path(directory) / name
+            relative = path.relative_to(payload).as_posix()
+            if relative.startswith(BYTECODE_ROOTS) and relative.endswith(".py"):
+                relative_name(relative)
+                sources.append((path, relative))
+                if len(sources) > 10000:
+                    raise DeliveryError("bytecode source count exceeds build bound")
+    for source, relative in sources:
+        before = _stamp(source.lstat())
+        source_bytes(source)
+        # Explicit cfile prevents an inherited PYTHONPYCACHEPREFIX from sending
+        # output outside this new bundle. Source filenames never expose Host paths.
+        cache = source.parent / "__pycache__" / (source.stem + ".cpython-313.pyc")
+        if bytecode_source(cache.relative_to(payload).as_posix()) != relative:
+            raise DeliveryError("bytecode source pairing differs")
+        if cache.exists() or cache.is_symlink():
+            raise DeliveryError("existing bytecode cache refused")
+        py_compile.compile(
+            str(source),
+            cfile=str(cache),
+            dfile=f"/opt/kf-onlinejudge/releases/{release_id}/payload/{relative}",
+            doraise=True,
+            optimize=0,
+            invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+        )
+        cache.chmod(0o444)
+        with cache.open("rb") as stream:
+            verify_header(stream.read(16), source)
+        if _stamp(source.lstat()) != before:
+            raise DeliveryError("source changed during bytecode compilation")
+    return len(sources)

scripts/release/manifest.py

@@ -21,6 +21,7 @@ from .policy import (
     PLATFORM,
     TOOLCHAINS,
     DeliveryError,
+    bytecode_source,
     canonical,
     digest,
     fields,
@@ -164,12 +165,17 @@ def _file_stamp(info: os.stat_result) -> tuple[int, ...]:
     )
 
 
-def _read_payload(row: tuple[Path, tuple[int, ...], dict]) -> dict:
-    path, expected, record = row
+def _read_payload(row: tuple[Path, tuple[int, ...], dict, Path | None]) -> dict:
+    path, expected, record, source = row
     descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
     with os.fdopen(descriptor, "rb") as stream:
         if _file_stamp(os.fstat(stream.fileno())) != expected:
             raise DeliveryError("payload file changed before byte verification")
+        if source is not None:
+            from .bytecode import verify_header
+
+            verify_header(stream.read(16), source)
+            stream.seek(0)
         checked = hashlib.file_digest(stream, "sha256").hexdigest()
         after = _file_stamp(os.fstat(stream.fileno()))
     if after != expected or _file_stamp(path.lstat()) != expected:
@@ -205,6 +211,7 @@ def inventory(root: Path, *, path_prefix: str = "") -> list[dict]:
                 raise DeliveryError("payload aggregate limit")
             relative = path.relative_to(root).as_posix()
             relative_name(path_prefix + relative)
+            source_name = bytecode_source(path_prefix + relative)
             entries.append(
                 (
                     path,
@@ -214,12 +221,15 @@ def inventory(root: Path, *, path_prefix: str = "") -> list[dict]:
                         "bytes": size,
                         "mode": "0555" if info.st_mode & 0o111 else "0444",
                     },
+                    root / source_name.removeprefix(path_prefix)
+                    if source_name is not None
+                    else None,
                 )
             )
     # The same finite four-call scheduler used for durable file copies also
     # bounds reads. It joins every dispatched reader before returning or raising.
     verified = file_copy.bounded_copies(entries, _read_payload)
-    for path, expected, _ in entries:
+    for path, expected, _, _ in entries:
         if _file_stamp(path.lstat()) != expected:
             raise DeliveryError("payload file changed before inventory completion")
     return sorted(verified, key=lambda x: x["path"])
@@ -339,6 +349,10 @@ def validate(data: dict) -> dict:
         or data["payload_bytes"] != total
     ):
         raise DeliveryError("bundle byte total")
+    for entry in files:
+        source = bytecode_source(entry["path"])
+        if source is not None and (source not in seen or entry["mode"] != "0444"):
+            raise DeliveryError("bytecode requires its inventoried source and read-only mode")
     required = {
         "bin/kf-sandbox",
         "bin/kf-worker",

scripts/release/policy.py

@@ -29,6 +29,24 @@ ROOTS = (
 FORBIDDEN = frozenset({".git", ".localplan", "docs", "__pycache__", ".env", ".ssh"})
 _UNSAFE_PARTS = FORBIDDEN | {".", ".."}
 _PAYLOAD_PATH = re.compile(r"[A-Za-z0-9_.+@-]{1,255}(?:/[A-Za-z0-9_.+@-]{1,255})*")
+BYTECODE_ROOTS = ("python/lib/python3.13/", "site/", "backend/src/", "delivery/")
+BYTECODE_TAG = ".cpython-313.pyc"
+
+
+def bytecode_source(value: str) -> str | None:
+    """A source-adjacent, unoptimized cache has exactly one fixed spelling."""
+    parent, _, leaf = value.rpartition("/")
+    directory, separator, cache = parent.rpartition("/")
+    if (
+        not separator
+        or cache != "__pycache__"
+        or not value.startswith(BYTECODE_ROOTS)
+        or value.split("/").count("__pycache__") != 1
+        or not leaf.endswith(BYTECODE_TAG)
+        or not leaf.removesuffix(BYTECODE_TAG)
+    ):
+        return None
+    return directory + "/" + leaf.removesuffix(BYTECODE_TAG) + ".py"
 
 
 class DeliveryError(ValueError):
@@ -63,9 +81,17 @@ def integer(value: Any, lower: int, upper: int, label: str) -> int:
 def _relative_name(value: str) -> str:
     if _PAYLOAD_PATH.fullmatch(value) is None:
         raise DeliveryError("noncanonical payload path")
-    if _UNSAFE_PARTS.intersection(value.split("/")):
+    parts = value.split("/")
+    cache_file = bytecode_source(value) is not None
+    cache_directory = (
+        value.startswith(BYTECODE_ROOTS)
+        and parts[-2:] == ["__pycache__", ".__directory__"]
+        and parts.count("__pycache__") == 1
+    )
+    unsafe = _UNSAFE_PARTS - {"__pycache__"} if cache_file or cache_directory else _UNSAFE_PARTS
+    if unsafe.intersection(parts):
         raise DeliveryError("private or unsafe payload path")
-    if not value.startswith(ROOTS) or value.endswith((".pyc", ".pyo")):
+    if not value.startswith(ROOTS) or (value.endswith((".pyc", ".pyo")) and not cache_file):
         raise DeliveryError("payload path outside delivery allowlist")
     return value